2501 files changed, 135462 insertions, 41529 deletions

diff --git a/.dir-locals.el b/.dir-locals.el
new file mode 100644
index 000000000..3ac0cfc6f
--- /dev/null
+++ b/.dir-locals.el
@@ -0,0 +1,2 @@
+((c-mode . ((c-file-style . "stroustrup")
+	    (indent-tabs-mode . nil))))
diff --git a/.gitignore b/.gitignore
index 61bc49263..88a80ff4a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,12 +19,13 @@
 /trace/generated-ust.c
 /ui/shader/texture-blit-frag.h
 /ui/shader/texture-blit-vert.h
-/libcacard/trace/generated-tracers.c
 *-timestamp
 /*-softmmu
 /*-darwin-user
 /*-linux-user
 /*-bsd-user
+/ivshmem-client
+/ivshmem-server
 /libdis*
 /libuser
 /linux-headers/asm
@@ -34,6 +35,7 @@
 /qapi-visit.[ch]
 /qapi-event.[ch]
 /qmp-commands.h
+/qmp-introspect.[ch]
 /qmp-marshal.c
 /qemu-doc.html
 /qemu-tech.html
@@ -49,6 +51,7 @@
 /qemu-ga
 /qemu-bridge-helper
 /qemu-monitor.texi
+/qemu-monitor-info.texi
 /qmp-commands.txt
 /vscclient
 /fsdev/virtfs-proxy-helper
@@ -58,6 +61,7 @@
 *.cp
 *.dvi
 *.exe
+*.msi
 *.dll
 *.so
 *.mo
diff --git a/.travis.yml b/.travis.yml
index 0ac170b46..6ca026094 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -54,15 +54,7 @@ matrix:
   include:
     # Make check target (we only do this once)
     - env:
-        - TARGETS=alpha-softmmu,arm-softmmu,aarch64-softmmu,cris-softmmu,
-                  i386-softmmu,x86_64-softmmu,m68k-softmmu,microblaze-softmmu,
-                  microblazeel-softmmu,mips-softmmu,mips64-softmmu,
-                  mips64el-softmmu,mipsel-softmmu,or32-softmmu,ppc-softmmu,
-                  ppc64-softmmu,ppcemb-softmmu,s390x-softmmu,sh4-softmmu,
-                  sh4eb-softmmu,sparc-softmmu,sparc64-softmmu,
-                  unicore32-softmmu,unicore32-linux-user,
-                  lm32-softmmu,moxie-softmmu,tricore-softmmu,xtensa-softmmu,
-                  xtensaeb-softmmu
+        - TARGETS=alpha-softmmu,arm-softmmu,aarch64-softmmu,cris-softmmu,i386-softmmu,x86_64-softmmu,m68k-softmmu,microblaze-softmmu,microblazeel-softmmu,mips-softmmu,mips64-softmmu,mips64el-softmmu,mipsel-softmmu,or32-softmmu,ppc-softmmu,ppc64-softmmu,ppcemb-softmmu,s390x-softmmu,sh4-softmmu,sh4eb-softmmu,sparc-softmmu,sparc64-softmmu,unicore32-softmmu,unicore32-linux-user,lm32-softmmu,moxie-softmmu,tricore-softmmu,xtensa-softmmu,xtensaeb-softmmu
           TEST_CMD="make check"
       compiler: gcc
     # Debug related options
diff --git a/CODING_STYLE b/CODING_STYLE
index d46cfa5f6..3c6978f83 100644
--- a/CODING_STYLE
+++ b/CODING_STYLE
@@ -87,10 +87,15 @@ Furthermore, it is the QEMU coding style.
 
 5. Declarations
 
-Mixed declarations (interleaving statements and declarations within blocks)
-are not allowed; declarations should be at the beginning of blocks.  In other
-words, the code should not generate warnings if using GCC's
--Wdeclaration-after-statement option.
+Mixed declarations (interleaving statements and declarations within
+blocks) are generally not allowed; declarations should be at the beginning
+of blocks.
+
+Every now and then, an exception is made for declarations inside a
+#ifdef or #ifndef block: if the code looks nicer, such declarations can
+be placed at the top of the block even if there are statements above.
+On the other hand, however, it's often best to move that #ifdef/#ifndef
+block to a separate function altogether.
 
 6. Conditional statements
 
diff --git a/MAINTAINERS b/MAINTAINERS
index 978b7174f..e8cee1e26 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -62,14 +62,22 @@ Guest CPU cores (TCG):
 ----------------------
 Overall
 L: qemu-devel@nongnu.org
-S: Odd fixes
+M: Paolo Bonzini <pbonzini@redhat.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+M: Richard Henderson <rth@twiddle.net>
+S: Maintained
 F: cpu-exec.c
+F: cpu-exec-common.c
+F: cpus.c
 F: cputlb.c
+F: exec.c
 F: softmmu_template.h
-F: translate-all.c
-F: include/exec/cpu_ldst.h
-F: include/exec/cpu_ldst_template.h
+F: translate-all.*
+F: translate-common.c
+F: include/exec/cpu*.h
+F: include/exec/exec-all.h
 F: include/exec/helper*.h
+F: include/exec/tb-hash.h
 
 Alpha
 M: Richard Henderson <rth@twiddle.net>
@@ -77,13 +85,18 @@ S: Maintained
 F: target-alpha/
 F: hw/alpha/
 F: tests/tcg/alpha/
+F: disas/alpha.c
 
 ARM
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: target-arm/
 F: hw/arm/
 F: hw/cpu/a*mpcore.c
+F: disas/arm.c
+F: disas/arm-a64.cc
+F: disas/libvixl/
 
 CRIS
 M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
@@ -91,6 +104,7 @@ S: Maintained
 F: target-cris/
 F: hw/cris/
 F: tests/tcg/cris/
+F: disas/cris.c
 
 LM32
 M: Michael Walle <michael@walle.cc>
@@ -114,6 +128,7 @@ M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
 S: Maintained
 F: target-microblaze/
 F: hw/microblaze/
+F: disas/microblaze.c
 
 MIPS
 M: Aurelien Jarno <aurelien@aurel32.net>
@@ -122,11 +137,13 @@ S: Maintained
 F: target-mips/
 F: hw/mips/
 F: tests/tcg/mips/
+F: disas/mips.c
 
 Moxie
 M: Anthony Green <green@moxielogic.com>
 S: Maintained
 F: target-moxie/
+F: disas/moxie.c
 
 OpenRISC
 M: Jia Liu <proljc@gmail.com>
@@ -141,6 +158,7 @@ L: qemu-ppc@nongnu.org
 S: Maintained
 F: target-ppc/
 F: hw/ppc/
+F: disas/ppc.c
 
 S390
 M: Richard Henderson <rth@twiddle.net>
@@ -148,12 +166,14 @@ M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/
 F: hw/s390x/
+F: disas/s390.c
 
 SH4
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Odd Fixes
 F: target-sh4/
 F: hw/sh4/
+F: disas/sh4.c
 
 SPARC
 M: Blue Swirl <blauwirbel@gmail.com>
@@ -162,6 +182,7 @@ S: Maintained
 F: target-sparc/
 F: hw/sparc/
 F: hw/sparc64/
+F: disas/sparc.c
 
 UniCore32
 M: Guan Xuetao <gxt@mprc.pku.edu.cn>
@@ -176,6 +197,7 @@ M: Eduardo Habkost <ehabkost@redhat.com>
 S: Maintained
 F: target-i386/
 F: hw/i386/
+F: disas/i386.c
 
 Xtensa
 M: Max Filippov <jcmvbkbc@gmail.com>
@@ -203,6 +225,7 @@ F: */kvm.*
 
 ARM
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: target-arm/kvm.c
 
@@ -222,9 +245,14 @@ M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/kvm.c
+F: target-s390x/ioinst.[ch]
+F: target-s390x/machine.c
 F: hw/intc/s390_flic.c
 F: hw/intc/s390_flic_kvm.c
 F: include/hw/s390x/s390_flic.h
+F: gdb-xml/s390*.xml
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 X86
 M: Paolo Bonzini <pbonzini@redhat.com>
@@ -242,6 +270,12 @@ L: xen-devel@lists.xensource.com
 S: Supported
 F: xen-*
 F: */xen*
+F: hw/char/xen_console.c
+F: hw/display/xenfb.c
+F: hw/net/xen_nic.c
+F: hw/xen/
+F: hw/xenpv/
+F: include/hw/xen/
 
 Hosts:
 ------
@@ -262,103 +296,163 @@ L: qemu-devel@nongnu.org
 M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: *win32*
+F: qemu.nsi
 
 ARM Machines
 ------------
 Allwinner-a10
-M: Li Guang <lig.fnst@cn.fujitsu.com>
+M: Beniamino Galvani <b.galvani@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
-F: hw/*/allwinner-a10*
-F: include/hw/*/allwinner-a10*
+F: hw/*/allwinner*
+F: include/hw/*/allwinner*
 F: hw/arm/cubieboard.c
 
+ARM PrimeCell
+M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/char/pl011.c
+F: hw/display/pl110*
+F: hw/dma/pl080.c
+F: hw/dma/pl330.c
+F: hw/gpio/pl061.c
+F: hw/input/pl050.c
+F: hw/intc/pl190.c
+F: hw/sd/pl181.c
+F: hw/timer/pl031.c
+F: include/hw/arm/primecell.h
+
+ARM cores
+M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/intc/arm*
+F: hw/intc/gic_internal.h
+F: hw/misc/a9scu.c
+F: hw/misc/arm11scu.c
+F: hw/timer/a9gtimer*
+F: hw/timer/arm_*
+F: include/hw/arm/arm.h
+F: include/hw/intc/arm*
+F: include/hw/misc/a9scu.h
+F: include/hw/misc/arm11scu.h
+F: include/hw/timer/a9gtimer.h
+F: include/hw/timer/arm_mptimer.h
+
 Exynos
 M: Evgeny Voevodin <e.voevodin@samsung.com>
 M: Maksim Kozlov <m.kozlov@samsung.com>
 M: Igor Mitsyanko <i.mitsyanko@gmail.com>
 M: Dmitry Solodkiy <d.solodkiy@samsung.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/exynos*
 
 Calxeda Highbank
 M: Rob Herring <robh@kernel.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/highbank.c
 F: hw/net/xgmac.c
 
 Canon DIGIC
 M: Antony Pavlov <antonynpavlov@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: include/hw/arm/digic.h
 F: hw/*/digic*
 
 Gumstix
 L: qemu-devel@nongnu.org
+L: qemu-arm@nongnu.org
 S: Orphan
 F: hw/arm/gumstix.c
 
 i.MX31
 M: Peter Chubb <peter.chubb@nicta.com.au>
+L: qemu-arm@nongnu.org
 S: Odd fixes
 F: hw/*/imx*
 F: hw/arm/kzm.c
 
 Integrator CP
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/integratorcp.c
 
-Mainstone
-L: qemu-devel@nongnu.org
-S: Orphan
-F: hw/arm/mainstone.c
-
 Musicpal
 M: Jan Kiszka <jan.kiszka@web.de>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/musicpal.c
 
 nSeries
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/nseries.c
 
 Palm
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/palm.c
 
 Real View
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/realview*
+F: hw/intc/realview_gic.c
+F: include/hw/intc/realview_gic.h
 
-Spitz
+PXA2XX
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
+F: hw/arm/mainstone.c
 F: hw/arm/spitz.c
+F: hw/arm/tosa.c
+F: hw/arm/z2.c
+F: hw/*/pxa2xx*
 
 Stellaris
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/stellaris*
 
 Versatile PB
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/versatile*
 
 Xilinx Zynq
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/xilinx_zynq.c
 F: hw/misc/zynq_slcr.c
 F: hw/*/cadence_*
 F: hw/ssi/xilinx_spips.c
 
+Xilinx ZynqMP
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/arm/xlnx-zynqmp.c
+F: hw/arm/xlnx-ep108.c
+F: include/hw/arm/xlnx-zynqmp.h
+
 ARM ACPI Subsystem
 M: Shannon Zhao <zhaoshenglong@huawei.com>
 M: Shannon Zhao <shannon.zhao@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/virt-acpi-build.c
 F: include/hw/arm/virt-acpi-build.h
@@ -405,7 +499,7 @@ S: Maintained
 F: hw/microblaze/petalogix_s3adsp1800_mmu.c
 
 petalogix_ml605
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
 S: Maintained
 F: hw/microblaze/petalogix_ml605_mmu.c
 
@@ -555,14 +649,14 @@ M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Christian Borntraeger <borntraeger@de.ibm.com>
 M: Alexander Graf <agraf@suse.de>
 S: Supported
-F: hw/s390x/s390-virtio-ccw.c
-F: hw/s390x/css.[hc]
-F: hw/s390x/sclp*.[hc]
-F: hw/s390x/ipl*.[hc]
-F: hw/s390x/*pci*.[hc]
+F: hw/char/sclp*.[hc]
+F: hw/s390x/
+X: hw/s390x/s390-virtio-bus.[ch]
 F: include/hw/s390x/
 F: pc-bios/s390-ccw/
-T: git git://github.com/cohuck/qemu virtio-ccw-upstr
+F: hw/watchdog/wdt_diag288.c
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 UniCore32 Machines
 -------------
@@ -591,6 +685,25 @@ F: hw/acpi/piix4.c
 F: hw/acpi/ich9.c
 F: include/hw/acpi/ich9.h
 F: include/hw/acpi/piix.h
+F: hw/misc/sga.c
+
+PC Chipset
+M: Michael S. Tsirkin <mst@redhat.com>
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Support
+F: hw/char/debugcon.c
+F: hw/char/parallel.c
+F: hw/char/serial*
+F: hw/dma/i8257*
+F: hw/i2c/pm_smbus.c
+F: hw/intc/apic*
+F: hw/intc/ioapic*
+F: hw/intc/i8259*
+F: hw/misc/debugexit.c
+F: hw/misc/pc-testdev.c
+F: hw/timer/hpet*
+F: hw/timer/i8254*
+F: hw/timer/mc146818rtc*
 
 
 Xtensa Machines
@@ -624,6 +737,7 @@ F: hw/block/cdrom.c
 F: hw/block/hd-geometry.c
 F: tests/ide-test.c
 F: tests/ahci-test.c
+F: tests/libqos/ahci*
 T: git git://github.com/jnsnow/qemu.git ide
 
 Floppy
@@ -632,6 +746,7 @@ L: qemu-block@nongnu.org
 S: Supported
 F: hw/block/fdc.c
 F: include/hw/block/fdc.h
+F: tests/fdc-test.c
 T: git git://github.com/jnsnow/qemu.git ide
 
 OMAP
@@ -639,19 +754,29 @@ M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: hw/*/omap*
 
+IPack
+M: Alberto Garcia <berto@igalia.com>
+S: Odd Fixes
+F: hw/char/ipoctal232.c
+F: hw/ipack/
+
 PCI
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
 F: include/hw/pci/*
+F: hw/misc/pci-testdev.c
 F: hw/pci/*
+F: hw/pci-bridge/*
 
-ACPI
+ACPI/SMBIOS
 M: Michael S. Tsirkin <mst@redhat.com>
 M: Igor Mammedov <imammedo@redhat.com>
 S: Supported
 F: include/hw/acpi/*
+F: include/hw/smbios/*
 F: hw/mem/*
 F: hw/acpi/*
+F: hw/smbios/*
 F: hw/i386/acpi-build.[hc]
 F: hw/i386/*dsl
 F: hw/arm/virt-acpi-build.c
@@ -670,6 +795,19 @@ M: Scott Wood <scottwood@freescale.com>
 L: qemu-ppc@nongnu.org
 S: Supported
 F: hw/ppc/e500*
+F: hw/pci-host/ppce500.c
+F: hw/net/fsl_etsec/
+
+Character devices
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Odd Fixes
+F: hw/char/
+
+Network devices
+M: Jason Wang <jasowang@redhat.com>
+S: Odd Fixes
+F: hw/net/
+T: git git://github.com/jasowang/qemu.git net
 
 SCSI
 M: Paolo Bonzini <pbonzini@redhat.com>
@@ -683,10 +821,17 @@ S: Orphan
 F: hw/scsi/lsi53c895a.c
 
 SSI
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 S: Maintained
 F: hw/ssi/*
 F: hw/block/m25p80.c
+X: hw/ssi/xilinx_*
+
+Xilinx SPI
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+S: Maintained
+F: hw/ssi/xilinx_*
 
 USB
 M: Gerd Hoffmann <kraxel@redhat.com>
@@ -718,6 +863,7 @@ F: net/vhost-user.c
 
 virtio-9p
 M: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+M: Greg Kurz <gkurz@linux.vnet.ibm.com>
 S: Supported
 F: hw/9pfs/
 F: fsdev/
@@ -738,7 +884,8 @@ M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Christian Borntraeger <borntraeger@de.ibm.com>
 S: Supported
 F: hw/s390x/virtio-ccw.[hc]
-T: git git://github.com/cohuck/qemu virtio-ccw-upstr
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 virtio-input
 M: Gerd Hoffmann <kraxel@redhat.com>
@@ -775,8 +922,9 @@ F: hw/scsi/megasas.c
 F: hw/scsi/mfi.h
 
 Xilinx EDK
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
 M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 S: Maintained
 F: hw/*/xilinx_*
 F: include/hw/xilinx.h
@@ -847,7 +995,7 @@ F: block/qapi.c
 F: qapi/block*.json
 T: git git://repo.or.cz/qemu/armbru.git block-next
 
-Character Devices
+Character device backends
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Maintained
 F: qemu-char.c
@@ -878,11 +1026,19 @@ F: include/hw/cpu/icc_bus.h
 F: hw/cpu/icc_bus.c
 
 Device Tree
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: device_tree.[ch]
 
+Error reporting
+M: Markus Armbruster <armbru@redhat.com>
+S: Supported
+F: include/qapi/error.h
+F: include/qemu/error-report.h
+F: util/error.c
+F: util/qemu-error.c
+
 GDB stub
 L: qemu-devel@nongnu.org
 S: Odd Fixes
@@ -895,6 +1051,7 @@ S: Supported
 F: include/exec/ioport.h
 F: ioport.c
 F: include/exec/memory.h
+F: include/exec/ram_addr.h
 F: memory.c
 F: include/exec/memory-internal.h
 F: exec.c
@@ -934,12 +1091,11 @@ F: hmp.c
 F: hmp-commands.hx
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
-Network device layer
-M: Stefan Hajnoczi <stefanha@redhat.com>
+Network device backends
 M: Jason Wang <jasowang@redhat.com>
 S: Maintained
 F: net/
-T: git git://github.com/stefanha/qemu.git net
+T: git git://github.com/jasowang/qemu.git net
 
 Netmap network backend
 M: Luigi Rizzo <rizzo@iet.unipi.it>
@@ -972,7 +1128,12 @@ M: Michael Roth <mdroth@linux.vnet.ibm.com>
 S: Supported
 F: qapi/
 X: qapi/*.json
+F: include/qapi/
+X: include/qapi/qmp/
+F: include/qapi/qmp/dispatch.h
 F: tests/qapi-schema/
+F: tests/test-*-visitor.c
+F: tests/test-qmp-*.c
 F: scripts/qapi*
 F: docs/qapi*
 T: git git://repo.or.cz/qemu/armbru.git qapi-next
@@ -989,6 +1150,14 @@ QObject
 M: Luiz Capitulino <lcapitulino@redhat.com>
 S: Maintained
 F: qobject/
+F: include/qapi/qmp/
+X: include/qapi/qmp/dispatch.h
+F: tests/check-qdict.c
+F: tests/check-qfloat.c
+F: tests/check-qint.c
+F: tests/check-qjson.c
+F: tests/check-qlist.c
+F: tests/check-qstring.c
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
 QEMU Guest Agent
@@ -1005,6 +1174,8 @@ F: include/qom/
 X: include/qom/cpu.h
 F: qom/
 X: qom/cpu.c
+F: tests/check-qom-interface.c
+F: tests/check-qom-proplist.c
 F: tests/qom-test.c
 
 QMP
@@ -1013,7 +1184,7 @@ S: Supported
 F: qmp.c
 F: monitor.c
 F: qmp-commands.hx
-F: docs/qmp/
+F: docs/*qmp-*
 F: scripts/qmp/
 T: git git://repo.or.cz/qemu/armbru.git qapi-next
 
@@ -1059,6 +1230,19 @@ F: crypto/
 F: include/crypto/
 F: tests/test-crypto-*
 
+Coroutines
+M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Kevin Wolf <kwolf@redhat.com>
+F: util/*coroutine*
+F: include/qemu/coroutine*
+F: tests/test-coroutine.c
+
+Buffers
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Odd fixes
+F: util/buffer.c
+F: include/qemu/buffer.h
+
 Usermode Emulation
 ------------------
 Overall
@@ -1088,54 +1272,61 @@ AArch64 target
 M: Claudio Fontana <claudio.fontana@huawei.com>
 M: Claudio Fontana <claudio.fontana@gmail.com>
 S: Maintained
+L: qemu-arm@nongnu.org
 F: tcg/aarch64/
+F: disas/arm-a64.cc
+F: disas/libvixl/
 
 ARM target
 M: Andrzej Zaborowski <balrogg@gmail.com>
 S: Maintained
+L: qemu-arm@nongnu.org
 F: tcg/arm/
+F: disas/arm.c
 
 i386 target
 L: qemu-devel@nongnu.org
 S: Maintained
 F: tcg/i386/
+F: disas/i386.c
 
 IA64 target
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Maintained
 F: tcg/ia64/
+F: disas/ia64.c
 
 MIPS target
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Maintained
 F: tcg/mips/
+F: disas/mips.c
 
 PPC
 M: Vassili Karpov (malc) <av1474@comtv.ru>
 S: Maintained
 F: tcg/ppc/
-
-PPC64 target
-M: Vassili Karpov (malc) <av1474@comtv.ru>
-S: Maintained
-F: tcg/ppc64/
+F: disas/ppc.c
 
 S390 target
 M: Alexander Graf <agraf@suse.de>
 M: Richard Henderson <rth@twiddle.net>
 S: Maintained
 F: tcg/s390/
+F: disas/s390.c
 
 SPARC target
 M: Blue Swirl <blauwirbel@gmail.com>
 S: Maintained
 F: tcg/sparc/
+F: disas/sparc.c
 
 TCI target
 M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: tcg/tci/
 F: tci.c
+F: disas/tci.c
 
 Stable branches
 ---------------
@@ -1288,6 +1479,7 @@ F: block/dmg.c
 
 parallels
 M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Denis V. Lunev <den@openvz.org>
 L: qemu-block@nongnu.org
 S: Supported
 F: block/parallels.c
@@ -1344,3 +1536,11 @@ M: Stefan Hajnoczi <stefanha@redhat.com>
 L: qemu-block@nongnu.org
 S: Supported
 F: tests/image-fuzzer/
+
+
+Documentation
+-------------
+Build system architecture
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Odd Fixes
+F: docs/build-system.txt
diff --git a/Makefile b/Makefile
index 340d9c8fa..930ac2796 100644
--- a/Makefile
+++ b/Makefile
@@ -52,6 +52,8 @@ endif
 GENERATED_HEADERS = config-host.h qemu-options.def
 GENERATED_HEADERS += qmp-commands.h qapi-types.h qapi-visit.h qapi-event.h
 GENERATED_SOURCES += qmp-marshal.c qapi-types.c qapi-visit.c qapi-event.c
+GENERATED_HEADERS += qmp-introspect.h
+GENERATED_SOURCES += qmp-introspect.c
 
 GENERATED_HEADERS += trace/generated-events.h
 GENERATED_SOURCES += trace/generated-events.c
@@ -88,7 +90,8 @@ LIBS+=-lz $(LIBS_TOOLS)
 HELPERS-$(CONFIG_LINUX) = qemu-bridge-helper$(EXESUF)
 
 ifdef BUILD_DOCS
-DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 qmp-commands.txt
+DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 qemu-ga.8
+DOCS+=qmp-commands.txt
 ifdef CONFIG_LINUX
 DOCS+=kvm_stat.1
 endif
@@ -148,18 +151,20 @@ dummy := $(call unnest-vars,, \
                 stub-obj-y \
                 util-obj-y \
                 qga-obj-y \
+                ivshmem-client-obj-y \
+                ivshmem-server-obj-y \
                 qga-vss-dll-obj-y \
                 block-obj-y \
                 block-obj-m \
+                crypto-obj-y \
+                crypto-aes-obj-y \
+                qom-obj-y \
                 common-obj-y \
                 common-obj-m)
 
 ifneq ($(wildcard config-host.mak),)
 include $(SRC_PATH)/tests/Makefile
 endif
-ifeq ($(CONFIG_SMARTCARD_NSS),y)
-include $(SRC_PATH)/libcacard/Makefile
-endif
 
 all: $(DOCS) $(TOOLS) $(HELPERS-y) recurse-all modules
 
@@ -172,6 +177,7 @@ SUBDIR_RULES=$(patsubst %,subdir-%, $(TARGET_DIRS))
 SOFTMMU_SUBDIR_RULES=$(filter %-softmmu,$(SUBDIR_RULES))
 
 $(SOFTMMU_SUBDIR_RULES): $(block-obj-y)
+$(SOFTMMU_SUBDIR_RULES): $(crypto-obj-y)
 $(SOFTMMU_SUBDIR_RULES): config-all-devices.mak
 
 subdir-%:
@@ -196,7 +202,7 @@ subdir-dtc:dtc/libfdt dtc/tests
 dtc/%:
 	mkdir -p $@
 
-$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y)
+$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y) $(qom-obj-y) $(crypto-aes-obj-$(CONFIG_USER_ONLY))
 
 ROMSUBDIR_RULES=$(patsubst %,romsubdir-%, $(ROMS))
 romsubdir-%:
@@ -226,9 +232,9 @@ util/module.o-cflags = -D'CONFIG_BLOCK_MODULES=$(block-modules)'
 
 qemu-img.o: qemu-img-cmds.h
 
-qemu-img$(EXESUF): qemu-img.o $(block-obj-y) libqemuutil.a libqemustub.a
-qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) libqemuutil.a libqemustub.a
-qemu-io$(EXESUF): qemu-io.o $(block-obj-y) libqemuutil.a libqemustub.a
+qemu-img$(EXESUF): qemu-img.o $(block-obj-y) $(crypto-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
+qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) $(crypto-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
+qemu-io$(EXESUF): qemu-io.o $(block-obj-y) $(crypto-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
 
 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
 
@@ -263,7 +269,7 @@ $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 
 qapi-modules = $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/qapi/common.json \
                $(SRC_PATH)/qapi/block.json $(SRC_PATH)/qapi/block-core.json \
-               $(SRC_PATH)/qapi/event.json
+               $(SRC_PATH)/qapi/event.json $(SRC_PATH)/qapi/introspect.json
 
 qapi-types.c qapi-types.h :\
 $(qapi-modules) $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
@@ -285,6 +291,11 @@ $(qapi-modules) $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
 		$(gen-out-type) -o "." -m $<, \
 		"  GEN   $@")
+qmp-introspect.h qmp-introspect.c :\
+$(qapi-modules) $(SRC_PATH)/scripts/qapi-introspect.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-introspect.py \
+		$(gen-out-type) -o "." $<, \
+		"  GEN   $@")
 
 QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)
 $(qga-obj-y) qemu-ga.o: $(QGALIB_GEN)
@@ -295,24 +306,30 @@ qemu-ga$(EXESUF): $(qga-obj-y) libqemuutil.a libqemustub.a
 ifdef QEMU_GA_MSI_ENABLED
 QEMU_GA_MSI=qemu-ga-$(ARCH).msi
 
-msi: ${QEMU_GA_MSI}
+msi: $(QEMU_GA_MSI)
 
-$(QEMU_GA_MSI): qemu-ga.exe
-
-ifdef QEMU_GA_MSI_WITH_VSS
-$(QEMU_GA_MSI): qga/vss-win32/qga-vss.dll
-endif
+$(QEMU_GA_MSI): qemu-ga.exe $(QGA_VSS_PROVIDER)
 
 $(QEMU_GA_MSI): config-host.mak
 
-$(QEMU_GA_MSI):  qga/installer/qemu-ga.wxs
-	$(call quiet-command,QEMU_GA_VERSION="$(QEMU_GA_VERSION)" QEMU_GA_MANUFACTURER="$(QEMU_GA_MANUFACTURER)" QEMU_GA_DISTRO="$(QEMU_GA_DISTRO)" \
+$(QEMU_GA_MSI):  $(SRC_PATH)/qga/installer/qemu-ga.wxs
+	$(call quiet-command,QEMU_GA_VERSION="$(QEMU_GA_VERSION)" QEMU_GA_MANUFACTURER="$(QEMU_GA_MANUFACTURER)" QEMU_GA_DISTRO="$(QEMU_GA_DISTRO)" BUILD_DIR="$(BUILD_DIR)" \
 	wixl -o $@ $(QEMU_GA_MSI_ARCH) $(QEMU_GA_MSI_WITH_VSS) $(QEMU_GA_MSI_MINGW_DLL_PATH) $<, "  WIXL  $@")
 else
 msi:
-	@echo MSI build not configured or dependency resolution failed (reconfigure with --enable-guest-agent-msi option)
+	@echo "MSI build not configured or dependency resolution failed (reconfigure with --enable-guest-agent-msi option)"
 endif
 
+ifneq ($(EXESUF),)
+.PHONY: qemu-ga
+qemu-ga: qemu-ga$(EXESUF) $(QGA_VSS_PROVIDER) $(QEMU_GA_MSI)
+endif
+
+ivshmem-client$(EXESUF): $(ivshmem-client-obj-y)
+	$(call LINK, $^)
+ivshmem-server$(EXESUF): $(ivshmem-server-obj-y) libqemuutil.a libqemustub.a
+	$(call LINK, $^)
+
 clean:
 # avoid old build problems by removing potentially incorrect old files
 	rm -f config.mak op-i386.h opc-i386.h gen-op-i386.h op-arm.h opc-arm.h gen-op-arm.h
@@ -344,7 +361,7 @@ qemu-%.tar.bz2:
 	$(SRC_PATH)/scripts/make-release "$(SRC_PATH)" "$(patsubst qemu-%.tar.bz2,%,$@)"
 
 distclean: clean
-	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi
+	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi qemu-monitor-info.texi
 	rm -f config-all-devices.mak config-all-disas.mak config.status
 	rm -f po/*.mo tests/qemu-iotests/common.env
 	rm -f roms/seabios/config.mak roms/vgabios/config.mak
@@ -400,6 +417,9 @@ ifneq ($(TOOLS),)
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man8"
 	$(INSTALL_DATA) qemu-nbd.8 "$(DESTDIR)$(mandir)/man8"
 endif
+ifneq (,$(findstring qemu-ga,$(TOOLS)))
+	$(INSTALL_DATA) qemu-ga.8 "$(DESTDIR)$(mandir)/man8"
+endif
 endif
 ifdef CONFIG_VIRTFS
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man1"
@@ -420,7 +440,7 @@ endif
 install: all $(if $(BUILD_DOCS),install-doc) \
 install-datadir install-localstatedir
 ifneq ($(TOOLS),)
-	$(call install-prog,$(TOOLS),$(DESTDIR)$(bindir))
+	$(call install-prog,$(subst qemu-ga,qemu-ga$(EXESUF),$(TOOLS)),$(DESTDIR)$(bindir))
 endif
 ifneq ($(CONFIG_MODULES),)
 	$(INSTALL_DIR) "$(DESTDIR)$(qemu_moddir)"
@@ -508,13 +528,16 @@ qemu-options.texi: $(SRC_PATH)/qemu-options.hx
 qemu-monitor.texi: $(SRC_PATH)/hmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
+qemu-monitor-info.texi: $(SRC_PATH)/hmp-commands-info.hx
+	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
+
 qmp-commands.txt: $(SRC_PATH)/qmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -q < $< > $@,"  GEN   $@")
 
 qemu-img-cmds.texi: $(SRC_PATH)/qemu-img-cmds.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
-qemu.1: qemu-doc.texi qemu-options.texi qemu-monitor.texi
+qemu.1: qemu-doc.texi qemu-options.texi qemu-monitor.texi qemu-monitor-info.texi
 	$(call quiet-command, \
 	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu.pod && \
 	  $(POD2MAN) --section=1 --center=" " --release=" " qemu.pod > $@, \
@@ -538,6 +561,12 @@ qemu-nbd.8: qemu-nbd.texi
 	  $(POD2MAN) --section=8 --center=" " --release=" " qemu-nbd.pod > $@, \
 	  "  GEN   $@")
 
+qemu-ga.8: qemu-ga.texi
+	$(call quiet-command, \
+	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu-ga.pod && \
+	  $(POD2MAN) --section=8 --center=" " --release=" " qemu-ga.pod > $@, \
+	  "  GEN   $@")
+
 kvm_stat.1: scripts/kvm/kvm_stat.texi
 	$(call quiet-command, \
 	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< kvm_stat.pod && \
@@ -551,7 +580,8 @@ pdf: qemu-doc.pdf qemu-tech.pdf
 
 qemu-doc.dvi qemu-doc.html qemu-doc.info qemu-doc.pdf: \
 	qemu-img.texi qemu-nbd.texi qemu-options.texi \
-	qemu-monitor.texi qemu-img-cmds.texi
+	qemu-monitor.texi qemu-img-cmds.texi qemu-ga.texi \
+	qemu-monitor-info.texi
 
 ifdef CONFIG_WIN32
 
@@ -601,6 +631,7 @@ endif # SIGNCODE
                 $(if $(DLL_PATH),-DDLLDIR="$(DLL_PATH)") \
                 -DSRCDIR="$(SRC_PATH)" \
                 -DOUTFILE="$(INSTALLER)" \
+                -DDISPLAYVERSION="$(VERSION)" \
                 $(SRC_PATH)/qemu.nsi
 	rm -r ${INSTDIR}
 ifdef SIGNCODE
diff --git a/Makefile.objs b/Makefile.objs
index f094eff15..77be052dd 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -1,8 +1,8 @@
 #######################################################################
 # Common libraries for tools and emulators
 stub-obj-y = stubs/
-util-obj-y = util/ qobject/ qapi/ qapi-types.o qapi-visit.o qapi-event.o
-util-obj-y += crypto/
+util-obj-y = util/ qobject/ qapi/
+util-obj-y += qmp-introspect.o qapi-types.o qapi-visit.o qapi-event.o
 
 #######################################################################
 # block-obj-y is code used by both qemu system emulation and qemu-img
@@ -15,24 +15,18 @@ block-obj-$(CONFIG_WIN32) += aio-win32.o
 block-obj-y += block/
 block-obj-y += qemu-io-cmds.o
 
-block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
-block-obj-y += qemu-coroutine-sleep.o
-block-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
-
 block-obj-m = block/
 
+#######################################################################
+# crypto-obj-y is code used by both qemu system emulation and qemu-img
 
-######################################################################
-# smartcard
+crypto-obj-y = crypto/
+crypto-aes-obj-y = crypto/
+
+#######################################################################
+# qom-obj-y is code used by both qemu system emulation and qemu-img
 
-libcacard-y += libcacard/cac.o libcacard/event.o
-libcacard-y += libcacard/vcard.o libcacard/vreader.o
-libcacard-y += libcacard/vcard_emul_nss.o
-libcacard-y += libcacard/vcard_emul_type.o
-libcacard-y += libcacard/card_7816.o
-libcacard-y += libcacard/vcardt.o
-libcacard/vcard_emul_nss.o-cflags := $(NSS_CFLAGS)
-libcacard/vcard_emul_nss.o-libs := $(NSS_LIBS)
+qom-obj-y = qom/
 
 ######################################################################
 # Target independent part of system emulation. The long term path is to
@@ -60,6 +54,8 @@ common-obj-y += audio/
 common-obj-y += hw/
 common-obj-y += accel.o
 
+common-obj-y += replay/
+
 common-obj-y += ui/
 common-obj-y += bt-host.o bt-vhci.o
 bt-host.o-cflags := $(BLUEZ_CFLAGS)
@@ -75,14 +71,13 @@ common-obj-y += backends/
 
 common-obj-$(CONFIG_SECCOMP) += qemu-seccomp.o
 
-common-obj-$(CONFIG_SMARTCARD_NSS) += $(libcacard-y)
-
 common-obj-$(CONFIG_FDT) += device_tree.o
 
 ######################################################################
 # qapi
 
 common-obj-y += qmp-marshal.o
+common-obj-y += qmp-introspect.o
 common-obj-y += qmp.o hmp.o
 endif
 
@@ -111,3 +106,8 @@ target-obj-y += trace/
 # by libqemuutil.a.  These should be moved to a separate .json schema.
 qga-obj-y = qga/
 qga-vss-dll-obj-y = qga/
+
+######################################################################
+# contrib
+ivshmem-client-obj-y = contrib/ivshmem-client/
+ivshmem-server-obj-y = contrib/ivshmem-server/
diff --git a/Makefile.target b/Makefile.target
index 3e7aafd72..962d0045f 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -7,7 +7,7 @@ include config-target.mak
 include config-devices.mak
 include $(SRC_PATH)/rules.mak
 
-$(call set-vpath, $(SRC_PATH))
+$(call set-vpath, $(SRC_PATH):$(BUILD_DIR))
 ifdef CONFIG_LINUX
 QEMU_CFLAGS += -I../linux-headers
 endif
@@ -85,8 +85,11 @@ all: $(PROGS) stap
 #########################################################
 # cpu emulator library
 obj-y = exec.o translate-all.o cpu-exec.o
+obj-y += translate-common.o
+obj-y += cpu-exec-common.o
 obj-y += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o
 obj-$(CONFIG_TCG_INTERPRETER) += tci.o
+obj-y += tcg/tcg-common.o
 obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o
 obj-y += fpu/softfloat.o
 obj-y += target-$(TARGET_BASE_ARCH)/
@@ -151,7 +154,7 @@ else
 obj-y += hw/$(TARGET_BASE_ARCH)/
 endif
 
-GENERATED_HEADERS += hmp-commands.h qmp-commands-old.h
+GENERATED_HEADERS += hmp-commands.h hmp-commands-info.h qmp-commands-old.h
 
 endif # CONFIG_SOFTMMU
 
@@ -170,12 +173,18 @@ target-obj-y-save := $(target-obj-y)
 dummy := $(call unnest-vars,.., \
                block-obj-y \
                block-obj-m \
+               crypto-obj-y \
+               crypto-aes-obj-y \
+               qom-obj-y \
                common-obj-y \
                common-obj-m)
 target-obj-y := $(target-obj-y-save)
 all-obj-y += $(common-obj-y)
 all-obj-y += $(target-obj-y)
+all-obj-y += $(qom-obj-y)
 all-obj-$(CONFIG_SOFTMMU) += $(block-obj-y)
+all-obj-$(CONFIG_USER_ONLY) += $(crypto-aes-obj-y)
+all-obj-$(CONFIG_SOFTMMU) += $(crypto-obj-y)
 
 $(QEMU_PROG_BUILD): config-devices.mak
 
@@ -193,6 +202,9 @@ gdbstub-xml.c: $(TARGET_XML_FILES) $(SRC_PATH)/scripts/feature_to_c.sh
 hmp-commands.h: $(SRC_PATH)/hmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
 
+hmp-commands-info.h: $(SRC_PATH)/hmp-commands-info.hx
+	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
+
 qmp-commands-old.h: $(SRC_PATH)/qmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
 
diff --git a/README b/README
index c7c990d89..f38193fc6 100644
--- a/README
+++ b/README
@@ -1,3 +1,107 @@
-Read the documentation in qemu-doc.html or on http://wiki.qemu-project.org
+         QEMU README
+         ===========
 
-- QEMU team
+QEMU is a generic and open source machine & userspace emulator and
+virtualizer.
+
+QEMU is capable of emulating a complete machine in software without any
+need for hardware virtualization support. By using dynamic translation,
+it achieves very good performance. QEMU can also integrate with the Xen
+and KVM hypervisors to provide emulated hardware while allowing the
+hypervisor to manage the CPU. With hypervisor support, QEMU can achieve
+near native performance for CPUs. When QEMU emulates CPUs directly it is
+capable of running operating systems made for one machine (e.g. an ARMv7
+board) on a different machine (e.g. an x86_64 PC board).
+
+QEMU is also capable of providing userspace API virtualization for Linux
+and BSD kernel interfaces. This allows binaries compiled against one
+architecture ABI (e.g. the Linux PPC64 ABI) to be run on a host using a
+different architecture ABI (e.g. the Linux x86_64 ABI). This does not
+involve any hardware emulation, simply CPU and syscall emulation.
+
+QEMU aims to fit into a variety of use cases. It can be invoked directly
+by users wishing to have full control over its behaviour and settings.
+It also aims to facilitate integration into higher level management
+layers, by providing a stable command line interface and monitor API.
+It is commonly invoked indirectly via the libvirt library when using
+open source applications such as oVirt, OpenStack and virt-manager.
+
+QEMU as a whole is released under the GNU General Public License,
+version 2. For full licensing details, consult the LICENSE file.
+
+
+Building
+========
+
+QEMU is multi-platform software intended to be buildable on all modern
+Linux platforms, OS-X, Win32 (via the Mingw64 toolchain) and a variety
+of other UNIX targets. The simple steps to build QEMU are:
+
+  mkdir build
+  cd build
+  ../configure
+  make
+
+Complete details of the process for building and configuring QEMU for
+all supported host platforms can be found in the qemu-tech.html file.
+Additional information can also be found online via the QEMU website:
+
+  http://qemu-project.org/Hosts/Linux
+  http://qemu-project.org/Hosts/W32
+
+
+Submitting patches
+==================
+
+The QEMU source code is maintained under the GIT version control system.
+
+   git clone git://git.qemu-project.org/qemu.git
+
+When submitting patches, the preferred approach is to use 'git
+format-patch' and/or 'git send-email' to format & send the mail to the
+qemu-devel@nongnu.org mailing list. All patches submitted must contain
+a 'Signed-off-by' line from the author. Patches should follow the
+guidelines set out in the HACKING and CODING_STYLE files.
+
+Additional information on submitting patches can be found online via
+the QEMU website
+
+  http://qemu-project.org/Contribute/SubmitAPatch
+  http://qemu-project.org/Contribute/TrivialPatches
+
+
+Bug reporting
+=============
+
+The QEMU project uses Launchpad as its primary upstream bug tracker. Bugs
+found when running code built from QEMU git or upstream released sources
+should be reported via:
+
+  https://bugs.launchpad.net/qemu/
+
+If using QEMU via an operating system vendor pre-built binary package, it
+is preferable to report bugs to the vendor's own bug tracker first. If
+the bug is also known to affect latest upstream code, it can also be
+reported via launchpad.
+
+For additional information on bug reporting consult:
+
+  http://qemu-project.org/Contribute/ReportABug
+
+
+Contact
+=======
+
+The QEMU community can be contacted in a number of ways, with the two
+main methods being email and IRC
+
+ - qemu-devel@nongnu.org
+   http://lists.nongnu.org/mailman/listinfo/qemu-devel
+ - #qemu on irc.oftc.net
+
+Information on additional methods of contacting the community can be
+found online via the QEMU website:
+
+  http://qemu-project.org/Contribute/StartHere
+
+-- End
diff --git a/VERSION b/VERSION
index 005119baa..3a6d2147d 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4.1
+2.5.1.1
diff --git a/aio-posix.c b/aio-posix.c
index d4770336c..482b31650 100644
--- a/aio-posix.c
+++ b/aio-posix.c
@@ -17,6 +17,9 @@
 #include "block/block.h"
 #include "qemu/queue.h"
 #include "qemu/sockets.h"
+#ifdef CONFIG_EPOLL
+#include <sys/epoll.h>
+#endif
 
 struct AioHandler
 {
@@ -25,9 +28,166 @@ struct AioHandler
     IOHandler *io_write;
     int deleted;
     void *opaque;
+    bool is_external;
     QLIST_ENTRY(AioHandler) node;
 };
 
+#ifdef CONFIG_EPOLL
+
+/* The fd number threashold to switch to epoll */
+#define EPOLL_ENABLE_THRESHOLD 64
+
+static void aio_epoll_disable(AioContext *ctx)
+{
+    ctx->epoll_available = false;
+    if (!ctx->epoll_enabled) {
+        return;
+    }
+    ctx->epoll_enabled = false;
+    close(ctx->epollfd);
+}
+
+static inline int epoll_events_from_pfd(int pfd_events)
+{
+    return (pfd_events & G_IO_IN ? EPOLLIN : 0) |
+           (pfd_events & G_IO_OUT ? EPOLLOUT : 0) |
+           (pfd_events & G_IO_HUP ? EPOLLHUP : 0) |
+           (pfd_events & G_IO_ERR ? EPOLLERR : 0);
+}
+
+static bool aio_epoll_try_enable(AioContext *ctx)
+{
+    AioHandler *node;
+    struct epoll_event event;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        int r;
+        if (node->deleted || !node->pfd.events) {
+            continue;
+        }
+        event.events = epoll_events_from_pfd(node->pfd.events);
+        event.data.ptr = node;
+        r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event);
+        if (r) {
+            return false;
+        }
+    }
+    ctx->epoll_enabled = true;
+    return true;
+}
+
+static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
+{
+    struct epoll_event event;
+    int r;
+
+    if (!ctx->epoll_enabled) {
+        return;
+    }
+    if (!node->pfd.events) {
+        r = epoll_ctl(ctx->epollfd, EPOLL_CTL_DEL, node->pfd.fd, &event);
+        if (r) {
+            aio_epoll_disable(ctx);
+        }
+    } else {
+        event.data.ptr = node;
+        event.events = epoll_events_from_pfd(node->pfd.events);
+        if (is_new) {
+            r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event);
+            if (r) {
+                aio_epoll_disable(ctx);
+            }
+        } else {
+            r = epoll_ctl(ctx->epollfd, EPOLL_CTL_MOD, node->pfd.fd, &event);
+            if (r) {
+                aio_epoll_disable(ctx);
+            }
+        }
+    }
+}
+
+static int aio_epoll(AioContext *ctx, GPollFD *pfds,
+                     unsigned npfd, int64_t timeout)
+{
+    AioHandler *node;
+    int i, ret = 0;
+    struct epoll_event events[128];
+
+    assert(npfd == 1);
+    assert(pfds[0].fd == ctx->epollfd);
+    if (timeout > 0) {
+        ret = qemu_poll_ns(pfds, npfd, timeout);
+    }
+    if (timeout <= 0 || ret > 0) {
+        ret = epoll_wait(ctx->epollfd, events,
+                         sizeof(events) / sizeof(events[0]),
+                         timeout);
+        if (ret <= 0) {
+            goto out;
+        }
+        for (i = 0; i < ret; i++) {
+            int ev = events[i].events;
+            node = events[i].data.ptr;
+            node->pfd.revents = (ev & EPOLLIN ? G_IO_IN : 0) |
+                (ev & EPOLLOUT ? G_IO_OUT : 0) |
+                (ev & EPOLLHUP ? G_IO_HUP : 0) |
+                (ev & EPOLLERR ? G_IO_ERR : 0);
+        }
+    }
+out:
+    return ret;
+}
+
+static bool aio_epoll_enabled(AioContext *ctx)
+{
+    /* Fall back to ppoll when external clients are disabled. */
+    return !aio_external_disabled(ctx) && ctx->epoll_enabled;
+}
+
+static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
+                                 unsigned npfd, int64_t timeout)
+{
+    if (!ctx->epoll_available) {
+        return false;
+    }
+    if (aio_epoll_enabled(ctx)) {
+        return true;
+    }
+    if (npfd >= EPOLL_ENABLE_THRESHOLD) {
+        if (aio_epoll_try_enable(ctx)) {
+            return true;
+        } else {
+            aio_epoll_disable(ctx);
+        }
+    }
+    return false;
+}
+
+#else
+
+static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
+{
+}
+
+static int aio_epoll(AioContext *ctx, GPollFD *pfds,
+                     unsigned npfd, int64_t timeout)
+{
+    assert(false);
+}
+
+static bool aio_epoll_enabled(AioContext *ctx)
+{
+    return false;
+}
+
+static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
+                          unsigned npfd, int64_t timeout)
+{
+    return false;
+}
+
+#endif
+
 static AioHandler *find_aio_handler(AioContext *ctx, int fd)
 {
     AioHandler *node;
@@ -43,11 +203,14 @@ static AioHandler *find_aio_handler(AioContext *ctx, int fd)
 
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque)
 {
     AioHandler *node;
+    bool is_new = false;
+    bool deleted = false;
 
     node = find_aio_handler(ctx, fd);
 
@@ -66,7 +229,7 @@ void aio_set_fd_handler(AioContext *ctx,
                  * releasing the walking_handlers lock.
                  */
                 QLIST_REMOVE(node, node);
-                g_free(node);
+                deleted = true;
             }
         }
     } else {
@@ -77,25 +240,32 @@ void aio_set_fd_handler(AioContext *ctx,
             QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
 
             g_source_add_poll(&ctx->source, &node->pfd);
+            is_new = true;
         }
         /* Update handler with latest information */
         node->io_read = io_read;
         node->io_write = io_write;
         node->opaque = opaque;
+        node->is_external = is_external;
 
         node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
         node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
     }
 
+    aio_epoll_update(ctx, node, is_new);
     aio_notify(ctx);
+    if (deleted) {
+        g_free(node);
+    }
 }
 
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *notifier,
+                            bool is_external,
                             EventNotifierHandler *io_read)
 {
     aio_set_fd_handler(ctx, event_notifier_get_fd(notifier),
-                       (IOHandler *)io_read, NULL, notifier);
+                       is_external, (IOHandler *)io_read, NULL, notifier);
 }
 
 bool aio_prepare(AioContext *ctx)
@@ -257,7 +427,9 @@ bool aio_poll(AioContext *ctx, bool blocking)
 
     /* fill pollfds */
     QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-        if (!node->deleted && node->pfd.events) {
+        if (!node->deleted && node->pfd.events
+            && !aio_epoll_enabled(ctx)
+            && aio_node_check(ctx, node->is_external)) {
             add_pollfd(node);
         }
     }
@@ -268,7 +440,17 @@ bool aio_poll(AioContext *ctx, bool blocking)
     if (timeout) {
         aio_context_release(ctx);
     }
-    ret = qemu_poll_ns((GPollFD *)pollfds, npfd, timeout);
+    if (aio_epoll_check_poll(ctx, pollfds, npfd, timeout)) {
+        AioHandler epoll_handler;
+
+        epoll_handler.pfd.fd = ctx->epollfd;
+        epoll_handler.pfd.events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR;
+        npfd = 0;
+        add_pollfd(&epoll_handler);
+        ret = aio_epoll(ctx, pollfds, npfd, timeout);
+    } else  {
+        ret = qemu_poll_ns(pollfds, npfd, timeout);
+    }
     if (blocking) {
         atomic_sub(&ctx->notify_me, 2);
     }
@@ -297,3 +479,16 @@ bool aio_poll(AioContext *ctx, bool blocking)
 
     return progress;
 }
+
+void aio_context_setup(AioContext *ctx, Error **errp)
+{
+#ifdef CONFIG_EPOLL
+    assert(!ctx->epollfd);
+    ctx->epollfd = epoll_create1(EPOLL_CLOEXEC);
+    if (ctx->epollfd == -1) {
+        ctx->epoll_available = false;
+    } else {
+        ctx->epoll_available = true;
+    }
+#endif
+}
diff --git a/aio-win32.c b/aio-win32.c
index 50a686745..cdc445608 100644
--- a/aio-win32.c
+++ b/aio-win32.c
@@ -28,11 +28,13 @@ struct AioHandler {
     GPollFD pfd;
     int deleted;
     void *opaque;
+    bool is_external;
     QLIST_ENTRY(AioHandler) node;
 };
 
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque)
@@ -86,6 +88,7 @@ void aio_set_fd_handler(AioContext *ctx,
         node->opaque = opaque;
         node->io_read = io_read;
         node->io_write = io_write;
+        node->is_external = is_external;
 
         event = event_notifier_get_handle(&ctx->notifier);
         WSAEventSelect(node->pfd.fd, event,
@@ -98,6 +101,7 @@ void aio_set_fd_handler(AioContext *ctx,
 
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *e,
+                            bool is_external,
                             EventNotifierHandler *io_notify)
 {
     AioHandler *node;
@@ -133,6 +137,7 @@ void aio_set_event_notifier(AioContext *ctx,
             node->e = e;
             node->pfd.fd = (uintptr_t)event_notifier_get_handle(e);
             node->pfd.events = G_IO_IN;
+            node->is_external = is_external;
             QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
 
             g_source_add_poll(&ctx->source, &node->pfd);
@@ -304,7 +309,8 @@ bool aio_poll(AioContext *ctx, bool blocking)
     /* fill fd sets */
     count = 0;
     QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-        if (!node->deleted && node->io_notify) {
+        if (!node->deleted && node->io_notify
+            && aio_node_check(ctx, node->is_external)) {
             events[count++] = event_notifier_get_handle(node->e);
         }
     }
@@ -363,3 +369,7 @@ bool aio_poll(AioContext *ctx, bool blocking)
     aio_context_release(ctx);
     return progress;
 }
+
+void aio_context_setup(AioContext *ctx, Error **errp)
+{
+}
diff --git a/arch_init.c b/arch_init.c
index 725c638ec..38f5fb9c2 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -26,7 +26,7 @@
 #include "sysemu/arch_init.h"
 #include "hw/pci/pci.h"
 #include "hw/audio/audio.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
 #include "qmp-commands.h"
diff --git a/async.c b/async.c
index efce14b63..e106072a4 100644
--- a/async.c
+++ b/async.c
@@ -59,6 +59,11 @@ QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque)
     return bh;
 }
 
+void aio_bh_call(QEMUBH *bh)
+{
+    bh->cb(bh->opaque);
+}
+
 /* Multiple occurrences of aio_bh_poll cannot be called concurrently */
 int aio_bh_poll(AioContext *ctx)
 {
@@ -84,7 +89,7 @@ int aio_bh_poll(AioContext *ctx)
                 ret = 1;
             }
             bh->idle = 0;
-            bh->cb(bh->opaque);
+            aio_bh_call(bh);
         }
     }
 
@@ -247,7 +252,7 @@ aio_ctx_finalize(GSource     *source)
     }
     qemu_mutex_unlock(&ctx->bh_lock);
 
-    aio_set_event_notifier(ctx, &ctx->notifier, NULL);
+    aio_set_event_notifier(ctx, &ctx->notifier, false, NULL);
     event_notifier_cleanup(&ctx->notifier);
     rfifolock_destroy(&ctx->lock);
     qemu_mutex_destroy(&ctx->bh_lock);
@@ -320,15 +325,22 @@ AioContext *aio_context_new(Error **errp)
 {
     int ret;
     AioContext *ctx;
+    Error *local_err = NULL;
+
     ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext));
+    aio_context_setup(ctx, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto fail;
+    }
     ret = event_notifier_init(&ctx->notifier, false);
     if (ret < 0) {
-        g_source_destroy(&ctx->source);
         error_setg_errno(errp, -ret, "Failed to initialize event notifier");
-        return NULL;
+        goto fail;
     }
     g_source_set_can_recurse(&ctx->source, true);
     aio_set_event_notifier(ctx, &ctx->notifier,
+                           false,
                            (EventNotifierHandler *)
                            event_notifier_dummy_cb);
     ctx->thread_pool = NULL;
@@ -339,6 +351,9 @@ AioContext *aio_context_new(Error **errp)
     ctx->notify_dummy_bh = aio_bh_new(ctx, notify_dummy_bh, NULL);
 
     return ctx;
+fail:
+    g_source_destroy(&ctx->source);
+    return NULL;
 }
 
 void aio_context_ref(AioContext *ctx)
diff --git a/backends/baum.c b/backends/baum.c
index a69aafff4..723c658ac 100644
--- a/backends/baum.c
+++ b/backends/baum.c
@@ -303,7 +303,7 @@ static int baum_eat_packet(BaumDriverState *baum, const uint8_t *buf, int len)
                 return 0;
             cur++;
         }
-        DPRINTF("Dropped %d bytes!\n", cur - buf);
+        DPRINTF("Dropped %td bytes!\n", cur - buf);
     }
 
 #define EAT(c) do {\
@@ -561,7 +561,10 @@ static void baum_close(struct CharDriverState *chr)
     g_free(baum);
 }
 
-CharDriverState *chr_baum_init(void)
+static CharDriverState *chr_baum_init(const char *id,
+                                      ChardevBackend *backend,
+                                      ChardevReturn *ret,
+                                      Error **errp)
 {
     BaumDriverState *baum;
     CharDriverState *chr;
@@ -586,14 +589,16 @@ CharDriverState *chr_baum_init(void)
 
     baum->brlapi_fd = brlapi__openConnection(handle, NULL, NULL);
     if (baum->brlapi_fd == -1) {
-        brlapi_perror("baum_init: brlapi_openConnection");
+        error_setg(errp, "brlapi__openConnection: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail_handle;
     }
 
     baum->cellCount_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, baum_cellCount_timer_cb, baum);
 
     if (brlapi__getDisplaySize(handle, &baum->x, &baum->y) == -1) {
-        brlapi_perror("baum_init: brlapi_getDisplaySize");
+        error_setg(errp, "brlapi__getDisplaySize: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail;
     }
 
@@ -609,7 +614,8 @@ CharDriverState *chr_baum_init(void)
         tty = BRLAPI_TTY_DEFAULT;
 
     if (brlapi__enterTtyMode(handle, tty, NULL) == -1) {
-        brlapi_perror("baum_init: brlapi_enterTtyMode");
+        error_setg(errp, "brlapi__enterTtyMode: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail;
     }
 
@@ -629,7 +635,8 @@ fail_handle:
 
 static void register_types(void)
 {
-    register_char_driver("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL);
+    register_char_driver("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL,
+                         chr_baum_init);
 }
 
 type_init(register_types);
diff --git a/backends/hostmem-file.c b/backends/hostmem-file.c
index 4b5536101..e9b6d21da 100644
--- a/backends/hostmem-file.c
+++ b/backends/hostmem-file.c
@@ -83,9 +83,7 @@ static void set_mem_path(Object *o, const char *str, Error **errp)
         error_setg(errp, "cannot change property value");
         return;
     }
-    if (fb->mem_path) {
-        g_free(fb->mem_path);
-    }
+    g_free(fb->mem_path);
     fb->mem_path = g_strdup(str);
 }
 
diff --git a/backends/hostmem.c b/backends/hostmem.c
index 41ba2af52..1b4eb4581 100644
--- a/backends/hostmem.c
+++ b/backends/hostmem.c
@@ -313,9 +313,11 @@ host_memory_backend_memory_complete(UserCreatable *uc, Error **errp)
         assert(maxnode <= MAX_NODES);
         if (mbind(ptr, sz, backend->policy,
                   maxnode ? backend->host_nodes : NULL, maxnode + 1, flags)) {
-            error_setg_errno(errp, errno,
-                             "cannot bind memory to host NUMA nodes");
-            return;
+            if (backend->policy != MPOL_DEFAULT || errno != ENOSYS) {
+                error_setg_errno(errp, errno,
+                                 "cannot bind memory to host NUMA nodes");
+                return;
+            }
         }
 #endif
         /* Preallocate memory after the NUMA policy has been instantiated.
diff --git a/backends/msmouse.c b/backends/msmouse.c
index 0119110a4..0126fa0b1 100644
--- a/backends/msmouse.c
+++ b/backends/msmouse.c
@@ -63,7 +63,10 @@ static void msmouse_chr_close (struct CharDriverState *chr)
     g_free (chr);
 }
 
-CharDriverState *qemu_chr_open_msmouse(void)
+static CharDriverState *qemu_chr_open_msmouse(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
     CharDriverState *chr;
 
@@ -79,7 +82,8 @@ CharDriverState *qemu_chr_open_msmouse(void)
 
 static void register_types(void)
 {
-    register_char_driver("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL);
+    register_char_driver("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL,
+                         qemu_chr_open_msmouse);
 }
 
 type_init(register_types);
diff --git a/backends/testdev.c b/backends/testdev.c
index eba396aeb..26d5c7307 100644
--- a/backends/testdev.c
+++ b/backends/testdev.c
@@ -108,13 +108,16 @@ static void testdev_close(struct CharDriverState *chr)
     g_free(testdev);
 }
 
-CharDriverState *chr_testdev_init(void)
+static CharDriverState *chr_testdev_init(const char *id,
+                                         ChardevBackend *backend,
+                                         ChardevReturn *ret,
+                                         Error **errp)
 {
     TestdevCharState *testdev;
     CharDriverState *chr;
 
-    testdev = g_malloc0(sizeof(TestdevCharState));
-    testdev->chr = chr = g_malloc0(sizeof(CharDriverState));
+    testdev = g_new0(TestdevCharState, 1);
+    testdev->chr = chr = g_new0(CharDriverState, 1);
 
     chr->opaque = testdev;
     chr->chr_write = testdev_write;
@@ -125,7 +128,8 @@ CharDriverState *chr_testdev_init(void)
 
 static void register_types(void)
 {
-    register_char_driver("testdev", CHARDEV_BACKEND_KIND_TESTDEV, NULL);
+    register_char_driver("testdev", CHARDEV_BACKEND_KIND_TESTDEV, NULL,
+                         chr_testdev_init);
 }
 
 type_init(register_types);
diff --git a/balloon.c b/balloon.c
index 5d69e8a00..0f45d1b5c 100644
--- a/balloon.c
+++ b/balloon.c
@@ -36,6 +36,17 @@
 static QEMUBalloonEvent *balloon_event_fn;
 static QEMUBalloonStatus *balloon_stat_fn;
 static void *balloon_opaque;
+static bool balloon_inhibited;
+
+bool qemu_balloon_is_inhibited(void)
+{
+    return balloon_inhibited;
+}
+
+void qemu_balloon_inhibit(bool state)
+{
+    balloon_inhibited = state;
+}
 
 static bool have_balloon(Error **errp)
 {
diff --git a/block.c b/block.c
index d088ee02f..3c172dd6a 100644
--- a/block.c
+++ b/block.c
@@ -33,7 +33,7 @@
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "qemu/notify.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/qapi.h"
 #include "qmp-commands.h"
 #include "qemu/timer.h"
@@ -73,8 +73,7 @@ struct BdrvDirtyBitmap {
 
 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
 
-static QTAILQ_HEAD(, BlockDriverState) bdrv_states =
-    QTAILQ_HEAD_INITIALIZER(bdrv_states);
+struct BdrvStates bdrv_states = QTAILQ_HEAD_INITIALIZER(bdrv_states);
 
 static QTAILQ_HEAD(, BlockDriverState) graph_bdrv_states =
     QTAILQ_HEAD_INITIALIZER(graph_bdrv_states);
@@ -85,8 +84,7 @@ static QLIST_HEAD(, BlockDriver) bdrv_drivers =
 static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
                              const char *reference, QDict *options, int flags,
                              BlockDriverState *parent,
-                             const BdrvChildRole *child_role,
-                             BlockDriver *drv, Error **errp);
+                             const BdrvChildRole *child_role, Error **errp);
 
 static void bdrv_dirty_bitmap_truncate(BlockDriverState *bs);
 /* If non-zero, use only whitelisted block drivers */
@@ -258,7 +256,6 @@ BlockDriverState *bdrv_new(void)
     for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
         QLIST_INIT(&bs->op_blockers[i]);
     }
-    bdrv_iostatus_disable(bs);
     notifier_list_init(&bs->close_notifiers);
     notifier_with_return_list_init(&bs->before_write_notifiers);
     qemu_co_queue_init(&bs->throttled_reqs[0]);
@@ -314,13 +311,6 @@ static int bdrv_is_whitelisted(BlockDriver *drv, bool read_only)
     return 0;
 }
 
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool read_only)
-{
-    BlockDriver *drv = bdrv_find_format(format_name);
-    return drv && bdrv_is_whitelisted(drv, read_only) ? drv : NULL;
-}
-
 typedef struct CreateCo {
     BlockDriver *drv;
     char *filename;
@@ -729,7 +719,7 @@ const BdrvChildRole child_format = {
 };
 
 /*
- * Returns the flags that bs->backing_hd should get, based on the given flags
+ * Returns the flags that bs->backing should get, based on the given flags
  * for the parent BDS
  */
 static int bdrv_backing_flags(int flags)
@@ -771,12 +761,15 @@ static void bdrv_assign_node_name(BlockDriverState *bs,
                                   const char *node_name,
                                   Error **errp)
 {
-    if (!node_name) {
-        return;
-    }
+    char *gen_node_name = NULL;
 
-    /* Check for empty string or invalid characters */
-    if (!id_wellformed(node_name)) {
+    if (!node_name) {
+        node_name = gen_node_name = id_generate(ID_BLOCK);
+    } else if (!id_wellformed(node_name)) {
+        /*
+         * Check for empty string or invalid characters, but not if it is
+         * generated (generated names use characters not available to the user)
+         */
         error_setg(errp, "Invalid node name");
         return;
     }
@@ -785,18 +778,20 @@ static void bdrv_assign_node_name(BlockDriverState *bs,
     if (blk_by_name(node_name)) {
         error_setg(errp, "node-name=%s is conflicting with a device id",
                    node_name);
-        return;
+        goto out;
     }
 
     /* takes care of avoiding duplicates node names */
     if (bdrv_find_node(node_name)) {
         error_setg(errp, "Duplicate node name");
-        return;
+        goto out;
     }
 
     /* copy node name into the bs and insert it into the graph list */
     pstrcpy(bs->node_name, sizeof(bs->node_name), node_name);
     QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs, node_list);
+out:
+    g_free(gen_node_name);
 }
 
 static QemuOptsList bdrv_runtime_opts = {
@@ -817,7 +812,7 @@ static QemuOptsList bdrv_runtime_opts = {
  *
  * Removes all processed options from *options.
  */
-static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
+static int bdrv_open_common(BlockDriverState *bs, BdrvChild *file,
     QDict *options, int flags, BlockDriver *drv, Error **errp)
 {
     int ret, open_flags;
@@ -831,7 +826,7 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
     assert(options != NULL && bs->options != options);
 
     if (file != NULL) {
-        filename = file->filename;
+        filename = file->bs->filename;
     } else {
         filename = qdict_get_try_str(options, "filename");
     }
@@ -860,7 +855,6 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
         goto fail_opts;
     }
 
-    bs->guest_block_size = 512;
     bs->request_alignment = 512;
     bs->zero_beyond_eof = true;
     open_flags = bdrv_open_flags(bs, flags);
@@ -997,13 +991,13 @@ static QDict *parse_json_filename(const char *filename, Error **errp)
  * block driver has been specified explicitly.
  */
 static int bdrv_fill_options(QDict **options, const char **pfilename,
-                             int *flags, BlockDriver *drv, Error **errp)
+                             int *flags, Error **errp)
 {
     const char *filename = *pfilename;
     const char *drvname;
     bool protocol = *flags & BDRV_O_PROTOCOL;
     bool parse_filename = false;
-    BlockDriver *tmp_drv;
+    BlockDriver *drv = NULL;
     Error *local_err = NULL;
 
     /* Parse json: pseudo-protocol */
@@ -1022,15 +1016,15 @@ static int bdrv_fill_options(QDict **options, const char **pfilename,
     }
 
     drvname = qdict_get_try_str(*options, "driver");
-
-    /* If the user has explicitly specified the driver, this choice should
-     * override the BDRV_O_PROTOCOL flag */
-    tmp_drv = drv;
-    if (!tmp_drv && drvname) {
-        tmp_drv = bdrv_find_format(drvname);
-    }
-    if (tmp_drv) {
-        protocol = tmp_drv->bdrv_file_open;
+    if (drvname) {
+        drv = bdrv_find_format(drvname);
+        if (!drv) {
+            error_setg(errp, "Unknown driver '%s'", drvname);
+            return -ENOENT;
+        }
+        /* If the user has explicitly specified the driver, this choice should
+         * override the BDRV_O_PROTOCOL flag */
+        protocol = drv->bdrv_file_open;
     }
 
     if (protocol) {
@@ -1054,33 +1048,18 @@ static int bdrv_fill_options(QDict **options, const char **pfilename,
     /* Find the right block driver */
     filename = qdict_get_try_str(*options, "filename");
 
-    if (drv) {
-        if (drvname) {
-            error_setg(errp, "Driver specified twice");
-            return -EINVAL;
-        }
-        drvname = drv->format_name;
-        qdict_put(*options, "driver", qstring_from_str(drvname));
-    } else {
-        if (!drvname && protocol) {
-            if (filename) {
-                drv = bdrv_find_protocol(filename, parse_filename, errp);
-                if (!drv) {
-                    return -EINVAL;
-                }
-
-                drvname = drv->format_name;
-                qdict_put(*options, "driver", qstring_from_str(drvname));
-            } else {
-                error_setg(errp, "Must specify either driver or file");
-                return -EINVAL;
-            }
-        } else if (drvname) {
-            drv = bdrv_find_format(drvname);
+    if (!drvname && protocol) {
+        if (filename) {
+            drv = bdrv_find_protocol(filename, parse_filename, errp);
             if (!drv) {
-                error_setg(errp, "Unknown driver '%s'", drvname);
-                return -ENOENT;
+                return -EINVAL;
             }
+
+            drvname = drv->format_name;
+            qdict_put(*options, "driver", qstring_from_str(drvname));
+        } else {
+            error_setg(errp, "Must specify either driver or file");
+            return -EINVAL;
         }
     }
 
@@ -1099,6 +1078,10 @@ static int bdrv_fill_options(QDict **options, const char **pfilename,
         }
     }
 
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        *flags |= BDRV_O_INCOMING;
+    }
+
     return 0;
 }
 
@@ -1113,6 +1096,7 @@ static BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs,
     };
 
     QLIST_INSERT_HEAD(&parent_bs->children, child, next);
+    QLIST_INSERT_HEAD(&child_bs->parents, child, next_parent);
 
     return child;
 }
@@ -1120,50 +1104,62 @@ static BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs,
 static void bdrv_detach_child(BdrvChild *child)
 {
     QLIST_REMOVE(child, next);
+    QLIST_REMOVE(child, next_parent);
     g_free(child);
 }
 
 void bdrv_unref_child(BlockDriverState *parent, BdrvChild *child)
 {
-    BlockDriverState *child_bs = child->bs;
+    BlockDriverState *child_bs;
+
+    if (child == NULL) {
+        return;
+    }
 
     if (child->bs->inherits_from == parent) {
         child->bs->inherits_from = NULL;
     }
 
+    child_bs = child->bs;
     bdrv_detach_child(child);
     bdrv_unref(child_bs);
 }
 
+/*
+ * Sets the backing file link of a BDS. A new reference is created; callers
+ * which don't need their own reference any more must call bdrv_unref().
+ */
 void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd)
 {
+    if (backing_hd) {
+        bdrv_ref(backing_hd);
+    }
 
-    if (bs->backing_hd) {
+    if (bs->backing) {
         assert(bs->backing_blocker);
-        bdrv_op_unblock_all(bs->backing_hd, bs->backing_blocker);
-        bdrv_detach_child(bs->backing_child);
+        bdrv_op_unblock_all(bs->backing->bs, bs->backing_blocker);
+        bdrv_unref_child(bs, bs->backing);
     } else if (backing_hd) {
         error_setg(&bs->backing_blocker,
                    "node is used as backing hd of '%s'",
                    bdrv_get_device_or_node_name(bs));
     }
 
-    bs->backing_hd = backing_hd;
     if (!backing_hd) {
         error_free(bs->backing_blocker);
         bs->backing_blocker = NULL;
-        bs->backing_child = NULL;
+        bs->backing = NULL;
         goto out;
     }
-    bs->backing_child = bdrv_attach_child(bs, backing_hd, &child_backing);
+    bs->backing = bdrv_attach_child(bs, backing_hd, &child_backing);
     bs->open_flags &= ~BDRV_O_NO_BACKING;
     pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_hd->filename);
     pstrcpy(bs->backing_format, sizeof(bs->backing_format),
             backing_hd->drv ? backing_hd->drv->format_name : "");
 
-    bdrv_op_block_all(bs->backing_hd, bs->backing_blocker);
+    bdrv_op_block_all(backing_hd, bs->backing_blocker);
     /* Otherwise we won't be able to commit due to check in bdrv_commit */
-    bdrv_op_unblock(bs->backing_hd, BLOCK_OP_TYPE_COMMIT_TARGET,
+    bdrv_op_unblock(backing_hd, BLOCK_OP_TYPE_COMMIT_TARGET,
                     bs->backing_blocker);
 out:
     bdrv_refresh_limits(bs, NULL);
@@ -1184,7 +1180,7 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
     BlockDriverState *backing_hd;
     Error *local_err = NULL;
 
-    if (bs->backing_hd != NULL) {
+    if (bs->backing != NULL) {
         QDECREF(options);
         goto free_exit;
     }
@@ -1224,11 +1220,10 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
         qdict_put(options, "driver", qstring_from_str(bs->backing_format));
     }
 
-    assert(bs->backing_hd == NULL);
+    assert(bs->backing == NULL);
     ret = bdrv_open_inherit(&backing_hd,
                             *backing_filename ? backing_filename : NULL,
-                            NULL, options, 0, bs, &child_backing,
-                            NULL, &local_err);
+                            NULL, options, 0, bs, &child_backing, &local_err);
     if (ret < 0) {
         bdrv_unref(backing_hd);
         backing_hd = NULL;
@@ -1239,7 +1234,10 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
         goto free_exit;
     }
 
+    /* Hook up the backing file link; drop our reference, bs owns the
+     * backing_hd reference now */
     bdrv_set_backing_hd(bs, backing_hd);
+    bdrv_unref(backing_hd);
 
 free_exit:
     g_free(backing_filename);
@@ -1291,7 +1289,7 @@ BdrvChild *bdrv_open_child(const char *filename,
 
     bs = NULL;
     ret = bdrv_open_inherit(&bs, filename, reference, image_options, 0,
-                            parent, child_role, NULL, errp);
+                            parent, child_role, errp);
     if (ret < 0) {
         goto done;
     }
@@ -1303,40 +1301,6 @@ done:
     return c;
 }
 
-/*
- * This is a version of bdrv_open_child() that returns 0/-EINVAL instead of
- * a BdrvChild object.
- *
- * If allow_none is true, no image will be opened if filename is false and no
- * BlockdevRef is given. *pbs will remain unchanged and 0 will be returned.
- *
- * To conform with the behavior of bdrv_open(), *pbs has to be NULL.
- */
-int bdrv_open_image(BlockDriverState **pbs, const char *filename,
-                    QDict *options, const char *bdref_key,
-                    BlockDriverState* parent, const BdrvChildRole *child_role,
-                    bool allow_none, Error **errp)
-{
-    Error *local_err = NULL;
-    BdrvChild *c;
-
-    assert(pbs);
-    assert(*pbs == NULL);
-
-    c = bdrv_open_child(filename, options, bdref_key, parent, child_role,
-                        allow_none, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return -EINVAL;
-    }
-
-    if (c != NULL) {
-        *pbs = c->bs;
-    }
-
-    return 0;
-}
-
 int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
 {
     /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
@@ -1385,11 +1349,13 @@ int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
               qstring_from_str("file"));
     qdict_put(snapshot_options, "file.filename",
               qstring_from_str(tmp_filename));
+    qdict_put(snapshot_options, "driver",
+              qstring_from_str("qcow2"));
 
     bs_snapshot = bdrv_new();
 
     ret = bdrv_open(&bs_snapshot, NULL, NULL, snapshot_options,
-                    flags, &bdrv_qcow2, &local_err);
+                    flags, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -1420,12 +1386,14 @@ out:
 static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
                              const char *reference, QDict *options, int flags,
                              BlockDriverState *parent,
-                             const BdrvChildRole *child_role,
-                             BlockDriver *drv, Error **errp)
+                             const BdrvChildRole *child_role, Error **errp)
 {
     int ret;
-    BlockDriverState *file = NULL, *bs;
+    BdrvChild *file = NULL;
+    BlockDriverState *bs;
+    BlockDriver *drv = NULL;
     const char *drvname;
+    const char *backing;
     Error *local_err = NULL;
     int snapshot_flags = 0;
 
@@ -1474,13 +1442,12 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
         flags = child_role->inherit_flags(parent->open_flags);
     }
 
-    ret = bdrv_fill_options(&options, &filename, &flags, drv, &local_err);
+    ret = bdrv_fill_options(&options, &filename, &flags, &local_err);
     if (local_err) {
         goto fail;
     }
 
     /* Find the right image format driver */
-    drv = NULL;
     drvname = qdict_get_try_str(options, "driver");
     if (drvname) {
         drv = bdrv_find_format(drvname);
@@ -1494,6 +1461,12 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
 
     assert(drvname || !(flags & BDRV_O_PROTOCOL));
 
+    backing = qdict_get_try_str(options, "backing");
+    if (backing && *backing == '\0') {
+        flags |= BDRV_O_NO_BACKING;
+        qdict_del(options, "backing");
+    }
+
     bs->open_flags = flags;
     bs->options = options;
     options = qdict_clone_shallow(options);
@@ -1508,11 +1481,12 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
             flags = bdrv_backing_flags(flags);
         }
 
-        assert(file == NULL);
         bs->open_flags = flags;
-        ret = bdrv_open_image(&file, filename, options, "file",
-                              bs, &child_file, true, &local_err);
-        if (ret < 0) {
+
+        file = bdrv_open_child(filename, options, "file", bs,
+                               &child_file, true, &local_err);
+        if (local_err) {
+            ret = -EINVAL;
             goto fail;
         }
     }
@@ -1520,7 +1494,7 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
     /* Image format probing */
     bs->probed = !drv;
     if (!drv && file) {
-        ret = find_image_format(file, filename, &drv, &local_err);
+        ret = find_image_format(file->bs, filename, &drv, &local_err);
         if (ret < 0) {
             goto fail;
         }
@@ -1543,7 +1517,7 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
     }
 
     if (file && (bs->file != file)) {
-        bdrv_unref(file);
+        bdrv_unref_child(bs, file);
         file = NULL;
     }
 
@@ -1560,15 +1534,6 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
 
     bdrv_refresh_filename(bs);
 
-    /* For snapshot=on, create a temporary qcow2 overlay. bs points to the
-     * temporary snapshot afterwards. */
-    if (snapshot_flags) {
-        ret = bdrv_append_temp_snapshot(bs, snapshot_flags, &local_err);
-        if (local_err) {
-            goto close_and_fail;
-        }
-    }
-
     /* Check if any unknown options were used */
     if (options && (qdict_size(options) != 0)) {
         const QDictEntry *entry = qdict_first(options);
@@ -1600,11 +1565,21 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
 
     QDECREF(options);
     *pbs = bs;
+
+    /* For snapshot=on, create a temporary qcow2 overlay. bs points to the
+     * temporary snapshot afterwards. */
+    if (snapshot_flags) {
+        ret = bdrv_append_temp_snapshot(bs, snapshot_flags, &local_err);
+        if (local_err) {
+            goto close_and_fail;
+        }
+    }
+
     return 0;
 
 fail:
     if (file != NULL) {
-        bdrv_unref(file);
+        bdrv_unref_child(bs, file);
     }
     QDECREF(bs->options);
     QDECREF(options);
@@ -1635,11 +1610,10 @@ close_and_fail:
 }
 
 int bdrv_open(BlockDriverState **pbs, const char *filename,
-              const char *reference, QDict *options, int flags,
-              BlockDriver *drv, Error **errp)
+              const char *reference, QDict *options, int flags, Error **errp)
 {
     return bdrv_open_inherit(pbs, filename, reference, options, flags, NULL,
-                             NULL, drv, errp);
+                             NULL, errp);
 }
 
 typedef struct BlockReopenQueueEntry {
@@ -1660,6 +1634,9 @@ typedef struct BlockReopenQueueEntry {
  *
  * bs is the BlockDriverState to add to the reopen queue.
  *
+ * options contains the changed options for the associated bs
+ * (the BlockReopenQueue takes ownership)
+ *
  * flags contains the open flags for the associated bs
  *
  * returns a pointer to bs_queue, which is either the newly allocated
@@ -1667,18 +1644,28 @@ typedef struct BlockReopenQueueEntry {
  *
  */
 BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags)
+                                    BlockDriverState *bs,
+                                    QDict *options, int flags)
 {
     assert(bs != NULL);
 
     BlockReopenQueueEntry *bs_entry;
     BdrvChild *child;
+    QDict *old_options;
 
     if (bs_queue == NULL) {
         bs_queue = g_new0(BlockReopenQueue, 1);
         QSIMPLEQ_INIT(bs_queue);
     }
 
+    if (!options) {
+        options = qdict_new();
+    }
+
+    old_options = qdict_clone_shallow(bs->options);
+    qdict_join(options, old_options, false);
+    QDECREF(old_options);
+
     /* bdrv_open() masks this flag out */
     flags &= ~BDRV_O_PROTOCOL;
 
@@ -1690,13 +1677,15 @@ BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
         }
 
         child_flags = child->role->inherit_flags(flags);
-        bdrv_reopen_queue(bs_queue, child->bs, child_flags);
+        /* TODO Pass down child flags (backing.*, extents.*, ...) */
+        bdrv_reopen_queue(bs_queue, child->bs, NULL, child_flags);
     }
 
     bs_entry = g_new0(BlockReopenQueueEntry, 1);
     QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
 
     bs_entry->state.bs = bs;
+    bs_entry->state.options = options;
     bs_entry->state.flags = flags;
 
     return bs_queue;
@@ -1749,6 +1738,7 @@ cleanup:
         if (ret && bs_entry->prepared) {
             bdrv_reopen_abort(&bs_entry->state);
         }
+        QDECREF(bs_entry->state.options);
         g_free(bs_entry);
     }
     g_free(bs_queue);
@@ -1761,7 +1751,7 @@ int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
 {
     int ret = -1;
     Error *local_err = NULL;
-    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
+    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, NULL, bdrv_flags);
 
     ret = bdrv_reopen_multiple(queue, &local_err);
     if (local_err != NULL) {
@@ -1811,8 +1801,7 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
 
     ret = bdrv_flush(reopen_state->bs);
     if (ret) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
-                  strerror(-ret));
+        error_setg_errno(errp, -ret, "Error flushing drive");
         goto error;
     }
 
@@ -1837,6 +1826,26 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
         goto error;
     }
 
+    /* Options that are not handled are only okay if they are unchanged
+     * compared to the old state. It is expected that some options are only
+     * used for the initial open, but not reopen (e.g. filename) */
+    if (qdict_size(reopen_state->options)) {
+        const QDictEntry *entry = qdict_first(reopen_state->options);
+
+        do {
+            QString *new_obj = qobject_to_qstring(entry->value);
+            const char *new = qstring_get_str(new_obj);
+            const char *old = qdict_get_try_str(reopen_state->bs->options,
+                                                entry->key);
+
+            if (!old || strcmp(new, old)) {
+                error_setg(errp, "Cannot change the option '%s'", entry->key);
+                ret = -EINVAL;
+                goto error;
+            }
+        } while ((entry = qdict_next(reopen_state->options, entry)));
+    }
+
     ret = 0;
 
 error:
@@ -1895,20 +1904,32 @@ void bdrv_close(BlockDriverState *bs)
     if (bs->job) {
         block_job_cancel_sync(bs->job);
     }
+
+    /* Disable I/O limits and drain all pending throttled requests */
+    if (bs->throttle_state) {
+        bdrv_io_limits_disable(bs);
+    }
+
     bdrv_drain(bs); /* complete I/O */
     bdrv_flush(bs);
     bdrv_drain(bs); /* in case flush left pending I/O */
     notifier_list_notify(&bs->close_notifiers, bs);
 
+    if (bs->blk) {
+        blk_dev_change_media_cb(bs->blk, false);
+    }
+
     if (bs->drv) {
         BdrvChild *child, *next;
 
         bs->drv->bdrv_close(bs);
+        bs->drv = NULL;
+
+        bdrv_set_backing_hd(bs, NULL);
 
-        if (bs->backing_hd) {
-            BlockDriverState *backing_hd = bs->backing_hd;
-            bdrv_set_backing_hd(bs, NULL);
-            bdrv_unref(backing_hd);
+        if (bs->file != NULL) {
+            bdrv_unref_child(bs, bs->file);
+            bs->file = NULL;
         }
 
         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
@@ -1922,7 +1943,6 @@ void bdrv_close(BlockDriverState *bs)
 
         g_free(bs->opaque);
         bs->opaque = NULL;
-        bs->drv = NULL;
         bs->copy_on_read = 0;
         bs->backing_file[0] = '\0';
         bs->backing_format[0] = '\0';
@@ -1935,20 +1955,6 @@ void bdrv_close(BlockDriverState *bs)
         bs->options = NULL;
         QDECREF(bs->full_open_options);
         bs->full_open_options = NULL;
-
-        if (bs->file != NULL) {
-            bdrv_unref(bs->file);
-            bs->file = NULL;
-        }
-    }
-
-    if (bs->blk) {
-        blk_dev_change_media_cb(bs->blk, false);
-    }
-
-    /*throttling disk I/O limits*/
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_disable(bs);
     }
 
     QLIST_FOREACH_SAFE(ban, &bs->aio_notifiers, list, ban_next) {
@@ -1970,21 +1976,25 @@ void bdrv_close_all(void)
     }
 }
 
+/* Note that bs->device_list.tqe_prev is initially null,
+ * and gets set to non-null by QTAILQ_INSERT_TAIL().  Establish
+ * the useful invariant "bs in bdrv_states iff bs->tqe_prev" by
+ * resetting it to null on remove.  */
+void bdrv_device_remove(BlockDriverState *bs)
+{
+    QTAILQ_REMOVE(&bdrv_states, bs, device_list);
+    bs->device_list.tqe_prev = NULL;
+}
+
 /* make a BlockDriverState anonymous by removing from bdrv_state and
  * graph_bdrv_state list.
    Also, NULL terminate the device_name to prevent double remove */
 void bdrv_make_anon(BlockDriverState *bs)
 {
-    /*
-     * Take care to remove bs from bdrv_states only when it's actually
-     * in it.  Note that bs->device_list.tqe_prev is initially null,
-     * and gets set to non-null by QTAILQ_INSERT_TAIL().  Establish
-     * the useful invariant "bs in bdrv_states iff bs->tqe_prev" by
-     * resetting it to null on remove.
-     */
+    /* Take care to remove bs from bdrv_states only when it's actually
+     * in it. */
     if (bs->device_list.tqe_prev) {
-        QTAILQ_REMOVE(&bdrv_states, bs, device_list);
-        bs->device_list.tqe_prev = NULL;
+        bdrv_device_remove(bs);
     }
     if (bs->node_name[0] != '\0') {
         QTAILQ_REMOVE(&graph_bdrv_states, bs, node_list);
@@ -1992,167 +2002,58 @@ void bdrv_make_anon(BlockDriverState *bs)
     bs->node_name[0] = '\0';
 }
 
-static void bdrv_rebind(BlockDriverState *bs)
-{
-    if (bs->drv && bs->drv->bdrv_rebind) {
-        bs->drv->bdrv_rebind(bs);
-    }
-}
-
+/* Fields that need to stay with the top-level BDS */
 static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
                                      BlockDriverState *bs_src)
 {
     /* move some fields that need to stay attached to the device */
 
     /* dev info */
-    bs_dest->guest_block_size   = bs_src->guest_block_size;
     bs_dest->copy_on_read       = bs_src->copy_on_read;
 
     bs_dest->enable_write_cache = bs_src->enable_write_cache;
 
-    /* i/o throttled req */
-    bs_dest->throttle_state     = bs_src->throttle_state,
-    bs_dest->io_limits_enabled  = bs_src->io_limits_enabled;
-    bs_dest->pending_reqs[0]    = bs_src->pending_reqs[0];
-    bs_dest->pending_reqs[1]    = bs_src->pending_reqs[1];
-    bs_dest->throttled_reqs[0]  = bs_src->throttled_reqs[0];
-    bs_dest->throttled_reqs[1]  = bs_src->throttled_reqs[1];
-    memcpy(&bs_dest->round_robin,
-           &bs_src->round_robin,
-           sizeof(bs_dest->round_robin));
-    memcpy(&bs_dest->throttle_timers,
-           &bs_src->throttle_timers,
-           sizeof(ThrottleTimers));
-
-    /* r/w error */
-    bs_dest->on_read_error      = bs_src->on_read_error;
-    bs_dest->on_write_error     = bs_src->on_write_error;
-
-    /* i/o status */
-    bs_dest->iostatus_enabled   = bs_src->iostatus_enabled;
-    bs_dest->iostatus           = bs_src->iostatus;
-
     /* dirty bitmap */
     bs_dest->dirty_bitmaps      = bs_src->dirty_bitmaps;
-
-    /* reference count */
-    bs_dest->refcnt             = bs_src->refcnt;
-
-    /* job */
-    bs_dest->job                = bs_src->job;
-
-    /* keep the same entry in bdrv_states */
-    bs_dest->device_list = bs_src->device_list;
-    bs_dest->blk = bs_src->blk;
-
-    memcpy(bs_dest->op_blockers, bs_src->op_blockers,
-           sizeof(bs_dest->op_blockers));
 }
 
-/*
- * Swap bs contents for two image chains while they are live,
- * while keeping required fields on the BlockDriverState that is
- * actually attached to a device.
- *
- * This will modify the BlockDriverState fields, and swap contents
- * between bs_new and bs_old. Both bs_new and bs_old are modified.
- *
- * bs_new must not be attached to a BlockBackend.
- *
- * This function does not create any image files.
- */
-void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
+static void change_parent_backing_link(BlockDriverState *from,
+                                       BlockDriverState *to)
 {
-    BlockDriverState tmp;
-    BdrvChild *child;
-
-    bdrv_drain(bs_new);
-    bdrv_drain(bs_old);
+    BdrvChild *c, *next;
 
-    /* The code needs to swap the node_name but simply swapping node_list won't
-     * work so first remove the nodes from the graph list, do the swap then
-     * insert them back if needed.
-     */
-    if (bs_new->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs_new, node_list);
-    }
-    if (bs_old->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs_old, node_list);
+    QLIST_FOREACH_SAFE(c, &from->parents, next_parent, next) {
+        assert(c->role != &child_backing);
+        c->bs = to;
+        QLIST_REMOVE(c, next_parent);
+        QLIST_INSERT_HEAD(&to->parents, c, next_parent);
+        bdrv_ref(to);
+        bdrv_unref(from);
     }
-
-    /* If the BlockDriverState is part of a throttling group acquire
-     * its lock since we're going to mess with the protected fields.
-     * Otherwise there's no need to worry since no one else can touch
-     * them. */
-    if (bs_old->throttle_state) {
-        throttle_group_lock(bs_old);
+    if (from->blk) {
+        blk_set_bs(from->blk, to);
+        if (!to->device_list.tqe_prev) {
+            QTAILQ_INSERT_BEFORE(from, to, device_list);
+        }
+        bdrv_device_remove(from);
     }
+}
 
-    /* bs_new must be unattached and shouldn't have anything fancy enabled */
-    assert(!bs_new->blk);
-    assert(QLIST_EMPTY(&bs_new->dirty_bitmaps));
-    assert(bs_new->job == NULL);
-    assert(bs_new->io_limits_enabled == false);
-    assert(bs_new->throttle_state == NULL);
-    assert(!throttle_timers_are_initialized(&bs_new->throttle_timers));
-
-    tmp = *bs_new;
-    *bs_new = *bs_old;
-    *bs_old = tmp;
+static void swap_feature_fields(BlockDriverState *bs_top,
+                                BlockDriverState *bs_new)
+{
+    BlockDriverState tmp;
 
-    /* there are some fields that should not be swapped, move them back */
-    bdrv_move_feature_fields(&tmp, bs_old);
-    bdrv_move_feature_fields(bs_old, bs_new);
+    bdrv_move_feature_fields(&tmp, bs_top);
+    bdrv_move_feature_fields(bs_top, bs_new);
     bdrv_move_feature_fields(bs_new, &tmp);
 
-    /* bs_new must remain unattached */
-    assert(!bs_new->blk);
-
-    /* Check a few fields that should remain attached to the device */
-    assert(bs_new->job == NULL);
-    assert(bs_new->io_limits_enabled == false);
-    assert(bs_new->throttle_state == NULL);
-    assert(!throttle_timers_are_initialized(&bs_new->throttle_timers));
-
-    /* Release the ThrottleGroup lock */
-    if (bs_old->throttle_state) {
-        throttle_group_unlock(bs_old);
-    }
-
-    /* insert the nodes back into the graph node list if needed */
-    if (bs_new->node_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs_new, node_list);
-    }
-    if (bs_old->node_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs_old, node_list);
+    assert(!bs_new->throttle_state);
+    if (bs_top->throttle_state) {
+        assert(bs_top->io_limits_enabled);
+        bdrv_io_limits_enable(bs_new, throttle_group_get_name(bs_top));
+        bdrv_io_limits_disable(bs_top);
     }
-
-    /*
-     * Update lh_first.le_prev for non-empty lists.
-     *
-     * The head of the op blocker list doesn't change because it is moved back
-     * in bdrv_move_feature_fields().
-     */
-    assert(QLIST_EMPTY(&bs_old->tracked_requests));
-    assert(QLIST_EMPTY(&bs_new->tracked_requests));
-
-    QLIST_FIX_HEAD_PTR(&bs_new->children, next);
-    QLIST_FIX_HEAD_PTR(&bs_old->children, next);
-
-    /* Update references in bs->opaque and children */
-    QLIST_FOREACH(child, &bs_old->children, next) {
-        if (child->bs->inherits_from == bs_new) {
-            child->bs->inherits_from = bs_old;
-        }
-    }
-    QLIST_FOREACH(child, &bs_new->children, next) {
-        if (child->bs->inherits_from == bs_old) {
-            child->bs->inherits_from = bs_new;
-        }
-    }
-
-    bdrv_rebind(bs_new);
-    bdrv_rebind(bs_old);
 }
 
 /*
@@ -2165,14 +2066,59 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
  * bs_new must not be attached to a BlockBackend.
  *
  * This function does not create any image files.
+ *
+ * bdrv_append() takes ownership of a bs_new reference and unrefs it because
+ * that's what the callers commonly need. bs_new will be referenced by the old
+ * parents of bs_top after bdrv_append() returns. If the caller needs to keep a
+ * reference of its own, it must call bdrv_ref().
  */
 void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top)
 {
-    bdrv_swap(bs_new, bs_top);
+    assert(!bdrv_requests_pending(bs_top));
+    assert(!bdrv_requests_pending(bs_new));
+
+    bdrv_ref(bs_top);
+    change_parent_backing_link(bs_top, bs_new);
 
-    /* The contents of 'tmp' will become bs_top, as we are
-     * swapping bs_new and bs_top contents. */
-    bdrv_set_backing_hd(bs_top, bs_new);
+    /* Some fields always stay on top of the backing file chain */
+    swap_feature_fields(bs_top, bs_new);
+
+    bdrv_set_backing_hd(bs_new, bs_top);
+    bdrv_unref(bs_top);
+
+    /* bs_new is now referenced by its new parents, we don't need the
+     * additional reference any more. */
+    bdrv_unref(bs_new);
+}
+
+void bdrv_replace_in_backing_chain(BlockDriverState *old, BlockDriverState *new)
+{
+    assert(!bdrv_requests_pending(old));
+    assert(!bdrv_requests_pending(new));
+
+    bdrv_ref(old);
+
+    if (old->blk) {
+        /* As long as these fields aren't in BlockBackend, but in the top-level
+         * BlockDriverState, it's not possible for a BDS to have two BBs.
+         *
+         * We really want to copy the fields from old to new, but we go for a
+         * swap instead so that pointers aren't duplicated and cause trouble.
+         * (Also, bdrv_swap() used to do the same.) */
+        assert(!new->blk);
+        swap_feature_fields(old, new);
+    }
+    change_parent_backing_link(old, new);
+
+    /* Change backing files if a previously independent node is added to the
+     * chain. For active commit, we replace top by its own (indirect) backing
+     * file and don't do anything here so we don't build a loop. */
+    if (new->backing == NULL && !bdrv_chain_contains(backing_bs(old), new)) {
+        bdrv_set_backing_hd(new, backing_bs(old));
+        bdrv_set_backing_hd(old, NULL);
+    }
+
+    bdrv_unref(old);
 }
 
 static void bdrv_delete(BlockDriverState *bs)
@@ -2224,20 +2170,20 @@ int bdrv_commit(BlockDriverState *bs)
     if (!drv)
         return -ENOMEDIUM;
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         return -ENOTSUP;
     }
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT_SOURCE, NULL) ||
-        bdrv_op_is_blocked(bs->backing_hd, BLOCK_OP_TYPE_COMMIT_TARGET, NULL)) {
+        bdrv_op_is_blocked(bs->backing->bs, BLOCK_OP_TYPE_COMMIT_TARGET, NULL)) {
         return -EBUSY;
     }
 
-    ro = bs->backing_hd->read_only;
-    open_flags =  bs->backing_hd->open_flags;
+    ro = bs->backing->bs->read_only;
+    open_flags =  bs->backing->bs->open_flags;
 
     if (ro) {
-        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
+        if (bdrv_reopen(bs->backing->bs, open_flags | BDRV_O_RDWR, NULL)) {
             return -EACCES;
         }
     }
@@ -2248,7 +2194,7 @@ int bdrv_commit(BlockDriverState *bs)
         goto ro_cleanup;
     }
 
-    backing_length = bdrv_getlength(bs->backing_hd);
+    backing_length = bdrv_getlength(bs->backing->bs);
     if (backing_length < 0) {
         ret = backing_length;
         goto ro_cleanup;
@@ -2258,7 +2204,7 @@ int bdrv_commit(BlockDriverState *bs)
      * grow the backing file image if possible.  If not possible,
      * we must return an error */
     if (length > backing_length) {
-        ret = bdrv_truncate(bs->backing_hd, length);
+        ret = bdrv_truncate(bs->backing->bs, length);
         if (ret < 0) {
             goto ro_cleanup;
         }
@@ -2267,7 +2213,7 @@ int bdrv_commit(BlockDriverState *bs)
     total_sectors = length >> BDRV_SECTOR_BITS;
 
     /* qemu_try_blockalign() for bs will choose an alignment that works for
-     * bs->backing_hd as well, so no need to compare the alignment manually. */
+     * bs->backing->bs as well, so no need to compare the alignment manually. */
     buf = qemu_try_blockalign(bs, COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
     if (buf == NULL) {
         ret = -ENOMEM;
@@ -2285,7 +2231,7 @@ int bdrv_commit(BlockDriverState *bs)
                 goto ro_cleanup;
             }
 
-            ret = bdrv_write(bs->backing_hd, sector, buf, n);
+            ret = bdrv_write(bs->backing->bs, sector, buf, n);
             if (ret < 0) {
                 goto ro_cleanup;
             }
@@ -2304,8 +2250,8 @@ int bdrv_commit(BlockDriverState *bs)
      * Make sure all data we wrote to the backing device is actually
      * stable on disk.
      */
-    if (bs->backing_hd) {
-        bdrv_flush(bs->backing_hd);
+    if (bs->backing) {
+        bdrv_flush(bs->backing->bs);
     }
 
     ret = 0;
@@ -2314,7 +2260,7 @@ ro_cleanup:
 
     if (ro) {
         /* ignoring error return here */
-        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
+        bdrv_reopen(bs->backing->bs, open_flags & ~BDRV_O_RDWR, NULL);
     }
 
     return ret;
@@ -2328,7 +2274,7 @@ int bdrv_commit_all(void)
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
         aio_context_acquire(aio_context);
-        if (bs->drv && bs->backing_hd) {
+        if (bs->drv && bs->backing) {
             int ret = bdrv_commit(bs);
             if (ret < 0) {
                 aio_context_release(aio_context);
@@ -2385,8 +2331,8 @@ int bdrv_change_backing_file(BlockDriverState *bs,
 BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
                                     BlockDriverState *bs)
 {
-    while (active && bs != active->backing_hd) {
-        active = active->backing_hd;
+    while (active && bs != backing_bs(active)) {
+        active = backing_bs(active);
     }
 
     return active;
@@ -2398,12 +2344,6 @@ BlockDriverState *bdrv_find_base(BlockDriverState *bs)
     return bdrv_find_overlay(bs, NULL);
 }
 
-typedef struct BlkIntermediateStates {
-    BlockDriverState *bs;
-    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
-} BlkIntermediateStates;
-
-
 /*
  * Drops images above 'base' up to and including 'top', and sets the image
  * above 'top' to have base as its backing file.
@@ -2436,15 +2376,9 @@ typedef struct BlkIntermediateStates {
 int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
                            BlockDriverState *base, const char *backing_file_str)
 {
-    BlockDriverState *intermediate;
-    BlockDriverState *base_bs = NULL;
     BlockDriverState *new_top_bs = NULL;
-    BlkIntermediateStates *intermediate_state, *next;
     int ret = -EIO;
 
-    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
-    QSIMPLEQ_INIT(&states_to_delete);
-
     if (!top->drv || !base->drv) {
         goto exit;
     }
@@ -2456,55 +2390,29 @@ int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
         goto exit;
     }
 
-    /* special case of new_top_bs->backing_hd already pointing to base - nothing
+    /* special case of new_top_bs->backing->bs already pointing to base - nothing
      * to do, no intermediate images */
-    if (new_top_bs->backing_hd == base) {
+    if (backing_bs(new_top_bs) == base) {
         ret = 0;
         goto exit;
     }
 
-    intermediate = top;
-
-    /* now we will go down through the list, and add each BDS we find
-     * into our deletion queue, until we hit the 'base'
-     */
-    while (intermediate) {
-        intermediate_state = g_new0(BlkIntermediateStates, 1);
-        intermediate_state->bs = intermediate;
-        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
-
-        if (intermediate->backing_hd == base) {
-            base_bs = intermediate->backing_hd;
-            break;
-        }
-        intermediate = intermediate->backing_hd;
-    }
-    if (base_bs == NULL) {
-        /* something went wrong, we did not end at the base. safely
-         * unravel everything, and exit with error */
+    /* Make sure that base is in the backing chain of top */
+    if (!bdrv_chain_contains(top, base)) {
         goto exit;
     }
 
     /* success - we can delete the intermediate states, and link top->base */
-    backing_file_str = backing_file_str ? backing_file_str : base_bs->filename;
+    backing_file_str = backing_file_str ? backing_file_str : base->filename;
     ret = bdrv_change_backing_file(new_top_bs, backing_file_str,
-                                   base_bs->drv ? base_bs->drv->format_name : "");
+                                   base->drv ? base->drv->format_name : "");
     if (ret) {
         goto exit;
     }
-    bdrv_set_backing_hd(new_top_bs, base_bs);
+    bdrv_set_backing_hd(new_top_bs, base);
 
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        /* so that bdrv_close() does not recursively close the chain */
-        bdrv_set_backing_hd(intermediate_state->bs, NULL);
-        bdrv_unref(intermediate_state->bs);
-    }
     ret = 0;
-
 exit:
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        g_free(intermediate_state);
-    }
     return ret;
 }
 
@@ -2547,7 +2455,7 @@ int64_t bdrv_get_allocated_file_size(BlockDriverState *bs)
         return drv->bdrv_get_allocated_file_size(bs);
     }
     if (bs->file) {
-        return bdrv_get_allocated_file_size(bs->file);
+        return bdrv_get_allocated_file_size(bs->file->bs);
     }
     return -ENOTSUP;
 }
@@ -2591,82 +2499,6 @@ void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr)
     *nb_sectors_ptr = nb_sectors < 0 ? 0 : nb_sectors;
 }
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error)
-{
-    bs->on_read_error = on_read_error;
-    bs->on_write_error = on_write_error;
-}
-
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read)
-{
-    return is_read ? bs->on_read_error : bs->on_write_error;
-}
-
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error)
-{
-    BlockdevOnError on_err = is_read ? bs->on_read_error : bs->on_write_error;
-
-    switch (on_err) {
-    case BLOCKDEV_ON_ERROR_ENOSPC:
-        return (error == ENOSPC) ?
-               BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_STOP:
-        return BLOCK_ERROR_ACTION_STOP;
-    case BLOCKDEV_ON_ERROR_REPORT:
-        return BLOCK_ERROR_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_IGNORE:
-        return BLOCK_ERROR_ACTION_IGNORE;
-    default:
-        abort();
-    }
-}
-
-static void send_qmp_error_event(BlockDriverState *bs,
-                                 BlockErrorAction action,
-                                 bool is_read, int error)
-{
-    IoOperationType optype;
-
-    optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
-    qapi_event_send_block_io_error(bdrv_get_device_name(bs), optype, action,
-                                   bdrv_iostatus_is_enabled(bs),
-                                   error == ENOSPC, strerror(error),
-                                   &error_abort);
-}
-
-/* This is done by device models because, while the block layer knows
- * about the error, it does not know whether an operation comes from
- * the device or the block layer (from a job, for example).
- */
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error)
-{
-    assert(error >= 0);
-
-    if (action == BLOCK_ERROR_ACTION_STOP) {
-        /* First set the iostatus, so that "info block" returns an iostatus
-         * that matches the events raised so far (an additional error iostatus
-         * is fine, but not a lost one).
-         */
-        bdrv_iostatus_set_err(bs, error);
-
-        /* Then raise the request to stop the VM and the event.
-         * qemu_system_vmstop_request_prepare has two effects.  First,
-         * it ensures that the STOP event always comes after the
-         * BLOCK_IO_ERROR event.  Second, it ensures that even if management
-         * can observe the STOP event and do a "cont" before the STOP
-         * event is issued, the VM will not stop.  In this case, vm_start()
-         * also ensures that the STOP/RESUME pair of events is emitted.
-         */
-        qemu_system_vmstop_request_prepare();
-        send_qmp_error_event(bs, action, is_read, error);
-        qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
-    } else {
-        send_qmp_error_event(bs, action, is_read, error);
-    }
-}
-
 int bdrv_is_read_only(BlockDriverState *bs)
 {
     return bs->read_only;
@@ -2696,25 +2528,27 @@ void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce)
 
 int bdrv_is_encrypted(BlockDriverState *bs)
 {
-    if (bs->backing_hd && bs->backing_hd->encrypted)
+    if (bs->backing && bs->backing->bs->encrypted) {
         return 1;
+    }
     return bs->encrypted;
 }
 
 int bdrv_key_required(BlockDriverState *bs)
 {
-    BlockDriverState *backing_hd = bs->backing_hd;
+    BdrvChild *backing = bs->backing;
 
-    if (backing_hd && backing_hd->encrypted && !backing_hd->valid_key)
+    if (backing && backing->bs->encrypted && !backing->bs->valid_key) {
         return 1;
+    }
     return (bs->encrypted && !bs->valid_key);
 }
 
 int bdrv_set_key(BlockDriverState *bs, const char *key)
 {
     int ret;
-    if (bs->backing_hd && bs->backing_hd->encrypted) {
-        ret = bdrv_set_key(bs->backing_hd, key);
+    if (bs->backing && bs->backing->bs->encrypted) {
+        ret = bdrv_set_key(bs->backing->bs, key);
         if (ret < 0)
             return ret;
         if (!bs->encrypted)
@@ -2858,7 +2692,12 @@ BlockDriverState *bdrv_lookup_bs(const char *device,
         blk = blk_by_name(device);
 
         if (blk) {
-            return blk_bs(blk);
+            bs = blk_bs(blk);
+            if (!bs) {
+                error_setg(errp, "Device '%s' has no medium", device);
+            }
+
+            return bs;
         }
     }
 
@@ -2881,7 +2720,7 @@ BlockDriverState *bdrv_lookup_bs(const char *device,
 bool bdrv_chain_contains(BlockDriverState *top, BlockDriverState *base)
 {
     while (top && top != base) {
-        top = top->backing_hd;
+        top = backing_bs(top);
     }
 
     return top != NULL;
@@ -2939,7 +2778,7 @@ int bdrv_has_zero_init(BlockDriverState *bs)
 
     /* If BS is a copy on write image, it is initialized to
        the contents of the base image, which may not be zeroes.  */
-    if (bs->backing_hd) {
+    if (bs->backing) {
         return 0;
     }
     if (bs->drv->bdrv_has_zero_init) {
@@ -2954,7 +2793,7 @@ bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs)
 {
     BlockDriverInfo bdi;
 
-    if (bs->backing_hd) {
+    if (bs->backing) {
         return false;
     }
 
@@ -2969,7 +2808,7 @@ bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs)
 {
     BlockDriverInfo bdi;
 
-    if (bs->backing_hd || !(bs->open_flags & BDRV_O_UNMAP)) {
+    if (bs->backing || !(bs->open_flags & BDRV_O_UNMAP)) {
         return false;
     }
 
@@ -2982,7 +2821,7 @@ bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs)
 
 const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
 {
-    if (bs->backing_hd && bs->backing_hd->encrypted)
+    if (bs->backing && bs->backing->bs->encrypted)
         return bs->backing_file;
     else if (bs->encrypted)
         return bs->filename;
@@ -3029,7 +2868,7 @@ int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
                           const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
@@ -3042,7 +2881,7 @@ int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
 int bdrv_debug_remove_breakpoint(BlockDriverState *bs, const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_remove_breakpoint) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_remove_breakpoint) {
@@ -3055,7 +2894,7 @@ int bdrv_debug_remove_breakpoint(BlockDriverState *bs, const char *tag)
 int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
 {
     while (bs && (!bs->drv || !bs->drv->bdrv_debug_resume)) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_resume) {
@@ -3068,7 +2907,7 @@ int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
 bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_is_suspended) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_is_suspended) {
@@ -3107,13 +2946,13 @@ BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
 
     is_protocol = path_has_protocol(backing_file);
 
-    for (curr_bs = bs; curr_bs->backing_hd; curr_bs = curr_bs->backing_hd) {
+    for (curr_bs = bs; curr_bs->backing; curr_bs = curr_bs->backing->bs) {
 
         /* If either of the filename paths is actually a protocol, then
          * compare unmodified paths; otherwise make paths relative */
         if (is_protocol || path_has_protocol(curr_bs->backing_file)) {
             if (strcmp(backing_file, curr_bs->backing_file) == 0) {
-                retval = curr_bs->backing_hd;
+                retval = curr_bs->backing->bs;
                 break;
             }
         } else {
@@ -3137,7 +2976,7 @@ BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
             }
 
             if (strcmp(backing_file_full, filename_full) == 0) {
-                retval = curr_bs->backing_hd;
+                retval = curr_bs->backing->bs;
                 break;
             }
         }
@@ -3155,11 +2994,11 @@ int bdrv_get_backing_file_depth(BlockDriverState *bs)
         return 0;
     }
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         return 0;
     }
 
-    return 1 + bdrv_get_backing_file_depth(bs->backing_hd);
+    return 1 + bdrv_get_backing_file_depth(bs->backing->bs);
 }
 
 void bdrv_init(void)
@@ -3190,7 +3029,7 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp)
     if (bs->drv->bdrv_invalidate_cache) {
         bs->drv->bdrv_invalidate_cache(bs, &local_err);
     } else if (bs->file) {
-        bdrv_invalidate_cache(bs->file, &local_err);
+        bdrv_invalidate_cache(bs->file->bs, &local_err);
     }
     if (local_err) {
         error_propagate(errp, local_err);
@@ -3228,15 +3067,23 @@ void bdrv_invalidate_cache_all(Error **errp)
 /**
  * Return TRUE if the media is present
  */
-int bdrv_is_inserted(BlockDriverState *bs)
+bool bdrv_is_inserted(BlockDriverState *bs)
 {
     BlockDriver *drv = bs->drv;
+    BdrvChild *child;
 
-    if (!drv)
-        return 0;
-    if (!drv->bdrv_is_inserted)
-        return 1;
-    return drv->bdrv_is_inserted(bs);
+    if (!drv) {
+        return false;
+    }
+    if (drv->bdrv_is_inserted) {
+        return drv->bdrv_is_inserted(bs);
+    }
+    QLIST_FOREACH(child, &bs->children, next) {
+        if (!bdrv_is_inserted(child->bs)) {
+            return false;
+        }
+    }
+    return true;
 }
 
 /**
@@ -3287,11 +3134,6 @@ void bdrv_lock_medium(BlockDriverState *bs, bool locked)
     }
 }
 
-void bdrv_set_guest_block_size(BlockDriverState *bs, int align)
-{
-    bs->guest_block_size = align;
-}
-
 BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs, const char *name)
 {
     BdrvDirtyBitmap *bm;
@@ -3566,10 +3408,25 @@ void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
     hbitmap_reset(bitmap->bitmap, cur_sector, nr_sectors);
 }
 
-void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap)
+void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out)
 {
     assert(bdrv_dirty_bitmap_enabled(bitmap));
-    hbitmap_reset_all(bitmap->bitmap);
+    if (!out) {
+        hbitmap_reset_all(bitmap->bitmap);
+    } else {
+        HBitmap *backup = bitmap->bitmap;
+        bitmap->bitmap = hbitmap_alloc(bitmap->size,
+                                       hbitmap_granularity(backup));
+        *out = backup;
+    }
+}
+
+void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in)
+{
+    HBitmap *tmp = bitmap->bitmap;
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    bitmap->bitmap = in;
+    hbitmap_free(tmp);
 }
 
 void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
@@ -3689,46 +3546,6 @@ bool bdrv_op_blocker_is_empty(BlockDriverState *bs)
     return true;
 }
 
-void bdrv_iostatus_enable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = true;
-    bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-}
-
-/* The I/O status is only enabled if the drive explicitly
- * enables it _and_ the VM is configured to stop on errors */
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs)
-{
-    return (bs->iostatus_enabled &&
-           (bs->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
-            bs->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
-            bs->on_read_error == BLOCKDEV_ON_ERROR_STOP));
-}
-
-void bdrv_iostatus_disable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = false;
-}
-
-void bdrv_iostatus_reset(BlockDriverState *bs)
-{
-    if (bdrv_iostatus_is_enabled(bs)) {
-        bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-        if (bs->job) {
-            block_job_iostatus_reset(bs->job);
-        }
-    }
-}
-
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
-{
-    assert(bdrv_iostatus_is_enabled(bs));
-    if (bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
-        bs->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
-                                         BLOCK_DEVICE_IO_STATUS_FAILED;
-    }
-}
-
 void bdrv_img_create(const char *filename, const char *fmt,
                      const char *base_filename, const char *base_fmt,
                      char *options, uint64_t img_size, int flags,
@@ -3739,7 +3556,6 @@ void bdrv_img_create(const char *filename, const char *fmt,
     const char *backing_fmt, *backing_file;
     int64_t size;
     BlockDriver *drv, *proto_drv;
-    BlockDriver *backing_drv = NULL;
     Error *local_err = NULL;
     int ret = 0;
 
@@ -3813,14 +3629,6 @@ void bdrv_img_create(const char *filename, const char *fmt,
     }
 
     backing_fmt = qemu_opt_get(opts, BLOCK_OPT_BACKING_FMT);
-    if (backing_fmt) {
-        backing_drv = bdrv_find_format(backing_fmt);
-        if (!backing_drv) {
-            error_setg(errp, "Unknown backing file format '%s'",
-                       backing_fmt);
-            goto out;
-        }
-    }
 
     // The size for the image must always be specified, with one exception:
     // If we are using a backing file, we can obtain the size from there
@@ -3831,6 +3639,7 @@ void bdrv_img_create(const char *filename, const char *fmt,
             char *full_backing = g_new0(char, PATH_MAX);
             int64_t size;
             int back_flags;
+            QDict *backing_options = NULL;
 
             bdrv_get_full_backing_filename_from_filename(filename, backing_file,
                                                          full_backing, PATH_MAX,
@@ -3844,9 +3653,15 @@ void bdrv_img_create(const char *filename, const char *fmt,
             back_flags =
                 flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
 
+            if (backing_fmt) {
+                backing_options = qdict_new();
+                qdict_put(backing_options, "driver",
+                          qstring_from_str(backing_fmt));
+            }
+
             bs = NULL;
-            ret = bdrv_open(&bs, full_backing, NULL, NULL, back_flags,
-                            backing_drv, &local_err);
+            ret = bdrv_open(&bs, full_backing, NULL, backing_options,
+                            back_flags, &local_err);
             g_free(full_backing);
             if (ret < 0) {
                 goto out;
@@ -3869,7 +3684,7 @@ void bdrv_img_create(const char *filename, const char *fmt,
     }
 
     if (!quiet) {
-        printf("Formatting '%s', fmt=%s", filename, fmt);
+        printf("Formatting '%s', fmt=%s ", filename, fmt);
         qemu_opts_print(opts, " ");
         puts("");
     }
@@ -3915,17 +3730,17 @@ void bdrv_detach_aio_context(BlockDriverState *bs)
         baf->detach_aio_context(baf->opaque);
     }
 
-    if (bs->io_limits_enabled) {
+    if (bs->throttle_state) {
         throttle_timers_detach_aio_context(&bs->throttle_timers);
     }
     if (bs->drv->bdrv_detach_aio_context) {
         bs->drv->bdrv_detach_aio_context(bs);
     }
     if (bs->file) {
-        bdrv_detach_aio_context(bs->file);
+        bdrv_detach_aio_context(bs->file->bs);
     }
-    if (bs->backing_hd) {
-        bdrv_detach_aio_context(bs->backing_hd);
+    if (bs->backing) {
+        bdrv_detach_aio_context(bs->backing->bs);
     }
 
     bs->aio_context = NULL;
@@ -3942,16 +3757,16 @@ void bdrv_attach_aio_context(BlockDriverState *bs,
 
     bs->aio_context = new_context;
 
-    if (bs->backing_hd) {
-        bdrv_attach_aio_context(bs->backing_hd, new_context);
+    if (bs->backing) {
+        bdrv_attach_aio_context(bs->backing->bs, new_context);
     }
     if (bs->file) {
-        bdrv_attach_aio_context(bs->file, new_context);
+        bdrv_attach_aio_context(bs->file->bs, new_context);
     }
     if (bs->drv->bdrv_attach_aio_context) {
         bs->drv->bdrv_attach_aio_context(bs, new_context);
     }
-    if (bs->io_limits_enabled) {
+    if (bs->throttle_state) {
         throttle_timers_attach_aio_context(&bs->throttle_timers, new_context);
     }
 
@@ -4077,7 +3892,8 @@ bool bdrv_is_first_non_filter(BlockDriverState *candidate)
     return false;
 }
 
-BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
+BlockDriverState *check_to_replace_node(BlockDriverState *parent_bs,
+                                        const char *node_name, Error **errp)
 {
     BlockDriverState *to_replace_bs = bdrv_find_node(node_name);
     AioContext *aio_context;
@@ -4100,7 +3916,7 @@ BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
      * Another benefit is that this tests exclude backing files which are
      * blocked by the backing blockers.
      */
-    if (!bdrv_is_first_non_filter(to_replace_bs)) {
+    if (!bdrv_recurse_is_first_non_filter(parent_bs, to_replace_bs)) {
         error_setg(errp, "Only top most non filter can be replaced");
         to_replace_bs = NULL;
         goto out;
@@ -4157,7 +3973,7 @@ void bdrv_refresh_filename(BlockDriverState *bs)
     /* This BDS's file name will most probably depend on its file's name, so
      * refresh that first */
     if (bs->file) {
-        bdrv_refresh_filename(bs->file);
+        bdrv_refresh_filename(bs->file->bs);
     }
 
     if (drv->bdrv_refresh_filename) {
@@ -4185,19 +4001,20 @@ void bdrv_refresh_filename(BlockDriverState *bs)
 
         /* If no specific options have been given for this BDS, the filename of
          * the underlying file should suffice for this one as well */
-        if (bs->file->exact_filename[0] && !has_open_options) {
-            strcpy(bs->exact_filename, bs->file->exact_filename);
+        if (bs->file->bs->exact_filename[0] && !has_open_options) {
+            strcpy(bs->exact_filename, bs->file->bs->exact_filename);
         }
         /* Reconstructing the full options QDict is simple for most format block
          * drivers, as long as the full options are known for the underlying
          * file BDS. The full options QDict of that file BDS should somehow
          * contain a representation of the filename, therefore the following
          * suffices without querying the (exact_)filename of this BDS. */
-        if (bs->file->full_open_options) {
+        if (bs->file->bs->full_open_options) {
             qdict_put_obj(opts, "driver",
                           QOBJECT(qstring_from_str(drv->format_name)));
-            QINCREF(bs->file->full_open_options);
-            qdict_put_obj(opts, "file", QOBJECT(bs->file->full_open_options));
+            QINCREF(bs->file->bs->full_open_options);
+            qdict_put_obj(opts, "file",
+                          QOBJECT(bs->file->bs->full_open_options));
 
             bs->full_open_options = opts;
         } else {
@@ -4240,14 +4057,3 @@ void bdrv_refresh_filename(BlockDriverState *bs)
         QDECREF(json);
     }
 }
-
-/* This accessor function purpose is to allow the device models to access the
- * BlockAcctStats structure embedded inside a BlockDriverState without being
- * aware of the BlockDriverState structure layout.
- * It will go away when the BlockAcctStats structure will be moved inside
- * the device models.
- */
-BlockAcctStats *bdrv_get_stats(BlockDriverState *bs)
-{
-    return &bs->stats;
-}
diff --git a/block/accounting.c b/block/accounting.c
index 01d594ffd..185025ec1 100644
--- a/block/accounting.c
+++ b/block/accounting.c
@@ -2,6 +2,7 @@
  * QEMU System Emulator block accounting
  *
  * Copyright (c) 2011 Christoph Hellwig
+ * Copyright (c) 2015 Igalia, S.L.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -25,6 +26,54 @@
 #include "block/accounting.h"
 #include "block/block_int.h"
 #include "qemu/timer.h"
+#include "sysemu/qtest.h"
+
+static QEMUClockType clock_type = QEMU_CLOCK_REALTIME;
+static const int qtest_latency_ns = NANOSECONDS_PER_SECOND / 1000;
+
+void block_acct_init(BlockAcctStats *stats, bool account_invalid,
+                     bool account_failed)
+{
+    stats->account_invalid = account_invalid;
+    stats->account_failed = account_failed;
+
+    if (qtest_enabled()) {
+        clock_type = QEMU_CLOCK_VIRTUAL;
+    }
+}
+
+void block_acct_cleanup(BlockAcctStats *stats)
+{
+    BlockAcctTimedStats *s, *next;
+    QSLIST_FOREACH_SAFE(s, &stats->intervals, entries, next) {
+        g_free(s);
+    }
+}
+
+void block_acct_add_interval(BlockAcctStats *stats, unsigned interval_length)
+{
+    BlockAcctTimedStats *s;
+    unsigned i;
+
+    s = g_new0(BlockAcctTimedStats, 1);
+    s->interval_length = interval_length;
+    QSLIST_INSERT_HEAD(&stats->intervals, s, entries);
+
+    for (i = 0; i < BLOCK_MAX_IOTYPE; i++) {
+        timed_average_init(&s->latency[i], clock_type,
+                           (uint64_t) interval_length * NANOSECONDS_PER_SECOND);
+    }
+}
+
+BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,
+                                              BlockAcctTimedStats *s)
+{
+    if (s == NULL) {
+        return QSLIST_FIRST(&stats->intervals);
+    } else {
+        return QSLIST_NEXT(s, entries);
+    }
+}
 
 void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
                       int64_t bytes, enum BlockAcctType type)
@@ -32,26 +81,69 @@ void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
     assert(type < BLOCK_MAX_IOTYPE);
 
     cookie->bytes = bytes;
-    cookie->start_time_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    cookie->start_time_ns = qemu_clock_get_ns(clock_type);
     cookie->type = type;
 }
 
 void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie)
 {
+    BlockAcctTimedStats *s;
+    int64_t time_ns = qemu_clock_get_ns(clock_type);
+    int64_t latency_ns = time_ns - cookie->start_time_ns;
+
+    if (qtest_enabled()) {
+        latency_ns = qtest_latency_ns;
+    }
+
     assert(cookie->type < BLOCK_MAX_IOTYPE);
 
     stats->nr_bytes[cookie->type] += cookie->bytes;
     stats->nr_ops[cookie->type]++;
-    stats->total_time_ns[cookie->type] +=
-        qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - cookie->start_time_ns;
+    stats->total_time_ns[cookie->type] += latency_ns;
+    stats->last_access_time_ns = time_ns;
+
+    QSLIST_FOREACH(s, &stats->intervals, entries) {
+        timed_average_account(&s->latency[cookie->type], latency_ns);
+    }
 }
 
+void block_acct_failed(BlockAcctStats *stats, BlockAcctCookie *cookie)
+{
+    assert(cookie->type < BLOCK_MAX_IOTYPE);
+
+    stats->failed_ops[cookie->type]++;
+
+    if (stats->account_failed) {
+        BlockAcctTimedStats *s;
+        int64_t time_ns = qemu_clock_get_ns(clock_type);
+        int64_t latency_ns = time_ns - cookie->start_time_ns;
+
+        if (qtest_enabled()) {
+            latency_ns = qtest_latency_ns;
+        }
 
-void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
-                               unsigned int nb_sectors)
+        stats->total_time_ns[cookie->type] += latency_ns;
+        stats->last_access_time_ns = time_ns;
+
+        QSLIST_FOREACH(s, &stats->intervals, entries) {
+            timed_average_account(&s->latency[cookie->type], latency_ns);
+        }
+    }
+}
+
+void block_acct_invalid(BlockAcctStats *stats, enum BlockAcctType type)
 {
-    if (stats->wr_highest_sector < sector_num + nb_sectors - 1) {
-        stats->wr_highest_sector = sector_num + nb_sectors - 1;
+    assert(type < BLOCK_MAX_IOTYPE);
+
+    /* block_acct_done() and block_acct_failed() update
+     * total_time_ns[], but this one does not. The reason is that
+     * invalid requests are accounted during their submission,
+     * therefore there's no actual I/O involved. */
+
+    stats->invalid_ops[type]++;
+
+    if (stats->account_invalid) {
+        stats->last_access_time_ns = qemu_clock_get_ns(clock_type);
     }
 }
 
@@ -61,3 +153,20 @@ void block_acct_merge_done(BlockAcctStats *stats, enum BlockAcctType type,
     assert(type < BLOCK_MAX_IOTYPE);
     stats->merged[type] += num_requests;
 }
+
+int64_t block_acct_idle_time_ns(BlockAcctStats *stats)
+{
+    return qemu_clock_get_ns(clock_type) - stats->last_access_time_ns;
+}
+
+double block_acct_queue_depth(BlockAcctTimedStats *stats,
+                              enum BlockAcctType type)
+{
+    uint64_t sum, elapsed;
+
+    assert(type < BLOCK_MAX_IOTYPE);
+
+    sum = timed_average_sum(&stats->latency[type], &elapsed);
+
+    return (double) sum / elapsed;
+}
diff --git a/block/backup.c b/block/backup.c
index 965654d52..705bb7766 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -21,6 +21,7 @@
 #include "block/blockjob.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "sysemu/block-backend.h"
 
 #define BACKUP_CLUSTER_BITS 16
 #define BACKUP_CLUSTER_SIZE (1 << BACKUP_CLUSTER_BITS)
@@ -89,7 +90,8 @@ static void cow_request_end(CowRequest *req)
 
 static int coroutine_fn backup_do_cow(BlockDriverState *bs,
                                       int64_t sector_num, int nb_sectors,
-                                      bool *error_is_read)
+                                      bool *error_is_read,
+                                      bool is_write_notifier)
 {
     BackupBlockJob *job = (BackupBlockJob *)bs->job;
     CowRequest cow_request;
@@ -129,8 +131,14 @@ static int coroutine_fn backup_do_cow(BlockDriverState *bs,
         iov.iov_len = n * BDRV_SECTOR_SIZE;
         qemu_iovec_init_external(&bounce_qiov, &iov, 1);
 
-        ret = bdrv_co_readv(bs, start * BACKUP_SECTORS_PER_CLUSTER, n,
-                            &bounce_qiov);
+        if (is_write_notifier) {
+            ret = bdrv_co_readv_no_serialising(bs,
+                                           start * BACKUP_SECTORS_PER_CLUSTER,
+                                           n, &bounce_qiov);
+        } else {
+            ret = bdrv_co_readv(bs, start * BACKUP_SECTORS_PER_CLUSTER, n,
+                                &bounce_qiov);
+        }
         if (ret < 0) {
             trace_backup_do_cow_read_fail(job, start, ret);
             if (error_is_read) {
@@ -190,7 +198,7 @@ static int coroutine_fn backup_before_write_notify(
     assert((req->offset & (BDRV_SECTOR_SIZE - 1)) == 0);
     assert((req->bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 
-    return backup_do_cow(req->bs, sector_num, nb_sectors, NULL);
+    return backup_do_cow(req->bs, sector_num, nb_sectors, NULL, true);
 }
 
 static void backup_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -208,7 +216,41 @@ static void backup_iostatus_reset(BlockJob *job)
 {
     BackupBlockJob *s = container_of(job, BackupBlockJob, common);
 
-    bdrv_iostatus_reset(s->target);
+    if (s->target->blk) {
+        blk_iostatus_reset(s->target->blk);
+    }
+}
+
+static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
+{
+    BdrvDirtyBitmap *bm;
+    BlockDriverState *bs = job->common.bs;
+
+    if (ret < 0 || block_job_is_cancelled(&job->common)) {
+        /* Merge the successor back into the parent, delete nothing. */
+        bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
+        assert(bm);
+    } else {
+        /* Everything is fine, delete this bitmap and install the backup. */
+        bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
+        assert(bm);
+    }
+}
+
+static void backup_commit(BlockJob *job)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+    if (s->sync_bitmap) {
+        backup_cleanup_sync_bitmap(s, 0);
+    }
+}
+
+static void backup_abort(BlockJob *job)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+    if (s->sync_bitmap) {
+        backup_cleanup_sync_bitmap(s, -1);
+    }
 }
 
 static const BlockJobDriver backup_job_driver = {
@@ -216,6 +258,8 @@ static const BlockJobDriver backup_job_driver = {
     .job_type       = BLOCK_JOB_TYPE_BACKUP,
     .set_speed      = backup_set_speed,
     .iostatus_reset = backup_iostatus_reset,
+    .commit         = backup_commit,
+    .abort          = backup_abort,
 };
 
 static BlockErrorAction backup_error_action(BackupBlockJob *job,
@@ -303,7 +347,8 @@ static int coroutine_fn backup_run_incremental(BackupBlockJob *job)
                     return ret;
                 }
                 ret = backup_do_cow(bs, cluster * BACKUP_SECTORS_PER_CLUSTER,
-                                    BACKUP_SECTORS_PER_CLUSTER, &error_is_read);
+                                    BACKUP_SECTORS_PER_CLUSTER, &error_is_read,
+                                    false);
                 if ((ret < 0) &&
                     backup_error_action(job, error_is_read, -ret) ==
                     BLOCK_ERROR_ACTION_REPORT) {
@@ -352,8 +397,10 @@ static void coroutine_fn backup_run(void *opaque)
     job->bitmap = hbitmap_alloc(end, 0);
 
     bdrv_set_enable_write_cache(target, true);
-    bdrv_set_on_error(target, on_target_error, on_target_error);
-    bdrv_iostatus_enable(target);
+    if (target->blk) {
+        blk_set_on_error(target->blk, on_target_error, on_target_error);
+        blk_iostatus_enable(target->blk);
+    }
 
     bdrv_add_before_write_notifier(bs, &before_write);
 
@@ -408,7 +455,7 @@ static void coroutine_fn backup_run(void *opaque)
             }
             /* FULL sync mode we copy the whole drive. */
             ret = backup_do_cow(bs, start * BACKUP_SECTORS_PER_CLUSTER,
-                    BACKUP_SECTORS_PER_CLUSTER, &error_is_read);
+                    BACKUP_SECTORS_PER_CLUSTER, &error_is_read, false);
             if (ret < 0) {
                 /* Depending on error action, fail now or retry cluster */
                 BlockErrorAction action =
@@ -428,22 +475,11 @@ static void coroutine_fn backup_run(void *opaque)
     /* wait until pending backup_do_cow() calls have completed */
     qemu_co_rwlock_wrlock(&job->flush_rwlock);
     qemu_co_rwlock_unlock(&job->flush_rwlock);
-
-    if (job->sync_bitmap) {
-        BdrvDirtyBitmap *bm;
-        if (ret < 0 || block_job_is_cancelled(&job->common)) {
-            /* Merge the successor back into the parent, delete nothing. */
-            bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
-            assert(bm);
-        } else {
-            /* Everything is fine, delete this bitmap and install the backup. */
-            bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
-            assert(bm);
-        }
-    }
     hbitmap_free(job->bitmap);
 
-    bdrv_iostatus_disable(target);
+    if (target->blk) {
+        blk_iostatus_disable(target->blk);
+    }
     bdrv_op_unblock_all(target, job->common.blocker);
 
     data = g_malloc(sizeof(*data));
@@ -457,7 +493,7 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockCompletionFunc *cb, void *opaque,
-                  Error **errp)
+                  BlockJobTxn *txn, Error **errp)
 {
     int64_t len;
 
@@ -472,7 +508,7 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
 
     if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
          on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, QERR_INVALID_PARAMETER, "on-source-error");
         return;
     }
@@ -539,6 +575,7 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
                        sync_bitmap : NULL;
     job->common.len = len;
     job->common.co = qemu_coroutine_create(backup_run);
+    block_job_txn_add_job(txn, &job->common);
     qemu_coroutine_enter(job->common.co, job);
     return;
 
diff --git a/block/blkdebug.c b/block/blkdebug.c
index bc247f46f..dee3a0edf 100644
--- a/block/blkdebug.c
+++ b/block/blkdebug.c
@@ -30,6 +30,7 @@
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qstring.h"
+#include "sysemu/qtest.h"
 
 typedef struct BDRVBlkdebugState {
     int state;
@@ -426,11 +427,11 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,
     /* Set initial state */
     s->state = 1;
 
-    /* Open the backing file */
-    assert(bs->file == NULL);
-    ret = bdrv_open_image(&bs->file, qemu_opt_get(opts, "x-image"), options, "image",
-                          bs, &child_file, false, &local_err);
-    if (ret < 0) {
+    /* Open the image file */
+    bs->file = bdrv_open_child(qemu_opt_get(opts, "x-image"), options, "image",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
         goto out;
     }
@@ -449,7 +450,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,
     goto out;
 
 fail_unref:
-    bdrv_unref(bs->file);
+    bdrv_unref_child(bs, bs->file);
 out:
     qemu_opts_del(opts);
     return ret;
@@ -510,7 +511,8 @@ static BlockAIOCB *blkdebug_aio_readv(BlockDriverState *bs,
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_readv(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
+    return bdrv_aio_readv(bs->file->bs, sector_num, qiov, nb_sectors,
+                          cb, opaque);
 }
 
 static BlockAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
@@ -532,7 +534,8 @@ static BlockAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
+    return bdrv_aio_writev(bs->file->bs, sector_num, qiov, nb_sectors,
+                           cb, opaque);
 }
 
 static BlockAIOCB *blkdebug_aio_flush(BlockDriverState *bs,
@@ -551,7 +554,7 @@ static BlockAIOCB *blkdebug_aio_flush(BlockDriverState *bs,
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_flush(bs->file, cb, opaque);
+    return bdrv_aio_flush(bs->file->bs, cb, opaque);
 }
 
 
@@ -581,9 +584,13 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)
     remove_rule(rule);
     QLIST_INSERT_HEAD(&s->suspended_reqs, &r, next);
 
-    printf("blkdebug: Suspended request '%s'\n", r.tag);
+    if (!qtest_enabled()) {
+        printf("blkdebug: Suspended request '%s'\n", r.tag);
+    }
     qemu_coroutine_yield();
-    printf("blkdebug: Resuming request '%s'\n", r.tag);
+    if (!qtest_enabled()) {
+        printf("blkdebug: Resuming request '%s'\n", r.tag);
+    }
 
     QLIST_REMOVE(&r, next);
     g_free(r.tag);
@@ -716,12 +723,12 @@ static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag)
 
 static int64_t blkdebug_getlength(BlockDriverState *bs)
 {
-    return bdrv_getlength(bs->file);
+    return bdrv_getlength(bs->file->bs);
 }
 
 static int blkdebug_truncate(BlockDriverState *bs, int64_t offset)
 {
-    return bdrv_truncate(bs->file, offset);
+    return bdrv_truncate(bs->file->bs, offset);
 }
 
 static void blkdebug_refresh_filename(BlockDriverState *bs)
@@ -741,24 +748,24 @@ static void blkdebug_refresh_filename(BlockDriverState *bs)
         }
     }
 
-    if (force_json && !bs->file->full_open_options) {
+    if (force_json && !bs->file->bs->full_open_options) {
         /* The config file cannot be recreated, so creating a plain filename
          * is impossible */
         return;
     }
 
-    if (!force_json && bs->file->exact_filename[0]) {
+    if (!force_json && bs->file->bs->exact_filename[0]) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
                  "blkdebug:%s:%s",
                  qdict_get_try_str(bs->options, "config") ?: "",
-                 bs->file->exact_filename);
+                 bs->file->bs->exact_filename);
     }
 
     opts = qdict_new();
     qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkdebug")));
 
-    QINCREF(bs->file->full_open_options);
-    qdict_put_obj(opts, "image", QOBJECT(bs->file->full_open_options));
+    QINCREF(bs->file->bs->full_open_options);
+    qdict_put_obj(opts, "image", QOBJECT(bs->file->bs->full_open_options));
 
     for (e = qdict_first(bs->options); e; e = qdict_next(bs->options, e)) {
         if (strcmp(qdict_entry_key(e), "x-image") &&
diff --git a/block/blkverify.c b/block/blkverify.c
index d277e6322..c5f8e8dcb 100644
--- a/block/blkverify.c
+++ b/block/blkverify.c
@@ -14,7 +14,7 @@
 #include "qapi/qmp/qstring.h"
 
 typedef struct {
-    BlockDriverState *test_file;
+    BdrvChild *test_file;
 } BDRVBlkverifyState;
 
 typedef struct BlkverifyAIOCB BlkverifyAIOCB;
@@ -123,26 +123,29 @@ static int blkverify_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* Open the raw file */
-    assert(bs->file == NULL);
-    ret = bdrv_open_image(&bs->file, qemu_opt_get(opts, "x-raw"), options,
-                          "raw", bs, &child_file, false, &local_err);
-    if (ret < 0) {
+    bs->file = bdrv_open_child(qemu_opt_get(opts, "x-raw"), options, "raw",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
         goto fail;
     }
 
     /* Open the test file */
-    assert(s->test_file == NULL);
-    ret = bdrv_open_image(&s->test_file, qemu_opt_get(opts, "x-image"), options,
-                          "test", bs, &child_format, false, &local_err);
-    if (ret < 0) {
+    s->test_file = bdrv_open_child(qemu_opt_get(opts, "x-image"), options,
+                                   "test", bs, &child_format, false,
+                                   &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
-        s->test_file = NULL;
         goto fail;
     }
 
     ret = 0;
 fail:
+    if (ret < 0) {
+        bdrv_unref_child(bs, bs->file);
+    }
     qemu_opts_del(opts);
     return ret;
 }
@@ -151,7 +154,7 @@ static void blkverify_close(BlockDriverState *bs)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_unref(s->test_file);
+    bdrv_unref_child(bs, s->test_file);
     s->test_file = NULL;
 }
 
@@ -159,7 +162,7 @@ static int64_t blkverify_getlength(BlockDriverState *bs)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    return bdrv_getlength(s->test_file);
+    return bdrv_getlength(s->test_file->bs);
 }
 
 static BlkverifyAIOCB *blkverify_aio_get(BlockDriverState *bs, bool is_write,
@@ -238,13 +241,13 @@ static BlockAIOCB *blkverify_aio_readv(BlockDriverState *bs,
                                             nb_sectors, cb, opaque);
 
     acb->verify = blkverify_verify_readv;
-    acb->buf = qemu_blockalign(bs->file, qiov->size);
+    acb->buf = qemu_blockalign(bs->file->bs, qiov->size);
     qemu_iovec_init(&acb->raw_qiov, acb->qiov->niov);
     qemu_iovec_clone(&acb->raw_qiov, qiov, acb->buf);
 
-    bdrv_aio_readv(s->test_file, sector_num, qiov, nb_sectors,
+    bdrv_aio_readv(s->test_file->bs, sector_num, qiov, nb_sectors,
                    blkverify_aio_cb, acb);
-    bdrv_aio_readv(bs->file, sector_num, &acb->raw_qiov, nb_sectors,
+    bdrv_aio_readv(bs->file->bs, sector_num, &acb->raw_qiov, nb_sectors,
                    blkverify_aio_cb, acb);
     return &acb->common;
 }
@@ -257,9 +260,9 @@ static BlockAIOCB *blkverify_aio_writev(BlockDriverState *bs,
     BlkverifyAIOCB *acb = blkverify_aio_get(bs, true, sector_num, qiov,
                                             nb_sectors, cb, opaque);
 
-    bdrv_aio_writev(s->test_file, sector_num, qiov, nb_sectors,
+    bdrv_aio_writev(s->test_file->bs, sector_num, qiov, nb_sectors,
                     blkverify_aio_cb, acb);
-    bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors,
+    bdrv_aio_writev(bs->file->bs, sector_num, qiov, nb_sectors,
                     blkverify_aio_cb, acb);
     return &acb->common;
 }
@@ -271,7 +274,7 @@ static BlockAIOCB *blkverify_aio_flush(BlockDriverState *bs,
     BDRVBlkverifyState *s = bs->opaque;
 
     /* Only flush test file, the raw file is not important */
-    return bdrv_aio_flush(s->test_file, cb, opaque);
+    return bdrv_aio_flush(s->test_file->bs, cb, opaque);
 }
 
 static bool blkverify_recurse_is_first_non_filter(BlockDriverState *bs,
@@ -279,13 +282,13 @@ static bool blkverify_recurse_is_first_non_filter(BlockDriverState *bs,
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bool perm = bdrv_recurse_is_first_non_filter(bs->file, candidate);
+    bool perm = bdrv_recurse_is_first_non_filter(bs->file->bs, candidate);
 
     if (perm) {
         return true;
     }
 
-    return bdrv_recurse_is_first_non_filter(s->test_file, candidate);
+    return bdrv_recurse_is_first_non_filter(s->test_file->bs, candidate);
 }
 
 /* Propagate AioContext changes to ->test_file */
@@ -293,7 +296,7 @@ static void blkverify_detach_aio_context(BlockDriverState *bs)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_detach_aio_context(s->test_file);
+    bdrv_detach_aio_context(s->test_file->bs);
 }
 
 static void blkverify_attach_aio_context(BlockDriverState *bs,
@@ -301,32 +304,38 @@ static void blkverify_attach_aio_context(BlockDriverState *bs,
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_attach_aio_context(s->test_file, new_context);
+    bdrv_attach_aio_context(s->test_file->bs, new_context);
 }
 
 static void blkverify_refresh_filename(BlockDriverState *bs)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    /* bs->file has already been refreshed */
-    bdrv_refresh_filename(s->test_file);
+    /* bs->file->bs has already been refreshed */
+    bdrv_refresh_filename(s->test_file->bs);
 
-    if (bs->file->full_open_options && s->test_file->full_open_options) {
+    if (bs->file->bs->full_open_options
+        && s->test_file->bs->full_open_options)
+    {
         QDict *opts = qdict_new();
         qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkverify")));
 
-        QINCREF(bs->file->full_open_options);
-        qdict_put_obj(opts, "raw", QOBJECT(bs->file->full_open_options));
-        QINCREF(s->test_file->full_open_options);
-        qdict_put_obj(opts, "test", QOBJECT(s->test_file->full_open_options));
+        QINCREF(bs->file->bs->full_open_options);
+        qdict_put_obj(opts, "raw", QOBJECT(bs->file->bs->full_open_options));
+        QINCREF(s->test_file->bs->full_open_options);
+        qdict_put_obj(opts, "test",
+                      QOBJECT(s->test_file->bs->full_open_options));
 
         bs->full_open_options = opts;
     }
 
-    if (bs->file->exact_filename[0] && s->test_file->exact_filename[0]) {
+    if (bs->file->bs->exact_filename[0]
+        && s->test_file->bs->exact_filename[0])
+    {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
                  "blkverify:%s:%s",
-                 bs->file->exact_filename, s->test_file->exact_filename);
+                 bs->file->bs->exact_filename,
+                 s->test_file->bs->exact_filename);
     }
 }
 
diff --git a/block/block-backend.c b/block/block-backend.c
index aee8a1202..419591f26 100644
--- a/block/block-backend.c
+++ b/block/block-backend.c
@@ -12,12 +12,17 @@
 
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
+#include "block/blockjob.h"
+#include "block/throttle-groups.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
 #include "qapi-event.h"
 
 /* Number of coroutines to reserve per attached device model */
 #define COROUTINE_POOL_RESERVATION 64
 
+static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb);
+
 struct BlockBackend {
     char *name;
     int refcnt;
@@ -29,15 +34,31 @@ struct BlockBackend {
     /* TODO change to DeviceState when all users are qdevified */
     const BlockDevOps *dev_ops;
     void *dev_opaque;
+
+    /* the block size for which the guest device expects atomicity */
+    int guest_block_size;
+
+    /* If the BDS tree is removed, some of its options are stored here (which
+     * can be used to restore those options in the new BDS on insert) */
+    BlockBackendRootState root_state;
+
+    /* I/O stats (display with "info blockstats"). */
+    BlockAcctStats stats;
+
+    BlockdevOnError on_read_error, on_write_error;
+    bool iostatus_enabled;
+    BlockDeviceIoStatus iostatus;
 };
 
 typedef struct BlockBackendAIOCB {
     BlockAIOCB common;
     QEMUBH *bh;
+    BlockBackend *blk;
     int ret;
 } BlockBackendAIOCB;
 
 static const AIOCBInfo block_backend_aiocb_info = {
+    .get_aio_context = blk_aiocb_get_aio_context,
     .aiocb_size = sizeof(BlockBackendAIOCB),
 };
 
@@ -126,7 +147,7 @@ BlockBackend *blk_new_open(const char *name, const char *filename,
         return NULL;
     }
 
-    ret = bdrv_open(&blk->bs, filename, reference, options, flags, NULL, errp);
+    ret = bdrv_open(&blk->bs, filename, reference, options, flags, errp);
     if (ret < 0) {
         blk_unref(blk);
         return NULL;
@@ -145,12 +166,17 @@ static void blk_delete(BlockBackend *blk)
         bdrv_unref(blk->bs);
         blk->bs = NULL;
     }
+    if (blk->root_state.throttle_state) {
+        g_free(blk->root_state.throttle_group);
+        throttle_group_unref(blk->root_state.throttle_state);
+    }
     /* Avoid double-remove after blk_hide_on_behalf_of_hmp_drive_del() */
     if (blk->name[0]) {
         QTAILQ_REMOVE(&blk_backends, blk, link);
     }
     g_free(blk->name);
     drive_info_del(blk->legacy_dinfo);
+    block_acct_cleanup(&blk->stats);
     g_free(blk);
 }
 
@@ -164,6 +190,11 @@ static void drive_info_del(DriveInfo *dinfo)
     g_free(dinfo);
 }
 
+int blk_get_refcnt(BlockBackend *blk)
+{
+    return blk ? blk->refcnt : 0;
+}
+
 /*
  * Increment @blk's reference count.
  * @blk must not be null.
@@ -239,6 +270,23 @@ BlockDriverState *blk_bs(BlockBackend *blk)
 }
 
 /*
+ * Changes the BlockDriverState attached to @blk
+ */
+void blk_set_bs(BlockBackend *blk, BlockDriverState *bs)
+{
+    bdrv_ref(bs);
+
+    if (blk->bs) {
+        blk->bs->blk = NULL;
+        bdrv_unref(blk->bs);
+    }
+    assert(bs->blk == NULL);
+
+    blk->bs = bs;
+    bs->blk = blk;
+}
+
+/*
  * Return @blk's DriveInfo if any, else null.
  */
 DriveInfo *blk_legacy_dinfo(BlockBackend *blk)
@@ -292,6 +340,29 @@ void blk_hide_on_behalf_of_hmp_drive_del(BlockBackend *blk)
 }
 
 /*
+ * Disassociates the currently associated BlockDriverState from @blk.
+ */
+void blk_remove_bs(BlockBackend *blk)
+{
+    blk_update_root_state(blk);
+
+    blk->bs->blk = NULL;
+    bdrv_unref(blk->bs);
+    blk->bs = NULL;
+}
+
+/*
+ * Associates a new BlockDriverState with @blk.
+ */
+void blk_insert_bs(BlockBackend *blk, BlockDriverState *bs)
+{
+    assert(!blk->bs && !bs->blk);
+    bdrv_ref(bs);
+    blk->bs = bs;
+    bs->blk = blk;
+}
+
+/*
  * Attach device model @dev to @blk.
  * Return 0 on success, -EBUSY when a device model is attached already.
  */
@@ -303,7 +374,7 @@ int blk_attach_dev(BlockBackend *blk, void *dev)
     }
     blk_ref(blk);
     blk->dev = dev;
-    bdrv_iostatus_reset(blk->bs);
+    blk_iostatus_reset(blk);
     return 0;
 }
 
@@ -330,7 +401,7 @@ void blk_detach_dev(BlockBackend *blk, void *dev)
     blk->dev = NULL;
     blk->dev_ops = NULL;
     blk->dev_opaque = NULL;
-    bdrv_set_guest_block_size(blk->bs, 512);
+    blk->guest_block_size = 512;
     blk_unref(blk);
 }
 
@@ -364,18 +435,15 @@ void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops,
 void blk_dev_change_media_cb(BlockBackend *blk, bool load)
 {
     if (blk->dev_ops && blk->dev_ops->change_media_cb) {
-        bool tray_was_closed = !blk_dev_is_tray_open(blk);
+        bool tray_was_open, tray_is_open;
 
+        tray_was_open = blk_dev_is_tray_open(blk);
         blk->dev_ops->change_media_cb(blk->dev_opaque, load);
-        if (tray_was_closed) {
-            /* tray open */
-            qapi_event_send_device_tray_moved(blk_name(blk),
-                                              true, &error_abort);
-        }
-        if (load) {
-            /* tray close */
-            qapi_event_send_device_tray_moved(blk_name(blk),
-                                              false, &error_abort);
+        tray_is_open = blk_dev_is_tray_open(blk);
+
+        if (tray_was_open != tray_is_open) {
+            qapi_event_send_device_tray_moved(blk_name(blk), tray_is_open,
+                                              &error_abort);
         }
     }
 }
@@ -390,6 +458,14 @@ bool blk_dev_has_removable_media(BlockBackend *blk)
 }
 
 /*
+ * Does @blk's attached device model have a tray?
+ */
+bool blk_dev_has_tray(BlockBackend *blk)
+{
+    return blk->dev_ops && blk->dev_ops->is_tray_open;
+}
+
+/*
  * Notify @blk's attached device model of a media eject request.
  * If @force is true, the medium is about to be yanked out forcefully.
  */
@@ -405,7 +481,7 @@ void blk_dev_eject_request(BlockBackend *blk, bool force)
  */
 bool blk_dev_is_tray_open(BlockBackend *blk)
 {
-    if (blk->dev_ops && blk->dev_ops->is_tray_open) {
+    if (blk_dev_has_tray(blk)) {
         return blk->dev_ops->is_tray_open(blk->dev_opaque);
     }
     return false;
@@ -435,7 +511,47 @@ void blk_dev_resize_cb(BlockBackend *blk)
 
 void blk_iostatus_enable(BlockBackend *blk)
 {
-    bdrv_iostatus_enable(blk->bs);
+    blk->iostatus_enabled = true;
+    blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+}
+
+/* The I/O status is only enabled if the drive explicitly
+ * enables it _and_ the VM is configured to stop on errors */
+bool blk_iostatus_is_enabled(const BlockBackend *blk)
+{
+    return (blk->iostatus_enabled &&
+           (blk->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
+            blk->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
+            blk->on_read_error == BLOCKDEV_ON_ERROR_STOP));
+}
+
+BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk)
+{
+    return blk->iostatus;
+}
+
+void blk_iostatus_disable(BlockBackend *blk)
+{
+    blk->iostatus_enabled = false;
+}
+
+void blk_iostatus_reset(BlockBackend *blk)
+{
+    if (blk_iostatus_is_enabled(blk)) {
+        blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+        if (blk->bs && blk->bs->job) {
+            block_job_iostatus_reset(blk->bs->job);
+        }
+    }
+}
+
+void blk_iostatus_set_err(BlockBackend *blk, int error)
+{
+    assert(blk_iostatus_is_enabled(blk));
+    if (blk->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+        blk->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
+                                          BLOCK_DEVICE_IO_STATUS_FAILED;
+    }
 }
 
 static int blk_check_byte_request(BlockBackend *blk, int64_t offset,
@@ -447,7 +563,7 @@ static int blk_check_byte_request(BlockBackend *blk, int64_t offset,
         return -EIO;
     }
 
-    if (!blk_is_inserted(blk)) {
+    if (!blk_is_available(blk)) {
         return -ENOMEDIUM;
     }
 
@@ -534,13 +650,15 @@ static void error_callback_bh(void *opaque)
     qemu_aio_unref(acb);
 }
 
-static BlockAIOCB *abort_aio_request(BlockBackend *blk, BlockCompletionFunc *cb,
-                                     void *opaque, int ret)
+BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque, int ret)
 {
     struct BlockBackendAIOCB *acb;
     QEMUBH *bh;
 
     acb = blk_aio_get(&block_backend_aiocb_info, blk, cb, opaque);
+    acb->blk = blk;
     acb->ret = ret;
 
     bh = aio_bh_new(blk_get_aio_context(blk), error_callback_bh, acb);
@@ -556,7 +674,7 @@ BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+        return blk_abort_aio_request(blk, cb, opaque, ret);
     }
 
     return bdrv_aio_write_zeroes(blk->bs, sector_num, nb_sectors, flags,
@@ -585,16 +703,28 @@ int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count)
 
 int64_t blk_getlength(BlockBackend *blk)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_getlength(blk->bs);
 }
 
 void blk_get_geometry(BlockBackend *blk, uint64_t *nb_sectors_ptr)
 {
-    bdrv_get_geometry(blk->bs, nb_sectors_ptr);
+    if (!blk->bs) {
+        *nb_sectors_ptr = 0;
+    } else {
+        bdrv_get_geometry(blk->bs, nb_sectors_ptr);
+    }
 }
 
 int64_t blk_nb_sectors(BlockBackend *blk)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_nb_sectors(blk->bs);
 }
 
@@ -604,7 +734,7 @@ BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+        return blk_abort_aio_request(blk, cb, opaque, ret);
     }
 
     return bdrv_aio_readv(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
@@ -616,7 +746,7 @@ BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+        return blk_abort_aio_request(blk, cb, opaque, ret);
     }
 
     return bdrv_aio_writev(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
@@ -625,6 +755,10 @@ BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
 BlockAIOCB *blk_aio_flush(BlockBackend *blk,
                           BlockCompletionFunc *cb, void *opaque)
 {
+    if (!blk_is_available(blk)) {
+        return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
+    }
+
     return bdrv_aio_flush(blk->bs, cb, opaque);
 }
 
@@ -634,7 +768,7 @@ BlockAIOCB *blk_aio_discard(BlockBackend *blk,
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+        return blk_abort_aio_request(blk, cb, opaque, ret);
     }
 
     return bdrv_aio_discard(blk->bs, sector_num, nb_sectors, cb, opaque);
@@ -666,12 +800,20 @@ int blk_aio_multiwrite(BlockBackend *blk, BlockRequest *reqs, int num_reqs)
 
 int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_ioctl(blk->bs, req, buf);
 }
 
 BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
                           BlockCompletionFunc *cb, void *opaque)
 {
+    if (!blk_is_available(blk)) {
+        return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
+    }
+
     return bdrv_aio_ioctl(blk->bs, req, buf, cb, opaque);
 }
 
@@ -687,11 +829,19 @@ int blk_co_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors)
 
 int blk_co_flush(BlockBackend *blk)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_co_flush(blk->bs);
 }
 
 int blk_flush(BlockBackend *blk)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_flush(blk->bs);
 }
 
@@ -702,7 +852,9 @@ int blk_flush_all(void)
 
 void blk_drain(BlockBackend *blk)
 {
-    bdrv_drain(blk->bs);
+    if (blk->bs) {
+        bdrv_drain(blk->bs);
+    }
 }
 
 void blk_drain_all(void)
@@ -710,76 +862,178 @@ void blk_drain_all(void)
     bdrv_drain_all();
 }
 
+void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
+                      BlockdevOnError on_write_error)
+{
+    blk->on_read_error = on_read_error;
+    blk->on_write_error = on_write_error;
+}
+
 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read)
 {
-    return bdrv_get_on_error(blk->bs, is_read);
+    return is_read ? blk->on_read_error : blk->on_write_error;
 }
 
 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
                                       int error)
 {
-    return bdrv_get_error_action(blk->bs, is_read, error);
+    BlockdevOnError on_err = blk_get_on_error(blk, is_read);
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        return (error == ENOSPC) ?
+               BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_STOP:
+        return BLOCK_ERROR_ACTION_STOP;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        return BLOCK_ERROR_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        return BLOCK_ERROR_ACTION_IGNORE;
+    default:
+        abort();
+    }
+}
+
+static void send_qmp_error_event(BlockBackend *blk,
+                                 BlockErrorAction action,
+                                 bool is_read, int error)
+{
+    IoOperationType optype;
+
+    optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
+    qapi_event_send_block_io_error(blk_name(blk), optype, action,
+                                   blk_iostatus_is_enabled(blk),
+                                   error == ENOSPC, strerror(error),
+                                   &error_abort);
 }
 
+/* This is done by device models because, while the block layer knows
+ * about the error, it does not know whether an operation comes from
+ * the device or the block layer (from a job, for example).
+ */
 void blk_error_action(BlockBackend *blk, BlockErrorAction action,
                       bool is_read, int error)
 {
-    bdrv_error_action(blk->bs, action, is_read, error);
+    assert(error >= 0);
+
+    if (action == BLOCK_ERROR_ACTION_STOP) {
+        /* First set the iostatus, so that "info block" returns an iostatus
+         * that matches the events raised so far (an additional error iostatus
+         * is fine, but not a lost one).
+         */
+        blk_iostatus_set_err(blk, error);
+
+        /* Then raise the request to stop the VM and the event.
+         * qemu_system_vmstop_request_prepare has two effects.  First,
+         * it ensures that the STOP event always comes after the
+         * BLOCK_IO_ERROR event.  Second, it ensures that even if management
+         * can observe the STOP event and do a "cont" before the STOP
+         * event is issued, the VM will not stop.  In this case, vm_start()
+         * also ensures that the STOP/RESUME pair of events is emitted.
+         */
+        qemu_system_vmstop_request_prepare();
+        send_qmp_error_event(blk, action, is_read, error);
+        qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
+    } else {
+        send_qmp_error_event(blk, action, is_read, error);
+    }
 }
 
 int blk_is_read_only(BlockBackend *blk)
 {
-    return bdrv_is_read_only(blk->bs);
+    if (blk->bs) {
+        return bdrv_is_read_only(blk->bs);
+    } else {
+        return blk->root_state.read_only;
+    }
 }
 
 int blk_is_sg(BlockBackend *blk)
 {
+    if (!blk->bs) {
+        return 0;
+    }
+
     return bdrv_is_sg(blk->bs);
 }
 
 int blk_enable_write_cache(BlockBackend *blk)
 {
-    return bdrv_enable_write_cache(blk->bs);
+    if (blk->bs) {
+        return bdrv_enable_write_cache(blk->bs);
+    } else {
+        return !!(blk->root_state.open_flags & BDRV_O_CACHE_WB);
+    }
 }
 
 void blk_set_enable_write_cache(BlockBackend *blk, bool wce)
 {
-    bdrv_set_enable_write_cache(blk->bs, wce);
+    if (blk->bs) {
+        bdrv_set_enable_write_cache(blk->bs, wce);
+    } else {
+        if (wce) {
+            blk->root_state.open_flags |= BDRV_O_CACHE_WB;
+        } else {
+            blk->root_state.open_flags &= ~BDRV_O_CACHE_WB;
+        }
+    }
 }
 
 void blk_invalidate_cache(BlockBackend *blk, Error **errp)
 {
+    if (!blk->bs) {
+        error_setg(errp, "Device '%s' has no medium", blk->name);
+        return;
+    }
+
     bdrv_invalidate_cache(blk->bs, errp);
 }
 
-int blk_is_inserted(BlockBackend *blk)
+bool blk_is_inserted(BlockBackend *blk)
+{
+    return blk->bs && bdrv_is_inserted(blk->bs);
+}
+
+bool blk_is_available(BlockBackend *blk)
 {
-    return bdrv_is_inserted(blk->bs);
+    return blk_is_inserted(blk) && !blk_dev_is_tray_open(blk);
 }
 
 void blk_lock_medium(BlockBackend *blk, bool locked)
 {
-    bdrv_lock_medium(blk->bs, locked);
+    if (blk->bs) {
+        bdrv_lock_medium(blk->bs, locked);
+    }
 }
 
 void blk_eject(BlockBackend *blk, bool eject_flag)
 {
-    bdrv_eject(blk->bs, eject_flag);
+    if (blk->bs) {
+        bdrv_eject(blk->bs, eject_flag);
+    }
 }
 
 int blk_get_flags(BlockBackend *blk)
 {
-    return bdrv_get_flags(blk->bs);
+    if (blk->bs) {
+        return bdrv_get_flags(blk->bs);
+    } else {
+        return blk->root_state.open_flags;
+    }
 }
 
 int blk_get_max_transfer_length(BlockBackend *blk)
 {
-    return blk->bs->bl.max_transfer_length;
+    if (blk->bs) {
+        return blk->bs->bl.max_transfer_length;
+    } else {
+        return 0;
+    }
 }
 
 void blk_set_guest_block_size(BlockBackend *blk, int align)
 {
-    bdrv_set_guest_block_size(blk->bs, align);
+    blk->guest_block_size = align;
 }
 
 void *blk_blockalign(BlockBackend *blk, size_t size)
@@ -789,40 +1043,64 @@ void *blk_blockalign(BlockBackend *blk, size_t size)
 
 bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp)
 {
+    if (!blk->bs) {
+        return false;
+    }
+
     return bdrv_op_is_blocked(blk->bs, op, errp);
 }
 
 void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason)
 {
-    bdrv_op_unblock(blk->bs, op, reason);
+    if (blk->bs) {
+        bdrv_op_unblock(blk->bs, op, reason);
+    }
 }
 
 void blk_op_block_all(BlockBackend *blk, Error *reason)
 {
-    bdrv_op_block_all(blk->bs, reason);
+    if (blk->bs) {
+        bdrv_op_block_all(blk->bs, reason);
+    }
 }
 
 void blk_op_unblock_all(BlockBackend *blk, Error *reason)
 {
-    bdrv_op_unblock_all(blk->bs, reason);
+    if (blk->bs) {
+        bdrv_op_unblock_all(blk->bs, reason);
+    }
 }
 
 AioContext *blk_get_aio_context(BlockBackend *blk)
 {
-    return bdrv_get_aio_context(blk->bs);
+    if (blk->bs) {
+        return bdrv_get_aio_context(blk->bs);
+    } else {
+        return qemu_get_aio_context();
+    }
+}
+
+static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb)
+{
+    BlockBackendAIOCB *blk_acb = DO_UPCAST(BlockBackendAIOCB, common, acb);
+    return blk_get_aio_context(blk_acb->blk);
 }
 
 void blk_set_aio_context(BlockBackend *blk, AioContext *new_context)
 {
-    bdrv_set_aio_context(blk->bs, new_context);
+    if (blk->bs) {
+        bdrv_set_aio_context(blk->bs, new_context);
+    }
 }
 
 void blk_add_aio_context_notifier(BlockBackend *blk,
         void (*attached_aio_context)(AioContext *new_context, void *opaque),
         void (*detach_aio_context)(void *opaque), void *opaque)
 {
-    bdrv_add_aio_context_notifier(blk->bs, attached_aio_context,
-                                  detach_aio_context, opaque);
+    if (blk->bs) {
+        bdrv_add_aio_context_notifier(blk->bs, attached_aio_context,
+                                      detach_aio_context, opaque);
+    }
 }
 
 void blk_remove_aio_context_notifier(BlockBackend *blk,
@@ -831,28 +1109,36 @@ void blk_remove_aio_context_notifier(BlockBackend *blk,
                                      void (*detach_aio_context)(void *),
                                      void *opaque)
 {
-    bdrv_remove_aio_context_notifier(blk->bs, attached_aio_context,
-                                     detach_aio_context, opaque);
+    if (blk->bs) {
+        bdrv_remove_aio_context_notifier(blk->bs, attached_aio_context,
+                                         detach_aio_context, opaque);
+    }
 }
 
 void blk_add_close_notifier(BlockBackend *blk, Notifier *notify)
 {
-    bdrv_add_close_notifier(blk->bs, notify);
+    if (blk->bs) {
+        bdrv_add_close_notifier(blk->bs, notify);
+    }
 }
 
 void blk_io_plug(BlockBackend *blk)
 {
-    bdrv_io_plug(blk->bs);
+    if (blk->bs) {
+        bdrv_io_plug(blk->bs);
+    }
 }
 
 void blk_io_unplug(BlockBackend *blk)
 {
-    bdrv_io_unplug(blk->bs);
+    if (blk->bs) {
+        bdrv_io_unplug(blk->bs);
+    }
 }
 
 BlockAcctStats *blk_get_stats(BlockBackend *blk)
 {
-    return bdrv_get_stats(blk->bs);
+    return &blk->stats;
 }
 
 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
@@ -885,6 +1171,10 @@ int blk_write_compressed(BlockBackend *blk, int64_t sector_num,
 
 int blk_truncate(BlockBackend *blk, int64_t offset)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_truncate(blk->bs, offset);
 }
 
@@ -901,20 +1191,94 @@ int blk_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors)
 int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
                      int64_t pos, int size)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_save_vmstate(blk->bs, buf, pos, size);
 }
 
 int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_load_vmstate(blk->bs, buf, pos, size);
 }
 
 int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_probe_blocksizes(blk->bs, bsz);
 }
 
 int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo)
 {
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
     return bdrv_probe_geometry(blk->bs, geo);
 }
+
+/*
+ * Updates the BlockBackendRootState object with data from the currently
+ * attached BlockDriverState.
+ */
+void blk_update_root_state(BlockBackend *blk)
+{
+    assert(blk->bs);
+
+    blk->root_state.open_flags    = blk->bs->open_flags;
+    blk->root_state.read_only     = blk->bs->read_only;
+    blk->root_state.detect_zeroes = blk->bs->detect_zeroes;
+
+    if (blk->root_state.throttle_group) {
+        g_free(blk->root_state.throttle_group);
+        throttle_group_unref(blk->root_state.throttle_state);
+    }
+    if (blk->bs->throttle_state) {
+        const char *name = throttle_group_get_name(blk->bs);
+        blk->root_state.throttle_group = g_strdup(name);
+        blk->root_state.throttle_state = throttle_group_incref(name);
+    } else {
+        blk->root_state.throttle_group = NULL;
+        blk->root_state.throttle_state = NULL;
+    }
+}
+
+/*
+ * Applies the information in the root state to the given BlockDriverState. This
+ * does not include the flags which have to be specified for bdrv_open(), use
+ * blk_get_open_flags_from_root_state() to inquire them.
+ */
+void blk_apply_root_state(BlockBackend *blk, BlockDriverState *bs)
+{
+    bs->detect_zeroes = blk->root_state.detect_zeroes;
+    if (blk->root_state.throttle_group) {
+        bdrv_io_limits_enable(bs, blk->root_state.throttle_group);
+    }
+}
+
+/*
+ * Returns the flags to be used for bdrv_open() of a BlockDriverState which is
+ * supposed to inherit the root state.
+ */
+int blk_get_open_flags_from_root_state(BlockBackend *blk)
+{
+    int bs_flags;
+
+    bs_flags = blk->root_state.read_only ? 0 : BDRV_O_RDWR;
+    bs_flags |= blk->root_state.open_flags & ~BDRV_O_RDWR;
+
+    return bs_flags;
+}
+
+BlockBackendRootState *blk_get_root_state(BlockBackend *blk)
+{
+    return &blk->root_state;
+}
diff --git a/block/bochs.c b/block/bochs.c
index 199ac2b9a..18949b9d4 100644
--- a/block/bochs.c
+++ b/block/bochs.c
@@ -103,7 +103,7 @@ static int bochs_open(BlockDriverState *bs, QDict *options, int flags,
 
     bs->read_only = 1; // no write support yet
 
-    ret = bdrv_pread(bs->file, 0, &bochs, sizeof(bochs));
+    ret = bdrv_pread(bs->file->bs, 0, &bochs, sizeof(bochs));
     if (ret < 0) {
         return ret;
     }
@@ -137,7 +137,7 @@ static int bochs_open(BlockDriverState *bs, QDict *options, int flags,
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(bs->file, le32_to_cpu(bochs.header), s->catalog_bitmap,
+    ret = bdrv_pread(bs->file->bs, le32_to_cpu(bochs.header), s->catalog_bitmap,
                      s->catalog_size * 4);
     if (ret < 0) {
         goto fail;
@@ -206,7 +206,7 @@ static int64_t seek_to_sector(BlockDriverState *bs, int64_t sector_num)
         (s->extent_blocks + s->bitmap_blocks));
 
     /* read in bitmap for current extent */
-    ret = bdrv_pread(bs->file, bitmap_offset + (extent_offset / 8),
+    ret = bdrv_pread(bs->file->bs, bitmap_offset + (extent_offset / 8),
                      &bitmap_entry, 1);
     if (ret < 0) {
         return ret;
@@ -229,7 +229,7 @@ static int bochs_read(BlockDriverState *bs, int64_t sector_num,
         if (block_offset < 0) {
             return block_offset;
         } else if (block_offset > 0) {
-            ret = bdrv_pread(bs->file, block_offset, buf, 512);
+            ret = bdrv_pread(bs->file->bs, block_offset, buf, 512);
             if (ret < 0) {
                 return ret;
             }
diff --git a/block/cloop.c b/block/cloop.c
index f328be06f..4190ae06d 100644
--- a/block/cloop.c
+++ b/block/cloop.c
@@ -66,7 +66,7 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
     bs->read_only = 1;
 
     /* read header */
-    ret = bdrv_pread(bs->file, 128, &s->block_size, 4);
+    ret = bdrv_pread(bs->file->bs, 128, &s->block_size, 4);
     if (ret < 0) {
         return ret;
     }
@@ -92,7 +92,7 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
         return -EINVAL;
     }
 
-    ret = bdrv_pread(bs->file, 128 + 4, &s->n_blocks, 4);
+    ret = bdrv_pread(bs->file->bs, 128 + 4, &s->n_blocks, 4);
     if (ret < 0) {
         return ret;
     }
@@ -123,7 +123,7 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(bs->file, 128 + 4 + 4, s->offsets, offsets_size);
+    ret = bdrv_pread(bs->file->bs, 128 + 4 + 4, s->offsets, offsets_size);
     if (ret < 0) {
         goto fail;
     }
@@ -203,8 +203,8 @@ static inline int cloop_read_block(BlockDriverState *bs, int block_num)
         int ret;
         uint32_t bytes = s->offsets[block_num + 1] - s->offsets[block_num];
 
-        ret = bdrv_pread(bs->file, s->offsets[block_num], s->compressed_block,
-                         bytes);
+        ret = bdrv_pread(bs->file->bs, s->offsets[block_num],
+                         s->compressed_block, bytes);
         if (ret != bytes) {
             return -1;
         }
diff --git a/block/commit.c b/block/commit.c
index 7312a5bdc..a5d02aa56 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -17,6 +17,7 @@
 #include "block/blockjob.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "sysemu/block-backend.h"
 
 enum {
     /*
@@ -213,7 +214,7 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
 
     if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
          on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, "Invalid parameter combination");
         return;
     }
@@ -235,14 +236,14 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
     orig_overlay_flags = bdrv_get_flags(overlay_bs);
 
     /* convert base & overlay_bs to r/w, if necessary */
-    if (!(orig_base_flags & BDRV_O_RDWR)) {
-        reopen_queue = bdrv_reopen_queue(reopen_queue, base,
-                                         orig_base_flags | BDRV_O_RDWR);
-    }
     if (!(orig_overlay_flags & BDRV_O_RDWR)) {
-        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs,
+        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs, NULL,
                                          orig_overlay_flags | BDRV_O_RDWR);
     }
+    if (!(orig_base_flags & BDRV_O_RDWR)) {
+        reopen_queue = bdrv_reopen_queue(reopen_queue, base, NULL,
+                                         orig_base_flags | BDRV_O_RDWR);
+    }
     if (reopen_queue) {
         bdrv_reopen_multiple(reopen_queue, &local_err);
         if (local_err != NULL) {
diff --git a/block/curl.c b/block/curl.c
index 032cc8ae2..89941826e 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -154,18 +154,20 @@ static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
     DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
     switch (action) {
         case CURL_POLL_IN:
-            aio_set_fd_handler(s->aio_context, fd, curl_multi_read,
-                               NULL, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               curl_multi_read, NULL, state);
             break;
         case CURL_POLL_OUT:
-            aio_set_fd_handler(s->aio_context, fd, NULL, curl_multi_do, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               NULL, curl_multi_do, state);
             break;
         case CURL_POLL_INOUT:
-            aio_set_fd_handler(s->aio_context, fd, curl_multi_read,
-                               curl_multi_do, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               curl_multi_read, curl_multi_do, state);
             break;
         case CURL_POLL_REMOVE:
-            aio_set_fd_handler(s->aio_context, fd, NULL, NULL, NULL);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               NULL, NULL, NULL);
             break;
     }
 
diff --git a/block/dmg.c b/block/dmg.c
index 9f2528169..546a6f533 100644
--- a/block/dmg.c
+++ b/block/dmg.c
@@ -85,7 +85,7 @@ static int read_uint64(BlockDriverState *bs, int64_t offset, uint64_t *result)
     uint64_t buffer;
     int ret;
 
-    ret = bdrv_pread(bs->file, offset, &buffer, 8);
+    ret = bdrv_pread(bs->file->bs, offset, &buffer, 8);
     if (ret < 0) {
         return ret;
     }
@@ -99,7 +99,7 @@ static int read_uint32(BlockDriverState *bs, int64_t offset, uint32_t *result)
     uint32_t buffer;
     int ret;
 
-    ret = bdrv_pread(bs->file, offset, &buffer, 4);
+    ret = bdrv_pread(bs->file->bs, offset, &buffer, 4);
     if (ret < 0) {
         return ret;
     }
@@ -354,7 +354,7 @@ static int dmg_read_resource_fork(BlockDriverState *bs, DmgHeaderState *ds,
         offset += 4;
 
         buffer = g_realloc(buffer, count);
-        ret = bdrv_pread(bs->file, offset, buffer, count);
+        ret = bdrv_pread(bs->file->bs, offset, buffer, count);
         if (ret < 0) {
             goto fail;
         }
@@ -391,7 +391,7 @@ static int dmg_read_plist_xml(BlockDriverState *bs, DmgHeaderState *ds,
 
     buffer = g_malloc(info_length + 1);
     buffer[info_length] = '\0';
-    ret = bdrv_pread(bs->file, info_begin, buffer, info_length);
+    ret = bdrv_pread(bs->file->bs, info_begin, buffer, info_length);
     if (ret != info_length) {
         ret = -EINVAL;
         goto fail;
@@ -446,7 +446,7 @@ static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
     ds.max_sectors_per_chunk = 1;
 
     /* locate the UDIF trailer */
-    offset = dmg_find_koly_offset(bs->file, errp);
+    offset = dmg_find_koly_offset(bs->file->bs, errp);
     if (offset < 0) {
         ret = offset;
         goto fail;
@@ -514,9 +514,9 @@ static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* initialize zlib engine */
-    s->compressed_chunk = qemu_try_blockalign(bs->file,
+    s->compressed_chunk = qemu_try_blockalign(bs->file->bs,
                                               ds.max_compressed_size + 1);
-    s->uncompressed_chunk = qemu_try_blockalign(bs->file,
+    s->uncompressed_chunk = qemu_try_blockalign(bs->file->bs,
                                                 512 * ds.max_sectors_per_chunk);
     if (s->compressed_chunk == NULL || s->uncompressed_chunk == NULL) {
         ret = -ENOMEM;
@@ -592,7 +592,7 @@ static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
         case 0x80000005: { /* zlib compressed */
             /* we need to buffer, because only the chunk as whole can be
              * inflated. */
-            ret = bdrv_pread(bs->file, s->offsets[chunk],
+            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                              s->compressed_chunk, s->lengths[chunk]);
             if (ret != s->lengths[chunk]) {
                 return -1;
@@ -616,7 +616,7 @@ static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
         case 0x80000006: /* bzip2 compressed */
             /* we need to buffer, because only the chunk as whole can be
              * inflated. */
-            ret = bdrv_pread(bs->file, s->offsets[chunk],
+            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                              s->compressed_chunk, s->lengths[chunk]);
             if (ret != s->lengths[chunk]) {
                 return -1;
@@ -641,7 +641,7 @@ static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
             break;
 #endif /* CONFIG_BZIP2 */
         case 1: /* copy */
-            ret = bdrv_pread(bs->file, s->offsets[chunk],
+            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                              s->uncompressed_chunk, s->lengths[chunk]);
             if (ret != s->lengths[chunk]) {
                 return -1;
diff --git a/block/gluster.c b/block/gluster.c
index 1eb3a8c39..0857c1464 100644
--- a/block/gluster.c
+++ b/block/gluster.c
@@ -429,28 +429,23 @@ static coroutine_fn int qemu_gluster_co_write_zeroes(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
     off_t size = nb_sectors * BDRV_SECTOR_SIZE;
     off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = size;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = size;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_zerofill_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
+    ret = glfs_zerofill_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 
 static inline bool gluster_supports_zerofill(void)
@@ -541,35 +536,30 @@ static coroutine_fn int qemu_gluster_co_rw(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int write)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
     size_t size = nb_sectors * BDRV_SECTOR_SIZE;
     off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = size;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = size;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
     if (write) {
         ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
-            &gluster_finish_aiocb, acb);
+            gluster_finish_aiocb, &acb);
     } else {
         ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
-            &gluster_finish_aiocb, acb);
+            gluster_finish_aiocb, &acb);
     }
 
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 
 static int qemu_gluster_truncate(BlockDriverState *bs, int64_t offset)
@@ -600,26 +590,21 @@ static coroutine_fn int qemu_gluster_co_writev(BlockDriverState *bs,
 static coroutine_fn int qemu_gluster_co_flush_to_disk(BlockDriverState *bs)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
 
-    acb->size = 0;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = 0;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_fsync_async(s->fd, &gluster_finish_aiocb, acb);
+    ret = glfs_fsync_async(s->fd, gluster_finish_aiocb, &acb);
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 
 #ifdef CONFIG_GLUSTERFS_DISCARD
@@ -627,28 +612,23 @@ static coroutine_fn int qemu_gluster_co_discard(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
     size_t size = nb_sectors * BDRV_SECTOR_SIZE;
     off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = 0;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = 0;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_discard_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
+    ret = glfs_discard_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 #endif
 
diff --git a/block/io.c b/block/io.c
index d4bc83b33..e00fb5d69 100644
--- a/block/io.c
+++ b/block/io.c
@@ -23,6 +23,7 @@
  */
 
 #include "trace.h"
+#include "sysemu/block-backend.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
 #include "block/throttle-groups.h"
@@ -156,38 +157,38 @@ void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
 
     /* Take some limits from the children as a default */
     if (bs->file) {
-        bdrv_refresh_limits(bs->file, &local_err);
+        bdrv_refresh_limits(bs->file->bs, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
         }
-        bs->bl.opt_transfer_length = bs->file->bl.opt_transfer_length;
-        bs->bl.max_transfer_length = bs->file->bl.max_transfer_length;
-        bs->bl.min_mem_alignment = bs->file->bl.min_mem_alignment;
-        bs->bl.opt_mem_alignment = bs->file->bl.opt_mem_alignment;
+        bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
+        bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
+        bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
+        bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
     } else {
         bs->bl.min_mem_alignment = 512;
         bs->bl.opt_mem_alignment = getpagesize();
     }
 
-    if (bs->backing_hd) {
-        bdrv_refresh_limits(bs->backing_hd, &local_err);
+    if (bs->backing) {
+        bdrv_refresh_limits(bs->backing->bs, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
         }
         bs->bl.opt_transfer_length =
             MAX(bs->bl.opt_transfer_length,
-                bs->backing_hd->bl.opt_transfer_length);
+                bs->backing->bs->bl.opt_transfer_length);
         bs->bl.max_transfer_length =
             MIN_NON_ZERO(bs->bl.max_transfer_length,
-                         bs->backing_hd->bl.max_transfer_length);
+                         bs->backing->bs->bl.max_transfer_length);
         bs->bl.opt_mem_alignment =
             MAX(bs->bl.opt_mem_alignment,
-                bs->backing_hd->bl.opt_mem_alignment);
+                bs->backing->bs->bl.opt_mem_alignment);
         bs->bl.min_mem_alignment =
             MAX(bs->bl.min_mem_alignment,
-                bs->backing_hd->bl.min_mem_alignment);
+                bs->backing->bs->bl.min_mem_alignment);
     }
 
     /* Then let the driver override it */
@@ -213,8 +214,10 @@ void bdrv_disable_copy_on_read(BlockDriverState *bs)
 }
 
 /* Check if any requests are in-flight (including throttled requests) */
-static bool bdrv_requests_pending(BlockDriverState *bs)
+bool bdrv_requests_pending(BlockDriverState *bs)
 {
+    BdrvChild *child;
+
     if (!QLIST_EMPTY(&bs->tracked_requests)) {
         return true;
     }
@@ -224,17 +227,31 @@ static bool bdrv_requests_pending(BlockDriverState *bs)
     if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) {
         return true;
     }
-    if (bs->file && bdrv_requests_pending(bs->file)) {
-        return true;
-    }
-    if (bs->backing_hd && bdrv_requests_pending(bs->backing_hd)) {
-        return true;
+
+    QLIST_FOREACH(child, &bs->children, next) {
+        if (bdrv_requests_pending(child->bs)) {
+            return true;
+        }
     }
+
     return false;
 }
 
+static void bdrv_drain_recurse(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    if (bs->drv && bs->drv->bdrv_drain) {
+        bs->drv->bdrv_drain(bs);
+    }
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_drain_recurse(child->bs);
+    }
+}
+
 /*
- * Wait for pending requests to complete on a single BlockDriverState subtree
+ * Wait for pending requests to complete on a single BlockDriverState subtree,
+ * and suspend block driver's internal I/O until next request arrives.
  *
  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
  * AioContext.
@@ -247,6 +264,7 @@ void bdrv_drain(BlockDriverState *bs)
 {
     bool busy = true;
 
+    bdrv_drain_recurse(bs);
     while (busy) {
         /* Keep iterating */
          bdrv_flush_io_queue(bs);
@@ -344,13 +362,14 @@ static void tracked_request_end(BdrvTrackedRequest *req)
 static void tracked_request_begin(BdrvTrackedRequest *req,
                                   BlockDriverState *bs,
                                   int64_t offset,
-                                  unsigned int bytes, bool is_write)
+                                  unsigned int bytes,
+                                  enum BdrvTrackedRequestType type)
 {
     *req = (BdrvTrackedRequest){
         .bs = bs,
         .offset         = offset,
         .bytes          = bytes,
-        .is_write       = is_write,
+        .type           = type,
         .co             = qemu_coroutine_self(),
         .serialising    = false,
         .overlap_offset = offset,
@@ -844,7 +863,9 @@ static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
         mark_request_serialising(req, bdrv_get_cluster_size(bs));
     }
 
-    wait_serialising_requests(req);
+    if (!(flags & BDRV_REQ_NO_SERIALISING)) {
+        wait_serialising_requests(req);
+    }
 
     if (flags & BDRV_REQ_COPY_ON_READ) {
         int pnum;
@@ -932,7 +953,8 @@ static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
         return ret;
     }
 
-    if (bs->copy_on_read) {
+    /* Don't do copy-on-read if we read data before write operation */
+    if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
         flags |= BDRV_REQ_COPY_ON_READ;
     }
 
@@ -966,7 +988,7 @@ static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
         bytes = ROUND_UP(bytes, align);
     }
 
-    tracked_request_begin(&req, bs, offset, bytes, false);
+    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
     ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
                               use_local_qiov ? &local_qiov : qiov,
                               flags);
@@ -1001,6 +1023,15 @@ int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
 }
 
+int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
+                            BDRV_REQ_NO_SERIALISING);
+}
+
 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
 {
@@ -1127,13 +1158,13 @@ static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
     if (ret < 0) {
         /* Do nothing, write notifier decided to fail this request */
     } else if (flags & BDRV_REQ_ZERO_WRITE) {
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_ZERO);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
         ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
     } else {
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV);
         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
     }
-    BLKDBG_EVENT(bs, BLKDBG_PWRITEV_DONE);
+    bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
 
     if (ret == 0 && !bs->enable_write_cache) {
         ret = bdrv_co_flush(bs);
@@ -1141,7 +1172,9 @@ static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
 
     bdrv_set_dirty(bs, sector_num, nb_sectors);
 
-    block_acct_highest_sector(&bs->stats, sector_num, nb_sectors);
+    if (bs->wr_highest_offset < offset + bytes) {
+        bs->wr_highest_offset = offset + bytes;
+    }
 
     if (ret >= 0) {
         bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
@@ -1182,13 +1215,13 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
         /* RMW the unaligned part before head. */
         mark_request_serialising(req, align);
         wait_serialising_requests(req);
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
         ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
                                   align, &local_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
 
         memset(buf + head_padding_bytes, 0, zero_bytes);
         ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
@@ -1220,13 +1253,13 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
         /* RMW the unaligned part after tail. */
         mark_request_serialising(req, align);
         wait_serialising_requests(req);
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
         ret = bdrv_aligned_preadv(bs, req, offset, align,
                                   align, &local_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
 
         memset(buf, 0, bytes);
         ret = bdrv_aligned_pwritev(bs, req, offset, align,
@@ -1276,7 +1309,7 @@ static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
      * Pad qiov with the read parts and be sure to have a tracked request not
      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
      */
-    tracked_request_begin(&req, bs, offset, bytes, true);
+    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
 
     if (!qiov) {
         ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
@@ -1297,13 +1330,13 @@ static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
         };
         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
 
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
         ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
                                   align, &head_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
 
         qemu_iovec_init(&local_qiov, qiov->niov + 2);
         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
@@ -1331,13 +1364,13 @@ static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
         };
         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
 
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
         ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
                                   align, &tail_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
 
         if (!use_local_qiov) {
             qemu_iovec_init(&local_qiov, qiov->niov + 1);
@@ -1486,7 +1519,7 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
 
     if (ret & BDRV_BLOCK_RAW) {
         assert(ret & BDRV_BLOCK_OFFSET_VALID);
-        return bdrv_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
+        return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
                                      *pnum, pnum);
     }
 
@@ -1495,8 +1528,8 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
     } else {
         if (bdrv_unallocated_blocks_are_zero(bs)) {
             ret |= BDRV_BLOCK_ZERO;
-        } else if (bs->backing_hd) {
-            BlockDriverState *bs2 = bs->backing_hd;
+        } else if (bs->backing) {
+            BlockDriverState *bs2 = bs->backing->bs;
             int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
             if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
                 ret |= BDRV_BLOCK_ZERO;
@@ -1509,7 +1542,7 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
         (ret & BDRV_BLOCK_OFFSET_VALID)) {
         int file_pnum;
 
-        ret2 = bdrv_co_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
+        ret2 = bdrv_co_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
                                         *pnum, &file_pnum);
         if (ret2 >= 0) {
             /* Ignore errors.  This is just providing extra information, it
@@ -1541,7 +1574,7 @@ static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
     int64_t ret = 0;
 
     assert(bs != base);
-    for (p = bs; p != base; p = p->backing_hd) {
+    for (p = bs; p != base; p = backing_bs(p)) {
         ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum);
         if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
             break;
@@ -1604,7 +1637,7 @@ int64_t bdrv_get_block_status(BlockDriverState *bs,
                               int64_t sector_num,
                               int nb_sectors, int *pnum)
 {
-    return bdrv_get_block_status_above(bs, bs->backing_hd,
+    return bdrv_get_block_status_above(bs, backing_bs(bs),
                                        sector_num, nb_sectors, pnum);
 }
 
@@ -1662,7 +1695,7 @@ int bdrv_is_allocated_above(BlockDriverState *top,
             n = pnum_inter;
         }
 
-        intermediate = intermediate->backing_hd;
+        intermediate = backing_bs(intermediate);
     }
 
     *pnum = n;
@@ -1713,7 +1746,7 @@ int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
     } else if (drv->bdrv_save_vmstate) {
         return drv->bdrv_save_vmstate(bs, qiov, pos);
     } else if (bs->file) {
-        return bdrv_writev_vmstate(bs->file, qiov, pos);
+        return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
     }
 
     return -ENOTSUP;
@@ -1728,7 +1761,7 @@ int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
     if (drv->bdrv_load_vmstate)
         return drv->bdrv_load_vmstate(bs, buf, pos, size);
     if (bs->file)
-        return bdrv_load_vmstate(bs->file, buf, pos, size);
+        return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
     return -ENOTSUP;
 }
 
@@ -1893,7 +1926,10 @@ static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
         }
     }
 
-    block_acct_merge_done(&bs->stats, BLOCK_ACCT_WRITE, num_reqs - outidx - 1);
+    if (bs->blk) {
+        block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE,
+                              num_reqs - outidx - 1);
+    }
 
     return outidx + 1;
 }
@@ -2208,7 +2244,7 @@ void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
 {
     BlockAIOCB *acb;
 
-    acb = g_slice_alloc(aiocb_info->aiocb_size);
+    acb = g_malloc(aiocb_info->aiocb_size);
     acb->aiocb_info = aiocb_info;
     acb->bs = bs;
     acb->cb = cb;
@@ -2228,7 +2264,7 @@ void qemu_aio_unref(void *p)
     BlockAIOCB *acb = p;
     assert(acb->refcnt > 0);
     if (--acb->refcnt == 0) {
-        g_slice_free1(acb->aiocb_info->aiocb_size, acb);
+        g_free(acb);
     }
 }
 
@@ -2298,18 +2334,20 @@ static void coroutine_fn bdrv_flush_co_entry(void *opaque)
 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
 {
     int ret;
+    BdrvTrackedRequest req;
 
     if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
         bdrv_is_sg(bs)) {
         return 0;
     }
 
+    tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
     /* Write back cached data to the OS even with cache=unsafe */
     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
     if (bs->drv->bdrv_co_flush_to_os) {
         ret = bs->drv->bdrv_co_flush_to_os(bs);
         if (ret < 0) {
-            return ret;
+            goto out;
         }
     }
 
@@ -2349,14 +2387,17 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
         ret = 0;
     }
     if (ret < 0) {
-        return ret;
+        goto out;
     }
 
     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
      * in the case of cache=unsafe, so there are no useless flushes.
      */
 flush_parent:
-    return bdrv_co_flush(bs->file);
+    ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
+out:
+    tracked_request_end(&req);
+    return ret;
 }
 
 int bdrv_flush(BlockDriverState *bs)
@@ -2399,6 +2440,7 @@ static void coroutine_fn bdrv_discard_co_entry(void *opaque)
 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
                                  int nb_sectors)
 {
+    BdrvTrackedRequest req;
     int max_discard, ret;
 
     if (!bs->drv) {
@@ -2421,6 +2463,8 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
         return 0;
     }
 
+    tracked_request_begin(&req, bs, sector_num, nb_sectors,
+                          BDRV_TRACKED_DISCARD);
     bdrv_set_dirty(bs, sector_num, nb_sectors);
 
     max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
@@ -2454,20 +2498,24 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
             acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
                                             bdrv_co_io_em_complete, &co);
             if (acb == NULL) {
-                return -EIO;
+                ret = -EIO;
+                goto out;
             } else {
                 qemu_coroutine_yield();
                 ret = co.ret;
             }
         }
         if (ret && ret != -ENOTSUP) {
-            return ret;
+            goto out;
         }
 
         sector_num += num;
         nb_sectors -= num;
     }
-    return 0;
+    ret = 0;
+out:
+    tracked_request_end(&req);
+    return ret;
 }
 
 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
@@ -2496,26 +2544,109 @@ int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
     return rwco.ret;
 }
 
-/* needed for generic scsi interface */
+typedef struct {
+    CoroutineIOCompletion *co;
+    QEMUBH *bh;
+} BdrvIoctlCompletionData;
 
-int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
+static void bdrv_ioctl_bh_cb(void *opaque)
+{
+    BdrvIoctlCompletionData *data = opaque;
+
+    bdrv_co_io_em_complete(data->co, -ENOTSUP);
+    qemu_bh_delete(data->bh);
+}
+
+static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
 {
     BlockDriver *drv = bs->drv;
+    BdrvTrackedRequest tracked_req;
+    CoroutineIOCompletion co = {
+        .coroutine = qemu_coroutine_self(),
+    };
+    BlockAIOCB *acb;
 
-    if (drv && drv->bdrv_ioctl)
-        return drv->bdrv_ioctl(bs, req, buf);
-    return -ENOTSUP;
+    tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
+    if (!drv || !drv->bdrv_aio_ioctl) {
+        co.ret = -ENOTSUP;
+        goto out;
+    }
+
+    acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
+    if (!acb) {
+        BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
+        data->bh = aio_bh_new(bdrv_get_aio_context(bs),
+                                bdrv_ioctl_bh_cb, data);
+        data->co = &co;
+        qemu_bh_schedule(data->bh);
+    }
+    qemu_coroutine_yield();
+out:
+    tracked_request_end(&tracked_req);
+    return co.ret;
+}
+
+typedef struct {
+    BlockDriverState *bs;
+    int req;
+    void *buf;
+    int ret;
+} BdrvIoctlCoData;
+
+static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
+{
+    BdrvIoctlCoData *data = opaque;
+    data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
+}
+
+/* needed for generic scsi interface */
+int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
+{
+    BdrvIoctlCoData data = {
+        .bs = bs,
+        .req = req,
+        .buf = buf,
+        .ret = -EINPROGRESS,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_co_ioctl_entry(&data);
+    } else {
+        Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
+        qemu_coroutine_enter(co, &data);
+    }
+    while (data.ret == -EINPROGRESS) {
+        aio_poll(bdrv_get_aio_context(bs), true);
+    }
+    return data.ret;
+}
+
+static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
+{
+    BlockAIOCBCoroutine *acb = opaque;
+    acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
+                                      acb->req.req, acb->req.buf);
+    bdrv_co_complete(acb);
 }
 
 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque)
 {
-    BlockDriver *drv = bs->drv;
+    BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
+                                            bs, cb, opaque);
+    Coroutine *co;
 
-    if (drv && drv->bdrv_aio_ioctl)
-        return drv->bdrv_aio_ioctl(bs, req, buf, cb, opaque);
-    return NULL;
+    acb->need_bh = true;
+    acb->req.error = -EINPROGRESS;
+    acb->req.req = req;
+    acb->req.buf = buf;
+    co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
+    qemu_coroutine_enter(co, acb);
+
+    bdrv_co_maybe_schedule_bh(acb);
+    return &acb->common;
 }
 
 void *qemu_blockalign(BlockDriverState *bs, size_t size)
@@ -2584,7 +2715,7 @@ void bdrv_io_plug(BlockDriverState *bs)
     if (drv && drv->bdrv_io_plug) {
         drv->bdrv_io_plug(bs);
     } else if (bs->file) {
-        bdrv_io_plug(bs->file);
+        bdrv_io_plug(bs->file->bs);
     }
 }
 
@@ -2594,7 +2725,7 @@ void bdrv_io_unplug(BlockDriverState *bs)
     if (drv && drv->bdrv_io_unplug) {
         drv->bdrv_io_unplug(bs);
     } else if (bs->file) {
-        bdrv_io_unplug(bs->file);
+        bdrv_io_unplug(bs->file->bs);
     }
 }
 
@@ -2604,7 +2735,24 @@ void bdrv_flush_io_queue(BlockDriverState *bs)
     if (drv && drv->bdrv_flush_io_queue) {
         drv->bdrv_flush_io_queue(bs);
     } else if (bs->file) {
-        bdrv_flush_io_queue(bs->file);
+        bdrv_flush_io_queue(bs->file->bs);
     }
     bdrv_start_throttled_reqs(bs);
 }
+
+void bdrv_drained_begin(BlockDriverState *bs)
+{
+    if (!bs->quiesce_counter++) {
+        aio_disable_external(bdrv_get_aio_context(bs));
+    }
+    bdrv_drain(bs);
+}
+
+void bdrv_drained_end(BlockDriverState *bs)
+{
+    assert(bs->quiesce_counter > 0);
+    if (--bs->quiesce_counter > 0) {
+        return;
+    }
+    aio_enable_external(bdrv_get_aio_context(bs));
+}
diff --git a/block/iscsi.c b/block/iscsi.c
index 93f1ee4c6..bd1f1bfcd 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -84,6 +84,7 @@ typedef struct IscsiTask {
     IscsiLun *iscsilun;
     QEMUTimer retry_timer;
     bool force_next_flush;
+    int err_code;
 } IscsiTask;
 
 typedef struct IscsiAIOCB {
@@ -96,6 +97,7 @@ typedef struct IscsiAIOCB {
     int status;
     int64_t sector_num;
     int nb_sectors;
+    int ret;
 #ifdef __linux__
     sg_io_hdr_t *ioh;
 #endif
@@ -169,19 +171,70 @@ static inline unsigned exp_random(double mean)
     return -mean * log((double)rand() / RAND_MAX);
 }
 
-/* SCSI_STATUS_TASK_SET_FULL and SCSI_STATUS_TIMEOUT were introduced
- * in libiscsi 1.10.0 as part of an enum. The LIBISCSI_API_VERSION
- * macro was introduced in 1.11.0. So use the API_VERSION macro as
- * a hint that the macros are defined and define them ourselves
- * otherwise to keep the required libiscsi version at 1.9.0 */
-#if !defined(LIBISCSI_API_VERSION)
-#define QEMU_SCSI_STATUS_TASK_SET_FULL  0x28
-#define QEMU_SCSI_STATUS_TIMEOUT        0x0f000002
-#else
-#define QEMU_SCSI_STATUS_TASK_SET_FULL  SCSI_STATUS_TASK_SET_FULL
-#define QEMU_SCSI_STATUS_TIMEOUT        SCSI_STATUS_TIMEOUT
+/* SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST was introduced in
+ * libiscsi 1.10.0, together with other constants we need.  Use it as
+ * a hint that we have to define them ourselves if needed, to keep the
+ * minimum required libiscsi version at 1.9.0.  We use an ASCQ macro for
+ * the test because SCSI_STATUS_* is an enum.
+ *
+ * To guard against future changes where SCSI_SENSE_ASCQ_* also becomes
+ * an enum, check against the LIBISCSI_API_VERSION macro, which was
+ * introduced in 1.11.0.  If it is present, there is no need to define
+ * anything.
+ */
+#if !defined(SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST) && \
+    !defined(LIBISCSI_API_VERSION)
+#define SCSI_STATUS_TASK_SET_FULL                          0x28
+#define SCSI_STATUS_TIMEOUT                                0x0f000002
+#define SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST    0x2600
+#define SCSI_SENSE_ASCQ_PARAMETER_LIST_LENGTH_ERROR        0x1a00
 #endif
 
+static int iscsi_translate_sense(struct scsi_sense *sense)
+{
+    int ret;
+
+    switch (sense->key) {
+    case SCSI_SENSE_NOT_READY:
+        return -EBUSY;
+    case SCSI_SENSE_DATA_PROTECTION:
+        return -EACCES;
+    case SCSI_SENSE_COMMAND_ABORTED:
+        return -ECANCELED;
+    case SCSI_SENSE_ILLEGAL_REQUEST:
+        /* Parse ASCQ */
+        break;
+    default:
+        return -EIO;
+    }
+    switch (sense->ascq) {
+    case SCSI_SENSE_ASCQ_PARAMETER_LIST_LENGTH_ERROR:
+    case SCSI_SENSE_ASCQ_INVALID_OPERATION_CODE:
+    case SCSI_SENSE_ASCQ_INVALID_FIELD_IN_CDB:
+    case SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST:
+        ret = -EINVAL;
+        break;
+    case SCSI_SENSE_ASCQ_LBA_OUT_OF_RANGE:
+        ret = -ENOSPC;
+        break;
+    case SCSI_SENSE_ASCQ_LOGICAL_UNIT_NOT_SUPPORTED:
+        ret = -ENOTSUP;
+        break;
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT:
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT_TRAY_CLOSED:
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT_TRAY_OPEN:
+        ret = -ENOMEDIUM;
+        break;
+    case SCSI_SENSE_ASCQ_WRITE_PROTECTED:
+        ret = -EACCES;
+        break;
+    default:
+        ret = -EIO;
+        break;
+    }
+    return ret;
+}
+
 static void
 iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
                         void *command_data, void *opaque)
@@ -203,11 +256,11 @@ iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
                 goto out;
             }
             if (status == SCSI_STATUS_BUSY ||
-                status == QEMU_SCSI_STATUS_TIMEOUT ||
-                status == QEMU_SCSI_STATUS_TASK_SET_FULL) {
+                status == SCSI_STATUS_TIMEOUT ||
+                status == SCSI_STATUS_TASK_SET_FULL) {
                 unsigned retry_time =
                     exp_random(iscsi_retry_times[iTask->retries - 1]);
-                if (status == QEMU_SCSI_STATUS_TIMEOUT) {
+                if (status == SCSI_STATUS_TIMEOUT) {
                     /* make sure the request is rescheduled AFTER the
                      * reconnect is initiated */
                     retry_time = EVENT_INTERVAL * 2;
@@ -226,6 +279,7 @@ iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
                 return;
             }
         }
+        iTask->err_code = iscsi_translate_sense(&task->sense);
         error_report("iSCSI Failure: %s", iscsi_get_error(iscsi));
     } else {
         iTask->iscsilun->force_next_flush |= iTask->force_next_flush;
@@ -291,8 +345,8 @@ iscsi_set_events(IscsiLun *iscsilun)
     int ev = iscsi_which_events(iscsi);
 
     if (ev != iscsilun->events) {
-        aio_set_fd_handler(iscsilun->aio_context,
-                           iscsi_get_fd(iscsi),
+        aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsi),
+                           false,
                            (ev & POLLIN) ? iscsi_process_read : NULL,
                            (ev & POLLOUT) ? iscsi_process_write : NULL,
                            iscsilun);
@@ -455,7 +509,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     iscsi_allocationmap_set(iscsilun, sector_num, nb_sectors);
@@ -644,7 +698,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     return 0;
@@ -683,7 +737,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     return 0;
@@ -703,7 +757,7 @@ iscsi_aio_ioctl_cb(struct iscsi_context *iscsi, int status,
     if (status < 0) {
         error_report("Failed to ioctl(SG_IO) to iSCSI lun. %s",
                      iscsi_get_error(iscsi));
-        acb->status = -EIO;
+        acb->status = iscsi_translate_sense(&acb->task->sense);
     }
 
     acb->ioh->driver_status = 0;
@@ -726,6 +780,38 @@ iscsi_aio_ioctl_cb(struct iscsi_context *iscsi, int status,
     iscsi_schedule_bh(acb);
 }
 
+static void iscsi_ioctl_bh_completion(void *opaque)
+{
+    IscsiAIOCB *acb = opaque;
+
+    qemu_bh_delete(acb->bh);
+    acb->common.cb(acb->common.opaque, acb->ret);
+    qemu_aio_unref(acb);
+}
+
+static void iscsi_ioctl_handle_emulated(IscsiAIOCB *acb, int req, void *buf)
+{
+    BlockDriverState *bs = acb->common.bs;
+    IscsiLun *iscsilun = bs->opaque;
+    int ret = 0;
+
+    switch (req) {
+    case SG_GET_VERSION_NUM:
+        *(int *)buf = 30000;
+        break;
+    case SG_GET_SCSI_ID:
+        ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
+        break;
+    default:
+        ret = -EINVAL;
+    }
+    assert(!acb->bh);
+    acb->bh = aio_bh_new(bdrv_get_aio_context(bs),
+                         iscsi_ioctl_bh_completion, acb);
+    acb->ret = ret;
+    qemu_bh_schedule(acb->bh);
+}
+
 static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque)
@@ -735,8 +821,6 @@ static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     struct iscsi_data data;
     IscsiAIOCB *acb;
 
-    assert(req == SG_IO);
-
     acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
@@ -745,6 +829,11 @@ static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     acb->buf         = NULL;
     acb->ioh         = buf;
 
+    if (req != SG_IO) {
+        iscsi_ioctl_handle_emulated(acb, req, buf);
+        return &acb->common;
+    }
+
     acb->task = malloc(sizeof(struct scsi_task));
     if (acb->task == NULL) {
         error_report("iSCSI: Failed to allocate task for scsi command. %s",
@@ -809,38 +898,6 @@ static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     return &acb->common;
 }
 
-static void ioctl_cb(void *opaque, int status)
-{
-    int *p_status = opaque;
-    *p_status = status;
-}
-
-static int iscsi_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    IscsiLun *iscsilun = bs->opaque;
-    int status;
-
-    switch (req) {
-    case SG_GET_VERSION_NUM:
-        *(int *)buf = 30000;
-        break;
-    case SG_GET_SCSI_ID:
-        ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
-        break;
-    case SG_IO:
-        status = -EINPROGRESS;
-        iscsi_aio_ioctl(bs, req, buf, ioctl_cb, &status);
-
-        while (status == -EINPROGRESS) {
-            aio_poll(iscsilun->aio_context, true);
-        }
-
-        return 0;
-    default:
-        return -1;
-    }
-    return 0;
-}
 #endif
 
 static int64_t
@@ -905,7 +962,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     iscsi_allocationmap_clear(iscsilun, sector_num, nb_sectors);
@@ -999,7 +1056,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     if (flags & BDRV_REQ_MAY_UNMAP) {
@@ -1280,9 +1337,8 @@ static void iscsi_detach_aio_context(BlockDriverState *bs)
 {
     IscsiLun *iscsilun = bs->opaque;
 
-    aio_set_fd_handler(iscsilun->aio_context,
-                       iscsi_get_fd(iscsilun->iscsi),
-                       NULL, NULL, NULL);
+    aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsilun->iscsi),
+                       false, NULL, NULL, NULL);
     iscsilun->events = 0;
 
     if (iscsilun->nop_timer) {
@@ -1772,7 +1828,6 @@ static BlockDriver bdrv_iscsi = {
     .bdrv_co_flush_to_disk = iscsi_co_flush,
 
 #ifdef __linux__
-    .bdrv_ioctl       = iscsi_ioctl,
     .bdrv_aio_ioctl   = iscsi_aio_ioctl,
 #endif
 
diff --git a/block/linux-aio.c b/block/linux-aio.c
index c991443c5..88b0520a8 100644
--- a/block/linux-aio.c
+++ b/block/linux-aio.c
@@ -287,7 +287,7 @@ void laio_detach_aio_context(void *s_, AioContext *old_context)
 {
     struct qemu_laio_state *s = s_;
 
-    aio_set_event_notifier(old_context, &s->e, NULL);
+    aio_set_event_notifier(old_context, &s->e, false, NULL);
     qemu_bh_delete(s->completion_bh);
 }
 
@@ -296,7 +296,8 @@ void laio_attach_aio_context(void *s_, AioContext *new_context)
     struct qemu_laio_state *s = s_;
 
     s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
-    aio_set_event_notifier(new_context, &s->e, qemu_laio_completion_cb);
+    aio_set_event_notifier(new_context, &s->e, false,
+                           qemu_laio_completion_cb);
 }
 
 void *laio_init(void)
diff --git a/block/mirror.c b/block/mirror.c
index b2fb4b9b1..0e8f5565a 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -14,6 +14,7 @@
 #include "trace.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
 #include "qemu/bitmap.h"
@@ -113,7 +114,7 @@ static void mirror_iteration_done(MirrorOp *op, int ret)
     }
 
     qemu_iovec_destroy(&op->qiov);
-    g_slice_free(MirrorOp, op);
+    g_free(op);
 
     if (s->waiting_for_io) {
         qemu_coroutine_enter(s->common.co, NULL);
@@ -246,6 +247,10 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
             trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight);
             break;
         }
+        if (IOV_MAX < nb_chunks + added_chunks) {
+            trace_mirror_break_iov_max(s, nb_chunks, added_chunks);
+            break;
+        }
 
         /* We have enough free space to copy these sectors.  */
         bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks);
@@ -260,7 +265,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
     } while (delay_ns == 0 && next_sector < end);
 
     /* Allocate a MirrorOp that is used as an AIO callback.  */
-    op = g_slice_new(MirrorOp);
+    op = g_new(MirrorOp, 1);
     op->s = s;
     op->sector_num = sector_num;
     op->nb_sectors = nb_sectors;
@@ -349,6 +354,11 @@ static void mirror_exit(BlockJob *job, void *opaque)
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
     MirrorExitData *data = opaque;
     AioContext *replace_aio_context = NULL;
+    BlockDriverState *src = s->common.bs;
+
+    /* Make sure that the source BDS doesn't go away before we called
+     * block_job_completed(). */
+    bdrv_ref(src);
 
     if (s->to_replace) {
         replace_aio_context = bdrv_get_aio_context(s->to_replace);
@@ -363,14 +373,7 @@ static void mirror_exit(BlockJob *job, void *opaque)
         if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) {
             bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL);
         }
-        bdrv_swap(s->target, to_replace);
-        if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) {
-            /* drop the bs loop chain formed by the swap: break the loop then
-             * trigger the unref from the top one */
-            BlockDriverState *p = s->base->backing_hd;
-            bdrv_set_backing_hd(s->base, NULL);
-            bdrv_unref(p);
-        }
+        bdrv_replace_in_backing_chain(to_replace, s->target);
     }
     if (s->to_replace) {
         bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
@@ -381,9 +384,12 @@ static void mirror_exit(BlockJob *job, void *opaque)
         aio_context_release(replace_aio_context);
     }
     g_free(s->replaces);
+    bdrv_op_unblock_all(s->target, s->common.blocker);
     bdrv_unref(s->target);
     block_job_completed(&s->common, data->ret);
     g_free(data);
+    bdrv_drained_end(src);
+    bdrv_unref(src);
 }
 
 static void coroutine_fn mirror_run(void *opaque)
@@ -427,7 +433,7 @@ static void coroutine_fn mirror_run(void *opaque)
      */
     bdrv_get_backing_filename(s->target, backing_filename,
                               sizeof(backing_filename));
-    if (backing_filename[0] && !s->target->backing_hd) {
+    if (backing_filename[0] && !s->target->backing) {
         ret = bdrv_get_info(s->target, &bdi);
         if (ret < 0) {
             goto immediate_exit;
@@ -451,6 +457,8 @@ static void coroutine_fn mirror_run(void *opaque)
     if (!s->is_none_mode) {
         /* First part, loop on the sectors and initialize the dirty bitmap.  */
         BlockDriverState *base = s->base;
+        bool mark_all_dirty = s->base == NULL && !bdrv_has_zero_init(s->target);
+
         for (sector_num = 0; sector_num < end; ) {
             /* Just to make sure we are not exceeding int limit. */
             int nb_sectors = MIN(INT_MAX >> BDRV_SECTOR_BITS,
@@ -473,7 +481,7 @@ static void coroutine_fn mirror_run(void *opaque)
             }
 
             assert(n > 0);
-            if (ret == 1) {
+            if (ret == 1 || mark_all_dirty) {
                 bdrv_set_dirty_bitmap(s->dirty_bitmap, sector_num, n);
             }
             sector_num += n;
@@ -594,10 +602,15 @@ immediate_exit:
     g_free(s->cow_bitmap);
     g_free(s->in_flight_bitmap);
     bdrv_release_dirty_bitmap(bs, s->dirty_bitmap);
-    bdrv_iostatus_disable(s->target);
+    if (s->target->blk) {
+        blk_iostatus_disable(s->target->blk);
+    }
 
     data = g_malloc(sizeof(*data));
     data->ret = ret;
+    /* Before we switch to target in mirror_exit, make sure data doesn't
+     * change. */
+    bdrv_drained_begin(s->common.bs);
     block_job_defer_to_main_loop(&s->common, mirror_exit, data);
 }
 
@@ -616,7 +629,9 @@ static void mirror_iostatus_reset(BlockJob *job)
 {
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
 
-    bdrv_iostatus_reset(s->target);
+    if (s->target->blk) {
+        blk_iostatus_reset(s->target->blk);
+    }
 }
 
 static void mirror_complete(BlockJob *job, Error **errp)
@@ -631,8 +646,7 @@ static void mirror_complete(BlockJob *job, Error **errp)
         return;
     }
     if (!s->synced) {
-        error_setg(errp, QERR_BLOCK_JOB_NOT_READY,
-                   bdrv_get_device_name(job->bs));
+        error_setg(errp, QERR_BLOCK_JOB_NOT_READY, job->id);
         return;
     }
 
@@ -640,9 +654,9 @@ static void mirror_complete(BlockJob *job, Error **errp)
     if (s->replaces) {
         AioContext *replace_aio_context;
 
-        s->to_replace = check_to_replace_node(s->replaces, &local_err);
+        s->to_replace = bdrv_find_node(s->replaces);
         if (!s->to_replace) {
-            error_propagate(errp, local_err);
+            error_setg(errp, "Node name '%s' not found", s->replaces);
             return;
         }
 
@@ -700,7 +714,7 @@ static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
 
     if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
          on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, QERR_INVALID_PARAMETER, "on-source-error");
         return;
     }
@@ -732,12 +746,17 @@ static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
     s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
     if (!s->dirty_bitmap) {
         g_free(s->replaces);
-        block_job_release(bs);
+        block_job_unref(&s->common);
         return;
     }
+
+    bdrv_op_block_all(s->target, s->common.blocker);
+
     bdrv_set_enable_write_cache(s->target, true);
-    bdrv_set_on_error(s->target, on_target_error, on_target_error);
-    bdrv_iostatus_enable(s->target);
+    if (s->target->blk) {
+        blk_set_on_error(s->target->blk, on_target_error, on_target_error);
+        blk_iostatus_enable(s->target->blk);
+    }
     s->common.co = qemu_coroutine_create(mirror_run);
     trace_mirror_start(bs, s, s->common.co, opaque);
     qemu_coroutine_enter(s->common.co, s);
@@ -760,7 +779,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
         return;
     }
     is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
-    base = mode == MIRROR_SYNC_MODE_TOP ? bs->backing_hd : NULL;
+    base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
     mirror_start_job(bs, target, replaces,
                      speed, granularity, buf_size,
                      on_source_error, on_target_error, unmap, cb, opaque, errp,
diff --git a/block/nbd-client.c b/block/nbd-client.c
index e1bb9198c..b7fd17a11 100644
--- a/block/nbd-client.c
+++ b/block/nbd-client.c
@@ -124,7 +124,7 @@ static int nbd_co_send_request(BlockDriverState *bs,
     s->send_coroutine = qemu_coroutine_self();
     aio_context = bdrv_get_aio_context(bs);
 
-    aio_set_fd_handler(aio_context, s->sock,
+    aio_set_fd_handler(aio_context, s->sock, false,
                        nbd_reply_ready, nbd_restart_write, bs);
     if (qiov) {
         if (!s->is_unix) {
@@ -144,7 +144,8 @@ static int nbd_co_send_request(BlockDriverState *bs,
     } else {
         rc = nbd_send_request(s->sock, request);
     }
-    aio_set_fd_handler(aio_context, s->sock, nbd_reply_ready, NULL, bs);
+    aio_set_fd_handler(aio_context, s->sock, false,
+                       nbd_reply_ready, NULL, bs);
     s->send_coroutine = NULL;
     qemu_co_mutex_unlock(&s->send_mutex);
     return rc;
@@ -348,14 +349,15 @@ int nbd_client_co_discard(BlockDriverState *bs, int64_t sector_num,
 void nbd_client_detach_aio_context(BlockDriverState *bs)
 {
     aio_set_fd_handler(bdrv_get_aio_context(bs),
-                       nbd_get_client_session(bs)->sock, NULL, NULL, NULL);
+                       nbd_get_client_session(bs)->sock,
+                       false, NULL, NULL, NULL);
 }
 
 void nbd_client_attach_aio_context(BlockDriverState *bs,
                                    AioContext *new_context)
 {
     aio_set_fd_handler(new_context, nbd_get_client_session(bs)->sock,
-                       nbd_reply_ready, NULL, bs);
+                       false, nbd_reply_ready, NULL, bs);
 }
 
 void nbd_client_close(BlockDriverState *bs)
diff --git a/block/nbd.c b/block/nbd.c
index 217618612..cd6a58777 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -43,7 +43,6 @@
 
 typedef struct BDRVNBDState {
     NbdClientSession client;
-    QemuOpts *socket_opts;
 } BDRVNBDState;
 
 static int nbd_parse_uri(const char *filename, QDict *options)
@@ -190,10 +189,10 @@ out:
     g_free(file);
 }
 
-static void nbd_config(BDRVNBDState *s, QDict *options, char **export,
-                       Error **errp)
+static SocketAddress *nbd_config(BDRVNBDState *s, QDict *options, char **export,
+                                 Error **errp)
 {
-    Error *local_err = NULL;
+    SocketAddress *saddr;
 
     if (qdict_haskey(options, "path") == qdict_haskey(options, "host")) {
         if (qdict_haskey(options, "path")) {
@@ -201,28 +200,37 @@ static void nbd_config(BDRVNBDState *s, QDict *options, char **export,
         } else {
             error_setg(errp, "one of path and host must be specified.");
         }
-        return;
+        return NULL;
     }
 
-    s->client.is_unix = qdict_haskey(options, "path");
-    s->socket_opts = qemu_opts_create(&socket_optslist, NULL, 0,
-                                      &error_abort);
+    saddr = g_new0(SocketAddress, 1);
 
-    qemu_opts_absorb_qdict(s->socket_opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
+    if (qdict_haskey(options, "path")) {
+        saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+        saddr->u.q_unix = g_new0(UnixSocketAddress, 1);
+        saddr->u.q_unix->path = g_strdup(qdict_get_str(options, "path"));
+        qdict_del(options, "path");
+    } else {
+        saddr->type = SOCKET_ADDRESS_KIND_INET;
+        saddr->u.inet = g_new0(InetSocketAddress, 1);
+        saddr->u.inet->host = g_strdup(qdict_get_str(options, "host"));
+        if (!qdict_get_try_str(options, "port")) {
+            saddr->u.inet->port = g_strdup_printf("%d", NBD_DEFAULT_PORT);
+        } else {
+            saddr->u.inet->port = g_strdup(qdict_get_str(options, "port"));
+        }
+        qdict_del(options, "host");
+        qdict_del(options, "port");
     }
 
-    if (!qemu_opt_get(s->socket_opts, "port")) {
-        qemu_opt_set_number(s->socket_opts, "port", NBD_DEFAULT_PORT,
-                            &error_abort);
-    }
+    s->client.is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
 
     *export = g_strdup(qdict_get_try_str(options, "export"));
     if (*export) {
         qdict_del(options, "export");
     }
+
+    return saddr;
 }
 
 NbdClientSession *nbd_get_client_session(BlockDriverState *bs)
@@ -231,26 +239,24 @@ NbdClientSession *nbd_get_client_session(BlockDriverState *bs)
     return &s->client;
 }
 
-static int nbd_establish_connection(BlockDriverState *bs, Error **errp)
+static int nbd_establish_connection(BlockDriverState *bs,
+                                    SocketAddress *saddr,
+                                    Error **errp)
 {
     BDRVNBDState *s = bs->opaque;
     int sock;
 
-    if (s->client.is_unix) {
-        sock = unix_connect_opts(s->socket_opts, errp, NULL, NULL);
-    } else {
-        sock = inet_connect_opts(s->socket_opts, errp, NULL, NULL);
-        if (sock >= 0) {
-            socket_set_nodelay(sock);
-        }
-    }
+    sock = socket_connect(saddr, errp, NULL, NULL);
 
-    /* Failed to establish connection */
     if (sock < 0) {
         logout("Failed to establish connection to NBD server\n");
         return -EIO;
     }
 
+    if (!s->client.is_unix) {
+        socket_set_nodelay(sock);
+    }
+
     return sock;
 }
 
@@ -260,19 +266,19 @@ static int nbd_open(BlockDriverState *bs, QDict *options, int flags,
     BDRVNBDState *s = bs->opaque;
     char *export = NULL;
     int result, sock;
-    Error *local_err = NULL;
+    SocketAddress *saddr;
 
     /* Pop the config into our state object. Exit if invalid. */
-    nbd_config(s, options, &export, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
+    saddr = nbd_config(s, options, &export, errp);
+    if (!saddr) {
         return -EINVAL;
     }
 
     /* establish TCP connection, return error if it fails
      * TODO: Configurable retry-until-timeout behaviour.
      */
-    sock = nbd_establish_connection(bs, errp);
+    sock = nbd_establish_connection(bs, saddr, errp);
+    qapi_free_SocketAddress(saddr);
     if (sock < 0) {
         g_free(export);
         return sock;
@@ -315,9 +321,6 @@ static int nbd_co_discard(BlockDriverState *bs, int64_t sector_num,
 
 static void nbd_close(BlockDriverState *bs)
 {
-    BDRVNBDState *s = bs->opaque;
-
-    qemu_opts_del(s->socket_opts);
     nbd_client_close(bs);
 }
 
diff --git a/block/nfs.c b/block/nfs.c
index 02eb4e464..fd79f8994 100644
--- a/block/nfs.c
+++ b/block/nfs.c
@@ -43,6 +43,7 @@ typedef struct NFSClient {
     int events;
     bool has_zero_init;
     AioContext *aio_context;
+    blkcnt_t st_blocks;
 } NFSClient;
 
 typedef struct NFSRPC {
@@ -62,11 +63,10 @@ static void nfs_set_events(NFSClient *client)
 {
     int ev = nfs_which_events(client->context);
     if (ev != client->events) {
-        aio_set_fd_handler(client->aio_context,
-                           nfs_get_fd(client->context),
+        aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                           false,
                            (ev & POLLIN) ? nfs_process_read : NULL,
-                           (ev & POLLOUT) ? nfs_process_write : NULL,
-                           client);
+                           (ev & POLLOUT) ? nfs_process_write : NULL, client);
 
     }
     client->events = ev;
@@ -241,9 +241,8 @@ static void nfs_detach_aio_context(BlockDriverState *bs)
 {
     NFSClient *client = bs->opaque;
 
-    aio_set_fd_handler(client->aio_context,
-                       nfs_get_fd(client->context),
-                       NULL, NULL, NULL);
+    aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                       false, NULL, NULL, NULL);
     client->events = 0;
 }
 
@@ -262,9 +261,8 @@ static void nfs_client_close(NFSClient *client)
         if (client->fh) {
             nfs_close(client->context, client->fh);
         }
-        aio_set_fd_handler(client->aio_context,
-                           nfs_get_fd(client->context),
-                           NULL, NULL, NULL);
+        aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                           false, NULL, NULL, NULL);
         nfs_destroy_context(client->context);
     }
     memset(client, 0, sizeof(NFSClient));
@@ -374,6 +372,7 @@ static int64_t nfs_client_open(NFSClient *client, const char *filename,
     }
 
     ret = DIV_ROUND_UP(st.st_size, BDRV_SECTOR_SIZE);
+    client->st_blocks = st.st_blocks;
     client->has_zero_init = S_ISREG(st.st_mode);
     goto out;
 fail:
@@ -464,6 +463,11 @@ static int64_t nfs_get_allocated_file_size(BlockDriverState *bs)
     NFSRPC task = {0};
     struct stat st;
 
+    if (bdrv_is_read_only(bs) &&
+        !(bs->open_flags & BDRV_O_NOCACHE)) {
+        return client->st_blocks * 512;
+    }
+
     task.st = &st;
     if (nfs_fstat_async(client->context, client->fh, nfs_co_generic_cb,
                         &task) != 0) {
@@ -484,6 +488,34 @@ static int nfs_file_truncate(BlockDriverState *bs, int64_t offset)
     return nfs_ftruncate(client->context, client->fh, offset);
 }
 
+/* Note that this will not re-establish a connection with the NFS server
+ * - it is effectively a NOP.  */
+static int nfs_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    NFSClient *client = state->bs->opaque;
+    struct stat st;
+    int ret = 0;
+
+    if (state->flags & BDRV_O_RDWR && bdrv_is_read_only(state->bs)) {
+        error_setg(errp, "Cannot open a read-only mount as read-write");
+        return -EACCES;
+    }
+
+    /* Update cache for read-only reopens */
+    if (!(state->flags & BDRV_O_RDWR)) {
+        ret = nfs_fstat(client->context, client->fh, &st);
+        if (ret < 0) {
+            error_setg(errp, "Failed to fstat file: %s",
+                       nfs_get_error(client->context));
+            return ret;
+        }
+        client->st_blocks = st.st_blocks;
+    }
+
+    return 0;
+}
+
 static BlockDriver bdrv_nfs = {
     .format_name                    = "nfs",
     .protocol_name                  = "nfs",
@@ -499,6 +531,7 @@ static BlockDriver bdrv_nfs = {
     .bdrv_file_open                 = nfs_file_open,
     .bdrv_close                     = nfs_file_close,
     .bdrv_create                    = nfs_file_create,
+    .bdrv_reopen_prepare            = nfs_reopen_prepare,
 
     .bdrv_co_readv                  = nfs_co_readv,
     .bdrv_co_writev                 = nfs_co_writev,
diff --git a/block/parallels.c b/block/parallels.c
index 046b56844..f689fdeaf 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -202,13 +202,13 @@ static int64_t allocate_clusters(BlockDriverState *bs, int64_t sector_num,
 
     to_allocate = (sector_num + *pnum + s->tracks - 1) / s->tracks - idx;
     space = to_allocate * s->tracks;
-    if (s->data_end + space > bdrv_getlength(bs->file) >> BDRV_SECTOR_BITS) {
+    if (s->data_end + space > bdrv_getlength(bs->file->bs) >> BDRV_SECTOR_BITS) {
         int ret;
         space += s->prealloc_size;
         if (s->prealloc_mode == PRL_PREALLOC_MODE_FALLOCATE) {
-            ret = bdrv_write_zeroes(bs->file, s->data_end, space, 0);
+            ret = bdrv_write_zeroes(bs->file->bs, s->data_end, space, 0);
         } else {
-            ret = bdrv_truncate(bs->file,
+            ret = bdrv_truncate(bs->file->bs,
                                 (s->data_end + space) << BDRV_SECTOR_BITS);
         }
         if (ret < 0) {
@@ -220,7 +220,7 @@ static int64_t allocate_clusters(BlockDriverState *bs, int64_t sector_num,
         s->bat_bitmap[idx + i] = cpu_to_le32(s->data_end / s->off_multiplier);
         s->data_end += s->tracks;
         bitmap_set(s->bat_dirty_bmap,
-                   bat_entry_off(idx) / s->bat_dirty_block, 1);
+                   bat_entry_off(idx + i) / s->bat_dirty_block, 1);
     }
 
     return bat2sect(s, idx) + sector_num % s->tracks;
@@ -244,7 +244,8 @@ static coroutine_fn int parallels_co_flush_to_os(BlockDriverState *bs)
         if (off + to_write > s->header_size) {
             to_write = s->header_size - off;
         }
-        ret = bdrv_pwrite(bs->file, off, (uint8_t *)s->header + off, to_write);
+        ret = bdrv_pwrite(bs->file->bs, off, (uint8_t *)s->header + off,
+                          to_write);
         if (ret < 0) {
             qemu_co_mutex_unlock(&s->lock);
             return ret;
@@ -303,7 +304,7 @@ static coroutine_fn int parallels_co_writev(BlockDriverState *bs,
         qemu_iovec_reset(&hd_qiov);
         qemu_iovec_concat(&hd_qiov, qiov, bytes_done, nbytes);
 
-        ret = bdrv_co_writev(bs->file, position, n, &hd_qiov);
+        ret = bdrv_co_writev(bs->file->bs, position, n, &hd_qiov);
         if (ret < 0) {
             break;
         }
@@ -343,7 +344,7 @@ static coroutine_fn int parallels_co_readv(BlockDriverState *bs,
             qemu_iovec_reset(&hd_qiov);
             qemu_iovec_concat(&hd_qiov, qiov, bytes_done, nbytes);
 
-            ret = bdrv_co_readv(bs->file, position, n, &hd_qiov);
+            ret = bdrv_co_readv(bs->file->bs, position, n, &hd_qiov);
             if (ret < 0) {
                 break;
             }
@@ -369,7 +370,7 @@ static int parallels_check(BlockDriverState *bs, BdrvCheckResult *res,
     bool flush_bat = false;
     int cluster_size = s->tracks << BDRV_SECTOR_BITS;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     if (size < 0) {
         res->check_errors++;
         return size;
@@ -424,7 +425,7 @@ static int parallels_check(BlockDriverState *bs, BdrvCheckResult *res,
     }
 
     if (flush_bat) {
-        ret = bdrv_pwrite_sync(bs->file, 0, s->header, s->header_size);
+        ret = bdrv_pwrite_sync(bs->file->bs, 0, s->header, s->header_size);
         if (ret < 0) {
             res->check_errors++;
             return ret;
@@ -440,7 +441,7 @@ static int parallels_check(BlockDriverState *bs, BdrvCheckResult *res,
                 size - res->image_end_offset);
         res->leaks += count;
         if (fix & BDRV_FIX_LEAKS) {
-            ret = bdrv_truncate(bs->file, res->image_end_offset);
+            ret = bdrv_truncate(bs->file->bs, res->image_end_offset);
             if (ret < 0) {
                 res->check_errors++;
                 return ret;
@@ -476,7 +477,7 @@ static int parallels_create(const char *filename, QemuOpts *opts, Error **errp)
 
     file = NULL;
     ret = bdrv_open(&file, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
+                    BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         return ret;
@@ -546,12 +547,13 @@ static int parallels_probe(const uint8_t *buf, int buf_size,
 static int parallels_update_header(BlockDriverState *bs)
 {
     BDRVParallelsState *s = bs->opaque;
-    unsigned size = MAX(bdrv_opt_mem_align(bs->file), sizeof(ParallelsHeader));
+    unsigned size = MAX(bdrv_opt_mem_align(bs->file->bs),
+                        sizeof(ParallelsHeader));
 
     if (size > s->header_size) {
         size = s->header_size;
     }
-    return bdrv_pwrite_sync(bs->file, 0, s->header, size);
+    return bdrv_pwrite_sync(bs->file->bs, 0, s->header, size);
 }
 
 static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
@@ -564,7 +566,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
     Error *local_err = NULL;
     char *buf;
 
-    ret = bdrv_pread(bs->file, 0, &ph, sizeof(ph));
+    ret = bdrv_pread(bs->file->bs, 0, &ph, sizeof(ph));
     if (ret < 0) {
         goto fail;
     }
@@ -603,8 +605,8 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     size = bat_entry_off(s->bat_size);
-    s->header_size = ROUND_UP(size, bdrv_opt_mem_align(bs->file));
-    s->header = qemu_try_blockalign(bs->file, s->header_size);
+    s->header_size = ROUND_UP(size, bdrv_opt_mem_align(bs->file->bs));
+    s->header = qemu_try_blockalign(bs->file->bs, s->header_size);
     if (s->header == NULL) {
         ret = -ENOMEM;
         goto fail;
@@ -619,7 +621,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
         s->header_size = size;
     }
 
-    ret = bdrv_pread(bs->file, 0, s->header, s->header_size);
+    ret = bdrv_pread(bs->file->bs, 0, s->header, s->header_size);
     if (ret < 0) {
         goto fail;
     }
@@ -663,8 +665,8 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
     if (local_err != NULL) {
         goto fail_options;
     }
-    if (!bdrv_has_zero_init(bs->file) ||
-            bdrv_truncate(bs->file, bdrv_getlength(bs->file)) != 0) {
+    if (!bdrv_has_zero_init(bs->file->bs) ||
+            bdrv_truncate(bs->file->bs, bdrv_getlength(bs->file->bs)) != 0) {
         s->prealloc_mode = PRL_PREALLOC_MODE_FALLOCATE;
     }
 
@@ -707,7 +709,7 @@ static void parallels_close(BlockDriverState *bs)
     }
 
     if (bs->open_flags & BDRV_O_RDWR) {
-        bdrv_truncate(bs->file, s->data_end << BDRV_SECTOR_BITS);
+        bdrv_truncate(bs->file->bs, s->data_end << BDRV_SECTOR_BITS);
     }
 
     g_free(s->bat_dirty_bmap);
diff --git a/block/qapi.c b/block/qapi.c
index 2ce509711..267f147fe 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -64,7 +64,7 @@ BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp)
     info->backing_file_depth = bdrv_get_backing_file_depth(bs);
     info->detect_zeroes = bs->detect_zeroes;
 
-    if (bs->io_limits_enabled) {
+    if (bs->throttle_state) {
         ThrottleConfig cfg;
 
         throttle_group_get_config(bs, &cfg);
@@ -110,8 +110,8 @@ BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp)
             qapi_free_BlockDeviceInfo(info);
             return NULL;
         }
-        if (bs0->drv && bs0->backing_hd) {
-            bs0 = bs0->backing_hd;
+        if (bs0->drv && bs0->backing) {
+            bs0 = bs0->backing->bs;
             (*p_image_info)->has_backing_image = true;
             p_image_info = &((*p_image_info)->backing_image);
         } else {
@@ -301,17 +301,17 @@ static void bdrv_query_info(BlockBackend *blk, BlockInfo **p_info,
         info->tray_open = blk_dev_is_tray_open(blk);
     }
 
-    if (bdrv_iostatus_is_enabled(bs)) {
+    if (blk_iostatus_is_enabled(blk)) {
         info->has_io_status = true;
-        info->io_status = bs->iostatus;
+        info->io_status = blk_iostatus(blk);
     }
 
-    if (!QLIST_EMPTY(&bs->dirty_bitmaps)) {
+    if (bs && !QLIST_EMPTY(&bs->dirty_bitmaps)) {
         info->has_dirty_bitmaps = true;
         info->dirty_bitmaps = bdrv_query_dirty_bitmaps(bs);
     }
 
-    if (bs->drv) {
+    if (bs && bs->drv) {
         info->has_inserted = true;
         info->inserted = bdrv_block_device_info(bs, errp);
         if (info->inserted == NULL) {
@@ -344,27 +344,82 @@ static BlockStats *bdrv_query_stats(const BlockDriverState *bs,
     }
 
     s->stats = g_malloc0(sizeof(*s->stats));
-    s->stats->rd_bytes = bs->stats.nr_bytes[BLOCK_ACCT_READ];
-    s->stats->wr_bytes = bs->stats.nr_bytes[BLOCK_ACCT_WRITE];
-    s->stats->rd_operations = bs->stats.nr_ops[BLOCK_ACCT_READ];
-    s->stats->wr_operations = bs->stats.nr_ops[BLOCK_ACCT_WRITE];
-    s->stats->rd_merged = bs->stats.merged[BLOCK_ACCT_READ];
-    s->stats->wr_merged = bs->stats.merged[BLOCK_ACCT_WRITE];
-    s->stats->wr_highest_offset =
-        bs->stats.wr_highest_sector * BDRV_SECTOR_SIZE;
-    s->stats->flush_operations = bs->stats.nr_ops[BLOCK_ACCT_FLUSH];
-    s->stats->wr_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_WRITE];
-    s->stats->rd_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_READ];
-    s->stats->flush_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_FLUSH];
+    if (bs->blk) {
+        BlockAcctStats *stats = blk_get_stats(bs->blk);
+        BlockAcctTimedStats *ts = NULL;
+
+        s->stats->rd_bytes = stats->nr_bytes[BLOCK_ACCT_READ];
+        s->stats->wr_bytes = stats->nr_bytes[BLOCK_ACCT_WRITE];
+        s->stats->rd_operations = stats->nr_ops[BLOCK_ACCT_READ];
+        s->stats->wr_operations = stats->nr_ops[BLOCK_ACCT_WRITE];
+
+        s->stats->failed_rd_operations = stats->failed_ops[BLOCK_ACCT_READ];
+        s->stats->failed_wr_operations = stats->failed_ops[BLOCK_ACCT_WRITE];
+        s->stats->failed_flush_operations = stats->failed_ops[BLOCK_ACCT_FLUSH];
+
+        s->stats->invalid_rd_operations = stats->invalid_ops[BLOCK_ACCT_READ];
+        s->stats->invalid_wr_operations = stats->invalid_ops[BLOCK_ACCT_WRITE];
+        s->stats->invalid_flush_operations =
+            stats->invalid_ops[BLOCK_ACCT_FLUSH];
+
+        s->stats->rd_merged = stats->merged[BLOCK_ACCT_READ];
+        s->stats->wr_merged = stats->merged[BLOCK_ACCT_WRITE];
+        s->stats->flush_operations = stats->nr_ops[BLOCK_ACCT_FLUSH];
+        s->stats->wr_total_time_ns = stats->total_time_ns[BLOCK_ACCT_WRITE];
+        s->stats->rd_total_time_ns = stats->total_time_ns[BLOCK_ACCT_READ];
+        s->stats->flush_total_time_ns = stats->total_time_ns[BLOCK_ACCT_FLUSH];
+
+        s->stats->has_idle_time_ns = stats->last_access_time_ns > 0;
+        if (s->stats->has_idle_time_ns) {
+            s->stats->idle_time_ns = block_acct_idle_time_ns(stats);
+        }
+
+        s->stats->account_invalid = stats->account_invalid;
+        s->stats->account_failed = stats->account_failed;
+
+        while ((ts = block_acct_interval_next(stats, ts))) {
+            BlockDeviceTimedStatsList *timed_stats =
+                g_malloc0(sizeof(*timed_stats));
+            BlockDeviceTimedStats *dev_stats = g_malloc0(sizeof(*dev_stats));
+            timed_stats->next = s->stats->timed_stats;
+            timed_stats->value = dev_stats;
+            s->stats->timed_stats = timed_stats;
+
+            TimedAverage *rd = &ts->latency[BLOCK_ACCT_READ];
+            TimedAverage *wr = &ts->latency[BLOCK_ACCT_WRITE];
+            TimedAverage *fl = &ts->latency[BLOCK_ACCT_FLUSH];
+
+            dev_stats->interval_length = ts->interval_length;
+
+            dev_stats->min_rd_latency_ns = timed_average_min(rd);
+            dev_stats->max_rd_latency_ns = timed_average_max(rd);
+            dev_stats->avg_rd_latency_ns = timed_average_avg(rd);
+
+            dev_stats->min_wr_latency_ns = timed_average_min(wr);
+            dev_stats->max_wr_latency_ns = timed_average_max(wr);
+            dev_stats->avg_wr_latency_ns = timed_average_avg(wr);
+
+            dev_stats->min_flush_latency_ns = timed_average_min(fl);
+            dev_stats->max_flush_latency_ns = timed_average_max(fl);
+            dev_stats->avg_flush_latency_ns = timed_average_avg(fl);
+
+            dev_stats->avg_rd_queue_depth =
+                block_acct_queue_depth(ts, BLOCK_ACCT_READ);
+            dev_stats->avg_wr_queue_depth =
+                block_acct_queue_depth(ts, BLOCK_ACCT_WRITE);
+        }
+    }
+
+    s->stats->wr_highest_offset = bs->wr_highest_offset;
 
     if (bs->file) {
         s->has_parent = true;
-        s->parent = bdrv_query_stats(bs->file, query_backing);
+        s->parent = bdrv_query_stats(bs->file->bs, query_backing);
     }
 
-    if (query_backing && bs->backing_hd) {
+    if (query_backing && bs->backing) {
         s->has_backing = true;
-        s->backing = bdrv_query_stats(bs->backing_hd, query_backing);
+        s->backing = bdrv_query_stats(bs->backing->bs, query_backing);
     }
 
     return s;
@@ -381,7 +436,9 @@ BlockInfoList *qmp_query_block(Error **errp)
         bdrv_query_info(blk, &info->value, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
-            goto err;
+            g_free(info);
+            qapi_free_BlockInfoList(head);
+            return NULL;
         }
 
         *p_next = info;
@@ -389,10 +446,6 @@ BlockInfoList *qmp_query_block(Error **errp)
     }
 
     return head;
-
- err:
-    qapi_free_BlockInfoList(head);
-    return NULL;
 }
 
 BlockStatsList *qmp_query_blockstats(bool has_query_nodes,
diff --git a/block/qcow.c b/block/qcow.c
index 01fba54ce..635085e27 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -100,7 +100,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
     int ret;
     QCowHeader header;
 
-    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
+    ret = bdrv_pread(bs->file->bs, 0, &header, sizeof(header));
     if (ret < 0) {
         goto fail;
     }
@@ -193,7 +193,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
         goto fail;
     }
 
-    ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
+    ret = bdrv_pread(bs->file->bs, s->l1_table_offset, s->l1_table,
                s->l1_size * sizeof(uint64_t));
     if (ret < 0) {
         goto fail;
@@ -205,7 +205,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
 
     /* alloc L2 cache (max. 64k * 16 * 8 = 8 MB) */
     s->l2_cache =
-        qemu_try_blockalign(bs->file,
+        qemu_try_blockalign(bs->file->bs,
                             s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
     if (s->l2_cache == NULL) {
         error_setg(errp, "Could not allocate L2 table cache");
@@ -224,7 +224,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
             ret = -EINVAL;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, header.backing_file_offset,
+        ret = bdrv_pread(bs->file->bs, header.backing_file_offset,
                    bs->backing_file, len);
         if (ret < 0) {
             goto fail;
@@ -369,13 +369,13 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
         if (!allocate)
             return 0;
         /* allocate a new l2 entry */
-        l2_offset = bdrv_getlength(bs->file);
+        l2_offset = bdrv_getlength(bs->file->bs);
         /* round to cluster size */
         l2_offset = (l2_offset + s->cluster_size - 1) & ~(s->cluster_size - 1);
         /* update the L1 entry */
         s->l1_table[l1_index] = l2_offset;
         tmp = cpu_to_be64(l2_offset);
-        if (bdrv_pwrite_sync(bs->file,
+        if (bdrv_pwrite_sync(bs->file->bs,
                 s->l1_table_offset + l1_index * sizeof(tmp),
                 &tmp, sizeof(tmp)) < 0)
             return 0;
@@ -405,11 +405,12 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
     l2_table = s->l2_cache + (min_index << s->l2_bits);
     if (new_l2_table) {
         memset(l2_table, 0, s->l2_size * sizeof(uint64_t));
-        if (bdrv_pwrite_sync(bs->file, l2_offset, l2_table,
+        if (bdrv_pwrite_sync(bs->file->bs, l2_offset, l2_table,
                 s->l2_size * sizeof(uint64_t)) < 0)
             return 0;
     } else {
-        if (bdrv_pread(bs->file, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) !=
+        if (bdrv_pread(bs->file->bs, l2_offset, l2_table,
+                       s->l2_size * sizeof(uint64_t)) !=
             s->l2_size * sizeof(uint64_t))
             return 0;
     }
@@ -430,20 +431,21 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
                overwritten */
             if (decompress_cluster(bs, cluster_offset) < 0)
                 return 0;
-            cluster_offset = bdrv_getlength(bs->file);
+            cluster_offset = bdrv_getlength(bs->file->bs);
             cluster_offset = (cluster_offset + s->cluster_size - 1) &
                 ~(s->cluster_size - 1);
             /* write the cluster content */
-            if (bdrv_pwrite(bs->file, cluster_offset, s->cluster_cache, s->cluster_size) !=
+            if (bdrv_pwrite(bs->file->bs, cluster_offset, s->cluster_cache,
+                            s->cluster_size) !=
                 s->cluster_size)
                 return -1;
         } else {
-            cluster_offset = bdrv_getlength(bs->file);
+            cluster_offset = bdrv_getlength(bs->file->bs);
             if (allocate == 1) {
                 /* round to cluster size */
                 cluster_offset = (cluster_offset + s->cluster_size - 1) &
                     ~(s->cluster_size - 1);
-                bdrv_truncate(bs->file, cluster_offset + s->cluster_size);
+                bdrv_truncate(bs->file->bs, cluster_offset + s->cluster_size);
                 /* if encrypted, we must initialize the cluster
                    content which won't be written */
                 if (bs->encrypted &&
@@ -463,7 +465,8 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
                                 errno = EIO;
                                 return -1;
                             }
-                            if (bdrv_pwrite(bs->file, cluster_offset + i * 512,
+                            if (bdrv_pwrite(bs->file->bs,
+                                            cluster_offset + i * 512,
                                             s->cluster_data, 512) != 512)
                                 return -1;
                         }
@@ -477,7 +480,7 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
         /* update L2 table */
         tmp = cpu_to_be64(cluster_offset);
         l2_table[l2_index] = tmp;
-        if (bdrv_pwrite_sync(bs->file, l2_offset + l2_index * sizeof(tmp),
+        if (bdrv_pwrite_sync(bs->file->bs, l2_offset + l2_index * sizeof(tmp),
                 &tmp, sizeof(tmp)) < 0)
             return 0;
     }
@@ -546,7 +549,7 @@ static int decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
     if (s->cluster_cache_offset != coffset) {
         csize = cluster_offset >> (63 - s->cluster_bits);
         csize &= (s->cluster_size - 1);
-        ret = bdrv_pread(bs->file, coffset, s->cluster_data, csize);
+        ret = bdrv_pread(bs->file->bs, coffset, s->cluster_data, csize);
         if (ret != csize)
             return -1;
         if (decompress_buffer(s->cluster_cache, s->cluster_size,
@@ -594,13 +597,13 @@ static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
         }
 
         if (!cluster_offset) {
-            if (bs->backing_hd) {
+            if (bs->backing) {
                 /* read from the base image */
                 hd_iov.iov_base = (void *)buf;
                 hd_iov.iov_len = n * 512;
                 qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
                 qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_readv(bs->backing_hd, sector_num,
+                ret = bdrv_co_readv(bs->backing->bs, sector_num,
                                     n, &hd_qiov);
                 qemu_co_mutex_lock(&s->lock);
                 if (ret < 0) {
@@ -625,7 +628,7 @@ static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
             hd_iov.iov_len = n * 512;
             qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
             qemu_co_mutex_unlock(&s->lock);
-            ret = bdrv_co_readv(bs->file,
+            ret = bdrv_co_readv(bs->file->bs,
                                 (cluster_offset >> 9) + index_in_cluster,
                                 n, &hd_qiov);
             qemu_co_mutex_lock(&s->lock);
@@ -727,7 +730,7 @@ static coroutine_fn int qcow_co_writev(BlockDriverState *bs, int64_t sector_num,
         hd_iov.iov_len = n * 512;
         qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
         qemu_co_mutex_unlock(&s->lock);
-        ret = bdrv_co_writev(bs->file,
+        ret = bdrv_co_writev(bs->file->bs,
                              (cluster_offset >> 9) + index_in_cluster,
                              n, &hd_qiov);
         qemu_co_mutex_lock(&s->lock);
@@ -793,7 +796,7 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
 
     qcow_bs = NULL;
     ret = bdrv_open(&qcow_bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
+                    BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto cleanup;
@@ -879,10 +882,10 @@ static int qcow_make_empty(BlockDriverState *bs)
     int ret;
 
     memset(s->l1_table, 0, l1_length);
-    if (bdrv_pwrite_sync(bs->file, s->l1_table_offset, s->l1_table,
+    if (bdrv_pwrite_sync(bs->file->bs, s->l1_table_offset, s->l1_table,
             l1_length) < 0)
         return -1;
-    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length);
+    ret = bdrv_truncate(bs->file->bs, s->l1_table_offset + l1_length);
     if (ret < 0)
         return ret;
 
@@ -962,7 +965,7 @@ static int qcow_write_compressed(BlockDriverState *bs, int64_t sector_num,
         }
 
         cluster_offset &= s->cluster_offset_mask;
-        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
+        ret = bdrv_pwrite(bs->file->bs, cluster_offset, out_buf, out_len);
         if (ret < 0) {
             goto fail;
         }
diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c
index 53b8afc3d..86dd7f2bd 100644
--- a/block/qcow2-cache.c
+++ b/block/qcow2-cache.c
@@ -22,16 +22,24 @@
  * THE SOFTWARE.
  */
 
+/* Needed for CONFIG_MADVISE */
+#include "config-host.h"
+
+#if defined(CONFIG_MADVISE) || defined(CONFIG_POSIX_MADVISE)
+#include <sys/mman.h>
+#endif
+
 #include "block/block_int.h"
 #include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "qcow2.h"
 #include "trace.h"
 
 typedef struct Qcow2CachedTable {
     int64_t  offset;
-    bool     dirty;
     uint64_t lru_counter;
     int      ref;
+    bool     dirty;
 } Qcow2CachedTable;
 
 struct Qcow2Cache {
@@ -41,34 +49,85 @@ struct Qcow2Cache {
     bool                    depends_on_flush;
     void                   *table_array;
     uint64_t                lru_counter;
+    uint64_t                cache_clean_lru_counter;
 };
 
 static inline void *qcow2_cache_get_table_addr(BlockDriverState *bs,
                     Qcow2Cache *c, int table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     return (uint8_t *) c->table_array + (size_t) table * s->cluster_size;
 }
 
 static inline int qcow2_cache_get_table_idx(BlockDriverState *bs,
                   Qcow2Cache *c, void *table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     ptrdiff_t table_offset = (uint8_t *) table - (uint8_t *) c->table_array;
     int idx = table_offset / s->cluster_size;
     assert(idx >= 0 && idx < c->size && table_offset % s->cluster_size == 0);
     return idx;
 }
 
+static void qcow2_cache_table_release(BlockDriverState *bs, Qcow2Cache *c,
+                                      int i, int num_tables)
+{
+#if QEMU_MADV_DONTNEED != QEMU_MADV_INVALID
+    BDRVQcow2State *s = bs->opaque;
+    void *t = qcow2_cache_get_table_addr(bs, c, i);
+    int align = getpagesize();
+    size_t mem_size = (size_t) s->cluster_size * num_tables;
+    size_t offset = QEMU_ALIGN_UP((uintptr_t) t, align) - (uintptr_t) t;
+    size_t length = QEMU_ALIGN_DOWN(mem_size - offset, align);
+    if (length > 0) {
+        qemu_madvise((uint8_t *) t + offset, length, QEMU_MADV_DONTNEED);
+    }
+#endif
+}
+
+static inline bool can_clean_entry(Qcow2Cache *c, int i)
+{
+    Qcow2CachedTable *t = &c->entries[i];
+    return t->ref == 0 && !t->dirty && t->offset != 0 &&
+        t->lru_counter <= c->cache_clean_lru_counter;
+}
+
+void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c)
+{
+    int i = 0;
+    while (i < c->size) {
+        int to_clean = 0;
+
+        /* Skip the entries that we don't need to clean */
+        while (i < c->size && !can_clean_entry(c, i)) {
+            i++;
+        }
+
+        /* And count how many we can clean in a row */
+        while (i < c->size && can_clean_entry(c, i)) {
+            c->entries[i].offset = 0;
+            c->entries[i].lru_counter = 0;
+            i++;
+            to_clean++;
+        }
+
+        if (to_clean > 0) {
+            qcow2_cache_table_release(bs, c, i - to_clean, to_clean);
+        }
+    }
+
+    c->cache_clean_lru_counter = c->lru_counter;
+}
+
 Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2Cache *c;
 
     c = g_new0(Qcow2Cache, 1);
     c->size = num_tables;
     c->entries = g_try_new0(Qcow2CachedTable, num_tables);
-    c->table_array = qemu_try_blockalign(bs->file,
+    c->table_array = qemu_try_blockalign(bs->file->bs,
                                          (size_t) num_tables * s->cluster_size);
 
     if (!c->entries || !c->table_array) {
@@ -113,7 +172,7 @@ static int qcow2_cache_flush_dependency(BlockDriverState *bs, Qcow2Cache *c)
 
 static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret = 0;
 
     if (!c->entries[i].dirty || !c->entries[i].offset) {
@@ -126,7 +185,7 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
     if (c->depends) {
         ret = qcow2_cache_flush_dependency(bs, c);
     } else if (c->depends_on_flush) {
-        ret = bdrv_flush(bs->file);
+        ret = bdrv_flush(bs->file->bs);
         if (ret >= 0) {
             c->depends_on_flush = false;
         }
@@ -157,7 +216,7 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
         BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE);
     }
 
-    ret = bdrv_pwrite(bs->file, c->entries[i].offset,
+    ret = bdrv_pwrite(bs->file->bs, c->entries[i].offset,
                       qcow2_cache_get_table_addr(bs, c, i), s->cluster_size);
     if (ret < 0) {
         return ret;
@@ -170,7 +229,7 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
 
 int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int result = 0;
     int ret;
     int i;
@@ -185,7 +244,7 @@ int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
     }
 
     if (result == 0) {
-        ret = bdrv_flush(bs->file);
+        ret = bdrv_flush(bs->file->bs);
         if (ret < 0) {
             result = ret;
         }
@@ -237,6 +296,8 @@ int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c)
         c->entries[i].lru_counter = 0;
     }
 
+    qcow2_cache_table_release(bs, c, 0, c->size);
+
     c->lru_counter = 0;
 
     return 0;
@@ -245,7 +306,7 @@ int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c)
 static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
     uint64_t offset, void **table, bool read_from_disk)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
     int ret;
     int lookup_index;
@@ -295,7 +356,8 @@ static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
             BLKDBG_EVENT(bs->file, BLKDBG_L2_LOAD);
         }
 
-        ret = bdrv_pread(bs->file, offset, qcow2_cache_get_table_addr(bs, c, i),
+        ret = bdrv_pread(bs->file->bs, offset,
+                         qcow2_cache_get_table_addr(bs, c, i),
                          s->cluster_size);
         if (ret < 0) {
             return ret;
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index 7e94fe70e..24a60e223 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -32,7 +32,7 @@
 int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
                         bool exact_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int new_l1_size2, ret, i;
     uint64_t *new_l1_table;
     int64_t old_l1_table_offset, old_l1_size;
@@ -72,7 +72,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
 #endif
 
     new_l1_size2 = sizeof(uint64_t) * new_l1_size;
-    new_l1_table = qemu_try_blockalign(bs->file,
+    new_l1_table = qemu_try_blockalign(bs->file->bs,
                                        align_offset(new_l1_size2, 512));
     if (new_l1_table == NULL) {
         return -ENOMEM;
@@ -105,7 +105,8 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
     BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_WRITE_TABLE);
     for(i = 0; i < s->l1_size; i++)
         new_l1_table[i] = cpu_to_be64(new_l1_table[i]);
-    ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset, new_l1_table, new_l1_size2);
+    ret = bdrv_pwrite_sync(bs->file->bs, new_l1_table_offset,
+                           new_l1_table, new_l1_size2);
     if (ret < 0)
         goto fail;
     for(i = 0; i < s->l1_size; i++)
@@ -115,7 +116,8 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
     BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ACTIVATE_TABLE);
     cpu_to_be32w((uint32_t*)data, new_l1_size);
     stq_be_p(data + 4, new_l1_table_offset);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size), data,sizeof(data));
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, l1_size),
+                           data, sizeof(data));
     if (ret < 0) {
         goto fail;
     }
@@ -148,7 +150,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
 static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
     uint64_t **l2_table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset, (void**) l2_table);
@@ -163,7 +165,7 @@ static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
 #define L1_ENTRIES_PER_SECTOR (512 / 8)
 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t buf[L1_ENTRIES_PER_SECTOR] = { 0 };
     int l1_start_index;
     int i, ret;
@@ -182,8 +184,9 @@ int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
-    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset + 8 * l1_start_index,
-        buf, sizeof(buf));
+    ret = bdrv_pwrite_sync(bs->file->bs,
+                           s->l1_table_offset + 8 * l1_start_index,
+                           buf, sizeof(buf));
     if (ret < 0) {
         return ret;
     }
@@ -203,7 +206,7 @@ int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
 
 static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t old_l2_offset;
     uint64_t *l2_table = NULL;
     int64_t l2_offset;
@@ -309,7 +312,7 @@ static int count_contiguous_clusters(int nb_clusters, int cluster_size,
     if (!offset)
         return 0;
 
-    assert(qcow2_get_cluster_type(first_entry) != QCOW2_CLUSTER_COMPRESSED);
+    assert(qcow2_get_cluster_type(first_entry) == QCOW2_CLUSTER_NORMAL);
 
     for (i = 0; i < nb_clusters; i++) {
         uint64_t l2_entry = be64_to_cpu(l2_table[i]) & mask;
@@ -321,14 +324,16 @@ static int count_contiguous_clusters(int nb_clusters, int cluster_size,
 	return i;
 }
 
-static int count_contiguous_free_clusters(int nb_clusters, uint64_t *l2_table)
+static int count_contiguous_clusters_by_type(int nb_clusters,
+                                             uint64_t *l2_table,
+                                             int wanted_type)
 {
     int i;
 
     for (i = 0; i < nb_clusters; i++) {
         int type = qcow2_get_cluster_type(be64_to_cpu(l2_table[i]));
 
-        if (type != QCOW2_CLUSTER_UNALLOCATED) {
+        if (type != wanted_type) {
             break;
         }
     }
@@ -339,7 +344,7 @@ static int count_contiguous_free_clusters(int nb_clusters, uint64_t *l2_table)
 /* The crypt function is compatible with the linux cryptoloop
    algorithm for < 4 GB images. NOTE: out_buf == in_buf is
    supported */
-int qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+int qcow2_encrypt_sectors(BDRVQcow2State *s, int64_t sector_num,
                           uint8_t *out_buf, const uint8_t *in_buf,
                           int nb_sectors, bool enc,
                           Error **errp)
@@ -387,7 +392,7 @@ static int coroutine_fn copy_sectors(BlockDriverState *bs,
                                      uint64_t cluster_offset,
                                      int n_start, int n_end)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QEMUIOVector qiov;
     struct iovec iov;
     int n, ret;
@@ -440,7 +445,8 @@ static int coroutine_fn copy_sectors(BlockDriverState *bs,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_COW_WRITE);
-    ret = bdrv_co_writev(bs->file, (cluster_offset >> 9) + n_start, n, &qiov);
+    ret = bdrv_co_writev(bs->file->bs, (cluster_offset >> 9) + n_start, n,
+                         &qiov);
     if (ret < 0) {
         goto out;
     }
@@ -469,7 +475,7 @@ out:
 int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
     int *num, uint64_t *cluster_offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int l2_index;
     uint64_t l1_index, l2_offset, *l2_table;
     int l1_bits, c;
@@ -499,7 +505,7 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
 
     *cluster_offset = 0;
 
-    /* seek the the l2 offset in the l1 table */
+    /* seek to the l2 offset in the l1 table */
 
     l1_index = offset >> l1_bits;
     if (l1_index >= s->l1_size) {
@@ -550,13 +556,14 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
             ret = -EIO;
             goto fail;
         }
-        c = count_contiguous_clusters(nb_clusters, s->cluster_size,
-                &l2_table[l2_index], QCOW_OFLAG_ZERO);
+        c = count_contiguous_clusters_by_type(nb_clusters, &l2_table[l2_index],
+                                              QCOW2_CLUSTER_ZERO);
         *cluster_offset = 0;
         break;
     case QCOW2_CLUSTER_UNALLOCATED:
         /* how many empty clusters ? */
-        c = count_contiguous_free_clusters(nb_clusters, &l2_table[l2_index]);
+        c = count_contiguous_clusters_by_type(nb_clusters, &l2_table[l2_index],
+                                              QCOW2_CLUSTER_UNALLOCATED);
         *cluster_offset = 0;
         break;
     case QCOW2_CLUSTER_NORMAL:
@@ -609,13 +616,13 @@ static int get_cluster_table(BlockDriverState *bs, uint64_t offset,
                              uint64_t **new_l2_table,
                              int *new_l2_index)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int l2_index;
     uint64_t l1_index, l2_offset;
     uint64_t *l2_table = NULL;
     int ret;
 
-    /* seek the the l2 offset in the l1 table */
+    /* seek to the l2 offset in the l1 table */
 
     l1_index = offset >> (s->l2_bits + s->cluster_bits);
     if (l1_index >= s->l1_size) {
@@ -683,7 +690,7 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
                                                uint64_t offset,
                                                int compressed_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index, ret;
     uint64_t *l2_table;
     int64_t cluster_offset;
@@ -728,7 +735,7 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
 
 static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     if (r->nb_sectors == 0) {
@@ -757,7 +764,7 @@ static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
 
 int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i, j = 0, l2_index, ret;
     uint64_t *old_cluster, *l2_table;
     uint64_t cluster_offset = m->alloc_offset;
@@ -817,7 +824,6 @@ int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
 
     /*
      * If this was a COW, we need to decrease the refcount of the old cluster.
-     * Also flush bs->file to get the right order for L2 and refcount update.
      *
      * Don't discard clusters that reach a refcount of 0 (e.g. compressed
      * clusters), the next write will reuse them anyway.
@@ -840,7 +846,7 @@ err:
  * write, but require COW to be performed (this includes yet unallocated space,
  * which must copy from the backing file)
  */
-static int count_cow_clusters(BDRVQcowState *s, int nb_clusters,
+static int count_cow_clusters(BDRVQcow2State *s, int nb_clusters,
     uint64_t *l2_table, int l2_index)
 {
     int i;
@@ -886,7 +892,7 @@ out:
 static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *cur_bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowL2Meta *old_alloc;
     uint64_t bytes = *cur_bytes;
 
@@ -959,7 +965,7 @@ static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
 static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index;
     uint64_t cluster_offset;
     uint64_t *l2_table;
@@ -1067,7 +1073,7 @@ out:
 static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
                                    uint64_t *host_offset, uint64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
                                          *host_offset, *nb_clusters);
@@ -1115,7 +1121,7 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
 static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index;
     uint64_t *l2_table;
     uint64_t entry;
@@ -1268,7 +1274,7 @@ fail:
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
     int *num, uint64_t *host_offset, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start, remaining;
     uint64_t cluster_offset;
     uint64_t cur_bytes;
@@ -1402,7 +1408,7 @@ static int decompress_buffer(uint8_t *out_buf, int out_buf_size,
 
 int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret, csize, nb_csectors, sector_offset;
     uint64_t coffset;
 
@@ -1412,7 +1418,8 @@ int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
         sector_offset = coffset & 511;
         csize = nb_csectors * 512 - sector_offset;
         BLKDBG_EVENT(bs->file, BLKDBG_READ_COMPRESSED);
-        ret = bdrv_read(bs->file, coffset >> 9, s->cluster_data, nb_csectors);
+        ret = bdrv_read(bs->file->bs, coffset >> 9, s->cluster_data,
+                        nb_csectors);
         if (ret < 0) {
             return ret;
         }
@@ -1434,7 +1441,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
                              uint64_t nb_clusters, enum qcow2_discard_type type,
                              bool full_discard)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table;
     int l2_index;
     int ret;
@@ -1469,7 +1476,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
          */
         switch (qcow2_get_cluster_type(old_l2_entry)) {
             case QCOW2_CLUSTER_UNALLOCATED:
-                if (full_discard || !bs->backing_hd) {
+                if (full_discard || !bs->backing) {
                     continue;
                 }
                 break;
@@ -1508,7 +1515,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
 int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
     int nb_sectors, enum qcow2_discard_type type, bool full_discard)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t end_offset;
     uint64_t nb_clusters;
     int ret;
@@ -1554,7 +1561,7 @@ fail:
 static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
                           uint64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table;
     int l2_index;
     int ret;
@@ -1591,7 +1598,7 @@ static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
 
 int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t nb_clusters;
     int ret;
 
@@ -1636,7 +1643,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                                       int64_t l1_entries,
                                       BlockDriverAmendStatusCB *status_cb)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bool is_active_l1 = (l1_table == s->l1_table);
     uint64_t *l2_table = NULL;
     int ret;
@@ -1645,7 +1652,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
     if (!is_active_l1) {
         /* inactive L2 tables require a buffer to be stored in when loading
          * them from disk */
-        l2_table = qemu_try_blockalign(bs->file, s->cluster_size);
+        l2_table = qemu_try_blockalign(bs->file->bs, s->cluster_size);
         if (l2_table == NULL) {
             return -ENOMEM;
         }
@@ -1679,8 +1686,8 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                     (void **)&l2_table);
         } else {
             /* load inactive L2 tables from disk */
-            ret = bdrv_read(bs->file, l2_offset / BDRV_SECTOR_SIZE,
-                    (void *)l2_table, s->cluster_sectors);
+            ret = bdrv_read(bs->file->bs, l2_offset / BDRV_SECTOR_SIZE,
+                            (void *)l2_table, s->cluster_sectors);
         }
         if (ret < 0) {
             goto fail;
@@ -1703,7 +1710,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
             }
 
             if (!preallocated) {
-                if (!bs->backing_hd) {
+                if (!bs->backing) {
                     /* not backed; therefore we can simply deallocate the
                      * cluster */
                     l2_table[j] = 0;
@@ -1754,7 +1761,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                 goto fail;
             }
 
-            ret = bdrv_write_zeroes(bs->file, offset / BDRV_SECTOR_SIZE,
+            ret = bdrv_write_zeroes(bs->file->bs, offset / BDRV_SECTOR_SIZE,
                                     s->cluster_sectors, 0);
             if (ret < 0) {
                 if (!preallocated) {
@@ -1787,8 +1794,8 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                     goto fail;
                 }
 
-                ret = bdrv_write(bs->file, l2_offset / BDRV_SECTOR_SIZE,
-                        (void *)l2_table, s->cluster_sectors);
+                ret = bdrv_write(bs->file->bs, l2_offset / BDRV_SECTOR_SIZE,
+                                 (void *)l2_table, s->cluster_sectors);
                 if (ret < 0) {
                     goto fail;
                 }
@@ -1823,7 +1830,7 @@ fail:
 int qcow2_expand_zero_clusters(BlockDriverState *bs,
                                BlockDriverAmendStatusCB *status_cb)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table = NULL;
     int64_t l1_entries = 0, visited_l1_entries = 0;
     int ret;
@@ -1861,8 +1868,9 @@ int qcow2_expand_zero_clusters(BlockDriverState *bs,
 
         l1_table = g_realloc(l1_table, l1_sectors * BDRV_SECTOR_SIZE);
 
-        ret = bdrv_read(bs->file, s->snapshots[i].l1_table_offset /
-                BDRV_SECTOR_SIZE, (void *)l1_table, l1_sectors);
+        ret = bdrv_read(bs->file->bs,
+                        s->snapshots[i].l1_table_offset / BDRV_SECTOR_SIZE,
+                        (void *)l1_table, l1_sectors);
         if (ret < 0) {
             goto fail;
         }
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index 0b6c302ee..820f412ab 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -82,7 +82,7 @@ static Qcow2SetRefcountFunc *const set_refcount_funcs[] = {
 
 int qcow2_refcount_init(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int refcount_table_size2, i;
     int ret;
 
@@ -101,7 +101,7 @@ int qcow2_refcount_init(BlockDriverState *bs)
             goto fail;
         }
         BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_LOAD);
-        ret = bdrv_pread(bs->file, s->refcount_table_offset,
+        ret = bdrv_pread(bs->file->bs, s->refcount_table_offset,
                          s->refcount_table, refcount_table_size2);
         if (ret < 0) {
             goto fail;
@@ -116,7 +116,7 @@ int qcow2_refcount_init(BlockDriverState *bs)
 
 void qcow2_refcount_close(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     g_free(s->refcount_table);
 }
 
@@ -214,7 +214,7 @@ static int load_refcount_block(BlockDriverState *bs,
                                int64_t refcount_block_offset,
                                void **refcount_block)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_LOAD);
@@ -231,7 +231,7 @@ static int load_refcount_block(BlockDriverState *bs,
 int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index,
                        uint64_t *refcount)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t refcount_table_index, block_index;
     int64_t refcount_block_offset;
     int ret;
@@ -274,7 +274,7 @@ int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index,
  * Rounds the refcount table size up to avoid growing the table for each single
  * refcount block that is allocated.
  */
-static unsigned int next_refcount_table_size(BDRVQcowState *s,
+static unsigned int next_refcount_table_size(BDRVQcow2State *s,
     unsigned int min_size)
 {
     unsigned int min_clusters = (min_size >> (s->cluster_bits - 3)) + 1;
@@ -290,7 +290,7 @@ static unsigned int next_refcount_table_size(BDRVQcowState *s,
 
 
 /* Checks if two offsets are described by the same refcount block */
-static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
+static int in_same_refcount_block(BDRVQcow2State *s, uint64_t offset_a,
     uint64_t offset_b)
 {
     uint64_t block_a = offset_a >> (s->cluster_bits + s->refcount_block_bits);
@@ -308,7 +308,7 @@ static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
 static int alloc_refcount_block(BlockDriverState *bs,
                                 int64_t cluster_index, void **refcount_block)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int refcount_table_index;
     int ret;
 
@@ -431,7 +431,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     if (refcount_table_index < s->refcount_table_size) {
         uint64_t data64 = cpu_to_be64(new_block);
         BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_HOOKUP);
-        ret = bdrv_pwrite_sync(bs->file,
+        ret = bdrv_pwrite_sync(bs->file->bs,
             s->refcount_table_offset + refcount_table_index * sizeof(uint64_t),
             &data64, sizeof(data64));
         if (ret < 0) {
@@ -535,7 +535,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
 
     /* Write refcount blocks to disk */
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS);
-    ret = bdrv_pwrite_sync(bs->file, meta_offset, new_blocks,
+    ret = bdrv_pwrite_sync(bs->file->bs, meta_offset, new_blocks,
         blocks_clusters * s->cluster_size);
     g_free(new_blocks);
     new_blocks = NULL;
@@ -549,7 +549,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE);
-    ret = bdrv_pwrite_sync(bs->file, table_offset, new_table,
+    ret = bdrv_pwrite_sync(bs->file->bs, table_offset, new_table,
         table_size * sizeof(uint64_t));
     if (ret < 0) {
         goto fail_table;
@@ -560,12 +560,16 @@ static int alloc_refcount_block(BlockDriverState *bs,
     }
 
     /* Hook up the new refcount table in the qcow2 header */
-    uint8_t data[12];
-    cpu_to_be64w((uint64_t*)data, table_offset);
-    cpu_to_be32w((uint32_t*)(data + 8), table_clusters);
+    struct QEMU_PACKED {
+        uint64_t d64;
+        uint32_t d32;
+    } data;
+    cpu_to_be64w(&data.d64, table_offset);
+    cpu_to_be32w(&data.d32, table_clusters);
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, refcount_table_offset),
-        data, sizeof(data));
+    ret = bdrv_pwrite_sync(bs->file->bs,
+                           offsetof(QCowHeader, refcount_table_offset),
+                           &data, sizeof(data));
     if (ret < 0) {
         goto fail_table;
     }
@@ -605,7 +609,7 @@ fail_block:
 
 void qcow2_process_discards(BlockDriverState *bs, int ret)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2DiscardRegion *d, *next;
 
     QTAILQ_FOREACH_SAFE(d, &s->discards, next, next) {
@@ -613,7 +617,7 @@ void qcow2_process_discards(BlockDriverState *bs, int ret)
 
         /* Discard is optional, ignore the return value */
         if (ret >= 0) {
-            bdrv_discard(bs->file,
+            bdrv_discard(bs->file->bs,
                          d->offset >> BDRV_SECTOR_BITS,
                          d->bytes >> BDRV_SECTOR_BITS);
         }
@@ -625,7 +629,7 @@ void qcow2_process_discards(BlockDriverState *bs, int ret)
 static void update_refcount_discard(BlockDriverState *bs,
                                     uint64_t offset, uint64_t length)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2DiscardRegion *d, *p, *next;
 
     QTAILQ_FOREACH(d, &s->discards, next) {
@@ -682,7 +686,7 @@ static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
                                                    bool decrease,
                                                    enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t start, last, cluster_offset;
     void *refcount_block = NULL;
     int64_t old_table_index = -1;
@@ -793,7 +797,7 @@ int qcow2_update_cluster_refcount(BlockDriverState *bs,
                                   uint64_t addend, bool decrease,
                                   enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     ret = update_refcount(bs, cluster_index << s->cluster_bits, 1, addend,
@@ -815,7 +819,7 @@ int qcow2_update_cluster_refcount(BlockDriverState *bs,
 /* return < 0 if error */
 static int64_t alloc_clusters_noref(BlockDriverState *bs, uint64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t i, nb_clusters, refcount;
     int ret;
 
@@ -878,7 +882,7 @@ int64_t qcow2_alloc_clusters(BlockDriverState *bs, uint64_t size)
 int64_t qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
                                 int64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t cluster_index, refcount;
     uint64_t i;
     int ret;
@@ -916,7 +920,7 @@ int64_t qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
    contiguous sectors. size must be <= cluster_size */
 int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t offset;
     size_t free_in_cluster;
     int ret;
@@ -949,11 +953,17 @@ int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
 
             if (!offset || ROUND_UP(offset, s->cluster_size) != new_cluster) {
                 offset = new_cluster;
+                free_in_cluster = s->cluster_size;
+            } else {
+                free_in_cluster += s->cluster_size;
             }
         }
 
         assert(offset);
         ret = update_refcount(bs, offset, size, 1, false, QCOW2_DISCARD_NEVER);
+        if (ret < 0) {
+            offset = 0;
+        }
     } while (ret == -EAGAIN);
     if (ret < 0) {
         return ret;
@@ -992,7 +1002,7 @@ void qcow2_free_clusters(BlockDriverState *bs,
 void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
                              int nb_clusters, enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     switch (qcow2_get_cluster_type(l2_entry)) {
     case QCOW2_CLUSTER_COMPRESSED:
@@ -1036,7 +1046,7 @@ void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
 int qcow2_update_snapshot_refcount(BlockDriverState *bs,
     int64_t l1_table_offset, int l1_size, int addend)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2, refcount;
     bool l1_allocated = false;
     int64_t old_offset, old_l2_offset;
@@ -1062,7 +1072,7 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
         }
         l1_allocated = true;
 
-        ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
+        ret = bdrv_pread(bs->file->bs, l1_table_offset, l1_table, l1_size2);
         if (ret < 0) {
             goto fail;
         }
@@ -1215,7 +1225,8 @@ fail:
             cpu_to_be64s(&l1_table[i]);
         }
 
-        ret = bdrv_pwrite_sync(bs->file, l1_table_offset, l1_table, l1_size2);
+        ret = bdrv_pwrite_sync(bs->file->bs, l1_table_offset,
+                               l1_table, l1_size2);
 
         for (i = 0; i < l1_size; i++) {
             be64_to_cpus(&l1_table[i]);
@@ -1233,7 +1244,7 @@ fail:
 /* refcount checking functions */
 
 
-static size_t refcount_array_byte_size(BDRVQcowState *s, uint64_t entries)
+static uint64_t refcount_array_byte_size(BDRVQcow2State *s, uint64_t entries)
 {
     /* This assertion holds because there is no way we can address more than
      * 2^(64 - 9) clusters at once (with cluster size 512 = 2^9, and because
@@ -1256,7 +1267,7 @@ static size_t refcount_array_byte_size(BDRVQcowState *s, uint64_t entries)
  * refcount array buffer will be aligned to a cluster boundary, and the newly
  * allocated area will be zeroed.
  */
-static int realloc_refcount_array(BDRVQcowState *s, void **array,
+static int realloc_refcount_array(BDRVQcow2State *s, void **array,
                                   int64_t *size, int64_t new_size)
 {
     int64_t old_byte_size, new_byte_size;
@@ -1298,7 +1309,7 @@ static int realloc_refcount_array(BDRVQcowState *s, void **array,
 /*
  * Increases the refcount for a range of clusters in a given refcount table.
  * This is used to construct a temporary refcount table out of L1 and L2 tables
- * which can be compared the the refcount table saved in the image.
+ * which can be compared to the refcount table saved in the image.
  *
  * Modifies the number of errors in res.
  */
@@ -1308,7 +1319,7 @@ static int inc_refcounts(BlockDriverState *bs,
                          int64_t *refcount_table_size,
                          int64_t offset, int64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start, last, cluster_offset, k, refcount;
     int ret;
 
@@ -1361,7 +1372,7 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
                               int64_t *refcount_table_size, int64_t l2_offset,
                               int flags)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table, l2_entry;
     uint64_t next_contiguous_offset = 0;
     int i, l2_size, nb_csectors, ret;
@@ -1370,7 +1381,7 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
     l2_size = s->l2_size * sizeof(uint64_t);
     l2_table = g_malloc(l2_size);
 
-    ret = bdrv_pread(bs->file, l2_offset, l2_table, l2_size);
+    ret = bdrv_pread(bs->file->bs, l2_offset, l2_table, l2_size);
     if (ret < 0) {
         fprintf(stderr, "ERROR: I/O error in check_refcounts_l2\n");
         res->check_errors++;
@@ -1481,7 +1492,7 @@ static int check_refcounts_l1(BlockDriverState *bs,
                               int64_t l1_table_offset, int l1_size,
                               int flags)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table = NULL, l2_offset, l1_size2;
     int i, ret;
 
@@ -1502,7 +1513,7 @@ static int check_refcounts_l1(BlockDriverState *bs,
             res->check_errors++;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
+        ret = bdrv_pread(bs->file->bs, l1_table_offset, l1_table, l1_size2);
         if (ret < 0) {
             fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
             res->check_errors++;
@@ -1558,7 +1569,7 @@ fail:
 static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
                               BdrvCheckMode fix)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table = qemu_blockalign(bs, s->cluster_size);
     int ret;
     uint64_t refcount;
@@ -1600,7 +1611,7 @@ static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
             }
         }
 
-        ret = bdrv_pread(bs->file, l2_offset, l2_table,
+        ret = bdrv_pread(bs->file->bs, l2_offset, l2_table,
                          s->l2_size * sizeof(uint64_t));
         if (ret < 0) {
             fprintf(stderr, "ERROR: Could not read L2 table: %s\n",
@@ -1652,7 +1663,8 @@ static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
                 goto fail;
             }
 
-            ret = bdrv_pwrite(bs->file, l2_offset, l2_table, s->cluster_size);
+            ret = bdrv_pwrite(bs->file->bs, l2_offset, l2_table,
+                              s->cluster_size);
             if (ret < 0) {
                 fprintf(stderr, "ERROR: Could not write L2 table: %s\n",
                         strerror(-ret));
@@ -1677,7 +1689,7 @@ static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
                            BdrvCheckMode fix, bool *rebuild,
                            void **refcount_table, int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i, size;
     int ret;
 
@@ -1707,11 +1719,11 @@ static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
                     goto resize_fail;
                 }
 
-                ret = bdrv_truncate(bs->file, offset + s->cluster_size);
+                ret = bdrv_truncate(bs->file->bs, offset + s->cluster_size);
                 if (ret < 0) {
                     goto resize_fail;
                 }
-                size = bdrv_getlength(bs->file);
+                size = bdrv_getlength(bs->file->bs);
                 if (size < 0) {
                     ret = size;
                     goto resize_fail;
@@ -1780,7 +1792,7 @@ static int calculate_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                                BdrvCheckMode fix, bool *rebuild,
                                void **refcount_table, int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i;
     QCowSnapshot *sn;
     int ret;
@@ -1844,7 +1856,7 @@ static void compare_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                               int64_t *highest_cluster,
                               void *refcount_table, int64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i;
     uint64_t refcount1, refcount2;
     int ret;
@@ -1921,7 +1933,7 @@ static int64_t alloc_clusters_imrt(BlockDriverState *bs,
                                    int64_t *imrt_nb_clusters,
                                    int64_t *first_free_cluster)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t cluster = *first_free_cluster, i;
     bool first_gap = true;
     int contiguous_free_clusters;
@@ -1991,7 +2003,7 @@ static int rebuild_refcount_structure(BlockDriverState *bs,
                                       void **refcount_table,
                                       int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t first_free_cluster = 0, reftable_offset = -1, cluster = 0;
     int64_t refblock_offset, refblock_start, refblock_index;
     uint32_t reftable_size = 0;
@@ -2085,7 +2097,7 @@ write_refblocks:
         on_disk_refblock = (void *)((char *) *refcount_table +
                                     refblock_index * s->cluster_size);
 
-        ret = bdrv_write(bs->file, refblock_offset / BDRV_SECTOR_SIZE,
+        ret = bdrv_write(bs->file->bs, refblock_offset / BDRV_SECTOR_SIZE,
                          on_disk_refblock, s->cluster_sectors);
         if (ret < 0) {
             fprintf(stderr, "ERROR writing refblock: %s\n", strerror(-ret));
@@ -2134,7 +2146,7 @@ write_refblocks:
     }
 
     assert(reftable_size < INT_MAX / sizeof(uint64_t));
-    ret = bdrv_pwrite(bs->file, reftable_offset, on_disk_reftable,
+    ret = bdrv_pwrite(bs->file->bs, reftable_offset, on_disk_reftable,
                       reftable_size * sizeof(uint64_t));
     if (ret < 0) {
         fprintf(stderr, "ERROR writing reftable: %s\n", strerror(-ret));
@@ -2146,8 +2158,8 @@ write_refblocks:
                  reftable_offset);
     cpu_to_be32w(&reftable_offset_and_clusters.reftable_clusters,
                  size_to_clusters(s, reftable_size * sizeof(uint64_t)));
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader,
-                                              refcount_table_offset),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader,
+                                                  refcount_table_offset),
                            &reftable_offset_and_clusters,
                            sizeof(reftable_offset_and_clusters));
     if (ret < 0) {
@@ -2178,14 +2190,14 @@ fail:
 int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                           BdrvCheckMode fix)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     BdrvCheckResult pre_compare_res;
     int64_t size, highest_cluster, nb_clusters;
     void *refcount_table = NULL;
     bool rebuild = false;
     int ret;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     if (size < 0) {
         res->check_errors++;
         return size;
@@ -2315,7 +2327,7 @@ fail:
 int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                                  int64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int chk = s->overlap_check & ~ign;
     int i, j;
 
@@ -2394,7 +2406,7 @@ int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                 return -ENOMEM;
             }
 
-            ret = bdrv_pread(bs->file, l1_ofs, l1, l1_sz2);
+            ret = bdrv_pread(bs->file->bs, l1_ofs, l1, l1_sz2);
             if (ret < 0) {
                 g_free(l1);
                 return ret;
diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index b6f58c13e..def720164 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -29,7 +29,7 @@
 
 void qcow2_free_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
 
     for(i = 0; i < s->nb_snapshots; i++) {
@@ -43,7 +43,7 @@ void qcow2_free_snapshots(BlockDriverState *bs)
 
 int qcow2_read_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshotHeader h;
     QCowSnapshotExtraData extra;
     QCowSnapshot *sn;
@@ -64,7 +64,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
     for(i = 0; i < s->nb_snapshots; i++) {
         /* Read statically sized part of the snapshot header */
         offset = align_offset(offset, 8);
-        ret = bdrv_pread(bs->file, offset, &h, sizeof(h));
+        ret = bdrv_pread(bs->file->bs, offset, &h, sizeof(h));
         if (ret < 0) {
             goto fail;
         }
@@ -83,7 +83,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
         name_size = be16_to_cpu(h.name_size);
 
         /* Read extra data */
-        ret = bdrv_pread(bs->file, offset, &extra,
+        ret = bdrv_pread(bs->file->bs, offset, &extra,
                          MIN(sizeof(extra), extra_data_size));
         if (ret < 0) {
             goto fail;
@@ -102,7 +102,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
 
         /* Read snapshot ID */
         sn->id_str = g_malloc(id_str_size + 1);
-        ret = bdrv_pread(bs->file, offset, sn->id_str, id_str_size);
+        ret = bdrv_pread(bs->file->bs, offset, sn->id_str, id_str_size);
         if (ret < 0) {
             goto fail;
         }
@@ -111,7 +111,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
 
         /* Read snapshot name */
         sn->name = g_malloc(name_size + 1);
-        ret = bdrv_pread(bs->file, offset, sn->name, name_size);
+        ret = bdrv_pread(bs->file->bs, offset, sn->name, name_size);
         if (ret < 0) {
             goto fail;
         }
@@ -136,7 +136,7 @@ fail:
 /* add at the end of the file a new list of snapshots */
 static int qcow2_write_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     QCowSnapshotHeader h;
     QCowSnapshotExtraData extra;
@@ -214,25 +214,25 @@ static int qcow2_write_snapshots(BlockDriverState *bs)
         h.name_size = cpu_to_be16(name_size);
         offset = align_offset(offset, 8);
 
-        ret = bdrv_pwrite(bs->file, offset, &h, sizeof(h));
+        ret = bdrv_pwrite(bs->file->bs, offset, &h, sizeof(h));
         if (ret < 0) {
             goto fail;
         }
         offset += sizeof(h);
 
-        ret = bdrv_pwrite(bs->file, offset, &extra, sizeof(extra));
+        ret = bdrv_pwrite(bs->file->bs, offset, &extra, sizeof(extra));
         if (ret < 0) {
             goto fail;
         }
         offset += sizeof(extra);
 
-        ret = bdrv_pwrite(bs->file, offset, sn->id_str, id_str_size);
+        ret = bdrv_pwrite(bs->file->bs, offset, sn->id_str, id_str_size);
         if (ret < 0) {
             goto fail;
         }
         offset += id_str_size;
 
-        ret = bdrv_pwrite(bs->file, offset, sn->name, name_size);
+        ret = bdrv_pwrite(bs->file->bs, offset, sn->name, name_size);
         if (ret < 0) {
             goto fail;
         }
@@ -254,7 +254,7 @@ static int qcow2_write_snapshots(BlockDriverState *bs)
     header_data.nb_snapshots        = cpu_to_be32(s->nb_snapshots);
     header_data.snapshots_offset    = cpu_to_be64(snapshots_offset);
 
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, nb_snapshots),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, nb_snapshots),
                            &header_data, sizeof(header_data));
     if (ret < 0) {
         goto fail;
@@ -278,7 +278,7 @@ fail:
 static void find_new_snapshot_id(BlockDriverState *bs,
                                  char *id_str, int id_str_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     int i;
     unsigned long id, id_max = 0;
@@ -296,7 +296,7 @@ static int find_snapshot_by_id_and_name(BlockDriverState *bs,
                                         const char *id,
                                         const char *name)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
 
     if (id && name) {
@@ -338,7 +338,7 @@ static int find_snapshot_by_id_or_name(BlockDriverState *bs,
 /* if no id is provided, a new one is constructed */
 int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *new_snapshot_list = NULL;
     QCowSnapshot *old_snapshot_list = NULL;
     QCowSnapshot sn1, *sn = &sn1;
@@ -396,7 +396,7 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
         goto fail;
     }
 
-    ret = bdrv_pwrite(bs->file, sn->l1_table_offset, l1_table,
+    ret = bdrv_pwrite(bs->file->bs, sn->l1_table_offset, l1_table,
                       s->l1_size * sizeof(uint64_t));
     if (ret < 0) {
         goto fail;
@@ -461,7 +461,7 @@ fail:
 /* copy the snapshot 'snapshot_name' into the current disk image */
 int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     int i, snapshot_index;
     int cur_l1_bytes, sn_l1_bytes;
@@ -509,7 +509,8 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
         goto fail;
     }
 
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, sn_l1_table, sn_l1_bytes);
+    ret = bdrv_pread(bs->file->bs, sn->l1_table_offset,
+                     sn_l1_table, sn_l1_bytes);
     if (ret < 0) {
         goto fail;
     }
@@ -526,7 +527,7 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
         goto fail;
     }
 
-    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset, sn_l1_table,
+    ret = bdrv_pwrite_sync(bs->file->bs, s->l1_table_offset, sn_l1_table,
                            cur_l1_bytes);
     if (ret < 0) {
         goto fail;
@@ -587,7 +588,7 @@ int qcow2_snapshot_delete(BlockDriverState *bs,
                           const char *name,
                           Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot sn;
     int snapshot_index, ret;
 
@@ -650,7 +651,7 @@ int qcow2_snapshot_delete(BlockDriverState *bs,
 
 int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QEMUSnapshotInfo *sn_tab, *sn_info;
     QCowSnapshot *sn;
     int i;
@@ -683,7 +684,7 @@ int qcow2_snapshot_load_tmp(BlockDriverState *bs,
                             Error **errp)
 {
     int i, snapshot_index;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     uint64_t *new_l1_table;
     int new_l1_bytes;
@@ -706,13 +707,14 @@ int qcow2_snapshot_load_tmp(BlockDriverState *bs,
         return -EFBIG;
     }
     new_l1_bytes = sn->l1_size * sizeof(uint64_t);
-    new_l1_table = qemu_try_blockalign(bs->file,
+    new_l1_table = qemu_try_blockalign(bs->file->bs,
                                        align_offset(new_l1_bytes, 512));
     if (new_l1_table == NULL) {
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, new_l1_table, new_l1_bytes);
+    ret = bdrv_pread(bs->file->bs, sn->l1_table_offset,
+                     new_l1_table, new_l1_bytes);
     if (ret < 0) {
         error_setg(errp, "Failed to read l1 table for snapshot");
         qemu_vfree(new_l1_table);
diff --git a/block/qcow2.c b/block/qcow2.c
index 76c331b38..88f56c886 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -85,7 +85,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
                                  uint64_t end_offset, void **p_feature_table,
                                  Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowExtension ext;
     uint64_t offset;
     int ret;
@@ -104,7 +104,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
         printf("attempting to read extended header in offset %lu\n", offset);
 #endif
 
-        ret = bdrv_pread(bs->file, offset, &ext, sizeof(ext));
+        ret = bdrv_pread(bs->file->bs, offset, &ext, sizeof(ext));
         if (ret < 0) {
             error_setg_errno(errp, -ret, "qcow2_read_extension: ERROR: "
                              "pread fail from offset %" PRIu64, offset);
@@ -132,7 +132,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
                            sizeof(bs->backing_format));
                 return 2;
             }
-            ret = bdrv_pread(bs->file, offset, bs->backing_format, ext.len);
+            ret = bdrv_pread(bs->file->bs, offset, bs->backing_format, ext.len);
             if (ret < 0) {
                 error_setg_errno(errp, -ret, "ERROR: ext_backing_format: "
                                  "Could not read format name");
@@ -148,7 +148,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
         case QCOW2_EXT_MAGIC_FEATURE_TABLE:
             if (p_feature_table != NULL) {
                 void* feature_table = g_malloc0(ext.len + 2 * sizeof(Qcow2Feature));
-                ret = bdrv_pread(bs->file, offset , feature_table, ext.len);
+                ret = bdrv_pread(bs->file->bs, offset , feature_table, ext.len);
                 if (ret < 0) {
                     error_setg_errno(errp, -ret, "ERROR: ext_feature_table: "
                                      "Could not read table");
@@ -169,7 +169,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
                 uext->len = ext.len;
                 QLIST_INSERT_HEAD(&s->unknown_header_ext, uext, next);
 
-                ret = bdrv_pread(bs->file, offset , uext->data, uext->len);
+                ret = bdrv_pread(bs->file->bs, offset , uext->data, uext->len);
                 if (ret < 0) {
                     error_setg_errno(errp, -ret, "ERROR: unknown extension: "
                                      "Could not read data");
@@ -187,7 +187,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
 
 static void cleanup_unknown_header_ext(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2UnknownHeaderExtension *uext, *next;
 
     QLIST_FOREACH_SAFE(uext, &s->unknown_header_ext, next, next) {
@@ -249,7 +249,7 @@ static void report_unsupported_feature(BlockDriverState *bs,
  */
 int qcow2_mark_dirty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t val;
     int ret;
 
@@ -260,12 +260,12 @@ int qcow2_mark_dirty(BlockDriverState *bs)
     }
 
     val = cpu_to_be64(s->incompatible_features | QCOW2_INCOMPAT_DIRTY);
-    ret = bdrv_pwrite(bs->file, offsetof(QCowHeader, incompatible_features),
+    ret = bdrv_pwrite(bs->file->bs, offsetof(QCowHeader, incompatible_features),
                       &val, sizeof(val));
     if (ret < 0) {
         return ret;
     }
-    ret = bdrv_flush(bs->file);
+    ret = bdrv_flush(bs->file->bs);
     if (ret < 0) {
         return ret;
     }
@@ -282,7 +282,7 @@ int qcow2_mark_dirty(BlockDriverState *bs)
  */
 static int qcow2_mark_clean(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
         int ret;
@@ -304,7 +304,7 @@ static int qcow2_mark_clean(BlockDriverState *bs)
  */
 int qcow2_mark_corrupt(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     s->incompatible_features |= QCOW2_INCOMPAT_CORRUPT;
     return qcow2_update_header(bs);
@@ -316,7 +316,7 @@ int qcow2_mark_corrupt(BlockDriverState *bs)
  */
 int qcow2_mark_consistent(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     if (s->incompatible_features & QCOW2_INCOMPAT_CORRUPT) {
         int ret = bdrv_flush(bs);
@@ -351,7 +351,7 @@ static int qcow2_check(BlockDriverState *bs, BdrvCheckResult *result,
 static int validate_table_offset(BlockDriverState *bs, uint64_t offset,
                                  uint64_t entries, size_t entry_len)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t size;
 
     /* Use signed INT64_MAX as the maximum even for uint64_t header fields,
@@ -467,6 +467,11 @@ static QemuOptsList qcow2_runtime_opts = {
             .type = QEMU_OPT_SIZE,
             .help = "Maximum refcount block cache size",
         },
+        {
+            .name = QCOW2_OPT_CACHE_CLEAN_INTERVAL,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Clean unused cache entries after this time (in seconds)",
+        },
         { /* end of list */ }
     },
 };
@@ -482,11 +487,54 @@ static const char *overlap_bool_option_names[QCOW2_OL_MAX_BITNR] = {
     [QCOW2_OL_INACTIVE_L2_BITNR]    = QCOW2_OPT_OVERLAP_INACTIVE_L2,
 };
 
+static void cache_clean_timer_cb(void *opaque)
+{
+    BlockDriverState *bs = opaque;
+    BDRVQcow2State *s = bs->opaque;
+    qcow2_cache_clean_unused(bs, s->l2_table_cache);
+    qcow2_cache_clean_unused(bs, s->refcount_block_cache);
+    timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+              (int64_t) s->cache_clean_interval * 1000);
+}
+
+static void cache_clean_timer_init(BlockDriverState *bs, AioContext *context)
+{
+    BDRVQcow2State *s = bs->opaque;
+    if (s->cache_clean_interval > 0) {
+        s->cache_clean_timer = aio_timer_new(context, QEMU_CLOCK_VIRTUAL,
+                                             SCALE_MS, cache_clean_timer_cb,
+                                             bs);
+        timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+                  (int64_t) s->cache_clean_interval * 1000);
+    }
+}
+
+static void cache_clean_timer_del(BlockDriverState *bs)
+{
+    BDRVQcow2State *s = bs->opaque;
+    if (s->cache_clean_timer) {
+        timer_del(s->cache_clean_timer);
+        timer_free(s->cache_clean_timer);
+        s->cache_clean_timer = NULL;
+    }
+}
+
+static void qcow2_detach_aio_context(BlockDriverState *bs)
+{
+    cache_clean_timer_del(bs);
+}
+
+static void qcow2_attach_aio_context(BlockDriverState *bs,
+                                     AioContext *new_context)
+{
+    cache_clean_timer_init(bs, new_context);
+}
+
 static void read_cache_sizes(BlockDriverState *bs, QemuOpts *opts,
                              uint64_t *l2_cache_size,
                              uint64_t *refcount_cache_size, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t combined_cache_size;
     bool l2_cache_size_set, refcount_cache_size_set, combined_cache_size_set;
 
@@ -541,22 +589,246 @@ static void read_cache_sizes(BlockDriverState *bs, QemuOpts *opts,
     }
 }
 
+typedef struct Qcow2ReopenState {
+    Qcow2Cache *l2_table_cache;
+    Qcow2Cache *refcount_block_cache;
+    bool use_lazy_refcounts;
+    int overlap_check;
+    bool discard_passthrough[QCOW2_DISCARD_MAX];
+    uint64_t cache_clean_interval;
+} Qcow2ReopenState;
+
+static int qcow2_update_options_prepare(BlockDriverState *bs,
+                                        Qcow2ReopenState *r,
+                                        QDict *options, int flags,
+                                        Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    QemuOpts *opts = NULL;
+    const char *opt_overlap_check, *opt_overlap_check_template;
+    int overlap_check_template = 0;
+    uint64_t l2_cache_size, refcount_cache_size;
+    int i;
+    Error *local_err = NULL;
+    int ret;
+
+    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* get L2 table/refcount block cache size from command line options */
+    read_cache_sizes(bs, opts, &l2_cache_size, &refcount_cache_size,
+                     &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    l2_cache_size /= s->cluster_size;
+    if (l2_cache_size < MIN_L2_CACHE_SIZE) {
+        l2_cache_size = MIN_L2_CACHE_SIZE;
+    }
+    if (l2_cache_size > INT_MAX) {
+        error_setg(errp, "L2 cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    refcount_cache_size /= s->cluster_size;
+    if (refcount_cache_size < MIN_REFCOUNT_CACHE_SIZE) {
+        refcount_cache_size = MIN_REFCOUNT_CACHE_SIZE;
+    }
+    if (refcount_cache_size > INT_MAX) {
+        error_setg(errp, "Refcount cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* alloc new L2 table/refcount block cache, flush old one */
+    if (s->l2_table_cache) {
+        ret = qcow2_cache_flush(bs, s->l2_table_cache);
+        if (ret) {
+            error_setg_errno(errp, -ret, "Failed to flush the L2 table cache");
+            goto fail;
+        }
+    }
+
+    if (s->refcount_block_cache) {
+        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+        if (ret) {
+            error_setg_errno(errp, -ret,
+                             "Failed to flush the refcount block cache");
+            goto fail;
+        }
+    }
+
+    r->l2_table_cache = qcow2_cache_create(bs, l2_cache_size);
+    r->refcount_block_cache = qcow2_cache_create(bs, refcount_cache_size);
+    if (r->l2_table_cache == NULL || r->refcount_block_cache == NULL) {
+        error_setg(errp, "Could not allocate metadata caches");
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    /* New interval for cache cleanup timer */
+    r->cache_clean_interval =
+        qemu_opt_get_number(opts, QCOW2_OPT_CACHE_CLEAN_INTERVAL,
+                            s->cache_clean_interval);
+    if (r->cache_clean_interval > UINT_MAX) {
+        error_setg(errp, "Cache clean interval too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* lazy-refcounts; flush if going from enabled to disabled */
+    r->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
+        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
+    if (r->use_lazy_refcounts && s->qcow_version < 3) {
+        error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
+                   "qemu 1.1 compatibility level");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    if (s->use_lazy_refcounts && !r->use_lazy_refcounts) {
+        ret = qcow2_mark_clean(bs);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Failed to disable lazy refcounts");
+            goto fail;
+        }
+    }
+
+    /* Overlap check options */
+    opt_overlap_check = qemu_opt_get(opts, QCOW2_OPT_OVERLAP);
+    opt_overlap_check_template = qemu_opt_get(opts, QCOW2_OPT_OVERLAP_TEMPLATE);
+    if (opt_overlap_check_template && opt_overlap_check &&
+        strcmp(opt_overlap_check_template, opt_overlap_check))
+    {
+        error_setg(errp, "Conflicting values for qcow2 options '"
+                   QCOW2_OPT_OVERLAP "' ('%s') and '" QCOW2_OPT_OVERLAP_TEMPLATE
+                   "' ('%s')", opt_overlap_check, opt_overlap_check_template);
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (!opt_overlap_check) {
+        opt_overlap_check = opt_overlap_check_template ?: "cached";
+    }
+
+    if (!strcmp(opt_overlap_check, "none")) {
+        overlap_check_template = 0;
+    } else if (!strcmp(opt_overlap_check, "constant")) {
+        overlap_check_template = QCOW2_OL_CONSTANT;
+    } else if (!strcmp(opt_overlap_check, "cached")) {
+        overlap_check_template = QCOW2_OL_CACHED;
+    } else if (!strcmp(opt_overlap_check, "all")) {
+        overlap_check_template = QCOW2_OL_ALL;
+    } else {
+        error_setg(errp, "Unsupported value '%s' for qcow2 option "
+                   "'overlap-check'. Allowed are any of the following: "
+                   "none, constant, cached, all", opt_overlap_check);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    r->overlap_check = 0;
+    for (i = 0; i < QCOW2_OL_MAX_BITNR; i++) {
+        /* overlap-check defines a template bitmask, but every flag may be
+         * overwritten through the associated boolean option */
+        r->overlap_check |=
+            qemu_opt_get_bool(opts, overlap_bool_option_names[i],
+                              overlap_check_template & (1 << i)) << i;
+    }
+
+    r->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
+    r->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
+    r->discard_passthrough[QCOW2_DISCARD_REQUEST] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
+                          flags & BDRV_O_UNMAP);
+    r->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
+    r->discard_passthrough[QCOW2_DISCARD_OTHER] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
+
+    ret = 0;
+fail:
+    qemu_opts_del(opts);
+    opts = NULL;
+    return ret;
+}
+
+static void qcow2_update_options_commit(BlockDriverState *bs,
+                                        Qcow2ReopenState *r)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int i;
+
+    if (s->l2_table_cache) {
+        qcow2_cache_destroy(bs, s->l2_table_cache);
+    }
+    if (s->refcount_block_cache) {
+        qcow2_cache_destroy(bs, s->refcount_block_cache);
+    }
+    s->l2_table_cache = r->l2_table_cache;
+    s->refcount_block_cache = r->refcount_block_cache;
+
+    s->overlap_check = r->overlap_check;
+    s->use_lazy_refcounts = r->use_lazy_refcounts;
+
+    for (i = 0; i < QCOW2_DISCARD_MAX; i++) {
+        s->discard_passthrough[i] = r->discard_passthrough[i];
+    }
+
+    if (s->cache_clean_interval != r->cache_clean_interval) {
+        cache_clean_timer_del(bs);
+        s->cache_clean_interval = r->cache_clean_interval;
+        cache_clean_timer_init(bs, bdrv_get_aio_context(bs));
+    }
+}
+
+static void qcow2_update_options_abort(BlockDriverState *bs,
+                                       Qcow2ReopenState *r)
+{
+    if (r->l2_table_cache) {
+        qcow2_cache_destroy(bs, r->l2_table_cache);
+    }
+    if (r->refcount_block_cache) {
+        qcow2_cache_destroy(bs, r->refcount_block_cache);
+    }
+}
+
+static int qcow2_update_options(BlockDriverState *bs, QDict *options,
+                                int flags, Error **errp)
+{
+    Qcow2ReopenState r = {};
+    int ret;
+
+    ret = qcow2_update_options_prepare(bs, &r, options, flags, errp);
+    if (ret >= 0) {
+        qcow2_update_options_commit(bs, &r);
+    } else {
+        qcow2_update_options_abort(bs, &r);
+    }
+
+    return ret;
+}
+
 static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
                       Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int len, i;
     int ret = 0;
     QCowHeader header;
-    QemuOpts *opts = NULL;
     Error *local_err = NULL;
     uint64_t ext_end;
     uint64_t l1_vm_state_index;
-    const char *opt_overlap_check, *opt_overlap_check_template;
-    int overlap_check_template = 0;
-    uint64_t l2_cache_size, refcount_cache_size;
 
-    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
+    ret = bdrv_pread(bs->file->bs, 0, &header, sizeof(header));
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not read qcow2 header");
         goto fail;
@@ -631,7 +903,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     if (header.header_length > sizeof(header)) {
         s->unknown_header_fields_size = header.header_length - sizeof(header);
         s->unknown_header_fields = g_malloc(s->unknown_header_fields_size);
-        ret = bdrv_pread(bs->file, sizeof(header), s->unknown_header_fields,
+        ret = bdrv_pread(bs->file->bs, sizeof(header), s->unknown_header_fields,
                          s->unknown_header_fields_size);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not read unknown qcow2 header "
@@ -784,14 +1056,14 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
 
 
     if (s->l1_size > 0) {
-        s->l1_table = qemu_try_blockalign(bs->file,
+        s->l1_table = qemu_try_blockalign(bs->file->bs,
             align_offset(s->l1_size * sizeof(uint64_t), 512));
         if (s->l1_table == NULL) {
             error_setg(errp, "Could not allocate L1 table");
             ret = -ENOMEM;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
+        ret = bdrv_pread(bs->file->bs, s->l1_table_offset, s->l1_table,
                          s->l1_size * sizeof(uint64_t));
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not read L1 table");
@@ -802,55 +1074,15 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
-    /* get L2 table/refcount block cache size from command line options */
-    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
-    qemu_opts_absorb_qdict(opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    read_cache_sizes(bs, opts, &l2_cache_size, &refcount_cache_size,
-                     &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    l2_cache_size /= s->cluster_size;
-    if (l2_cache_size < MIN_L2_CACHE_SIZE) {
-        l2_cache_size = MIN_L2_CACHE_SIZE;
-    }
-    if (l2_cache_size > INT_MAX) {
-        error_setg(errp, "L2 cache size too big");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    refcount_cache_size /= s->cluster_size;
-    if (refcount_cache_size < MIN_REFCOUNT_CACHE_SIZE) {
-        refcount_cache_size = MIN_REFCOUNT_CACHE_SIZE;
-    }
-    if (refcount_cache_size > INT_MAX) {
-        error_setg(errp, "Refcount cache size too big");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    /* alloc L2 table/refcount block cache */
-    s->l2_table_cache = qcow2_cache_create(bs, l2_cache_size);
-    s->refcount_block_cache = qcow2_cache_create(bs, refcount_cache_size);
-    if (s->l2_table_cache == NULL || s->refcount_block_cache == NULL) {
-        error_setg(errp, "Could not allocate metadata caches");
-        ret = -ENOMEM;
+    /* Parse driver-specific options */
+    ret = qcow2_update_options(bs, options, flags, errp);
+    if (ret < 0) {
         goto fail;
     }
 
     s->cluster_cache = g_malloc(s->cluster_size);
     /* one more sector for decompressed data alignment */
-    s->cluster_data = qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
+    s->cluster_data = qemu_try_blockalign(bs->file->bs, QCOW_MAX_CRYPT_CLUSTERS
                                                     * s->cluster_size + 512);
     if (s->cluster_data == NULL) {
         error_setg(errp, "Could not allocate temporary cluster buffer");
@@ -887,7 +1119,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
             ret = -EINVAL;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, header.backing_file_offset,
+        ret = bdrv_pread(bs->file->bs, header.backing_file_offset,
                          bs->backing_file, len);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not read backing file name");
@@ -932,70 +1164,6 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
-    /* Enable lazy_refcounts according to image and command line options */
-    s->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
-        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
-
-    s->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
-    s->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
-    s->discard_passthrough[QCOW2_DISCARD_REQUEST] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
-                          flags & BDRV_O_UNMAP);
-    s->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
-    s->discard_passthrough[QCOW2_DISCARD_OTHER] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
-
-    opt_overlap_check = qemu_opt_get(opts, QCOW2_OPT_OVERLAP);
-    opt_overlap_check_template = qemu_opt_get(opts, QCOW2_OPT_OVERLAP_TEMPLATE);
-    if (opt_overlap_check_template && opt_overlap_check &&
-        strcmp(opt_overlap_check_template, opt_overlap_check))
-    {
-        error_setg(errp, "Conflicting values for qcow2 options '"
-                   QCOW2_OPT_OVERLAP "' ('%s') and '" QCOW2_OPT_OVERLAP_TEMPLATE
-                   "' ('%s')", opt_overlap_check, opt_overlap_check_template);
-        ret = -EINVAL;
-        goto fail;
-    }
-    if (!opt_overlap_check) {
-        opt_overlap_check = opt_overlap_check_template ?: "cached";
-    }
-
-    if (!strcmp(opt_overlap_check, "none")) {
-        overlap_check_template = 0;
-    } else if (!strcmp(opt_overlap_check, "constant")) {
-        overlap_check_template = QCOW2_OL_CONSTANT;
-    } else if (!strcmp(opt_overlap_check, "cached")) {
-        overlap_check_template = QCOW2_OL_CACHED;
-    } else if (!strcmp(opt_overlap_check, "all")) {
-        overlap_check_template = QCOW2_OL_ALL;
-    } else {
-        error_setg(errp, "Unsupported value '%s' for qcow2 option "
-                   "'overlap-check'. Allowed are either of the following: "
-                   "none, constant, cached, all", opt_overlap_check);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    s->overlap_check = 0;
-    for (i = 0; i < QCOW2_OL_MAX_BITNR; i++) {
-        /* overlap-check defines a template bitmask, but every flag may be
-         * overwritten through the associated boolean option */
-        s->overlap_check |=
-            qemu_opt_get_bool(opts, overlap_bool_option_names[i],
-                              overlap_check_template & (1 << i)) << i;
-    }
-
-    qemu_opts_del(opts);
-    opts = NULL;
-
-    if (s->use_lazy_refcounts && s->qcow_version < 3) {
-        error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
-                   "qemu 1.1 compatibility level");
-        ret = -EINVAL;
-        goto fail;
-    }
-
 #ifdef DEBUG_ALLOC
     {
         BdrvCheckResult result = {0};
@@ -1005,7 +1173,6 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     return ret;
 
  fail:
-    qemu_opts_del(opts);
     g_free(s->unknown_header_fields);
     cleanup_unknown_header_ext(bs);
     qcow2_free_snapshots(bs);
@@ -1013,6 +1180,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
+    cache_clean_timer_del(bs);
     if (s->l2_table_cache) {
         qcow2_cache_destroy(bs, s->l2_table_cache);
     }
@@ -1026,14 +1194,14 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
 
 static void qcow2_refresh_limits(BlockDriverState *bs, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     bs->bl.write_zeroes_alignment = s->cluster_sectors;
 }
 
 static int qcow2_set_key(BlockDriverState *bs, const char *key)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint8_t keybuf[16];
     int len, i;
     Error *err = NULL;
@@ -1066,32 +1234,58 @@ static int qcow2_set_key(BlockDriverState *bs, const char *key)
     return 0;
 }
 
-/* We have no actual commit/abort logic for qcow2, but we need to write out any
- * unwritten data if we reopen read-only. */
 static int qcow2_reopen_prepare(BDRVReopenState *state,
                                 BlockReopenQueue *queue, Error **errp)
 {
+    Qcow2ReopenState *r;
     int ret;
 
+    r = g_new0(Qcow2ReopenState, 1);
+    state->opaque = r;
+
+    ret = qcow2_update_options_prepare(state->bs, r, state->options,
+                                       state->flags, errp);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* We need to write out any unwritten data if we reopen read-only. */
     if ((state->flags & BDRV_O_RDWR) == 0) {
         ret = bdrv_flush(state->bs);
         if (ret < 0) {
-            return ret;
+            goto fail;
         }
 
         ret = qcow2_mark_clean(state->bs);
         if (ret < 0) {
-            return ret;
+            goto fail;
         }
     }
 
     return 0;
+
+fail:
+    qcow2_update_options_abort(state->bs, r);
+    g_free(r);
+    return ret;
+}
+
+static void qcow2_reopen_commit(BDRVReopenState *state)
+{
+    qcow2_update_options_commit(state->bs, state->opaque);
+    g_free(state->opaque);
+}
+
+static void qcow2_reopen_abort(BDRVReopenState *state)
+{
+    qcow2_update_options_abort(state->bs, state->opaque);
+    g_free(state->opaque);
 }
 
 static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, int *pnum)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t cluster_offset;
     int index_in_cluster, ret;
     int64_t status = 0;
@@ -1138,7 +1332,7 @@ int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
 static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
                           int remaining_sectors, QEMUIOVector *qiov)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int index_in_cluster, n1;
     int ret;
     int cur_nr_sectors; /* number of sectors in current iteration */
@@ -1175,9 +1369,9 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
         switch (ret) {
         case QCOW2_CLUSTER_UNALLOCATED:
 
-            if (bs->backing_hd) {
+            if (bs->backing) {
                 /* read from the base image */
-                n1 = qcow2_backing_read1(bs->backing_hd, &hd_qiov,
+                n1 = qcow2_backing_read1(bs->backing->bs, &hd_qiov,
                     sector_num, cur_nr_sectors);
                 if (n1 > 0) {
                     QEMUIOVector local_qiov;
@@ -1188,7 +1382,7 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
 
                     BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
                     qemu_co_mutex_unlock(&s->lock);
-                    ret = bdrv_co_readv(bs->backing_hd, sector_num,
+                    ret = bdrv_co_readv(bs->backing->bs, sector_num,
                                         n1, &local_qiov);
                     qemu_co_mutex_lock(&s->lock);
 
@@ -1235,8 +1429,9 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
                  */
                 if (!cluster_data) {
                     cluster_data =
-                        qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
-                                                      * s->cluster_size);
+                        qemu_try_blockalign(bs->file->bs,
+                                            QCOW_MAX_CRYPT_CLUSTERS
+                                            * s->cluster_size);
                     if (cluster_data == NULL) {
                         ret = -ENOMEM;
                         goto fail;
@@ -1252,7 +1447,7 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
 
             BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
             qemu_co_mutex_unlock(&s->lock);
-            ret = bdrv_co_readv(bs->file,
+            ret = bdrv_co_readv(bs->file->bs,
                                 (cluster_offset >> 9) + index_in_cluster,
                                 cur_nr_sectors, &hd_qiov);
             qemu_co_mutex_lock(&s->lock);
@@ -1300,7 +1495,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
                            int remaining_sectors,
                            QEMUIOVector *qiov)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int index_in_cluster;
     int ret;
     int cur_nr_sectors; /* number of sectors in current iteration */
@@ -1349,7 +1544,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
             Error *err = NULL;
             assert(s->cipher);
             if (!cluster_data) {
-                cluster_data = qemu_try_blockalign(bs->file,
+                cluster_data = qemu_try_blockalign(bs->file->bs,
                                                    QCOW_MAX_CRYPT_CLUSTERS
                                                    * s->cluster_size);
                 if (cluster_data == NULL) {
@@ -1386,7 +1581,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
         BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         trace_qcow2_writev_data(qemu_coroutine_self(),
                                 (cluster_offset >> 9) + index_in_cluster);
-        ret = bdrv_co_writev(bs->file,
+        ret = bdrv_co_writev(bs->file->bs,
                              (cluster_offset >> 9) + index_in_cluster,
                              cur_nr_sectors, &hd_qiov);
         qemu_co_mutex_lock(&s->lock);
@@ -1446,7 +1641,7 @@ fail:
 
 static void qcow2_close(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
@@ -1471,6 +1666,7 @@ static void qcow2_close(BlockDriverState *bs)
         }
     }
 
+    cache_clean_timer_del(bs);
     qcow2_cache_destroy(bs, s->l2_table_cache);
     qcow2_cache_destroy(bs, s->refcount_block_cache);
 
@@ -1491,7 +1687,7 @@ static void qcow2_close(BlockDriverState *bs)
 
 static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int flags = s->flags;
     QCryptoCipher *cipher = NULL;
     QDict *options;
@@ -1508,13 +1704,13 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
 
     qcow2_close(bs);
 
-    bdrv_invalidate_cache(bs->file, &local_err);
+    bdrv_invalidate_cache(bs->file->bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
 
-    memset(s, 0, sizeof(BDRVQcowState));
+    memset(s, 0, sizeof(BDRVQcow2State));
     options = qdict_clone_shallow(bs->options);
 
     ret = qcow2_open(bs, options, flags, &local_err);
@@ -1561,7 +1757,7 @@ static size_t header_ext_add(char *buf, uint32_t magic, const void *s,
  */
 int qcow2_update_header(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowHeader *header;
     char *buf;
     size_t buflen = s->cluster_size;
@@ -1716,7 +1912,7 @@ int qcow2_update_header(BlockDriverState *bs)
     }
 
     /* Write the new header */
-    ret = bdrv_pwrite(bs->file, 0, header, s->cluster_size);
+    ret = bdrv_pwrite(bs->file->bs, 0, header, s->cluster_size);
     if (ret < 0) {
         goto fail;
     }
@@ -1730,7 +1926,7 @@ fail:
 static int qcow2_change_backing_file(BlockDriverState *bs,
     const char *backing_file, const char *backing_fmt)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
     pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
@@ -1796,7 +1992,8 @@ static int preallocate(BlockDriverState *bs)
     if (host_offset != 0) {
         uint8_t buf[BDRV_SECTOR_SIZE];
         memset(buf, 0, BDRV_SECTOR_SIZE);
-        ret = bdrv_write(bs->file, (host_offset >> BDRV_SECTOR_BITS) + num - 1,
+        ret = bdrv_write(bs->file->bs,
+                         (host_offset >> BDRV_SECTOR_BITS) + num - 1,
                          buf, 1);
         if (ret < 0) {
             return ret;
@@ -1812,8 +2009,10 @@ static int qcow2_create2(const char *filename, int64_t total_size,
                          QemuOpts *opts, int version, int refcount_order,
                          Error **errp)
 {
-    /* Calculate cluster_bits */
     int cluster_bits;
+    QDict *options;
+
+    /* Calculate cluster_bits */
     cluster_bits = ctz32(cluster_size);
     if (cluster_bits < MIN_CLUSTER_BITS || cluster_bits > MAX_CLUSTER_BITS ||
         (1 << cluster_bits) != cluster_size)
@@ -1912,7 +2111,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
 
     bs = NULL;
     ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
+                    &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         return ret;
@@ -1971,9 +2170,11 @@ static int qcow2_create2(const char *filename, int64_t total_size,
      * refcount of the cluster that is occupied by the header and the refcount
      * table)
      */
-    ret = bdrv_open(&bs, filename, NULL, NULL,
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow2"));
+    ret = bdrv_open(&bs, filename, NULL, options,
                     BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH,
-                    &bdrv_qcow2, &local_err);
+                    &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -2009,7 +2210,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
 
     /* And if we're supposed to preallocate metadata, do that now */
     if (prealloc != PREALLOC_MODE_OFF) {
-        BDRVQcowState *s = bs->opaque;
+        BDRVQcow2State *s = bs->opaque;
         qemu_co_mutex_lock(&s->lock);
         ret = preallocate(bs);
         qemu_co_mutex_unlock(&s->lock);
@@ -2023,9 +2224,11 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     bs = NULL;
 
     /* Reopen the image without BDRV_O_NO_FLUSH to flush it before returning */
-    ret = bdrv_open(&bs, filename, NULL, NULL,
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow2"));
+    ret = bdrv_open(&bs, filename, NULL, options,
                     BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_BACKING,
-                    &bdrv_qcow2, &local_err);
+                    &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto out;
@@ -2142,7 +2345,7 @@ static coroutine_fn int qcow2_co_write_zeroes(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
 {
     int ret;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     /* Emulate misaligned zero writes */
     if (sector_num % s->cluster_sectors || nb_sectors % s->cluster_sectors) {
@@ -2162,7 +2365,7 @@ static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors)
 {
     int ret;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     qemu_co_mutex_lock(&s->lock);
     ret = qcow2_discard_clusters(bs, sector_num << BDRV_SECTOR_BITS,
@@ -2173,7 +2376,7 @@ static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
 
 static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t new_l1_size;
     int ret;
 
@@ -2202,7 +2405,7 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
 
     /* write updated header.size */
     offset = cpu_to_be64(offset);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, size),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, size),
                            &offset, sizeof(uint64_t));
     if (ret < 0) {
         return ret;
@@ -2217,7 +2420,7 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
 static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
                                   const uint8_t *buf, int nb_sectors)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     z_stream strm;
     int ret, out_len;
     uint8_t *out_buf;
@@ -2226,8 +2429,8 @@ static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
     if (nb_sectors == 0) {
         /* align end of file to a sector boundary to ease reading with
            sector based I/Os */
-        cluster_offset = bdrv_getlength(bs->file);
-        return bdrv_truncate(bs->file, cluster_offset);
+        cluster_offset = bdrv_getlength(bs->file->bs);
+        return bdrv_truncate(bs->file->bs, cluster_offset);
     }
 
     if (nb_sectors != s->cluster_sectors) {
@@ -2294,7 +2497,7 @@ static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
         }
 
         BLKDBG_EVENT(bs->file, BLKDBG_WRITE_COMPRESSED);
-        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
+        ret = bdrv_pwrite(bs->file->bs, cluster_offset, out_buf, out_len);
         if (ret < 0) {
             goto fail;
         }
@@ -2308,7 +2511,7 @@ fail:
 
 static int make_completely_empty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret, l1_clusters;
     int64_t offset;
     uint64_t *new_reftable = NULL;
@@ -2343,7 +2546,7 @@ static int make_completely_empty(BlockDriverState *bs)
     /* After this call, neither the in-memory nor the on-disk refcount
      * information accurately describe the actual references */
 
-    ret = bdrv_write_zeroes(bs->file, s->l1_table_offset / BDRV_SECTOR_SIZE,
+    ret = bdrv_write_zeroes(bs->file->bs, s->l1_table_offset / BDRV_SECTOR_SIZE,
                             l1_clusters * s->cluster_sectors, 0);
     if (ret < 0) {
         goto fail_broken_refcounts;
@@ -2357,7 +2560,7 @@ static int make_completely_empty(BlockDriverState *bs)
      * overwrite parts of the existing refcount and L1 table, which is not
      * an issue because the dirty flag is set, complete data loss is in fact
      * desired and partial data loss is consequently fine as well */
-    ret = bdrv_write_zeroes(bs->file, s->cluster_size / BDRV_SECTOR_SIZE,
+    ret = bdrv_write_zeroes(bs->file->bs, s->cluster_size / BDRV_SECTOR_SIZE,
                             (2 + l1_clusters) * s->cluster_size /
                             BDRV_SECTOR_SIZE, 0);
     /* This call (even if it failed overall) may have overwritten on-disk
@@ -2377,7 +2580,7 @@ static int make_completely_empty(BlockDriverState *bs)
     cpu_to_be64w(&l1_ofs_rt_ofs_cls.l1_offset, 3 * s->cluster_size);
     cpu_to_be64w(&l1_ofs_rt_ofs_cls.reftable_offset, s->cluster_size);
     cpu_to_be32w(&l1_ofs_rt_ofs_cls.reftable_clusters, 1);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_table_offset),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, l1_table_offset),
                            &l1_ofs_rt_ofs_cls, sizeof(l1_ofs_rt_ofs_cls));
     if (ret < 0) {
         goto fail_broken_refcounts;
@@ -2408,7 +2611,7 @@ static int make_completely_empty(BlockDriverState *bs)
 
     /* Enter the first refblock into the reftable */
     rt_entry = cpu_to_be64(2 * s->cluster_size);
-    ret = bdrv_pwrite_sync(bs->file, s->cluster_size,
+    ret = bdrv_pwrite_sync(bs->file->bs, s->cluster_size,
                            &rt_entry, sizeof(rt_entry));
     if (ret < 0) {
         goto fail_broken_refcounts;
@@ -2433,7 +2636,7 @@ static int make_completely_empty(BlockDriverState *bs)
         goto fail;
     }
 
-    ret = bdrv_truncate(bs->file, (3 + l1_clusters) * s->cluster_size);
+    ret = bdrv_truncate(bs->file->bs, (3 + l1_clusters) * s->cluster_size);
     if (ret < 0) {
         goto fail;
     }
@@ -2456,7 +2659,7 @@ fail:
 
 static int qcow2_make_empty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start_sector;
     int sector_step = INT_MAX / BDRV_SECTOR_SIZE;
     int l1_clusters, ret = 0;
@@ -2497,7 +2700,7 @@ static int qcow2_make_empty(BlockDriverState *bs)
 
 static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     qemu_co_mutex_lock(&s->lock);
@@ -2521,7 +2724,7 @@ static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
 
 static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bdi->unallocated_blocks_are_zero = true;
     bdi->can_write_zeroes_with_unmap = (s->qcow_version >= 3);
     bdi->cluster_size = s->cluster_size;
@@ -2531,22 +2734,20 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 
 static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     ImageInfoSpecific *spec_info = g_new(ImageInfoSpecific, 1);
 
     *spec_info = (ImageInfoSpecific){
-        .kind  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
-        {
-            .qcow2 = g_new(ImageInfoSpecificQCow2, 1),
-        },
+        .type  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
+        .u.qcow2 = g_new(ImageInfoSpecificQCow2, 1),
     };
     if (s->qcow_version == 2) {
-        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
+        *spec_info->u.qcow2 = (ImageInfoSpecificQCow2){
             .compat             = g_strdup("0.10"),
             .refcount_bits      = s->refcount_bits,
         };
     } else if (s->qcow_version == 3) {
-        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
+        *spec_info->u.qcow2 = (ImageInfoSpecificQCow2){
             .compat             = g_strdup("1.1"),
             .lazy_refcounts     = s->compatible_features &
                                   QCOW2_COMPAT_LAZY_REFCOUNTS,
@@ -2564,11 +2765,11 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
 #if 0
 static void dump_refcounts(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t nb_clusters, k, k1, size;
     int refcount;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     nb_clusters = size_to_clusters(s, size);
     for(k = 0; k < nb_clusters;) {
         k1 = k;
@@ -2585,7 +2786,7 @@ static void dump_refcounts(BlockDriverState *bs)
 static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
                               int64_t pos)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t total_sectors = bs->total_sectors;
     bool zero_beyond_eof = bs->zero_beyond_eof;
     int ret;
@@ -2606,7 +2807,7 @@ static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
 static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
                               int64_t pos, int size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bool zero_beyond_eof = bs->zero_beyond_eof;
     int ret;
 
@@ -2625,7 +2826,7 @@ static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
 static int qcow2_downgrade(BlockDriverState *bs, int target_version,
                            BlockDriverAmendStatusCB *status_cb)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int current_version = s->qcow_version;
     int ret;
 
@@ -2689,7 +2890,7 @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
 static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
                                BlockDriverAmendStatusCB *status_cb)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int old_version = s->qcow_version, new_version = old_version;
     uint64_t new_size = 0;
     const char *backing_file = NULL, *backing_format = NULL;
@@ -2836,7 +3037,7 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
 void qcow2_signal_corruption(BlockDriverState *bs, bool fatal, int64_t offset,
                              int64_t size, const char *message_format, ...)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     const char *node_name;
     char *message;
     va_list ap;
@@ -2937,11 +3138,13 @@ static QemuOptsList qcow2_create_opts = {
 
 BlockDriver bdrv_qcow2 = {
     .format_name        = "qcow2",
-    .instance_size      = sizeof(BDRVQcowState),
+    .instance_size      = sizeof(BDRVQcow2State),
     .bdrv_probe         = qcow2_probe,
     .bdrv_open          = qcow2_open,
     .bdrv_close         = qcow2_close,
     .bdrv_reopen_prepare  = qcow2_reopen_prepare,
+    .bdrv_reopen_commit   = qcow2_reopen_commit,
+    .bdrv_reopen_abort    = qcow2_reopen_abort,
     .bdrv_create        = qcow2_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
     .bdrv_co_get_block_status = qcow2_co_get_block_status,
@@ -2977,6 +3180,9 @@ BlockDriver bdrv_qcow2 = {
     .create_opts         = &qcow2_create_opts,
     .bdrv_check          = qcow2_check,
     .bdrv_amend_options  = qcow2_amend_options,
+
+    .bdrv_detach_aio_context  = qcow2_detach_aio_context,
+    .bdrv_attach_aio_context  = qcow2_attach_aio_context,
 };
 
 static void bdrv_qcow2_init(void)
diff --git a/block/qcow2.h b/block/qcow2.h
index 4b5a6afc8..b8c500b9d 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -26,7 +26,7 @@
 #define BLOCK_QCOW2_H
 
 #include "crypto/cipher.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 //#define DEBUG_ALLOC
 //#define DEBUG_ALLOC2
@@ -96,6 +96,7 @@
 #define QCOW2_OPT_CACHE_SIZE "cache-size"
 #define QCOW2_OPT_L2_CACHE_SIZE "l2-cache-size"
 #define QCOW2_OPT_REFCOUNT_CACHE_SIZE "refcount-cache-size"
+#define QCOW2_OPT_CACHE_CLEAN_INTERVAL "cache-clean-interval"
 
 typedef struct QCowHeader {
     uint32_t magic;
@@ -221,7 +222,7 @@ typedef uint64_t Qcow2GetRefcountFunc(const void *refcount_array,
 typedef void Qcow2SetRefcountFunc(void *refcount_array,
                                   uint64_t index, uint64_t value);
 
-typedef struct BDRVQcowState {
+typedef struct BDRVQcow2State {
     int cluster_bits;
     int cluster_size;
     int cluster_sectors;
@@ -239,6 +240,8 @@ typedef struct BDRVQcowState {
 
     Qcow2Cache* l2_table_cache;
     Qcow2Cache* refcount_block_cache;
+    QEMUTimer *cache_clean_timer;
+    unsigned cache_clean_interval;
 
     uint8_t *cluster_cache;
     uint8_t *cluster_data;
@@ -290,9 +293,7 @@ typedef struct BDRVQcowState {
      * override) */
     char *image_backing_file;
     char *image_backing_format;
-} BDRVQcowState;
-
-struct QCowAIOCB;
+} BDRVQcow2State;
 
 typedef struct Qcow2COWRegion {
     /**
@@ -402,28 +403,28 @@ typedef enum QCow2MetadataOverlap {
 
 #define REFT_OFFSET_MASK 0xfffffffffffffe00ULL
 
-static inline int64_t start_of_cluster(BDRVQcowState *s, int64_t offset)
+static inline int64_t start_of_cluster(BDRVQcow2State *s, int64_t offset)
 {
     return offset & ~(s->cluster_size - 1);
 }
 
-static inline int64_t offset_into_cluster(BDRVQcowState *s, int64_t offset)
+static inline int64_t offset_into_cluster(BDRVQcow2State *s, int64_t offset)
 {
     return offset & (s->cluster_size - 1);
 }
 
-static inline uint64_t size_to_clusters(BDRVQcowState *s, uint64_t size)
+static inline uint64_t size_to_clusters(BDRVQcow2State *s, uint64_t size)
 {
     return (size + (s->cluster_size - 1)) >> s->cluster_bits;
 }
 
-static inline int64_t size_to_l1(BDRVQcowState *s, int64_t size)
+static inline int64_t size_to_l1(BDRVQcow2State *s, int64_t size)
 {
     int shift = s->cluster_bits + s->l2_bits;
     return (size + (1ULL << shift) - 1) >> shift;
 }
 
-static inline int offset_to_l2_index(BDRVQcowState *s, int64_t offset)
+static inline int offset_to_l2_index(BDRVQcow2State *s, int64_t offset)
 {
     return (offset >> s->cluster_bits) & (s->l2_size - 1);
 }
@@ -434,12 +435,12 @@ static inline int64_t align_offset(int64_t offset, int n)
     return offset;
 }
 
-static inline int64_t qcow2_vm_state_offset(BDRVQcowState *s)
+static inline int64_t qcow2_vm_state_offset(BDRVQcow2State *s)
 {
     return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
 }
 
-static inline uint64_t qcow2_max_refcount_clusters(BDRVQcowState *s)
+static inline uint64_t qcow2_max_refcount_clusters(BDRVQcow2State *s)
 {
     return QCOW_MAX_REFTABLE_SIZE >> s->cluster_bits;
 }
@@ -458,7 +459,7 @@ static inline int qcow2_get_cluster_type(uint64_t l2_entry)
 }
 
 /* Check whether refcounts are eager or lazy */
-static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s)
+static inline bool qcow2_need_accurate_refcounts(BDRVQcow2State *s)
 {
     return !(s->incompatible_features & QCOW2_INCOMPAT_DIRTY);
 }
@@ -534,7 +535,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index);
 void qcow2_l2_cache_reset(BlockDriverState *bs);
 int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
-int qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+int qcow2_encrypt_sectors(BDRVQcow2State *s, int64_t sector_num,
                           uint8_t *out_buf, const uint8_t *in_buf,
                           int nb_sectors, bool enc, Error **errp);
 
@@ -581,6 +582,7 @@ int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
     Qcow2Cache *dependency);
 void qcow2_cache_depends_on_flush(Qcow2Cache *c);
 
+void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c);
 int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c);
 
 int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
diff --git a/block/qed-table.c b/block/qed-table.c
index 513aa872c..f4219b8ac 100644
--- a/block/qed-table.c
+++ b/block/qed-table.c
@@ -63,7 +63,7 @@ static void qed_read_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
     read_table_cb->iov.iov_len = s->header.cluster_size * s->header.table_size,
 
     qemu_iovec_init_external(qiov, &read_table_cb->iov, 1);
-    bdrv_aio_readv(s->bs->file, offset / BDRV_SECTOR_SIZE, qiov,
+    bdrv_aio_readv(s->bs->file->bs, offset / BDRV_SECTOR_SIZE, qiov,
                    qiov->size / BDRV_SECTOR_SIZE,
                    qed_read_table_cb, read_table_cb);
 }
@@ -152,7 +152,7 @@ static void qed_write_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
     /* Adjust for offset into table */
     offset += start * sizeof(uint64_t);
 
-    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_writev(s->bs->file->bs, offset / BDRV_SECTOR_SIZE,
                     &write_table_cb->qiov,
                     write_table_cb->qiov.size / BDRV_SECTOR_SIZE,
                     qed_write_table_cb, write_table_cb);
diff --git a/block/qed.c b/block/qed.c
index 954ed007c..9b8889503 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -82,7 +82,7 @@ int qed_write_header_sync(BDRVQEDState *s)
     int ret;
 
     qed_header_cpu_to_le(&s->header, &le);
-    ret = bdrv_pwrite(s->bs->file, 0, &le, sizeof(le));
+    ret = bdrv_pwrite(s->bs->file->bs, 0, &le, sizeof(le));
     if (ret != sizeof(le)) {
         return ret;
     }
@@ -119,7 +119,7 @@ static void qed_write_header_read_cb(void *opaque, int ret)
     /* Update header */
     qed_header_cpu_to_le(&s->header, (QEDHeader *)write_header_cb->buf);
 
-    bdrv_aio_writev(s->bs->file, 0, &write_header_cb->qiov,
+    bdrv_aio_writev(s->bs->file->bs, 0, &write_header_cb->qiov,
                     write_header_cb->nsectors, qed_write_header_cb,
                     write_header_cb);
 }
@@ -152,7 +152,7 @@ static void qed_write_header(BDRVQEDState *s, BlockCompletionFunc cb,
     write_header_cb->iov.iov_len = len;
     qemu_iovec_init_external(&write_header_cb->qiov, &write_header_cb->iov, 1);
 
-    bdrv_aio_readv(s->bs->file, 0, &write_header_cb->qiov, nsectors,
+    bdrv_aio_readv(s->bs->file->bs, 0, &write_header_cb->qiov, nsectors,
                    qed_write_header_read_cb, write_header_cb);
 }
 
@@ -354,12 +354,6 @@ static void qed_cancel_need_check_timer(BDRVQEDState *s)
     timer_del(s->need_check_timer);
 }
 
-static void bdrv_qed_rebind(BlockDriverState *bs)
-{
-    BDRVQEDState *s = bs->opaque;
-    s->bs = bs;
-}
-
 static void bdrv_qed_detach_aio_context(BlockDriverState *bs)
 {
     BDRVQEDState *s = bs->opaque;
@@ -381,6 +375,18 @@ static void bdrv_qed_attach_aio_context(BlockDriverState *bs,
     }
 }
 
+static void bdrv_qed_drain(BlockDriverState *bs)
+{
+    BDRVQEDState *s = bs->opaque;
+
+    /* Cancel timer and start doing I/O that were meant to happen as if it
+     * fired, that way we get bdrv_drain() taking care of the ongoing requests
+     * correctly. */
+    qed_cancel_need_check_timer(s);
+    qed_plug_allocating_write_reqs(s);
+    bdrv_aio_flush(s->bs, qed_clear_need_check, s);
+}
+
 static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
                          Error **errp)
 {
@@ -392,7 +398,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
     s->bs = bs;
     QSIMPLEQ_INIT(&s->allocating_write_reqs);
 
-    ret = bdrv_pread(bs->file, 0, &le_header, sizeof(le_header));
+    ret = bdrv_pread(bs->file->bs, 0, &le_header, sizeof(le_header));
     if (ret < 0) {
         return ret;
     }
@@ -416,7 +422,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* Round down file size to the last cluster */
-    file_size = bdrv_getlength(bs->file);
+    file_size = bdrv_getlength(bs->file->bs);
     if (file_size < 0) {
         return file_size;
     }
@@ -452,7 +458,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
             return -EINVAL;
         }
 
-        ret = qed_read_string(bs->file, s->header.backing_filename_offset,
+        ret = qed_read_string(bs->file->bs, s->header.backing_filename_offset,
                               s->header.backing_filename_size, bs->backing_file,
                               sizeof(bs->backing_file));
         if (ret < 0) {
@@ -471,7 +477,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
      * feature is no longer valid.
      */
     if ((s->header.autoclear_features & ~QED_AUTOCLEAR_FEATURE_MASK) != 0 &&
-        !bdrv_is_read_only(bs->file) && !(flags & BDRV_O_INCOMING)) {
+        !bdrv_is_read_only(bs->file->bs) && !(flags & BDRV_O_INCOMING)) {
         s->header.autoclear_features &= QED_AUTOCLEAR_FEATURE_MASK;
 
         ret = qed_write_header_sync(s);
@@ -480,7 +486,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
         }
 
         /* From here on only known autoclear feature bits are valid */
-        bdrv_flush(bs->file);
+        bdrv_flush(bs->file->bs);
     }
 
     s->l1_table = qed_alloc_table(s);
@@ -498,7 +504,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
          * potentially inconsistent images to be opened read-only.  This can
          * aid data recovery from an otherwise inconsistent image.
          */
-        if (!bdrv_is_read_only(bs->file) &&
+        if (!bdrv_is_read_only(bs->file->bs) &&
             !(flags & BDRV_O_INCOMING)) {
             BdrvCheckResult result = {0};
 
@@ -541,7 +547,7 @@ static void bdrv_qed_close(BlockDriverState *bs)
     bdrv_qed_detach_aio_context(bs);
 
     /* Ensure writes reach stable storage */
-    bdrv_flush(bs->file);
+    bdrv_flush(bs->file->bs);
 
     /* Clean shutdown, no check required on next open */
     if (s->header.features & QED_F_NEED_CHECK) {
@@ -583,7 +589,7 @@ static int qed_create(const char *filename, uint32_t cluster_size,
 
     bs = NULL;
     ret = bdrv_open(&bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_PROTOCOL, NULL,
+                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_PROTOCOL,
                     &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
@@ -772,8 +778,8 @@ static void qed_read_backing_file(BDRVQEDState *s, uint64_t pos,
     /* If there is a backing file, get its length.  Treat the absence of a
      * backing file like a zero length backing file.
      */
-    if (s->bs->backing_hd) {
-        int64_t l = bdrv_getlength(s->bs->backing_hd);
+    if (s->bs->backing) {
+        int64_t l = bdrv_getlength(s->bs->backing->bs);
         if (l < 0) {
             cb(opaque, l);
             return;
@@ -802,7 +808,7 @@ static void qed_read_backing_file(BDRVQEDState *s, uint64_t pos,
     qemu_iovec_concat(*backing_qiov, qiov, 0, size);
 
     BLKDBG_EVENT(s->bs->file, BLKDBG_READ_BACKING_AIO);
-    bdrv_aio_readv(s->bs->backing_hd, pos / BDRV_SECTOR_SIZE,
+    bdrv_aio_readv(s->bs->backing->bs, pos / BDRV_SECTOR_SIZE,
                    *backing_qiov, size / BDRV_SECTOR_SIZE, cb, opaque);
 }
 
@@ -839,7 +845,7 @@ static void qed_copy_from_backing_file_write(void *opaque, int ret)
     }
 
     BLKDBG_EVENT(s->bs->file, BLKDBG_COW_WRITE);
-    bdrv_aio_writev(s->bs->file, copy_cb->offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_writev(s->bs->file->bs, copy_cb->offset / BDRV_SECTOR_SIZE,
                     &copy_cb->qiov, copy_cb->qiov.size / BDRV_SECTOR_SIZE,
                     qed_copy_from_backing_file_cb, copy_cb);
 }
@@ -1055,7 +1061,7 @@ static void qed_aio_write_flush_before_l2_update(void *opaque, int ret)
     QEDAIOCB *acb = opaque;
     BDRVQEDState *s = acb_to_s(acb);
 
-    if (!bdrv_aio_flush(s->bs->file, qed_aio_write_l2_update_cb, opaque)) {
+    if (!bdrv_aio_flush(s->bs->file->bs, qed_aio_write_l2_update_cb, opaque)) {
         qed_aio_complete(acb, -EIO);
     }
 }
@@ -1081,7 +1087,7 @@ static void qed_aio_write_main(void *opaque, int ret)
     if (acb->find_cluster_ret == QED_CLUSTER_FOUND) {
         next_fn = qed_aio_next_io;
     } else {
-        if (s->bs->backing_hd) {
+        if (s->bs->backing) {
             next_fn = qed_aio_write_flush_before_l2_update;
         } else {
             next_fn = qed_aio_write_l2_update_cb;
@@ -1089,7 +1095,7 @@ static void qed_aio_write_main(void *opaque, int ret)
     }
 
     BLKDBG_EVENT(s->bs->file, BLKDBG_WRITE_AIO);
-    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_writev(s->bs->file->bs, offset / BDRV_SECTOR_SIZE,
                     &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
                     next_fn, acb);
 }
@@ -1139,7 +1145,7 @@ static void qed_aio_write_prefill(void *opaque, int ret)
 static bool qed_should_set_need_check(BDRVQEDState *s)
 {
     /* The flush before L2 update path ensures consistency */
-    if (s->bs->backing_hd) {
+    if (s->bs->backing) {
         return false;
     }
 
@@ -1321,7 +1327,7 @@ static void qed_aio_read_data(void *opaque, int ret,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
-    bdrv_aio_readv(bs->file, offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_readv(bs->file->bs, offset / BDRV_SECTOR_SIZE,
                    &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
                    qed_aio_next_io, acb);
     return;
@@ -1443,7 +1449,7 @@ static int coroutine_fn bdrv_qed_co_write_zeroes(BlockDriverState *bs,
     struct iovec iov;
 
     /* Refuse if there are untouched backing file sectors */
-    if (bs->backing_hd) {
+    if (bs->backing) {
         if (qed_offset_into_cluster(s, sector_num * BDRV_SECTOR_SIZE) != 0) {
             return -ENOTSUP;
         }
@@ -1580,7 +1586,7 @@ static int bdrv_qed_change_backing_file(BlockDriverState *bs,
     }
 
     /* Write new header */
-    ret = bdrv_pwrite_sync(bs->file, 0, buffer, buffer_len);
+    ret = bdrv_pwrite_sync(bs->file->bs, 0, buffer, buffer_len);
     g_free(buffer);
     if (ret == 0) {
         memcpy(&s->header, &new_header, sizeof(new_header));
@@ -1596,7 +1602,7 @@ static void bdrv_qed_invalidate_cache(BlockDriverState *bs, Error **errp)
 
     bdrv_qed_close(bs);
 
-    bdrv_invalidate_cache(bs->file, &local_err);
+    bdrv_invalidate_cache(bs->file->bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1664,7 +1670,6 @@ static BlockDriver bdrv_qed = {
     .supports_backing         = true,
 
     .bdrv_probe               = bdrv_qed_probe,
-    .bdrv_rebind              = bdrv_qed_rebind,
     .bdrv_open                = bdrv_qed_open,
     .bdrv_close               = bdrv_qed_close,
     .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
@@ -1683,6 +1688,7 @@ static BlockDriver bdrv_qed = {
     .bdrv_check               = bdrv_qed_check,
     .bdrv_detach_aio_context  = bdrv_qed_detach_aio_context,
     .bdrv_attach_aio_context  = bdrv_qed_attach_aio_context,
+    .bdrv_drain               = bdrv_qed_drain,
 };
 
 static void bdrv_qed_init(void)
diff --git a/block/quorum.c b/block/quorum.c
index 2f6c45f76..e640688eb 100644
--- a/block/quorum.c
+++ b/block/quorum.c
@@ -64,7 +64,7 @@ typedef struct QuorumVotes {
 
 /* the following structure holds the state of one quorum instance */
 typedef struct BDRVQuorumState {
-    BlockDriverState **bs; /* children BlockDriverStates */
+    BdrvChild **children;  /* children BlockDriverStates */
     int num_children;      /* children count */
     int threshold;         /* if less than threshold children reads gave the
                             * same result a quorum error occurs.
@@ -336,7 +336,7 @@ static void quorum_report_bad_versions(BDRVQuorumState *s,
             continue;
         }
         QLIST_FOREACH(item, &version->items, next) {
-            quorum_report_bad(acb, s->bs[item->index]->node_name, 0);
+            quorum_report_bad(acb, s->children[item->index]->bs->node_name, 0);
         }
     }
 }
@@ -369,8 +369,9 @@ static bool quorum_rewrite_bad_versions(BDRVQuorumState *s, QuorumAIOCB *acb,
             continue;
         }
         QLIST_FOREACH(item, &version->items, next) {
-            bdrv_aio_writev(s->bs[item->index], acb->sector_num, acb->qiov,
-                            acb->nb_sectors, quorum_rewrite_aio_cb, acb);
+            bdrv_aio_writev(s->children[item->index]->bs, acb->sector_num,
+                            acb->qiov, acb->nb_sectors, quorum_rewrite_aio_cb,
+                            acb);
         }
     }
 
@@ -639,14 +640,15 @@ static BlockAIOCB *read_quorum_children(QuorumAIOCB *acb)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        acb->qcrs[i].buf = qemu_blockalign(s->bs[i], acb->qiov->size);
+        acb->qcrs[i].buf = qemu_blockalign(s->children[i]->bs, acb->qiov->size);
         qemu_iovec_init(&acb->qcrs[i].qiov, acb->qiov->niov);
         qemu_iovec_clone(&acb->qcrs[i].qiov, acb->qiov, acb->qcrs[i].buf);
     }
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_aio_readv(s->bs[i], acb->sector_num, &acb->qcrs[i].qiov,
-                       acb->nb_sectors, quorum_aio_cb, &acb->qcrs[i]);
+        acb->qcrs[i].aiocb = bdrv_aio_readv(s->children[i]->bs, acb->sector_num,
+                                            &acb->qcrs[i].qiov, acb->nb_sectors,
+                                            quorum_aio_cb, &acb->qcrs[i]);
     }
 
     return &acb->common;
@@ -656,14 +658,15 @@ static BlockAIOCB *read_fifo_child(QuorumAIOCB *acb)
 {
     BDRVQuorumState *s = acb->common.bs->opaque;
 
-    acb->qcrs[acb->child_iter].buf = qemu_blockalign(s->bs[acb->child_iter],
-                                                     acb->qiov->size);
+    acb->qcrs[acb->child_iter].buf =
+        qemu_blockalign(s->children[acb->child_iter]->bs, acb->qiov->size);
     qemu_iovec_init(&acb->qcrs[acb->child_iter].qiov, acb->qiov->niov);
     qemu_iovec_clone(&acb->qcrs[acb->child_iter].qiov, acb->qiov,
                      acb->qcrs[acb->child_iter].buf);
-    bdrv_aio_readv(s->bs[acb->child_iter], acb->sector_num,
-                   &acb->qcrs[acb->child_iter].qiov, acb->nb_sectors,
-                   quorum_aio_cb, &acb->qcrs[acb->child_iter]);
+    acb->qcrs[acb->child_iter].aiocb =
+        bdrv_aio_readv(s->children[acb->child_iter]->bs, acb->sector_num,
+                       &acb->qcrs[acb->child_iter].qiov, acb->nb_sectors,
+                       quorum_aio_cb, &acb->qcrs[acb->child_iter]);
 
     return &acb->common;
 }
@@ -702,8 +705,8 @@ static BlockAIOCB *quorum_aio_writev(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        acb->qcrs[i].aiocb = bdrv_aio_writev(s->bs[i], sector_num, qiov,
-                                             nb_sectors, &quorum_aio_cb,
+        acb->qcrs[i].aiocb = bdrv_aio_writev(s->children[i]->bs, sector_num,
+                                             qiov, nb_sectors, &quorum_aio_cb,
                                              &acb->qcrs[i]);
     }
 
@@ -717,12 +720,12 @@ static int64_t quorum_getlength(BlockDriverState *bs)
     int i;
 
     /* check that all file have the same length */
-    result = bdrv_getlength(s->bs[0]);
+    result = bdrv_getlength(s->children[0]->bs);
     if (result < 0) {
         return result;
     }
     for (i = 1; i < s->num_children; i++) {
-        int64_t value = bdrv_getlength(s->bs[i]);
+        int64_t value = bdrv_getlength(s->children[i]->bs);
         if (value < 0) {
             return value;
         }
@@ -741,7 +744,7 @@ static void quorum_invalidate_cache(BlockDriverState *bs, Error **errp)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_invalidate_cache(s->bs[i], &local_err);
+        bdrv_invalidate_cache(s->children[i]->bs, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
@@ -762,7 +765,7 @@ static coroutine_fn int quorum_co_flush(BlockDriverState *bs)
     error_votes.compare = quorum_64bits_compare;
 
     for (i = 0; i < s->num_children; i++) {
-        result = bdrv_co_flush(s->bs[i]);
+        result = bdrv_co_flush(s->children[i]->bs);
         result_value.l = result;
         quorum_count_vote(&error_votes, &result_value, i);
     }
@@ -782,7 +785,7 @@ static bool quorum_recurse_is_first_non_filter(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bool perm = bdrv_recurse_is_first_non_filter(s->bs[i],
+        bool perm = bdrv_recurse_is_first_non_filter(s->children[i]->bs,
                                                      candidate);
         if (perm) {
             return true;
@@ -889,6 +892,12 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     s->threshold = qemu_opt_get_number(opts, QUORUM_OPT_VOTE_THRESHOLD, 0);
+    /* and validate it against s->num_children */
+    ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
+    if (ret < 0) {
+        goto exit;
+    }
+
     ret = parse_read_pattern(qemu_opt_get(opts, QUORUM_OPT_READ_PATTERN));
     if (ret < 0) {
         error_setg(&local_err, "Please set read-pattern as fifo or quorum");
@@ -897,12 +906,6 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
     s->read_pattern = ret;
 
     if (s->read_pattern == QUORUM_READ_PATTERN_QUORUM) {
-        /* and validate it against s->num_children */
-        ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
-        if (ret < 0) {
-            goto exit;
-        }
-
         /* is the driver in blkverify mode */
         if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false) &&
             s->num_children == 2 && s->threshold == 2) {
@@ -922,8 +925,8 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
-    /* allocate the children BlockDriverState array */
-    s->bs = g_new0(BlockDriverState *, s->num_children);
+    /* allocate the children array */
+    s->children = g_new0(BdrvChild *, s->num_children);
     opened = g_new0(bool, s->num_children);
 
     for (i = 0; i < s->num_children; i++) {
@@ -931,9 +934,10 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
         ret = snprintf(indexstr, 32, "children.%d", i);
         assert(ret < 32);
 
-        ret = bdrv_open_image(&s->bs[i], NULL, options, indexstr, bs,
-                              &child_format, false, &local_err);
-        if (ret < 0) {
+        s->children[i] = bdrv_open_child(NULL, options, indexstr, bs,
+                                         &child_format, false, &local_err);
+        if (local_err) {
+            ret = -EINVAL;
             goto close_exit;
         }
 
@@ -949,9 +953,9 @@ close_exit:
         if (!opened[i]) {
             continue;
         }
-        bdrv_unref(s->bs[i]);
+        bdrv_unref_child(bs, s->children[i]);
     }
-    g_free(s->bs);
+    g_free(s->children);
     g_free(opened);
 exit:
     qemu_opts_del(opts);
@@ -968,10 +972,10 @@ static void quorum_close(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_unref(s->bs[i]);
+        bdrv_unref_child(bs, s->children[i]);
     }
 
-    g_free(s->bs);
+    g_free(s->children);
 }
 
 static void quorum_detach_aio_context(BlockDriverState *bs)
@@ -980,7 +984,7 @@ static void quorum_detach_aio_context(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_detach_aio_context(s->bs[i]);
+        bdrv_detach_aio_context(s->children[i]->bs);
     }
 }
 
@@ -991,7 +995,7 @@ static void quorum_attach_aio_context(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_attach_aio_context(s->bs[i], new_context);
+        bdrv_attach_aio_context(s->children[i]->bs, new_context);
     }
 }
 
@@ -1003,16 +1007,17 @@ static void quorum_refresh_filename(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_refresh_filename(s->bs[i]);
-        if (!s->bs[i]->full_open_options) {
+        bdrv_refresh_filename(s->children[i]->bs);
+        if (!s->children[i]->bs->full_open_options) {
             return;
         }
     }
 
     children = qlist_new();
     for (i = 0; i < s->num_children; i++) {
-        QINCREF(s->bs[i]->full_open_options);
-        qlist_append_obj(children, QOBJECT(s->bs[i]->full_open_options));
+        QINCREF(s->children[i]->bs->full_open_options);
+        qlist_append_obj(children,
+                         QOBJECT(s->children[i]->bs->full_open_options));
     }
 
     opts = qdict_new();
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 855febed5..2fff1843c 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -127,11 +127,6 @@ do { \
 
 #define FTYPE_FILE   0
 #define FTYPE_CD     1
-#define FTYPE_FD     2
-
-/* if the FD is not accessed during that time (in ns), we try to
-   reopen it to see if the disk has been changed */
-#define FD_OPEN_TIMEOUT (1000000000)
 
 #define MAX_BLOCKSIZE	4096
 
@@ -141,13 +136,6 @@ typedef struct BDRVRawState {
     int open_flags;
     size_t buf_align;
 
-#if defined(__linux__)
-    /* linux floppy specific */
-    int64_t fd_open_time;
-    int64_t fd_error_time;
-    int fd_got_error;
-    int fd_media_changed;
-#endif
 #ifdef CONFIG_LINUX_AIO
     int use_aio;
     void *aio_ctx;
@@ -519,7 +507,16 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
                      "future QEMU versions.\n",
                      bs->filename);
     }
-#endif
+#else
+    if (bdrv_flags & BDRV_O_NATIVE_AIO) {
+        error_printf("WARNING: aio=native was specified for '%s', but "
+                     "is not supported in this build. Falling back to "
+                     "aio=threads.\n"
+                     "         This will become an error condition in "
+                     "future QEMU versions.\n",
+                     bs->filename);
+    }
+#endif /* !defined(CONFIG_LINUX_AIO) */
 
     s->has_discard = true;
     s->has_write_zeroes = true;
@@ -626,7 +623,7 @@ static int raw_reopen_prepare(BDRVReopenState *state,
     }
 #endif
 
-    if (s->type == FTYPE_FD || s->type == FTYPE_CD) {
+    if (s->type == FTYPE_CD) {
         raw_s->open_flags |= O_NONBLOCK;
     }
 
@@ -670,11 +667,17 @@ static int raw_reopen_prepare(BDRVReopenState *state,
 
     /* If we cannot use fcntl, or fcntl failed, fall back to qemu_open() */
     if (raw_s->fd == -1) {
-        assert(!(raw_s->open_flags & O_CREAT));
-        raw_s->fd = qemu_open(state->bs->filename, raw_s->open_flags);
-        if (raw_s->fd == -1) {
-            error_setg_errno(errp, errno, "Could not reopen file");
-            ret = -1;
+        const char *normalized_filename = state->bs->filename;
+        ret = raw_normalize_devicepath(&normalized_filename);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not normalize device path");
+        } else {
+            assert(!(raw_s->open_flags & O_CREAT));
+            raw_s->fd = qemu_open(normalized_filename, raw_s->open_flags);
+            if (raw_s->fd == -1) {
+                error_setg_errno(errp, errno, "Could not reopen file");
+                ret = -1;
+            }
         }
     }
 
@@ -780,7 +783,6 @@ static int hdev_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
 {
     BDRVRawState *s = bs->opaque;
     struct hd_geometry ioctl_geo = {0};
-    uint32_t blksize;
 
     /* If DASD, get its geometry */
     if (check_for_dasd(s->fd) < 0) {
@@ -800,12 +802,6 @@ static int hdev_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
     }
     geo->heads = ioctl_geo.heads;
     geo->sectors = ioctl_geo.sectors;
-    if (!probe_physical_blocksize(s->fd, &blksize)) {
-        /* overwrite cyls: HDIO_GETGEO result is incorrect for big drives */
-        geo->cylinders = bdrv_nb_sectors(bs) / (blksize / BDRV_SECTOR_SIZE)
-                                             / (geo->heads * geo->sectors);
-        return 0;
-    }
     geo->cylinders = ioctl_geo.cylinders;
 
     return 0;
@@ -1253,7 +1249,7 @@ static int aio_worker(void *arg)
         break;
     }
 
-    g_slice_free(RawPosixAIOData, aiocb);
+    g_free(aiocb);
     return ret;
 }
 
@@ -1261,7 +1257,7 @@ static int paio_submit_co(BlockDriverState *bs, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         int type)
 {
-    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+    RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
@@ -1286,7 +1282,7 @@ static BlockAIOCB *paio_submit(BlockDriverState *bs, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockCompletionFunc *cb, void *opaque, int type)
 {
-    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+    RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
@@ -1642,7 +1638,7 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
         goto out;
     }
 
-    fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
+    fd = qemu_open(filename, O_RDWR | O_CREAT | O_TRUNC | O_BINARY,
                    0644);
     if (fd < 0) {
         result = -errno;
@@ -1973,8 +1969,8 @@ BlockDriver bdrv_file = {
 
 #if defined(__APPLE__) && defined(__MACH__)
 static kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator );
-static kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize );
-
+static kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
+                                CFIndex maxPathSize, int flags);
 kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator )
 {
     kern_return_t       kernResult;
@@ -2001,7 +1997,8 @@ kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator )
     return kernResult;
 }
 
-kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize )
+kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
+                         CFIndex maxPathSize, int flags)
 {
     io_object_t     nextMedia;
     kern_return_t   kernResult = KERN_FAILURE;
@@ -2014,7 +2011,9 @@ kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex ma
         if ( bsdPathAsCFString ) {
             size_t devPathLength;
             strcpy( bsdPath, _PATH_DEV );
-            strcat( bsdPath, "r" );
+            if (flags & BDRV_O_NOCACHE) {
+                strcat(bsdPath, "r");
+            }
             devPathLength = strlen( bsdPath );
             if ( CFStringGetCString( bsdPathAsCFString, bsdPath + devPathLength, maxPathSize - devPathLength, kCFStringEncodingASCII ) ) {
                 kernResult = KERN_SUCCESS;
@@ -2126,8 +2125,8 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
         int fd;
 
         kernResult = FindEjectableCDMedia( &mediaIterator );
-        kernResult = GetBSDPath( mediaIterator, bsdPath, sizeof( bsdPath ) );
-
+        kernResult = GetBSDPath(mediaIterator, bsdPath, sizeof(bsdPath),
+                                flags);
         if ( bsdPath[ 0 ] != '\0' ) {
             strcat(bsdPath,"s0");
             /* some CDs don't have a partition 0 */
@@ -2172,53 +2171,6 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
 }
 
 #if defined(__linux__)
-/* Note: we do not have a reliable method to detect if the floppy is
-   present. The current method is to try to open the floppy at every
-   I/O and to keep it opened during a few hundreds of ms. */
-static int fd_open(BlockDriverState *bs)
-{
-    BDRVRawState *s = bs->opaque;
-    int last_media_present;
-
-    if (s->type != FTYPE_FD)
-        return 0;
-    last_media_present = (s->fd >= 0);
-    if (s->fd >= 0 &&
-        (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->fd_open_time) >= FD_OPEN_TIMEOUT) {
-        qemu_close(s->fd);
-        s->fd = -1;
-        DPRINTF("Floppy closed\n");
-    }
-    if (s->fd < 0) {
-        if (s->fd_got_error &&
-            (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->fd_error_time) < FD_OPEN_TIMEOUT) {
-            DPRINTF("No floppy (open delayed)\n");
-            return -EIO;
-        }
-        s->fd = qemu_open(bs->filename, s->open_flags & ~O_NONBLOCK);
-        if (s->fd < 0) {
-            s->fd_error_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-            s->fd_got_error = 1;
-            if (last_media_present)
-                s->fd_media_changed = 1;
-            DPRINTF("No floppy\n");
-            return -EIO;
-        }
-        DPRINTF("Floppy opened\n");
-    }
-    if (!last_media_present)
-        s->fd_media_changed = 1;
-    s->fd_open_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-    s->fd_got_error = 0;
-    return 0;
-}
-
-static int hdev_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    BDRVRawState *s = bs->opaque;
-
-    return ioctl(s->fd, req, buf);
-}
 
 static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
@@ -2231,7 +2183,7 @@ static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
     if (fd_open(bs) < 0)
         return NULL;
 
-    acb = g_slice_new(RawPosixAIOData);
+    acb = g_new(RawPosixAIOData, 1);
     acb->bs = bs;
     acb->aio_type = QEMU_AIO_IOCTL;
     acb->aio_fildes = s->fd;
@@ -2241,8 +2193,8 @@ static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
     pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
     return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque);
 }
+#endif /* linux */
 
-#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static int fd_open(BlockDriverState *bs)
 {
     BDRVRawState *s = bs->opaque;
@@ -2252,14 +2204,6 @@ static int fd_open(BlockDriverState *bs)
         return 0;
     return -EIO;
 }
-#else /* !linux && !FreeBSD */
-
-static int fd_open(BlockDriverState *bs)
-{
-    return 0;
-}
-
-#endif /* !linux && !FreeBSD */
 
 static coroutine_fn BlockAIOCB *hdev_aio_discard(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors,
@@ -2303,17 +2247,22 @@ static int hdev_create(const char *filename, QemuOpts *opts,
     int64_t total_size = 0;
     bool has_prefix;
 
-    /* This function is used by all three protocol block drivers and therefore
-     * any of these three prefixes may be given.
+    /* This function is used by both protocol block drivers and therefore either
+     * of these prefixes may be given.
      * The return value has to be stored somewhere, otherwise this is an error
      * due to -Werror=unused-value. */
     has_prefix =
         strstart(filename, "host_device:", &filename) ||
-        strstart(filename, "host_cdrom:" , &filename) ||
-        strstart(filename, "host_floppy:", &filename);
+        strstart(filename, "host_cdrom:" , &filename);
 
     (void)has_prefix;
 
+    ret = raw_normalize_devicepath(&filename);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not normalize device path");
+        return ret;
+    }
+
     /* Read out options */
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
                           BDRV_SECTOR_SIZE);
@@ -2379,160 +2328,10 @@ static BlockDriver bdrv_host_device = {
 
     /* generic scsi device */
 #ifdef __linux__
-    .bdrv_ioctl         = hdev_ioctl,
     .bdrv_aio_ioctl     = hdev_aio_ioctl,
 #endif
 };
 
-#ifdef __linux__
-static void floppy_parse_filename(const char *filename, QDict *options,
-                                  Error **errp)
-{
-    /* The prefix is optional, just as for "file". */
-    strstart(filename, "host_floppy:", &filename);
-
-    qdict_put_obj(options, "filename", QOBJECT(qstring_from_str(filename)));
-}
-
-static int floppy_open(BlockDriverState *bs, QDict *options, int flags,
-                       Error **errp)
-{
-    BDRVRawState *s = bs->opaque;
-    Error *local_err = NULL;
-    int ret;
-
-    s->type = FTYPE_FD;
-
-    /* open will not fail even if no floppy is inserted, so add O_NONBLOCK */
-    ret = raw_open_common(bs, options, flags, O_NONBLOCK, &local_err);
-    if (ret) {
-        if (local_err) {
-            error_propagate(errp, local_err);
-        }
-        return ret;
-    }
-
-    /* close fd so that we can reopen it as needed */
-    qemu_close(s->fd);
-    s->fd = -1;
-    s->fd_media_changed = 1;
-
-    error_report("Host floppy pass-through is deprecated");
-    error_printf("Support for it will be removed in a future release.\n");
-    return 0;
-}
-
-static int floppy_probe_device(const char *filename)
-{
-    int fd, ret;
-    int prio = 0;
-    struct floppy_struct fdparam;
-    struct stat st;
-
-    if (strstart(filename, "/dev/fd", NULL) &&
-        !strstart(filename, "/dev/fdset/", NULL) &&
-        !strstart(filename, "/dev/fd/", NULL)) {
-        prio = 50;
-    }
-
-    fd = qemu_open(filename, O_RDONLY | O_NONBLOCK);
-    if (fd < 0) {
-        goto out;
-    }
-    ret = fstat(fd, &st);
-    if (ret == -1 || !S_ISBLK(st.st_mode)) {
-        goto outc;
-    }
-
-    /* Attempt to detect via a floppy specific ioctl */
-    ret = ioctl(fd, FDGETPRM, &fdparam);
-    if (ret >= 0)
-        prio = 100;
-
-outc:
-    qemu_close(fd);
-out:
-    return prio;
-}
-
-
-static int floppy_is_inserted(BlockDriverState *bs)
-{
-    return fd_open(bs) >= 0;
-}
-
-static int floppy_media_changed(BlockDriverState *bs)
-{
-    BDRVRawState *s = bs->opaque;
-    int ret;
-
-    /*
-     * XXX: we do not have a true media changed indication.
-     * It does not work if the floppy is changed without trying to read it.
-     */
-    fd_open(bs);
-    ret = s->fd_media_changed;
-    s->fd_media_changed = 0;
-    DPRINTF("Floppy changed=%d\n", ret);
-    return ret;
-}
-
-static void floppy_eject(BlockDriverState *bs, bool eject_flag)
-{
-    BDRVRawState *s = bs->opaque;
-    int fd;
-
-    if (s->fd >= 0) {
-        qemu_close(s->fd);
-        s->fd = -1;
-    }
-    fd = qemu_open(bs->filename, s->open_flags | O_NONBLOCK);
-    if (fd >= 0) {
-        if (ioctl(fd, FDEJECT, 0) < 0)
-            perror("FDEJECT");
-        qemu_close(fd);
-    }
-}
-
-static BlockDriver bdrv_host_floppy = {
-    .format_name        = "host_floppy",
-    .protocol_name      = "host_floppy",
-    .instance_size      = sizeof(BDRVRawState),
-    .bdrv_needs_filename = true,
-    .bdrv_probe_device	= floppy_probe_device,
-    .bdrv_parse_filename = floppy_parse_filename,
-    .bdrv_file_open     = floppy_open,
-    .bdrv_close         = raw_close,
-    .bdrv_reopen_prepare = raw_reopen_prepare,
-    .bdrv_reopen_commit  = raw_reopen_commit,
-    .bdrv_reopen_abort   = raw_reopen_abort,
-    .bdrv_create         = hdev_create,
-    .create_opts         = &raw_create_opts,
-
-    .bdrv_aio_readv     = raw_aio_readv,
-    .bdrv_aio_writev    = raw_aio_writev,
-    .bdrv_aio_flush	= raw_aio_flush,
-    .bdrv_refresh_limits = raw_refresh_limits,
-    .bdrv_io_plug = raw_aio_plug,
-    .bdrv_io_unplug = raw_aio_unplug,
-    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
-
-    .bdrv_truncate      = raw_truncate,
-    .bdrv_getlength      = raw_getlength,
-    .has_variable_length = true,
-    .bdrv_get_allocated_file_size
-                        = raw_get_allocated_file_size,
-
-    .bdrv_detach_aio_context = raw_detach_aio_context,
-    .bdrv_attach_aio_context = raw_attach_aio_context,
-
-    /* removable device support */
-    .bdrv_is_inserted   = floppy_is_inserted,
-    .bdrv_media_changed = floppy_media_changed,
-    .bdrv_eject         = floppy_eject,
-};
-#endif
-
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static void cdrom_parse_filename(const char *filename, QDict *options,
                                  Error **errp)
@@ -2588,15 +2387,13 @@ out:
     return prio;
 }
 
-static int cdrom_is_inserted(BlockDriverState *bs)
+static bool cdrom_is_inserted(BlockDriverState *bs)
 {
     BDRVRawState *s = bs->opaque;
     int ret;
 
     ret = ioctl(s->fd, CDROM_DRIVE_STATUS, CDSL_CURRENT);
-    if (ret == CDS_DISC_OK)
-        return 1;
-    return 0;
+    return ret == CDS_DISC_OK;
 }
 
 static void cdrom_eject(BlockDriverState *bs, bool eject_flag)
@@ -2663,7 +2460,6 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_lock_medium   = cdrom_lock_medium,
 
     /* generic scsi device */
-    .bdrv_ioctl         = hdev_ioctl,
     .bdrv_aio_ioctl     = hdev_aio_ioctl,
 };
 #endif /* __linux__ */
@@ -2722,7 +2518,7 @@ static int cdrom_reopen(BlockDriverState *bs)
     return 0;
 }
 
-static int cdrom_is_inserted(BlockDriverState *bs)
+static bool cdrom_is_inserted(BlockDriverState *bs)
 {
     return raw_getlength(bs) > 0;
 }
@@ -2810,7 +2606,6 @@ static void bdrv_file_init(void)
     bdrv_register(&bdrv_file);
     bdrv_register(&bdrv_host_device);
 #ifdef __linux__
-    bdrv_register(&bdrv_host_floppy);
     bdrv_register(&bdrv_host_cdrom);
 #endif
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
diff --git a/block/raw-win32.c b/block/raw-win32.c
index 68f2338ac..2d0907a82 100644
--- a/block/raw-win32.c
+++ b/block/raw-win32.c
@@ -119,9 +119,9 @@ static int aio_worker(void *arg)
     case QEMU_AIO_WRITE:
         count = handle_aiocb_rw(aiocb);
         if (count == aiocb->aio_nbytes) {
-            count = 0;
+            ret = 0;
         } else {
-            count = -EINVAL;
+            ret = -EINVAL;
         }
         break;
     case QEMU_AIO_FLUSH:
@@ -135,7 +135,7 @@ static int aio_worker(void *arg)
         break;
     }
 
-    g_slice_free(RawWin32AIOData, aiocb);
+    g_free(aiocb);
     return ret;
 }
 
@@ -143,7 +143,7 @@ static BlockAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockCompletionFunc *cb, void *opaque, int type)
 {
-    RawWin32AIOData *acb = g_slice_new(RawWin32AIOData);
+    RawWin32AIOData *acb = g_new(RawWin32AIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
diff --git a/block/raw_bsd.c b/block/raw_bsd.c
index e3d2d0468..915d6fd0e 100644
--- a/block/raw_bsd.c
+++ b/block/raw_bsd.c
@@ -52,7 +52,7 @@ static int coroutine_fn raw_co_readv(BlockDriverState *bs, int64_t sector_num,
                                      int nb_sectors, QEMUIOVector *qiov)
 {
     BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
-    return bdrv_co_readv(bs->file, sector_num, nb_sectors, qiov);
+    return bdrv_co_readv(bs->file->bs, sector_num, nb_sectors, qiov);
 }
 
 static int coroutine_fn raw_co_writev(BlockDriverState *bs, int64_t sector_num,
@@ -75,7 +75,7 @@ static int coroutine_fn raw_co_writev(BlockDriverState *bs, int64_t sector_num,
             return 0;
         }
 
-        buf = qemu_try_blockalign(bs->file, 512);
+        buf = qemu_try_blockalign(bs->file->bs, 512);
         if (!buf) {
             ret = -ENOMEM;
             goto fail;
@@ -102,7 +102,7 @@ static int coroutine_fn raw_co_writev(BlockDriverState *bs, int64_t sector_num,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
-    ret = bdrv_co_writev(bs->file, sector_num, nb_sectors, qiov);
+    ret = bdrv_co_writev(bs->file->bs, sector_num, nb_sectors, qiov);
 
 fail:
     if (qiov == &local_qiov) {
@@ -125,58 +125,48 @@ static int coroutine_fn raw_co_write_zeroes(BlockDriverState *bs,
                                             int64_t sector_num, int nb_sectors,
                                             BdrvRequestFlags flags)
 {
-    return bdrv_co_write_zeroes(bs->file, sector_num, nb_sectors, flags);
+    return bdrv_co_write_zeroes(bs->file->bs, sector_num, nb_sectors, flags);
 }
 
 static int coroutine_fn raw_co_discard(BlockDriverState *bs,
                                        int64_t sector_num, int nb_sectors)
 {
-    return bdrv_co_discard(bs->file, sector_num, nb_sectors);
+    return bdrv_co_discard(bs->file->bs, sector_num, nb_sectors);
 }
 
 static int64_t raw_getlength(BlockDriverState *bs)
 {
-    return bdrv_getlength(bs->file);
+    return bdrv_getlength(bs->file->bs);
 }
 
 static int raw_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
-    return bdrv_get_info(bs->file, bdi);
+    return bdrv_get_info(bs->file->bs, bdi);
 }
 
 static void raw_refresh_limits(BlockDriverState *bs, Error **errp)
 {
-    bs->bl = bs->file->bl;
+    bs->bl = bs->file->bs->bl;
 }
 
 static int raw_truncate(BlockDriverState *bs, int64_t offset)
 {
-    return bdrv_truncate(bs->file, offset);
-}
-
-static int raw_is_inserted(BlockDriverState *bs)
-{
-    return bdrv_is_inserted(bs->file);
+    return bdrv_truncate(bs->file->bs, offset);
 }
 
 static int raw_media_changed(BlockDriverState *bs)
 {
-    return bdrv_media_changed(bs->file);
+    return bdrv_media_changed(bs->file->bs);
 }
 
 static void raw_eject(BlockDriverState *bs, bool eject_flag)
 {
-    bdrv_eject(bs->file, eject_flag);
+    bdrv_eject(bs->file->bs, eject_flag);
 }
 
 static void raw_lock_medium(BlockDriverState *bs, bool locked)
 {
-    bdrv_lock_medium(bs->file, locked);
-}
-
-static int raw_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    return bdrv_ioctl(bs->file, req, buf);
+    bdrv_lock_medium(bs->file->bs, locked);
 }
 
 static BlockAIOCB *raw_aio_ioctl(BlockDriverState *bs,
@@ -184,12 +174,12 @@ static BlockAIOCB *raw_aio_ioctl(BlockDriverState *bs,
                                  BlockCompletionFunc *cb,
                                  void *opaque)
 {
-    return bdrv_aio_ioctl(bs->file, req, buf, cb, opaque);
+    return bdrv_aio_ioctl(bs->file->bs, req, buf, cb, opaque);
 }
 
 static int raw_has_zero_init(BlockDriverState *bs)
 {
-    return bdrv_has_zero_init(bs->file);
+    return bdrv_has_zero_init(bs->file->bs);
 }
 
 static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
@@ -207,7 +197,7 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
 static int raw_open(BlockDriverState *bs, QDict *options, int flags,
                     Error **errp)
 {
-    bs->sg = bs->file->sg;
+    bs->sg = bs->file->bs->sg;
 
     if (bs->probed && !bdrv_is_read_only(bs)) {
         fprintf(stderr,
@@ -217,7 +207,7 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags,
                 "raw images, write operations on block 0 will be restricted.\n"
                 "         Specify the 'raw' format explicitly to remove the "
                 "restrictions.\n",
-                bs->file->filename);
+                bs->file->bs->filename);
     }
 
     return 0;
@@ -237,12 +227,12 @@ static int raw_probe(const uint8_t *buf, int buf_size, const char *filename)
 
 static int raw_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
 {
-    return bdrv_probe_blocksizes(bs->file, bsz);
+    return bdrv_probe_blocksizes(bs->file->bs, bsz);
 }
 
 static int raw_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
 {
-    return bdrv_probe_geometry(bs->file, geo);
+    return bdrv_probe_geometry(bs->file->bs, geo);
 }
 
 BlockDriver bdrv_raw = {
@@ -264,11 +254,9 @@ BlockDriver bdrv_raw = {
     .bdrv_refresh_limits  = &raw_refresh_limits,
     .bdrv_probe_blocksizes = &raw_probe_blocksizes,
     .bdrv_probe_geometry  = &raw_probe_geometry,
-    .bdrv_is_inserted     = &raw_is_inserted,
     .bdrv_media_changed   = &raw_media_changed,
     .bdrv_eject           = &raw_eject,
     .bdrv_lock_medium     = &raw_lock_medium,
-    .bdrv_ioctl           = &raw_ioctl,
     .bdrv_aio_ioctl       = &raw_aio_ioctl,
     .create_opts          = &raw_create_opts,
     .bdrv_has_zero_init   = &raw_has_zero_init
diff --git a/block/sheepdog.c b/block/sheepdog.c
index 9585beb73..d80e4ed18 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -28,7 +28,6 @@
 #define SD_OP_READ_OBJ       0x02
 #define SD_OP_WRITE_OBJ      0x03
 /* 0x04 is used internally by Sheepdog */
-#define SD_OP_DISCARD_OBJ    0x05
 
 #define SD_OP_NEW_VDI        0x11
 #define SD_OP_LOCK_VDI       0x12
@@ -318,7 +317,7 @@ enum AIOCBState {
     AIOCB_DISCARD_OBJ,
 };
 
-#define AIOCBOverwrapping(x, y)                                 \
+#define AIOCBOverlapping(x, y)                                 \
     (!(x->max_affect_data_idx < y->min_affect_data_idx          \
        || y->max_affect_data_idx < x->min_affect_data_idx))
 
@@ -342,6 +341,15 @@ struct SheepdogAIOCB {
     uint32_t min_affect_data_idx;
     uint32_t max_affect_data_idx;
 
+    /*
+     * The difference between affect_data_idx and dirty_data_idx:
+     * affect_data_idx represents range of index of all request types.
+     * dirty_data_idx represents range of index updated by COW requests.
+     * dirty_data_idx is used for updating an inode object.
+     */
+    uint32_t min_dirty_data_idx;
+    uint32_t max_dirty_data_idx;
+
     QLIST_ENTRY(SheepdogAIOCB) aiocb_siblings;
 };
 
@@ -351,9 +359,6 @@ typedef struct BDRVSheepdogState {
 
     SheepdogInode inode;
 
-    uint32_t min_dirty_data_idx;
-    uint32_t max_dirty_data_idx;
-
     char name[SD_MAX_VDI_LEN];
     bool is_snapshot;
     uint32_t cache_flags;
@@ -373,10 +378,15 @@ typedef struct BDRVSheepdogState {
     QLIST_HEAD(inflight_aio_head, AIOReq) inflight_aio_head;
     QLIST_HEAD(failed_aio_head, AIOReq) failed_aio_head;
 
-    CoQueue overwrapping_queue;
+    CoQueue overlapping_queue;
     QLIST_HEAD(inflight_aiocb_head, SheepdogAIOCB) inflight_aiocb_head;
 } BDRVSheepdogState;
 
+typedef struct BDRVSheepdogReopenState {
+    int fd;
+    int cache_flags;
+} BDRVSheepdogReopenState;
+
 static const char * sd_strerror(int err)
 {
     int i;
@@ -556,6 +566,9 @@ static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
     acb->max_affect_data_idx = (acb->sector_num * BDRV_SECTOR_SIZE +
                               acb->nb_sectors * BDRV_SECTOR_SIZE) / object_size;
 
+    acb->min_dirty_data_idx = UINT32_MAX;
+    acb->max_dirty_data_idx = 0;
+
     return acb;
 }
 
@@ -638,14 +651,16 @@ static coroutine_fn void do_co_req(void *opaque)
     unsigned int *rlen = srco->rlen;
 
     co = qemu_coroutine_self();
-    aio_set_fd_handler(srco->aio_context, sockfd, NULL, restart_co_req, co);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       NULL, restart_co_req, co);
 
     ret = send_co_req(sockfd, hdr, data, wlen);
     if (ret < 0) {
         goto out;
     }
 
-    aio_set_fd_handler(srco->aio_context, sockfd, restart_co_req, NULL, co);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       restart_co_req, NULL, co);
 
     ret = qemu_co_recv(sockfd, hdr, sizeof(*hdr));
     if (ret != sizeof(*hdr)) {
@@ -670,7 +685,8 @@ static coroutine_fn void do_co_req(void *opaque)
 out:
     /* there is at most one request for this sockfd, so it is safe to
      * set each handler to NULL. */
-    aio_set_fd_handler(srco->aio_context, sockfd, NULL, NULL, NULL);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       NULL, NULL, NULL);
 
     srco->ret = ret;
     srco->finished = true;
@@ -722,7 +738,8 @@ static coroutine_fn void reconnect_to_sdog(void *opaque)
     BDRVSheepdogState *s = opaque;
     AIOReq *aio_req, *next;
 
-    aio_set_fd_handler(s->aio_context, s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(s->aio_context, s->fd, false, NULL,
+                       NULL, NULL);
     close(s->fd);
     s->fd = -1;
 
@@ -819,8 +836,8 @@ static void coroutine_fn aio_read_response(void *opaque)
              */
             if (rsp.result == SD_RES_SUCCESS) {
                 s->inode.data_vdi_id[idx] = s->inode.vdi_id;
-                s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
-                s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
+                acb->max_dirty_data_idx = MAX(idx, acb->max_dirty_data_idx);
+                acb->min_dirty_data_idx = MIN(idx, acb->min_dirty_data_idx);
             }
         }
         break;
@@ -847,10 +864,6 @@ static void coroutine_fn aio_read_response(void *opaque)
             rsp.result = SD_RES_SUCCESS;
             s->discard_supported = false;
             break;
-        case SD_RES_SUCCESS:
-            idx = data_oid_to_idx(aio_req->oid);
-            s->inode.data_vdi_id[idx] = 0;
-            break;
         default:
             break;
         }
@@ -929,7 +942,8 @@ static int get_sheep_fd(BDRVSheepdogState *s, Error **errp)
         return fd;
     }
 
-    aio_set_fd_handler(s->aio_context, fd, co_read_response, NULL, s);
+    aio_set_fd_handler(s->aio_context, fd, false,
+                       co_read_response, NULL, s);
     return fd;
 }
 
@@ -1165,7 +1179,13 @@ static void coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
         hdr.flags = SD_FLAG_CMD_WRITE | flags;
         break;
     case AIOCB_DISCARD_OBJ:
-        hdr.opcode = SD_OP_DISCARD_OBJ;
+        hdr.opcode = SD_OP_WRITE_OBJ;
+        hdr.flags = SD_FLAG_CMD_WRITE | flags;
+        s->inode.data_vdi_id[data_oid_to_idx(oid)] = 0;
+        offset = offsetof(SheepdogInode,
+                          data_vdi_id[data_oid_to_idx(oid)]);
+        oid = vid_to_vdi_oid(s->inode.vdi_id);
+        wlen = datalen = sizeof(uint32_t);
         break;
     }
 
@@ -1184,7 +1204,7 @@ static void coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 
     qemu_co_mutex_lock(&s->lock);
     s->co_send = qemu_coroutine_self();
-    aio_set_fd_handler(s->aio_context, s->fd,
+    aio_set_fd_handler(s->aio_context, s->fd, false,
                        co_read_response, co_write_request, s);
     socket_set_cork(s->fd, 1);
 
@@ -1203,7 +1223,8 @@ static void coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
     }
 out:
     socket_set_cork(s->fd, 0);
-    aio_set_fd_handler(s->aio_context, s->fd, co_read_response, NULL, s);
+    aio_set_fd_handler(s->aio_context, s->fd, false,
+                       co_read_response, NULL, s);
     s->co_send = NULL;
     qemu_co_mutex_unlock(&s->lock);
 }
@@ -1353,7 +1374,8 @@ static void sd_detach_aio_context(BlockDriverState *bs)
 {
     BDRVSheepdogState *s = bs->opaque;
 
-    aio_set_fd_handler(s->aio_context, s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(s->aio_context, s->fd, false, NULL,
+                       NULL, NULL);
 }
 
 static void sd_attach_aio_context(BlockDriverState *bs,
@@ -1362,7 +1384,8 @@ static void sd_attach_aio_context(BlockDriverState *bs,
     BDRVSheepdogState *s = bs->opaque;
 
     s->aio_context = new_context;
-    aio_set_fd_handler(new_context, s->fd, co_read_response, NULL, s);
+    aio_set_fd_handler(new_context, s->fd, false,
+                       co_read_response, NULL, s);
 }
 
 /* TODO Convert to fine grained options */
@@ -1466,18 +1489,17 @@ static int sd_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     memcpy(&s->inode, buf, sizeof(s->inode));
-    s->min_dirty_data_idx = UINT32_MAX;
-    s->max_dirty_data_idx = 0;
 
     bs->total_sectors = s->inode.vdi_size / BDRV_SECTOR_SIZE;
     pstrcpy(s->name, sizeof(s->name), vdi);
     qemu_co_mutex_init(&s->lock);
-    qemu_co_queue_init(&s->overwrapping_queue);
+    qemu_co_queue_init(&s->overlapping_queue);
     qemu_opts_del(opts);
     g_free(buf);
     return 0;
 out:
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd,
+                       false, NULL, NULL, NULL);
     if (s->fd >= 0) {
         closesocket(s->fd);
     }
@@ -1486,6 +1508,70 @@ out:
     return ret;
 }
 
+static int sd_reopen_prepare(BDRVReopenState *state, BlockReopenQueue *queue,
+                             Error **errp)
+{
+    BDRVSheepdogState *s = state->bs->opaque;
+    BDRVSheepdogReopenState *re_s;
+    int ret = 0;
+
+    re_s = state->opaque = g_new0(BDRVSheepdogReopenState, 1);
+
+    re_s->cache_flags = SD_FLAG_CMD_CACHE;
+    if (state->flags & BDRV_O_NOCACHE) {
+        re_s->cache_flags = SD_FLAG_CMD_DIRECT;
+    }
+
+    re_s->fd = get_sheep_fd(s, errp);
+    if (re_s->fd < 0) {
+        ret = re_s->fd;
+        return ret;
+    }
+
+    return ret;
+}
+
+static void sd_reopen_commit(BDRVReopenState *state)
+{
+    BDRVSheepdogReopenState *re_s = state->opaque;
+    BDRVSheepdogState *s = state->bs->opaque;
+
+    if (s->fd) {
+        aio_set_fd_handler(s->aio_context, s->fd, false,
+                           NULL, NULL, NULL);
+        closesocket(s->fd);
+    }
+
+    s->fd = re_s->fd;
+    s->cache_flags = re_s->cache_flags;
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+
+    return;
+}
+
+static void sd_reopen_abort(BDRVReopenState *state)
+{
+    BDRVSheepdogReopenState *re_s = state->opaque;
+    BDRVSheepdogState *s = state->bs->opaque;
+
+    if (re_s == NULL) {
+        return;
+    }
+
+    if (re_s->fd) {
+        aio_set_fd_handler(s->aio_context, re_s->fd, false,
+                           NULL, NULL, NULL);
+        closesocket(re_s->fd);
+    }
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+
+    return;
+}
+
 static int do_sd_create(BDRVSheepdogState *s, uint32_t *vdi_id, int snapshot,
                         Error **errp)
 {
@@ -1554,7 +1640,7 @@ static int sd_prealloc(const char *filename, Error **errp)
     int ret;
 
     ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, errp);
+                    errp);
     if (ret < 0) {
         goto out_with_err_set;
     }
@@ -1746,8 +1832,7 @@ static int sd_create(const char *filename, QemuOpts *opts,
         }
 
         bs = NULL;
-        ret = bdrv_open(&bs, backing_file, NULL, NULL, BDRV_O_PROTOCOL, NULL,
-                        errp);
+        ret = bdrv_open(&bs, backing_file, NULL, NULL, BDRV_O_PROTOCOL, errp);
         if (ret < 0) {
             goto out;
         }
@@ -1861,7 +1946,8 @@ static void sd_close(BlockDriverState *bs)
         error_report("%s, %s", sd_strerror(rsp->result), s->name);
     }
 
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd,
+                       false, NULL, NULL, NULL);
     closesocket(s->fd);
     g_free(s->host_spec);
 }
@@ -1923,16 +2009,16 @@ static void coroutine_fn sd_write_done(SheepdogAIOCB *acb)
     AIOReq *aio_req;
     uint32_t offset, data_len, mn, mx;
 
-    mn = s->min_dirty_data_idx;
-    mx = s->max_dirty_data_idx;
+    mn = acb->min_dirty_data_idx;
+    mx = acb->max_dirty_data_idx;
     if (mn <= mx) {
         /* we need to update the vdi object. */
         offset = sizeof(s->inode) - sizeof(s->inode.data_vdi_id) +
             mn * sizeof(s->inode.data_vdi_id[0]);
         data_len = (mx - mn + 1) * sizeof(s->inode.data_vdi_id[0]);
 
-        s->min_dirty_data_idx = UINT32_MAX;
-        s->max_dirty_data_idx = 0;
+        acb->min_dirty_data_idx = UINT32_MAX;
+        acb->max_dirty_data_idx = 0;
 
         iov.iov_base = &s->inode;
         iov.iov_len = sizeof(s->inode);
@@ -2141,7 +2227,9 @@ static int coroutine_fn sd_co_rw_vector(void *p)
         }
 
         aio_req = alloc_aio_req(s, acb, oid, len, offset, flags, create,
-                                old_oid, done);
+                                old_oid,
+                                acb->aiocb_type == AIOCB_DISCARD_OBJ ?
+                                0 : done);
         QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
 
         add_aio_request(s, aio_req, acb->qiov->iov, acb->qiov->niov,
@@ -2158,12 +2246,12 @@ out:
     return 1;
 }
 
-static bool check_overwrapping_aiocb(BDRVSheepdogState *s, SheepdogAIOCB *aiocb)
+static bool check_overlapping_aiocb(BDRVSheepdogState *s, SheepdogAIOCB *aiocb)
 {
     SheepdogAIOCB *cb;
 
     QLIST_FOREACH(cb, &s->inflight_aiocb_head, aiocb_siblings) {
-        if (AIOCBOverwrapping(aiocb, cb)) {
+        if (AIOCBOverlapping(aiocb, cb)) {
             return true;
         }
     }
@@ -2192,15 +2280,15 @@ static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
     acb->aiocb_type = AIOCB_WRITE_UDATA;
 
 retry:
-    if (check_overwrapping_aiocb(s, acb)) {
-        qemu_co_queue_wait(&s->overwrapping_queue);
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
         goto retry;
     }
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
         QLIST_REMOVE(acb, aiocb_siblings);
-        qemu_co_queue_restart_all(&s->overwrapping_queue);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
@@ -2208,7 +2296,7 @@ retry:
     qemu_coroutine_yield();
 
     QLIST_REMOVE(acb, aiocb_siblings);
-    qemu_co_queue_restart_all(&s->overwrapping_queue);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
 
     return acb->ret;
 }
@@ -2225,15 +2313,15 @@ static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
     acb->aio_done_func = sd_finish_aiocb;
 
 retry:
-    if (check_overwrapping_aiocb(s, acb)) {
-        qemu_co_queue_wait(&s->overwrapping_queue);
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
         goto retry;
     }
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
         QLIST_REMOVE(acb, aiocb_siblings);
-        qemu_co_queue_restart_all(&s->overwrapping_queue);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
@@ -2241,7 +2329,7 @@ retry:
     qemu_coroutine_yield();
 
     QLIST_REMOVE(acb, aiocb_siblings);
-    qemu_co_queue_restart_all(&s->overwrapping_queue);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
     return acb->ret;
 }
 
@@ -2577,28 +2665,36 @@ static coroutine_fn int sd_co_discard(BlockDriverState *bs, int64_t sector_num,
                                       int nb_sectors)
 {
     SheepdogAIOCB *acb;
-    QEMUIOVector dummy;
     BDRVSheepdogState *s = bs->opaque;
     int ret;
+    QEMUIOVector discard_iov;
+    struct iovec iov;
+    uint32_t zero = 0;
 
     if (!s->discard_supported) {
             return 0;
     }
 
-    acb = sd_aio_setup(bs, &dummy, sector_num, nb_sectors);
+    memset(&discard_iov, 0, sizeof(discard_iov));
+    memset(&iov, 0, sizeof(iov));
+    iov.iov_base = &zero;
+    iov.iov_len = sizeof(zero);
+    discard_iov.iov = &iov;
+    discard_iov.niov = 1;
+    acb = sd_aio_setup(bs, &discard_iov, sector_num, nb_sectors);
     acb->aiocb_type = AIOCB_DISCARD_OBJ;
     acb->aio_done_func = sd_finish_aiocb;
 
 retry:
-    if (check_overwrapping_aiocb(s, acb)) {
-        qemu_co_queue_wait(&s->overwrapping_queue);
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
         goto retry;
     }
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
         QLIST_REMOVE(acb, aiocb_siblings);
-        qemu_co_queue_restart_all(&s->overwrapping_queue);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
@@ -2606,7 +2702,7 @@ retry:
     qemu_coroutine_yield();
 
     QLIST_REMOVE(acb, aiocb_siblings);
-    qemu_co_queue_restart_all(&s->overwrapping_queue);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
 
     return acb->ret;
 }
@@ -2703,6 +2799,9 @@ static BlockDriver bdrv_sheepdog = {
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -2736,6 +2835,9 @@ static BlockDriver bdrv_sheepdog_tcp = {
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -2769,6 +2871,9 @@ static BlockDriver bdrv_sheepdog_unix = {
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
diff --git a/block/snapshot.c b/block/snapshot.c
index 49e143e99..6e9fa8da9 100644
--- a/block/snapshot.c
+++ b/block/snapshot.c
@@ -149,7 +149,7 @@ int bdrv_can_snapshot(BlockDriverState *bs)
 
     if (!drv->bdrv_snapshot_create) {
         if (bs->file != NULL) {
-            return bdrv_can_snapshot(bs->file);
+            return bdrv_can_snapshot(bs->file->bs);
         }
         return 0;
     }
@@ -168,7 +168,7 @@ int bdrv_snapshot_create(BlockDriverState *bs,
         return drv->bdrv_snapshot_create(bs, sn_info);
     }
     if (bs->file) {
-        return bdrv_snapshot_create(bs->file, sn_info);
+        return bdrv_snapshot_create(bs->file->bs, sn_info);
     }
     return -ENOTSUP;
 }
@@ -188,10 +188,10 @@ int bdrv_snapshot_goto(BlockDriverState *bs,
 
     if (bs->file) {
         drv->bdrv_close(bs);
-        ret = bdrv_snapshot_goto(bs->file, snapshot_id);
+        ret = bdrv_snapshot_goto(bs->file->bs, snapshot_id);
         open_ret = drv->bdrv_open(bs, NULL, bs->open_flags, NULL);
         if (open_ret < 0) {
-            bdrv_unref(bs->file);
+            bdrv_unref(bs->file->bs);
             bs->drv = NULL;
             return open_ret;
         }
@@ -245,7 +245,7 @@ int bdrv_snapshot_delete(BlockDriverState *bs,
         return drv->bdrv_snapshot_delete(bs, snapshot_id, name, errp);
     }
     if (bs->file) {
-        return bdrv_snapshot_delete(bs->file, snapshot_id, name, errp);
+        return bdrv_snapshot_delete(bs->file->bs, snapshot_id, name, errp);
     }
     error_setg(errp, "Block format '%s' used by device '%s' "
                "does not support internal snapshot deletion",
@@ -253,9 +253,9 @@ int bdrv_snapshot_delete(BlockDriverState *bs,
     return -ENOTSUP;
 }
 
-void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
-                                        const char *id_or_name,
-                                        Error **errp)
+int bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                       const char *id_or_name,
+                                       Error **errp)
 {
     int ret;
     Error *local_err = NULL;
@@ -270,6 +270,7 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
     if (ret < 0) {
         error_propagate(errp, local_err);
     }
+    return ret;
 }
 
 int bdrv_snapshot_list(BlockDriverState *bs,
@@ -283,7 +284,7 @@ int bdrv_snapshot_list(BlockDriverState *bs,
         return drv->bdrv_snapshot_list(bs, psn_info);
     }
     if (bs->file) {
-        return bdrv_snapshot_list(bs->file, psn_info);
+        return bdrv_snapshot_list(bs->file->bs, psn_info);
     }
     return -ENOTSUP;
 }
@@ -356,3 +357,130 @@ int bdrv_snapshot_load_tmp_by_id_or_name(BlockDriverState *bs,
 
     return ret;
 }
+
+
+/* Group operations. All block drivers are involved.
+ * These functions will properly handle dataplane (take aio_context_acquire
+ * when appropriate for appropriate block drivers) */
+
+bool bdrv_all_can_snapshot(BlockDriverState **first_bad_bs)
+{
+    bool ok = true;
+    BlockDriverState *bs = NULL;
+
+    while (ok && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_is_inserted(bs) && !bdrv_is_read_only(bs)) {
+            ok = bdrv_can_snapshot(bs);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return ok;
+}
+
+int bdrv_all_delete_snapshot(const char *name, BlockDriverState **first_bad_bs,
+                             Error **err)
+{
+    int ret = 0;
+    BlockDriverState *bs = NULL;
+    QEMUSnapshotInfo sn1, *snapshot = &sn1;
+
+    while (ret == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs) &&
+                bdrv_snapshot_find(bs, snapshot, name) >= 0) {
+            ret = bdrv_snapshot_delete_by_id_or_name(bs, name, err);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return ret;
+}
+
+
+int bdrv_all_goto_snapshot(const char *name, BlockDriverState **first_bad_bs)
+{
+    int err = 0;
+    BlockDriverState *bs = NULL;
+
+    while (err == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            err = bdrv_snapshot_goto(bs, name);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return err;
+}
+
+int bdrv_all_find_snapshot(const char *name, BlockDriverState **first_bad_bs)
+{
+    QEMUSnapshotInfo sn;
+    int err = 0;
+    BlockDriverState *bs = NULL;
+
+    while (err == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            err = bdrv_snapshot_find(bs, &sn, name);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return err;
+}
+
+int bdrv_all_create_snapshot(QEMUSnapshotInfo *sn,
+                             BlockDriverState *vm_state_bs,
+                             uint64_t vm_state_size,
+                             BlockDriverState **first_bad_bs)
+{
+    int err = 0;
+    BlockDriverState *bs = NULL;
+
+    while (err == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bs == vm_state_bs) {
+            sn->vm_state_size = vm_state_size;
+            err = bdrv_snapshot_create(bs, sn);
+        } else if (bdrv_can_snapshot(bs)) {
+            sn->vm_state_size = 0;
+            err = bdrv_snapshot_create(bs, sn);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return err;
+}
+
+BlockDriverState *bdrv_all_find_vmstate_bs(void)
+{
+    bool not_found = true;
+    BlockDriverState *bs = NULL;
+
+    while (not_found && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        not_found = !bdrv_can_snapshot(bs);
+        aio_context_release(ctx);
+    }
+    return bs;
+}
diff --git a/block/ssh.c b/block/ssh.c
index 8d0673903..af025c08a 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -193,7 +193,7 @@ sftp_error_report(BDRVSSHState *s, const char *fs, ...)
 static int parse_uri(const char *filename, QDict *options, Error **errp)
 {
     URI *uri = NULL;
-    QueryParams *qp = NULL;
+    QueryParams *qp;
     int i;
 
     uri = uri_parse(filename);
@@ -249,9 +249,6 @@ static int parse_uri(const char *filename, QDict *options, Error **errp)
     return 0;
 
  err:
-    if (qp) {
-      query_params_free(qp);
-    }
     if (uri) {
       uri_free(uri);
     }
@@ -803,14 +800,15 @@ static coroutine_fn void set_fd_handler(BDRVSSHState *s, BlockDriverState *bs)
             rd_handler, wr_handler);
 
     aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock,
-                       rd_handler, wr_handler, co);
+                       false, rd_handler, wr_handler, co);
 }
 
 static coroutine_fn void clear_fd_handler(BDRVSSHState *s,
                                           BlockDriverState *bs)
 {
     DPRINTF("s->sock=%d", s->sock);
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock,
+                       false, NULL, NULL, NULL);
 }
 
 /* A non-blocking call returned EAGAIN, so yield, ensuring the
diff --git a/block/stream.c b/block/stream.c
index ab0bd057f..25af7eff6 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -16,6 +16,7 @@
 #include "block/blockjob.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "sysemu/block-backend.h"
 
 enum {
     /*
@@ -52,34 +53,6 @@ static int coroutine_fn stream_populate(BlockDriverState *bs,
     return bdrv_co_copy_on_readv(bs, sector_num, nb_sectors, &qiov);
 }
 
-static void close_unused_images(BlockDriverState *top, BlockDriverState *base,
-                                const char *base_id)
-{
-    BlockDriverState *intermediate;
-    intermediate = top->backing_hd;
-
-    /* Must assign before bdrv_delete() to prevent traversing dangling pointer
-     * while we delete backing image instances.
-     */
-    bdrv_set_backing_hd(top, base);
-
-    while (intermediate) {
-        BlockDriverState *unused;
-
-        /* reached base */
-        if (intermediate == base) {
-            break;
-        }
-
-        unused = intermediate;
-        intermediate = intermediate->backing_hd;
-        bdrv_set_backing_hd(unused, NULL);
-        bdrv_unref(unused);
-    }
-
-    bdrv_refresh_limits(top, NULL);
-}
-
 typedef struct {
     int ret;
     bool reached_end;
@@ -101,7 +74,7 @@ static void stream_complete(BlockJob *job, void *opaque)
             }
         }
         data->ret = bdrv_change_backing_file(job->bs, base_id, base_fmt);
-        close_unused_images(job->bs, base, base_id);
+        bdrv_set_backing_hd(job->bs, base);
     }
 
     g_free(s->backing_file_str);
@@ -121,7 +94,7 @@ static void coroutine_fn stream_run(void *opaque)
     int n = 0;
     void *buf;
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         block_job_completed(&s->common, 0);
         return;
     }
@@ -166,7 +139,7 @@ wait:
         } else if (ret >= 0) {
             /* Copy if allocated in the intermediate images.  Limit to the
              * known-unallocated area [sector_num, sector_num+n).  */
-            ret = bdrv_is_allocated_above(bs->backing_hd, base,
+            ret = bdrv_is_allocated_above(backing_bs(bs), base,
                                           sector_num, n, &n);
 
             /* Finish early if end of backing file has been reached */
@@ -250,7 +223,7 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
 
     if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
          on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, QERR_INVALID_PARAMETER, "on-error");
         return;
     }
diff --git a/block/throttle-groups.c b/block/throttle-groups.c
index 1abc6fcae..13b5baa5d 100644
--- a/block/throttle-groups.c
+++ b/block/throttle-groups.c
@@ -33,8 +33,7 @@
  * its own locking.
  *
  * This locking is however handled internally in this file, so it's
- * mostly transparent to outside users (but see the documentation in
- * throttle_groups_lock()).
+ * transparent to outside users.
  *
  * The whole ThrottleGroup structure is private and invisible to
  * outside users, that only use it through its ThrottleState.
@@ -76,9 +75,9 @@ static QTAILQ_HEAD(, ThrottleGroup) throttle_groups =
  * created.
  *
  * @name: the name of the ThrottleGroup
- * @ret:  the ThrottleGroup
+ * @ret:  the ThrottleState member of the ThrottleGroup
  */
-static ThrottleGroup *throttle_group_incref(const char *name)
+ThrottleState *throttle_group_incref(const char *name)
 {
     ThrottleGroup *tg = NULL;
     ThrottleGroup *iter;
@@ -108,7 +107,7 @@ static ThrottleGroup *throttle_group_incref(const char *name)
 
     qemu_mutex_unlock(&throttle_groups_lock);
 
-    return tg;
+    return &tg->ts;
 }
 
 /* Decrease the reference count of a ThrottleGroup.
@@ -116,10 +115,12 @@ static ThrottleGroup *throttle_group_incref(const char *name)
  * When the reference count reaches zero the ThrottleGroup is
  * destroyed.
  *
- * @tg:  The ThrottleGroup to unref
+ * @ts:  The ThrottleGroup to unref, given by its ThrottleState member
  */
-static void throttle_group_unref(ThrottleGroup *tg)
+void throttle_group_unref(ThrottleState *ts)
 {
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+
     qemu_mutex_lock(&throttle_groups_lock);
     if (--tg->refcount == 0) {
         QTAILQ_REMOVE(&throttle_groups, tg, list);
@@ -401,7 +402,8 @@ static void write_timer_cb(void *opaque)
 void throttle_group_register_bs(BlockDriverState *bs, const char *groupname)
 {
     int i;
-    ThrottleGroup *tg = throttle_group_incref(groupname);
+    ThrottleState *ts = throttle_group_incref(groupname);
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
     int clock_type = QEMU_CLOCK_REALTIME;
 
     if (qtest_enabled()) {
@@ -409,7 +411,7 @@ void throttle_group_register_bs(BlockDriverState *bs, const char *groupname)
         clock_type = QEMU_CLOCK_VIRTUAL;
     }
 
-    bs->throttle_state = &tg->ts;
+    bs->throttle_state = ts;
 
     qemu_mutex_lock(&tg->lock);
     /* If the ThrottleGroup is new set this BlockDriverState as the token */
@@ -435,6 +437,9 @@ void throttle_group_register_bs(BlockDriverState *bs, const char *groupname)
  * list, destroying the timers and setting the throttle_state pointer
  * to NULL.
  *
+ * The BlockDriverState must not have pending throttled requests, so
+ * the caller has to drain them first.
+ *
  * The group will be destroyed if it's empty after this operation.
  *
  * @bs: the BlockDriverState to remove
@@ -444,6 +449,10 @@ void throttle_group_unregister_bs(BlockDriverState *bs)
     ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
     int i;
 
+    assert(bs->pending_reqs[0] == 0 && bs->pending_reqs[1] == 0);
+    assert(qemu_co_queue_empty(&bs->throttled_reqs[0]));
+    assert(qemu_co_queue_empty(&bs->throttled_reqs[1]));
+
     qemu_mutex_lock(&tg->lock);
     for (i = 0; i < 2; i++) {
         if (tg->tokens[i] == bs) {
@@ -461,38 +470,10 @@ void throttle_group_unregister_bs(BlockDriverState *bs)
     throttle_timers_destroy(&bs->throttle_timers);
     qemu_mutex_unlock(&tg->lock);
 
-    throttle_group_unref(tg);
+    throttle_group_unref(&tg->ts);
     bs->throttle_state = NULL;
 }
 
-/* Acquire the lock of this throttling group.
- *
- * You won't normally need to use this. None of the functions from the
- * ThrottleGroup API require you to acquire the lock since all of them
- * deal with it internally.
- *
- * This should only be used in exceptional cases when you want to
- * access the protected fields of a BlockDriverState directly
- * (e.g. bdrv_swap()).
- *
- * @bs: a BlockDriverState that is member of the group
- */
-void throttle_group_lock(BlockDriverState *bs)
-{
-    ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
-    qemu_mutex_lock(&tg->lock);
-}
-
-/* Release the lock of this throttling group.
- *
- * See the comments in throttle_group_lock().
- */
-void throttle_group_unlock(BlockDriverState *bs)
-{
-    ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
-    qemu_mutex_unlock(&tg->lock);
-}
-
 static void throttle_groups_init(void)
 {
     qemu_mutex_init(&throttle_groups_lock);
diff --git a/block/vdi.c b/block/vdi.c
index 7642ef359..17f435fad 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -53,7 +53,7 @@
 #include "block/block_int.h"
 #include "qemu/module.h"
 #include "migration/migration.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
@@ -399,7 +399,7 @@ static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
 
     logout("\n");
 
-    ret = bdrv_read(bs->file, 0, (uint8_t *)&header, 1);
+    ret = bdrv_read(bs->file->bs, 0, (uint8_t *)&header, 1);
     if (ret < 0) {
         goto fail;
     }
@@ -490,13 +490,14 @@ static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
 
     bmap_size = header.blocks_in_image * sizeof(uint32_t);
     bmap_size = DIV_ROUND_UP(bmap_size, SECTOR_SIZE);
-    s->bmap = qemu_try_blockalign(bs->file, bmap_size * SECTOR_SIZE);
+    s->bmap = qemu_try_blockalign(bs->file->bs, bmap_size * SECTOR_SIZE);
     if (s->bmap == NULL) {
         ret = -ENOMEM;
         goto fail;
     }
 
-    ret = bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size);
+    ret = bdrv_read(bs->file->bs, s->bmap_sector, (uint8_t *)s->bmap,
+                    bmap_size);
     if (ret < 0) {
         goto fail_free_bmap;
     }
@@ -585,7 +586,7 @@ static int vdi_co_read(BlockDriverState *bs,
             uint64_t offset = s->header.offset_data / SECTOR_SIZE +
                               (uint64_t)bmap_entry * s->block_sectors +
                               sector_in_block;
-            ret = bdrv_read(bs->file, offset, buf, n_sectors);
+            ret = bdrv_read(bs->file->bs, offset, buf, n_sectors);
         }
         logout("%u sectors read\n", n_sectors);
 
@@ -653,7 +654,7 @@ static int vdi_co_write(BlockDriverState *bs,
              * acquire the lock and thus the padded cluster is written before
              * the other coroutines can write to the affected area. */
             qemu_co_mutex_lock(&s->write_lock);
-            ret = bdrv_write(bs->file, offset, block, s->block_sectors);
+            ret = bdrv_write(bs->file->bs, offset, block, s->block_sectors);
             qemu_co_mutex_unlock(&s->write_lock);
         } else {
             uint64_t offset = s->header.offset_data / SECTOR_SIZE +
@@ -669,7 +670,7 @@ static int vdi_co_write(BlockDriverState *bs,
              * that that write operation has returned (there may be other writes
              * in flight, but they do not concern this very operation). */
             qemu_co_mutex_unlock(&s->write_lock);
-            ret = bdrv_write(bs->file, offset, buf, n_sectors);
+            ret = bdrv_write(bs->file->bs, offset, buf, n_sectors);
         }
 
         nb_sectors -= n_sectors;
@@ -694,7 +695,7 @@ static int vdi_co_write(BlockDriverState *bs,
         assert(VDI_IS_ALLOCATED(bmap_first));
         *header = s->header;
         vdi_header_to_le(header);
-        ret = bdrv_write(bs->file, 0, block, 1);
+        ret = bdrv_write(bs->file->bs, 0, block, 1);
         g_free(block);
         block = NULL;
 
@@ -712,7 +713,7 @@ static int vdi_co_write(BlockDriverState *bs,
         base = ((uint8_t *)&s->bmap[0]) + bmap_first * SECTOR_SIZE;
         logout("will write %u block map sectors starting from entry %u\n",
                n_sectors, bmap_first);
-        ret = bdrv_write(bs->file, offset, base, n_sectors);
+        ret = bdrv_write(bs->file->bs, offset, base, n_sectors);
     }
 
     return ret;
@@ -764,7 +765,7 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
         goto exit;
     }
     ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
+                    &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto exit;
diff --git a/block/vhdx-log.c b/block/vhdx-log.c
index 47fec63c6..47ae4b135 100644
--- a/block/vhdx-log.c
+++ b/block/vhdx-log.c
@@ -81,7 +81,7 @@ static int vhdx_log_peek_hdr(BlockDriverState *bs, VHDXLogEntries *log,
 
     offset = log->offset + read;
 
-    ret = bdrv_pread(bs->file, offset, hdr, sizeof(VHDXLogEntryHeader));
+    ret = bdrv_pread(bs->file->bs, offset, hdr, sizeof(VHDXLogEntryHeader));
     if (ret < 0) {
         goto exit;
     }
@@ -141,7 +141,7 @@ static int vhdx_log_read_sectors(BlockDriverState *bs, VHDXLogEntries *log,
         }
         offset = log->offset + read;
 
-        ret = bdrv_pread(bs->file, offset, buffer, VHDX_LOG_SECTOR_SIZE);
+        ret = bdrv_pread(bs->file->bs, offset, buffer, VHDX_LOG_SECTOR_SIZE);
         if (ret < 0) {
             goto exit;
         }
@@ -191,7 +191,8 @@ static int vhdx_log_write_sectors(BlockDriverState *bs, VHDXLogEntries *log,
             /* full */
             break;
         }
-        ret = bdrv_pwrite(bs->file, offset, buffer_tmp, VHDX_LOG_SECTOR_SIZE);
+        ret = bdrv_pwrite(bs->file->bs, offset, buffer_tmp,
+                          VHDX_LOG_SECTOR_SIZE);
         if (ret < 0) {
             goto exit;
         }
@@ -353,7 +354,7 @@ static int vhdx_log_read_desc(BlockDriverState *bs, BDRVVHDXState *s,
     }
 
     desc_sectors = vhdx_compute_desc_sectors(hdr.descriptor_count);
-    desc_entries = qemu_try_blockalign(bs->file,
+    desc_entries = qemu_try_blockalign(bs->file->bs,
                                        desc_sectors * VHDX_LOG_SECTOR_SIZE);
     if (desc_entries == NULL) {
         ret = -ENOMEM;
@@ -462,7 +463,7 @@ static int vhdx_log_flush_desc(BlockDriverState *bs, VHDXLogDescriptor *desc,
 
     /* count is only > 1 if we are writing zeroes */
     for (i = 0; i < count; i++) {
-        ret = bdrv_pwrite_sync(bs->file, file_offset, buffer,
+        ret = bdrv_pwrite_sync(bs->file->bs, file_offset, buffer,
                                VHDX_LOG_SECTOR_SIZE);
         if (ret < 0) {
             goto exit;
@@ -509,7 +510,7 @@ static int vhdx_log_flush(BlockDriverState *bs, BDRVVHDXState *s,
         /* if the log shows a FlushedFileOffset larger than our current file
          * size, then that means the file has been truncated / corrupted, and
          * we must refused to open it / use it */
-        if (hdr_tmp.flushed_file_offset > bdrv_getlength(bs->file)) {
+        if (hdr_tmp.flushed_file_offset > bdrv_getlength(bs->file->bs)) {
             ret = -EINVAL;
             goto exit;
         }
@@ -539,12 +540,12 @@ static int vhdx_log_flush(BlockDriverState *bs, BDRVVHDXState *s,
                 goto exit;
             }
         }
-        if (bdrv_getlength(bs->file) < desc_entries->hdr.last_file_offset) {
+        if (bdrv_getlength(bs->file->bs) < desc_entries->hdr.last_file_offset) {
             new_file_size = desc_entries->hdr.last_file_offset;
             if (new_file_size % (1024*1024)) {
                 /* round up to nearest 1MB boundary */
                 new_file_size = ((new_file_size >> 20) + 1) << 20;
-                bdrv_truncate(bs->file, new_file_size);
+                bdrv_truncate(bs->file->bs, new_file_size);
             }
         }
         qemu_vfree(desc_entries);
@@ -908,8 +909,8 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
                 .sequence_number     = s->log.sequence,
                 .descriptor_count    = sectors,
                 .reserved            = 0,
-                .flushed_file_offset = bdrv_getlength(bs->file),
-                .last_file_offset    = bdrv_getlength(bs->file),
+                .flushed_file_offset = bdrv_getlength(bs->file->bs),
+                .last_file_offset    = bdrv_getlength(bs->file->bs),
               };
 
     new_hdr.log_guid = header->log_guid;
@@ -940,7 +941,7 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
 
         if (i == 0 && leading_length) {
             /* partial sector at the front of the buffer */
-            ret = bdrv_pread(bs->file, file_offset, merged_sector,
+            ret = bdrv_pread(bs->file->bs, file_offset, merged_sector,
                              VHDX_LOG_SECTOR_SIZE);
             if (ret < 0) {
                 goto exit;
@@ -950,7 +951,7 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
             sector_write = merged_sector;
         } else if (i == sectors - 1 && trailing_length) {
             /* partial sector at the end of the buffer */
-            ret = bdrv_pread(bs->file,
+            ret = bdrv_pread(bs->file->bs,
                             file_offset,
                             merged_sector + trailing_length,
                             VHDX_LOG_SECTOR_SIZE - trailing_length);
diff --git a/block/vhdx.c b/block/vhdx.c
index 0776de717..2fe9a5e0c 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -375,7 +375,7 @@ static int vhdx_update_header(BlockDriverState *bs, BDRVVHDXState *s,
         inactive_header->log_guid = *log_guid;
     }
 
-    ret = vhdx_write_header(bs->file, inactive_header, header_offset, true);
+    ret = vhdx_write_header(bs->file->bs, inactive_header, header_offset, true);
     if (ret < 0) {
         goto exit;
     }
@@ -427,7 +427,8 @@ static void vhdx_parse_header(BlockDriverState *bs, BDRVVHDXState *s,
     /* We have to read the whole VHDX_HEADER_SIZE instead of
      * sizeof(VHDXHeader), because the checksum is over the whole
      * region */
-    ret = bdrv_pread(bs->file, VHDX_HEADER1_OFFSET, buffer, VHDX_HEADER_SIZE);
+    ret = bdrv_pread(bs->file->bs, VHDX_HEADER1_OFFSET, buffer,
+                     VHDX_HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
@@ -443,7 +444,8 @@ static void vhdx_parse_header(BlockDriverState *bs, BDRVVHDXState *s,
         }
     }
 
-    ret = bdrv_pread(bs->file, VHDX_HEADER2_OFFSET, buffer, VHDX_HEADER_SIZE);
+    ret = bdrv_pread(bs->file->bs, VHDX_HEADER2_OFFSET, buffer,
+                     VHDX_HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
@@ -516,7 +518,7 @@ static int vhdx_open_region_tables(BlockDriverState *bs, BDRVVHDXState *s)
      * whole block */
     buffer = qemu_blockalign(bs, VHDX_HEADER_BLOCK_SIZE);
 
-    ret = bdrv_pread(bs->file, VHDX_REGION_TABLE_OFFSET, buffer,
+    ret = bdrv_pread(bs->file->bs, VHDX_REGION_TABLE_OFFSET, buffer,
                      VHDX_HEADER_BLOCK_SIZE);
     if (ret < 0) {
         goto fail;
@@ -629,7 +631,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
 
     buffer = qemu_blockalign(bs, VHDX_METADATA_TABLE_MAX_SIZE);
 
-    ret = bdrv_pread(bs->file, s->metadata_rt.file_offset, buffer,
+    ret = bdrv_pread(bs->file->bs, s->metadata_rt.file_offset, buffer,
                      VHDX_METADATA_TABLE_MAX_SIZE);
     if (ret < 0) {
         goto exit;
@@ -732,7 +734,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
         goto exit;
     }
 
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.file_parameters_entry.offset
                                          + s->metadata_rt.file_offset,
                      &s->params,
@@ -767,7 +769,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
     /* determine virtual disk size, logical sector size,
      * and phys sector size */
 
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.virtual_disk_size_entry.offset
                                            + s->metadata_rt.file_offset,
                      &s->virtual_disk_size,
@@ -775,7 +777,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
     if (ret < 0) {
         goto exit;
     }
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.logical_sector_size_entry.offset
                                              + s->metadata_rt.file_offset,
                      &s->logical_sector_size,
@@ -783,7 +785,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
     if (ret < 0) {
         goto exit;
     }
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.phys_sector_size_entry.offset
                                           + s->metadata_rt.file_offset,
                      &s->physical_sector_size,
@@ -906,7 +908,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags,
     QLIST_INIT(&s->regions);
 
     /* validate the file signature */
-    ret = bdrv_pread(bs->file, 0, &signature, sizeof(uint64_t));
+    ret = bdrv_pread(bs->file->bs, 0, &signature, sizeof(uint64_t));
     if (ret < 0) {
         goto fail;
     }
@@ -959,13 +961,13 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* s->bat is freed in vhdx_close() */
-    s->bat = qemu_try_blockalign(bs->file, s->bat_rt.length);
+    s->bat = qemu_try_blockalign(bs->file->bs, s->bat_rt.length);
     if (s->bat == NULL) {
         ret = -ENOMEM;
         goto fail;
     }
 
-    ret = bdrv_pread(bs->file, s->bat_offset, s->bat, s->bat_rt.length);
+    ret = bdrv_pread(bs->file->bs, s->bat_offset, s->bat, s->bat_rt.length);
     if (ret < 0) {
         goto fail;
     }
@@ -1118,7 +1120,7 @@ static coroutine_fn int vhdx_co_readv(BlockDriverState *bs, int64_t sector_num,
                 break;
             case PAYLOAD_BLOCK_FULLY_PRESENT:
                 qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_readv(bs->file,
+                ret = bdrv_co_readv(bs->file->bs,
                                     sinfo.file_offset >> BDRV_SECTOR_BITS,
                                     sinfo.sectors_avail, &hd_qiov);
                 qemu_co_mutex_lock(&s->lock);
@@ -1156,12 +1158,12 @@ exit:
 static int vhdx_allocate_block(BlockDriverState *bs, BDRVVHDXState *s,
                                     uint64_t *new_offset)
 {
-    *new_offset = bdrv_getlength(bs->file);
+    *new_offset = bdrv_getlength(bs->file->bs);
 
     /* per the spec, the address for a block is in units of 1MB */
     *new_offset = ROUND_UP(*new_offset, 1024 * 1024);
 
-    return bdrv_truncate(bs->file, *new_offset + s->block_size);
+    return bdrv_truncate(bs->file->bs, *new_offset + s->block_size);
 }
 
 /*
@@ -1260,7 +1262,7 @@ static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num,
                 /* Queue another write of zero buffers if the underlying file
                  * does not zero-fill on file extension */
 
-                if (bdrv_has_zero_init(bs->file) == 0) {
+                if (bdrv_has_zero_init(bs->file->bs) == 0) {
                     use_zero_buffers = true;
 
                     /* zero fill the front, if any */
@@ -1327,7 +1329,7 @@ static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num,
                 }
                 /* block exists, so we can just overwrite it */
                 qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_writev(bs->file,
+                ret = bdrv_co_writev(bs->file->bs,
                                     sinfo.file_offset >> BDRV_SECTOR_BITS,
                                     sectors_to_write, &hd_qiov);
                 qemu_co_mutex_lock(&s->lock);
@@ -1454,7 +1456,7 @@ static int vhdx_create_new_metadata(BlockDriverState *bs,
     uint32_t offset = 0;
     void *buffer = NULL;
     void *entry_buffer;
-    VHDXMetadataTableHeader *md_table;;
+    VHDXMetadataTableHeader *md_table;
     VHDXMetadataTableEntry  *md_table_entry;
 
     /* Metadata entries */
@@ -1842,7 +1844,7 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
 
     bs = NULL;
     ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
+                    &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto exit;
diff --git a/block/vmdk.c b/block/vmdk.c
index fbaab67c8..e46271a80 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -87,7 +87,7 @@ typedef struct {
 #define L2_CACHE_SIZE 16
 
 typedef struct VmdkExtent {
-    BlockDriverState *file;
+    BdrvChild *file;
     bool flat;
     bool compressed;
     bool has_marker;
@@ -222,7 +222,7 @@ static void vmdk_free_extents(BlockDriverState *bs)
         g_free(e->l1_backup_table);
         g_free(e->type);
         if (e->file != bs->file) {
-            bdrv_unref(e->file);
+            bdrv_unref_child(bs, e->file);
         }
     }
     g_free(s->extents);
@@ -248,7 +248,7 @@ static uint32_t vmdk_read_cid(BlockDriverState *bs, int parent)
     BDRVVmdkState *s = bs->opaque;
     int ret;
 
-    ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
+    ret = bdrv_pread(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
         return 0;
     }
@@ -278,7 +278,7 @@ static int vmdk_write_cid(BlockDriverState *bs, uint32_t cid)
     BDRVVmdkState *s = bs->opaque;
     int ret;
 
-    ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
+    ret = bdrv_pread(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
         return ret;
     }
@@ -297,7 +297,7 @@ static int vmdk_write_cid(BlockDriverState *bs, uint32_t cid)
         pstrcat(desc, sizeof(desc), tmp_desc);
     }
 
-    ret = bdrv_pwrite_sync(bs->file, s->desc_offset, desc, DESC_SIZE);
+    ret = bdrv_pwrite_sync(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
         return ret;
     }
@@ -308,10 +308,11 @@ static int vmdk_write_cid(BlockDriverState *bs, uint32_t cid)
 static int vmdk_is_cid_valid(BlockDriverState *bs)
 {
     BDRVVmdkState *s = bs->opaque;
-    BlockDriverState *p_bs = bs->backing_hd;
     uint32_t cur_pcid;
 
-    if (!s->cid_checked && p_bs) {
+    if (!s->cid_checked && bs->backing) {
+        BlockDriverState *p_bs = bs->backing->bs;
+
         cur_pcid = vmdk_read_cid(p_bs, 0);
         if (s->parent_cid != cur_pcid) {
             /* CID not valid */
@@ -340,7 +341,7 @@ static int vmdk_parent_open(BlockDriverState *bs)
     int ret;
 
     desc[DESC_SIZE] = '\0';
-    ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
+    ret = bdrv_pread(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
         return ret;
     }
@@ -367,7 +368,7 @@ static int vmdk_parent_open(BlockDriverState *bs)
 /* Create and append extent to the extent array. Return the added VmdkExtent
  * address. return NULL if allocation failed. */
 static int vmdk_add_extent(BlockDriverState *bs,
-                           BlockDriverState *file, bool flat, int64_t sectors,
+                           BdrvChild *file, bool flat, int64_t sectors,
                            int64_t l1_offset, int64_t l1_backup_offset,
                            uint32_t l1_size,
                            int l2_size, uint64_t cluster_sectors,
@@ -392,7 +393,7 @@ static int vmdk_add_extent(BlockDriverState *bs,
         return -EFBIG;
     }
 
-    nb_sectors = bdrv_nb_sectors(file);
+    nb_sectors = bdrv_nb_sectors(file->bs);
     if (nb_sectors < 0) {
         return nb_sectors;
     }
@@ -439,14 +440,14 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(extent->file,
+    ret = bdrv_pread(extent->file->bs,
                      extent->l1_table_offset,
                      extent->l1_table,
                      l1_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read l1 table from extent '%s'",
-                         extent->file->filename);
+                         extent->file->bs->filename);
         goto fail_l1;
     }
     for (i = 0; i < extent->l1_size; i++) {
@@ -459,14 +460,14 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
             ret = -ENOMEM;
             goto fail_l1;
         }
-        ret = bdrv_pread(extent->file,
+        ret = bdrv_pread(extent->file->bs,
                          extent->l1_backup_table_offset,
                          extent->l1_backup_table,
                          l1_size);
         if (ret < 0) {
             error_setg_errno(errp, -ret,
                              "Could not read l1 backup table from extent '%s'",
-                             extent->file->filename);
+                             extent->file->bs->filename);
             goto fail_l1b;
         }
         for (i = 0; i < extent->l1_size; i++) {
@@ -485,7 +486,7 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
 }
 
 static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
-                                 BlockDriverState *file,
+                                 BdrvChild *file,
                                  int flags, Error **errp)
 {
     int ret;
@@ -493,11 +494,11 @@ static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
     VMDK3Header header;
     VmdkExtent *extent;
 
-    ret = bdrv_pread(file, sizeof(magic), &header, sizeof(header));
+    ret = bdrv_pread(file->bs, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read header from file '%s'",
-                         file->filename);
+                         file->bs->filename);
         return ret;
     }
     ret = vmdk_add_extent(bs, file, false,
@@ -559,7 +560,7 @@ static char *vmdk_read_desc(BlockDriverState *file, uint64_t desc_offset,
 }
 
 static int vmdk_open_vmdk4(BlockDriverState *bs,
-                           BlockDriverState *file,
+                           BdrvChild *file,
                            int flags, QDict *options, Error **errp)
 {
     int ret;
@@ -569,18 +570,19 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
     VmdkExtent *extent;
     BDRVVmdkState *s = bs->opaque;
     int64_t l1_backup_offset = 0;
+    bool compressed;
 
-    ret = bdrv_pread(file, sizeof(magic), &header, sizeof(header));
+    ret = bdrv_pread(file->bs, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read header from file '%s'",
-                         file->filename);
+                         file->bs->filename);
         return -EINVAL;
     }
     if (header.capacity == 0) {
         uint64_t desc_offset = le64_to_cpu(header.desc_offset);
         if (desc_offset) {
-            char *buf = vmdk_read_desc(file, desc_offset << 9, errp);
+            char *buf = vmdk_read_desc(file->bs, desc_offset << 9, errp);
             if (!buf) {
                 return -EINVAL;
             }
@@ -620,8 +622,8 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
             } QEMU_PACKED eos_marker;
         } QEMU_PACKED footer;
 
-        ret = bdrv_pread(file,
-            bs->file->total_sectors * 512 - 1536,
+        ret = bdrv_pread(file->bs,
+            bs->file->bs->total_sectors * 512 - 1536,
             &footer, sizeof(footer));
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Failed to read footer");
@@ -643,6 +645,8 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
         header = footer.header;
     }
 
+    compressed =
+        le16_to_cpu(header.compressAlgorithm) == VMDK4_COMPRESSION_DEFLATE;
     if (le32_to_cpu(header.version) > 3) {
         char buf[64];
         snprintf(buf, sizeof(buf), "VMDK version %" PRId32,
@@ -650,7 +654,8 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
         error_setg(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
                    bdrv_get_device_or_node_name(bs), "vmdk", buf);
         return -ENOTSUP;
-    } else if (le32_to_cpu(header.version) == 3 && (flags & BDRV_O_RDWR)) {
+    } else if (le32_to_cpu(header.version) == 3 && (flags & BDRV_O_RDWR) &&
+               !compressed) {
         /* VMware KB 2064959 explains that version 3 added support for
          * persistent changed block tracking (CBT), and backup software can
          * read it as version=1 if it doesn't care about the changed area
@@ -675,7 +680,7 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
     if (le32_to_cpu(header.flags) & VMDK4_FLAG_RGD) {
         l1_backup_offset = le64_to_cpu(header.rgd_offset) << 9;
     }
-    if (bdrv_nb_sectors(file) < le64_to_cpu(header.grain_offset)) {
+    if (bdrv_nb_sectors(file->bs) < le64_to_cpu(header.grain_offset)) {
         error_setg(errp, "File truncated, expecting at least %" PRId64 " bytes",
                    (int64_t)(le64_to_cpu(header.grain_offset)
                              * BDRV_SECTOR_SIZE));
@@ -739,8 +744,7 @@ static int vmdk_parse_description(const char *desc, const char *opt_name,
 }
 
 /* Open an extent file and append to bs array */
-static int vmdk_open_sparse(BlockDriverState *bs,
-                            BlockDriverState *file, int flags,
+static int vmdk_open_sparse(BlockDriverState *bs, BdrvChild *file, int flags,
                             char *buf, QDict *options, Error **errp)
 {
     uint32_t magic;
@@ -773,10 +777,11 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
     int64_t sectors = 0;
     int64_t flat_offset;
     char *extent_path;
-    BlockDriverState *extent_file;
+    BdrvChild *extent_file;
     BDRVVmdkState *s = bs->opaque;
     VmdkExtent *extent;
     char extent_opt_prefix[32];
+    Error *local_err = NULL;
 
     while (*p) {
         /* parse extent line in one of below formats:
@@ -819,22 +824,22 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             !desc_file_path[0])
         {
             error_setg(errp, "Cannot use relative extent paths with VMDK "
-                       "descriptor file '%s'", bs->file->filename);
+                       "descriptor file '%s'", bs->file->bs->filename);
             return -EINVAL;
         }
 
         extent_path = g_malloc0(PATH_MAX);
         path_combine(extent_path, PATH_MAX, desc_file_path, fname);
-        extent_file = NULL;
 
         ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);
         assert(ret < 32);
 
-        ret = bdrv_open_image(&extent_file, extent_path, options,
-                              extent_opt_prefix, bs, &child_file, false, errp);
+        extent_file = bdrv_open_child(extent_path, options, extent_opt_prefix,
+                                      bs, &child_file, false, &local_err);
         g_free(extent_path);
-        if (ret) {
-            return ret;
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return -EINVAL;
         }
 
         /* save to extents array */
@@ -844,13 +849,13 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             ret = vmdk_add_extent(bs, extent_file, true, sectors,
                             0, 0, 0, 0, 0, &extent, errp);
             if (ret < 0) {
-                bdrv_unref(extent_file);
+                bdrv_unref_child(bs, extent_file);
                 return ret;
             }
             extent->flat_start_offset = flat_offset << 9;
         } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
             /* SPARSE extent and VMFSSPARSE extent are both "COWD" sparse file*/
-            char *buf = vmdk_read_desc(extent_file, 0, errp);
+            char *buf = vmdk_read_desc(extent_file->bs, 0, errp);
             if (!buf) {
                 ret = -EINVAL;
             } else {
@@ -859,13 +864,13 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             }
             g_free(buf);
             if (ret) {
-                bdrv_unref(extent_file);
+                bdrv_unref_child(bs, extent_file);
                 return ret;
             }
             extent = &s->extents[s->num_extents - 1];
         } else {
             error_setg(errp, "Unsupported extent type '%s'", type);
-            bdrv_unref(extent_file);
+            bdrv_unref_child(bs, extent_file);
             return -ENOTSUP;
         }
         extent->type = g_strdup(type);
@@ -905,7 +910,8 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
     }
     s->create_type = g_strdup(ct);
     s->desc_offset = 0;
-    ret = vmdk_parse_extents(buf, bs, bs->file->exact_filename, options, errp);
+    ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,
+                             errp);
 exit:
     return ret;
 }
@@ -918,7 +924,7 @@ static int vmdk_open(BlockDriverState *bs, QDict *options, int flags,
     BDRVVmdkState *s = bs->opaque;
     uint32_t magic;
 
-    buf = vmdk_read_desc(bs->file, 0, errp);
+    buf = vmdk_read_desc(bs->file->bs, 0, errp);
     if (!buf) {
         return -EINVAL;
     }
@@ -927,7 +933,8 @@ static int vmdk_open(BlockDriverState *bs, QDict *options, int flags,
     switch (magic) {
         case VMDK3_MAGIC:
         case VMDK4_MAGIC:
-            ret = vmdk_open_sparse(bs, bs->file, flags, buf, options, errp);
+            ret = vmdk_open_sparse(bs, bs->file, flags, buf, options,
+                                   errp);
             s->desc_offset = 0x200;
             break;
         default:
@@ -1004,7 +1011,7 @@ static int get_whole_cluster(BlockDriverState *bs,
     cluster_bytes = extent->cluster_sectors << BDRV_SECTOR_BITS;
     whole_grain = qemu_blockalign(bs, cluster_bytes);
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         memset(whole_grain, 0,  skip_start_sector << BDRV_SECTOR_BITS);
         memset(whole_grain + (skip_end_sector << BDRV_SECTOR_BITS), 0,
                cluster_bytes - (skip_end_sector << BDRV_SECTOR_BITS));
@@ -1013,22 +1020,22 @@ static int get_whole_cluster(BlockDriverState *bs,
     assert(skip_end_sector <= extent->cluster_sectors);
     /* we will be here if it's first write on non-exist grain(cluster).
      * try to read from parent image, if exist */
-    if (bs->backing_hd && !vmdk_is_cid_valid(bs)) {
+    if (bs->backing && !vmdk_is_cid_valid(bs)) {
         ret = VMDK_ERROR;
         goto exit;
     }
 
     /* Read backing data before skip range */
     if (skip_start_sector > 0) {
-        if (bs->backing_hd) {
-            ret = bdrv_read(bs->backing_hd, sector_num,
+        if (bs->backing) {
+            ret = bdrv_read(bs->backing->bs, sector_num,
                             whole_grain, skip_start_sector);
             if (ret < 0) {
                 ret = VMDK_ERROR;
                 goto exit;
             }
         }
-        ret = bdrv_write(extent->file, cluster_sector_num, whole_grain,
+        ret = bdrv_write(extent->file->bs, cluster_sector_num, whole_grain,
                          skip_start_sector);
         if (ret < 0) {
             ret = VMDK_ERROR;
@@ -1037,8 +1044,8 @@ static int get_whole_cluster(BlockDriverState *bs,
     }
     /* Read backing data after skip range */
     if (skip_end_sector < extent->cluster_sectors) {
-        if (bs->backing_hd) {
-            ret = bdrv_read(bs->backing_hd, sector_num + skip_end_sector,
+        if (bs->backing) {
+            ret = bdrv_read(bs->backing->bs, sector_num + skip_end_sector,
                             whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
                             extent->cluster_sectors - skip_end_sector);
             if (ret < 0) {
@@ -1046,7 +1053,7 @@ static int get_whole_cluster(BlockDriverState *bs,
                 goto exit;
             }
         }
-        ret = bdrv_write(extent->file, cluster_sector_num + skip_end_sector,
+        ret = bdrv_write(extent->file->bs, cluster_sector_num + skip_end_sector,
                          whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
                          extent->cluster_sectors - skip_end_sector);
         if (ret < 0) {
@@ -1066,7 +1073,7 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data,
     offset = cpu_to_le32(offset);
     /* update L2 table */
     if (bdrv_pwrite_sync(
-                extent->file,
+                extent->file->bs,
                 ((int64_t)m_data->l2_offset * 512)
                     + (m_data->l2_index * sizeof(offset)),
                 &offset, sizeof(offset)) < 0) {
@@ -1076,7 +1083,7 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data,
     if (extent->l1_backup_table_offset != 0) {
         m_data->l2_offset = extent->l1_backup_table[m_data->l1_index];
         if (bdrv_pwrite_sync(
-                    extent->file,
+                    extent->file->bs,
                     ((int64_t)m_data->l2_offset * 512)
                         + (m_data->l2_index * sizeof(offset)),
                     &offset, sizeof(offset)) < 0) {
@@ -1166,7 +1173,7 @@ static int get_cluster_offset(BlockDriverState *bs,
     }
     l2_table = extent->l2_cache + (min_index * extent->l2_size);
     if (bdrv_pread(
-                extent->file,
+                extent->file->bs,
                 (int64_t)l2_offset * 512,
                 l2_table,
                 extent->l2_size * sizeof(uint32_t)
@@ -1320,12 +1327,16 @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset,
         write_len = buf_len + sizeof(VmdkGrainMarker);
     }
     write_offset = cluster_offset + offset_in_cluster,
-    ret = bdrv_pwrite(extent->file, write_offset, write_buf, write_len);
+    ret = bdrv_pwrite(extent->file->bs, write_offset, write_buf, write_len);
 
     write_end_sector = DIV_ROUND_UP(write_offset + write_len, BDRV_SECTOR_SIZE);
 
-    extent->next_cluster_sector = MAX(extent->next_cluster_sector,
-                                      write_end_sector);
+    if (extent->compressed) {
+        extent->next_cluster_sector = write_end_sector;
+    } else {
+        extent->next_cluster_sector = MAX(extent->next_cluster_sector,
+                                          write_end_sector);
+    }
 
     if (ret != write_len) {
         ret = ret < 0 ? ret : -EIO;
@@ -1351,7 +1362,7 @@ static int vmdk_read_extent(VmdkExtent *extent, int64_t cluster_offset,
 
 
     if (!extent->compressed) {
-        ret = bdrv_pread(extent->file,
+        ret = bdrv_pread(extent->file->bs,
                           cluster_offset + offset_in_cluster,
                           buf, nb_sectors * 512);
         if (ret == nb_sectors * 512) {
@@ -1365,7 +1376,7 @@ static int vmdk_read_extent(VmdkExtent *extent, int64_t cluster_offset,
     buf_bytes = cluster_bytes * 2;
     cluster_buf = g_malloc(buf_bytes);
     uncomp_buf = g_malloc(cluster_bytes);
-    ret = bdrv_pread(extent->file,
+    ret = bdrv_pread(extent->file->bs,
                 cluster_offset,
                 cluster_buf, buf_bytes);
     if (ret < 0) {
@@ -1427,11 +1438,11 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
         }
         if (ret != VMDK_OK) {
             /* if not allocated, try to read from parent image, if exist */
-            if (bs->backing_hd && ret != VMDK_ZEROED) {
+            if (bs->backing && ret != VMDK_ZEROED) {
                 if (!vmdk_is_cid_valid(bs)) {
                     return -EINVAL;
                 }
-                ret = bdrv_read(bs->backing_hd, sector_num, buf, n);
+                ret = bdrv_read(bs->backing->bs, sector_num, buf, n);
                 if (ret < 0) {
                     return ret;
                 }
@@ -1632,7 +1643,7 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
 
     assert(bs == NULL);
     ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
+                    &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto exit;
@@ -1647,7 +1658,13 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
     }
     magic = cpu_to_be32(VMDK4_MAGIC);
     memset(&header, 0, sizeof(header));
-    header.version = zeroed_grain ? 2 : 1;
+    if (compress) {
+        header.version = 3;
+    } else if (zeroed_grain) {
+        header.version = 2;
+    } else {
+        header.version = 1;
+    }
     header.flags = VMDK4_FLAG_RGD | VMDK4_FLAG_NL_DETECT
                    | (compress ? VMDK4_FLAG_COMPRESS | VMDK4_FLAG_MARKER : 0)
                    | (zeroed_grain ? VMDK4_FLAG_ZERO_GRAIN : 0);
@@ -1905,8 +1922,7 @@ static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
             ret = -ENOENT;
             goto exit;
         }
-        ret = bdrv_open(&bs, full_backing, NULL, NULL, BDRV_O_NO_BACKING, NULL,
-                        errp);
+        ret = bdrv_open(&bs, full_backing, NULL, NULL, BDRV_O_NO_BACKING, errp);
         g_free(full_backing);
         if (ret != 0) {
             goto exit;
@@ -1977,7 +1993,7 @@ static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
     }
     assert(new_bs == NULL);
     ret = bdrv_open(&new_bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
+                    BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto exit;
@@ -2032,7 +2048,7 @@ static coroutine_fn int vmdk_co_flush(BlockDriverState *bs)
     int ret = 0;
 
     for (i = 0; i < s->num_extents; i++) {
-        err = bdrv_co_flush(s->extents[i].file);
+        err = bdrv_co_flush(s->extents[i].file->bs);
         if (err < 0) {
             ret = err;
         }
@@ -2047,7 +2063,7 @@ static int64_t vmdk_get_allocated_file_size(BlockDriverState *bs)
     int64_t r;
     BDRVVmdkState *s = bs->opaque;
 
-    ret = bdrv_get_allocated_file_size(bs->file);
+    ret = bdrv_get_allocated_file_size(bs->file->bs);
     if (ret < 0) {
         return ret;
     }
@@ -2055,7 +2071,7 @@ static int64_t vmdk_get_allocated_file_size(BlockDriverState *bs)
         if (s->extents[i].file == bs->file) {
             continue;
         }
-        r = bdrv_get_allocated_file_size(s->extents[i].file);
+        r = bdrv_get_allocated_file_size(s->extents[i].file->bs);
         if (r < 0) {
             return r;
         }
@@ -2073,7 +2089,7 @@ static int vmdk_has_zero_init(BlockDriverState *bs)
      * return 0. */
     for (i = 0; i < s->num_extents; i++) {
         if (s->extents[i].flat) {
-            if (!bdrv_has_zero_init(s->extents[i].file)) {
+            if (!bdrv_has_zero_init(s->extents[i].file->bs)) {
                 return 0;
             }
         }
@@ -2086,7 +2102,7 @@ static ImageInfo *vmdk_get_extent_info(VmdkExtent *extent)
     ImageInfo *info = g_new0(ImageInfo, 1);
 
     *info = (ImageInfo){
-        .filename         = g_strdup(extent->file->filename),
+        .filename         = g_strdup(extent->file->bs->filename),
         .format           = g_strdup(extent->type),
         .virtual_size     = extent->sectors * BDRV_SECTOR_SIZE,
         .compressed       = extent->compressed,
@@ -2132,7 +2148,9 @@ static int vmdk_check(BlockDriverState *bs, BdrvCheckResult *result,
                     PRId64 "\n", sector_num);
             break;
         }
-        if (ret == VMDK_OK && cluster_offset >= bdrv_getlength(extent->file)) {
+        if (ret == VMDK_OK &&
+            cluster_offset >= bdrv_getlength(extent->file->bs))
+        {
             fprintf(stderr,
                     "ERROR: cluster offset for sector %"
                     PRId64 " points after EOF\n", sector_num);
@@ -2153,19 +2171,19 @@ static ImageInfoSpecific *vmdk_get_specific_info(BlockDriverState *bs)
     ImageInfoList **next;
 
     *spec_info = (ImageInfoSpecific){
-        .kind = IMAGE_INFO_SPECIFIC_KIND_VMDK,
+        .type = IMAGE_INFO_SPECIFIC_KIND_VMDK,
         {
             .vmdk = g_new0(ImageInfoSpecificVmdk, 1),
         },
     };
 
-    *spec_info->vmdk = (ImageInfoSpecificVmdk) {
+    *spec_info->u.vmdk = (ImageInfoSpecificVmdk) {
         .create_type = g_strdup(s->create_type),
         .cid = s->cid,
         .parent_cid = s->parent_cid,
     };
 
-    next = &spec_info->vmdk->extents;
+    next = &spec_info->u.vmdk->extents;
     for (i = 0; i < s->num_extents; i++) {
         *next = g_new0(ImageInfoList, 1);
         (*next)->value = vmdk_get_extent_info(&s->extents[i]);
@@ -2208,7 +2226,7 @@ static void vmdk_detach_aio_context(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_extents; i++) {
-        bdrv_detach_aio_context(s->extents[i].file);
+        bdrv_detach_aio_context(s->extents[i].file->bs);
     }
 }
 
@@ -2219,7 +2237,7 @@ static void vmdk_attach_aio_context(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_extents; i++) {
-        bdrv_attach_aio_context(s->extents[i].file, new_context);
+        bdrv_attach_aio_context(s->extents[i].file->bs, new_context);
     }
 }
 
diff --git a/block/vpc.c b/block/vpc.c
index 3e385d9fb..299d37309 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -172,14 +172,14 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     int disk_type = VHD_DYNAMIC;
     int ret;
 
-    ret = bdrv_pread(bs->file, 0, s->footer_buf, HEADER_SIZE);
+    ret = bdrv_pread(bs->file->bs, 0, s->footer_buf, HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
 
     footer = (VHDFooter *) s->footer_buf;
     if (strncmp(footer->creator, "conectix", 8)) {
-        int64_t offset = bdrv_getlength(bs->file);
+        int64_t offset = bdrv_getlength(bs->file->bs);
         if (offset < 0) {
             ret = offset;
             goto fail;
@@ -189,7 +189,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
         }
 
         /* If a fixed disk, the footer is found only at the end of the file */
-        ret = bdrv_pread(bs->file, offset-HEADER_SIZE, s->footer_buf,
+        ret = bdrv_pread(bs->file->bs, offset-HEADER_SIZE, s->footer_buf,
                          HEADER_SIZE);
         if (ret < 0) {
             goto fail;
@@ -232,7 +232,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     if (disk_type == VHD_DYNAMIC) {
-        ret = bdrv_pread(bs->file, be64_to_cpu(footer->data_offset), buf,
+        ret = bdrv_pread(bs->file->bs, be64_to_cpu(footer->data_offset), buf,
                          HEADER_SIZE);
         if (ret < 0) {
             goto fail;
@@ -280,7 +280,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
 
         pagetable_size = (uint64_t) s->max_table_entries * 4;
 
-        s->pagetable = qemu_try_blockalign(bs->file, pagetable_size);
+        s->pagetable = qemu_try_blockalign(bs->file->bs, pagetable_size);
         if (s->pagetable == NULL) {
             ret = -ENOMEM;
             goto fail;
@@ -288,7 +288,8 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
 
         s->bat_offset = be64_to_cpu(dyndisk_header->table_offset);
 
-        ret = bdrv_pread(bs->file, s->bat_offset, s->pagetable, pagetable_size);
+        ret = bdrv_pread(bs->file->bs, s->bat_offset, s->pagetable,
+                         pagetable_size);
         if (ret < 0) {
             goto fail;
         }
@@ -308,7 +309,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
             }
         }
 
-        if (s->free_data_block_offset > bdrv_getlength(bs->file)) {
+        if (s->free_data_block_offset > bdrv_getlength(bs->file->bs)) {
             error_setg(errp, "block-vpc: free_data_block_offset points after "
                              "the end of file. The image has been truncated.");
             ret = -EINVAL;
@@ -383,7 +384,7 @@ static inline int64_t get_sector_offset(BlockDriverState *bs,
 
         s->last_bitmap_offset = bitmap_offset;
         memset(bitmap, 0xff, s->bitmap_size);
-        bdrv_pwrite_sync(bs->file, bitmap_offset, bitmap, s->bitmap_size);
+        bdrv_pwrite_sync(bs->file->bs, bitmap_offset, bitmap, s->bitmap_size);
     }
 
     return block_offset;
@@ -401,7 +402,7 @@ static int rewrite_footer(BlockDriverState* bs)
     BDRVVPCState *s = bs->opaque;
     int64_t offset = s->free_data_block_offset;
 
-    ret = bdrv_pwrite_sync(bs->file, offset, s->footer_buf, HEADER_SIZE);
+    ret = bdrv_pwrite_sync(bs->file->bs, offset, s->footer_buf, HEADER_SIZE);
     if (ret < 0)
         return ret;
 
@@ -436,7 +437,7 @@ static int64_t alloc_block(BlockDriverState* bs, int64_t sector_num)
 
     // Initialize the block's bitmap
     memset(bitmap, 0xff, s->bitmap_size);
-    ret = bdrv_pwrite_sync(bs->file, s->free_data_block_offset, bitmap,
+    ret = bdrv_pwrite_sync(bs->file->bs, s->free_data_block_offset, bitmap,
         s->bitmap_size);
     if (ret < 0) {
         return ret;
@@ -451,7 +452,7 @@ static int64_t alloc_block(BlockDriverState* bs, int64_t sector_num)
     // Write BAT entry to disk
     bat_offset = s->bat_offset + (4 * index);
     bat_value = cpu_to_be32(s->pagetable[index]);
-    ret = bdrv_pwrite_sync(bs->file, bat_offset, &bat_value, 4);
+    ret = bdrv_pwrite_sync(bs->file->bs, bat_offset, &bat_value, 4);
     if (ret < 0)
         goto fail;
 
@@ -485,7 +486,7 @@ static int vpc_read(BlockDriverState *bs, int64_t sector_num,
     VHDFooter *footer = (VHDFooter *) s->footer_buf;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_read(bs->file, sector_num, buf, nb_sectors);
+        return bdrv_read(bs->file->bs, sector_num, buf, nb_sectors);
     }
     while (nb_sectors > 0) {
         offset = get_sector_offset(bs, sector_num, 0);
@@ -499,7 +500,7 @@ static int vpc_read(BlockDriverState *bs, int64_t sector_num,
         if (offset == -1) {
             memset(buf, 0, sectors * BDRV_SECTOR_SIZE);
         } else {
-            ret = bdrv_pread(bs->file, offset, buf,
+            ret = bdrv_pread(bs->file->bs, offset, buf,
                 sectors * BDRV_SECTOR_SIZE);
             if (ret != sectors * BDRV_SECTOR_SIZE) {
                 return -1;
@@ -534,7 +535,7 @@ static int vpc_write(BlockDriverState *bs, int64_t sector_num,
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_write(bs->file, sector_num, buf, nb_sectors);
+        return bdrv_write(bs->file->bs, sector_num, buf, nb_sectors);
     }
     while (nb_sectors > 0) {
         offset = get_sector_offset(bs, sector_num, 1);
@@ -551,7 +552,8 @@ static int vpc_write(BlockDriverState *bs, int64_t sector_num,
                 return -1;
         }
 
-        ret = bdrv_pwrite(bs->file, offset, buf, sectors * BDRV_SECTOR_SIZE);
+        ret = bdrv_pwrite(bs->file->bs, offset, buf,
+                          sectors * BDRV_SECTOR_SIZE);
         if (ret != sectors * BDRV_SECTOR_SIZE) {
             return -1;
         }
@@ -794,7 +796,7 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
         goto out;
     }
     ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
+                    &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -878,7 +880,7 @@ static int vpc_has_zero_init(BlockDriverState *bs)
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_has_zero_init(bs->file);
+        return bdrv_has_zero_init(bs->file->bs);
     } else {
         return 1;
     }
diff --git a/block/vvfat.c b/block/vvfat.c
index 206869712..b184eca6f 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -985,12 +985,6 @@ static BDRVVVFATState *vvv = NULL;
 static int enable_write_target(BDRVVVFATState *s, Error **errp);
 static int is_consistent(BDRVVVFATState *s);
 
-static void vvfat_rebind(BlockDriverState *bs)
-{
-    BDRVVVFATState *s = bs->opaque;
-    s->bs = bs;
-}
-
 static QemuOptsList runtime_opts = {
     .name = "vvfat",
     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
@@ -2923,9 +2917,12 @@ static BlockDriver vvfat_write_target = {
 static int enable_write_target(BDRVVVFATState *s, Error **errp)
 {
     BlockDriver *bdrv_qcow = NULL;
+    BlockDriverState *backing;
     QemuOpts *opts = NULL;
     int ret;
     int size = sector2cluster(s, s->sector_count);
+    QDict *options;
+
     s->used_clusters = calloc(size, 1);
 
     array_init(&(s->commits), sizeof(commit_t));
@@ -2956,9 +2953,11 @@ static int enable_write_target(BDRVVVFATState *s, Error **errp)
     }
 
     s->qcow = NULL;
-    ret = bdrv_open(&s->qcow, s->qcow_filename, NULL, NULL,
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow"));
+    ret = bdrv_open(&s->qcow, s->qcow_filename, NULL, options,
                     BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH,
-                    bdrv_qcow, errp);
+                    errp);
     if (ret < 0) {
         goto err;
     }
@@ -2967,10 +2966,13 @@ static int enable_write_target(BDRVVVFATState *s, Error **errp)
     unlink(s->qcow_filename);
 #endif
 
-    bdrv_set_backing_hd(s->bs, bdrv_new());
-    s->bs->backing_hd->drv = &vvfat_write_target;
-    s->bs->backing_hd->opaque = g_new(void *, 1);
-    *(void**)s->bs->backing_hd->opaque = s;
+    backing = bdrv_new();
+    bdrv_set_backing_hd(s->bs, backing);
+    bdrv_unref(backing);
+
+    s->bs->backing->bs->drv = &vvfat_write_target;
+    s->bs->backing->bs->opaque = g_new(void *, 1);
+    *(void**)s->bs->backing->bs->opaque = s;
 
     return 0;
 
@@ -3004,7 +3006,6 @@ static BlockDriver bdrv_vvfat = {
     .bdrv_parse_filename    = vvfat_parse_filename,
     .bdrv_file_open         = vvfat_open,
     .bdrv_close             = vvfat_close,
-    .bdrv_rebind            = vvfat_rebind,
 
     .bdrv_read              = vvfat_co_read,
     .bdrv_write             = vvfat_co_write,
diff --git a/block/win32-aio.c b/block/win32-aio.c
index 64e86827b..bbf2f01c1 100644
--- a/block/win32-aio.c
+++ b/block/win32-aio.c
@@ -174,7 +174,7 @@ int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile)
 void win32_aio_detach_aio_context(QEMUWin32AIOState *aio,
                                   AioContext *old_context)
 {
-    aio_set_event_notifier(old_context, &aio->e, NULL);
+    aio_set_event_notifier(old_context, &aio->e, false, NULL);
     aio->is_aio_context_attached = false;
 }
 
@@ -182,7 +182,8 @@ void win32_aio_attach_aio_context(QEMUWin32AIOState *aio,
                                   AioContext *new_context)
 {
     aio->is_aio_context_attached = true;
-    aio_set_event_notifier(new_context, &aio->e, win32_aio_completion_cb);
+    aio_set_event_notifier(new_context, &aio->e, false,
+                           win32_aio_completion_cb);
 }
 
 QEMUWin32AIOState *win32_aio_init(void)
diff --git a/block/write-threshold.c b/block/write-threshold.c
index a53c1f5e6..0fe38917c 100644
--- a/block/write-threshold.c
+++ b/block/write-threshold.c
@@ -11,7 +11,7 @@
  */
 
 #include "block/block_int.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/write-threshold.h"
 #include "qemu/notify.h"
 #include "qapi-event.h"
diff --git a/blockdev.c b/blockdev.c
index 62a4586cd..e20e0c1c0 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -124,14 +124,16 @@ void blockdev_mark_auto_del(BlockBackend *blk)
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
+    if (bs) {
+        aio_context = bdrv_get_aio_context(bs);
+        aio_context_acquire(aio_context);
 
-    if (bs->job) {
-        block_job_cancel(bs->job);
-    }
+        if (bs->job) {
+            block_job_cancel(bs->job);
+        }
 
-    aio_context_release(aio_context);
+        aio_context_release(aio_context);
+    }
 
     dinfo->auto_del = 1;
 }
@@ -229,8 +231,8 @@ bool drive_check_orphaned(void)
             dinfo->type != IF_NONE) {
             fprintf(stderr, "Warning: Orphaned drive without device: "
                     "id=%s,file=%s,if=%s,bus=%d,unit=%d\n",
-                    blk_name(blk), blk_bs(blk)->filename, if_name[dinfo->type],
-                    dinfo->bus, dinfo->unit);
+                    blk_name(blk), blk_bs(blk) ? blk_bs(blk)->filename : "",
+                    if_name[dinfo->type], dinfo->bus, dinfo->unit);
             rs = true;
         }
     }
@@ -281,32 +283,6 @@ typedef struct {
     BlockDriverState *bs;
 } BDRVPutRefBH;
 
-static void bdrv_put_ref_bh(void *opaque)
-{
-    BDRVPutRefBH *s = opaque;
-
-    bdrv_unref(s->bs);
-    qemu_bh_delete(s->bh);
-    g_free(s);
-}
-
-/*
- * Release a BDS reference in a BH
- *
- * It is not safe to use bdrv_unref() from a callback function when the callers
- * still need the BlockDriverState.  In such cases we schedule a BH to release
- * the reference.
- */
-static void bdrv_put_ref_bh_schedule(BlockDriverState *bs)
-{
-    BDRVPutRefBH *s;
-
-    s = g_new(BDRVPutRefBH, 1);
-    s->bh = qemu_bh_new(bdrv_put_ref_bh, s);
-    s->bs = bs;
-    qemu_bh_schedule(s->bh);
-}
-
 static int parse_block_error_action(const char *buf, bool is_read, Error **errp)
 {
     if (!strcmp(buf, "ignore")) {
@@ -324,6 +300,45 @@ static int parse_block_error_action(const char *buf, bool is_read, Error **errp)
     }
 }
 
+static bool parse_stats_intervals(BlockAcctStats *stats, QList *intervals,
+                                  Error **errp)
+{
+    const QListEntry *entry;
+    for (entry = qlist_first(intervals); entry; entry = qlist_next(entry)) {
+        switch (qobject_type(entry->value)) {
+
+        case QTYPE_QSTRING: {
+            unsigned long long length;
+            const char *str = qstring_get_str(qobject_to_qstring(entry->value));
+            if (parse_uint_full(str, &length, 10) == 0 &&
+                length > 0 && length <= UINT_MAX) {
+                block_acct_add_interval(stats, (unsigned) length);
+            } else {
+                error_setg(errp, "Invalid interval length: %s", str);
+                return false;
+            }
+            break;
+        }
+
+        case QTYPE_QINT: {
+            int64_t length = qint_get_int(qobject_to_qint(entry->value));
+            if (length > 0 && length <= UINT_MAX) {
+                block_acct_add_interval(stats, (unsigned) length);
+            } else {
+                error_setg(errp, "Invalid interval length: %" PRId64, length);
+                return false;
+            }
+            break;
+        }
+
+        default:
+            error_setg(errp, "The specification of stats-intervals is invalid");
+            return false;
+        }
+    }
+    return true;
+}
+
 static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
 {
     if (throttle_conflicting(cfg)) {
@@ -337,30 +352,148 @@ static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
         return false;
     }
 
+    if (throttle_max_is_missing_limit(cfg)) {
+        error_setg(errp, "bps_max/iops_max require corresponding"
+                         " bps/iops values");
+        return false;
+    }
+
     return true;
 }
 
 typedef enum { MEDIA_DISK, MEDIA_CDROM } DriveMediaType;
 
+/* All parameters but @opts are optional and may be set to NULL. */
+static void extract_common_blockdev_options(QemuOpts *opts, int *bdrv_flags,
+    const char **throttling_group, ThrottleConfig *throttle_cfg,
+    BlockdevDetectZeroesOptions *detect_zeroes, Error **errp)
+{
+    const char *discard;
+    Error *local_error = NULL;
+    const char *aio;
+
+    if (bdrv_flags) {
+        if (!qemu_opt_get_bool(opts, "read-only", false)) {
+            *bdrv_flags |= BDRV_O_RDWR;
+        }
+        if (qemu_opt_get_bool(opts, "copy-on-read", false)) {
+            *bdrv_flags |= BDRV_O_COPY_ON_READ;
+        }
+
+        if ((discard = qemu_opt_get(opts, "discard")) != NULL) {
+            if (bdrv_parse_discard_flags(discard, bdrv_flags) != 0) {
+                error_setg(errp, "Invalid discard option");
+                return;
+            }
+        }
+
+        if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_WB, true)) {
+            *bdrv_flags |= BDRV_O_CACHE_WB;
+        }
+        if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_DIRECT, false)) {
+            *bdrv_flags |= BDRV_O_NOCACHE;
+        }
+        if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_NO_FLUSH, false)) {
+            *bdrv_flags |= BDRV_O_NO_FLUSH;
+        }
+
+        if ((aio = qemu_opt_get(opts, "aio")) != NULL) {
+            if (!strcmp(aio, "native")) {
+                *bdrv_flags |= BDRV_O_NATIVE_AIO;
+            } else if (!strcmp(aio, "threads")) {
+                /* this is the default */
+            } else {
+               error_setg(errp, "invalid aio option");
+               return;
+            }
+        }
+    }
+
+    /* disk I/O throttling */
+    if (throttling_group) {
+        *throttling_group = qemu_opt_get(opts, "throttling.group");
+    }
+
+    if (throttle_cfg) {
+        memset(throttle_cfg, 0, sizeof(*throttle_cfg));
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].avg =
+            qemu_opt_get_number(opts, "throttling.bps-total", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].avg  =
+            qemu_opt_get_number(opts, "throttling.bps-read", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].avg =
+            qemu_opt_get_number(opts, "throttling.bps-write", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].avg =
+            qemu_opt_get_number(opts, "throttling.iops-total", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].avg =
+            qemu_opt_get_number(opts, "throttling.iops-read", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].avg =
+            qemu_opt_get_number(opts, "throttling.iops-write", 0);
+
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].max =
+            qemu_opt_get_number(opts, "throttling.bps-total-max", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].max  =
+            qemu_opt_get_number(opts, "throttling.bps-read-max", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].max =
+            qemu_opt_get_number(opts, "throttling.bps-write-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].max =
+            qemu_opt_get_number(opts, "throttling.iops-total-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].max =
+            qemu_opt_get_number(opts, "throttling.iops-read-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].max =
+            qemu_opt_get_number(opts, "throttling.iops-write-max", 0);
+
+        throttle_cfg->op_size =
+            qemu_opt_get_number(opts, "throttling.iops-size", 0);
+
+        if (!check_throttle_config(throttle_cfg, errp)) {
+            return;
+        }
+    }
+
+    if (detect_zeroes) {
+        *detect_zeroes =
+            qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
+                            qemu_opt_get(opts, "detect-zeroes"),
+                            BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
+                            BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
+                            &local_error);
+        if (local_error) {
+            error_propagate(errp, local_error);
+            return;
+        }
+
+        if (bdrv_flags &&
+            *detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
+            !(*bdrv_flags & BDRV_O_UNMAP))
+        {
+            error_setg(errp, "setting detect-zeroes to unmap is not allowed "
+                             "without setting discard operation to unmap");
+            return;
+        }
+    }
+}
+
 /* Takes the ownership of bs_opts */
 static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
                                    Error **errp)
 {
     const char *buf;
-    int ro = 0;
     int bdrv_flags = 0;
     int on_read_error, on_write_error;
+    bool account_invalid, account_failed;
     BlockBackend *blk;
     BlockDriverState *bs;
     ThrottleConfig cfg;
     int snapshot = 0;
-    bool copy_on_read;
     Error *error = NULL;
     QemuOpts *opts;
+    QDict *interval_dict = NULL;
+    QList *interval_list = NULL;
     const char *id;
     bool has_driver_specific_opts;
-    BlockdevDetectZeroesOptions detect_zeroes;
-    const char *throttling_group;
+    BlockdevDetectZeroesOptions detect_zeroes =
+        BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF;
+    const char *throttling_group = NULL;
 
     /* Check common options by copying from bs_opts to opts, all other options
      * stay in bs_opts for processing by bdrv_open(). */
@@ -385,38 +518,25 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
 
     /* extract parameters */
     snapshot = qemu_opt_get_bool(opts, "snapshot", 0);
-    ro = qemu_opt_get_bool(opts, "read-only", 0);
-    copy_on_read = qemu_opt_get_bool(opts, "copy-on-read", false);
 
-    if ((buf = qemu_opt_get(opts, "discard")) != NULL) {
-        if (bdrv_parse_discard_flags(buf, &bdrv_flags) != 0) {
-            error_setg(errp, "invalid discard option");
-            goto early_err;
-        }
-    }
+    account_invalid = qemu_opt_get_bool(opts, "stats-account-invalid", true);
+    account_failed = qemu_opt_get_bool(opts, "stats-account-failed", true);
 
-    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_WB, true)) {
-        bdrv_flags |= BDRV_O_CACHE_WB;
-    }
-    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_DIRECT, false)) {
-        bdrv_flags |= BDRV_O_NOCACHE;
-    }
-    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_NO_FLUSH, false)) {
-        bdrv_flags |= BDRV_O_NO_FLUSH;
+    qdict_extract_subqdict(bs_opts, &interval_dict, "stats-intervals.");
+    qdict_array_split(interval_dict, &interval_list);
+
+    if (qdict_size(interval_dict) != 0) {
+        error_setg(errp, "Invalid option stats-intervals.%s",
+                   qdict_first(interval_dict)->key);
+        goto early_err;
     }
 
-#ifdef CONFIG_LINUX_AIO
-    if ((buf = qemu_opt_get(opts, "aio")) != NULL) {
-        if (!strcmp(buf, "native")) {
-            bdrv_flags |= BDRV_O_NATIVE_AIO;
-        } else if (!strcmp(buf, "threads")) {
-            /* this is the default */
-        } else {
-           error_setg(errp, "invalid aio option");
-           goto early_err;
-        }
+    extract_common_blockdev_options(opts, &bdrv_flags, &throttling_group, &cfg,
+                                    &detect_zeroes, &error);
+    if (error) {
+        error_propagate(errp, error);
+        goto early_err;
     }
-#endif
 
     if ((buf = qemu_opt_get(opts, "format")) != NULL) {
         if (is_help_option(buf)) {
@@ -433,43 +553,6 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
         qdict_put(bs_opts, "driver", qstring_from_str(buf));
     }
 
-    /* disk I/O throttling */
-    memset(&cfg, 0, sizeof(cfg));
-    cfg.buckets[THROTTLE_BPS_TOTAL].avg =
-        qemu_opt_get_number(opts, "throttling.bps-total", 0);
-    cfg.buckets[THROTTLE_BPS_READ].avg  =
-        qemu_opt_get_number(opts, "throttling.bps-read", 0);
-    cfg.buckets[THROTTLE_BPS_WRITE].avg =
-        qemu_opt_get_number(opts, "throttling.bps-write", 0);
-    cfg.buckets[THROTTLE_OPS_TOTAL].avg =
-        qemu_opt_get_number(opts, "throttling.iops-total", 0);
-    cfg.buckets[THROTTLE_OPS_READ].avg =
-        qemu_opt_get_number(opts, "throttling.iops-read", 0);
-    cfg.buckets[THROTTLE_OPS_WRITE].avg =
-        qemu_opt_get_number(opts, "throttling.iops-write", 0);
-
-    cfg.buckets[THROTTLE_BPS_TOTAL].max =
-        qemu_opt_get_number(opts, "throttling.bps-total-max", 0);
-    cfg.buckets[THROTTLE_BPS_READ].max  =
-        qemu_opt_get_number(opts, "throttling.bps-read-max", 0);
-    cfg.buckets[THROTTLE_BPS_WRITE].max =
-        qemu_opt_get_number(opts, "throttling.bps-write-max", 0);
-    cfg.buckets[THROTTLE_OPS_TOTAL].max =
-        qemu_opt_get_number(opts, "throttling.iops-total-max", 0);
-    cfg.buckets[THROTTLE_OPS_READ].max =
-        qemu_opt_get_number(opts, "throttling.iops-read-max", 0);
-    cfg.buckets[THROTTLE_OPS_WRITE].max =
-        qemu_opt_get_number(opts, "throttling.iops-write-max", 0);
-
-    cfg.op_size = qemu_opt_get_number(opts, "throttling.iops-size", 0);
-
-    throttling_group = qemu_opt_get(opts, "throttling.group");
-
-    if (!check_throttle_config(&cfg, &error)) {
-        error_propagate(errp, error);
-        goto early_err;
-    }
-
     on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
     if ((buf = qemu_opt_get(opts, "werror")) != NULL) {
         on_write_error = parse_block_error_action(buf, 0, &error);
@@ -488,34 +571,34 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
         }
     }
 
-    detect_zeroes =
-        qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
-                        qemu_opt_get(opts, "detect-zeroes"),
-                        BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
-                        BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
-                        &error);
-    if (error) {
-        error_propagate(errp, error);
-        goto early_err;
-    }
-
-    if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
-        !(bdrv_flags & BDRV_O_UNMAP)) {
-        error_setg(errp, "setting detect-zeroes to unmap is not allowed "
-                         "without setting discard operation to unmap");
-        goto early_err;
+    if (snapshot) {
+        /* always use cache=unsafe with snapshot */
+        bdrv_flags &= ~BDRV_O_CACHE_MASK;
+        bdrv_flags |= (BDRV_O_SNAPSHOT|BDRV_O_CACHE_WB|BDRV_O_NO_FLUSH);
     }
 
     /* init */
     if ((!file || !*file) && !has_driver_specific_opts) {
-        blk = blk_new_with_bs(qemu_opts_id(opts), errp);
+        BlockBackendRootState *blk_rs;
+
+        blk = blk_new(qemu_opts_id(opts), errp);
         if (!blk) {
             goto early_err;
         }
 
-        bs = blk_bs(blk);
-        bs->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
-        bs->read_only = ro;
+        blk_rs = blk_get_root_state(blk);
+        blk_rs->open_flags    = bdrv_flags;
+        blk_rs->read_only     = !(bdrv_flags & BDRV_O_RDWR);
+        blk_rs->detect_zeroes = detect_zeroes;
+
+        if (throttle_enabled(&cfg)) {
+            if (!throttling_group) {
+                throttling_group = blk_name(blk);
+            }
+            blk_rs->throttle_group = g_strdup(throttling_group);
+            blk_rs->throttle_state = throttle_group_incref(throttling_group);
+            blk_rs->throttle_state->cfg = cfg;
+        }
 
         QDECREF(bs_opts);
     } else {
@@ -523,58 +606,102 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
             file = NULL;
         }
 
-        if (snapshot) {
-            /* always use cache=unsafe with snapshot */
-            bdrv_flags &= ~BDRV_O_CACHE_MASK;
-            bdrv_flags |= (BDRV_O_SNAPSHOT|BDRV_O_CACHE_WB|BDRV_O_NO_FLUSH);
-        }
-
-        if (copy_on_read) {
-            bdrv_flags |= BDRV_O_COPY_ON_READ;
-        }
-
-        if (runstate_check(RUN_STATE_INMIGRATE)) {
-            bdrv_flags |= BDRV_O_INCOMING;
-        }
-
-        bdrv_flags |= ro ? 0 : BDRV_O_RDWR;
-
         blk = blk_new_open(qemu_opts_id(opts), file, NULL, bs_opts, bdrv_flags,
                            errp);
         if (!blk) {
             goto err_no_bs_opts;
         }
         bs = blk_bs(blk);
-    }
 
-    bs->detect_zeroes = detect_zeroes;
+        bs->detect_zeroes = detect_zeroes;
 
-    bdrv_set_on_error(bs, on_read_error, on_write_error);
+        /* disk I/O throttling */
+        if (throttle_enabled(&cfg)) {
+            if (!throttling_group) {
+                throttling_group = blk_name(blk);
+            }
+            bdrv_io_limits_enable(bs, throttling_group);
+            bdrv_set_io_limits(bs, &cfg);
+        }
 
-    /* disk I/O throttling */
-    if (throttle_enabled(&cfg)) {
-        if (!throttling_group) {
-            throttling_group = blk_name(blk);
+        if (bdrv_key_required(bs)) {
+            autostart = 0;
         }
-        bdrv_io_limits_enable(bs, throttling_group);
-        bdrv_set_io_limits(bs, &cfg);
-    }
 
-    if (bdrv_key_required(bs)) {
-        autostart = 0;
+        block_acct_init(blk_get_stats(blk), account_invalid, account_failed);
+
+        if (!parse_stats_intervals(blk_get_stats(blk), interval_list, errp)) {
+            blk_unref(blk);
+            blk = NULL;
+            goto err_no_bs_opts;
+        }
     }
 
+    blk_set_on_error(blk, on_read_error, on_write_error);
+
 err_no_bs_opts:
     qemu_opts_del(opts);
+    QDECREF(interval_dict);
+    QDECREF(interval_list);
     return blk;
 
 early_err:
     qemu_opts_del(opts);
+    QDECREF(interval_dict);
+    QDECREF(interval_list);
 err_no_opts:
     QDECREF(bs_opts);
     return NULL;
 }
 
+static QemuOptsList qemu_root_bds_opts;
+
+/* Takes the ownership of bs_opts */
+static BlockDriverState *bds_tree_init(QDict *bs_opts, Error **errp)
+{
+    BlockDriverState *bs;
+    QemuOpts *opts;
+    Error *local_error = NULL;
+    BlockdevDetectZeroesOptions detect_zeroes;
+    int ret;
+    int bdrv_flags = 0;
+
+    opts = qemu_opts_create(&qemu_root_bds_opts, NULL, 1, errp);
+    if (!opts) {
+        goto fail;
+    }
+
+    qemu_opts_absorb_qdict(opts, bs_opts, &local_error);
+    if (local_error) {
+        error_propagate(errp, local_error);
+        goto fail;
+    }
+
+    extract_common_blockdev_options(opts, &bdrv_flags, NULL, NULL,
+                                    &detect_zeroes, &local_error);
+    if (local_error) {
+        error_propagate(errp, local_error);
+        goto fail;
+    }
+
+    bs = NULL;
+    ret = bdrv_open(&bs, NULL, NULL, bs_opts, bdrv_flags, errp);
+    if (ret < 0) {
+        goto fail_no_bs_opts;
+    }
+
+    bs->detect_zeroes = detect_zeroes;
+
+fail_no_bs_opts:
+    qemu_opts_del(opts);
+    return bs;
+
+fail:
+    qemu_opts_del(opts);
+    QDECREF(bs_opts);
+    return NULL;
+}
+
 static void qemu_opt_rename(QemuOpts *opts, const char *from, const char *to,
                             Error **errp)
 {
@@ -1033,12 +1160,26 @@ void hmp_commit(Monitor *mon, const QDict *qdict)
     if (!strcmp(device, "all")) {
         ret = bdrv_commit_all();
     } else {
+        BlockDriverState *bs;
+        AioContext *aio_context;
+
         blk = blk_by_name(device);
         if (!blk) {
             monitor_printf(mon, "Device '%s' not found\n", device);
             return;
         }
-        ret = bdrv_commit(blk_bs(blk));
+        if (!blk_is_available(blk)) {
+            monitor_printf(mon, "Device '%s' has no medium\n", device);
+            return;
+        }
+
+        bs = blk_bs(blk);
+        aio_context = bdrv_get_aio_context(bs);
+        aio_context_acquire(aio_context);
+
+        ret = bdrv_commit(bs);
+
+        aio_context_release(aio_context);
     }
     if (ret < 0) {
         monitor_printf(mon, "'commit' error for '%s': %s\n", device,
@@ -1046,16 +1187,17 @@ void hmp_commit(Monitor *mon, const QDict *qdict)
     }
 }
 
-static void blockdev_do_action(int kind, void *data, Error **errp)
+static void blockdev_do_action(TransactionActionKind type, void *data,
+                               Error **errp)
 {
     TransactionAction action;
     TransactionActionList list;
 
-    action.kind = kind;
-    action.data = data;
+    action.type = type;
+    action.u.data = data;
     list.value = &action;
     list.next = NULL;
-    qmp_transaction(&list, errp);
+    qmp_transaction(&list, false, NULL, errp);
 }
 
 void qmp_blockdev_snapshot_sync(bool has_device, const char *device,
@@ -1066,7 +1208,7 @@ void qmp_blockdev_snapshot_sync(bool has_device, const char *device,
                                 bool has_format, const char *format,
                                 bool has_mode, NewImageMode mode, Error **errp)
 {
-    BlockdevSnapshot snapshot = {
+    BlockdevSnapshotSync snapshot = {
         .has_device = has_device,
         .device = (char *) device,
         .has_node_name = has_node_name,
@@ -1083,6 +1225,18 @@ void qmp_blockdev_snapshot_sync(bool has_device, const char *device,
                        &snapshot, errp);
 }
 
+void qmp_blockdev_snapshot(const char *node, const char *overlay,
+                           Error **errp)
+{
+    BlockdevSnapshot snapshot_data = {
+        .node = (char *) node,
+        .overlay = (char *) overlay
+    };
+
+    blockdev_do_action(TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT,
+                       &snapshot_data, errp);
+}
+
 void qmp_blockdev_snapshot_internal_sync(const char *device,
                                          const char *name,
                                          Error **errp)
@@ -1117,7 +1271,9 @@ SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
                   "Device '%s' not found", device);
         return NULL;
     }
-    bs = blk_bs(blk);
+
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
 
     if (!has_id) {
         id = NULL;
@@ -1129,11 +1285,14 @@ SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
 
     if (!id && !name) {
         error_setg(errp, "Name or id must be provided");
-        return NULL;
+        goto out_aio_context;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out_aio_context;
+    }
+    bs = blk_bs(blk);
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE, errp)) {
         goto out_aio_context;
@@ -1240,43 +1399,75 @@ static BdrvDirtyBitmap *block_dirty_bitmap_lookup(const char *node,
 
 /* New and old BlockDriverState structs for atomic group operations */
 
-typedef struct BlkTransactionState BlkTransactionState;
+typedef struct BlkActionState BlkActionState;
 
-/* Only prepare() may fail. In a single transaction, only one of commit() or
-   abort() will be called, clean() will always be called if it present. */
-typedef struct BdrvActionOps {
-    /* Size of state struct, in bytes. */
+/**
+ * BlkActionOps:
+ * Table of operations that define an Action.
+ *
+ * @instance_size: Size of state struct, in bytes.
+ * @prepare: Prepare the work, must NOT be NULL.
+ * @commit: Commit the changes, can be NULL.
+ * @abort: Abort the changes on fail, can be NULL.
+ * @clean: Clean up resources after all transaction actions have called
+ *         commit() or abort(). Can be NULL.
+ *
+ * Only prepare() may fail. In a single transaction, only one of commit() or
+ * abort() will be called. clean() will always be called if it is present.
+ */
+typedef struct BlkActionOps {
     size_t instance_size;
-    /* Prepare the work, must NOT be NULL. */
-    void (*prepare)(BlkTransactionState *common, Error **errp);
-    /* Commit the changes, can be NULL. */
-    void (*commit)(BlkTransactionState *common);
-    /* Abort the changes on fail, can be NULL. */
-    void (*abort)(BlkTransactionState *common);
-    /* Clean up resource in the end, can be NULL. */
-    void (*clean)(BlkTransactionState *common);
-} BdrvActionOps;
+    void (*prepare)(BlkActionState *common, Error **errp);
+    void (*commit)(BlkActionState *common);
+    void (*abort)(BlkActionState *common);
+    void (*clean)(BlkActionState *common);
+} BlkActionOps;
 
-/*
- * This structure must be arranged as first member in child type, assuming
- * that compiler will also arrange it to the same address with parent instance.
- * Later it will be used in free().
+/**
+ * BlkActionState:
+ * Describes one Action's state within a Transaction.
+ *
+ * @action: QAPI-defined enum identifying which Action to perform.
+ * @ops: Table of ActionOps this Action can perform.
+ * @block_job_txn: Transaction which this action belongs to.
+ * @entry: List membership for all Actions in this Transaction.
+ *
+ * This structure must be arranged as first member in a subclassed type,
+ * assuming that the compiler will also arrange it to the same offsets as the
+ * base class.
  */
-struct BlkTransactionState {
+struct BlkActionState {
     TransactionAction *action;
-    const BdrvActionOps *ops;
-    QSIMPLEQ_ENTRY(BlkTransactionState) entry;
+    const BlkActionOps *ops;
+    BlockJobTxn *block_job_txn;
+    TransactionProperties *txn_props;
+    QSIMPLEQ_ENTRY(BlkActionState) entry;
 };
 
 /* internal snapshot private data */
 typedef struct InternalSnapshotState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
     AioContext *aio_context;
     QEMUSnapshotInfo sn;
+    bool created;
 } InternalSnapshotState;
 
-static void internal_snapshot_prepare(BlkTransactionState *common,
+
+static int action_check_completion_mode(BlkActionState *s, Error **errp)
+{
+    if (s->txn_props->completion_mode != ACTION_COMPLETION_MODE_INDIVIDUAL) {
+        error_setg(errp,
+                   "Action '%s' does not support Transaction property "
+                   "completion-mode = %s",
+                   TransactionActionKind_lookup[s->action->type],
+                   ActionCompletionMode_lookup[s->txn_props->completion_mode]);
+        return -1;
+    }
+    return 0;
+}
+
+static void internal_snapshot_prepare(BlkActionState *common,
                                       Error **errp)
 {
     Error *local_err = NULL;
@@ -1291,9 +1482,9 @@ static void internal_snapshot_prepare(BlkTransactionState *common,
     InternalSnapshotState *state;
     int ret1;
 
-    g_assert(common->action->kind ==
+    g_assert(common->action->type ==
              TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC);
-    internal = common->action->blockdev_snapshot_internal_sync;
+    internal = common->action->u.blockdev_snapshot_internal_sync;
     state = DO_UPCAST(InternalSnapshotState, common, common);
 
     /* 1. parse input */
@@ -1301,22 +1492,29 @@ static void internal_snapshot_prepare(BlkTransactionState *common,
     name = internal->name;
 
     /* 2. check for validation */
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
     blk = blk_by_name(device);
     if (!blk) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
     /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
+    state->aio_context = blk_get_aio_context(blk);
     aio_context_acquire(state->aio_context);
 
-    if (!bdrv_is_inserted(bs)) {
+    if (!blk_is_available(blk)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         return;
     }
+    bs = blk_bs(blk);
+
+    state->bs = bs;
+    bdrv_drained_begin(bs);
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, errp)) {
         return;
@@ -1369,10 +1567,10 @@ static void internal_snapshot_prepare(BlkTransactionState *common,
     }
 
     /* 4. succeed, mark a snapshot is created */
-    state->bs = bs;
+    state->created = true;
 }
 
-static void internal_snapshot_abort(BlkTransactionState *common)
+static void internal_snapshot_abort(BlkActionState *common)
 {
     InternalSnapshotState *state =
                              DO_UPCAST(InternalSnapshotState, common, common);
@@ -1380,7 +1578,7 @@ static void internal_snapshot_abort(BlkTransactionState *common)
     QEMUSnapshotInfo *sn = &state->sn;
     Error *local_error = NULL;
 
-    if (!bs) {
+    if (!state->created) {
         return;
     }
 
@@ -1395,91 +1593,83 @@ static void internal_snapshot_abort(BlkTransactionState *common)
     }
 }
 
-static void internal_snapshot_clean(BlkTransactionState *common)
+static void internal_snapshot_clean(BlkActionState *common)
 {
     InternalSnapshotState *state = DO_UPCAST(InternalSnapshotState,
                                              common, common);
 
     if (state->aio_context) {
+        if (state->bs) {
+            bdrv_drained_end(state->bs);
+        }
         aio_context_release(state->aio_context);
     }
 }
 
 /* external snapshot private data */
 typedef struct ExternalSnapshotState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *old_bs;
     BlockDriverState *new_bs;
     AioContext *aio_context;
 } ExternalSnapshotState;
 
-static void external_snapshot_prepare(BlkTransactionState *common,
+static void external_snapshot_prepare(BlkActionState *common,
                                       Error **errp)
 {
-    BlockDriver *drv;
-    int flags, ret;
+    int flags = 0, ret;
     QDict *options = NULL;
     Error *local_err = NULL;
-    bool has_device = false;
+    /* Device and node name of the image to generate the snapshot from */
     const char *device;
-    bool has_node_name = false;
     const char *node_name;
-    bool has_snapshot_node_name = false;
-    const char *snapshot_node_name;
+    /* Reference to the new image (for 'blockdev-snapshot') */
+    const char *snapshot_ref;
+    /* File name of the new image (for 'blockdev-snapshot-sync') */
     const char *new_image_file;
-    const char *format = "qcow2";
-    enum NewImageMode mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
     TransactionAction *action = common->action;
 
-    /* get parameters */
-    g_assert(action->kind == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC);
-
-    has_device = action->blockdev_snapshot_sync->has_device;
-    device = action->blockdev_snapshot_sync->device;
-    has_node_name = action->blockdev_snapshot_sync->has_node_name;
-    node_name = action->blockdev_snapshot_sync->node_name;
-    has_snapshot_node_name =
-        action->blockdev_snapshot_sync->has_snapshot_node_name;
-    snapshot_node_name = action->blockdev_snapshot_sync->snapshot_node_name;
-
-    new_image_file = action->blockdev_snapshot_sync->snapshot_file;
-    if (action->blockdev_snapshot_sync->has_format) {
-        format = action->blockdev_snapshot_sync->format;
-    }
-    if (action->blockdev_snapshot_sync->has_mode) {
-        mode = action->blockdev_snapshot_sync->mode;
+    /* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar
+     * purpose but a different set of parameters */
+    switch (action->type) {
+    case TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT:
+        {
+            BlockdevSnapshot *s = action->u.blockdev_snapshot;
+            device = s->node;
+            node_name = s->node;
+            new_image_file = NULL;
+            snapshot_ref = s->overlay;
+        }
+        break;
+    case TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC:
+        {
+            BlockdevSnapshotSync *s = action->u.blockdev_snapshot_sync;
+            device = s->has_device ? s->device : NULL;
+            node_name = s->has_node_name ? s->node_name : NULL;
+            new_image_file = s->snapshot_file;
+            snapshot_ref = NULL;
+        }
+        break;
+    default:
+        g_assert_not_reached();
     }
 
     /* start processing */
-    drv = bdrv_find_format(format);
-    if (!drv) {
-        error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
-        return;
-    }
-
-    state->old_bs = bdrv_lookup_bs(has_device ? device : NULL,
-                                   has_node_name ? node_name : NULL,
-                                   &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    if (has_node_name && !has_snapshot_node_name) {
-        error_setg(errp, "New snapshot node name missing");
+    if (action_check_completion_mode(common, errp) < 0) {
         return;
     }
 
-    if (has_snapshot_node_name && bdrv_find_node(snapshot_node_name)) {
-        error_setg(errp, "New snapshot node name already existing");
+    state->old_bs = bdrv_lookup_bs(device, node_name, errp);
+    if (!state->old_bs) {
         return;
     }
 
     /* Acquire AioContext now so any threads operating on old_bs stop */
     state->aio_context = bdrv_get_aio_context(state->old_bs);
     aio_context_acquire(state->aio_context);
+    bdrv_drained_begin(state->old_bs);
 
     if (!bdrv_is_inserted(state->old_bs)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
@@ -1503,38 +1693,79 @@ static void external_snapshot_prepare(BlkTransactionState *common,
         return;
     }
 
-    flags = state->old_bs->open_flags;
+    if (action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC) {
+        BlockdevSnapshotSync *s = action->u.blockdev_snapshot_sync;
+        const char *format = s->has_format ? s->format : "qcow2";
+        enum NewImageMode mode;
+        const char *snapshot_node_name =
+            s->has_snapshot_node_name ? s->snapshot_node_name : NULL;
 
-    /* create new image w/backing file */
-    if (mode != NEW_IMAGE_MODE_EXISTING) {
-        bdrv_img_create(new_image_file, format,
-                        state->old_bs->filename,
-                        state->old_bs->drv->format_name,
-                        NULL, -1, flags, &local_err, false);
-        if (local_err) {
-            error_propagate(errp, local_err);
+        if (node_name && !snapshot_node_name) {
+            error_setg(errp, "New snapshot node name missing");
             return;
         }
-    }
 
-    if (has_snapshot_node_name) {
+        if (snapshot_node_name &&
+            bdrv_lookup_bs(snapshot_node_name, snapshot_node_name, NULL)) {
+            error_setg(errp, "New snapshot node name already in use");
+            return;
+        }
+
+        flags = state->old_bs->open_flags;
+
+        /* create new image w/backing file */
+        mode = s->has_mode ? s->mode : NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+        if (mode != NEW_IMAGE_MODE_EXISTING) {
+            bdrv_img_create(new_image_file, format,
+                            state->old_bs->filename,
+                            state->old_bs->drv->format_name,
+                            NULL, -1, flags, &local_err, false);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+        }
+
         options = qdict_new();
-        qdict_put(options, "node-name",
-                  qstring_from_str(snapshot_node_name));
+        if (s->has_snapshot_node_name) {
+            qdict_put(options, "node-name",
+                      qstring_from_str(snapshot_node_name));
+        }
+        qdict_put(options, "driver", qstring_from_str(format));
+
+        flags |= BDRV_O_NO_BACKING;
     }
 
-    /* TODO Inherit bs->options or only take explicit options with an
-     * extended QMP command? */
     assert(state->new_bs == NULL);
-    ret = bdrv_open(&state->new_bs, new_image_file, NULL, options,
-                    flags | BDRV_O_NO_BACKING, drv, &local_err);
+    ret = bdrv_open(&state->new_bs, new_image_file, snapshot_ref, options,
+                    flags, errp);
     /* We will manually add the backing_hd field to the bs later */
     if (ret != 0) {
-        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (state->new_bs->blk != NULL) {
+        error_setg(errp, "The snapshot is already in use by %s",
+                   blk_name(state->new_bs->blk));
+        return;
+    }
+
+    if (bdrv_op_is_blocked(state->new_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
+                           errp)) {
+        return;
+    }
+
+    if (state->new_bs->backing != NULL) {
+        error_setg(errp, "The snapshot already has a backing image");
+        return;
+    }
+
+    if (!state->new_bs->drv->supports_backing) {
+        error_setg(errp, "The snapshot does not support backing images");
     }
 }
 
-static void external_snapshot_commit(BlkTransactionState *common)
+static void external_snapshot_commit(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
@@ -1546,41 +1777,57 @@ static void external_snapshot_commit(BlkTransactionState *common)
     /* We don't need (or want) to use the transactional
      * bdrv_reopen_multiple() across all the entries at once, because we
      * don't want to abort all of them if one of them fails the reopen */
-    bdrv_reopen(state->new_bs, state->new_bs->open_flags & ~BDRV_O_RDWR,
+    bdrv_reopen(state->old_bs, state->old_bs->open_flags & ~BDRV_O_RDWR,
                 NULL);
-
-    aio_context_release(state->aio_context);
 }
 
-static void external_snapshot_abort(BlkTransactionState *common)
+static void external_snapshot_abort(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
     if (state->new_bs) {
         bdrv_unref(state->new_bs);
     }
+}
+
+static void external_snapshot_clean(BlkActionState *common)
+{
+    ExternalSnapshotState *state =
+                             DO_UPCAST(ExternalSnapshotState, common, common);
     if (state->aio_context) {
+        bdrv_drained_end(state->old_bs);
         aio_context_release(state->aio_context);
     }
 }
 
 typedef struct DriveBackupState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
     AioContext *aio_context;
     BlockJob *job;
 } DriveBackupState;
 
-static void drive_backup_prepare(BlkTransactionState *common, Error **errp)
+static void do_drive_backup(const char *device, const char *target,
+                            bool has_format, const char *format,
+                            enum MirrorSyncMode sync,
+                            bool has_mode, enum NewImageMode mode,
+                            bool has_speed, int64_t speed,
+                            bool has_bitmap, const char *bitmap,
+                            bool has_on_source_error,
+                            BlockdevOnError on_source_error,
+                            bool has_on_target_error,
+                            BlockdevOnError on_target_error,
+                            BlockJobTxn *txn, Error **errp);
+
+static void drive_backup_prepare(BlkActionState *common, Error **errp)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
-    BlockDriverState *bs;
     BlockBackend *blk;
     DriveBackup *backup;
     Error *local_err = NULL;
 
-    assert(common->action->kind == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
-    backup = common->action->drive_backup;
+    assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
+    backup = common->action->u.drive_backup;
 
     blk = blk_by_name(backup->device);
     if (!blk) {
@@ -1588,31 +1835,36 @@ static void drive_backup_prepare(BlkTransactionState *common, Error **errp)
                   "Device '%s' not found", backup->device);
         return;
     }
-    bs = blk_bs(blk);
+
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, backup->device);
+        return;
+    }
 
     /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
+    state->aio_context = blk_get_aio_context(blk);
     aio_context_acquire(state->aio_context);
-
-    qmp_drive_backup(backup->device, backup->target,
-                     backup->has_format, backup->format,
-                     backup->sync,
-                     backup->has_mode, backup->mode,
-                     backup->has_speed, backup->speed,
-                     backup->has_bitmap, backup->bitmap,
-                     backup->has_on_source_error, backup->on_source_error,
-                     backup->has_on_target_error, backup->on_target_error,
-                     &local_err);
+    bdrv_drained_begin(blk_bs(blk));
+    state->bs = blk_bs(blk);
+
+    do_drive_backup(backup->device, backup->target,
+                    backup->has_format, backup->format,
+                    backup->sync,
+                    backup->has_mode, backup->mode,
+                    backup->has_speed, backup->speed,
+                    backup->has_bitmap, backup->bitmap,
+                    backup->has_on_source_error, backup->on_source_error,
+                    backup->has_on_target_error, backup->on_target_error,
+                    common->block_job_txn, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
 
-    state->bs = bs;
     state->job = state->bs->job;
 }
 
-static void drive_backup_abort(BlkTransactionState *common)
+static void drive_backup_abort(BlkActionState *common)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
     BlockDriverState *bs = state->bs;
@@ -1623,72 +1875,85 @@ static void drive_backup_abort(BlkTransactionState *common)
     }
 }
 
-static void drive_backup_clean(BlkTransactionState *common)
+static void drive_backup_clean(BlkActionState *common)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
 
     if (state->aio_context) {
+        bdrv_drained_end(state->bs);
         aio_context_release(state->aio_context);
     }
 }
 
 typedef struct BlockdevBackupState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
     BlockJob *job;
     AioContext *aio_context;
 } BlockdevBackupState;
 
-static void blockdev_backup_prepare(BlkTransactionState *common, Error **errp)
+static void do_blockdev_backup(const char *device, const char *target,
+                               enum MirrorSyncMode sync,
+                               bool has_speed, int64_t speed,
+                               bool has_on_source_error,
+                               BlockdevOnError on_source_error,
+                               bool has_on_target_error,
+                               BlockdevOnError on_target_error,
+                               BlockJobTxn *txn, Error **errp);
+
+static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
     BlockdevBackup *backup;
-    BlockDriverState *bs, *target;
-    BlockBackend *blk;
+    BlockBackend *blk, *target;
     Error *local_err = NULL;
 
-    assert(common->action->kind == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
-    backup = common->action->blockdev_backup;
+    assert(common->action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
+    backup = common->action->u.blockdev_backup;
 
     blk = blk_by_name(backup->device);
     if (!blk) {
         error_setg(errp, "Device '%s' not found", backup->device);
         return;
     }
-    bs = blk_bs(blk);
 
-    blk = blk_by_name(backup->target);
-    if (!blk) {
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, backup->device);
+        return;
+    }
+
+    target = blk_by_name(backup->target);
+    if (!target) {
         error_setg(errp, "Device '%s' not found", backup->target);
         return;
     }
-    target = blk_bs(blk);
 
     /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
-    if (state->aio_context != bdrv_get_aio_context(target)) {
+    state->aio_context = blk_get_aio_context(blk);
+    if (state->aio_context != blk_get_aio_context(target)) {
         state->aio_context = NULL;
         error_setg(errp, "Backup between two IO threads is not implemented");
         return;
     }
     aio_context_acquire(state->aio_context);
-
-    qmp_blockdev_backup(backup->device, backup->target,
-                        backup->sync,
-                        backup->has_speed, backup->speed,
-                        backup->has_on_source_error, backup->on_source_error,
-                        backup->has_on_target_error, backup->on_target_error,
-                        &local_err);
+    state->bs = blk_bs(blk);
+    bdrv_drained_begin(state->bs);
+
+    do_blockdev_backup(backup->device, backup->target,
+                       backup->sync,
+                       backup->has_speed, backup->speed,
+                       backup->has_on_source_error, backup->on_source_error,
+                       backup->has_on_target_error, backup->on_target_error,
+                       common->block_job_txn, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
 
-    state->bs = bs;
     state->job = state->bs->job;
 }
 
-static void blockdev_backup_abort(BlkTransactionState *common)
+static void blockdev_backup_abort(BlkActionState *common)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
     BlockDriverState *bs = state->bs;
@@ -1699,31 +1964,148 @@ static void blockdev_backup_abort(BlkTransactionState *common)
     }
 }
 
-static void blockdev_backup_clean(BlkTransactionState *common)
+static void blockdev_backup_clean(BlkActionState *common)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
 
     if (state->aio_context) {
+        bdrv_drained_end(state->bs);
         aio_context_release(state->aio_context);
     }
 }
 
-static void abort_prepare(BlkTransactionState *common, Error **errp)
+typedef struct BlockDirtyBitmapState {
+    BlkActionState common;
+    BdrvDirtyBitmap *bitmap;
+    BlockDriverState *bs;
+    AioContext *aio_context;
+    HBitmap *backup;
+    bool prepared;
+} BlockDirtyBitmapState;
+
+static void block_dirty_bitmap_add_prepare(BlkActionState *common,
+                                           Error **errp)
+{
+    Error *local_err = NULL;
+    BlockDirtyBitmapAdd *action;
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
+    action = common->action->u.block_dirty_bitmap_add;
+    /* AIO context taken and released within qmp_block_dirty_bitmap_add */
+    qmp_block_dirty_bitmap_add(action->node, action->name,
+                               action->has_granularity, action->granularity,
+                               &local_err);
+
+    if (!local_err) {
+        state->prepared = true;
+    } else {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void block_dirty_bitmap_add_abort(BlkActionState *common)
+{
+    BlockDirtyBitmapAdd *action;
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    action = common->action->u.block_dirty_bitmap_add;
+    /* Should not be able to fail: IF the bitmap was added via .prepare(),
+     * then the node reference and bitmap name must have been valid.
+     */
+    if (state->prepared) {
+        qmp_block_dirty_bitmap_remove(action->node, action->name, &error_abort);
+    }
+}
+
+static void block_dirty_bitmap_clear_prepare(BlkActionState *common,
+                                             Error **errp)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+    BlockDirtyBitmap *action;
+
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
+    action = common->action->u.block_dirty_bitmap_clear;
+    state->bitmap = block_dirty_bitmap_lookup(action->node,
+                                              action->name,
+                                              &state->bs,
+                                              &state->aio_context,
+                                              errp);
+    if (!state->bitmap) {
+        return;
+    }
+
+    if (bdrv_dirty_bitmap_frozen(state->bitmap)) {
+        error_setg(errp, "Cannot modify a frozen bitmap");
+        return;
+    } else if (!bdrv_dirty_bitmap_enabled(state->bitmap)) {
+        error_setg(errp, "Cannot clear a disabled bitmap");
+        return;
+    }
+
+    bdrv_clear_dirty_bitmap(state->bitmap, &state->backup);
+    /* AioContext is released in .clean() */
+}
+
+static void block_dirty_bitmap_clear_abort(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    bdrv_undo_clear_dirty_bitmap(state->bitmap, state->backup);
+}
+
+static void block_dirty_bitmap_clear_commit(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    hbitmap_free(state->backup);
+}
+
+static void block_dirty_bitmap_clear_clean(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    if (state->aio_context) {
+        aio_context_release(state->aio_context);
+    }
+}
+
+static void abort_prepare(BlkActionState *common, Error **errp)
 {
     error_setg(errp, "Transaction aborted using Abort action");
 }
 
-static void abort_commit(BlkTransactionState *common)
+static void abort_commit(BlkActionState *common)
 {
     g_assert_not_reached(); /* this action never succeeds */
 }
 
-static const BdrvActionOps actions[] = {
+static const BlkActionOps actions[] = {
+    [TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT] = {
+        .instance_size = sizeof(ExternalSnapshotState),
+        .prepare  = external_snapshot_prepare,
+        .commit   = external_snapshot_commit,
+        .abort = external_snapshot_abort,
+        .clean = external_snapshot_clean,
+    },
     [TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC] = {
         .instance_size = sizeof(ExternalSnapshotState),
         .prepare  = external_snapshot_prepare,
         .commit   = external_snapshot_commit,
         .abort = external_snapshot_abort,
+        .clean = external_snapshot_clean,
     },
     [TRANSACTION_ACTION_KIND_DRIVE_BACKUP] = {
         .instance_size = sizeof(DriveBackupState),
@@ -1738,7 +2120,7 @@ static const BdrvActionOps actions[] = {
         .clean = blockdev_backup_clean,
     },
     [TRANSACTION_ACTION_KIND_ABORT] = {
-        .instance_size = sizeof(BlkTransactionState),
+        .instance_size = sizeof(BlkActionState),
         .prepare = abort_prepare,
         .commit = abort_commit,
     },
@@ -1748,40 +2130,85 @@ static const BdrvActionOps actions[] = {
         .abort = internal_snapshot_abort,
         .clean = internal_snapshot_clean,
     },
+    [TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_ADD] = {
+        .instance_size = sizeof(BlockDirtyBitmapState),
+        .prepare = block_dirty_bitmap_add_prepare,
+        .abort = block_dirty_bitmap_add_abort,
+    },
+    [TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_CLEAR] = {
+        .instance_size = sizeof(BlockDirtyBitmapState),
+        .prepare = block_dirty_bitmap_clear_prepare,
+        .commit = block_dirty_bitmap_clear_commit,
+        .abort = block_dirty_bitmap_clear_abort,
+        .clean = block_dirty_bitmap_clear_clean,
+    }
 };
 
+/**
+ * Allocate a TransactionProperties structure if necessary, and fill
+ * that structure with desired defaults if they are unset.
+ */
+static TransactionProperties *get_transaction_properties(
+    TransactionProperties *props)
+{
+    if (!props) {
+        props = g_new0(TransactionProperties, 1);
+    }
+
+    if (!props->has_completion_mode) {
+        props->has_completion_mode = true;
+        props->completion_mode = ACTION_COMPLETION_MODE_INDIVIDUAL;
+    }
+
+    return props;
+}
+
 /*
  * 'Atomic' group operations.  The operations are performed as a set, and if
  * any fail then we roll back all operations in the group.
  */
-void qmp_transaction(TransactionActionList *dev_list, Error **errp)
+void qmp_transaction(TransactionActionList *dev_list,
+                     bool has_props,
+                     struct TransactionProperties *props,
+                     Error **errp)
 {
     TransactionActionList *dev_entry = dev_list;
-    BlkTransactionState *state, *next;
+    BlockJobTxn *block_job_txn = NULL;
+    BlkActionState *state, *next;
     Error *local_err = NULL;
 
-    QSIMPLEQ_HEAD(snap_bdrv_states, BlkTransactionState) snap_bdrv_states;
+    QSIMPLEQ_HEAD(snap_bdrv_states, BlkActionState) snap_bdrv_states;
     QSIMPLEQ_INIT(&snap_bdrv_states);
 
+    /* Does this transaction get canceled as a group on failure?
+     * If not, we don't really need to make a BlockJobTxn.
+     */
+    props = get_transaction_properties(props);
+    if (props->completion_mode != ACTION_COMPLETION_MODE_INDIVIDUAL) {
+        block_job_txn = block_job_txn_new();
+    }
+
     /* drain all i/o before any operations */
     bdrv_drain_all();
 
     /* We don't do anything in this loop that commits us to the operations */
     while (NULL != dev_entry) {
         TransactionAction *dev_info = NULL;
-        const BdrvActionOps *ops;
+        const BlkActionOps *ops;
 
         dev_info = dev_entry->value;
         dev_entry = dev_entry->next;
 
-        assert(dev_info->kind < ARRAY_SIZE(actions));
+        assert(dev_info->type < ARRAY_SIZE(actions));
 
-        ops = &actions[dev_info->kind];
+        ops = &actions[dev_info->type];
         assert(ops->instance_size > 0);
 
         state = g_malloc0(ops->instance_size);
         state->ops = ops;
         state->action = dev_info;
+        state->block_job_txn = block_job_txn;
+        state->txn_props = props;
         QSIMPLEQ_INSERT_TAIL(&snap_bdrv_states, state, entry);
 
         state->ops->prepare(state, &local_err);
@@ -1814,42 +2241,91 @@ exit:
         }
         g_free(state);
     }
+    if (!has_props) {
+        qapi_free_TransactionProperties(props);
+    }
+    block_job_txn_unref(block_job_txn);
 }
 
+void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
+{
+    Error *local_err = NULL;
+
+    qmp_blockdev_open_tray(device, has_force, force, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    qmp_x_blockdev_remove_medium(device, errp);
+}
 
-static void eject_device(BlockBackend *blk, int force, Error **errp)
+void qmp_block_passwd(bool has_device, const char *device,
+                      bool has_node_name, const char *node_name,
+                      const char *password, Error **errp)
 {
-    BlockDriverState *bs = blk_bs(blk);
+    Error *local_err = NULL;
+    BlockDriverState *bs;
     AioContext *aio_context;
 
+    bs = bdrv_lookup_bs(has_device ? device : NULL,
+                        has_node_name ? node_name : NULL,
+                        &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     aio_context = bdrv_get_aio_context(bs);
     aio_context_acquire(aio_context);
 
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
-        goto out;
+    bdrv_add_key(bs, password, errp);
+
+    aio_context_release(aio_context);
+}
+
+void qmp_blockdev_open_tray(const char *device, bool has_force, bool force,
+                            Error **errp)
+{
+    BlockBackend *blk;
+    bool locked;
+
+    if (!has_force) {
+        force = false;
+    }
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return;
     }
+
     if (!blk_dev_has_removable_media(blk)) {
-        error_setg(errp, "Device '%s' is not removable",
-                   bdrv_get_device_name(bs));
-        goto out;
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
     }
 
-    if (blk_dev_is_medium_locked(blk) && !blk_dev_is_tray_open(blk)) {
-        blk_dev_eject_request(blk, force);
-        if (!force) {
-            error_setg(errp, "Device '%s' is locked",
-                       bdrv_get_device_name(bs));
-            goto out;
-        }
+    if (!blk_dev_has_tray(blk)) {
+        /* Ignore this command on tray-less devices */
+        return;
     }
 
-    bdrv_close(bs);
+    if (blk_dev_is_tray_open(blk)) {
+        return;
+    }
 
-out:
-    aio_context_release(aio_context);
+    locked = blk_dev_is_medium_locked(blk);
+    if (locked) {
+        blk_dev_eject_request(blk, force);
+    }
+
+    if (!locked || force) {
+        blk_dev_change_media_cb(blk, false);
+    }
 }
 
-void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
+void qmp_blockdev_close_tray(const char *device, Error **errp)
 {
     BlockBackend *blk;
 
@@ -1860,92 +2336,235 @@ void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
         return;
     }
 
-    eject_device(blk, force, errp);
+    if (!blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (!blk_dev_has_tray(blk)) {
+        /* Ignore this command on tray-less devices */
+        return;
+    }
+
+    if (!blk_dev_is_tray_open(blk)) {
+        return;
+    }
+
+    blk_dev_change_media_cb(blk, true);
 }
 
-void qmp_block_passwd(bool has_device, const char *device,
-                      bool has_node_name, const char *node_name,
-                      const char *password, Error **errp)
+void qmp_x_blockdev_remove_medium(const char *device, Error **errp)
 {
-    Error *local_err = NULL;
+    BlockBackend *blk;
     BlockDriverState *bs;
     AioContext *aio_context;
+    bool has_device;
 
-    bs = bdrv_lookup_bs(has_device ? device : NULL,
-                        has_node_name ? node_name : NULL,
-                        &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return;
+    }
+
+    /* For BBs without a device, we can exchange the BDS tree at will */
+    has_device = blk_get_attached_dev(blk);
+
+    if (has_device && !blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (has_device && blk_dev_has_tray(blk) && !blk_dev_is_tray_open(blk)) {
+        error_setg(errp, "Tray of device '%s' is not open", device);
+        return;
+    }
+
+    bs = blk_bs(blk);
+    if (!bs) {
         return;
     }
 
     aio_context = bdrv_get_aio_context(bs);
     aio_context_acquire(aio_context);
 
-    bdrv_add_key(bs, password, errp);
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
+        goto out;
+    }
+
+    /* This follows the convention established by bdrv_make_anon() */
+    if (bs->device_list.tqe_prev) {
+        bdrv_device_remove(bs);
+    }
+
+    blk_remove_bs(blk);
+
+    if (!blk_dev_has_tray(blk)) {
+        /* For tray-less devices, blockdev-open-tray is a no-op (or may not be
+         * called at all); therefore, the medium needs to be ejected here.
+         * Do it after blk_remove_bs() so blk_is_inserted(blk) returns the @load
+         * value passed here (i.e. false). */
+        blk_dev_change_media_cb(blk, false);
+    }
 
+out:
     aio_context_release(aio_context);
 }
 
-/* Assumes AioContext is held */
-static void qmp_bdrv_open_encrypted(BlockDriverState *bs, const char *filename,
-                                    int bdrv_flags, BlockDriver *drv,
-                                    const char *password, Error **errp)
+static void qmp_blockdev_insert_anon_medium(const char *device,
+                                            BlockDriverState *bs, Error **errp)
 {
-    Error *local_err = NULL;
-    int ret;
+    BlockBackend *blk;
+    bool has_device;
 
-    ret = bdrv_open(&bs, filename, NULL, NULL, bdrv_flags, drv, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
 
-    bdrv_add_key(bs, password, errp);
+    /* For BBs without a device, we can exchange the BDS tree at will */
+    has_device = blk_get_attached_dev(blk);
+
+    if (has_device && !blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (has_device && blk_dev_has_tray(blk) && !blk_dev_is_tray_open(blk)) {
+        error_setg(errp, "Tray of device '%s' is not open", device);
+        return;
+    }
+
+    if (blk_bs(blk)) {
+        error_setg(errp, "There already is a medium in device '%s'", device);
+        return;
+    }
+
+    blk_insert_bs(blk, bs);
+
+    QTAILQ_INSERT_TAIL(&bdrv_states, bs, device_list);
+
+    if (!blk_dev_has_tray(blk)) {
+        /* For tray-less devices, blockdev-close-tray is a no-op (or may not be
+         * called at all); therefore, the medium needs to be pushed into the
+         * slot here.
+         * Do it after blk_insert_bs() so blk_is_inserted(blk) returns the @load
+         * value passed here (i.e. true). */
+        blk_dev_change_media_cb(blk, true);
+    }
 }
 
-void qmp_change_blockdev(const char *device, const char *filename,
-                         const char *format, Error **errp)
+void qmp_x_blockdev_insert_medium(const char *device, const char *node_name,
+                                  Error **errp)
 {
-    BlockBackend *blk;
     BlockDriverState *bs;
-    AioContext *aio_context;
-    BlockDriver *drv = NULL;
-    int bdrv_flags;
+
+    bs = bdrv_find_node(node_name);
+    if (!bs) {
+        error_setg(errp, "Node '%s' not found", node_name);
+        return;
+    }
+
+    if (bs->blk) {
+        error_setg(errp, "Node '%s' is already in use by '%s'", node_name,
+                   blk_name(bs->blk));
+        return;
+    }
+
+    qmp_blockdev_insert_anon_medium(device, bs, errp);
+}
+
+void qmp_blockdev_change_medium(const char *device, const char *filename,
+                                bool has_format, const char *format,
+                                bool has_read_only,
+                                BlockdevChangeReadOnlyMode read_only,
+                                Error **errp)
+{
+    BlockBackend *blk;
+    BlockDriverState *medium_bs = NULL;
+    int bdrv_flags, ret;
+    QDict *options = NULL;
     Error *err = NULL;
 
     blk = blk_by_name(device);
     if (!blk) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                   "Device '%s' not found", device);
-        return;
+        goto fail;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
+    if (blk_bs(blk)) {
+        blk_update_root_state(blk);
+    }
 
-    if (format) {
-        drv = bdrv_find_whitelisted_format(format, bs->read_only);
-        if (!drv) {
-            error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
+    bdrv_flags = blk_get_open_flags_from_root_state(blk);
+
+    if (!has_read_only) {
+        read_only = BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN;
     }
 
-    eject_device(blk, 0, &err);
+    switch (read_only) {
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN:
+        break;
+
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_READ_ONLY:
+        bdrv_flags &= ~BDRV_O_RDWR;
+        break;
+
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_READ_WRITE:
+        bdrv_flags |= BDRV_O_RDWR;
+        break;
+
+    default:
+        abort();
+    }
+
+    if (has_format) {
+        options = qdict_new();
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
+
+    assert(!medium_bs);
+    ret = bdrv_open(&medium_bs, filename, NULL, options, bdrv_flags, errp);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    blk_apply_root_state(blk, medium_bs);
+
+    bdrv_add_key(medium_bs, NULL, &err);
     if (err) {
         error_propagate(errp, err);
-        goto out;
+        goto fail;
     }
 
-    bdrv_flags = bdrv_is_read_only(bs) ? 0 : BDRV_O_RDWR;
-    bdrv_flags |= bdrv_is_snapshot(bs) ? BDRV_O_SNAPSHOT : 0;
+    qmp_blockdev_open_tray(device, false, false, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
 
-    qmp_bdrv_open_encrypted(bs, filename, bdrv_flags, drv, NULL, errp);
+    qmp_x_blockdev_remove_medium(device, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
 
-out:
-    aio_context_release(aio_context);
+    qmp_blockdev_insert_anon_medium(device, medium_bs, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
+
+    qmp_blockdev_close_tray(device, errp);
+
+fail:
+    /* If the medium has been inserted, the device has its own reference, so
+     * ours must be relinquished; and if it has not been inserted successfully,
+     * the reference must be relinquished anyway */
+    bdrv_unref(medium_bs);
 }
 
 /* throttling disk I/O limits */
@@ -1982,7 +2601,15 @@ void qmp_block_set_io_throttle(const char *device, int64_t bps, int64_t bps_rd,
                   "Device '%s' not found", device);
         return;
     }
+
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
+
     bs = blk_bs(blk);
+    if (!bs) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
 
     memset(&cfg, 0, sizeof(cfg));
     cfg.buckets[THROTTLE_BPS_TOTAL].avg = bps;
@@ -2017,27 +2644,25 @@ void qmp_block_set_io_throttle(const char *device, int64_t bps, int64_t bps_rd,
     }
 
     if (!check_throttle_config(&cfg, errp)) {
-        return;
+        goto out;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
-
     if (throttle_enabled(&cfg)) {
         /* Enable I/O limits if they're not enabled yet, otherwise
          * just update the throttling group. */
-        if (!bs->io_limits_enabled) {
+        if (!bs->throttle_state) {
             bdrv_io_limits_enable(bs, has_group ? group : device);
         } else if (has_group) {
             bdrv_io_limits_update_group(bs, group);
         }
         /* Set the new throttling configuration */
         bdrv_set_io_limits(bs, &cfg);
-    } else if (bs->io_limits_enabled) {
+    } else if (bs->throttle_state) {
         /* If all throttling settings are set to 0, disable I/O limits */
         bdrv_io_limits_disable(bs);
     }
 
+out:
     aio_context_release(aio_context);
 }
 
@@ -2131,7 +2756,7 @@ void qmp_block_dirty_bitmap_clear(const char *node, const char *name,
         goto out;
     }
 
-    bdrv_clear_dirty_bitmap(bitmap);
+    bdrv_clear_dirty_bitmap(bitmap, NULL);
 
  out:
     aio_context_release(aio_context);
@@ -2150,7 +2775,6 @@ void hmp_drive_del(Monitor *mon, const QDict *qdict)
         error_report("Device '%s' not found", id);
         return;
     }
-    bs = blk_bs(blk);
 
     if (!blk_legacy_dinfo(blk)) {
         error_report("Deleting device added with blockdev-add"
@@ -2158,16 +2782,19 @@ void hmp_drive_del(Monitor *mon, const QDict *qdict)
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
-        error_report_err(local_err);
-        aio_context_release(aio_context);
-        return;
-    }
+    bs = blk_bs(blk);
+    if (bs) {
+        if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
+            error_report_err(local_err);
+            aio_context_release(aio_context);
+            return;
+        }
 
-    bdrv_close(bs);
+        bdrv_close(bs);
+    }
 
     /* if we have a device attached to this BlockDriverState
      * then we need to make the drive anonymous until the device
@@ -2177,8 +2804,8 @@ void hmp_drive_del(Monitor *mon, const QDict *qdict)
     if (blk_get_attached_dev(blk)) {
         blk_hide_on_behalf_of_hmp_drive_del(blk);
         /* Further I/O must not pause the guest */
-        bdrv_set_on_error(bs, BLOCKDEV_ON_ERROR_REPORT,
-                          BLOCKDEV_ON_ERROR_REPORT);
+        blk_set_on_error(blk, BLOCKDEV_ON_ERROR_REPORT,
+                         BLOCKDEV_ON_ERROR_REPORT);
     } else {
         blk_unref(blk);
     }
@@ -2272,8 +2899,6 @@ static void block_job_cb(void *opaque, int ret)
     } else {
         block_job_event_completed(bs->job, msg);
     }
-
-    bdrv_put_ref_bh_schedule(bs);
 }
 
 void qmp_block_stream(const char *device,
@@ -2300,11 +2925,16 @@ void qmp_block_stream(const char *device,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_STREAM, errp)) {
         goto out;
     }
@@ -2375,11 +3005,16 @@ void qmp_block_commit(const char *device,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT_SOURCE, errp)) {
         goto out;
     }
@@ -2444,15 +3079,17 @@ out:
     aio_context_release(aio_context);
 }
 
-void qmp_drive_backup(const char *device, const char *target,
-                      bool has_format, const char *format,
-                      enum MirrorSyncMode sync,
-                      bool has_mode, enum NewImageMode mode,
-                      bool has_speed, int64_t speed,
-                      bool has_bitmap, const char *bitmap,
-                      bool has_on_source_error, BlockdevOnError on_source_error,
-                      bool has_on_target_error, BlockdevOnError on_target_error,
-                      Error **errp)
+static void do_drive_backup(const char *device, const char *target,
+                            bool has_format, const char *format,
+                            enum MirrorSyncMode sync,
+                            bool has_mode, enum NewImageMode mode,
+                            bool has_speed, int64_t speed,
+                            bool has_bitmap, const char *bitmap,
+                            bool has_on_source_error,
+                            BlockdevOnError on_source_error,
+                            bool has_on_target_error,
+                            BlockdevOnError on_target_error,
+                            BlockJobTxn *txn, Error **errp)
 {
     BlockBackend *blk;
     BlockDriverState *bs;
@@ -2460,7 +3097,7 @@ void qmp_drive_backup(const char *device, const char *target,
     BlockDriverState *source = NULL;
     BdrvDirtyBitmap *bmap = NULL;
     AioContext *aio_context;
-    BlockDriver *drv = NULL;
+    QDict *options = NULL;
     Error *local_err = NULL;
     int flags;
     int64_t size;
@@ -2485,28 +3122,21 @@ void qmp_drive_backup(const char *device, const char *target,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
     /* Although backup_run has this check too, we need to use bs->drv below, so
      * do an early check redundantly. */
-    if (!bdrv_is_inserted(bs)) {
+    if (!blk_is_available(blk)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         goto out;
     }
+    bs = blk_bs(blk);
 
     if (!has_format) {
         format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
     }
-    if (format) {
-        drv = bdrv_find_format(format);
-        if (!drv) {
-            error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
-    }
 
     /* Early check to avoid creating target */
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
@@ -2518,7 +3148,7 @@ void qmp_drive_backup(const char *device, const char *target,
     /* See if we have a backing HD we can use to create our new image
      * on top of. */
     if (sync == MIRROR_SYNC_MODE_TOP) {
-        source = bs->backing_hd;
+        source = backing_bs(bs);
         if (!source) {
             sync = MIRROR_SYNC_MODE_FULL;
         }
@@ -2534,7 +3164,7 @@ void qmp_drive_backup(const char *device, const char *target,
     }
 
     if (mode != NEW_IMAGE_MODE_EXISTING) {
-        assert(format && drv);
+        assert(format);
         if (source) {
             bdrv_img_create(target, format, source->filename,
                             source->drv->format_name, NULL,
@@ -2550,8 +3180,13 @@ void qmp_drive_backup(const char *device, const char *target,
         goto out;
     }
 
+    if (format) {
+        options = qdict_new();
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
+
     target_bs = NULL;
-    ret = bdrv_open(&target_bs, target, NULL, NULL, flags, drv, &local_err);
+    ret = bdrv_open(&target_bs, target, NULL, options, flags, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -2563,13 +3198,14 @@ void qmp_drive_backup(const char *device, const char *target,
         bmap = bdrv_find_dirty_bitmap(bs, bitmap);
         if (!bmap) {
             error_setg(errp, "Bitmap '%s' could not be found", bitmap);
+            bdrv_unref(target_bs);
             goto out;
         }
     }
 
     backup_start(bs, target_bs, speed, sync, bmap,
                  on_source_error, on_target_error,
-                 block_job_cb, bs, &local_err);
+                 block_job_cb, bs, txn, &local_err);
     if (local_err != NULL) {
         bdrv_unref(target_bs);
         error_propagate(errp, local_err);
@@ -2580,21 +3216,39 @@ out:
     aio_context_release(aio_context);
 }
 
+void qmp_drive_backup(const char *device, const char *target,
+                      bool has_format, const char *format,
+                      enum MirrorSyncMode sync,
+                      bool has_mode, enum NewImageMode mode,
+                      bool has_speed, int64_t speed,
+                      bool has_bitmap, const char *bitmap,
+                      bool has_on_source_error, BlockdevOnError on_source_error,
+                      bool has_on_target_error, BlockdevOnError on_target_error,
+                      Error **errp)
+{
+    return do_drive_backup(device, target, has_format, format, sync,
+                           has_mode, mode, has_speed, speed,
+                           has_bitmap, bitmap,
+                           has_on_source_error, on_source_error,
+                           has_on_target_error, on_target_error,
+                           NULL, errp);
+}
+
 BlockDeviceInfoList *qmp_query_named_block_nodes(Error **errp)
 {
     return bdrv_named_nodes_list(errp);
 }
 
-void qmp_blockdev_backup(const char *device, const char *target,
+void do_blockdev_backup(const char *device, const char *target,
                          enum MirrorSyncMode sync,
                          bool has_speed, int64_t speed,
                          bool has_on_source_error,
                          BlockdevOnError on_source_error,
                          bool has_on_target_error,
                          BlockdevOnError on_target_error,
-                         Error **errp)
+                         BlockJobTxn *txn, Error **errp)
 {
-    BlockBackend *blk;
+    BlockBackend *blk, *target_blk;
     BlockDriverState *bs;
     BlockDriverState *target_bs;
     Error *local_err = NULL;
@@ -2615,22 +3269,32 @@ void qmp_blockdev_backup(const char *device, const char *target,
         error_setg(errp, "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    blk = blk_by_name(target);
-    if (!blk) {
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
+    target_blk = blk_by_name(target);
+    if (!target_blk) {
         error_setg(errp, "Device '%s' not found", target);
         goto out;
     }
-    target_bs = blk_bs(blk);
+
+    if (!blk_is_available(target_blk)) {
+        error_setg(errp, "Device '%s' has no medium", target);
+        goto out;
+    }
+    target_bs = blk_bs(target_blk);
 
     bdrv_ref(target_bs);
     bdrv_set_aio_context(target_bs, aio_context);
     backup_start(bs, target_bs, speed, sync, NULL, on_source_error,
-                 on_target_error, block_job_cb, bs, &local_err);
+                 on_target_error, block_job_cb, bs, txn, &local_err);
     if (local_err != NULL) {
         bdrv_unref(target_bs);
         error_propagate(errp, local_err);
@@ -2639,6 +3303,21 @@ out:
     aio_context_release(aio_context);
 }
 
+void qmp_blockdev_backup(const char *device, const char *target,
+                         enum MirrorSyncMode sync,
+                         bool has_speed, int64_t speed,
+                         bool has_on_source_error,
+                         BlockdevOnError on_source_error,
+                         bool has_on_target_error,
+                         BlockdevOnError on_target_error,
+                         Error **errp)
+{
+    do_blockdev_backup(device, target, sync, has_speed, speed,
+                       has_on_source_error, on_source_error,
+                       has_on_target_error, on_target_error,
+                       NULL, errp);
+}
+
 void qmp_drive_mirror(const char *device, const char *target,
                       bool has_format, const char *format,
                       bool has_node_name, const char *node_name,
@@ -2657,9 +3336,8 @@ void qmp_drive_mirror(const char *device, const char *target,
     BlockDriverState *bs;
     BlockDriverState *source, *target_bs;
     AioContext *aio_context;
-    BlockDriver *drv = NULL;
     Error *local_err = NULL;
-    QDict *options = NULL;
+    QDict *options;
     int flags;
     int64_t size;
     int ret;
@@ -2703,33 +3381,26 @@ void qmp_drive_mirror(const char *device, const char *target,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    if (!bdrv_is_inserted(bs)) {
+    if (!blk_is_available(blk)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         goto out;
     }
+    bs = blk_bs(blk);
 
     if (!has_format) {
         format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
     }
-    if (format) {
-        drv = bdrv_find_format(format);
-        if (!drv) {
-            error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
-    }
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR, errp)) {
         goto out;
     }
 
     flags = bs->open_flags | BDRV_O_RDWR;
-    source = bs->backing_hd;
+    source = backing_bs(bs);
     if (!source && sync == MIRROR_SYNC_MODE_TOP) {
         sync = MIRROR_SYNC_MODE_FULL;
     }
@@ -2754,7 +3425,7 @@ void qmp_drive_mirror(const char *device, const char *target,
             goto out;
         }
 
-        to_replace_bs = check_to_replace_node(replaces, &local_err);
+        to_replace_bs = check_to_replace_node(bs, replaces, &local_err);
 
         if (!to_replace_bs) {
             error_propagate(errp, local_err);
@@ -2777,7 +3448,7 @@ void qmp_drive_mirror(const char *device, const char *target,
         && mode != NEW_IMAGE_MODE_EXISTING)
     {
         /* create new image w/o backing file */
-        assert(format && drv);
+        assert(format);
         bdrv_img_create(target, format,
                         NULL, NULL, NULL, size, flags, &local_err, false);
     } else {
@@ -2801,17 +3472,20 @@ void qmp_drive_mirror(const char *device, const char *target,
         goto out;
     }
 
+    options = qdict_new();
     if (has_node_name) {
-        options = qdict_new();
         qdict_put(options, "node-name", qstring_from_str(node_name));
     }
+    if (format) {
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
 
     /* Mirroring takes care of copy-on-write using the source's backing
      * file.
      */
     target_bs = NULL;
     ret = bdrv_open(&target_bs, target, NULL, options,
-                    flags | BDRV_O_NO_BACKING, drv, &local_err);
+                    flags | BDRV_O_NO_BACKING, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -2845,17 +3519,22 @@ static BlockJob *find_block_job(const char *device, AioContext **aio_context,
     BlockBackend *blk;
     BlockDriverState *bs;
 
+    *aio_context = NULL;
+
     blk = blk_by_name(device);
     if (!blk) {
         goto notfound;
     }
-    bs = blk_bs(blk);
 
-    *aio_context = bdrv_get_aio_context(bs);
+    *aio_context = blk_get_aio_context(blk);
     aio_context_acquire(*aio_context);
 
+    if (!blk_is_available(blk)) {
+        goto notfound;
+    }
+    bs = blk_bs(blk);
+
     if (!bs->job) {
-        aio_context_release(*aio_context);
         goto notfound;
     }
 
@@ -2864,7 +3543,10 @@ static BlockJob *find_block_job(const char *device, AioContext **aio_context,
 notfound:
     error_set(errp, ERROR_CLASS_DEVICE_NOT_ACTIVE,
               "No active block job on device '%s'", device);
-    *aio_context = NULL;
+    if (*aio_context) {
+        aio_context_release(*aio_context);
+        *aio_context = NULL;
+    }
     return NULL;
 }
 
@@ -2971,11 +3653,16 @@ void qmp_change_backing_file(const char *device,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     image_bs = bdrv_lookup_bs(NULL, image_node_name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -3042,17 +3729,12 @@ out:
 void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
 {
     QmpOutputVisitor *ov = qmp_output_visitor_new();
-    BlockBackend *blk;
+    BlockDriverState *bs;
+    BlockBackend *blk = NULL;
     QObject *obj;
     QDict *qdict;
     Error *local_err = NULL;
 
-    /* Require an ID in the top level */
-    if (!options->has_id) {
-        error_setg(errp, "Block device needs an ID");
-        goto fail;
-    }
-
     /* TODO Sort it out in raw-posix and drive_new(): Reject aio=native with
      * cache.direct=false instead of silently switching to aio=threads, except
      * when called from drive_new().
@@ -3080,14 +3762,33 @@ void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
 
     qdict_flatten(qdict);
 
-    blk = blockdev_init(NULL, qdict, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        goto fail;
+    if (options->has_id) {
+        blk = blockdev_init(NULL, qdict, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            goto fail;
+        }
+
+        bs = blk_bs(blk);
+    } else {
+        if (!qdict_get_try_str(qdict, "node-name")) {
+            error_setg(errp, "'id' and/or 'node-name' need to be specified for "
+                       "the root node");
+            goto fail;
+        }
+
+        bs = bds_tree_init(qdict, errp);
+        if (!bs) {
+            goto fail;
+        }
     }
 
-    if (bdrv_key_required(blk_bs(blk))) {
-        blk_unref(blk);
+    if (bs && bdrv_key_required(bs)) {
+        if (blk) {
+            blk_unref(blk);
+        } else {
+            bdrv_unref(bs);
+        }
         error_setg(errp, "blockdev-add doesn't support encrypted devices");
         goto fail;
     }
@@ -3096,6 +3797,72 @@ fail:
     qmp_output_visitor_cleanup(ov);
 }
 
+void qmp_x_blockdev_del(bool has_id, const char *id,
+                        bool has_node_name, const char *node_name, Error **errp)
+{
+    AioContext *aio_context;
+    BlockBackend *blk;
+    BlockDriverState *bs;
+
+    if (has_id && has_node_name) {
+        error_setg(errp, "Only one of id and node-name must be specified");
+        return;
+    } else if (!has_id && !has_node_name) {
+        error_setg(errp, "No block device specified");
+        return;
+    }
+
+    if (has_id) {
+        blk = blk_by_name(id);
+        if (!blk) {
+            error_setg(errp, "Cannot find block backend %s", id);
+            return;
+        }
+        if (blk_get_refcnt(blk) > 1) {
+            error_setg(errp, "Block backend %s is in use", id);
+            return;
+        }
+        bs = blk_bs(blk);
+        aio_context = blk_get_aio_context(blk);
+    } else {
+        bs = bdrv_find_node(node_name);
+        if (!bs) {
+            error_setg(errp, "Cannot find node %s", node_name);
+            return;
+        }
+        blk = bs->blk;
+        if (blk) {
+            error_setg(errp, "Node %s is in use by %s",
+                       node_name, blk_name(blk));
+            return;
+        }
+        aio_context = bdrv_get_aio_context(bs);
+    }
+
+    aio_context_acquire(aio_context);
+
+    if (bs) {
+        if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, errp)) {
+            goto out;
+        }
+
+        if (bs->refcnt > 1 || !QLIST_EMPTY(&bs->parents)) {
+            error_setg(errp, "Block device %s is in use",
+                       bdrv_get_device_or_node_name(bs));
+            goto out;
+        }
+    }
+
+    if (blk) {
+        blk_unref(blk);
+    } else {
+        bdrv_unref(bs);
+    }
+
+out:
+    aio_context_release(aio_context);
+}
+
 BlockJobInfoList *qmp_query_block_jobs(Error **errp)
 {
     BlockJobInfoList *head = NULL, **p_next = &head;
@@ -3227,6 +3994,57 @@ QemuOptsList qemu_common_drive_opts = {
             .name = "detect-zeroes",
             .type = QEMU_OPT_STRING,
             .help = "try to optimize zero writes (off, on, unmap)",
+        },{
+            .name = "stats-account-invalid",
+            .type = QEMU_OPT_BOOL,
+            .help = "whether to account for invalid I/O operations "
+                    "in the statistics",
+        },{
+            .name = "stats-account-failed",
+            .type = QEMU_OPT_BOOL,
+            .help = "whether to account for failed I/O operations "
+                    "in the statistics",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_root_bds_opts = {
+    .name = "root-bds",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_common_drive_opts.head),
+    .desc = {
+        {
+            .name = "discard",
+            .type = QEMU_OPT_STRING,
+            .help = "discard operation (ignore/off, unmap/on)",
+        },{
+            .name = "cache.writeback",
+            .type = QEMU_OPT_BOOL,
+            .help = "enables writeback mode for any caches",
+        },{
+            .name = "cache.direct",
+            .type = QEMU_OPT_BOOL,
+            .help = "enables use of O_DIRECT (bypass the host page cache)",
+        },{
+            .name = "cache.no-flush",
+            .type = QEMU_OPT_BOOL,
+            .help = "ignore any flush requests for the device",
+        },{
+            .name = "aio",
+            .type = QEMU_OPT_STRING,
+            .help = "host AIO implementation (threads, native)",
+        },{
+            .name = "read-only",
+            .type = QEMU_OPT_BOOL,
+            .help = "open drive file as read-only",
+        },{
+            .name = "copy-on-read",
+            .type = QEMU_OPT_BOOL,
+            .help = "copy read data from backing file into image file",
+        },{
+            .name = "detect-zeroes",
+            .type = QEMU_OPT_STRING,
+            .help = "try to optimize zero writes (off, on, unmap)",
         },
         { /* end of list */ }
     },
diff --git a/blockjob.c b/blockjob.c
index 62bb90663..80adb9d52 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -29,13 +29,27 @@
 #include "block/block.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qjson.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qmp-commands.h"
 #include "qemu/timer.h"
 #include "qapi-event.h"
 
+/* Transactional group of block jobs */
+struct BlockJobTxn {
+
+    /* Is this txn being cancelled? */
+    bool aborting;
+
+    /* List of jobs */
+    QLIST_HEAD(, BlockJob) jobs;
+
+    /* Reference count */
+    int refcnt;
+};
+
 void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
                        int64_t speed, BlockCompletionFunc *cb,
                        void *opaque, Error **errp)
@@ -54,10 +68,12 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
     bdrv_op_unblock(bs, BLOCK_OP_TYPE_DATAPLANE, job->blocker);
 
     job->driver        = driver;
+    job->id            = g_strdup(bdrv_get_device_name(bs));
     job->bs            = bs;
     job->cb            = cb;
     job->opaque        = opaque;
     job->busy          = true;
+    job->refcnt        = 1;
     bs->job = job;
 
     /* Only set speed when necessary to avoid NotSupported error */
@@ -66,7 +82,7 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
 
         block_job_set_speed(job, speed, &local_err);
         if (local_err) {
-            block_job_release(bs);
+            block_job_unref(job);
             error_propagate(errp, local_err);
             return NULL;
         }
@@ -74,14 +90,101 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
     return job;
 }
 
-void block_job_release(BlockDriverState *bs)
+void block_job_ref(BlockJob *job)
 {
-    BlockJob *job = bs->job;
+    ++job->refcnt;
+}
 
-    bs->job = NULL;
-    bdrv_op_unblock_all(bs, job->blocker);
-    error_free(job->blocker);
-    g_free(job);
+void block_job_unref(BlockJob *job)
+{
+    if (--job->refcnt == 0) {
+        job->bs->job = NULL;
+        bdrv_op_unblock_all(job->bs, job->blocker);
+        bdrv_unref(job->bs);
+        error_free(job->blocker);
+        g_free(job->id);
+        g_free(job);
+    }
+}
+
+static void block_job_completed_single(BlockJob *job)
+{
+    if (!job->ret) {
+        if (job->driver->commit) {
+            job->driver->commit(job);
+        }
+    } else {
+        if (job->driver->abort) {
+            job->driver->abort(job);
+        }
+    }
+    job->cb(job->opaque, job->ret);
+    if (job->txn) {
+        block_job_txn_unref(job->txn);
+    }
+    block_job_unref(job);
+}
+
+static void block_job_completed_txn_abort(BlockJob *job)
+{
+    AioContext *ctx;
+    BlockJobTxn *txn = job->txn;
+    BlockJob *other_job, *next;
+
+    if (txn->aborting) {
+        /*
+         * We are cancelled by another job, which will handle everything.
+         */
+        return;
+    }
+    txn->aborting = true;
+    /* We are the first failed job. Cancel other jobs. */
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        ctx = bdrv_get_aio_context(other_job->bs);
+        aio_context_acquire(ctx);
+    }
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        if (other_job == job || other_job->completed) {
+            /* Other jobs are "effectively" cancelled by us, set the status for
+             * them; this job, however, may or may not be cancelled, depending
+             * on the caller, so leave it. */
+            if (other_job != job) {
+                other_job->cancelled = true;
+            }
+            continue;
+        }
+        block_job_cancel_sync(other_job);
+        assert(other_job->completed);
+    }
+    QLIST_FOREACH_SAFE(other_job, &txn->jobs, txn_list, next) {
+        ctx = bdrv_get_aio_context(other_job->bs);
+        block_job_completed_single(other_job);
+        aio_context_release(ctx);
+    }
+}
+
+static void block_job_completed_txn_success(BlockJob *job)
+{
+    AioContext *ctx;
+    BlockJobTxn *txn = job->txn;
+    BlockJob *other_job, *next;
+    /*
+     * Successful completion, see if there are other running jobs in this
+     * txn.
+     */
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        if (!other_job->completed) {
+            return;
+        }
+    }
+    /* We are the last completed job, commit the transaction. */
+    QLIST_FOREACH_SAFE(other_job, &txn->jobs, txn_list, next) {
+        ctx = bdrv_get_aio_context(other_job->bs);
+        aio_context_acquire(ctx);
+        assert(other_job->ret == 0);
+        block_job_completed_single(other_job);
+        aio_context_release(ctx);
+    }
 }
 
 void block_job_completed(BlockJob *job, int ret)
@@ -89,8 +192,16 @@ void block_job_completed(BlockJob *job, int ret)
     BlockDriverState *bs = job->bs;
 
     assert(bs->job == job);
-    job->cb(job->opaque, ret);
-    block_job_release(bs);
+    assert(!job->completed);
+    job->completed = true;
+    job->ret = ret;
+    if (!job->txn) {
+        block_job_completed_single(job);
+    } else if (ret < 0 || block_job_is_cancelled(job)) {
+        block_job_completed_txn_abort(job);
+    } else {
+        block_job_completed_txn_success(job);
+    }
 }
 
 void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -113,8 +224,7 @@ void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
 void block_job_complete(BlockJob *job, Error **errp)
 {
     if (job->pause_count || job->cancelled || !job->driver->complete) {
-        error_setg(errp, QERR_BLOCK_JOB_NOT_READY,
-                   bdrv_get_device_name(job->bs));
+        error_setg(errp, QERR_BLOCK_JOB_NOT_READY, job->id);
         return;
     }
 
@@ -176,43 +286,29 @@ struct BlockFinishData {
     int ret;
 };
 
-static void block_job_finish_cb(void *opaque, int ret)
-{
-    struct BlockFinishData *data = opaque;
-
-    data->cancelled = block_job_is_cancelled(data->job);
-    data->ret = ret;
-    data->cb(data->opaque, ret);
-}
-
 static int block_job_finish_sync(BlockJob *job,
                                  void (*finish)(BlockJob *, Error **errp),
                                  Error **errp)
 {
-    struct BlockFinishData data;
     BlockDriverState *bs = job->bs;
     Error *local_err = NULL;
+    int ret;
 
     assert(bs->job == job);
 
-    /* Set up our own callback to store the result and chain to
-     * the original callback.
-     */
-    data.job = job;
-    data.cb = job->cb;
-    data.opaque = job->opaque;
-    data.ret = -EINPROGRESS;
-    job->cb = block_job_finish_cb;
-    job->opaque = &data;
+    block_job_ref(job);
     finish(job, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
+        block_job_unref(job);
         return -EBUSY;
     }
-    while (data.ret == -EINPROGRESS) {
+    while (!job->completed) {
         aio_poll(bdrv_get_aio_context(bs), true);
     }
-    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
+    ret = (job->cancelled && job->ret == 0) ? -ECANCELED : job->ret;
+    block_job_unref(job);
+    return ret;
 }
 
 /* A wrapper around block_job_cancel() taking an Error ** parameter so it may be
@@ -269,7 +365,7 @@ BlockJobInfo *block_job_query(BlockJob *job)
 {
     BlockJobInfo *info = g_new0(BlockJobInfo, 1);
     info->type      = g_strdup(BlockJobType_lookup[job->driver->job_type]);
-    info->device    = g_strdup(bdrv_get_device_name(job->bs));
+    info->device    = g_strdup(job->id);
     info->len       = job->len;
     info->busy      = job->busy;
     info->paused    = job->pause_count > 0;
@@ -291,7 +387,7 @@ static void block_job_iostatus_set_err(BlockJob *job, int error)
 void block_job_event_cancelled(BlockJob *job)
 {
     qapi_event_send_block_job_cancelled(job->driver->job_type,
-                                        bdrv_get_device_name(job->bs),
+                                        job->id,
                                         job->len,
                                         job->offset,
                                         job->speed,
@@ -301,7 +397,7 @@ void block_job_event_cancelled(BlockJob *job)
 void block_job_event_completed(BlockJob *job, const char *msg)
 {
     qapi_event_send_block_job_completed(job->driver->job_type,
-                                        bdrv_get_device_name(job->bs),
+                                        job->id,
                                         job->len,
                                         job->offset,
                                         job->speed,
@@ -315,7 +411,7 @@ void block_job_event_ready(BlockJob *job)
     job->ready = true;
 
     qapi_event_send_block_job_ready(job->driver->job_type,
-                                    bdrv_get_device_name(job->bs),
+                                    job->id,
                                     job->len,
                                     job->offset,
                                     job->speed, &error_abort);
@@ -344,7 +440,7 @@ BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
     default:
         abort();
     }
-    qapi_event_send_block_job_error(bdrv_get_device_name(job->bs),
+    qapi_event_send_block_job_error(job->id,
                                     is_read ? IO_OPERATION_TYPE_READ :
                                     IO_OPERATION_TYPE_WRITE,
                                     action, &error_abort);
@@ -353,8 +449,8 @@ BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
         job->user_paused = true;
         block_job_pause(job);
         block_job_iostatus_set_err(job, error);
-        if (bs != job->bs) {
-            bdrv_iostatus_set_err(bs, error);
+        if (bs->blk && bs != job->bs) {
+            blk_iostatus_set_err(bs->blk, error);
         }
     }
     return action;
@@ -404,3 +500,36 @@ void block_job_defer_to_main_loop(BlockJob *job,
 
     qemu_bh_schedule(data->bh);
 }
+
+BlockJobTxn *block_job_txn_new(void)
+{
+    BlockJobTxn *txn = g_new0(BlockJobTxn, 1);
+    QLIST_INIT(&txn->jobs);
+    txn->refcnt = 1;
+    return txn;
+}
+
+static void block_job_txn_ref(BlockJobTxn *txn)
+{
+    txn->refcnt++;
+}
+
+void block_job_txn_unref(BlockJobTxn *txn)
+{
+    if (txn && --txn->refcnt == 0) {
+        g_free(txn);
+    }
+}
+
+void block_job_txn_add_job(BlockJobTxn *txn, BlockJob *job)
+{
+    if (!txn) {
+        return;
+    }
+
+    assert(!job->txn);
+    job->txn = txn;
+
+    QLIST_INSERT_HEAD(&txn->jobs, job, txn_list);
+    block_job_txn_ref(txn);
+}
diff --git a/bsd-user/elfload.c b/bsd-user/elfload.c
index 2bf57eb1f..59a7bdf0c 100644
--- a/bsd-user/elfload.c
+++ b/bsd-user/elfload.c
@@ -740,8 +740,7 @@ static void padzero(abi_ulong elf_bss, abi_ulong last_bss)
            size must be known */
         if (qemu_real_host_page_size < qemu_host_page_size) {
             abi_ulong end_addr, end_addr1;
-            end_addr1 = (elf_bss + qemu_real_host_page_size - 1) &
-                ~(qemu_real_host_page_size - 1);
+            end_addr1 = REAL_HOST_PAGE_ALIGN(elf_bss);
             end_addr = HOST_PAGE_ALIGN(elf_bss);
             if (end_addr1 < end_addr) {
                 mmap((void *)g2h(end_addr1), end_addr - end_addr1,
@@ -1355,9 +1354,7 @@ int load_elf_binary(struct linux_binprm * bprm, struct target_pt_regs * regs,
             }
         }
         if (!bprm->p) {
-            if (elf_interpreter) {
-                free(elf_interpreter);
-            }
+            free(elf_interpreter);
             free (elf_phdata);
             close(bprm->fd);
             return -E2BIG;
@@ -1371,7 +1368,6 @@ int load_elf_binary(struct linux_binprm * bprm, struct target_pt_regs * regs,
     info->mmap = 0;
     elf_entry = (abi_ulong) elf_ex.e_entry;
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * In case where user has not explicitly set the guest_base, we
      * probe here that should we set it automatically.
@@ -1392,7 +1388,6 @@ int load_elf_binary(struct linux_binprm * bprm, struct target_pt_regs * regs,
             }
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     /* Do this so that we can load the interpreter, if need be.  We will
        change some of these later */
diff --git a/bsd-user/main.c b/bsd-user/main.c
index ee68daa39..adf2de0d9 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -35,12 +35,10 @@
 #include "qemu/envlist.h"
 
 int singlestep;
-#if defined(CONFIG_USE_GUEST_BASE)
 unsigned long mmap_min_addr;
 unsigned long guest_base;
 int have_guest_base;
 unsigned long reserved_va;
-#endif
 
 static const char *interp_prefix = CONFIG_QEMU_INTERP_PREFIX;
 const char *qemu_uname_release;
@@ -110,7 +108,7 @@ void cpu_list_unlock(void)
 
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 static void write_dt(void *ptr, unsigned long addr, unsigned long limit,
@@ -682,9 +680,7 @@ static void usage(void)
            "-drop-ld-preload  drop LD_PRELOAD for target process\n"
            "-E var=value      sets/modifies targets environment variable(s)\n"
            "-U var            unsets targets environment variable(s)\n"
-#if defined(CONFIG_USE_GUEST_BASE)
            "-B address        set guest_base address to address\n"
-#endif
            "-bsd type         select emulated BSD type FreeBSD/NetBSD/OpenBSD (default)\n"
            "\n"
            "Debug options:\n"
@@ -830,11 +826,9 @@ int main(int argc, char **argv)
 #endif
                 exit(1);
             }
-#if defined(CONFIG_USE_GUEST_BASE)
         } else if (!strcmp(r, "B")) {
            guest_base = strtol(argv[optind++], NULL, 0);
            have_guest_base = 1;
-#endif
         } else if (!strcmp(r, "drop-ld-preload")) {
             (void) envlist_unsetenv(envlist, "LD_PRELOAD");
         } else if (!strcmp(r, "bsd")) {
@@ -923,7 +917,6 @@ int main(int argc, char **argv)
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * Now that page sizes are configured in cpu_init() we can do
      * proper page alignment for guest_base.
@@ -950,7 +943,6 @@ int main(int argc, char **argv)
             fclose(fp);
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     if (loader_exec(filename, argv+optind, target_environ, regs, info) != 0) {
         printf("Error loading %s\n", filename);
@@ -964,9 +956,7 @@ int main(int argc, char **argv)
     free(target_environ);
 
     if (qemu_log_enabled()) {
-#if defined(CONFIG_USE_GUEST_BASE)
         qemu_log("guest_base  0x%lx\n", guest_base);
-#endif
         log_page_dump();
 
         qemu_log("start_brk   0x" TARGET_ABI_FMT_lx "\n", info->start_brk);
@@ -986,12 +976,10 @@ int main(int argc, char **argv)
     syscall_init();
     signal_init();
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /* Now that we've loaded the binary, GUEST_BASE is fixed.  Delay
        generating the prologue until now so that the prologue can take
        the real value of GUEST_BASE into account.  */
     tcg_prologue_init(&tcg_ctx);
-#endif
 
     /* build Task State */
     memset(ts, 0, sizeof(TaskState));
diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h
index 5362297fe..735cb4042 100644
--- a/bsd-user/qemu.h
+++ b/bsd-user/qemu.h
@@ -101,9 +101,7 @@ typedef struct TaskState {
 
 void init_task_state(TaskState *ts);
 extern const char *qemu_uname_release;
-#if defined(CONFIG_USE_GUEST_BASE)
 extern unsigned long mmap_min_addr;
-#endif
 
 /* ??? See if we can avoid exposing so much of the loader internals.  */
 /*
@@ -213,8 +211,6 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
                        abi_ulong new_addr);
 int target_msync(abi_ulong start, abi_ulong len, int flags);
 extern unsigned long last_brk;
-void mmap_lock(void);
-void mmap_unlock(void);
 void cpu_list_lock(void);
 void cpu_list_unlock(void);
 #if defined(CONFIG_USE_NPTL)
diff --git a/bsd-user/signal.c b/bsd-user/signal.c
index 445f69e83..e4ee2d0e8 100644
--- a/bsd-user/signal.c
+++ b/bsd-user/signal.c
@@ -21,7 +21,6 @@
 #include <string.h>
 #include <stdarg.h>
 #include <unistd.h>
-#include <signal.h>
 #include <errno.h>
 
 #include "qemu.h"
diff --git a/configure b/configure
index cd219d8e3..6ca6c6488 100755
--- a/configure
+++ b/configure
@@ -8,6 +8,9 @@
 CLICOLOR_FORCE= GREP_OPTIONS=
 unset CLICOLOR_FORCE GREP_OPTIONS
 
+# Don't allow CCACHE, if present, to use cached results of compile tests!
+export CCACHE_RECACHE=yes
+
 # Temporary directory used for files created while
 # configure runs. Since it is in the build directory
 # we can safely blow away any previous version of it
@@ -242,7 +245,6 @@ vnc="yes"
 sparse="no"
 uuid=""
 vde=""
-vnc_tls=""
 vnc_sasl=""
 vnc_jpeg=""
 vnc_png=""
@@ -262,6 +264,7 @@ rdma=""
 gprof="no"
 debug_tcg="no"
 debug="no"
+fortify_source=""
 strip_opt="yes"
 tcg_interpreter="no"
 bigendian="no"
@@ -293,7 +296,6 @@ cocoa="no"
 softmmu="yes"
 linux_user="no"
 bsd_user="no"
-guest_base="yes"
 aix="no"
 blobs="yes"
 pkgversion=""
@@ -304,7 +306,7 @@ trace_backends="nop"
 trace_file="trace"
 spice=""
 rbd=""
-smartcard_nss=""
+smartcard=""
 libusb=""
 usb_redir=""
 opengl=""
@@ -330,14 +332,19 @@ glusterfs_zerofill="no"
 archipelago="no"
 gtk=""
 gtkabi=""
+gtk_gl="no"
 gnutls=""
 gnutls_hash=""
+nettle=""
+gcrypt=""
 vte=""
+virglrenderer=""
 tpm="yes"
 libssh2=""
 vhdx=""
 numa=""
 tcmalloc="no"
+jemalloc="no"
 
 # parse CC options first
 for opt do
@@ -720,6 +727,8 @@ if test "$mingw32" = "yes" ; then
   QEMU_CFLAGS="-DWIN32_LEAN_AND_MEAN -DWINVER=0x501 $QEMU_CFLAGS"
   # enable C99/POSIX format strings (needs mingw32-runtime 3.15 or later)
   QEMU_CFLAGS="-D__USE_MINGW_ANSI_STDIO=1 $QEMU_CFLAGS"
+  # MinGW needs -mthreads for TLS and macro _MT.
+  QEMU_CFLAGS="-mthreads $QEMU_CFLAGS"
   LIBS="-lwinmm -lws2_32 -liphlpapi $LIBS"
   write_c_skeleton;
   if compile_prog "" "-liberty" ; then
@@ -733,7 +742,7 @@ if test "$mingw32" = "yes" ; then
   sysconfdir="\${prefix}"
   local_statedir=
   confsuffix=""
-  libs_qga="-lws2_32 -lwinmm -lpowrprof -liphlpapi $libs_qga"
+  libs_qga="-lws2_32 -lwinmm -lpowrprof -liphlpapi -lnetapi32 $libs_qga"
 fi
 
 werror=""
@@ -784,6 +793,9 @@ for opt do
   --enable-modules)
       modules="yes"
   ;;
+  --disable-modules)
+      modules="no"
+  ;;
   --cpu=*)
   ;;
   --target-list=*) target_list="$optarg"
@@ -873,6 +885,7 @@ for opt do
       debug_tcg="yes"
       debug="yes"
       strip_opt="no"
+      fortify_source="no"
   ;;
   --enable-sparse) sparse="yes"
   ;;
@@ -880,10 +893,6 @@ for opt do
   ;;
   --disable-strip) strip_opt="no"
   ;;
-  --disable-vnc-tls) vnc_tls="no"
-  ;;
-  --enable-vnc-tls) vnc_tls="yes"
-  ;;
   --disable-vnc-sasl) vnc_sasl="no"
   ;;
   --enable-vnc-sasl) vnc_sasl="yes"
@@ -956,7 +965,6 @@ for opt do
   ;;
   --enable-cocoa)
       cocoa="yes" ;
-      sdl="no" ;
       audio_drv_list="coreaudio `echo $audio_drv_list | sed s,coreaudio,,g`"
   ;;
   --disable-system) softmmu="no"
@@ -976,10 +984,6 @@ for opt do
   ;;
   --enable-bsd-user) bsd_user="yes"
   ;;
-  --enable-guest-base) guest_base="yes"
-  ;;
-  --disable-guest-base) guest_base="no"
-  ;;
   --enable-pie) pie="yes"
   ;;
   --disable-pie) pie="no"
@@ -1046,9 +1050,9 @@ for opt do
   ;;
   --enable-xfsctl) xfs="yes"
   ;;
-  --disable-smartcard-nss) smartcard_nss="no"
+  --disable-smartcard) smartcard="no"
   ;;
-  --enable-smartcard-nss) smartcard_nss="yes"
+  --enable-smartcard) smartcard="yes"
   ;;
   --disable-libusb) libusb="no"
   ;;
@@ -1119,6 +1123,14 @@ for opt do
   ;;
   --enable-gnutls) gnutls="yes"
   ;;
+  --disable-nettle) nettle="no"
+  ;;
+  --enable-nettle) nettle="yes"
+  ;;
+  --disable-gcrypt) gcrypt="no"
+  ;;
+  --enable-gcrypt) gcrypt="yes"
+  ;;
   --enable-rdma) rdma="yes"
   ;;
   --disable-rdma) rdma="no"
@@ -1129,6 +1141,10 @@ for opt do
   ;;
   --enable-vte) vte="yes"
   ;;
+  --disable-virglrenderer) virglrenderer="no"
+  ;;
+  --enable-virglrenderer) virglrenderer="yes"
+  ;;
   --disable-tpm) tpm="no"
   ;;
   --enable-tpm) tpm="yes"
@@ -1149,6 +1165,10 @@ for opt do
   ;;
   --enable-tcmalloc) tcmalloc="yes"
   ;;
+  --disable-jemalloc) jemalloc="no"
+  ;;
+  --enable-jemalloc) jemalloc="yes"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"
@@ -1163,18 +1183,14 @@ fi
 
 # Note that if the Python conditional here evaluates True we will exit
 # with status 1 which is a shell 'false' value.
-if ! $python -c 'import sys; sys.exit(sys.version_info < (2,4) or sys.version_info >= (3,))'; then
-  error_exit "Cannot use '$python', Python 2.4 or later is required." \
+if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then
+  error_exit "Cannot use '$python', Python 2.6 or later is required." \
       "Note that Python 3 or later is not yet supported." \
       "Use --python=/path/to/python to specify a supported Python."
 fi
 
-# The -B switch was added in Python 2.6.
-# If it is supplied, compiled files are not written.
-# Use it for Python versions which support it.
-if $python -B -c 'import sys; sys.exit(0)' 2>/dev/null; then
-  python="$python -B"
-fi
+# Suppress writing compiled files
+python="$python -B"
 
 case "$cpu" in
     ppc)
@@ -1315,7 +1331,6 @@ disabled with --disable-FEATURE, default is enabled if available:
   user            supported user emulation targets
   linux-user      all linux usermode emulation targets
   bsd-user        all BSD usermode emulation targets
-  guest-base      GUEST_BASE support for usermode emulation targets
   docs            build documentation
   guest-agent     build the QEMU Guest Agent
   guest-agent-msi build guest agent Windows MSI installation package
@@ -1326,6 +1341,8 @@ disabled with --disable-FEATURE, default is enabled if available:
   sparse          sparse checker
 
   gnutls          GNUTLS cryptography support
+  nettle          nettle cryptography support
+  gcrypt          libgcrypt cryptography support
   sdl             SDL UI
   --with-sdlabi     select preferred SDL ABI 1.2 or 2.0
   gtk             gtk UI
@@ -1333,7 +1350,6 @@ disabled with --disable-FEATURE, default is enabled if available:
   vte             vte support for the gtk UI
   curses          curses UI
   vnc             VNC UI support
-  vnc-tls         TLS encryption for VNC server
   vnc-sasl        SASL encryption for VNC server
   vnc-jpeg        JPEG lossy compression for VNC server
   vnc-png         PNG compression for VNC server
@@ -1358,7 +1374,7 @@ disabled with --disable-FEATURE, default is enabled if available:
   rbd             rados block device (rbd)
   libiscsi        iscsi support
   libnfs          nfs support
-  smartcard-nss   smartcard nss support
+  smartcard       smartcard support (libcacard)
   libusb          libusb (for usb passthrough)
   usb-redir       usb network redirection support
   lzo             support of lzo compression library
@@ -1374,6 +1390,7 @@ disabled with --disable-FEATURE, default is enabled if available:
   vhdx            support for the Microsoft VHDX image format
   numa            libnuma support
   tcmalloc        tcmalloc support
+  jemalloc        jemalloc support
 
 NOTE: The object files are built at the place where configure is launched
 EOF
@@ -1411,6 +1428,9 @@ if compile_object ; then
 else
     error_exit "\"$cc\" either does not exist or does not work"
 fi
+if ! compile_prog ; then
+    error_exit "\"$cc\" cannot build an executable (is your linker broken?)"
+fi
 
 # Check that the C++ compiler exists and works with the C compiler
 if has $cxx; then
@@ -1471,6 +1491,16 @@ for flag in $gcc_flags; do
 done
 
 if test "$stack_protector" != "no"; then
+  cat > $TMPC << EOF
+int main(int argc, char *argv[])
+{
+    char arr[64], *p = arr, *c = argv[0];
+    while (*c) {
+        *p++ = *c++;
+    }
+    return 0;
+}
+EOF
   gcc_flags="-fstack-protector-strong -fstack-protector-all"
   sp_on=0
   for flag in $gcc_flags; do
@@ -1711,6 +1741,21 @@ else
 fi
 
 ##########################################
+# cocoa implies not SDL or GTK
+# (the cocoa UI code currently assumes it is always the active UI
+# and doesn't interact well with other UI frontend code)
+if test "$cocoa" = "yes"; then
+    if test "$sdl" = "yes"; then
+        error_exit "Cocoa and SDL UIs cannot both be enabled at once"
+    fi
+    if test "$gtk" = "yes"; then
+        error_exit "Cocoa and GTK UIs cannot both be enabled at once"
+    fi
+    gtk=no
+    sdl=no
+fi
+
+##########################################
 # L2TPV3 probe
 
 cat > $TMPC <<EOF
@@ -1725,6 +1770,37 @@ else
 fi
 
 ##########################################
+# MinGW / Mingw-w64 localtime_r/gmtime_r check
+
+if test "$mingw32" = "yes"; then
+    # Some versions of MinGW / Mingw-w64 lack localtime_r
+    # and gmtime_r entirely.
+    #
+    # Some versions of Mingw-w64 define a macro for
+    # localtime_r/gmtime_r.
+    #
+    # Some versions of Mingw-w64 will define functions
+    # for localtime_r/gmtime_r, but only if you have
+    # _POSIX_THREAD_SAFE_FUNCTIONS defined. For fun
+    # though, unistd.h and pthread.h both define
+    # that for you.
+    #
+    # So this #undef localtime_r and #include <unistd.h>
+    # are not in fact redundant.
+cat > $TMPC << EOF
+#include <unistd.h>
+#include <time.h>
+#undef localtime_r
+int main(void) { localtime_r(NULL, NULL); return 0; }
+EOF
+    if compile_prog "" "" ; then
+        localtime_r="yes"
+    else
+        localtime_r="no"
+    fi
+fi
+
+##########################################
 # pkg-config probe
 
 if ! has "$pkg_config_exe"; then
@@ -1827,16 +1903,34 @@ fi
 # libseccomp check
 
 if test "$seccomp" != "no" ; then
-    if test "$cpu" = "i386" || test "$cpu" = "x86_64" &&
-        $pkg_config --atleast-version=2.1.1 libseccomp; then
+    case "$cpu" in
+    i386|x86_64)
+        libseccomp_minver="2.1.0"
+        ;;
+    arm|aarch64)
+        libseccomp_minver="2.2.3"
+        ;;
+    *)
+        libseccomp_minver=""
+        ;;
+    esac
+
+    if test "$libseccomp_minver" != "" &&
+       $pkg_config --atleast-version=$libseccomp_minver libseccomp ; then
         libs_softmmu="$libs_softmmu `$pkg_config --libs libseccomp`"
         QEMU_CFLAGS="$QEMU_CFLAGS `$pkg_config --cflags libseccomp`"
-	seccomp="yes"
+        seccomp="yes"
     else
-	if test "$seccomp" = "yes"; then
-            feature_not_found "libseccomp" "Install libseccomp devel >= 2.1.1"
-	fi
-	seccomp="no"
+        if test "$seccomp" = "yes" ; then
+            if test "$libseccomp_minver" != "" ; then
+                feature_not_found "libseccomp" \
+                    "Install libseccomp devel >= $libseccomp_minver"
+            else
+                feature_not_found "libseccomp" \
+                    "libseccomp is not supported for host cpu $cpu"
+            fi
+        fi
+        seccomp="no"
     fi
 fi
 ##########################################
@@ -1867,6 +1961,51 @@ EOF
   elif
       cat > $TMPC <<EOF &&
 #include <xenctrl.h>
+#include <stdint.h>
+int main(void) {
+  xc_interface *xc = NULL;
+  xen_domain_handle_t handle;
+  xc_domain_create(xc, 0, handle, 0, NULL, NULL);
+  return 0;
+}
+EOF
+      compile_prog "" "$xen_libs"
+    then
+    xen_ctrl_version=470
+    xen=yes
+
+  # Xen 4.6
+  elif
+      cat > $TMPC <<EOF &&
+#include <xenctrl.h>
+#include <xenstore.h>
+#include <stdint.h>
+#include <xen/hvm/hvm_info_table.h>
+#if !defined(HVM_MAX_VCPUS)
+# error HVM_MAX_VCPUS not defined
+#endif
+int main(void) {
+  xc_interface *xc;
+  xs_daemon_open();
+  xc = xc_interface_open(0, 0, 0);
+  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
+  xc_gnttab_open(NULL, 0);
+  xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
+  xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
+  xc_hvm_create_ioreq_server(xc, 0, HVM_IOREQSRV_BUFIOREQ_ATOMIC, NULL);
+  xc_reserved_device_memory_map(xc, 0, 0, 0, 0, NULL, 0);
+  return 0;
+}
+EOF
+      compile_prog "" "$xen_libs"
+    then
+    xen_ctrl_version=460
+    xen=yes
+
+  # Xen 4.5
+  elif
+      cat > $TMPC <<EOF &&
+#include <xenctrl.h>
 #include <xenstore.h>
 #include <stdint.h>
 #include <xen/hvm/hvm_info_table.h>
@@ -2181,20 +2320,76 @@ else
     gnutls_hash="no"
 fi
 
-if test "$gnutls_gcrypt" != "no"; then
-    if has "libgcrypt-config"; then
+
+# If user didn't give a --disable/enable-gcrypt flag,
+# then mark as disabled if user requested nettle
+# explicitly, or if gnutls links to nettle
+if test -z "$gcrypt"
+then
+    if test "$nettle" = "yes" || test "$gnutls_nettle" = "yes"
+    then
+        gcrypt="no"
+    fi
+fi
+
+# If user didn't give a --disable/enable-nettle flag,
+# then mark as disabled if user requested gcrypt
+# explicitly, or if gnutls links to gcrypt
+if test -z "$nettle"
+then
+    if test "$gcrypt" = "yes" || test "$gnutls_gcrypt" = "yes"
+    then
+        nettle="no"
+    fi
+fi
+
+has_libgcrypt_config() {
+    if ! has "libgcrypt-config"
+    then
+	return 1
+    fi
+
+    if test -n "$cross_prefix"
+    then
+	host=`libgcrypt-config --host`
+	if test "$host-" != $cross_prefix
+	then
+	    return 1
+	fi
+    fi
+
+    return 0
+}
+
+if test "$gcrypt" != "no"; then
+    if has_libgcrypt_config; then
         gcrypt_cflags=`libgcrypt-config --cflags`
         gcrypt_libs=`libgcrypt-config --libs`
+        # Debian has remove -lgpg-error from libgcrypt-config
+        # as it "spreads unnecessary dependencies" which in
+        # turn breaks static builds...
+        if test "$static" = "yes"
+        then
+            gcrypt_libs="$gcrypt_libs -lgpg-error"
+        fi
         libs_softmmu="$gcrypt_libs $libs_softmmu"
         libs_tools="$gcrypt_libs $libs_tools"
         QEMU_CFLAGS="$QEMU_CFLAGS $gcrypt_cflags"
+        gcrypt="yes"
+        if test -z "$nettle"; then
+           nettle="no"
+        fi
     else
-        feature_not_found "gcrypt" "Install gcrypt devel"
+        if test "$gcrypt" = "yes"; then
+            feature_not_found "gcrypt" "Install gcrypt devel"
+        else
+            gcrypt="no"
+        fi
     fi
 fi
 
 
-if test "$gnutls_nettle" != "no"; then
+if test "$nettle" != "no"; then
     if $pkg_config --exists "nettle"; then
         nettle_cflags=`$pkg_config --cflags nettle`
         nettle_libs=`$pkg_config --libs nettle`
@@ -2202,11 +2397,34 @@ if test "$gnutls_nettle" != "no"; then
         libs_softmmu="$nettle_libs $libs_softmmu"
         libs_tools="$nettle_libs $libs_tools"
         QEMU_CFLAGS="$QEMU_CFLAGS $nettle_cflags"
+        nettle="yes"
     else
-        feature_not_found "nettle" "Install nettle devel"
+        if test "$nettle" = "yes"; then
+            feature_not_found "nettle" "Install nettle devel"
+        else
+            nettle="no"
+        fi
     fi
 fi
 
+if test "$gcrypt" = "yes" && test "$nettle" = "yes"
+then
+    error_exit "Only one of gcrypt & nettle can be enabled"
+fi
+
+##########################################
+# libtasn1 - only for the TLS creds/session test suite
+
+tasn1=yes
+tasn1_cflags=""
+tasn1_libs=""
+if $pkg_config --exists "libtasn1"; then
+    tasn1_cflags=`$pkg_config --cflags libtasn1`
+    tasn1_libs=`$pkg_config --libs libtasn1`
+else
+    tasn1=no
+fi
+
 
 ##########################################
 # VTE probe
@@ -2287,9 +2505,7 @@ EOF
     if test "$_sdlversion" -lt 121 ; then
       sdl_too_old=yes
     else
-      if test "$cocoa" = "no" ; then
-        sdl=yes
-      fi
+      sdl=yes
     fi
 
     # static link with sdl ? (note: sdl.pc's --static --libs is broken)
@@ -2353,28 +2569,6 @@ EOF
   fi
 fi
 
-##########################################
-# VNC TLS/WS detection
-if test "$vnc" = "yes" -a "$vnc_tls" != "no" ; then
-  cat > $TMPC <<EOF
-#include <gnutls/gnutls.h>
-int main(void) { gnutls_session_t s; gnutls_init(&s, GNUTLS_SERVER); return 0; }
-EOF
-  vnc_tls_cflags=`$pkg_config --cflags gnutls 2> /dev/null`
-  vnc_tls_libs=`$pkg_config --libs gnutls 2> /dev/null`
-  if compile_prog "$vnc_tls_cflags" "$vnc_tls_libs" ; then
-    if test "$vnc_tls" != "no" ; then
-      vnc_tls=yes
-    fi
-    libs_softmmu="$vnc_tls_libs $libs_softmmu"
-    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_tls_cflags"
-  else
-    if test "$vnc_tls" = "yes" ; then
-      feature_not_found "vnc-tls" "Install gnutls devel"
-    fi
-    vnc_tls=no
-  fi
-fi
 
 ##########################################
 # VNC SASL detection
@@ -3149,28 +3343,17 @@ fi
 libs_softmmu="$libs_softmmu $fdt_libs"
 
 ##########################################
-# opengl probe (for sdl2, milkymist-tmu2)
-
-# GLX probe, used by milkymist-tmu2
-# this is temporary, code will be switched to egl mid-term.
-cat > $TMPC << EOF
-#include <X11/Xlib.h>
-#include <GL/gl.h>
-#include <GL/glx.h>
-int main(void) { glBegin(0); glXQueryVersion(0,0,0); return 0; }
-EOF
-if compile_prog "" "-lGL -lX11" ; then
-  have_glx=yes
-else
-  have_glx=no
-fi
+# opengl probe (for sdl2, gtk, milkymist-tmu2)
 
 if test "$opengl" != "no" ; then
-  opengl_pkgs="gl glesv2 epoxy egl"
-  if $pkg_config $opengl_pkgs x11 && test "$have_glx" = "yes"; then
+  opengl_pkgs="epoxy"
+  if $pkg_config $opengl_pkgs x11; then
     opengl_cflags="$($pkg_config --cflags $opengl_pkgs) $x11_cflags"
     opengl_libs="$($pkg_config --libs $opengl_pkgs) $x11_libs"
     opengl=yes
+    if test "$gtk" = "yes" && $pkg_config --exists "$gtkpackage >= 3.16"; then
+        gtk_gl="yes"
+    fi
   else
     if test "$opengl" = "yes" ; then
       feature_not_found "opengl" "Please install opengl (mesa) devel pkgs: $opengl_pkgs"
@@ -3361,6 +3544,11 @@ EOF
   fi
 fi
 
+if test "$tcmalloc" = "yes" && test "$jemalloc" = "yes" ; then
+    echo "ERROR: tcmalloc && jemalloc can't be used at the same time"
+    exit 1
+fi
+
 ##########################################
 # tcmalloc probe
 
@@ -3378,6 +3566,22 @@ EOF
 fi
 
 ##########################################
+# jemalloc probe
+
+if test "$jemalloc" = "yes" ; then
+  cat > $TMPC << EOF
+#include <stdlib.h>
+int main(void) { malloc(1); return 0; }
+EOF
+
+  if compile_prog "" "-ljemalloc" ; then
+    LIBS="-ljemalloc $LIBS"
+  else
+    feature_not_found "jemalloc" "install jemalloc devel"
+  fi
+fi
+
+##########################################
 # signalfd probe
 signalfd="no"
 cat > $TMPC << EOF
@@ -3405,6 +3609,22 @@ if compile_prog "" "" ; then
   eventfd=yes
 fi
 
+# check if memfd is supported
+memfd=no
+cat > $TMPC << EOF
+#include <sys/memfd.h>
+
+int main(void)
+{
+    return memfd_create("foo", MFD_ALLOW_SEALING);
+}
+EOF
+if compile_prog "" "" ; then
+  memfd=yes
+fi
+
+
+
 # check for fallocate
 fallocate=no
 cat > $TMPC << EOF
@@ -3760,34 +3980,20 @@ EOF
   fi
 fi
 
-# check for libcacard for smartcard support
+# check for smartcard support
 smartcard_cflags=""
-# TODO - what's the minimal nss version we support?
-if test "$smartcard_nss" != "no"; then
-  cat > $TMPC << EOF
-#include <pk11pub.h>
-int main(void) { PK11_FreeSlot(0); return 0; }
-EOF
-    # FIXME: do not include $glib_* in here
-    nss_libs="$($pkg_config --libs nss 2>/dev/null) $glib_libs"
-    nss_cflags="$($pkg_config --cflags nss 2>/dev/null) $glib_cflags"
-    test_cflags="$nss_cflags"
-    # The header files in nss < 3.13.3 have a bug which causes them to
-    # emit a warning. If we're going to compile QEMU with -Werror, then
-    # test that the headers don't have this bug. Otherwise we would pass
-    # the configure test but fail to compile QEMU later.
-    if test "$werror" = "yes"; then
-        test_cflags="-Werror $test_cflags"
-    fi
-    if test -n "$libtool" &&
-       $pkg_config --atleast-version=3.12.8 nss && \
-      compile_prog "$test_cflags" "$nss_libs"; then
-        smartcard_nss="yes"
+if test "$smartcard" != "no"; then
+    if $pkg_config libcacard; then
+        libcacard_cflags=$($pkg_config --cflags libcacard)
+        libcacard_libs=$($pkg_config --libs libcacard)
+        QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
+        libs_softmmu="$libs_softmmu $libcacard_libs"
+        smartcard="yes"
     else
-        if test "$smartcard_nss" = "yes"; then
-            feature_not_found "nss" "Install nss devel >= 3.12.8"
+        if test "$smartcard" = "yes"; then
+            feature_not_found "smartcard" "Install libcacard devel"
         fi
-        smartcard_nss="no"
+        smartcard="no"
     fi
 fi
 
@@ -3845,6 +4051,7 @@ EOF
     guest_agent_with_vss="yes"
     QEMU_CFLAGS="$QEMU_CFLAGS $vss_win32_include"
     libs_qga="-lole32 -loleaut32 -lshlwapi -luuid -lstdc++ -Wl,--enable-stdcall-fixup $libs_qga"
+    qga_vss_provider="qga/vss-win32/qga-vss.dll qga/vss-win32/qga-vss.tlb"
   else
     if test "$vss_win32_sdk" != "" ; then
       echo "ERROR: Please download and install Microsoft VSS SDK:"
@@ -3899,55 +4106,24 @@ EOF
 fi
 
 ##########################################
-# Guest agent Window MSI  package
-
-if test "$guest_agent" != yes; then
-  if test "$guest_agent_msi" = yes; then
-    error_exit "MSI guest agent package requires guest agent enabled"
-  fi
-  guest_agent_msi=no
-elif test "$mingw32" != "yes"; then
-  if test "$guest_agent_msi" = "yes"; then
-    error_exit "MSI guest agent package is available only for MinGW Windows cross-compilation"
-  fi
-  guest_agent_msi=no
-elif ! has wixl; then
-  if test "$guest_agent_msi" = "yes"; then
-    error_exit "MSI guest agent package requires wixl tool installed ( usually from msitools package )"
-  fi
-  guest_agent_msi=no
-fi
-
-if test "$guest_agent_msi" != "no"; then
-  if test "$guest_agent_with_vss" = "yes"; then
-    QEMU_GA_MSI_WITH_VSS="-D InstallVss"
-  fi
-
-  if test "$QEMU_GA_MANUFACTURER" = ""; then
-    QEMU_GA_MANUFACTURER=QEMU
-  fi
-
-  if test "$QEMU_GA_DISTRO" = ""; then
-    QEMU_GA_DISTRO=Linux
-  fi
+# virgl renderer probe
 
-  if test "$QEMU_GA_VERSION" = ""; then
-      QEMU_GA_VERSION=`cat $source_path/VERSION`
+if test "$virglrenderer" != "no" ; then
+  cat > $TMPC << EOF
+#include <virglrenderer.h>
+int main(void) { virgl_renderer_poll(); return 0; }
+EOF
+  virgl_cflags=$($pkg_config --cflags virglrenderer 2>/dev/null)
+  virgl_libs=$($pkg_config --libs virglrenderer 2>/dev/null)
+  if $pkg_config virglrenderer >/dev/null 2>&1 && \
+     compile_prog "$virgl_cflags" "$virgl_libs" ; then
+    virglrenderer="yes"
+  else
+    if test "$virglrenderer" = "yes" ; then
+      feature_not_found "virglrenderer"
+    fi
+    virglrenderer="no"
   fi
-
-  QEMU_GA_MSI_MINGW_DLL_PATH="-D Mingw_dlls=`$pkg_config --variable=prefix glib-2.0`/bin"
-  
-  case "$cpu" in
-  x86_64)
-    QEMU_GA_MSI_ARCH="-a x64 -D Arch=64"
-    ;;
-  i386)
-    QEMU_GA_MSI_ARCH="-D Arch=32"
-    ;;
-  *)
-    error_exit "CPU $cpu not supported for building installation package"
-    ;;
-  esac
 fi
 
 ##########################################
@@ -4279,6 +4455,7 @@ fi
 # check if ccache is interfering with
 # semantic analysis of macros
 
+unset CCACHE_CPP2
 ccache_cpp2=no
 cat > $TMPC << EOF
 static const int Z = 1;
@@ -4302,6 +4479,20 @@ if ! compile_object "-Werror"; then
     ccache_cpp2=yes
 fi
 
+#################################################
+# clang does not support glibc + FORTIFY_SOURCE.
+
+if test "$fortify_source" != "no"; then
+  if echo | $cc -dM -E - | grep __clang__ > /dev/null 2>&1 ; then
+    fortify_source="no";
+  elif test -n "$cxx" &&
+       echo | $cxx -dM -E - | grep __clang__ >/dev/null 2>&1 ; then
+    fortify_source="no";
+  else
+    fortify_source="yes"
+  fi
+fi
+
 ##########################################
 # End of CC checks
 # After here, no more $cc or $ld runs
@@ -4309,8 +4500,10 @@ fi
 if test "$gcov" = "yes" ; then
   CFLAGS="-fprofile-arcs -ftest-coverage -g $CFLAGS"
   LDFLAGS="-fprofile-arcs -ftest-coverage $LDFLAGS"
-elif test "$debug" = "no" ; then
+elif test "$fortify_source" = "yes" ; then
   CFLAGS="-O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $CFLAGS"
+elif test "$debug" = "no"; then
+  CFLAGS="-O2 $CFLAGS"
 fi
 
 ##########################################
@@ -4375,6 +4568,7 @@ if test "$want_tools" = "yes" ; then
   tools="qemu-img\$(EXESUF) qemu-io\$(EXESUF) $tools"
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
     tools="qemu-nbd\$(EXESUF) $tools"
+    tools="ivshmem-client\$(EXESUF) ivshmem-server\$(EXESUF) $tools"
   fi
 fi
 if test "$softmmu" = yes ; then
@@ -4390,12 +4584,12 @@ if test "$softmmu" = yes ; then
     fi
   fi
 fi
+
+# Probe for guest agent support/options
+
 if [ "$guest_agent" != "no" ]; then
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" -o "$mingw32" = "yes" ] ; then
-      tools="qemu-ga\$(EXESUF) $tools"
-      if [ "$mingw32" = "yes" -a "$guest_agent_with_vss" = "yes" ]; then
-        tools="qga/vss-win32/qga-vss.dll qga/vss-win32/qga-vss.tlb $tools"
-      fi
+      tools="qemu-ga $tools"
       guest_agent=yes
   elif [ "$guest_agent" != yes ]; then
       guest_agent=no
@@ -4404,6 +4598,63 @@ if [ "$guest_agent" != "no" ]; then
   fi
 fi
 
+# Guest agent Window MSI  package
+
+if test "$guest_agent" != yes; then
+  if test "$guest_agent_msi" = yes; then
+    error_exit "MSI guest agent package requires guest agent enabled"
+  fi
+  guest_agent_msi=no
+elif test "$mingw32" != "yes"; then
+  if test "$guest_agent_msi" = "yes"; then
+    error_exit "MSI guest agent package is available only for MinGW Windows cross-compilation"
+  fi
+  guest_agent_msi=no
+elif ! has wixl; then
+  if test "$guest_agent_msi" = "yes"; then
+    error_exit "MSI guest agent package requires wixl tool installed ( usually from msitools package )"
+  fi
+  guest_agent_msi=no
+else
+  # we support qemu-ga, mingw32, and wixl: default to MSI enabled if it wasn't
+  # disabled explicitly
+  if test "$guest_agent_msi" != "no"; then
+    guest_agent_msi=yes
+  fi
+fi
+
+if test "$guest_agent_msi" = "yes"; then
+  if test "$guest_agent_with_vss" = "yes"; then
+    QEMU_GA_MSI_WITH_VSS="-D InstallVss"
+  fi
+
+  if test "$QEMU_GA_MANUFACTURER" = ""; then
+    QEMU_GA_MANUFACTURER=QEMU
+  fi
+
+  if test "$QEMU_GA_DISTRO" = ""; then
+    QEMU_GA_DISTRO=Linux
+  fi
+
+  if test "$QEMU_GA_VERSION" = ""; then
+      QEMU_GA_VERSION=`cat $source_path/VERSION`
+  fi
+
+  QEMU_GA_MSI_MINGW_DLL_PATH="-D Mingw_dlls=`$pkg_config --variable=prefix glib-2.0`/bin"
+
+  case "$cpu" in
+  x86_64)
+    QEMU_GA_MSI_ARCH="-a x64 -D Arch=64"
+    ;;
+  i386)
+    QEMU_GA_MSI_ARCH="-D Arch=32"
+    ;;
+  *)
+    error_exit "CPU $cpu not supported for building installation package"
+    ;;
+  esac
+fi
+
 # Mac OS X ships with a broken assembler
 roms=
 if test \( "$cpu" = "i386" -o "$cpu" = "x86_64" \) -a \
@@ -4503,12 +4754,19 @@ fi
 echo "pixman            $pixman"
 echo "SDL support       $sdl"
 echo "GTK support       $gtk"
+echo "GTK GL support    $gtk_gl"
 echo "GNUTLS support    $gnutls"
 echo "GNUTLS hash       $gnutls_hash"
-echo "GNUTLS gcrypt     $gnutls_gcrypt"
-echo "GNUTLS nettle     $gnutls_nettle ${gnutls_nettle+($nettle_version)}"
+echo "libgcrypt         $gcrypt"
+if test "$nettle" = "yes"; then
+    echo "nettle            $nettle ($nettle_version)"
+else
+    echo "nettle            $nettle"
+fi
+echo "libtasn1          $tasn1"
 echo "VTE support       $vte"
 echo "curses support    $curses"
+echo "virgl support     $virglrenderer"
 echo "curl support      $curl"
 echo "mingw32 support   $mingw32"
 echo "Audio drivers     $audio_drv_list"
@@ -4517,7 +4775,6 @@ echo "Block whitelist (ro) $block_drv_ro_whitelist"
 echo "VirtFS support    $virtfs"
 echo "VNC support       $vnc"
 if test "$vnc" = "yes" ; then
-    echo "VNC TLS support   $vnc_tls"
     echo "VNC SASL support  $vnc_sasl"
     echo "VNC JPEG support  $vnc_jpeg"
     echo "VNC PNG support   $vnc_png"
@@ -4532,7 +4789,6 @@ fi
 echo "brlapi support    $brlapi"
 echo "bluez  support    $bluez"
 echo "Documentation     $docs"
-echo "GUEST_BASE        $guest_base"
 echo "PIE               $pie"
 echo "vde support       $vde"
 echo "netmap support    $netmap"
@@ -4563,7 +4819,7 @@ echo "spice support     $spice"
 fi
 echo "rbd support       $rbd"
 echo "xfsctl support    $xfs"
-echo "nss used          $smartcard_nss"
+echo "smartcard support $smartcard"
 echo "libusb            $libusb"
 echo "usb net redir     $usb_redir"
 echo "OpenGL support    $opengl"
@@ -4572,6 +4828,7 @@ echo "libnfs support    $libnfs"
 echo "build guest agent $guest_agent"
 echo "QGA VSS support   $guest_agent_with_vss"
 echo "QGA w32 disk info $guest_agent_ntddscsi"
+echo "QGA MSI support   $guest_agent_msi"
 echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine"
 echo "coroutine pool    $coroutine_pool"
@@ -4589,6 +4846,7 @@ echo "snappy support    $snappy"
 echo "bzip2 support     $bzip2"
 echo "NUMA host support $numa"
 echo "tcmalloc support  $tcmalloc"
+echo "jemalloc support  $jemalloc"
 
 if test "$sdl_too_old" = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -4646,12 +4904,13 @@ if test "$mingw32" = "yes" ; then
   echo "CONFIG_PRODUCTVERSION=$version_major,$version_minor,$version_subminor,$version_micro" >> $config_host_mak
   if test "$guest_agent_with_vss" = "yes" ; then
     echo "CONFIG_QGA_VSS=y" >> $config_host_mak
+    echo "QGA_VSS_PROVIDER=$qga_vss_provider" >> $config_host_mak
     echo "WIN_SDK=\"$win_sdk\"" >> $config_host_mak
   fi
   if test "$guest_agent_ntddscsi" = "yes" ; then
     echo "CONFIG_QGA_NTDDDISK=y" >> $config_host_mak
   fi
-  if test "$guest_agent_msi" != "no"; then
+  if test "$guest_agent_msi" = "yes"; then
     echo "QEMU_GA_MSI_ENABLED=yes" >> $config_host_mak  
     echo "QEMU_GA_MSI_MINGW_DLL_PATH=${QEMU_GA_MSI_MINGW_DLL_PATH}" >> $config_host_mak
     echo "QEMU_GA_MSI_WITH_VSS=${QEMU_GA_MSI_WITH_VSS}" >> $config_host_mak
@@ -4724,9 +4983,6 @@ echo "CONFIG_BDRV_RO_WHITELIST=$block_drv_ro_whitelist" >> $config_host_mak
 if test "$vnc" = "yes" ; then
   echo "CONFIG_VNC=y" >> $config_host_mak
 fi
-if test "$vnc_tls" = "yes" ; then
-  echo "CONFIG_VNC_TLS=y" >> $config_host_mak
-fi
 if test "$vnc_sasl" = "yes" ; then
   echo "CONFIG_VNC_SASL=y" >> $config_host_mak
 fi
@@ -4785,6 +5041,9 @@ fi
 if test "$eventfd" = "yes" ; then
   echo "CONFIG_EVENTFD=y" >> $config_host_mak
 fi
+if test "$memfd" = "yes" ; then
+  echo "CONFIG_MEMFD=y" >> $config_host_mak
+fi
 if test "$fallocate" = "yes" ; then
   echo "CONFIG_FALLOCATE=y" >> $config_host_mak
 fi
@@ -4862,6 +5121,9 @@ if test "$gtk" = "yes" ; then
   echo "CONFIG_GTK=y" >> $config_host_mak
   echo "CONFIG_GTKABI=$gtkabi" >> $config_host_mak
   echo "GTK_CFLAGS=$gtk_cflags" >> $config_host_mak
+  if test "$gtk_gl" = "yes" ; then
+    echo "CONFIG_GTK_GL=y" >> $config_host_mak
+  fi
 fi
 if test "$gnutls" = "yes" ; then
   echo "CONFIG_GNUTLS=y" >> $config_host_mak
@@ -4869,17 +5131,25 @@ fi
 if test "$gnutls_hash" = "yes" ; then
   echo "CONFIG_GNUTLS_HASH=y" >> $config_host_mak
 fi
-if test "$gnutls_gcrypt" = "yes" ; then
-  echo "CONFIG_GNUTLS_GCRYPT=y" >> $config_host_mak
+if test "$gcrypt" = "yes" ; then
+  echo "CONFIG_GCRYPT=y" >> $config_host_mak
 fi
-if test "$gnutls_nettle" = "yes" ; then
-  echo "CONFIG_GNUTLS_NETTLE=y" >> $config_host_mak
+if test "$nettle" = "yes" ; then
+  echo "CONFIG_NETTLE=y" >> $config_host_mak
   echo "CONFIG_NETTLE_VERSION_MAJOR=${nettle_version%%.*}" >> $config_host_mak
 fi
+if test "$tasn1" = "yes" ; then
+  echo "CONFIG_TASN1=y" >> $config_host_mak
+fi
 if test "$vte" = "yes" ; then
   echo "CONFIG_VTE=y" >> $config_host_mak
   echo "VTE_CFLAGS=$vte_cflags" >> $config_host_mak
 fi
+if test "$virglrenderer" = "yes" ; then
+  echo "CONFIG_VIRGL=y" >> $config_host_mak
+  echo "VIRGL_CFLAGS=$virgl_cflags" >> $config_host_mak
+  echo "VIRGL_LIBS=$virgl_libs" >> $config_host_mak
+fi
 if test "$xen" = "yes" ; then
   echo "CONFIG_XEN_BACKEND=y" >> $config_host_mak
   echo "CONFIG_XEN_CTRL_INTERFACE_VERSION=$xen_ctrl_version" >> $config_host_mak
@@ -4937,10 +5207,8 @@ if test "$spice" = "yes" ; then
   echo "CONFIG_SPICE=y" >> $config_host_mak
 fi
 
-if test "$smartcard_nss" = "yes" ; then
-  echo "CONFIG_SMARTCARD_NSS=y" >> $config_host_mak
-  echo "NSS_LIBS=$nss_libs" >> $config_host_mak
-  echo "NSS_CFLAGS=$nss_cflags" >> $config_host_mak
+if test "$smartcard" = "yes" ; then
+  echo "CONFIG_SMARTCARD=y" >> $config_host_mak
 fi
 
 if test "$libusb" = "yes" ; then
@@ -4992,6 +5260,9 @@ fi
 if test "$zero_malloc" = "yes" ; then
   echo "CONFIG_ZERO_MALLOC=y" >> $config_host_mak
 fi
+if test "$localtime_r" = "yes" ; then
+  echo "CONFIG_LOCALTIME_R=y" >> $config_host_mak
+fi
 if test "$qom_cast_debug" = "yes" ; then
   echo "CONFIG_QOM_CAST_DEBUG=y" >> $config_host_mak
 fi
@@ -5199,6 +5470,8 @@ echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "DSOSUF=$DSOSUF" >> $config_host_mak
 echo "LDFLAGS_SHARED=$LDFLAGS_SHARED" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
+echo "TASN1_LIBS=$tasn1_libs" >> $config_host_mak
+echo "TASN1_CFLAGS=$tasn1_cflags" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
 echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
 if test "$gcov" = "yes" ; then
@@ -5379,7 +5652,9 @@ case "$target_name" in
     echo "TARGET_ABI32=y" >> $config_target_mak
   ;;
   s390x)
-    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml s390-vx.xml"
+    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml s390-vx.xml s390-cr.xml s390-virt.xml"
+  ;;
+  tilegx)
   ;;
   tricore)
   ;;
@@ -5442,6 +5717,7 @@ case "$target_name" in
       echo "CONFIG_KVM=y" >> $config_target_mak
       if test "$vhost_net" = "yes" ; then
         echo "CONFIG_VHOST_NET=y" >> $config_target_mak
+        echo "CONFIG_VHOST_NET_TEST_$target_name=y" >> $config_host_mak
       fi
     fi
 esac
@@ -5469,9 +5745,6 @@ fi
 if test "$target_user_only" = "yes" -a "$bflt" = "yes"; then
   echo "TARGET_HAS_BFLT=y" >> $config_target_mak
 fi
-if test "$target_user_only" = "yes" -a "$guest_base" = "yes"; then
-  echo "CONFIG_USE_GUEST_BASE=y" >> $config_target_mak
-fi
 if test "$target_bsd_user" = "yes" ; then
   echo "CONFIG_BSD_USER=y" >> $config_target_mak
 fi
@@ -5481,91 +5754,76 @@ fi
 cflags=""
 ldflags=""
 
+disas_config() {
+  echo "CONFIG_${1}_DIS=y" >> $config_target_mak
+  echo "CONFIG_${1}_DIS=y" >> config-all-disas.mak
+}
+
 for i in $ARCH $TARGET_BASE_ARCH ; do
   case "$i" in
   alpha)
-    echo "CONFIG_ALPHA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ALPHA_DIS=y"  >> config-all-disas.mak
+    disas_config "ALPHA"
   ;;
   aarch64)
     if test -n "${cxx}"; then
-      echo "CONFIG_ARM_A64_DIS=y"  >> $config_target_mak
-      echo "CONFIG_ARM_A64_DIS=y"  >> config-all-disas.mak
+      disas_config "ARM_A64"
     fi
   ;;
   arm)
-    echo "CONFIG_ARM_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ARM_DIS=y"  >> config-all-disas.mak
+    disas_config "ARM"
     if test -n "${cxx}"; then
-      echo "CONFIG_ARM_A64_DIS=y"  >> $config_target_mak
-      echo "CONFIG_ARM_A64_DIS=y"  >> config-all-disas.mak
+      disas_config "ARM_A64"
     fi
   ;;
   cris)
-    echo "CONFIG_CRIS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_CRIS_DIS=y"  >> config-all-disas.mak
+    disas_config "CRIS"
   ;;
   hppa)
-    echo "CONFIG_HPPA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_HPPA_DIS=y"  >> config-all-disas.mak
+    disas_config "HPPA"
   ;;
   i386|x86_64|x32)
-    echo "CONFIG_I386_DIS=y"  >> $config_target_mak
-    echo "CONFIG_I386_DIS=y"  >> config-all-disas.mak
+    disas_config "I386"
   ;;
   ia64*)
-    echo "CONFIG_IA64_DIS=y"  >> $config_target_mak
-    echo "CONFIG_IA64_DIS=y"  >> config-all-disas.mak
+    disas_config "IA64"
   ;;
   lm32)
-    echo "CONFIG_LM32_DIS=y"  >> $config_target_mak
-    echo "CONFIG_LM32_DIS=y"  >> config-all-disas.mak
+    disas_config "LM32"
   ;;
   m68k)
-    echo "CONFIG_M68K_DIS=y"  >> $config_target_mak
-    echo "CONFIG_M68K_DIS=y"  >> config-all-disas.mak
+    disas_config "M68K"
   ;;
   microblaze*)
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> config-all-disas.mak
+    disas_config "MICROBLAZE"
   ;;
   mips*)
-    echo "CONFIG_MIPS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MIPS_DIS=y"  >> config-all-disas.mak
+    disas_config "MIPS"
   ;;
   moxie*)
-    echo "CONFIG_MOXIE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MOXIE_DIS=y"  >> config-all-disas.mak
+    disas_config "MOXIE"
   ;;
   or32)
-    echo "CONFIG_OPENRISC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_OPENRISC_DIS=y"  >> config-all-disas.mak
+    disas_config "OPENRISC"
   ;;
   ppc*)
-    echo "CONFIG_PPC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_PPC_DIS=y"  >> config-all-disas.mak
+    disas_config "PPC"
   ;;
   s390*)
-    echo "CONFIG_S390_DIS=y"  >> $config_target_mak
-    echo "CONFIG_S390_DIS=y"  >> config-all-disas.mak
+    disas_config "S390"
   ;;
   sh4)
-    echo "CONFIG_SH4_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SH4_DIS=y"  >> config-all-disas.mak
+    disas_config "SH4"
   ;;
   sparc*)
-    echo "CONFIG_SPARC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SPARC_DIS=y"  >> config-all-disas.mak
+    disas_config "SPARC"
   ;;
   xtensa*)
-    echo "CONFIG_XTENSA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_XTENSA_DIS=y"  >> config-all-disas.mak
+    disas_config "XTENSA"
   ;;
   esac
 done
 if test "$tcg_interpreter" = "yes" ; then
-  echo "CONFIG_TCI_DIS=y"  >> $config_target_mak
-  echo "CONFIG_TCI_DIS=y"  >> config-all-disas.mak
+  disas_config "TCI"
 fi
 
 case "$ARCH" in
@@ -5600,10 +5858,6 @@ if [ "$pixman" = "internal" ]; then
   echo "config-host.h: subdir-pixman" >> $config_host_mak
 fi
 
-if test "$rdma" = "yes" ; then
-echo "CONFIG_RDMA=y" >> $config_host_mak
-fi
-
 if [ "$dtc_internal" = "yes" ]; then
   echo "config-host.h: subdir-dtc" >> $config_host_mak
 fi
diff --git a/contrib/ivshmem-client/Makefile.objs b/contrib/ivshmem-client/Makefile.objs
new file mode 100644
index 000000000..bfab2d20d
--- /dev/null
+++ b/contrib/ivshmem-client/Makefile.objs
@@ -0,0 +1 @@
+ivshmem-client-obj-y = ivshmem-client.o main.o
diff --git a/contrib/ivshmem-client/ivshmem-client.c b/contrib/ivshmem-client/ivshmem-client.c
new file mode 100644
index 000000000..31619d80e
--- /dev/null
+++ b/contrib/ivshmem-client/ivshmem-client.c
@@ -0,0 +1,446 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "qemu-common.h"
+#include "qemu/queue.h"
+
+#include "ivshmem-client.h"
+
+/* log a message on stdout if verbose=1 */
+#define IVSHMEM_CLIENT_DEBUG(client, fmt, ...) do { \
+        if ((client)->verbose) {         \
+            printf(fmt, ## __VA_ARGS__); \
+        }                                \
+    } while (0)
+
+/* read message from the unix socket */
+static int
+ivshmem_client_read_one_msg(IvshmemClient *client, int64_t *index, int *fd)
+{
+    int ret;
+    struct msghdr msg;
+    struct iovec iov[1];
+    union {
+        struct cmsghdr cmsg;
+        char control[CMSG_SPACE(sizeof(int))];
+    } msg_control;
+    struct cmsghdr *cmsg;
+
+    iov[0].iov_base = index;
+    iov[0].iov_len = sizeof(*index);
+
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+    msg.msg_control = &msg_control;
+    msg.msg_controllen = sizeof(msg_control);
+
+    ret = recvmsg(client->sock_fd, &msg, 0);
+    if (ret < sizeof(*index)) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read message: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+    if (ret == 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "lost connection to server\n");
+        return -1;
+    }
+
+    *index = GINT64_FROM_LE(*index);
+    *fd = -1;
+
+    for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+
+        if (cmsg->cmsg_len != CMSG_LEN(sizeof(int)) ||
+            cmsg->cmsg_level != SOL_SOCKET ||
+            cmsg->cmsg_type != SCM_RIGHTS) {
+            continue;
+        }
+
+        memcpy(fd, CMSG_DATA(cmsg), sizeof(*fd));
+    }
+
+    return 0;
+}
+
+/* free a peer when the server advertises a disconnection or when the
+ * client is freed */
+static void
+ivshmem_client_free_peer(IvshmemClient *client, IvshmemClientPeer *peer)
+{
+    unsigned vector;
+
+    QTAILQ_REMOVE(&client->peer_list, peer, next);
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        close(peer->vectors[vector]);
+    }
+
+    g_free(peer);
+}
+
+/* handle message coming from server (new peer, new vectors) */
+static int
+ivshmem_client_handle_server_msg(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    int64_t peer_id;
+    int ret, fd;
+
+    ret = ivshmem_client_read_one_msg(client, &peer_id, &fd);
+    if (ret < 0) {
+        return -1;
+    }
+
+    /* can return a peer or the local client */
+    peer = ivshmem_client_search_peer(client, peer_id);
+
+    /* delete peer */
+    if (fd == -1) {
+
+        if (peer == NULL || peer == &client->local) {
+            IVSHMEM_CLIENT_DEBUG(client, "receive delete for invalid "
+                                 "peer %" PRId64 "\n", peer_id);
+            return -1;
+        }
+
+        IVSHMEM_CLIENT_DEBUG(client, "delete peer id = %" PRId64 "\n", peer_id);
+        ivshmem_client_free_peer(client, peer);
+        return 0;
+    }
+
+    /* new peer */
+    if (peer == NULL) {
+        peer = g_malloc0(sizeof(*peer));
+        peer->id = peer_id;
+        peer->vectors_count = 0;
+        QTAILQ_INSERT_TAIL(&client->peer_list, peer, next);
+        IVSHMEM_CLIENT_DEBUG(client, "new peer id = %" PRId64 "\n", peer_id);
+    }
+
+    /* new vector */
+    IVSHMEM_CLIENT_DEBUG(client, "  new vector %d (fd=%d) for peer id %"
+                         PRId64 "\n", peer->vectors_count, fd, peer->id);
+    if (peer->vectors_count >= G_N_ELEMENTS(peer->vectors)) {
+        IVSHMEM_CLIENT_DEBUG(client, "Too many vectors received, failing");
+        return -1;
+    }
+
+    peer->vectors[peer->vectors_count] = fd;
+    peer->vectors_count++;
+
+    return 0;
+}
+
+/* init a new ivshmem client */
+int
+ivshmem_client_init(IvshmemClient *client, const char *unix_sock_path,
+                    IvshmemClientNotifCb notif_cb, void *notif_arg,
+                    bool verbose)
+{
+    int ret;
+    unsigned i;
+
+    memset(client, 0, sizeof(*client));
+
+    ret = snprintf(client->unix_sock_path, sizeof(client->unix_sock_path),
+                   "%s", unix_sock_path);
+
+    if (ret < 0 || ret >= sizeof(client->unix_sock_path)) {
+        IVSHMEM_CLIENT_DEBUG(client, "could not copy unix socket path\n");
+        return -1;
+    }
+
+    for (i = 0; i < IVSHMEM_CLIENT_MAX_VECTORS; i++) {
+        client->local.vectors[i] = -1;
+    }
+
+    QTAILQ_INIT(&client->peer_list);
+    client->local.id = -1;
+
+    client->notif_cb = notif_cb;
+    client->notif_arg = notif_arg;
+    client->verbose = verbose;
+    client->shm_fd = -1;
+    client->sock_fd = -1;
+
+    return 0;
+}
+
+/* create and connect to the unix socket */
+int
+ivshmem_client_connect(IvshmemClient *client)
+{
+    struct sockaddr_un sun;
+    int fd, ret;
+    int64_t tmp;
+
+    IVSHMEM_CLIENT_DEBUG(client, "connect to client %s\n",
+                         client->unix_sock_path);
+
+    client->sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (client->sock_fd < 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot create socket: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    sun.sun_family = AF_UNIX;
+    ret = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
+                   client->unix_sock_path);
+    if (ret < 0 || ret >= sizeof(sun.sun_path)) {
+        IVSHMEM_CLIENT_DEBUG(client, "could not copy unix socket path\n");
+        goto err_close;
+    }
+
+    if (connect(client->sock_fd, (struct sockaddr *)&sun, sizeof(sun)) < 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot connect to %s: %s\n", sun.sun_path,
+                             strerror(errno));
+        goto err_close;
+    }
+
+    /* first, we expect a protocol version */
+    if (ivshmem_client_read_one_msg(client, &tmp, &fd) < 0 ||
+        (tmp != IVSHMEM_PROTOCOL_VERSION) || fd != -1) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server\n");
+        goto err_close;
+    }
+
+    /* then, we expect our index + a fd == -1 */
+    if (ivshmem_client_read_one_msg(client, &client->local.id, &fd) < 0 ||
+        client->local.id < 0 || fd != -1) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server (2)\n");
+        goto err_close;
+    }
+    IVSHMEM_CLIENT_DEBUG(client, "our_id=%" PRId64 "\n", client->local.id);
+
+    /* now, we expect shared mem fd + a -1 index, note that shm fd
+     * is not used */
+    if (ivshmem_client_read_one_msg(client, &tmp, &fd) < 0 ||
+        tmp != -1 || fd < 0) {
+        if (fd >= 0) {
+            close(fd);
+        }
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server (3)\n");
+        goto err_close;
+    }
+    client->shm_fd = fd;
+    IVSHMEM_CLIENT_DEBUG(client, "shm_fd=%d\n", fd);
+
+    return 0;
+
+err_close:
+    close(client->sock_fd);
+    client->sock_fd = -1;
+    return -1;
+}
+
+/* close connection to the server, and free all peer structures */
+void
+ivshmem_client_close(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    unsigned i;
+
+    IVSHMEM_CLIENT_DEBUG(client, "close client\n");
+
+    while ((peer = QTAILQ_FIRST(&client->peer_list)) != NULL) {
+        ivshmem_client_free_peer(client, peer);
+    }
+
+    close(client->shm_fd);
+    client->shm_fd = -1;
+    close(client->sock_fd);
+    client->sock_fd = -1;
+    client->local.id = -1;
+    for (i = 0; i < IVSHMEM_CLIENT_MAX_VECTORS; i++) {
+        close(client->local.vectors[i]);
+        client->local.vectors[i] = -1;
+    }
+    client->local.vectors_count = 0;
+}
+
+/* get the fd_set according to the unix socket and peer list */
+void
+ivshmem_client_get_fds(const IvshmemClient *client, fd_set *fds, int *maxfd)
+{
+    int fd;
+    unsigned vector;
+
+    FD_SET(client->sock_fd, fds);
+    if (client->sock_fd >= *maxfd) {
+        *maxfd = client->sock_fd + 1;
+    }
+
+    for (vector = 0; vector < client->local.vectors_count; vector++) {
+        fd = client->local.vectors[vector];
+        FD_SET(fd, fds);
+        if (fd >= *maxfd) {
+            *maxfd = fd + 1;
+        }
+    }
+}
+
+/* handle events from eventfd: just print a message on notification */
+static int
+ivshmem_client_handle_event(IvshmemClient *client, const fd_set *cur, int maxfd)
+{
+    IvshmemClientPeer *peer;
+    uint64_t kick;
+    unsigned i;
+    int ret;
+
+    peer = &client->local;
+
+    for (i = 0; i < peer->vectors_count; i++) {
+        if (peer->vectors[i] >= maxfd || !FD_ISSET(peer->vectors[i], cur)) {
+            continue;
+        }
+
+        ret = read(peer->vectors[i], &kick, sizeof(kick));
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret != sizeof(kick)) {
+            IVSHMEM_CLIENT_DEBUG(client, "invalid read size = %d\n", ret);
+            errno = EINVAL;
+            return -1;
+        }
+        IVSHMEM_CLIENT_DEBUG(client, "received event on fd %d vector %d: %"
+                             PRIu64 "\n", peer->vectors[i], i, kick);
+        if (client->notif_cb != NULL) {
+            client->notif_cb(client, peer, i, client->notif_arg);
+        }
+    }
+
+    return 0;
+}
+
+/* read and handle new messages on the given fd_set */
+int
+ivshmem_client_handle_fds(IvshmemClient *client, fd_set *fds, int maxfd)
+{
+    if (client->sock_fd < maxfd && FD_ISSET(client->sock_fd, fds) &&
+        ivshmem_client_handle_server_msg(client) < 0 && errno != EINTR) {
+        IVSHMEM_CLIENT_DEBUG(client, "ivshmem_client_handle_server_msg() "
+                             "failed\n");
+        return -1;
+    } else if (ivshmem_client_handle_event(client, fds, maxfd) < 0 &&
+               errno != EINTR) {
+        IVSHMEM_CLIENT_DEBUG(client, "ivshmem_client_handle_event() failed\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+/* send a notification on a vector of a peer */
+int
+ivshmem_client_notify(const IvshmemClient *client,
+                      const IvshmemClientPeer *peer, unsigned vector)
+{
+    uint64_t kick;
+    int fd;
+
+    if (vector >= peer->vectors_count) {
+        IVSHMEM_CLIENT_DEBUG(client, "invalid vector %u on peer %" PRId64 "\n",
+                             vector, peer->id);
+        return -1;
+    }
+    fd = peer->vectors[vector];
+    IVSHMEM_CLIENT_DEBUG(client, "notify peer %" PRId64
+                         " on vector %d, fd %d\n", peer->id, vector, fd);
+
+    kick = 1;
+    if (write(fd, &kick, sizeof(kick)) != sizeof(kick)) {
+        fprintf(stderr, "could not write to %d: %s\n", peer->vectors[vector],
+                strerror(errno));
+        return -1;
+    }
+    return 0;
+}
+
+/* send a notification to all vectors of a peer */
+int
+ivshmem_client_notify_all_vects(const IvshmemClient *client,
+                                const IvshmemClientPeer *peer)
+{
+    unsigned vector;
+    int ret = 0;
+
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        if (ivshmem_client_notify(client, peer, vector) < 0) {
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+/* send a notification to all peers */
+int
+ivshmem_client_notify_broadcast(const IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    int ret = 0;
+
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        if (ivshmem_client_notify_all_vects(client, peer) < 0) {
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+/* lookup peer from its id */
+IvshmemClientPeer *
+ivshmem_client_search_peer(IvshmemClient *client, int64_t peer_id)
+{
+    IvshmemClientPeer *peer;
+
+    if (peer_id == client->local.id) {
+        return &client->local;
+    }
+
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        if (peer->id == peer_id) {
+            return peer;
+        }
+    }
+    return NULL;
+}
+
+/* dump our info, the list of peers their vectors on stdout */
+void
+ivshmem_client_dump(const IvshmemClient *client)
+{
+    const IvshmemClientPeer *peer;
+    unsigned vector;
+
+    /* dump local infos */
+    peer = &client->local;
+    printf("our_id = %" PRId64 "\n", peer->id);
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        printf("  vector %d is enabled (fd=%d)\n", vector,
+               peer->vectors[vector]);
+    }
+
+    /* dump peers */
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        printf("peer_id = %" PRId64 "\n", peer->id);
+
+        for (vector = 0; vector < peer->vectors_count; vector++) {
+            printf("  vector %d is enabled (fd=%d)\n", vector,
+                   peer->vectors[vector]);
+        }
+    }
+}
diff --git a/contrib/ivshmem-client/ivshmem-client.h b/contrib/ivshmem-client/ivshmem-client.h
new file mode 100644
index 000000000..3a4f80968
--- /dev/null
+++ b/contrib/ivshmem-client/ivshmem-client.h
@@ -0,0 +1,213 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef _IVSHMEM_CLIENT_H_
+#define _IVSHMEM_CLIENT_H_
+
+/**
+ * This file provides helper to implement an ivshmem client. It is used
+ * on the host to ask QEMU to send an interrupt to an ivshmem PCI device in a
+ * guest. QEMU also implements an ivshmem client similar to this one, they both
+ * connect to an ivshmem server.
+ *
+ * A standalone ivshmem client based on this file is provided for debug/test
+ * purposes.
+ */
+
+#include <limits.h>
+#include <sys/select.h>
+
+#include "qemu/queue.h"
+#include "hw/misc/ivshmem.h"
+
+/**
+ * Maximum number of notification vectors supported by the client
+ */
+#define IVSHMEM_CLIENT_MAX_VECTORS 64
+
+/**
+ * Structure storing a peer
+ *
+ * Each time a client connects to an ivshmem server, it is advertised to
+ * all connected clients through the unix socket. When our ivshmem
+ * client receives a notification, it creates a IvshmemClientPeer
+ * structure to store the infos of this peer.
+ *
+ * This structure is also used to store the information of our own
+ * client in (IvshmemClient)->local.
+ */
+typedef struct IvshmemClientPeer {
+    QTAILQ_ENTRY(IvshmemClientPeer) next;    /**< next in list*/
+    int64_t id;                              /**< the id of the peer */
+    int vectors[IVSHMEM_CLIENT_MAX_VECTORS]; /**< one fd per vector */
+    unsigned vectors_count;                  /**< number of vectors */
+} IvshmemClientPeer;
+QTAILQ_HEAD(IvshmemClientPeerList, IvshmemClientPeer);
+
+typedef struct IvshmemClientPeerList IvshmemClientPeerList;
+typedef struct IvshmemClient IvshmemClient;
+
+/**
+ * Typedef of callback function used when our IvshmemClient receives a
+ * notification from a peer.
+ */
+typedef void (*IvshmemClientNotifCb)(
+    const IvshmemClient *client,
+    const IvshmemClientPeer *peer,
+    unsigned vect, void *arg);
+
+/**
+ * Structure describing an ivshmem client
+ *
+ * This structure stores all information related to our client: the name
+ * of the server unix socket, the list of peers advertised by the
+ * server, our own client information, and a pointer the notification
+ * callback function used when we receive a notification from a peer.
+ */
+struct IvshmemClient {
+    char unix_sock_path[PATH_MAX];      /**< path to unix sock */
+    int sock_fd;                        /**< unix sock filedesc */
+    int shm_fd;                         /**< shm file descriptor */
+
+    IvshmemClientPeerList peer_list;    /**< list of peers */
+    IvshmemClientPeer local;            /**< our own infos */
+
+    IvshmemClientNotifCb notif_cb;      /**< notification callback */
+    void *notif_arg;                    /**< notification argument */
+
+    bool verbose;                       /**< true to enable debug */
+};
+
+/**
+ * Initialize an ivshmem client
+ *
+ * @client:         A pointer to an uninitialized IvshmemClient structure
+ * @unix_sock_path: The pointer to the unix socket file name
+ * @notif_cb:       If not NULL, the pointer to the function to be called when
+ *                  our IvshmemClient receives a notification from a peer
+ * @notif_arg:      Opaque pointer given as-is to the notification callback
+ *                  function
+ * @verbose:        True to enable debug
+ *
+ * Returns:         0 on success, or a negative value on error
+ */
+int ivshmem_client_init(IvshmemClient *client, const char *unix_sock_path,
+                        IvshmemClientNotifCb notif_cb, void *notif_arg,
+                        bool verbose);
+
+/**
+ * Connect to the server
+ *
+ * Connect to the server unix socket, and read the first initial
+ * messages sent by the server, giving the ID of the client and the file
+ * descriptor of the shared memory.
+ *
+ * @client: The ivshmem client
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_connect(IvshmemClient *client);
+
+/**
+ * Close connection to the server and free all peer structures
+ *
+ * @client: The ivshmem client
+ */
+void ivshmem_client_close(IvshmemClient *client);
+
+/**
+ * Fill a fd_set with file descriptors to be monitored
+ *
+ * This function will fill a fd_set with all file descriptors
+ * that must be polled (unix server socket and peers eventfd). The
+ * function will not initialize the fd_set, it is up to the caller
+ * to do this.
+ *
+ * @client: The ivshmem client
+ * @fds:    The fd_set to be updated
+ * @maxfd:  Must be set to the max file descriptor + 1 in fd_set. This value is
+ *          updated if this function adds a greater fd in fd_set.
+ */
+void ivshmem_client_get_fds(const IvshmemClient *client, fd_set *fds,
+                            int *maxfd);
+
+/**
+ * Read and handle new messages
+ *
+ * Given a fd_set filled by select(), handle incoming messages from
+ * server or peers.
+ *
+ * @client: The ivshmem client
+ * @fds:    The fd_set containing the file descriptors to be checked. Note
+ *          that file descriptors that are not related to our client are
+ *          ignored.
+ * @maxfd:  The maximum fd in fd_set, plus one.
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_handle_fds(IvshmemClient *client, fd_set *fds, int maxfd);
+
+/**
+ * Send a notification to a vector of a peer
+ *
+ * @client: The ivshmem client
+ * @peer:   The peer to be notified
+ * @vector: The number of the vector
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_notify(const IvshmemClient *client,
+                          const IvshmemClientPeer *peer, unsigned vector);
+
+/**
+ * Send a notification to all vectors of a peer
+ *
+ * @client: The ivshmem client
+ * @peer:   The peer to be notified
+ *
+ * Returns: 0 on success, or a negative value on error (at least one
+ *          notification failed)
+ */
+int ivshmem_client_notify_all_vects(const IvshmemClient *client,
+                                    const IvshmemClientPeer *peer);
+
+/**
+ * Broadcat a notification to all vectors of all peers
+ *
+ * @client: The ivshmem client
+ *
+ * Returns: 0 on success, or a negative value on error (at least one
+ *          notification failed)
+ */
+int ivshmem_client_notify_broadcast(const IvshmemClient *client);
+
+/**
+ * Search a peer from its identifier
+ *
+ * Return the peer structure from its peer_id. If the given peer_id is
+ * the local id, the function returns the local peer structure.
+ *
+ * @client:  The ivshmem client
+ * @peer_id: The identifier of the peer structure
+ *
+ * Returns:  The peer structure, or NULL if not found
+ */
+IvshmemClientPeer *
+ivshmem_client_search_peer(IvshmemClient *client, int64_t peer_id);
+
+/**
+ * Dump information of this ivshmem client on stdout
+ *
+ * Dump the id and the vectors of the given ivshmem client and the list
+ * of its peers and their vectors on stdout.
+ *
+ * @client: The ivshmem client
+ */
+void ivshmem_client_dump(const IvshmemClient *client);
+
+#endif /* _IVSHMEM_CLIENT_H_ */
diff --git a/contrib/ivshmem-client/main.c b/contrib/ivshmem-client/main.c
new file mode 100644
index 000000000..c004870ae
--- /dev/null
+++ b/contrib/ivshmem-client/main.c
@@ -0,0 +1,240 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu-common.h"
+
+#include "ivshmem-client.h"
+
+#define IVSHMEM_CLIENT_DEFAULT_VERBOSE        0
+#define IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH "/tmp/ivshmem_socket"
+
+typedef struct IvshmemClientArgs {
+    bool verbose;
+    const char *unix_sock_path;
+} IvshmemClientArgs;
+
+/* show ivshmem_client_usage and exit with given error code */
+static void
+ivshmem_client_usage(const char *name, int code)
+{
+    fprintf(stderr, "%s [opts]\n", name);
+    fprintf(stderr, "  -h: show this help\n");
+    fprintf(stderr, "  -v: verbose mode\n");
+    fprintf(stderr, "  -S <unix_sock_path>: path to the unix socket\n"
+                    "     to connect to.\n"
+                    "     default=%s\n", IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH);
+    exit(code);
+}
+
+/* parse the program arguments, exit on error */
+static void
+ivshmem_client_parse_args(IvshmemClientArgs *args, int argc, char *argv[])
+{
+    int c;
+
+    while ((c = getopt(argc, argv,
+                       "h"  /* help */
+                       "v"  /* verbose */
+                       "S:" /* unix_sock_path */
+                      )) != -1) {
+
+        switch (c) {
+        case 'h': /* help */
+            ivshmem_client_usage(argv[0], 0);
+            break;
+
+        case 'v': /* verbose */
+            args->verbose = 1;
+            break;
+
+        case 'S': /* unix_sock_path */
+            args->unix_sock_path = optarg;
+            break;
+
+        default:
+            ivshmem_client_usage(argv[0], 1);
+            break;
+        }
+    }
+}
+
+/* show command line help */
+static void
+ivshmem_client_cmdline_help(void)
+{
+    printf("dump: dump peers (including us)\n"
+           "int <peer> <vector>: notify one vector on a peer\n"
+           "int <peer> all: notify all vectors of a peer\n"
+           "int all: notify all vectors of all peers (excepting us)\n");
+}
+
+/* read stdin and handle commands */
+static int
+ivshmem_client_handle_stdin_command(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    char buf[128];
+    char *s, *token;
+    int ret;
+    int peer_id, vector;
+
+    memset(buf, 0, sizeof(buf));
+    ret = read(0, buf, sizeof(buf) - 1);
+    if (ret < 0) {
+        return -1;
+    }
+
+    s = buf;
+    while ((token = strsep(&s, "\n\r;")) != NULL) {
+        if (!strcmp(token, "")) {
+            continue;
+        }
+        if (!strcmp(token, "?")) {
+            ivshmem_client_cmdline_help();
+        }
+        if (!strcmp(token, "help")) {
+            ivshmem_client_cmdline_help();
+        } else if (!strcmp(token, "dump")) {
+            ivshmem_client_dump(client);
+        } else if (!strcmp(token, "int all")) {
+            ivshmem_client_notify_broadcast(client);
+        } else if (sscanf(token, "int %d %d", &peer_id, &vector) == 2) {
+            peer = ivshmem_client_search_peer(client, peer_id);
+            if (peer == NULL) {
+                printf("cannot find peer_id = %d\n", peer_id);
+                continue;
+            }
+            ivshmem_client_notify(client, peer, vector);
+        } else if (sscanf(token, "int %d all", &peer_id) == 1) {
+            peer = ivshmem_client_search_peer(client, peer_id);
+            if (peer == NULL) {
+                printf("cannot find peer_id = %d\n", peer_id);
+                continue;
+            }
+            ivshmem_client_notify_all_vects(client, peer);
+        } else {
+            printf("invalid command, type help\n");
+        }
+    }
+
+    printf("cmd> ");
+    fflush(stdout);
+    return 0;
+}
+
+/* listen on stdin (command line), on unix socket (notifications of new
+ * and dead peers), and on eventfd (IRQ request) */
+static int
+ivshmem_client_poll_events(IvshmemClient *client)
+{
+    fd_set fds;
+    int ret, maxfd;
+
+    while (1) {
+
+        FD_ZERO(&fds);
+        FD_SET(0, &fds); /* add stdin in fd_set */
+        maxfd = 1;
+
+        ivshmem_client_get_fds(client, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            fprintf(stderr, "select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (FD_ISSET(0, &fds) &&
+            ivshmem_client_handle_stdin_command(client) < 0 && errno != EINTR) {
+            fprintf(stderr, "ivshmem_client_handle_stdin_command() failed\n");
+            break;
+        }
+
+        if (ivshmem_client_handle_fds(client, &fds, maxfd) < 0) {
+            fprintf(stderr, "ivshmem_client_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return ret;
+}
+
+/* callback when we receive a notification (just display it) */
+static void
+ivshmem_client_notification_cb(const IvshmemClient *client,
+                               const IvshmemClientPeer *peer,
+                               unsigned vect, void *arg)
+{
+    (void)client;
+    (void)arg;
+    printf("receive notification from peer_id=%" PRId64 " vector=%u\n",
+           peer->id, vect);
+}
+
+int
+main(int argc, char *argv[])
+{
+    struct sigaction sa;
+    IvshmemClient client;
+    IvshmemClientArgs args = {
+        .verbose = IVSHMEM_CLIENT_DEFAULT_VERBOSE,
+        .unix_sock_path = IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH,
+    };
+
+    /* parse arguments, will exit on error */
+    ivshmem_client_parse_args(&args, argc, argv);
+
+    /* Ignore SIGPIPE, see this link for more info:
+     * http://www.mail-archive.com/libevent-users@monkey.org/msg01606.html */
+    sa.sa_handler = SIG_IGN;
+    sa.sa_flags = 0;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+        sigaction(SIGPIPE, &sa, 0) == -1) {
+        perror("failed to ignore SIGPIPE; sigaction");
+        return 1;
+    }
+
+    ivshmem_client_cmdline_help();
+    printf("cmd> ");
+    fflush(stdout);
+
+    if (ivshmem_client_init(&client, args.unix_sock_path,
+                            ivshmem_client_notification_cb, NULL,
+                            args.verbose) < 0) {
+        fprintf(stderr, "cannot init client\n");
+        return 1;
+    }
+
+    while (1) {
+        if (ivshmem_client_connect(&client) < 0) {
+            fprintf(stderr, "cannot connect to server, retry in 1 second\n");
+            sleep(1);
+            continue;
+        }
+
+        fprintf(stdout, "listen on server socket %d\n", client.sock_fd);
+
+        if (ivshmem_client_poll_events(&client) == 0) {
+            continue;
+        }
+
+        /* disconnected from server, reset all peers */
+        fprintf(stdout, "disconnected from server\n");
+
+        ivshmem_client_close(&client);
+    }
+
+    return 0;
+}
diff --git a/contrib/ivshmem-server/Makefile.objs b/contrib/ivshmem-server/Makefile.objs
new file mode 100644
index 000000000..c060dd369
--- /dev/null
+++ b/contrib/ivshmem-server/Makefile.objs
@@ -0,0 +1 @@
+ivshmem-server-obj-y = ivshmem-server.o main.o
diff --git a/contrib/ivshmem-server/ivshmem-server.c b/contrib/ivshmem-server/ivshmem-server.c
new file mode 100644
index 000000000..d9e26b057
--- /dev/null
+++ b/contrib/ivshmem-server/ivshmem-server.c
@@ -0,0 +1,493 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+#include "qemu-common.h"
+#include "qemu/sockets.h"
+
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#ifdef CONFIG_LINUX
+#include <sys/vfs.h>
+#endif
+
+#include "ivshmem-server.h"
+
+/* log a message on stdout if verbose=1 */
+#define IVSHMEM_SERVER_DEBUG(server, fmt, ...) do { \
+        if ((server)->verbose) {         \
+            printf(fmt, ## __VA_ARGS__); \
+        }                                \
+    } while (0)
+
+/** maximum size of a huge page, used by ivshmem_server_ftruncate() */
+#define IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE (1024 * 1024 * 1024)
+
+/** default listen backlog (number of sockets not accepted) */
+#define IVSHMEM_SERVER_LISTEN_BACKLOG 10
+
+/* send message to a client unix socket */
+static int
+ivshmem_server_send_one_msg(int sock_fd, int64_t peer_id, int fd)
+{
+    int ret;
+    struct msghdr msg;
+    struct iovec iov[1];
+    union {
+        struct cmsghdr cmsg;
+        char control[CMSG_SPACE(sizeof(int))];
+    } msg_control;
+    struct cmsghdr *cmsg;
+
+    peer_id = GINT64_TO_LE(peer_id);
+    iov[0].iov_base = &peer_id;
+    iov[0].iov_len = sizeof(peer_id);
+
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+
+    /* if fd is specified, add it in a cmsg */
+    if (fd >= 0) {
+        memset(&msg_control, 0, sizeof(msg_control));
+        msg.msg_control = &msg_control;
+        msg.msg_controllen = sizeof(msg_control);
+        cmsg = CMSG_FIRSTHDR(&msg);
+        cmsg->cmsg_level = SOL_SOCKET;
+        cmsg->cmsg_type = SCM_RIGHTS;
+        cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+        memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd));
+    }
+
+    ret = sendmsg(sock_fd, &msg, 0);
+    if (ret <= 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/* free a peer when the server advertises a disconnection or when the
+ * server is freed */
+static void
+ivshmem_server_free_peer(IvshmemServer *server, IvshmemServerPeer *peer)
+{
+    unsigned vector;
+    IvshmemServerPeer *other_peer;
+
+    IVSHMEM_SERVER_DEBUG(server, "free peer %" PRId64 "\n", peer->id);
+    close(peer->sock_fd);
+    QTAILQ_REMOVE(&server->peer_list, peer, next);
+
+    /* advertise the deletion to other peers */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id, -1);
+    }
+
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        event_notifier_cleanup(&peer->vectors[vector]);
+    }
+
+    g_free(peer);
+}
+
+/* send the peer id and the shm_fd just after a new client connection */
+static int
+ivshmem_server_send_initial_info(IvshmemServer *server, IvshmemServerPeer *peer)
+{
+    int ret;
+
+    /* send our protocol version first */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, IVSHMEM_PROTOCOL_VERSION,
+                                      -1);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send version: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    /* send the peer id to the client */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, peer->id, -1);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send peer id: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    /* send the shm_fd */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, -1, server->shm_fd);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send shm fd: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/* handle message on listening unix socket (new client connection) */
+static int
+ivshmem_server_handle_new_conn(IvshmemServer *server)
+{
+    IvshmemServerPeer *peer, *other_peer;
+    struct sockaddr_un unaddr;
+    socklen_t unaddr_len;
+    int newfd;
+    unsigned i;
+
+    /* accept the incoming connection */
+    unaddr_len = sizeof(unaddr);
+    newfd = qemu_accept(server->sock_fd,
+                        (struct sockaddr *)&unaddr, &unaddr_len);
+
+    if (newfd < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot accept() %s\n", strerror(errno));
+        return -1;
+    }
+
+    qemu_set_nonblock(newfd);
+    IVSHMEM_SERVER_DEBUG(server, "accept()=%d\n", newfd);
+
+    /* allocate new structure for this peer */
+    peer = g_malloc0(sizeof(*peer));
+    peer->sock_fd = newfd;
+
+    /* get an unused peer id */
+    /* XXX: this could use id allocation such as Linux IDA, or simply
+     * a free-list */
+    for (i = 0; i < G_MAXUINT16; i++) {
+        if (ivshmem_server_search_peer(server, server->cur_id) == NULL) {
+            break;
+        }
+        server->cur_id++;
+    }
+    if (i == G_MAXUINT16) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot allocate new client id\n");
+        close(newfd);
+        g_free(peer);
+        return -1;
+    }
+    peer->id = server->cur_id++;
+
+    /* create eventfd, one per vector */
+    peer->vectors_count = server->n_vectors;
+    for (i = 0; i < peer->vectors_count; i++) {
+        if (event_notifier_init(&peer->vectors[i], FALSE) < 0) {
+            IVSHMEM_SERVER_DEBUG(server, "cannot create eventfd\n");
+            goto fail;
+        }
+    }
+
+    /* send peer id and shm fd */
+    if (ivshmem_server_send_initial_info(server, peer) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send initial info\n");
+        goto fail;
+    }
+
+    /* advertise the new peer to others */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        for (i = 0; i < peer->vectors_count; i++) {
+            ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id,
+                                        peer->vectors[i].wfd);
+        }
+    }
+
+    /* advertise the other peers to the new one */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        for (i = 0; i < peer->vectors_count; i++) {
+            ivshmem_server_send_one_msg(peer->sock_fd, other_peer->id,
+                                        other_peer->vectors[i].wfd);
+        }
+    }
+
+    /* advertise the new peer to itself */
+    for (i = 0; i < peer->vectors_count; i++) {
+        ivshmem_server_send_one_msg(peer->sock_fd, peer->id,
+                                    event_notifier_get_fd(&peer->vectors[i]));
+    }
+
+    QTAILQ_INSERT_TAIL(&server->peer_list, peer, next);
+    IVSHMEM_SERVER_DEBUG(server, "new peer id = %" PRId64 "\n",
+                         peer->id);
+    return 0;
+
+fail:
+    while (i--) {
+        event_notifier_cleanup(&peer->vectors[i]);
+    }
+    close(newfd);
+    g_free(peer);
+    return -1;
+}
+
+/* Try to ftruncate a file to next power of 2 of shmsize.
+ * If it fails; all power of 2 above shmsize are tested until
+ * we reach the maximum huge page size. This is useful
+ * if the shm file is in a hugetlbfs that cannot be truncated to the
+ * shm_size value. */
+static int
+ivshmem_server_ftruncate(int fd, unsigned shmsize)
+{
+    int ret;
+    struct stat mapstat;
+
+    /* align shmsize to next power of 2 */
+    shmsize = pow2ceil(shmsize);
+
+    if (fstat(fd, &mapstat) != -1 && mapstat.st_size == shmsize) {
+        return 0;
+    }
+
+    while (shmsize <= IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE) {
+        ret = ftruncate(fd, shmsize);
+        if (ret == 0) {
+            return ret;
+        }
+        shmsize *= 2;
+    }
+
+    return -1;
+}
+
+/* Init a new ivshmem server */
+int
+ivshmem_server_init(IvshmemServer *server, const char *unix_sock_path,
+                    const char *shm_path, size_t shm_size, unsigned n_vectors,
+                    bool verbose)
+{
+    int ret;
+
+    memset(server, 0, sizeof(*server));
+    server->verbose = verbose;
+
+    ret = snprintf(server->unix_sock_path, sizeof(server->unix_sock_path),
+                   "%s", unix_sock_path);
+    if (ret < 0 || ret >= sizeof(server->unix_sock_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
+        return -1;
+    }
+    ret = snprintf(server->shm_path, sizeof(server->shm_path),
+                   "%s", shm_path);
+    if (ret < 0 || ret >= sizeof(server->shm_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy shm path\n");
+        return -1;
+    }
+
+    server->shm_size = shm_size;
+    server->n_vectors = n_vectors;
+
+    QTAILQ_INIT(&server->peer_list);
+
+    return 0;
+}
+
+#ifdef CONFIG_LINUX
+
+#define HUGETLBFS_MAGIC       0x958458f6
+
+static long gethugepagesize(const char *path)
+{
+    struct statfs fs;
+    int ret;
+
+    do {
+        ret = statfs(path, &fs);
+    } while (ret != 0 && errno == EINTR);
+
+    if (ret != 0) {
+        return -1;
+    }
+
+    if (fs.f_type != HUGETLBFS_MAGIC) {
+        return -1;
+    }
+
+    return fs.f_bsize;
+}
+#endif
+
+/* open shm, create and bind to the unix socket */
+int
+ivshmem_server_start(IvshmemServer *server)
+{
+    struct sockaddr_un sun;
+    int shm_fd, sock_fd, ret;
+
+    /* open shm file */
+#ifdef CONFIG_LINUX
+    long hpagesize;
+
+    hpagesize = gethugepagesize(server->shm_path);
+    if (hpagesize < 0 && errno != ENOENT) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot stat shm file %s: %s\n",
+                             server->shm_path, strerror(errno));
+    }
+
+    if (hpagesize > 0) {
+        gchar *filename = g_strdup_printf("%s/ivshmem.XXXXXX", server->shm_path);
+        IVSHMEM_SERVER_DEBUG(server, "Using hugepages: %s\n", server->shm_path);
+        shm_fd = mkstemp(filename);
+        unlink(filename);
+        g_free(filename);
+    } else
+#endif
+    {
+        IVSHMEM_SERVER_DEBUG(server, "Using POSIX shared memory: %s\n",
+                             server->shm_path);
+        shm_fd = shm_open(server->shm_path, O_CREAT|O_RDWR, S_IRWXU);
+    }
+
+    if (shm_fd < 0) {
+        fprintf(stderr, "cannot open shm file %s: %s\n", server->shm_path,
+                strerror(errno));
+        return -1;
+    }
+    if (ivshmem_server_ftruncate(shm_fd, server->shm_size) < 0) {
+        fprintf(stderr, "ftruncate(%s) failed: %s\n", server->shm_path,
+                strerror(errno));
+        goto err_close_shm;
+    }
+
+    IVSHMEM_SERVER_DEBUG(server, "create & bind socket %s\n",
+                         server->unix_sock_path);
+
+    /* create the unix listening socket */
+    sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (sock_fd < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot create socket: %s\n",
+                             strerror(errno));
+        goto err_close_shm;
+    }
+
+    sun.sun_family = AF_UNIX;
+    ret = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
+                   server->unix_sock_path);
+    if (ret < 0 || ret >= sizeof(sun.sun_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
+        goto err_close_sock;
+    }
+    if (bind(sock_fd, (struct sockaddr *)&sun, sizeof(sun)) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot connect to %s: %s\n", sun.sun_path,
+                             strerror(errno));
+        goto err_close_sock;
+    }
+
+    if (listen(sock_fd, IVSHMEM_SERVER_LISTEN_BACKLOG) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "listen() failed: %s\n", strerror(errno));
+        goto err_close_sock;
+    }
+
+    server->sock_fd = sock_fd;
+    server->shm_fd = shm_fd;
+
+    return 0;
+
+err_close_sock:
+    close(sock_fd);
+err_close_shm:
+    close(shm_fd);
+    return -1;
+}
+
+/* close connections to clients, the unix socket and the shm fd */
+void
+ivshmem_server_close(IvshmemServer *server)
+{
+    IvshmemServerPeer *peer, *npeer;
+
+    IVSHMEM_SERVER_DEBUG(server, "close server\n");
+
+    QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, npeer) {
+        ivshmem_server_free_peer(server, peer);
+    }
+
+    unlink(server->unix_sock_path);
+    close(server->sock_fd);
+    close(server->shm_fd);
+    server->sock_fd = -1;
+    server->shm_fd = -1;
+}
+
+/* get the fd_set according to the unix socket and the peer list */
+void
+ivshmem_server_get_fds(const IvshmemServer *server, fd_set *fds, int *maxfd)
+{
+    IvshmemServerPeer *peer;
+
+    if (server->sock_fd == -1) {
+        return;
+    }
+
+    FD_SET(server->sock_fd, fds);
+    if (server->sock_fd >= *maxfd) {
+        *maxfd = server->sock_fd + 1;
+    }
+
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        FD_SET(peer->sock_fd, fds);
+        if (peer->sock_fd >= *maxfd) {
+            *maxfd = peer->sock_fd + 1;
+        }
+    }
+}
+
+/* process incoming messages on the sockets in fd_set */
+int
+ivshmem_server_handle_fds(IvshmemServer *server, fd_set *fds, int maxfd)
+{
+    IvshmemServerPeer *peer, *peer_next;
+
+    if (server->sock_fd < maxfd && FD_ISSET(server->sock_fd, fds) &&
+        ivshmem_server_handle_new_conn(server) < 0 && errno != EINTR) {
+        IVSHMEM_SERVER_DEBUG(server, "ivshmem_server_handle_new_conn() "
+                             "failed\n");
+        return -1;
+    }
+
+    QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, peer_next) {
+        /* any message from a peer socket result in a close() */
+        IVSHMEM_SERVER_DEBUG(server, "peer->sock_fd=%d\n", peer->sock_fd);
+        if (peer->sock_fd < maxfd && FD_ISSET(peer->sock_fd, fds)) {
+            ivshmem_server_free_peer(server, peer);
+        }
+    }
+
+    return 0;
+}
+
+/* lookup peer from its id */
+IvshmemServerPeer *
+ivshmem_server_search_peer(IvshmemServer *server, int64_t peer_id)
+{
+    IvshmemServerPeer *peer;
+
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        if (peer->id == peer_id) {
+            return peer;
+        }
+    }
+    return NULL;
+}
+
+/* dump our info, the list of peers their vectors on stdout */
+void
+ivshmem_server_dump(const IvshmemServer *server)
+{
+    const IvshmemServerPeer *peer;
+    unsigned vector;
+
+    /* dump peers */
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        printf("peer_id = %" PRId64 "\n", peer->id);
+
+        for (vector = 0; vector < peer->vectors_count; vector++) {
+            printf("  vector %d is enabled (fd=%d)\n", vector,
+                   event_notifier_get_fd(&peer->vectors[vector]));
+        }
+    }
+}
diff --git a/contrib/ivshmem-server/ivshmem-server.h b/contrib/ivshmem-server/ivshmem-server.h
new file mode 100644
index 000000000..c9359a0a8
--- /dev/null
+++ b/contrib/ivshmem-server/ivshmem-server.h
@@ -0,0 +1,167 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef _IVSHMEM_SERVER_H_
+#define _IVSHMEM_SERVER_H_
+
+/**
+ * The ivshmem server is a daemon that creates a unix socket in listen
+ * mode. The ivshmem clients (qemu or ivshmem-client) connect to this
+ * unix socket. For each client, the server will create some eventfd
+ * (see EVENTFD(2)), one per vector. These fd are transmitted to all
+ * clients using the SCM_RIGHTS cmsg message. Therefore, each client is
+ * able to send a notification to another client without beeing
+ * "profixied" by the server.
+ *
+ * We use this mechanism to send interruptions between guests.
+ * qemu is able to transform an event on a eventfd into a PCI MSI-x
+ * interruption in the guest.
+ *
+ * The ivshmem server is also able to share the file descriptor
+ * associated to the ivshmem shared memory.
+ */
+
+#include <limits.h>
+#include <sys/select.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "qemu/event_notifier.h"
+#include "qemu/queue.h"
+#include "hw/misc/ivshmem.h"
+
+/**
+ * Maximum number of notification vectors supported by the server
+ */
+#define IVSHMEM_SERVER_MAX_VECTORS 64
+
+/**
+ * Structure storing a peer
+ *
+ * Each time a client connects to an ivshmem server, a new
+ * IvshmemServerPeer structure is created. This peer and all its
+ * vectors are advertised to all connected clients through the connected
+ * unix sockets.
+ */
+typedef struct IvshmemServerPeer {
+    QTAILQ_ENTRY(IvshmemServerPeer) next;    /**< next in list*/
+    int sock_fd;                             /**< connected unix sock */
+    int64_t id;                              /**< the id of the peer */
+    EventNotifier vectors[IVSHMEM_SERVER_MAX_VECTORS]; /**< one per vector */
+    unsigned vectors_count;                  /**< number of vectors */
+} IvshmemServerPeer;
+QTAILQ_HEAD(IvshmemServerPeerList, IvshmemServerPeer);
+
+typedef struct IvshmemServerPeerList IvshmemServerPeerList;
+
+/**
+ * Structure describing an ivshmem server
+ *
+ * This structure stores all information related to our server: the name
+ * of the server unix socket and the list of connected peers.
+ */
+typedef struct IvshmemServer {
+    char unix_sock_path[PATH_MAX];   /**< path to unix socket */
+    int sock_fd;                     /**< unix sock file descriptor */
+    char shm_path[PATH_MAX];         /**< path to shm */
+    size_t shm_size;                 /**< size of shm */
+    int shm_fd;                      /**< shm file descriptor */
+    unsigned n_vectors;              /**< number of vectors */
+    uint16_t cur_id;                 /**< id to be given to next client */
+    bool verbose;                    /**< true in verbose mode */
+    IvshmemServerPeerList peer_list; /**< list of peers */
+} IvshmemServer;
+
+/**
+ * Initialize an ivshmem server
+ *
+ * @server:         A pointer to an uninitialized IvshmemServer structure
+ * @unix_sock_path: The pointer to the unix socket file name
+ * @shm_path:       Path to the shared memory. The path corresponds to a POSIX
+ *                  shm name or a hugetlbfs mount point.
+ * @shm_size:       Size of shared memory
+ * @n_vectors:      Number of interrupt vectors per client
+ * @verbose:        True to enable verbose mode
+ *
+ * Returns:         0 on success, or a negative value on error
+ */
+int
+ivshmem_server_init(IvshmemServer *server, const char *unix_sock_path,
+                    const char *shm_path, size_t shm_size, unsigned n_vectors,
+                    bool verbose);
+
+/**
+ * Open the shm, then create and bind to the unix socket
+ *
+ * @server: The pointer to the initialized IvshmemServer structure
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_server_start(IvshmemServer *server);
+
+/**
+ * Close the server
+ *
+ * Close connections to all clients, close the unix socket and the
+ * shared memory file descriptor. The structure remains initialized, so
+ * it is possible to call ivshmem_server_start() again after a call to
+ * ivshmem_server_close().
+ *
+ * @server: The ivshmem server
+ */
+void ivshmem_server_close(IvshmemServer *server);
+
+/**
+ * Fill a fd_set with file descriptors to be monitored
+ *
+ * This function will fill a fd_set with all file descriptors that must
+ * be polled (unix server socket and peers unix socket). The function
+ * will not initialize the fd_set, it is up to the caller to do it.
+ *
+ * @server: The ivshmem server
+ * @fds:    The fd_set to be updated
+ * @maxfd:  Must be set to the max file descriptor + 1 in fd_set. This value is
+ *          updated if this function adds a greater fd in fd_set.
+ */
+void
+ivshmem_server_get_fds(const IvshmemServer *server, fd_set *fds, int *maxfd);
+
+/**
+ * Read and handle new messages
+ *
+ * Given a fd_set (for instance filled by a call to select()), handle
+ * incoming messages from peers.
+ *
+ * @server: The ivshmem server
+ * @fds:    The fd_set containing the file descriptors to be checked. Note that
+ *          file descriptors that are not related to our server are ignored.
+ * @maxfd:  The maximum fd in fd_set, plus one.
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_server_handle_fds(IvshmemServer *server, fd_set *fds, int maxfd);
+
+/**
+ * Search a peer from its identifier
+ *
+ * @server:  The ivshmem server
+ * @peer_id: The identifier of the peer structure
+ *
+ * Returns:  The peer structure, or NULL if not found
+ */
+IvshmemServerPeer *
+ivshmem_server_search_peer(IvshmemServer *server, int64_t peer_id);
+
+/**
+ * Dump information of this ivshmem server and its peers on stdout
+ *
+ * @server: The ivshmem server
+ */
+void ivshmem_server_dump(const IvshmemServer *server);
+
+#endif /* _IVSHMEM_SERVER_H_ */
diff --git a/contrib/ivshmem-server/main.c b/contrib/ivshmem-server/main.c
new file mode 100644
index 000000000..54ff001c2
--- /dev/null
+++ b/contrib/ivshmem-server/main.c
@@ -0,0 +1,263 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu-common.h"
+
+#include "ivshmem-server.h"
+
+#define IVSHMEM_SERVER_DEFAULT_VERBOSE        0
+#define IVSHMEM_SERVER_DEFAULT_FOREGROUND     0
+#define IVSHMEM_SERVER_DEFAULT_PID_FILE       "/var/run/ivshmem-server.pid"
+#define IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH "/tmp/ivshmem_socket"
+#define IVSHMEM_SERVER_DEFAULT_SHM_PATH       "ivshmem"
+#define IVSHMEM_SERVER_DEFAULT_SHM_SIZE       (4*1024*1024)
+#define IVSHMEM_SERVER_DEFAULT_N_VECTORS      1
+
+/* used to quit on signal SIGTERM */
+static int ivshmem_server_quit;
+
+/* arguments given by the user */
+typedef struct IvshmemServerArgs {
+    bool verbose;
+    bool foreground;
+    const char *pid_file;
+    const char *unix_socket_path;
+    const char *shm_path;
+    uint64_t shm_size;
+    unsigned n_vectors;
+} IvshmemServerArgs;
+
+/* show ivshmem_server_usage and exit with given error code */
+static void
+ivshmem_server_usage(const char *name, int code)
+{
+    fprintf(stderr, "%s [opts]\n", name);
+    fprintf(stderr, "  -h: show this help\n");
+    fprintf(stderr, "  -v: verbose mode\n");
+    fprintf(stderr, "  -F: foreground mode (default is to daemonize)\n");
+    fprintf(stderr, "  -p <pid_file>: path to the PID file (used in daemon\n"
+                    "     mode only).\n"
+                    "     Default=%s\n", IVSHMEM_SERVER_DEFAULT_SHM_PATH);
+    fprintf(stderr, "  -S <unix_socket_path>: path to the unix socket\n"
+                    "     to listen to.\n"
+                    "     Default=%s\n", IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH);
+    fprintf(stderr, "  -m <shm_path>: path to the shared memory.\n"
+                    "     The path corresponds to a POSIX shm name or a\n"
+                    "     hugetlbfs mount point.\n"
+                    "     default=%s\n", IVSHMEM_SERVER_DEFAULT_SHM_PATH);
+    fprintf(stderr, "  -l <size>: size of shared memory in bytes. The suffix\n"
+                    "     K, M and G can be used (ex: 1K means 1024).\n"
+                    "     default=%u\n", IVSHMEM_SERVER_DEFAULT_SHM_SIZE);
+    fprintf(stderr, "  -n <n_vects>: number of vectors.\n"
+                    "     default=%u\n", IVSHMEM_SERVER_DEFAULT_N_VECTORS);
+
+    exit(code);
+}
+
+/* parse the program arguments, exit on error */
+static void
+ivshmem_server_parse_args(IvshmemServerArgs *args, int argc, char *argv[])
+{
+    int c;
+    unsigned long long v;
+    Error *errp = NULL;
+
+    while ((c = getopt(argc, argv,
+                       "h"  /* help */
+                       "v"  /* verbose */
+                       "F"  /* foreground */
+                       "p:" /* pid_file */
+                       "S:" /* unix_socket_path */
+                       "m:" /* shm_path */
+                       "l:" /* shm_size */
+                       "n:" /* n_vectors */
+                      )) != -1) {
+
+        switch (c) {
+        case 'h': /* help */
+            ivshmem_server_usage(argv[0], 0);
+            break;
+
+        case 'v': /* verbose */
+            args->verbose = 1;
+            break;
+
+        case 'F': /* foreground */
+            args->foreground = 1;
+            break;
+
+        case 'p': /* pid_file */
+            args->pid_file = optarg;
+            break;
+
+        case 'S': /* unix_socket_path */
+            args->unix_socket_path = optarg;
+            break;
+
+        case 'm': /* shm_path */
+            args->shm_path = optarg;
+            break;
+
+        case 'l': /* shm_size */
+            parse_option_size("shm_size", optarg, &args->shm_size, &errp);
+            if (errp) {
+                fprintf(stderr, "cannot parse shm size: %s\n",
+                        error_get_pretty(errp));
+                error_free(errp);
+                ivshmem_server_usage(argv[0], 1);
+            }
+            break;
+
+        case 'n': /* n_vectors */
+            if (parse_uint_full(optarg, &v, 0) < 0) {
+                fprintf(stderr, "cannot parse n_vectors\n");
+                ivshmem_server_usage(argv[0], 1);
+            }
+            args->n_vectors = v;
+            break;
+
+        default:
+            ivshmem_server_usage(argv[0], 1);
+            break;
+        }
+    }
+
+    if (args->n_vectors > IVSHMEM_SERVER_MAX_VECTORS) {
+        fprintf(stderr, "too many requested vectors (max is %d)\n",
+                IVSHMEM_SERVER_MAX_VECTORS);
+        ivshmem_server_usage(argv[0], 1);
+    }
+
+    if (args->verbose == 1 && args->foreground == 0) {
+        fprintf(stderr, "cannot use verbose in daemon mode\n");
+        ivshmem_server_usage(argv[0], 1);
+    }
+}
+
+/* wait for events on listening server unix socket and connected client
+ * sockets */
+static int
+ivshmem_server_poll_events(IvshmemServer *server)
+{
+    fd_set fds;
+    int ret = 0, maxfd;
+
+    while (!ivshmem_server_quit) {
+
+        FD_ZERO(&fds);
+        maxfd = 0;
+        ivshmem_server_get_fds(server, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            fprintf(stderr, "select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (ivshmem_server_handle_fds(server, &fds, maxfd) < 0) {
+            fprintf(stderr, "ivshmem_server_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return ret;
+}
+
+static void
+ivshmem_server_quit_cb(int signum)
+{
+    ivshmem_server_quit = 1;
+}
+
+int
+main(int argc, char *argv[])
+{
+    IvshmemServer server;
+    struct sigaction sa, sa_quit;
+    IvshmemServerArgs args = {
+        .verbose = IVSHMEM_SERVER_DEFAULT_VERBOSE,
+        .foreground = IVSHMEM_SERVER_DEFAULT_FOREGROUND,
+        .pid_file = IVSHMEM_SERVER_DEFAULT_PID_FILE,
+        .unix_socket_path = IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH,
+        .shm_path = IVSHMEM_SERVER_DEFAULT_SHM_PATH,
+        .shm_size = IVSHMEM_SERVER_DEFAULT_SHM_SIZE,
+        .n_vectors = IVSHMEM_SERVER_DEFAULT_N_VECTORS,
+    };
+    int ret = 1;
+
+    /* parse arguments, will exit on error */
+    ivshmem_server_parse_args(&args, argc, argv);
+
+    /* Ignore SIGPIPE, see this link for more info:
+     * http://www.mail-archive.com/libevent-users@monkey.org/msg01606.html */
+    sa.sa_handler = SIG_IGN;
+    sa.sa_flags = 0;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+        sigaction(SIGPIPE, &sa, 0) == -1) {
+        perror("failed to ignore SIGPIPE; sigaction");
+        goto err;
+    }
+
+    sa_quit.sa_handler = ivshmem_server_quit_cb;
+    sa_quit.sa_flags = 0;
+    if (sigemptyset(&sa_quit.sa_mask) == -1 ||
+        sigaction(SIGTERM, &sa_quit, 0) == -1) {
+        perror("failed to add SIGTERM handler; sigaction");
+        goto err;
+    }
+
+    /* init the ivshms structure */
+    if (ivshmem_server_init(&server, args.unix_socket_path, args.shm_path,
+                            args.shm_size, args.n_vectors, args.verbose) < 0) {
+        fprintf(stderr, "cannot init server\n");
+        goto err;
+    }
+
+    /* start the ivshmem server (open shm & unix socket) */
+    if (ivshmem_server_start(&server) < 0) {
+        fprintf(stderr, "cannot bind\n");
+        goto err;
+    }
+
+    /* daemonize if asked to */
+    if (!args.foreground) {
+        FILE *fp;
+
+        if (qemu_daemon(1, 1) < 0) {
+            fprintf(stderr, "cannot daemonize: %s\n", strerror(errno));
+            goto err_close;
+        }
+
+        /* write pid file */
+        fp = fopen(args.pid_file, "w");
+        if (fp == NULL) {
+            fprintf(stderr, "cannot write pid file: %s\n", strerror(errno));
+            goto err_close;
+        }
+
+        fprintf(fp, "%d\n", (int) getpid());
+        fclose(fp);
+    }
+
+    ivshmem_server_poll_events(&server);
+    fprintf(stdout, "server disconnected\n");
+    ret = 0;
+
+err_close:
+    ivshmem_server_close(&server);
+err:
+    return ret;
+}
diff --git a/cpu-exec-common.c b/cpu-exec-common.c
new file mode 100644
index 000000000..43edf3677
--- /dev/null
+++ b/cpu-exec-common.c
@@ -0,0 +1,82 @@
+/*
+ *  emulator main execution loop
+ *
+ *  Copyright (c) 2003-2005 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "config.h"
+#include "cpu.h"
+#include "sysemu/cpus.h"
+#include "exec/memory-internal.h"
+
+bool exit_request;
+CPUState *tcg_current_cpu;
+
+/* exit the current TB from a signal handler. The host registers are
+   restored in a state compatible with the CPU emulator
+ */
+#if defined(CONFIG_SOFTMMU)
+void cpu_resume_from_signal(CPUState *cpu, void *puc)
+{
+    /* XXX: restore cpu registers saved in host registers */
+
+    cpu->exception_index = -1;
+    siglongjmp(cpu->jmp_env, 1);
+}
+
+void cpu_reloading_memory_map(void)
+{
+    if (qemu_in_vcpu_thread()) {
+        /* The guest can in theory prolong the RCU critical section as long
+         * as it feels like. The major problem with this is that because it
+         * can do multiple reconfigurations of the memory map within the
+         * critical section, we could potentially accumulate an unbounded
+         * collection of memory data structures awaiting reclamation.
+         *
+         * Because the only thing we're currently protecting with RCU is the
+         * memory data structures, it's sufficient to break the critical section
+         * in this callback, which we know will get called every time the
+         * memory map is rearranged.
+         *
+         * (If we add anything else in the system that uses RCU to protect
+         * its data structures, we will need to implement some other mechanism
+         * to force TCG CPUs to exit the critical section, at which point this
+         * part of this callback might become unnecessary.)
+         *
+         * This pair matches cpu_exec's rcu_read_lock()/rcu_read_unlock(), which
+         * only protects cpu->as->dispatch. Since we know our caller is about
+         * to reload it, it's safe to split the critical section.
+         */
+        rcu_read_unlock();
+        rcu_read_lock();
+    }
+}
+#endif
+
+void cpu_loop_exit(CPUState *cpu)
+{
+    cpu->current_tb = NULL;
+    siglongjmp(cpu->jmp_env, 1);
+}
+
+void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc)
+{
+    if (pc) {
+        cpu_restore_state(cpu, pc);
+    }
+    cpu->current_tb = NULL;
+    siglongjmp(cpu->jmp_env, 1);
+}
diff --git a/cpu-exec.c b/cpu-exec.c
index 75694f3bb..c88d0ffdc 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -25,9 +25,12 @@
 #include "sysemu/qtest.h"
 #include "qemu/timer.h"
 #include "exec/address-spaces.h"
-#include "exec/memory-internal.h"
 #include "qemu/rcu.h"
 #include "exec/tb-hash.h"
+#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
+#include "hw/i386/apic.h"
+#endif
+#include "sysemu/replay.h"
 
 /* -icount align implementation. */
 
@@ -128,52 +131,6 @@ static void init_delay_params(SyncClocks *sc, const CPUState *cpu)
 }
 #endif /* CONFIG USER ONLY */
 
-void cpu_loop_exit(CPUState *cpu)
-{
-    cpu->current_tb = NULL;
-    siglongjmp(cpu->jmp_env, 1);
-}
-
-/* exit the current TB from a signal handler. The host registers are
-   restored in a state compatible with the CPU emulator
- */
-#if defined(CONFIG_SOFTMMU)
-void cpu_resume_from_signal(CPUState *cpu, void *puc)
-{
-    /* XXX: restore cpu registers saved in host registers */
-
-    cpu->exception_index = -1;
-    siglongjmp(cpu->jmp_env, 1);
-}
-
-void cpu_reload_memory_map(CPUState *cpu)
-{
-    AddressSpaceDispatch *d;
-
-    if (qemu_in_vcpu_thread()) {
-        /* Do not let the guest prolong the critical section as much as it
-         * as it desires.
-         *
-         * Currently, this is prevented by the I/O thread's periodinc kicking
-         * of the VCPU thread (iothread_requesting_mutex, qemu_cpu_kick_thread)
-         * but this will go away once TCG's execution moves out of the global
-         * mutex.
-         *
-         * This pair matches cpu_exec's rcu_read_lock()/rcu_read_unlock(), which
-         * only protects cpu->as->dispatch.  Since we reload it below, we can
-         * split the critical section.
-         */
-        rcu_read_unlock();
-        rcu_read_lock();
-    }
-
-    /* The CPU and TLB are protected by the iothread lock.  */
-    d = atomic_rcu_read(&cpu->as->dispatch);
-    cpu->memory_dispatch = d;
-    tlb_flush(cpu, 1);
-}
-#endif
-
 /* Execute a TB, and fix up the CPU state afterwards if necessary */
 static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
 {
@@ -196,7 +153,7 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
     }
 #endif /* DEBUG_DISAS */
 
-    cpu->can_do_io = 0;
+    cpu->can_do_io = !use_icount;
     next_tb = tcg_qemu_tb_exec(env, tb_ptr);
     cpu->can_do_io = 1;
     trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK),
@@ -228,22 +185,19 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
 /* Execute the code without caching the generated code. An interpreter
    could be used if available. */
 static void cpu_exec_nocache(CPUState *cpu, int max_cycles,
-                             TranslationBlock *orig_tb)
+                             TranslationBlock *orig_tb, bool ignore_icount)
 {
     TranslationBlock *tb;
-    target_ulong pc = orig_tb->pc;
-    target_ulong cs_base = orig_tb->cs_base;
-    uint64_t flags = orig_tb->flags;
 
     /* Should never happen.
        We only end up here when an existing TB is too long.  */
     if (max_cycles > CF_COUNT_MASK)
         max_cycles = CF_COUNT_MASK;
 
-    /* tb_gen_code can flush our orig_tb, invalidate it now */
-    tb_phys_invalidate(orig_tb, -1);
-    tb = tb_gen_code(cpu, pc, cs_base, flags,
-                     max_cycles | CF_NOCACHE);
+    tb = tb_gen_code(cpu, orig_tb->pc, orig_tb->cs_base, orig_tb->flags,
+                     max_cycles | CF_NOCACHE
+                         | (ignore_icount ? CF_IGNORE_ICOUNT : 0));
+    tb->orig_tb = tcg_ctx.tb_ctx.tb_invalidated_flag ? NULL : orig_tb;
     cpu->current_tb = tb;
     /* execute the generated code */
     trace_exec_tb_nocache(tb, tb->pc);
@@ -253,10 +207,10 @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles,
     tb_free(tb);
 }
 
-static TranslationBlock *tb_find_slow(CPUState *cpu,
-                                      target_ulong pc,
-                                      target_ulong cs_base,
-                                      uint64_t flags)
+static TranslationBlock *tb_find_physical(CPUState *cpu,
+                                          target_ulong pc,
+                                          target_ulong cs_base,
+                                          uint64_t flags)
 {
     CPUArchState *env = (CPUArchState *)cpu->env_ptr;
     TranslationBlock *tb, **ptb1;
@@ -273,8 +227,9 @@ static TranslationBlock *tb_find_slow(CPUState *cpu,
     ptb1 = &tcg_ctx.tb_ctx.tb_phys_hash[h];
     for(;;) {
         tb = *ptb1;
-        if (!tb)
-            goto not_found;
+        if (!tb) {
+            return NULL;
+        }
         if (tb->pc == pc &&
             tb->page_addr[0] == phys_page1 &&
             tb->cs_base == cs_base &&
@@ -286,25 +241,59 @@ static TranslationBlock *tb_find_slow(CPUState *cpu,
                 virt_page2 = (pc & TARGET_PAGE_MASK) +
                     TARGET_PAGE_SIZE;
                 phys_page2 = get_page_addr_code(env, virt_page2);
-                if (tb->page_addr[1] == phys_page2)
-                    goto found;
+                if (tb->page_addr[1] == phys_page2) {
+                    break;
+                }
             } else {
-                goto found;
+                break;
             }
         }
         ptb1 = &tb->phys_hash_next;
     }
- not_found:
-   /* if no translated code available, then translate it now */
-    tb = tb_gen_code(cpu, pc, cs_base, flags, 0);
 
- found:
-    /* Move the last found TB to the head of the list */
-    if (likely(*ptb1)) {
-        *ptb1 = tb->phys_hash_next;
-        tb->phys_hash_next = tcg_ctx.tb_ctx.tb_phys_hash[h];
-        tcg_ctx.tb_ctx.tb_phys_hash[h] = tb;
+    /* Move the TB to the head of the list */
+    *ptb1 = tb->phys_hash_next;
+    tb->phys_hash_next = tcg_ctx.tb_ctx.tb_phys_hash[h];
+    tcg_ctx.tb_ctx.tb_phys_hash[h] = tb;
+    return tb;
+}
+
+static TranslationBlock *tb_find_slow(CPUState *cpu,
+                                      target_ulong pc,
+                                      target_ulong cs_base,
+                                      uint64_t flags)
+{
+    TranslationBlock *tb;
+
+    tb = tb_find_physical(cpu, pc, cs_base, flags);
+    if (tb) {
+        goto found;
+    }
+
+#ifdef CONFIG_USER_ONLY
+    /* mmap_lock is needed by tb_gen_code, and mmap_lock must be
+     * taken outside tb_lock.  Since we're momentarily dropping
+     * tb_lock, there's a chance that our desired tb has been
+     * translated.
+     */
+    tb_unlock();
+    mmap_lock();
+    tb_lock();
+    tb = tb_find_physical(cpu, pc, cs_base, flags);
+    if (tb) {
+        mmap_unlock();
+        goto found;
     }
+#endif
+
+    /* if no translated code available, then translate it now */
+    tb = tb_gen_code(cpu, pc, cs_base, flags, 0);
+
+#ifdef CONFIG_USER_ONLY
+    mmap_unlock();
+#endif
+
+found:
     /* we add the TB in the virtual pc hash table */
     cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)] = tb;
     return tb;
@@ -345,8 +334,6 @@ static void cpu_handle_debug_exception(CPUState *cpu)
 
 /* main execution loop */
 
-volatile sig_atomic_t exit_request;
-
 int cpu_exec(CPUState *cpu)
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
@@ -360,30 +347,29 @@ int cpu_exec(CPUState *cpu)
     uintptr_t next_tb;
     SyncClocks sc;
 
-    /* This must be volatile so it is not trashed by longjmp() */
-    volatile bool have_tb_lock = false;
+    /* replay_interrupt may need current_cpu */
+    current_cpu = cpu;
 
     if (cpu->halted) {
+#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
+        if ((cpu->interrupt_request & CPU_INTERRUPT_POLL)
+            && replay_interrupt()) {
+            apic_poll_irq(x86_cpu->apic_state);
+            cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL);
+        }
+#endif
         if (!cpu_has_work(cpu)) {
+            current_cpu = NULL;
             return EXCP_HALTED;
         }
 
         cpu->halted = 0;
     }
 
-    current_cpu = cpu;
-
-    /* As long as current_cpu is null, up to the assignment just above,
-     * requests by other threads to exit the execution loop are expected to
-     * be issued using the exit_request global. We must make sure that our
-     * evaluation of the global value is performed past the current_cpu
-     * value transition point, which requires a memory barrier as well as
-     * an instruction scheduling constraint on modern architectures.  */
-    smp_mb();
-
+    atomic_mb_set(&tcg_current_cpu, cpu);
     rcu_read_lock();
 
-    if (unlikely(exit_request)) {
+    if (unlikely(atomic_mb_read(&exit_request))) {
         cpu->exit_request = 1;
     }
 
@@ -421,10 +407,22 @@ int cpu_exec(CPUState *cpu)
                     cpu->exception_index = -1;
                     break;
 #else
-                    cc->do_interrupt(cpu);
-                    cpu->exception_index = -1;
+                    if (replay_exception()) {
+                        cc->do_interrupt(cpu);
+                        cpu->exception_index = -1;
+                    } else if (!replay_has_interrupt()) {
+                        /* give a chance to iothread in replay mode */
+                        ret = EXCP_INTERRUPT;
+                        break;
+                    }
 #endif
                 }
+            } else if (replay_has_exception()
+                       && cpu->icount_decr.u16.low + cpu->icount_extra == 0) {
+                /* try to cause an exception pending in the log */
+                cpu_exec_nocache(cpu, 1, tb_find_fast(cpu), true);
+                ret = -1;
+                break;
             }
 
             next_tb = 0; /* force lookup of first TB */
@@ -440,30 +438,40 @@ int cpu_exec(CPUState *cpu)
                         cpu->exception_index = EXCP_DEBUG;
                         cpu_loop_exit(cpu);
                     }
-                    if (interrupt_request & CPU_INTERRUPT_HALT) {
+                    if (replay_mode == REPLAY_MODE_PLAY
+                        && !replay_has_interrupt()) {
+                        /* Do nothing */
+                    } else if (interrupt_request & CPU_INTERRUPT_HALT) {
+                        replay_interrupt();
                         cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
                         cpu->halted = 1;
                         cpu->exception_index = EXCP_HLT;
                         cpu_loop_exit(cpu);
                     }
 #if defined(TARGET_I386)
-                    if (interrupt_request & CPU_INTERRUPT_INIT) {
+                    else if (interrupt_request & CPU_INTERRUPT_INIT) {
+                        replay_interrupt();
                         cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0);
                         do_cpu_init(x86_cpu);
                         cpu->exception_index = EXCP_HALTED;
                         cpu_loop_exit(cpu);
                     }
 #else
-                    if (interrupt_request & CPU_INTERRUPT_RESET) {
+                    else if (interrupt_request & CPU_INTERRUPT_RESET) {
+                        replay_interrupt();
                         cpu_reset(cpu);
+                        cpu_loop_exit(cpu);
                     }
 #endif
                     /* The target hook has 3 exit conditions:
                        False when the interrupt isn't processed,
                        True when it is, and we should restart on a new TB,
                        and via longjmp via cpu_loop_exit.  */
-                    if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
-                        next_tb = 0;
+                    else {
+                        replay_interrupt();
+                        if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
+                            next_tb = 0;
+                        }
                     }
                     /* Don't use the cached interrupt_request value,
                        do_interrupt may have updated the EXITTB flag. */
@@ -474,13 +482,13 @@ int cpu_exec(CPUState *cpu)
                         next_tb = 0;
                     }
                 }
-                if (unlikely(cpu->exit_request)) {
+                if (unlikely(cpu->exit_request
+                             || replay_has_interrupt())) {
                     cpu->exit_request = 0;
                     cpu->exception_index = EXCP_INTERRUPT;
                     cpu_loop_exit(cpu);
                 }
-                spin_lock(&tcg_ctx.tb_ctx.tb_lock);
-                have_tb_lock = true;
+                tb_lock();
                 tb = tb_find_fast(cpu);
                 /* Note: we do it here to avoid a gcc bug on Mac OS X when
                    doing it in tb_find_slow */
@@ -498,24 +506,19 @@ int cpu_exec(CPUState *cpu)
                 /* see if we can patch the calling TB. When the TB
                    spans two pages, we cannot safely do a direct
                    jump. */
-                if (next_tb != 0 && tb->page_addr[1] == -1) {
+                if (next_tb != 0 && tb->page_addr[1] == -1
+                    && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
                     tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK),
                                 next_tb & TB_EXIT_MASK, tb);
                 }
-                have_tb_lock = false;
-                spin_unlock(&tcg_ctx.tb_ctx.tb_lock);
-
-                /* cpu_interrupt might be called while translating the
-                   TB, but before it is linked into a potentially
-                   infinite loop and becomes env->current_tb. Avoid
-                   starting execution if there is a pending interrupt. */
-                cpu->current_tb = tb;
-                barrier();
+                tb_unlock();
                 if (likely(!cpu->exit_request)) {
                     trace_exec_tb(tb, tb->pc);
                     tc_ptr = tb->tc_ptr;
                     /* execute the generated code */
+                    cpu->current_tb = tb;
                     next_tb = cpu_tb_exec(cpu, tc_ptr);
+                    cpu->current_tb = NULL;
                     switch (next_tb & TB_EXIT_MASK) {
                     case TB_EXIT_REQUESTED:
                         /* Something asked us to stop executing
@@ -523,8 +526,12 @@ int cpu_exec(CPUState *cpu)
                          * loop. Whatever requested the exit will also
                          * have set something else (eg exit_request or
                          * interrupt_request) which we will handle
-                         * next time around the loop.
+                         * next time around the loop.  But we need to
+                         * ensure the tcg_exit_req read in generated code
+                         * comes before the next read of cpu->exit_request
+                         * or cpu->interrupt_request.
                          */
+                        smp_rmb();
                         next_tb = 0;
                         break;
                     case TB_EXIT_ICOUNT_EXPIRED:
@@ -541,7 +548,7 @@ int cpu_exec(CPUState *cpu)
                             if (insns_left > 0) {
                                 /* Execute remaining instructions.  */
                                 tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
-                                cpu_exec_nocache(cpu, insns_left, tb);
+                                cpu_exec_nocache(cpu, insns_left, tb, false);
                                 align_clocks(&sc, cpu);
                             }
                             cpu->exception_index = EXCP_INTERRUPT;
@@ -554,7 +561,6 @@ int cpu_exec(CPUState *cpu)
                         break;
                     }
                 }
-                cpu->current_tb = NULL;
                 /* Try to align the host and virtual clocks
                    if the guest is in advance */
                 align_clocks(&sc, cpu);
@@ -562,19 +568,28 @@ int cpu_exec(CPUState *cpu)
                    only be set by a memory fault) */
             } /* for(;;) */
         } else {
-            /* Reload env after longjmp - the compiler may have smashed all
-             * local variables as longjmp is marked 'noreturn'. */
+#if defined(__clang__) || !QEMU_GNUC_PREREQ(4, 6)
+            /* Some compilers wrongly smash all local variables after
+             * siglongjmp. There were bug reports for gcc 4.5.0 and clang.
+             * Reload essential local variables here for those compilers.
+             * Newer versions of gcc would complain about this code (-Wclobbered). */
             cpu = current_cpu;
             cc = CPU_GET_CLASS(cpu);
-            cpu->can_do_io = 1;
 #ifdef TARGET_I386
             x86_cpu = X86_CPU(cpu);
             env = &x86_cpu->env;
 #endif
-            if (have_tb_lock) {
-                spin_unlock(&tcg_ctx.tb_ctx.tb_lock);
-                have_tb_lock = false;
-            }
+#else /* buggy compiler */
+            /* Assert that the compiler does not smash local variables. */
+            g_assert(cpu == current_cpu);
+            g_assert(cc == CPU_GET_CLASS(cpu));
+#ifdef TARGET_I386
+            g_assert(x86_cpu == X86_CPU(cpu));
+            g_assert(env == &x86_cpu->env);
+#endif
+#endif /* buggy compiler */
+            cpu->can_do_io = 1;
+            tb_lock_reset();
         }
     } /* for(;;) */
 
@@ -583,5 +598,8 @@ int cpu_exec(CPUState *cpu)
 
     /* fail safe : never use current_cpu outside cpu_exec() */
     current_cpu = NULL;
+
+    /* Does not need atomic_mb_set because a spurious wakeup is okay.  */
+    atomic_set(&tcg_current_cpu, NULL);
     return ret;
 }
diff --git a/cpus.c b/cpus.c
index 7841fd8ae..747f14dc7 100644
--- a/cpus.c
+++ b/cpus.c
@@ -42,6 +42,7 @@
 #include "qemu/seqlock.h"
 #include "qapi-event.h"
 #include "hw/nmi.h"
+#include "sysemu/replay.h"
 
 #ifndef _WIN32
 #include "qemu/compatfd.h"
@@ -69,6 +70,14 @@ static CPUState *next_cpu;
 int64_t max_delay;
 int64_t max_advance;
 
+/* vcpu throttling controls */
+static QEMUTimer *throttle_timer;
+static unsigned int throttle_percentage;
+
+#define CPU_THROTTLE_PCT_MIN 1
+#define CPU_THROTTLE_PCT_MAX 99
+#define CPU_THROTTLE_TIMESLICE_NS 10000000
+
 bool cpu_is_stopped(CPUState *cpu)
 {
     return cpu->stopped || !runstate_is_running();
@@ -145,7 +154,7 @@ int64_t cpu_get_icount_raw(void)
 
     icount = timers_state.qemu_icount;
     if (cpu) {
-        if (!cpu_can_do_io(cpu)) {
+        if (!cpu->can_do_io) {
             fprintf(stderr, "Bad icount read\n");
             exit(1);
         }
@@ -191,7 +200,7 @@ int64_t cpu_get_ticks(void)
 
     ticks = timers_state.cpu_ticks_offset;
     if (timers_state.cpu_ticks_enabled) {
-        ticks += cpu_get_real_ticks();
+        ticks += cpu_get_host_ticks();
     }
 
     if (timers_state.cpu_ticks_prev > ticks) {
@@ -239,7 +248,7 @@ void cpu_enable_ticks(void)
     /* Here, the really thing protected by seqlock is cpu_clock_offset. */
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (!timers_state.cpu_ticks_enabled) {
-        timers_state.cpu_ticks_offset -= cpu_get_real_ticks();
+        timers_state.cpu_ticks_offset -= cpu_get_host_ticks();
         timers_state.cpu_clock_offset -= get_clock();
         timers_state.cpu_ticks_enabled = 1;
     }
@@ -255,7 +264,7 @@ void cpu_disable_ticks(void)
     /* Here, the really thing protected by seqlock is cpu_clock_offset. */
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (timers_state.cpu_ticks_enabled) {
-        timers_state.cpu_ticks_offset += cpu_get_real_ticks();
+        timers_state.cpu_ticks_offset += cpu_get_host_ticks();
         timers_state.cpu_clock_offset = cpu_get_clock_locked();
         timers_state.cpu_ticks_enabled = 0;
     }
@@ -326,7 +335,7 @@ static int64_t qemu_icount_round(int64_t count)
     return (count + (1 << icount_time_shift) - 1) >> icount_time_shift;
 }
 
-static void icount_warp_rt(void *opaque)
+static void icount_warp_rt(void)
 {
     /* The icount_warp_timer is rescheduled soon after vm_clock_warp_start
      * changes from -1 to another value, so the race here is okay.
@@ -337,7 +346,8 @@ static void icount_warp_rt(void *opaque)
 
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (runstate_is_running()) {
-        int64_t clock = cpu_get_clock_locked();
+        int64_t clock = REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT,
+                                     cpu_get_clock_locked());
         int64_t warp_delta;
 
         warp_delta = clock - vm_clock_warp_start;
@@ -360,6 +370,11 @@ static void icount_warp_rt(void *opaque)
     }
 }
 
+static void icount_dummy_timer(void *opaque)
+{
+    (void)opaque;
+}
+
 void qtest_clock_warp(int64_t dest)
 {
     int64_t clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
@@ -395,6 +410,18 @@ void qemu_clock_warp(QEMUClockType type)
         return;
     }
 
+    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
+     * do not fire, so computing the deadline does not make sense.
+     */
+    if (!runstate_is_running()) {
+        return;
+    }
+
+    /* warp clock deterministically in record/replay mode */
+    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP)) {
+        return;
+    }
+
     if (icount_sleep) {
         /*
          * If the CPUs have been sleeping, advance QEMU_CLOCK_VIRTUAL timer now.
@@ -404,7 +431,7 @@ void qemu_clock_warp(QEMUClockType type)
          * the CPU starts running, in case the CPU is woken by an event other
          * than the earliest QEMU_CLOCK_VIRTUAL timer.
          */
-        icount_warp_rt(NULL);
+        icount_warp_rt();
         timer_del(icount_warp_timer);
     }
     if (!all_cpu_threads_idle()) {
@@ -505,10 +532,80 @@ static const VMStateDescription vmstate_timers = {
     }
 };
 
+static void cpu_throttle_thread(void *opaque)
+{
+    CPUState *cpu = opaque;
+    double pct;
+    double throttle_ratio;
+    long sleeptime_ns;
+
+    if (!cpu_throttle_get_percentage()) {
+        return;
+    }
+
+    pct = (double)cpu_throttle_get_percentage()/100;
+    throttle_ratio = pct / (1 - pct);
+    sleeptime_ns = (long)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS);
+
+    qemu_mutex_unlock_iothread();
+    atomic_set(&cpu->throttle_thread_scheduled, 0);
+    g_usleep(sleeptime_ns / 1000); /* Convert ns to us for usleep call */
+    qemu_mutex_lock_iothread();
+}
+
+static void cpu_throttle_timer_tick(void *opaque)
+{
+    CPUState *cpu;
+    double pct;
+
+    /* Stop the timer if needed */
+    if (!cpu_throttle_get_percentage()) {
+        return;
+    }
+    CPU_FOREACH(cpu) {
+        if (!atomic_xchg(&cpu->throttle_thread_scheduled, 1)) {
+            async_run_on_cpu(cpu, cpu_throttle_thread, cpu);
+        }
+    }
+
+    pct = (double)cpu_throttle_get_percentage()/100;
+    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
+                                   CPU_THROTTLE_TIMESLICE_NS / (1-pct));
+}
+
+void cpu_throttle_set(int new_throttle_pct)
+{
+    /* Ensure throttle percentage is within valid range */
+    new_throttle_pct = MIN(new_throttle_pct, CPU_THROTTLE_PCT_MAX);
+    new_throttle_pct = MAX(new_throttle_pct, CPU_THROTTLE_PCT_MIN);
+
+    atomic_set(&throttle_percentage, new_throttle_pct);
+
+    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
+                                       CPU_THROTTLE_TIMESLICE_NS);
+}
+
+void cpu_throttle_stop(void)
+{
+    atomic_set(&throttle_percentage, 0);
+}
+
+bool cpu_throttle_active(void)
+{
+    return (cpu_throttle_get_percentage() != 0);
+}
+
+int cpu_throttle_get_percentage(void)
+{
+    return atomic_read(&throttle_percentage);
+}
+
 void cpu_ticks_init(void)
 {
     seqlock_init(&timers_state.vm_clock_seqlock, NULL);
     vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
+    throttle_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
+                                           cpu_throttle_timer_tick, NULL);
 }
 
 void configure_icount(QemuOpts *opts, Error **errp)
@@ -527,7 +624,7 @@ void configure_icount(QemuOpts *opts, Error **errp)
     icount_sleep = qemu_opt_get_bool(opts, "sleep", true);
     if (icount_sleep) {
         icount_warp_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
-                                         icount_warp_rt, NULL);
+                                         icount_dummy_timer, NULL);
     }
 
     icount_align_option = qemu_opt_get_bool(opts, "align", false);
@@ -616,15 +713,6 @@ void cpu_synchronize_all_post_init(void)
     }
 }
 
-void cpu_clean_all_dirty(void)
-{
-    CPUState *cpu;
-
-    CPU_FOREACH(cpu) {
-        cpu_clean_state(cpu);
-    }
-}
-
 static int do_vm_stop(RunState state)
 {
     int ret = 0;
@@ -661,14 +749,6 @@ static void cpu_handle_guest_debug(CPUState *cpu)
     cpu->stopped = true;
 }
 
-static void cpu_signal(int sig)
-{
-    if (current_cpu) {
-        cpu_exit(current_cpu);
-    }
-    exit_request = 1;
-}
-
 #ifdef CONFIG_LINUX
 static void sigbus_reraise(void)
 {
@@ -781,29 +861,11 @@ static void qemu_kvm_init_cpu_signals(CPUState *cpu)
     }
 }
 
-static void qemu_tcg_init_cpu_signals(void)
-{
-    sigset_t set;
-    struct sigaction sigact;
-
-    memset(&sigact, 0, sizeof(sigact));
-    sigact.sa_handler = cpu_signal;
-    sigaction(SIG_IPI, &sigact, NULL);
-
-    sigemptyset(&set);
-    sigaddset(&set, SIG_IPI);
-    pthread_sigmask(SIG_UNBLOCK, &set, NULL);
-}
-
 #else /* _WIN32 */
 static void qemu_kvm_init_cpu_signals(CPUState *cpu)
 {
     abort();
 }
-
-static void qemu_tcg_init_cpu_signals(void)
-{
-}
 #endif /* _WIN32 */
 
 static QemuMutex qemu_global_mutex;
@@ -812,9 +874,6 @@ static unsigned iothread_requesting_mutex;
 
 static QemuThread io_thread;
 
-static QemuThread *tcg_cpu_thread;
-static QemuCond *tcg_halt_cond;
-
 /* cpu creation */
 static QemuCond qemu_cpu_cond;
 /* system init */
@@ -845,6 +904,8 @@ void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     wi.func = func;
     wi.data = data;
     wi.free = false;
+
+    qemu_mutex_lock(&cpu->work_mutex);
     if (cpu->queued_work_first == NULL) {
         cpu->queued_work_first = &wi;
     } else {
@@ -853,9 +914,10 @@ void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     cpu->queued_work_last = &wi;
     wi.next = NULL;
     wi.done = false;
+    qemu_mutex_unlock(&cpu->work_mutex);
 
     qemu_cpu_kick(cpu);
-    while (!wi.done) {
+    while (!atomic_mb_read(&wi.done)) {
         CPUState *self_cpu = current_cpu;
 
         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
@@ -876,6 +938,8 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     wi->func = func;
     wi->data = data;
     wi->free = true;
+
+    qemu_mutex_lock(&cpu->work_mutex);
     if (cpu->queued_work_first == NULL) {
         cpu->queued_work_first = wi;
     } else {
@@ -884,6 +948,7 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     cpu->queued_work_last = wi;
     wi->next = NULL;
     wi->done = false;
+    qemu_mutex_unlock(&cpu->work_mutex);
 
     qemu_cpu_kick(cpu);
 }
@@ -896,15 +961,23 @@ static void flush_queued_work(CPUState *cpu)
         return;
     }
 
-    while ((wi = cpu->queued_work_first)) {
+    qemu_mutex_lock(&cpu->work_mutex);
+    while (cpu->queued_work_first != NULL) {
+        wi = cpu->queued_work_first;
         cpu->queued_work_first = wi->next;
+        if (!cpu->queued_work_first) {
+            cpu->queued_work_last = NULL;
+        }
+        qemu_mutex_unlock(&cpu->work_mutex);
         wi->func(wi->data);
-        wi->done = true;
+        qemu_mutex_lock(&cpu->work_mutex);
         if (wi->free) {
             g_free(wi);
+        } else {
+            atomic_mb_set(&wi->done, true);
         }
     }
-    cpu->queued_work_last = NULL;
+    qemu_mutex_unlock(&cpu->work_mutex);
     qemu_cond_broadcast(&qemu_work_cond);
 }
 
@@ -913,21 +986,19 @@ static void qemu_wait_io_event_common(CPUState *cpu)
     if (cpu->stop) {
         cpu->stop = false;
         cpu->stopped = true;
-        qemu_cond_signal(&qemu_pause_cond);
+        qemu_cond_broadcast(&qemu_pause_cond);
     }
     flush_queued_work(cpu);
     cpu->thread_kicked = false;
 }
 
-static void qemu_tcg_wait_io_event(void)
+static void qemu_tcg_wait_io_event(CPUState *cpu)
 {
-    CPUState *cpu;
-
     while (all_cpu_threads_idle()) {
        /* Start accounting real time to the virtual clock if the CPUs
           are idle.  */
         qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
-        qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
+        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
 
     while (iothread_requesting_mutex) {
@@ -1041,7 +1112,6 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     rcu_register_thread();
 
     qemu_mutex_lock_iothread();
-    qemu_tcg_init_cpu_signals();
     qemu_thread_get_self(cpu->thread);
 
     CPU_FOREACH(cpu) {
@@ -1053,7 +1123,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
 
     /* wait for initial kick-off after machine start */
     while (first_cpu->stopped) {
-        qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
+        qemu_cond_wait(first_cpu->halt_cond, &qemu_global_mutex);
 
         /* process any pending work */
         CPU_FOREACH(cpu) {
@@ -1062,7 +1132,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     }
 
     /* process any pending work */
-    exit_request = 1;
+    atomic_mb_set(&exit_request, 1);
 
     while (1) {
         tcg_exec_all();
@@ -1074,7 +1144,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
                 qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
             }
         }
-        qemu_tcg_wait_io_event();
+        qemu_tcg_wait_io_event(QTAILQ_FIRST(&cpus));
     }
 
     return NULL;
@@ -1085,61 +1155,47 @@ static void qemu_cpu_kick_thread(CPUState *cpu)
 #ifndef _WIN32
     int err;
 
+    if (cpu->thread_kicked) {
+        return;
+    }
+    cpu->thread_kicked = true;
     err = pthread_kill(cpu->thread->thread, SIG_IPI);
     if (err) {
         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
         exit(1);
     }
 #else /* _WIN32 */
-    if (!qemu_cpu_is_self(cpu)) {
-        CONTEXT tcgContext;
-
-        if (SuspendThread(cpu->hThread) == (DWORD)-1) {
-            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
-                    GetLastError());
-            exit(1);
-        }
-
-        /* On multi-core systems, we are not sure that the thread is actually
-         * suspended until we can get the context.
-         */
-        tcgContext.ContextFlags = CONTEXT_CONTROL;
-        while (GetThreadContext(cpu->hThread, &tcgContext) != 0) {
-            continue;
-        }
-
-        cpu_signal(0);
+    abort();
+#endif
+}
 
-        if (ResumeThread(cpu->hThread) == (DWORD)-1) {
-            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
-                    GetLastError());
-            exit(1);
-        }
+static void qemu_cpu_kick_no_halt(void)
+{
+    CPUState *cpu;
+    /* Ensure whatever caused the exit has reached the CPU threads before
+     * writing exit_request.
+     */
+    atomic_mb_set(&exit_request, 1);
+    cpu = atomic_mb_read(&tcg_current_cpu);
+    if (cpu) {
+        cpu_exit(cpu);
     }
-#endif
 }
 
 void qemu_cpu_kick(CPUState *cpu)
 {
     qemu_cond_broadcast(cpu->halt_cond);
-    if (!tcg_enabled() && !cpu->thread_kicked) {
+    if (tcg_enabled()) {
+        qemu_cpu_kick_no_halt();
+    } else {
         qemu_cpu_kick_thread(cpu);
-        cpu->thread_kicked = true;
     }
 }
 
 void qemu_cpu_kick_self(void)
 {
-#ifndef _WIN32
     assert(current_cpu);
-
-    if (!current_cpu->thread_kicked) {
-        qemu_cpu_kick_thread(current_cpu);
-        current_cpu->thread_kicked = true;
-    }
-#else
-    abort();
-#endif
+    qemu_cpu_kick_thread(current_cpu);
 }
 
 bool qemu_cpu_is_self(CPUState *cpu)
@@ -1171,7 +1227,7 @@ void qemu_mutex_lock_iothread(void)
         atomic_dec(&iothread_requesting_mutex);
     } else {
         if (qemu_mutex_trylock(&qemu_global_mutex)) {
-            qemu_cpu_kick_thread(first_cpu);
+            qemu_cpu_kick_no_halt();
             qemu_mutex_lock(&qemu_global_mutex);
         }
         atomic_dec(&iothread_requesting_mutex);
@@ -1251,6 +1307,8 @@ void resume_all_vcpus(void)
 static void qemu_tcg_init_vcpu(CPUState *cpu)
 {
     char thread_name[VCPU_THREAD_NAME_SIZE];
+    static QemuCond *tcg_halt_cond;
+    static QemuThread *tcg_cpu_thread;
 
     tcg_cpu_address_space_init(cpu, cpu->as);
 
@@ -1329,7 +1387,7 @@ void cpu_stop_current(void)
         current_cpu->stop = false;
         current_cpu->stopped = true;
         cpu_exit(current_cpu);
-        qemu_cond_signal(&qemu_pause_cond);
+        qemu_cond_broadcast(&qemu_pause_cond);
     }
 }
 
@@ -1357,12 +1415,36 @@ int vm_stop_force_state(RunState state)
         return vm_stop(state);
     } else {
         runstate_set(state);
+
+        bdrv_drain_all();
         /* Make sure to return an error if the flush in a previous vm_stop()
          * failed. */
         return bdrv_flush_all();
     }
 }
 
+static int64_t tcg_get_icount_limit(void)
+{
+    int64_t deadline;
+
+    if (replay_mode != REPLAY_MODE_PLAY) {
+        deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
+
+        /* Maintain prior (possibly buggy) behaviour where if no deadline
+         * was set (as there is no QEMU_CLOCK_VIRTUAL timer) or it is more than
+         * INT32_MAX nanoseconds ahead, we still use INT32_MAX
+         * nanoseconds.
+         */
+        if ((deadline < 0) || (deadline > INT32_MAX)) {
+            deadline = INT32_MAX;
+        }
+
+        return qemu_icount_round(deadline);
+    } else {
+        return replay_get_instructions();
+    }
+}
+
 static int tcg_cpu_exec(CPUState *cpu)
 {
     int ret;
@@ -1375,24 +1457,12 @@ static int tcg_cpu_exec(CPUState *cpu)
 #endif
     if (use_icount) {
         int64_t count;
-        int64_t deadline;
         int decr;
         timers_state.qemu_icount -= (cpu->icount_decr.u16.low
                                     + cpu->icount_extra);
         cpu->icount_decr.u16.low = 0;
         cpu->icount_extra = 0;
-        deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
-
-        /* Maintain prior (possibly buggy) behaviour where if no deadline
-         * was set (as there is no QEMU_CLOCK_VIRTUAL timer) or it is more than
-         * INT32_MAX nanoseconds ahead, we still use INT32_MAX
-         * nanoseconds.
-         */
-        if ((deadline < 0) || (deadline > INT32_MAX)) {
-            deadline = INT32_MAX;
-        }
-
-        count = qemu_icount_round(deadline);
+        count = tcg_get_icount_limit();
         timers_state.qemu_icount += count;
         decr = (count > 0xffff) ? 0xffff : count;
         count -= decr;
@@ -1410,6 +1480,7 @@ static int tcg_cpu_exec(CPUState *cpu)
                         + cpu->icount_extra);
         cpu->icount_decr.u32 = 0;
         cpu->icount_extra = 0;
+        replay_account_executed_instructions();
     }
     return ret;
 }
@@ -1440,7 +1511,9 @@ static void tcg_exec_all(void)
             break;
         }
     }
-    exit_request = 0;
+
+    /* Pairs with smp_wmb in qemu_cpu_kick.  */
+    atomic_mb_set(&exit_request, 0);
 }
 
 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
diff --git a/cputlb.c b/cputlb.c
index a50608676..bf1d50add 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -69,6 +69,47 @@ void tlb_flush(CPUState *cpu, int flush_global)
     tlb_flush_count++;
 }
 
+static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
+{
+    CPUArchState *env = cpu->env_ptr;
+
+#if defined(DEBUG_TLB)
+    printf("tlb_flush_by_mmuidx:");
+#endif
+    /* must reset current TB so that interrupts cannot modify the
+       links while we are modifying them */
+    cpu->current_tb = NULL;
+
+    for (;;) {
+        int mmu_idx = va_arg(argp, int);
+
+        if (mmu_idx < 0) {
+            break;
+        }
+
+#if defined(DEBUG_TLB)
+        printf(" %d", mmu_idx);
+#endif
+
+        memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
+        memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
+    }
+
+#if defined(DEBUG_TLB)
+    printf("\n");
+#endif
+
+    memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
+}
+
+void tlb_flush_by_mmuidx(CPUState *cpu, ...)
+{
+    va_list argp;
+    va_start(argp, cpu);
+    v_tlb_flush_by_mmuidx(cpu, argp);
+    va_end(argp);
+}
+
 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
 {
     if (addr == (tlb_entry->addr_read &
@@ -121,6 +162,62 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr)
     tb_flush_jmp_cache(cpu, addr);
 }
 
+void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
+{
+    CPUArchState *env = cpu->env_ptr;
+    int i, k;
+    va_list argp;
+
+    va_start(argp, addr);
+
+#if defined(DEBUG_TLB)
+    printf("tlb_flush_page_by_mmu_idx: " TARGET_FMT_lx, addr);
+#endif
+    /* Check if we need to flush due to large pages.  */
+    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
+#if defined(DEBUG_TLB)
+        printf(" forced full flush ("
+               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
+               env->tlb_flush_addr, env->tlb_flush_mask);
+#endif
+        v_tlb_flush_by_mmuidx(cpu, argp);
+        va_end(argp);
+        return;
+    }
+    /* must reset current TB so that interrupts cannot modify the
+       links while we are modifying them */
+    cpu->current_tb = NULL;
+
+    addr &= TARGET_PAGE_MASK;
+    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+
+    for (;;) {
+        int mmu_idx = va_arg(argp, int);
+
+        if (mmu_idx < 0) {
+            break;
+        }
+
+#if defined(DEBUG_TLB)
+        printf(" %d", mmu_idx);
+#endif
+
+        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
+
+        /* check whether there are vltb entries that need to be flushed */
+        for (k = 0; k < CPU_VTLB_SIZE; k++) {
+            tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
+        }
+    }
+    va_end(argp);
+
+#if defined(DEBUG_TLB)
+    printf("\n");
+#endif
+
+    tb_flush_jmp_cache(cpu, addr);
+}
+
 /* update the TLBs so that writes to code in the virtual page 'addr'
    can be detected */
 void tlb_protect_code(ram_addr_t ram_addr)
@@ -165,27 +262,24 @@ static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
     return ram_addr;
 }
 
-void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length)
+void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 {
-    CPUState *cpu;
     CPUArchState *env;
 
-    CPU_FOREACH(cpu) {
-        int mmu_idx;
+    int mmu_idx;
 
-        env = cpu->env_ptr;
-        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-            unsigned int i;
+    env = cpu->env_ptr;
+    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+        unsigned int i;
 
-            for (i = 0; i < CPU_TLB_SIZE; i++) {
-                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
-                                      start1, length);
-            }
+        for (i = 0; i < CPU_TLB_SIZE; i++) {
+            tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
+                                  start1, length);
+        }
 
-            for (i = 0; i < CPU_VTLB_SIZE; i++) {
-                tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
-                                      start1, length);
-            }
+        for (i = 0; i < CPU_VTLB_SIZE; i++) {
+            tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
+                                  start1, length);
         }
     }
 }
@@ -199,8 +293,9 @@ static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
 
 /* update the TLB corresponding to virtual page vaddr
    so that it is no longer dirty */
-void tlb_set_dirty(CPUArchState *env, target_ulong vaddr)
+void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 {
+    CPUArchState *env = cpu->env_ptr;
     int i;
     int mmu_idx;
 
@@ -355,7 +450,7 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
     CPUState *cpu = ENV_GET_CPU(env1);
 
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    mmu_idx = cpu_mmu_index(env1);
+    mmu_idx = cpu_mmu_index(env1, true);
     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
                  (addr & TARGET_PAGE_MASK))) {
         cpu_ldub_code(env1, addr);
diff --git a/crypto/Makefile.objs b/crypto/Makefile.objs
index b05013831..b2a0e0b38 100644
--- a/crypto/Makefile.objs
+++ b/crypto/Makefile.objs
@@ -1,5 +1,12 @@
-util-obj-y += init.o
-util-obj-y += hash.o
-util-obj-y += aes.o
-util-obj-y += desrfb.o
-util-obj-y += cipher.o
+crypto-obj-y = init.o
+crypto-obj-y += hash.o
+crypto-obj-y += aes.o
+crypto-obj-y += desrfb.o
+crypto-obj-y += cipher.o
+crypto-obj-y += tlscreds.o
+crypto-obj-y += tlscredsanon.o
+crypto-obj-y += tlscredsx509.o
+crypto-obj-y += tlssession.o
+
+# Let the userspace emulators avoid linking gnutls/etc
+crypto-aes-obj-y = aes.o
diff --git a/crypto/cipher-builtin.c b/crypto/cipher-builtin.c
index 30f4853c8..39e31a7f4 100644
--- a/crypto/cipher-builtin.c
+++ b/crypto/cipher-builtin.c
@@ -25,8 +25,7 @@ typedef struct QCryptoCipherBuiltinAES QCryptoCipherBuiltinAES;
 struct QCryptoCipherBuiltinAES {
     AES_KEY encrypt_key;
     AES_KEY decrypt_key;
-    uint8_t *iv;
-    size_t niv;
+    uint8_t iv[AES_BLOCK_SIZE];
 };
 typedef struct QCryptoCipherBuiltinDESRFB QCryptoCipherBuiltinDESRFB;
 struct QCryptoCipherBuiltinDESRFB {
@@ -40,6 +39,7 @@ struct QCryptoCipherBuiltin {
         QCryptoCipherBuiltinAES aes;
         QCryptoCipherBuiltinDESRFB desrfb;
     } state;
+    size_t blocksize;
     void (*free)(QCryptoCipher *cipher);
     int (*setiv)(QCryptoCipher *cipher,
                  const uint8_t *iv, size_t niv,
@@ -61,7 +61,6 @@ static void qcrypto_cipher_free_aes(QCryptoCipher *cipher)
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
-    g_free(ctxt->state.aes.iv);
     g_free(ctxt);
     cipher->opaque = NULL;
 }
@@ -145,15 +144,13 @@ static int qcrypto_cipher_setiv_aes(QCryptoCipher *cipher,
                                      Error **errp)
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
-    if (niv != 16) {
-        error_setg(errp, "IV must be 16 bytes not %zu", niv);
+    if (niv != AES_BLOCK_SIZE) {
+        error_setg(errp, "IV must be %d bytes not %zu",
+                   AES_BLOCK_SIZE, niv);
         return -1;
     }
 
-    g_free(ctxt->state.aes.iv);
-    ctxt->state.aes.iv = g_new0(uint8_t, niv);
-    memcpy(ctxt->state.aes.iv, iv, niv);
-    ctxt->state.aes.niv = niv;
+    memcpy(ctxt->state.aes.iv, iv, AES_BLOCK_SIZE);
 
     return 0;
 }
@@ -185,6 +182,7 @@ static int qcrypto_cipher_init_aes(QCryptoCipher *cipher,
         goto error;
     }
 
+    ctxt->blocksize = AES_BLOCK_SIZE;
     ctxt->free = qcrypto_cipher_free_aes;
     ctxt->setiv = qcrypto_cipher_setiv_aes;
     ctxt->encrypt = qcrypto_cipher_encrypt_aes;
@@ -286,6 +284,7 @@ static int qcrypto_cipher_init_des_rfb(QCryptoCipher *cipher,
     memcpy(ctxt->state.desrfb.key, key, nkey);
     ctxt->state.desrfb.nkey = nkey;
 
+    ctxt->blocksize = 8;
     ctxt->free = qcrypto_cipher_free_des_rfb;
     ctxt->setiv = qcrypto_cipher_setiv_des_rfb;
     ctxt->encrypt = qcrypto_cipher_encrypt_des_rfb;
@@ -374,6 +373,12 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
+    if (len % ctxt->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctxt->blocksize);
+        return -1;
+    }
+
     return ctxt->encrypt(cipher, in, out, len, errp);
 }
 
@@ -386,6 +391,12 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
+    if (len % ctxt->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctxt->blocksize);
+        return -1;
+    }
+
     return ctxt->decrypt(cipher, in, out, len, errp);
 }
 
diff --git a/crypto/cipher-gcrypt.c b/crypto/cipher-gcrypt.c
index 8cfc56250..c4f811487 100644
--- a/crypto/cipher-gcrypt.c
+++ b/crypto/cipher-gcrypt.c
@@ -34,6 +34,11 @@ bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
     }
 }
 
+typedef struct QCryptoCipherGcrypt QCryptoCipherGcrypt;
+struct QCryptoCipherGcrypt {
+    gcry_cipher_hd_t handle;
+    size_t blocksize;
+};
 
 QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
                                   QCryptoCipherMode mode,
@@ -41,7 +46,7 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
                                   Error **errp)
 {
     QCryptoCipher *cipher;
-    gcry_cipher_hd_t handle;
+    QCryptoCipherGcrypt *ctx;
     gcry_error_t err;
     int gcryalg, gcrymode;
 
@@ -87,7 +92,9 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
     cipher->alg = alg;
     cipher->mode = mode;
 
-    err = gcry_cipher_open(&handle, gcryalg, gcrymode, 0);
+    ctx = g_new0(QCryptoCipherGcrypt, 1);
+
+    err = gcry_cipher_open(&ctx->handle, gcryalg, gcrymode, 0);
     if (err != 0) {
         error_setg(errp, "Cannot initialize cipher: %s",
                    gcry_strerror(err));
@@ -100,10 +107,12 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
          * bizarre RFB variant of DES :-)
          */
         uint8_t *rfbkey = qcrypto_cipher_munge_des_rfb_key(key, nkey);
-        err = gcry_cipher_setkey(handle, rfbkey, nkey);
+        err = gcry_cipher_setkey(ctx->handle, rfbkey, nkey);
         g_free(rfbkey);
+        ctx->blocksize = 8;
     } else {
-        err = gcry_cipher_setkey(handle, key, nkey);
+        err = gcry_cipher_setkey(ctx->handle, key, nkey);
+        ctx->blocksize = 16;
     }
     if (err != 0) {
         error_setg(errp, "Cannot set key: %s",
@@ -111,11 +120,12 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         goto error;
     }
 
-    cipher->opaque = handle;
+    cipher->opaque = ctx;
     return cipher;
 
  error:
-    gcry_cipher_close(handle);
+    gcry_cipher_close(ctx->handle);
+    g_free(ctx);
     g_free(cipher);
     return NULL;
 }
@@ -123,12 +133,13 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
 
 void qcrypto_cipher_free(QCryptoCipher *cipher)
 {
-    gcry_cipher_hd_t handle;
+    QCryptoCipherGcrypt *ctx;
     if (!cipher) {
         return;
     }
-    handle = cipher->opaque;
-    gcry_cipher_close(handle);
+    ctx = cipher->opaque;
+    gcry_cipher_close(ctx->handle);
+    g_free(ctx);
     g_free(cipher);
 }
 
@@ -139,10 +150,16 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
                            size_t len,
                            Error **errp)
 {
-    gcry_cipher_hd_t handle = cipher->opaque;
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
     gcry_error_t err;
 
-    err = gcry_cipher_encrypt(handle,
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
+    err = gcry_cipher_encrypt(ctx->handle,
                               out, len,
                               in, len);
     if (err != 0) {
@@ -161,10 +178,16 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
                            size_t len,
                            Error **errp)
 {
-    gcry_cipher_hd_t handle = cipher->opaque;
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
     gcry_error_t err;
 
-    err = gcry_cipher_decrypt(handle,
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
+    err = gcry_cipher_decrypt(ctx->handle,
                               out, len,
                               in, len);
     if (err != 0) {
@@ -180,11 +203,17 @@ int qcrypto_cipher_setiv(QCryptoCipher *cipher,
                          const uint8_t *iv, size_t niv,
                          Error **errp)
 {
-    gcry_cipher_hd_t handle = cipher->opaque;
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
     gcry_error_t err;
 
-    gcry_cipher_reset(handle);
-    err = gcry_cipher_setiv(handle, iv, niv);
+    if (niv != ctx->blocksize) {
+        error_setg(errp, "Expected IV size %zu not %zu",
+                   ctx->blocksize, niv);
+        return -1;
+    }
+
+    gcry_cipher_reset(ctx->handle);
+    err = gcry_cipher_setiv(ctx->handle, iv, niv);
     if (err != 0) {
         error_setg(errp, "Cannot set IV: %s",
                    gcry_strerror(err));
diff --git a/crypto/cipher-nettle.c b/crypto/cipher-nettle.c
index b01cb1c85..7449338d3 100644
--- a/crypto/cipher-nettle.c
+++ b/crypto/cipher-nettle.c
@@ -69,7 +69,7 @@ struct QCryptoCipherNettle {
     nettle_cipher_func *alg_encrypt;
     nettle_cipher_func *alg_decrypt;
     uint8_t *iv;
-    size_t niv;
+    size_t blocksize;
 };
 
 bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
@@ -125,7 +125,7 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         ctx->alg_encrypt = des_encrypt_wrapper;
         ctx->alg_decrypt = des_decrypt_wrapper;
 
-        ctx->niv = DES_BLOCK_SIZE;
+        ctx->blocksize = DES_BLOCK_SIZE;
         break;
 
     case QCRYPTO_CIPHER_ALG_AES_128:
@@ -140,14 +140,14 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         ctx->alg_encrypt = aes_encrypt_wrapper;
         ctx->alg_decrypt = aes_decrypt_wrapper;
 
-        ctx->niv = AES_BLOCK_SIZE;
+        ctx->blocksize = AES_BLOCK_SIZE;
         break;
     default:
         error_setg(errp, "Unsupported cipher algorithm %d", alg);
         goto error;
     }
 
-    ctx->iv = g_new0(uint8_t, ctx->niv);
+    ctx->iv = g_new0(uint8_t, ctx->blocksize);
     cipher->opaque = ctx;
 
     return cipher;
@@ -184,6 +184,12 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherNettle *ctx = cipher->opaque;
 
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
     switch (cipher->mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
         ctx->alg_encrypt(ctx->ctx_encrypt, len, out, in);
@@ -191,7 +197,7 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
 
     case QCRYPTO_CIPHER_MODE_CBC:
         cbc_encrypt(ctx->ctx_encrypt, ctx->alg_encrypt,
-                    ctx->niv, ctx->iv,
+                    ctx->blocksize, ctx->iv,
                     len, out, in);
         break;
     default:
@@ -211,6 +217,12 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherNettle *ctx = cipher->opaque;
 
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
     switch (cipher->mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
         ctx->alg_decrypt(ctx->ctx_decrypt ? ctx->ctx_decrypt : ctx->ctx_encrypt,
@@ -219,7 +231,7 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
 
     case QCRYPTO_CIPHER_MODE_CBC:
         cbc_decrypt(ctx->ctx_decrypt ? ctx->ctx_decrypt : ctx->ctx_encrypt,
-                    ctx->alg_decrypt, ctx->niv, ctx->iv,
+                    ctx->alg_decrypt, ctx->blocksize, ctx->iv,
                     len, out, in);
         break;
     default:
@@ -235,9 +247,9 @@ int qcrypto_cipher_setiv(QCryptoCipher *cipher,
                          Error **errp)
 {
     QCryptoCipherNettle *ctx = cipher->opaque;
-    if (niv != ctx->niv) {
+    if (niv != ctx->blocksize) {
         error_setg(errp, "Expected IV size %zu not %zu",
-                   ctx->niv, niv);
+                   ctx->blocksize, niv);
         return -1;
     }
     memcpy(ctx->iv, iv, niv);
diff --git a/crypto/cipher.c b/crypto/cipher.c
index 024a00cb5..c8bd18053 100644
--- a/crypto/cipher.c
+++ b/crypto/cipher.c
@@ -47,7 +47,7 @@ qcrypto_cipher_validate_key_length(QCryptoCipherAlgorithm alg,
     return true;
 }
 
-#if defined(CONFIG_GNUTLS_GCRYPT) || defined(CONFIG_GNUTLS_NETTLE)
+#if defined(CONFIG_GCRYPT) || defined(CONFIG_NETTLE)
 static uint8_t *
 qcrypto_cipher_munge_des_rfb_key(const uint8_t *key,
                                  size_t nkey)
@@ -63,11 +63,11 @@ qcrypto_cipher_munge_des_rfb_key(const uint8_t *key,
     }
     return ret;
 }
-#endif /* CONFIG_GNUTLS_GCRYPT || CONFIG_GNUTLS_NETTLE */
+#endif /* CONFIG_GCRYPT || CONFIG_NETTLE */
 
-#ifdef CONFIG_GNUTLS_GCRYPT
+#ifdef CONFIG_GCRYPT
 #include "crypto/cipher-gcrypt.c"
-#elif defined CONFIG_GNUTLS_NETTLE
+#elif defined CONFIG_NETTLE
 #include "crypto/cipher-nettle.c"
 #else
 #include "crypto/cipher-builtin.c"
diff --git a/crypto/init.c b/crypto/init.c
index 7447882c7..d94faacdf 100644
--- a/crypto/init.c
+++ b/crypto/init.c
@@ -24,8 +24,9 @@
 #ifdef CONFIG_GNUTLS
 #include <gnutls/gnutls.h>
 #include <gnutls/crypto.h>
+#endif
 
-#ifdef CONFIG_GNUTLS_GCRYPT
+#ifdef CONFIG_GCRYPT
 #include <gcrypt.h>
 #endif
 
@@ -37,6 +38,7 @@
  *  - When GNUTLS >= 2.12, we must not initialize gcrypt threading
  *    because GNUTLS will do that itself
  *  - When GNUTLS < 2.12 we must always initialize gcrypt threading
+ *  - When GNUTLS is disabled we must always initialize gcrypt threading
  *
  * But....
  *
@@ -47,12 +49,15 @@
  *
  *   - gcrypt < 1.6.0
  * AND
- *   - gnutls < 2.12
+ *      - gnutls < 2.12
+ *   OR
+ *      - gnutls is disabled
  *
  */
 
-#if (defined(CONFIG_GNUTLS_GCRYPT) &&           \
-     (!defined(GNUTLS_VERSION_NUMBER) ||        \
+#if (defined(CONFIG_GCRYPT) &&                  \
+     (!defined(CONFIG_GNUTLS) ||                \
+      !defined(GNUTLS_VERSION_NUMBER) ||       \
       (GNUTLS_VERSION_NUMBER < 0x020c00)) &&    \
      (!defined(GCRYPT_VERSION_NUMBER) ||        \
       (GCRYPT_VERSION_NUMBER < 0x010600)))
@@ -113,6 +118,7 @@ static struct gcry_thread_cbs qcrypto_gcrypt_thread_impl = {
 
 int qcrypto_init(Error **errp)
 {
+#ifdef CONFIG_GNUTLS
     int ret;
     ret = gnutls_global_init();
     if (ret < 0) {
@@ -125,8 +131,9 @@ int qcrypto_init(Error **errp)
     gnutls_global_set_log_level(10);
     gnutls_global_set_log_function(qcrypto_gnutls_log);
 #endif
+#endif
 
-#ifdef CONFIG_GNUTLS_GCRYPT
+#ifdef CONFIG_GCRYPT
     if (!gcry_check_version(GCRYPT_VERSION)) {
         error_setg(errp, "Unable to initialize gcrypt");
         return -1;
@@ -139,12 +146,3 @@ int qcrypto_init(Error **errp)
 
     return 0;
 }
-
-#else /* ! CONFIG_GNUTLS */
-
-int qcrypto_init(Error **errp G_GNUC_UNUSED)
-{
-    return 0;
-}
-
-#endif /* ! CONFIG_GNUTLS */
diff --git a/crypto/tlscreds.c b/crypto/tlscreds.c
new file mode 100644
index 000000000..e7d9c1cfa
--- /dev/null
+++ b/crypto/tlscreds.c
@@ -0,0 +1,251 @@
+/*
+ * QEMU crypto TLS credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/tlscredspriv.h"
+#include "trace.h"
+
+#define DH_BITS 2048
+
+#ifdef CONFIG_GNUTLS
+int
+qcrypto_tls_creds_get_dh_params_file(QCryptoTLSCreds *creds,
+                                     const char *filename,
+                                     gnutls_dh_params_t *dh_params,
+                                     Error **errp)
+{
+    int ret;
+
+    trace_qcrypto_tls_creds_load_dh(creds, filename ? filename : "<generated>");
+
+    if (filename == NULL) {
+        ret = gnutls_dh_params_init(dh_params);
+        if (ret < 0) {
+            error_setg(errp, "Unable to initialize DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+        ret = gnutls_dh_params_generate2(*dh_params, DH_BITS);
+        if (ret < 0) {
+            gnutls_dh_params_deinit(*dh_params);
+            *dh_params = NULL;
+            error_setg(errp, "Unable to generate DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+    } else {
+        GError *gerr = NULL;
+        gchar *contents;
+        gsize len;
+        gnutls_datum_t data;
+        if (!g_file_get_contents(filename,
+                                 &contents,
+                                 &len,
+                                 &gerr)) {
+
+            error_setg(errp, "%s", gerr->message);
+            g_error_free(gerr);
+            return -1;
+        }
+        data.data = (unsigned char *)contents;
+        data.size = len;
+        ret = gnutls_dh_params_init(dh_params);
+        if (ret < 0) {
+            g_free(contents);
+            error_setg(errp, "Unable to initialize DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+        ret = gnutls_dh_params_import_pkcs3(*dh_params,
+                                            &data,
+                                            GNUTLS_X509_FMT_PEM);
+        g_free(contents);
+        if (ret < 0) {
+            gnutls_dh_params_deinit(*dh_params);
+            *dh_params = NULL;
+            error_setg(errp, "Unable to load DH parameters from %s: %s",
+                       filename, gnutls_strerror(ret));
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+int
+qcrypto_tls_creds_get_path(QCryptoTLSCreds *creds,
+                           const char *filename,
+                           bool required,
+                           char **cred,
+                           Error **errp)
+{
+    struct stat sb;
+    int ret = -1;
+
+    if (!creds->dir) {
+        if (required) {
+            error_setg(errp, "Missing 'dir' property value");
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    *cred = g_strdup_printf("%s/%s", creds->dir, filename);
+
+    if (stat(*cred, &sb) < 0) {
+        if (errno == ENOENT && !required) {
+            ret = 0;
+        } else {
+            error_setg_errno(errp, errno,
+                             "Unable to access credentials %s",
+                             *cred);
+        }
+        g_free(*cred);
+        *cred = NULL;
+        goto cleanup;
+    }
+
+    ret = 0;
+ cleanup:
+    trace_qcrypto_tls_creds_get_path(creds, filename,
+                                     *cred ? *cred : "<none>");
+    return ret;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_prop_set_verify(Object *obj,
+                                  bool value,
+                                  Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->verifyPeer = value;
+}
+
+
+static bool
+qcrypto_tls_creds_prop_get_verify(Object *obj,
+                                  Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return creds->verifyPeer;
+}
+
+
+static void
+qcrypto_tls_creds_prop_set_dir(Object *obj,
+                               const char *value,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->dir = g_strdup(value);
+}
+
+
+static char *
+qcrypto_tls_creds_prop_get_dir(Object *obj,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return g_strdup(creds->dir);
+}
+
+
+static void
+qcrypto_tls_creds_prop_set_endpoint(Object *obj,
+                                    int value,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->endpoint = value;
+}
+
+
+static int
+qcrypto_tls_creds_prop_get_endpoint(Object *obj,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return creds->endpoint;
+}
+
+
+static void
+qcrypto_tls_creds_init(Object *obj)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->verifyPeer = true;
+
+    object_property_add_bool(obj, "verify-peer",
+                             qcrypto_tls_creds_prop_get_verify,
+                             qcrypto_tls_creds_prop_set_verify,
+                             NULL);
+    object_property_add_str(obj, "dir",
+                            qcrypto_tls_creds_prop_get_dir,
+                            qcrypto_tls_creds_prop_set_dir,
+                            NULL);
+    object_property_add_enum(obj, "endpoint",
+                             "QCryptoTLSCredsEndpoint",
+                             QCryptoTLSCredsEndpoint_lookup,
+                             qcrypto_tls_creds_prop_get_endpoint,
+                             qcrypto_tls_creds_prop_set_endpoint,
+                             NULL);
+}
+
+
+static void
+qcrypto_tls_creds_finalize(Object *obj)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    g_free(creds->dir);
+}
+
+
+static const TypeInfo qcrypto_tls_creds_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_QCRYPTO_TLS_CREDS,
+    .instance_size = sizeof(QCryptoTLSCreds),
+    .instance_init = qcrypto_tls_creds_init,
+    .instance_finalize = qcrypto_tls_creds_finalize,
+    .class_size = sizeof(QCryptoTLSCredsClass),
+    .abstract = true,
+};
+
+
+static void
+qcrypto_tls_creds_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_info);
+}
+
+
+type_init(qcrypto_tls_creds_register_types);
diff --git a/crypto/tlscredsanon.c b/crypto/tlscredsanon.c
new file mode 100644
index 000000000..c3fcdaff0
--- /dev/null
+++ b/crypto/tlscredsanon.c
@@ -0,0 +1,223 @@
+/*
+ * QEMU crypto TLS anonymous credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredspriv.h"
+#include "qom/object_interfaces.h"
+#include "trace.h"
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static int
+qcrypto_tls_creds_anon_load(QCryptoTLSCredsAnon *creds,
+                            Error **errp)
+{
+    char *dhparams = NULL;
+    int ret;
+    int rv = -1;
+
+    trace_qcrypto_tls_creds_anon_load(creds,
+            creds->parent_obj.dir ? creds->parent_obj.dir : "<nodir>");
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_DH_PARAMS,
+                                       false, &dhparams, errp) < 0) {
+            goto cleanup;
+        }
+
+        ret = gnutls_anon_allocate_server_credentials(&creds->data.server);
+        if (ret < 0) {
+            error_setg(errp, "Cannot allocate credentials: %s",
+                       gnutls_strerror(ret));
+            goto cleanup;
+        }
+
+        if (qcrypto_tls_creds_get_dh_params_file(&creds->parent_obj, dhparams,
+                                                 &creds->parent_obj.dh_params,
+                                                 errp) < 0) {
+            goto cleanup;
+        }
+
+        gnutls_anon_set_server_dh_params(creds->data.server,
+                                         creds->parent_obj.dh_params);
+    } else {
+        ret = gnutls_anon_allocate_client_credentials(&creds->data.client);
+        if (ret < 0) {
+            error_setg(errp, "Cannot allocate credentials: %s",
+                       gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    rv = 0;
+ cleanup:
+    g_free(dhparams);
+    return rv;
+}
+
+
+static void
+qcrypto_tls_creds_anon_unload(QCryptoTLSCredsAnon *creds)
+{
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT) {
+        if (creds->data.client) {
+            gnutls_anon_free_client_credentials(creds->data.client);
+            creds->data.client = NULL;
+        }
+    } else {
+        if (creds->data.server) {
+            gnutls_anon_free_server_credentials(creds->data.server);
+            creds->data.server = NULL;
+        }
+    }
+    if (creds->parent_obj.dh_params) {
+        gnutls_dh_params_deinit(creds->parent_obj.dh_params);
+        creds->parent_obj.dh_params = NULL;
+    }
+}
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_load(QCryptoTLSCredsAnon *creds G_GNUC_UNUSED,
+                            Error **errp)
+{
+    error_setg(errp, "TLS credentials support requires GNUTLS");
+}
+
+
+static void
+qcrypto_tls_creds_anon_unload(QCryptoTLSCredsAnon *creds G_GNUC_UNUSED)
+{
+    /* nada */
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_prop_set_loaded(Object *obj,
+                                       bool value,
+                                       Error **errp)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    if (value) {
+        qcrypto_tls_creds_anon_load(creds, errp);
+    } else {
+        qcrypto_tls_creds_anon_unload(creds);
+    }
+}
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static bool
+qcrypto_tls_creds_anon_prop_get_loaded(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        return creds->data.server != NULL;
+    } else {
+        return creds->data.client != NULL;
+    }
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static bool
+qcrypto_tls_creds_anon_prop_get_loaded(Object *obj G_GNUC_UNUSED,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_tls_creds_anon_init(Object *obj)
+{
+    object_property_add_bool(obj, "loaded",
+                             qcrypto_tls_creds_anon_prop_get_loaded,
+                             qcrypto_tls_creds_anon_prop_set_loaded,
+                             NULL);
+}
+
+
+static void
+qcrypto_tls_creds_anon_finalize(Object *obj)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    qcrypto_tls_creds_anon_unload(creds);
+}
+
+
+static void
+qcrypto_tls_creds_anon_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_tls_creds_anon_complete;
+}
+
+
+static const TypeInfo qcrypto_tls_creds_anon_info = {
+    .parent = TYPE_QCRYPTO_TLS_CREDS,
+    .name = TYPE_QCRYPTO_TLS_CREDS_ANON,
+    .instance_size = sizeof(QCryptoTLSCredsAnon),
+    .instance_init = qcrypto_tls_creds_anon_init,
+    .instance_finalize = qcrypto_tls_creds_anon_finalize,
+    .class_size = sizeof(QCryptoTLSCredsAnonClass),
+    .class_init = qcrypto_tls_creds_anon_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_tls_creds_anon_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_anon_info);
+}
+
+
+type_init(qcrypto_tls_creds_anon_register_types);
diff --git a/crypto/tlscredspriv.h b/crypto/tlscredspriv.h
new file mode 100644
index 000000000..9222be4a9
--- /dev/null
+++ b/crypto/tlscredspriv.h
@@ -0,0 +1,42 @@
+/*
+ * QEMU crypto TLS credential support private helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_PRIV_H__
+#define QCRYPTO_TLSCRED_PRIV_H__
+
+#include "crypto/tlscreds.h"
+
+#ifdef CONFIG_GNUTLS
+
+int qcrypto_tls_creds_get_path(QCryptoTLSCreds *creds,
+                               const char *filename,
+                               bool required,
+                               char **cred,
+                               Error **errp);
+
+int qcrypto_tls_creds_get_dh_params_file(QCryptoTLSCreds *creds,
+                                         const char *filename,
+                                         gnutls_dh_params_t *dh_params,
+                                         Error **errp);
+
+#endif
+
+#endif /* QCRYPTO_TLSCRED_PRIV_H__ */
+
diff --git a/crypto/tlscredsx509.c b/crypto/tlscredsx509.c
new file mode 100644
index 000000000..26f18cbb4
--- /dev/null
+++ b/crypto/tlscredsx509.c
@@ -0,0 +1,815 @@
+/*
+ * QEMU crypto TLS x509 credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/tlscredsx509.h"
+#include "crypto/tlscredspriv.h"
+#include "qom/object_interfaces.h"
+#include "trace.h"
+
+
+#ifdef CONFIG_GNUTLS
+
+#include <gnutls/x509.h>
+
+
+static int
+qcrypto_tls_creds_check_cert_times(gnutls_x509_crt_t cert,
+                                   const char *certFile,
+                                   bool isServer,
+                                   bool isCA,
+                                   Error **errp)
+{
+    time_t now = time(NULL);
+
+    if (now == ((time_t)-1)) {
+        error_setg_errno(errp, errno, "cannot get current time");
+        return -1;
+    }
+
+    if (gnutls_x509_crt_get_expiration_time(cert) < now) {
+        error_setg(errp,
+                   (isCA ?
+                    "The CA certificate %s has expired" :
+                    (isServer ?
+                     "The server certificate %s has expired" :
+                     "The client certificate %s has expired")),
+                   certFile);
+        return -1;
+    }
+
+    if (gnutls_x509_crt_get_activation_time(cert) > now) {
+        error_setg(errp,
+                   (isCA ?
+                    "The CA certificate %s is not yet active" :
+                    (isServer ?
+                     "The server certificate %s is not yet active" :
+                     "The client certificate %s is not yet active")),
+                   certFile);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+#if LIBGNUTLS_VERSION_NUMBER >= 2
+/*
+ * The gnutls_x509_crt_get_basic_constraints function isn't
+ * available in GNUTLS 1.0.x branches. This isn't critical
+ * though, since gnutls_certificate_verify_peers2 will do
+ * pretty much the same check at runtime, so we can just
+ * disable this code
+ */
+static int
+qcrypto_tls_creds_check_cert_basic_constraints(QCryptoTLSCredsX509 *creds,
+                                               gnutls_x509_crt_t cert,
+                                               const char *certFile,
+                                               bool isServer,
+                                               bool isCA,
+                                               Error **errp)
+{
+    int status;
+
+    status = gnutls_x509_crt_get_basic_constraints(cert, NULL, NULL, NULL);
+    trace_qcrypto_tls_creds_x509_check_basic_constraints(
+        creds, certFile, status);
+
+    if (status > 0) { /* It is a CA cert */
+        if (!isCA) {
+            error_setg(errp, isServer ?
+                       "The certificate %s basic constraints show a CA, "
+                       "but we need one for a server" :
+                       "The certificate %s basic constraints show a CA, "
+                       "but we need one for a client",
+                       certFile);
+            return -1;
+        }
+    } else if (status == 0) { /* It is not a CA cert */
+        if (isCA) {
+            error_setg(errp,
+                       "The certificate %s basic constraints do not "
+                       "show a CA",
+                       certFile);
+            return -1;
+        }
+    } else if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+        /* Missing basicConstraints */
+        if (isCA) {
+            error_setg(errp,
+                       "The certificate %s is missing basic constraints "
+                       "for a CA",
+                       certFile);
+            return -1;
+        }
+    } else { /* General error */
+        error_setg(errp,
+                   "Unable to query certificate %s basic constraints: %s",
+                   certFile, gnutls_strerror(status));
+        return -1;
+    }
+
+    return 0;
+}
+#endif
+
+
+static int
+qcrypto_tls_creds_check_cert_key_usage(QCryptoTLSCredsX509 *creds,
+                                       gnutls_x509_crt_t cert,
+                                       const char *certFile,
+                                       bool isCA,
+                                       Error **errp)
+{
+    int status;
+    unsigned int usage = 0;
+    unsigned int critical = 0;
+
+    status = gnutls_x509_crt_get_key_usage(cert, &usage, &critical);
+    trace_qcrypto_tls_creds_x509_check_key_usage(
+        creds, certFile, status, usage, critical);
+
+    if (status < 0) {
+        if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+            usage = isCA ? GNUTLS_KEY_KEY_CERT_SIGN :
+                GNUTLS_KEY_DIGITAL_SIGNATURE|GNUTLS_KEY_KEY_ENCIPHERMENT;
+        } else {
+            error_setg(errp,
+                       "Unable to query certificate %s key usage: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+    }
+
+    if (isCA) {
+        if (!(usage & GNUTLS_KEY_KEY_CERT_SIGN)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit "
+                           "certificate signing", certFile);
+                return -1;
+            }
+        }
+    } else {
+        if (!(usage & GNUTLS_KEY_DIGITAL_SIGNATURE)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit digital "
+                           "signature", certFile);
+                return -1;
+            }
+        }
+        if (!(usage & GNUTLS_KEY_KEY_ENCIPHERMENT)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit key "
+                           "encipherment", certFile);
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert_key_purpose(QCryptoTLSCredsX509 *creds,
+                                         gnutls_x509_crt_t cert,
+                                         const char *certFile,
+                                         bool isServer,
+                                         Error **errp)
+{
+    int status;
+    size_t i;
+    unsigned int purposeCritical;
+    unsigned int critical;
+    char *buffer = NULL;
+    size_t size;
+    bool allowClient = false, allowServer = false;
+
+    critical = 0;
+    for (i = 0; ; i++) {
+        size = 0;
+        status = gnutls_x509_crt_get_key_purpose_oid(cert, i, buffer,
+                                                     &size, NULL);
+
+        if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+
+            /* If there is no data at all, then we must allow
+               client/server to pass */
+            if (i == 0) {
+                allowServer = allowClient = true;
+            }
+            break;
+        }
+        if (status != GNUTLS_E_SHORT_MEMORY_BUFFER) {
+            error_setg(errp,
+                       "Unable to query certificate %s key purpose: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+
+        buffer = g_new0(char, size);
+
+        status = gnutls_x509_crt_get_key_purpose_oid(cert, i, buffer,
+                                                     &size, &purposeCritical);
+
+        if (status < 0) {
+            trace_qcrypto_tls_creds_x509_check_key_purpose(
+                creds, certFile, status, "<none>", purposeCritical);
+            g_free(buffer);
+            error_setg(errp,
+                       "Unable to query certificate %s key purpose: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+        trace_qcrypto_tls_creds_x509_check_key_purpose(
+            creds, certFile, status, buffer, purposeCritical);
+        if (purposeCritical) {
+            critical = true;
+        }
+
+        if (g_str_equal(buffer, GNUTLS_KP_TLS_WWW_SERVER)) {
+            allowServer = true;
+        } else if (g_str_equal(buffer, GNUTLS_KP_TLS_WWW_CLIENT)) {
+            allowClient = true;
+        } else if (g_str_equal(buffer, GNUTLS_KP_ANY)) {
+            allowServer = allowClient = true;
+        }
+
+        g_free(buffer);
+        buffer = NULL;
+    }
+
+    if (isServer) {
+        if (!allowServer) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s purpose does not allow "
+                           "use with a TLS server", certFile);
+                return -1;
+            }
+        }
+    } else {
+        if (!allowClient) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s purpose does not allow use "
+                           "with a TLS client", certFile);
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert(QCryptoTLSCredsX509 *creds,
+                             gnutls_x509_crt_t cert,
+                             const char *certFile,
+                             bool isServer,
+                             bool isCA,
+                             Error **errp)
+{
+    if (qcrypto_tls_creds_check_cert_times(cert, certFile,
+                                           isServer, isCA,
+                                           errp) < 0) {
+        return -1;
+    }
+
+#if LIBGNUTLS_VERSION_NUMBER >= 2
+    if (qcrypto_tls_creds_check_cert_basic_constraints(creds,
+                                                       cert, certFile,
+                                                       isServer, isCA,
+                                                       errp) < 0) {
+        return -1;
+    }
+#endif
+
+    if (qcrypto_tls_creds_check_cert_key_usage(creds,
+                                               cert, certFile,
+                                               isCA, errp) < 0) {
+        return -1;
+    }
+
+    if (!isCA &&
+        qcrypto_tls_creds_check_cert_key_purpose(creds,
+                                                 cert, certFile,
+                                                 isServer, errp) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert_pair(gnutls_x509_crt_t cert,
+                                  const char *certFile,
+                                  gnutls_x509_crt_t *cacerts,
+                                  size_t ncacerts,
+                                  const char *cacertFile,
+                                  bool isServer,
+                                  Error **errp)
+{
+    unsigned int status;
+
+    if (gnutls_x509_crt_list_verify(&cert, 1,
+                                    cacerts, ncacerts,
+                                    NULL, 0,
+                                    0, &status) < 0) {
+        error_setg(errp, isServer ?
+                   "Unable to verify server certificate %s against "
+                   "CA certificate %s" :
+                   "Unable to verify client certificate %s against "
+                   "CA certificate %s",
+                   certFile, cacertFile);
+        return -1;
+    }
+
+    if (status != 0) {
+        const char *reason = "Invalid certificate";
+
+        if (status & GNUTLS_CERT_INVALID) {
+            reason = "The certificate is not trusted";
+        }
+
+        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
+            reason = "The certificate hasn't got a known issuer";
+        }
+
+        if (status & GNUTLS_CERT_REVOKED) {
+            reason = "The certificate has been revoked";
+        }
+
+#ifndef GNUTLS_1_0_COMPAT
+        if (status & GNUTLS_CERT_INSECURE_ALGORITHM) {
+            reason = "The certificate uses an insecure algorithm";
+        }
+#endif
+
+        error_setg(errp,
+                   "Our own certificate %s failed validation against %s: %s",
+                   certFile, cacertFile, reason);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static gnutls_x509_crt_t
+qcrypto_tls_creds_load_cert(QCryptoTLSCredsX509 *creds,
+                            const char *certFile,
+                            bool isServer,
+                            Error **errp)
+{
+    gnutls_datum_t data;
+    gnutls_x509_crt_t cert = NULL;
+    char *buf = NULL;
+    gsize buflen;
+    GError *gerr;
+    int ret = -1;
+
+    trace_qcrypto_tls_creds_x509_load_cert(creds, isServer, certFile);
+
+    if (gnutls_x509_crt_init(&cert) < 0) {
+        error_setg(errp, "Unable to initialize certificate");
+        goto cleanup;
+    }
+
+    if (!g_file_get_contents(certFile, &buf, &buflen, &gerr)) {
+        error_setg(errp, "Cannot load CA cert list %s: %s",
+                   certFile, gerr->message);
+        g_error_free(gerr);
+        goto cleanup;
+    }
+
+    data.data = (unsigned char *)buf;
+    data.size = strlen(buf);
+
+    if (gnutls_x509_crt_import(cert, &data, GNUTLS_X509_FMT_PEM) < 0) {
+        error_setg(errp, isServer ?
+                   "Unable to import server certificate %s" :
+                   "Unable to import client certificate %s",
+                   certFile);
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    if (ret != 0) {
+        gnutls_x509_crt_deinit(cert);
+        cert = NULL;
+    }
+    g_free(buf);
+    return cert;
+}
+
+
+static int
+qcrypto_tls_creds_load_ca_cert_list(QCryptoTLSCredsX509 *creds,
+                                    const char *certFile,
+                                    gnutls_x509_crt_t *certs,
+                                    unsigned int certMax,
+                                    size_t *ncerts,
+                                    Error **errp)
+{
+    gnutls_datum_t data;
+    char *buf = NULL;
+    gsize buflen;
+    int ret = -1;
+    GError *gerr = NULL;
+
+    *ncerts = 0;
+    trace_qcrypto_tls_creds_x509_load_cert_list(creds, certFile);
+
+    if (!g_file_get_contents(certFile, &buf, &buflen, &gerr)) {
+        error_setg(errp, "Cannot load CA cert list %s: %s",
+                   certFile, gerr->message);
+        g_error_free(gerr);
+        goto cleanup;
+    }
+
+    data.data = (unsigned char *)buf;
+    data.size = strlen(buf);
+
+    if (gnutls_x509_crt_list_import(certs, &certMax, &data,
+                                    GNUTLS_X509_FMT_PEM, 0) < 0) {
+        error_setg(errp,
+                   "Unable to import CA certificate list %s",
+                   certFile);
+        goto cleanup;
+    }
+    *ncerts = certMax;
+
+    ret = 0;
+
+ cleanup:
+    g_free(buf);
+    return ret;
+}
+
+
+#define MAX_CERTS 16
+static int
+qcrypto_tls_creds_x509_sanity_check(QCryptoTLSCredsX509 *creds,
+                                    bool isServer,
+                                    const char *cacertFile,
+                                    const char *certFile,
+                                    Error **errp)
+{
+    gnutls_x509_crt_t cert = NULL;
+    gnutls_x509_crt_t cacerts[MAX_CERTS];
+    size_t ncacerts = 0;
+    size_t i;
+    int ret = -1;
+
+    memset(cacerts, 0, sizeof(cacerts));
+    if (certFile &&
+        access(certFile, R_OK) == 0) {
+        cert = qcrypto_tls_creds_load_cert(creds,
+                                           certFile, isServer,
+                                           errp);
+        if (!cert) {
+            goto cleanup;
+        }
+    }
+    if (access(cacertFile, R_OK) == 0) {
+        if (qcrypto_tls_creds_load_ca_cert_list(creds,
+                                                cacertFile, cacerts,
+                                                MAX_CERTS, &ncacerts,
+                                                errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (cert &&
+        qcrypto_tls_creds_check_cert(creds,
+                                     cert, certFile, isServer,
+                                     false, errp) < 0) {
+        goto cleanup;
+    }
+
+    for (i = 0; i < ncacerts; i++) {
+        if (qcrypto_tls_creds_check_cert(creds,
+                                         cacerts[i], cacertFile,
+                                         isServer, true, errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (cert && ncacerts &&
+        qcrypto_tls_creds_check_cert_pair(cert, certFile, cacerts,
+                                          ncacerts, cacertFile,
+                                          isServer, errp) < 0) {
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    if (cert) {
+        gnutls_x509_crt_deinit(cert);
+    }
+    for (i = 0; i < ncacerts; i++) {
+        gnutls_x509_crt_deinit(cacerts[i]);
+    }
+    return ret;
+}
+
+
+static int
+qcrypto_tls_creds_x509_load(QCryptoTLSCredsX509 *creds,
+                            Error **errp)
+{
+    char *cacert = NULL, *cacrl = NULL, *cert = NULL,
+        *key = NULL, *dhparams = NULL;
+    int ret;
+    int rv = -1;
+
+    trace_qcrypto_tls_creds_x509_load(creds,
+            creds->parent_obj.dir ? creds->parent_obj.dir : "<nodir>");
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CERT,
+                                       true, &cacert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CRL,
+                                       false, &cacrl, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_SERVER_CERT,
+                                       true, &cert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_SERVER_KEY,
+                                       true, &key, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_DH_PARAMS,
+                                       false, &dhparams, errp) < 0) {
+            goto cleanup;
+        }
+    } else {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CERT,
+                                       true, &cacert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CLIENT_CERT,
+                                       false, &cert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CLIENT_KEY,
+                                       false, &key, errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (creds->sanityCheck &&
+        qcrypto_tls_creds_x509_sanity_check(creds,
+            creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+            cacert, cert, errp) < 0) {
+        goto cleanup;
+    }
+
+    ret = gnutls_certificate_allocate_credentials(&creds->data);
+    if (ret < 0) {
+        error_setg(errp, "Cannot allocate credentials: '%s'",
+                   gnutls_strerror(ret));
+        goto cleanup;
+    }
+
+    ret = gnutls_certificate_set_x509_trust_file(creds->data,
+                                                 cacert,
+                                                 GNUTLS_X509_FMT_PEM);
+    if (ret < 0) {
+        error_setg(errp, "Cannot load CA certificate '%s': %s",
+                   cacert, gnutls_strerror(ret));
+        goto cleanup;
+    }
+
+    if (cert != NULL && key != NULL) {
+        ret = gnutls_certificate_set_x509_key_file(creds->data,
+                                                   cert, key,
+                                                   GNUTLS_X509_FMT_PEM);
+        if (ret < 0) {
+            error_setg(errp, "Cannot load certificate '%s' & key '%s': %s",
+                       cert, key, gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    if (cacrl != NULL) {
+        ret = gnutls_certificate_set_x509_crl_file(creds->data,
+                                                   cacrl,
+                                                   GNUTLS_X509_FMT_PEM);
+        if (ret < 0) {
+            error_setg(errp, "Cannot load CRL '%s': %s",
+                       cacrl, gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_dh_params_file(&creds->parent_obj, dhparams,
+                                                 &creds->parent_obj.dh_params,
+                                                 errp) < 0) {
+            goto cleanup;
+        }
+        gnutls_certificate_set_dh_params(creds->data,
+                                         creds->parent_obj.dh_params);
+    }
+
+    rv = 0;
+ cleanup:
+    g_free(cacert);
+    g_free(cacrl);
+    g_free(cert);
+    g_free(key);
+    g_free(dhparams);
+    return rv;
+}
+
+
+static void
+qcrypto_tls_creds_x509_unload(QCryptoTLSCredsX509 *creds)
+{
+    if (creds->data) {
+        gnutls_certificate_free_credentials(creds->data);
+        creds->data = NULL;
+    }
+    if (creds->parent_obj.dh_params) {
+        gnutls_dh_params_deinit(creds->parent_obj.dh_params);
+        creds->parent_obj.dh_params = NULL;
+    }
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_load(QCryptoTLSCredsX509 *creds G_GNUC_UNUSED,
+                            Error **errp)
+{
+    error_setg(errp, "TLS credentials support requires GNUTLS");
+}
+
+
+static void
+qcrypto_tls_creds_x509_unload(QCryptoTLSCredsX509 *creds G_GNUC_UNUSED)
+{
+    /* nada */
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_loaded(Object *obj,
+                                       bool value,
+                                       Error **errp)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    if (value) {
+        qcrypto_tls_creds_x509_load(creds, errp);
+    } else {
+        qcrypto_tls_creds_x509_unload(creds);
+    }
+}
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_loaded(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return creds->data != NULL;
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_loaded(Object *obj G_GNUC_UNUSED,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_sanity(Object *obj,
+                                       bool value,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->sanityCheck = value;
+}
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_sanity(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return creds->sanityCheck;
+}
+
+
+static void
+qcrypto_tls_creds_x509_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_tls_creds_x509_init(Object *obj)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->sanityCheck = true;
+
+    object_property_add_bool(obj, "loaded",
+                             qcrypto_tls_creds_x509_prop_get_loaded,
+                             qcrypto_tls_creds_x509_prop_set_loaded,
+                             NULL);
+    object_property_add_bool(obj, "sanity-check",
+                             qcrypto_tls_creds_x509_prop_get_sanity,
+                             qcrypto_tls_creds_x509_prop_set_sanity,
+                             NULL);
+}
+
+
+static void
+qcrypto_tls_creds_x509_finalize(Object *obj)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    qcrypto_tls_creds_x509_unload(creds);
+}
+
+
+static void
+qcrypto_tls_creds_x509_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_tls_creds_x509_complete;
+}
+
+
+static const TypeInfo qcrypto_tls_creds_x509_info = {
+    .parent = TYPE_QCRYPTO_TLS_CREDS,
+    .name = TYPE_QCRYPTO_TLS_CREDS_X509,
+    .instance_size = sizeof(QCryptoTLSCredsX509),
+    .instance_init = qcrypto_tls_creds_x509_init,
+    .instance_finalize = qcrypto_tls_creds_x509_finalize,
+    .class_size = sizeof(QCryptoTLSCredsX509Class),
+    .class_init = qcrypto_tls_creds_x509_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_tls_creds_x509_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_x509_info);
+}
+
+
+type_init(qcrypto_tls_creds_x509_register_types);
diff --git a/crypto/tlssession.c b/crypto/tlssession.c
new file mode 100644
index 000000000..373552942
--- /dev/null
+++ b/crypto/tlssession.c
@@ -0,0 +1,574 @@
+/*
+ * QEMU crypto TLS session support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/tlssession.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredsx509.h"
+#include "qemu/acl.h"
+#include "trace.h"
+
+#ifdef CONFIG_GNUTLS
+
+
+#include <gnutls/x509.h>
+
+
+struct QCryptoTLSSession {
+    QCryptoTLSCreds *creds;
+    gnutls_session_t handle;
+    char *hostname;
+    char *aclname;
+    bool handshakeComplete;
+    QCryptoTLSSessionWriteFunc writeFunc;
+    QCryptoTLSSessionReadFunc readFunc;
+    void *opaque;
+    char *peername;
+};
+
+
+void
+qcrypto_tls_session_free(QCryptoTLSSession *session)
+{
+    if (!session) {
+        return;
+    }
+
+    gnutls_deinit(session->handle);
+    g_free(session->hostname);
+    g_free(session->peername);
+    g_free(session->aclname);
+    object_unref(OBJECT(session->creds));
+    g_free(session);
+}
+
+
+static ssize_t
+qcrypto_tls_session_push(void *opaque, const void *buf, size_t len)
+{
+    QCryptoTLSSession *session = opaque;
+
+    if (!session->writeFunc) {
+        errno = EIO;
+        return -1;
+    };
+
+    return session->writeFunc(buf, len, session->opaque);
+}
+
+
+static ssize_t
+qcrypto_tls_session_pull(void *opaque, void *buf, size_t len)
+{
+    QCryptoTLSSession *session = opaque;
+
+    if (!session->readFunc) {
+        errno = EIO;
+        return -1;
+    };
+
+    return session->readFunc(buf, len, session->opaque);
+}
+
+
+QCryptoTLSSession *
+qcrypto_tls_session_new(QCryptoTLSCreds *creds,
+                        const char *hostname,
+                        const char *aclname,
+                        QCryptoTLSCredsEndpoint endpoint,
+                        Error **errp)
+{
+    QCryptoTLSSession *session;
+    int ret;
+
+    session = g_new0(QCryptoTLSSession, 1);
+    trace_qcrypto_tls_session_new(
+        session, creds, hostname ? hostname : "<none>",
+        aclname ? aclname : "<none>", endpoint);
+
+    if (hostname) {
+        session->hostname = g_strdup(hostname);
+    }
+    if (aclname) {
+        session->aclname = g_strdup(aclname);
+    }
+    session->creds = creds;
+    object_ref(OBJECT(creds));
+
+    if (creds->endpoint != endpoint) {
+        error_setg(errp, "Credentials endpoint doesn't match session");
+        goto error;
+    }
+
+    if (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        ret = gnutls_init(&session->handle, GNUTLS_SERVER);
+    } else {
+        ret = gnutls_init(&session->handle, GNUTLS_CLIENT);
+    }
+    if (ret < 0) {
+        error_setg(errp, "Cannot initialize TLS session: %s",
+                   gnutls_strerror(ret));
+        goto error;
+    }
+
+    if (object_dynamic_cast(OBJECT(creds),
+                            TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+        QCryptoTLSCredsAnon *acreds = QCRYPTO_TLS_CREDS_ANON(creds);
+
+        ret = gnutls_priority_set_direct(session->handle,
+                                         "NORMAL:+ANON-DH", NULL);
+        if (ret < 0) {
+            error_setg(errp, "Unable to set TLS session priority: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+        if (creds->endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            ret = gnutls_credentials_set(session->handle,
+                                         GNUTLS_CRD_ANON,
+                                         acreds->data.server);
+        } else {
+            ret = gnutls_credentials_set(session->handle,
+                                         GNUTLS_CRD_ANON,
+                                         acreds->data.client);
+        }
+        if (ret < 0) {
+            error_setg(errp, "Cannot set session credentials: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+    } else if (object_dynamic_cast(OBJECT(creds),
+                                   TYPE_QCRYPTO_TLS_CREDS_X509)) {
+        QCryptoTLSCredsX509 *tcreds = QCRYPTO_TLS_CREDS_X509(creds);
+
+        ret = gnutls_set_default_priority(session->handle);
+        if (ret < 0) {
+            error_setg(errp, "Cannot set default TLS session priority: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+        ret = gnutls_credentials_set(session->handle,
+                                     GNUTLS_CRD_CERTIFICATE,
+                                     tcreds->data);
+        if (ret < 0) {
+            error_setg(errp, "Cannot set session credentials: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+
+        if (creds->endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            /* This requests, but does not enforce a client cert.
+             * The cert checking code later does enforcement */
+            gnutls_certificate_server_set_request(session->handle,
+                                                  GNUTLS_CERT_REQUEST);
+        }
+    } else {
+        error_setg(errp, "Unsupported TLS credentials type %s",
+                   object_get_typename(OBJECT(creds)));
+        goto error;
+    }
+
+    gnutls_transport_set_ptr(session->handle, session);
+    gnutls_transport_set_push_function(session->handle,
+                                       qcrypto_tls_session_push);
+    gnutls_transport_set_pull_function(session->handle,
+                                       qcrypto_tls_session_pull);
+
+    return session;
+
+ error:
+    qcrypto_tls_session_free(session);
+    return NULL;
+}
+
+static int
+qcrypto_tls_session_check_certificate(QCryptoTLSSession *session,
+                                      Error **errp)
+{
+    int ret;
+    unsigned int status;
+    const gnutls_datum_t *certs;
+    unsigned int nCerts, i;
+    time_t now;
+    gnutls_x509_crt_t cert = NULL;
+
+    now = time(NULL);
+    if (now == ((time_t)-1)) {
+        error_setg_errno(errp, errno, "Cannot get current time");
+        return -1;
+    }
+
+    ret = gnutls_certificate_verify_peers2(session->handle, &status);
+    if (ret < 0) {
+        error_setg(errp, "Verify failed: %s", gnutls_strerror(ret));
+        return -1;
+    }
+
+    if (status != 0) {
+        const char *reason = "Invalid certificate";
+
+        if (status & GNUTLS_CERT_INVALID) {
+            reason = "The certificate is not trusted";
+        }
+
+        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
+            reason = "The certificate hasn't got a known issuer";
+        }
+
+        if (status & GNUTLS_CERT_REVOKED) {
+            reason = "The certificate has been revoked";
+        }
+
+        if (status & GNUTLS_CERT_INSECURE_ALGORITHM) {
+            reason = "The certificate uses an insecure algorithm";
+        }
+
+        error_setg(errp, "%s", reason);
+        return -1;
+    }
+
+    certs = gnutls_certificate_get_peers(session->handle, &nCerts);
+    if (!certs) {
+        error_setg(errp, "No certificate peers");
+        return -1;
+    }
+
+    for (i = 0; i < nCerts; i++) {
+        ret = gnutls_x509_crt_init(&cert);
+        if (ret < 0) {
+            error_setg(errp, "Cannot initialize certificate: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+
+        ret = gnutls_x509_crt_import(cert, &certs[i], GNUTLS_X509_FMT_DER);
+        if (ret < 0) {
+            error_setg(errp, "Cannot import certificate: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_expiration_time(cert) < now) {
+            error_setg(errp, "The certificate has expired");
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_activation_time(cert) > now) {
+            error_setg(errp, "The certificate is not yet activated");
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_activation_time(cert) > now) {
+            error_setg(errp, "The certificate is not yet activated");
+            goto error;
+        }
+
+        if (i == 0) {
+            size_t dnameSize = 1024;
+            session->peername = g_malloc(dnameSize);
+        requery:
+            ret = gnutls_x509_crt_get_dn(cert, session->peername, &dnameSize);
+            if (ret < 0) {
+                if (ret == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+                    session->peername = g_realloc(session->peername,
+                                                  dnameSize);
+                    goto requery;
+                }
+                error_setg(errp, "Cannot get client distinguished name: %s",
+                           gnutls_strerror(ret));
+                goto error;
+            }
+            if (session->aclname) {
+                qemu_acl *acl = qemu_acl_find(session->aclname);
+                int allow;
+                if (!acl) {
+                    error_setg(errp, "Cannot find ACL %s",
+                               session->aclname);
+                    goto error;
+                }
+
+                allow = qemu_acl_party_is_allowed(acl, session->peername);
+
+                if (!allow) {
+                    error_setg(errp, "TLS x509 ACL check for %s is denied",
+                               session->peername);
+                    goto error;
+                }
+            }
+            if (session->hostname) {
+                if (!gnutls_x509_crt_check_hostname(cert, session->hostname)) {
+                    error_setg(errp,
+                               "Certificate does not match the hostname %s",
+                               session->hostname);
+                    goto error;
+                }
+            }
+        }
+
+        gnutls_x509_crt_deinit(cert);
+    }
+
+    return 0;
+
+ error:
+    gnutls_x509_crt_deinit(cert);
+    return -1;
+}
+
+
+int
+qcrypto_tls_session_check_credentials(QCryptoTLSSession *session,
+                                      Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(session->creds),
+                            TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+        return 0;
+    } else if (object_dynamic_cast(OBJECT(session->creds),
+                            TYPE_QCRYPTO_TLS_CREDS_X509)) {
+        if (session->creds->verifyPeer) {
+            return qcrypto_tls_session_check_certificate(session,
+                                                         errp);
+        } else {
+            return 0;
+        }
+    } else {
+        error_setg(errp, "Unexpected credential type %s",
+                   object_get_typename(OBJECT(session->creds)));
+        return -1;
+    }
+}
+
+
+void
+qcrypto_tls_session_set_callbacks(QCryptoTLSSession *session,
+                                  QCryptoTLSSessionWriteFunc writeFunc,
+                                  QCryptoTLSSessionReadFunc readFunc,
+                                  void *opaque)
+{
+    session->writeFunc = writeFunc;
+    session->readFunc = readFunc;
+    session->opaque = opaque;
+}
+
+
+ssize_t
+qcrypto_tls_session_write(QCryptoTLSSession *session,
+                          const char *buf,
+                          size_t len)
+{
+    ssize_t ret = gnutls_record_send(session->handle, buf, len);
+
+    if (ret < 0) {
+        switch (ret) {
+        case GNUTLS_E_AGAIN:
+            errno = EAGAIN;
+            break;
+        case GNUTLS_E_INTERRUPTED:
+            errno = EINTR;
+            break;
+        default:
+            errno = EIO;
+            break;
+        }
+        ret = -1;
+    }
+
+    return ret;
+}
+
+
+ssize_t
+qcrypto_tls_session_read(QCryptoTLSSession *session,
+                         char *buf,
+                         size_t len)
+{
+    ssize_t ret = gnutls_record_recv(session->handle, buf, len);
+
+    if (ret < 0) {
+        switch (ret) {
+        case GNUTLS_E_AGAIN:
+            errno = EAGAIN;
+            break;
+        case GNUTLS_E_INTERRUPTED:
+            errno = EINTR;
+            break;
+        default:
+            errno = EIO;
+            break;
+        }
+        ret = -1;
+    }
+
+    return ret;
+}
+
+
+int
+qcrypto_tls_session_handshake(QCryptoTLSSession *session,
+                              Error **errp)
+{
+    int ret = gnutls_handshake(session->handle);
+    if (ret == 0) {
+        session->handshakeComplete = true;
+    } else {
+        if (ret == GNUTLS_E_INTERRUPTED ||
+            ret == GNUTLS_E_AGAIN) {
+            ret = 1;
+        } else {
+            error_setg(errp, "TLS handshake failed: %s",
+                       gnutls_strerror(ret));
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *session)
+{
+    if (session->handshakeComplete) {
+        return QCRYPTO_TLS_HANDSHAKE_COMPLETE;
+    } else if (gnutls_record_get_direction(session->handle) == 0) {
+        return QCRYPTO_TLS_HANDSHAKE_RECVING;
+    } else {
+        return QCRYPTO_TLS_HANDSHAKE_SENDING;
+    }
+}
+
+
+int
+qcrypto_tls_session_get_key_size(QCryptoTLSSession *session,
+                                 Error **errp)
+{
+    gnutls_cipher_algorithm_t cipher;
+    int ssf;
+
+    cipher = gnutls_cipher_get(session->handle);
+    ssf = gnutls_cipher_get_key_size(cipher);
+    if (!ssf) {
+        error_setg(errp, "Cannot get TLS cipher key size");
+        return -1;
+    }
+    return ssf;
+}
+
+
+char *
+qcrypto_tls_session_get_peer_name(QCryptoTLSSession *session)
+{
+    if (session->peername) {
+        return g_strdup(session->peername);
+    }
+    return NULL;
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+QCryptoTLSSession *
+qcrypto_tls_session_new(QCryptoTLSCreds *creds G_GNUC_UNUSED,
+                        const char *hostname G_GNUC_UNUSED,
+                        const char *aclname G_GNUC_UNUSED,
+                        QCryptoTLSCredsEndpoint endpoint G_GNUC_UNUSED,
+                        Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return NULL;
+}
+
+
+void
+qcrypto_tls_session_free(QCryptoTLSSession *sess G_GNUC_UNUSED)
+{
+}
+
+
+int
+qcrypto_tls_session_check_credentials(QCryptoTLSSession *sess G_GNUC_UNUSED,
+                                      Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+void
+qcrypto_tls_session_set_callbacks(
+    QCryptoTLSSession *sess G_GNUC_UNUSED,
+    QCryptoTLSSessionWriteFunc writeFunc G_GNUC_UNUSED,
+    QCryptoTLSSessionReadFunc readFunc G_GNUC_UNUSED,
+    void *opaque G_GNUC_UNUSED)
+{
+}
+
+
+ssize_t
+qcrypto_tls_session_write(QCryptoTLSSession *sess,
+                          const char *buf,
+                          size_t len)
+{
+    errno = -EIO;
+    return -1;
+}
+
+
+ssize_t
+qcrypto_tls_session_read(QCryptoTLSSession *sess,
+                         char *buf,
+                         size_t len)
+{
+    errno = -EIO;
+    return -1;
+}
+
+
+int
+qcrypto_tls_session_handshake(QCryptoTLSSession *sess,
+                              Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *sess)
+{
+    return QCRYPTO_TLS_HANDSHAKE_COMPLETE;
+}
+
+
+int
+qcrypto_tls_session_get_key_size(QCryptoTLSSession *sess,
+                                 Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+char *
+qcrypto_tls_session_get_peer_name(QCryptoTLSSession *sess)
+{
+    return NULL;
+}
+
+#endif
diff --git a/default-configs/aarch64-linux-user.mak b/default-configs/aarch64-linux-user.mak
index 3df7de5b8..0a5b08a00 100644
--- a/default-configs/aarch64-linux-user.mak
+++ b/default-configs/aarch64-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for aarch64-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak
index 74f1db3a0..d9b90a50d 100644
--- a/default-configs/arm-softmmu.mak
+++ b/default-configs/arm-softmmu.mak
@@ -28,6 +28,7 @@ CONFIG_SSI_M25P80=y
 CONFIG_LAN9118=y
 CONFIG_SMC91C111=y
 CONFIG_ALLWINNER_EMAC=y
+CONFIG_IMX_FEC=y
 CONFIG_DS1338=y
 CONFIG_PFLASH_CFI01=y
 CONFIG_PFLASH_CFI02=y
@@ -98,7 +99,13 @@ CONFIG_ALLWINNER_A10_PIT=y
 CONFIG_ALLWINNER_A10_PIC=y
 CONFIG_ALLWINNER_A10=y
 
+CONFIG_FSL_IMX31=y
+CONFIG_FSL_IMX25=y
+
+CONFIG_IMX_I2C=y
+
 CONFIG_XIO3130=y
 CONFIG_IOH3420=y
 CONFIG_I82801B11=y
 CONFIG_ACPI=y
+CONFIG_SMBIOS=y
diff --git a/default-configs/i386-softmmu.mak b/default-configs/i386-softmmu.mak
index 48b57623f..43c96d1c9 100644
--- a/default-configs/i386-softmmu.mak
+++ b/default-configs/i386-softmmu.mak
@@ -7,6 +7,7 @@ CONFIG_QXL=$(CONFIG_SPICE)
 CONFIG_VGA_ISA=y
 CONFIG_VGA_CIRRUS=y
 CONFIG_VMWARE_VGA=y
+CONFIG_VIRTIO_VGA=y
 CONFIG_VMMOUSE=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
@@ -43,9 +44,9 @@ CONFIG_LPC_ICH9=y
 CONFIG_PCI_Q35=y
 CONFIG_APIC=y
 CONFIG_IOAPIC=y
-CONFIG_ICC_BUS=y
 CONFIG_PVPANIC=y
 CONFIG_MEM_HOTPLUG=y
 CONFIG_XIO3130=y
 CONFIG_IOH3420=y
 CONFIG_I82801B11=y
+CONFIG_SMBIOS=y
diff --git a/default-configs/pci.mak b/default-configs/pci.mak
index 7e10903ba..f250119e1 100644
--- a/default-configs/pci.mak
+++ b/default-configs/pci.mak
@@ -35,5 +35,5 @@ CONFIG_SDHCI=y
 CONFIG_EDU=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
-CONFIG_IVSHMEM=$(CONFIG_KVM)
+CONFIG_IVSHMEM=$(CONFIG_POSIX)
 CONFIG_ROCKER=y
diff --git a/default-configs/ppc64-softmmu.mak b/default-configs/ppc64-softmmu.mak
index ab62cc7ff..bb71b23ee 100644
--- a/default-configs/ppc64-softmmu.mak
+++ b/default-configs/ppc64-softmmu.mak
@@ -3,6 +3,7 @@
 include pci.mak
 include sound.mak
 include usb.mak
+CONFIG_VIRTIO_VGA=y
 CONFIG_ISA_MMIO=y
 CONFIG_ESCC=y
 CONFIG_M48T59=y
@@ -52,3 +53,4 @@ CONFIG_XICS_KVM=$(and $(CONFIG_PSERIES),$(CONFIG_KVM))
 # For PReP
 CONFIG_MC146818RTC=y
 CONFIG_ISA_TESTDEV=y
+CONFIG_MEM_HOTPLUG=y
diff --git a/default-configs/tilegx-linux-user.mak b/default-configs/tilegx-linux-user.mak
new file mode 100644
index 000000000..3e47493af
--- /dev/null
+++ b/default-configs/tilegx-linux-user.mak
@@ -0,0 +1 @@
+# Default configuration for tilegx-linux-user
diff --git a/default-configs/x86_64-softmmu.mak b/default-configs/x86_64-softmmu.mak
index 4962ed70a..dfb80954d 100644
--- a/default-configs/x86_64-softmmu.mak
+++ b/default-configs/x86_64-softmmu.mak
@@ -44,9 +44,9 @@ CONFIG_LPC_ICH9=y
 CONFIG_PCI_Q35=y
 CONFIG_APIC=y
 CONFIG_IOAPIC=y
-CONFIG_ICC_BUS=y
 CONFIG_PVPANIC=y
 CONFIG_MEM_HOTPLUG=y
 CONFIG_XIO3130=y
 CONFIG_IOH3420=y
 CONFIG_I82801B11=y
+CONFIG_SMBIOS=y
diff --git a/disas.c b/disas.c
index 69a606691..4e119449e 100644
--- a/disas.c
+++ b/disas.c
@@ -72,14 +72,6 @@ generic_print_address (bfd_vma addr, struct disassemble_info *info)
     (*info->fprintf_func) (info->stream, "0x%" PRIx64, addr);
 }
 
-/* Print address in hex, truncated to the width of a target virtual address. */
-static void
-generic_print_target_address(bfd_vma addr, struct disassemble_info *info)
-{
-    uint64_t mask = ~0ULL >> (64 - TARGET_VIRT_ADDR_SPACE_BITS);
-    generic_print_address(addr & mask, info);
-}
-
 /* Print address in hex, truncated to the width of a host virtual address. */
 static void
 generic_print_host_address(bfd_vma addr, struct disassemble_info *info)
@@ -201,7 +193,7 @@ void target_disas(FILE *out, CPUState *cpu, target_ulong code,
     s.info.read_memory_func = target_read_memory;
     s.info.buffer_vma = code;
     s.info.buffer_length = size;
-    s.info.print_address_func = generic_print_target_address;
+    s.info.print_address_func = generic_print_address;
 
 #ifdef TARGET_WORDS_BIGENDIAN
     s.info.endian = BFD_ENDIAN_BIG;
@@ -222,11 +214,6 @@ void target_disas(FILE *out, CPUState *cpu, target_ulong code,
         s.info.mach = bfd_mach_i386_i386;
     }
     s.info.print_insn = print_insn_i386;
-#elif defined(TARGET_SPARC)
-    s.info.print_insn = print_insn_sparc;
-#ifdef TARGET_SPARC64
-    s.info.mach = bfd_mach_sparc_v9b;
-#endif
 #elif defined(TARGET_PPC)
     if ((flags >> 16) & 1) {
         s.info.endian = BFD_ENDIAN_LITTLE;
@@ -243,29 +230,6 @@ void target_disas(FILE *out, CPUState *cpu, target_ulong code,
     }
     s.info.disassembler_options = (char *)"any";
     s.info.print_insn = print_insn_ppc;
-#elif defined(TARGET_M68K)
-    s.info.print_insn = print_insn_m68k;
-#elif defined(TARGET_MIPS)
-#ifdef TARGET_WORDS_BIGENDIAN
-    s.info.print_insn = print_insn_big_mips;
-#else
-    s.info.print_insn = print_insn_little_mips;
-#endif
-#elif defined(TARGET_SH4)
-    s.info.mach = bfd_mach_sh4;
-    s.info.print_insn = print_insn_sh;
-#elif defined(TARGET_ALPHA)
-    s.info.mach = bfd_mach_alpha_ev6;
-    s.info.print_insn = print_insn_alpha;
-#elif defined(TARGET_S390X)
-    s.info.mach = bfd_mach_s390_64;
-    s.info.print_insn = print_insn_s390;
-#elif defined(TARGET_MOXIE)
-    s.info.mach = bfd_arch_moxie;
-    s.info.print_insn = print_insn_moxie;
-#elif defined(TARGET_LM32)
-    s.info.mach = bfd_mach_lm32;
-    s.info.print_insn = print_insn_lm32;
 #endif
     if (s.info.print_insn == NULL) {
         s.info.print_insn = print_insn_od_target;
@@ -400,16 +364,6 @@ monitor_read_memory (bfd_vma memaddr, bfd_byte *myaddr, int length,
     return 0;
 }
 
-static int GCC_FMT_ATTR(2, 3)
-monitor_fprintf(FILE *stream, const char *fmt, ...)
-{
-    va_list ap;
-    va_start(ap, fmt);
-    monitor_vprintf((Monitor *)stream, fmt, ap);
-    va_end(ap);
-    return 0;
-}
-
 /* Disassembler for the monitor.
    See target_disas for a description of flags. */
 void monitor_disas(Monitor *mon, CPUState *cpu,
@@ -424,7 +378,7 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
     s.cpu = cpu;
     monitor_disas_is_physical = is_physical;
     s.info.read_memory_func = monitor_read_memory;
-    s.info.print_address_func = generic_print_target_address;
+    s.info.print_address_func = generic_print_address;
 
     s.info.buffer_vma = pc;
 
@@ -447,13 +401,6 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
         s.info.mach = bfd_mach_i386_i386;
     }
     s.info.print_insn = print_insn_i386;
-#elif defined(TARGET_ALPHA)
-    s.info.print_insn = print_insn_alpha;
-#elif defined(TARGET_SPARC)
-    s.info.print_insn = print_insn_sparc;
-#ifdef TARGET_SPARC64
-    s.info.mach = bfd_mach_sparc_v9b;
-#endif
 #elif defined(TARGET_PPC)
     if (flags & 0xFFFF) {
         /* If we have a precise definition of the instruction set, use it. */
@@ -469,26 +416,6 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
         s.info.endian = BFD_ENDIAN_LITTLE;
     }
     s.info.print_insn = print_insn_ppc;
-#elif defined(TARGET_M68K)
-    s.info.print_insn = print_insn_m68k;
-#elif defined(TARGET_MIPS)
-#ifdef TARGET_WORDS_BIGENDIAN
-    s.info.print_insn = print_insn_big_mips;
-#else
-    s.info.print_insn = print_insn_little_mips;
-#endif
-#elif defined(TARGET_SH4)
-    s.info.mach = bfd_mach_sh4;
-    s.info.print_insn = print_insn_sh;
-#elif defined(TARGET_S390X)
-    s.info.mach = bfd_mach_s390_64;
-    s.info.print_insn = print_insn_s390;
-#elif defined(TARGET_MOXIE)
-    s.info.mach = bfd_arch_moxie;
-    s.info.print_insn = print_insn_moxie;
-#elif defined(TARGET_LM32)
-    s.info.mach = bfd_mach_lm32;
-    s.info.print_insn = print_insn_lm32;
 #endif
     if (!s.info.print_insn) {
         monitor_printf(mon, "0x" TARGET_FMT_lx
diff --git a/disas/arm.c b/disas/arm.c
index 616524653..7a7354b76 100644
--- a/disas/arm.c
+++ b/disas/arm.c
@@ -1779,7 +1779,7 @@ print_insn_coprocessor (bfd_vma pc, struct disassemble_info *info, long given,
 
 			/* Is ``imm'' a negative number?  */
 			if (imm & 0x40)
-			  imm |= (-1 << 7);
+			  imm |= (~0u << 7);
 
 			func (stream, "%d", imm);
 		      }
diff --git a/disas/cris.c b/disas/cris.c
index 1b76a09db..4482a4113 100644
--- a/disas/cris.c
+++ b/disas/cris.c
@@ -2492,7 +2492,7 @@ print_with_operands (const struct cris_opcode *opcodep,
 	  = spec_reg_info ((insn >> 12) & 15, disdata->distype);
 
 	if (sregp->name == NULL)
-	  /* Should have been caught as a non-match eariler.  */
+	  /* Should have been caught as a non-match earlier.  */
 	  *tp++ = '?';
 	else
 	  {
diff --git a/disas/i386.c b/disas/i386.c
index 00ceca9c5..c63d4a06e 100644
--- a/disas/i386.c
+++ b/disas/i386.c
@@ -357,7 +357,7 @@ fetch_data(struct disassemble_info *info, bfd_byte *addr)
 #define Rd { OP_R, d_mode }
 #define Rm { OP_R, m_mode }
 #define Ib { OP_I, b_mode }
-#define sIb { OP_sI, b_mode }	/* sign extened byte */
+#define sIb { OP_sI, b_mode }	/* sign extended byte */
 #define Iv { OP_I, v_mode }
 #define Iq { OP_I, q_mode }
 #define Iv64 { OP_I64, v_mode }
diff --git a/disas/ia64.c b/disas/ia64.c
index a8fe26c41..d7c7bdfc5 100644
--- a/disas/ia64.c
+++ b/disas/ia64.c
@@ -18,7 +18,6 @@
    along with this file; see the file COPYING.  If not, see
    <http://www.gnu.org/licenses/>. */
 
-#include <assert.h>
 #include <string.h>
 
 #include "disas/bfd.h"
diff --git a/disas/microblaze.c b/disas/microblaze.c
index c14ab89b7..5d1590795 100644
--- a/disas/microblaze.c
+++ b/disas/microblaze.c
@@ -582,7 +582,6 @@ static const char pvr_register_prefix[] = "rpvr";
 #endif /* MICROBLAZE_OPC */
 
 #include "disas/bfd.h"
-#include <strings.h>
 
 #define get_field_rd(instr) get_field(instr, RD_MASK, RD_LOW)
 #define get_field_r1(instr) get_field(instr, RA_MASK, RA_LOW)
@@ -604,25 +603,6 @@ char * get_field_special (long instr, struct op_code_struct * op);
 unsigned long read_insn_microblaze (bfd_vma memaddr, 
 		      struct disassemble_info *info,
 		      struct op_code_struct **opr);
-enum microblaze_instr get_insn_microblaze (long inst,
-  		     bfd_boolean *isunsignedimm,
-  		     enum microblaze_instr_type *insn_type,
-  		     short *delay_slots);
-short get_delay_slots_microblaze (long inst);
-enum microblaze_instr microblaze_decode_insn (long insn,
-		        int *rd, 
-			int *ra, 
-			int *rb, 
-			int *imm);
-unsigned long
-microblaze_get_target_address (long inst,
-			       bfd_boolean immfound,
-			       int immval,
-			       long pcval,
-			       long r1val,
-			       long r2val,
-			       bfd_boolean *targetvalid,
-			       bfd_boolean *unconditionalbranch);
 
 static char *
 get_field (long instr, long mask, unsigned short low)
@@ -968,134 +948,3 @@ print_insn_microblaze (bfd_vma memaddr, struct disassemble_info * info)
   /* Say how many bytes we consumed? */
   return 4;
 }
-
-enum microblaze_instr
-get_insn_microblaze (long inst,
-  		     bfd_boolean *isunsignedimm,
-  		     enum microblaze_instr_type *insn_type,
-  		     short *delay_slots)
-{
-  struct op_code_struct * op;
-  *isunsignedimm = FALSE;
-
-  /* Just a linear search of the table.  */
-  for (op = opcodes; op->name != 0; op ++)
-    if (op->bit_sequence == (inst & op->opcode_mask))
-      break;
-
-  if (op->name == 0)
-    return invalid_inst;
-  else {
-    *isunsignedimm = (op->inst_type == INST_TYPE_RD_R1_UNSIGNED_IMM);
-    *insn_type = op->instr_type;
-    *delay_slots = op->delay_slots;
-    return op->instr;
-  }
-}
-
-short
-get_delay_slots_microblaze (long inst)
-{
-  bfd_boolean isunsignedimm;
-  enum microblaze_instr_type insn_type;
-  enum microblaze_instr op;
-  short delay_slots;
-
-  op = get_insn_microblaze( inst, &isunsignedimm, &insn_type, &delay_slots);
-  if (op == invalid_inst)
-    return 0;
-  else 
-    return delay_slots;
-}
-
-enum microblaze_instr
-microblaze_decode_insn (long insn,
-		        int *rd, 
-			int *ra, 
-			int *rb, 
-			int *imm)
-{
-  enum microblaze_instr op;
-  bfd_boolean t1;
-  enum microblaze_instr_type t2;
-  short t3;
-
-  op = get_insn_microblaze(insn, &t1, &t2, &t3);
-  *rd = (insn & RD_MASK) >> RD_LOW;
-  *ra = (insn & RA_MASK) >> RA_LOW;
-  *rb = (insn & RB_MASK) >> RB_LOW;
-  t3 = (insn & IMM_MASK) >> IMM_LOW;
-  *imm = (int) t3;
-  return (op);
-}
-
-unsigned long
-microblaze_get_target_address (long inst,
-			       bfd_boolean immfound,
-			       int immval,
-			       long pcval,
-			       long r1val,
-			       long r2val,
-			       bfd_boolean *targetvalid,
-			       bfd_boolean *unconditionalbranch)
-{
-  struct op_code_struct * op;
-  long targetaddr = 0;
-
-  *unconditionalbranch = FALSE;
-  /* Just a linear search of the table.  */
-  for (op = opcodes; op->name != 0; op ++)
-    if (op->bit_sequence == (inst & op->opcode_mask))
-      break;
-
-  if (op->name == 0) {
-    *targetvalid = FALSE;
-  } else if (op->instr_type == branch_inst) {
-    switch (op->inst_type) {
-    case INST_TYPE_R2:
-      *unconditionalbranch = TRUE;
-      /* fallthru */
-    case INST_TYPE_RD_R2:
-    case INST_TYPE_R1_R2:
-      targetaddr = r2val;
-      *targetvalid = TRUE;
-      if (op->inst_offset_type == INST_PC_OFFSET)
-	targetaddr += pcval;
-      break;
-    case INST_TYPE_IMM:
-      *unconditionalbranch = TRUE;
-      /* fallthru */
-    case INST_TYPE_RD_IMM:
-    case INST_TYPE_R1_IMM:
-      if (immfound) {
-	targetaddr = (immval << 16) & 0xffff0000;
-	targetaddr |= (get_int_field_imm(inst) & 0x0000ffff);
-      } else {
-	targetaddr = get_int_field_imm(inst);
-	if (targetaddr & 0x8000)
-	  targetaddr |= 0xFFFF0000;
-      }
-      if (op->inst_offset_type == INST_PC_OFFSET)
-	targetaddr += pcval;
-      *targetvalid = TRUE;
-      break;
-    default:
-      *targetvalid = FALSE;
-      break;
-    }
-  } else if (op->instr_type == return_inst) {
-      if (immfound) {
-	targetaddr = (immval << 16) & 0xffff0000;
-	targetaddr |= (get_int_field_imm(inst) & 0x0000ffff);
-      } else {
-	targetaddr = get_int_field_imm(inst);
-	if (targetaddr & 0x8000)
-	  targetaddr |= 0xFFFF0000;
-      }
-      targetaddr += r1val;
-      *targetvalid = TRUE;
-  } else {
-    *targetvalid = FALSE;
-  }
-  return targetaddr;
-}
diff --git a/disas/mips.c b/disas/mips.c
index 01336a838..bf0bbaf86 100644
--- a/disas/mips.c
+++ b/disas/mips.c
@@ -2420,9 +2420,11 @@ const struct mips_opcode mips_builtin_opcodes[] =
 {"hibernate","",        0x42000023, 0xffffffff,	0, 			0,		V1	},
 {"ins",     "t,r,+A,+B", 0x7c000004, 0xfc00003f, WR_t|RD_s,    		0,		I33	},
 {"jr",      "s",	0x00000008, 0xfc1fffff,	UBD|RD_s,		0,		I1	},
+{"jr",      "s",	0x00000009, 0xfc1fffff,	UBD|RD_s,		0,		I32R6	}, /* jalr */
 /* jr.hb is officially MIPS{32,64}R2, but it works on R1 as jr with
    the same hazard barrier effect.  */
 {"jr.hb",   "s",	0x00000408, 0xfc1fffff,	UBD|RD_s,		0,		I32	},
+{"jr.hb",   "s",	0x00000409, 0xfc1fffff,	UBD|RD_s,		0,		I32R6	}, /* jalr.hb */
 {"j",       "s",	0x00000008, 0xfc1fffff,	UBD|RD_s,		0,		I1	}, /* jr */
 /* SVR4 PIC code requires special handling for j, so it must be a
    macro.  */
diff --git a/disas/s390.c b/disas/s390.c
index 974460c81..c29bc4e69 100644
--- a/disas/s390.c
+++ b/disas/s390.c
@@ -613,7 +613,7 @@ static const struct s390_operand s390_operands[] =
       names of the instruction format that you can find in the principals
       of operation.
    2) the last part of the definition (y in INSTR_x_y) gives you an idea
-      which operands the binary represenation of the instruction has.
+      which operands the binary representation of the instruction has.
       The meanings of the letters in y are:
       a - access register
       c - control register
@@ -627,7 +627,7 @@ static const struct s390_operand s390_operands[] =
       m - mode field, 4 bit
       0 - operand skipped.
       The order of the letters reflects the layout of the format in
-      storage and not the order of the paramaters of the instructions.
+      storage and not the order of the parameters of the instructions.
       The use of the letters is not a 100% match with the PoP but it is
       quite close.
 
diff --git a/disas/sparc.c b/disas/sparc.c
index f4e356573..59a1e36cf 100644
--- a/disas/sparc.c
+++ b/disas/sparc.c
@@ -2622,8 +2622,7 @@ build_hash_table (const sparc_opcode **opcode_table,
 
   memset (hash_table, 0, HASH_SIZE * sizeof (hash_table[0]));
   memset (hash_count, 0, HASH_SIZE * sizeof (hash_count[0]));
-  if (hash_buf != NULL)
-    free (hash_buf);
+  free(hash_buf);
   hash_buf = malloc (sizeof (* hash_buf) * num_opcodes);
   for (i = num_opcodes - 1; i >= 0; --i)
     {
diff --git a/docs/bitmaps.md b/docs/bitmaps.md
index fa87f077f..a2e8d5116 100644
--- a/docs/bitmaps.md
+++ b/docs/bitmaps.md
@@ -19,12 +19,20 @@ which is included at the end of this document.
 * A dirty bitmap's name is unique to the node, but bitmaps attached to different
   nodes can share the same name.
 
+* Dirty bitmaps created for internal use by QEMU may be anonymous and have no
+  name, but any user-created bitmaps may not be. There can be any number of
+  anonymous bitmaps per node.
+
+* The name of a user-created bitmap must not be empty ("").
+
 ## Bitmap Modes
 
 * A Bitmap can be "frozen," which means that it is currently in-use by a backup
   operation and cannot be deleted, renamed, written to, reset,
   etc.
 
+* The normal operating mode for a bitmap is "active."
+
 ## Basic QMP Usage
 
 ### Supported Commands ###
@@ -97,11 +105,7 @@ which is included at the end of this document.
 }
 ```
 
-## Transactions (Not yet implemented)
-
-* Transactional commands are forthcoming in a future version,
-  and are not yet available for use. This section serves as
-  documentation of intent for their design and usage.
+## Transactions
 
 ### Justification
 
@@ -323,6 +327,155 @@ full backup as a backing image.
       "event": "BLOCK_JOB_COMPLETED" }
     ```
 
+### Partial Transactional Failures
+
+* Sometimes, a transaction will succeed in launching and return success,
+  but then later the backup jobs themselves may fail. It is possible that
+  a management application may have to deal with a partial backup failure
+  after a successful transaction.
+
+* If multiple backup jobs are specified in a single transaction, when one of
+  them fails, it will not interact with the other backup jobs in any way.
+
+* The job(s) that succeeded will clear the dirty bitmap associated with the
+  operation, but the job(s) that failed will not. It is not "safe" to delete
+  any incremental backups that were created successfully in this scenario,
+  even though others failed.
+
+#### Example
+
+* QMP example highlighting two backup jobs:
+
+    ```json
+    { "execute": "transaction",
+      "arguments": {
+        "actions": [
+          { "type": "drive-backup",
+            "data": { "device": "drive0", "bitmap": "bitmap0",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d0-incr-1.qcow2" } },
+          { "type": "drive-backup",
+            "data": { "device": "drive1", "bitmap": "bitmap1",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d1-incr-1.qcow2" } },
+        ]
+      }
+    }
+    ```
+
+* QMP example response, highlighting one success and one failure:
+    * Acknowledgement that the Transaction was accepted and jobs were launched:
+        ```json
+        { "return": {} }
+        ```
+
+    * Later, QEMU sends notice that the first job was completed:
+        ```json
+        { "timestamp": { "seconds": 1447192343, "microseconds": 615698 },
+          "data": { "device": "drive0", "type": "backup",
+                     "speed": 0, "len": 67108864, "offset": 67108864 },
+          "event": "BLOCK_JOB_COMPLETED"
+        }
+        ```
+
+    * Later yet, QEMU sends notice that the second job has failed:
+        ```json
+        { "timestamp": { "seconds": 1447192399, "microseconds": 683015 },
+          "data": { "device": "drive1", "action": "report",
+                    "operation": "read" },
+          "event": "BLOCK_JOB_ERROR" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447192399, "microseconds": 685853 },
+          "data": { "speed": 0, "offset": 0, "len": 67108864,
+                    "error": "Input/output error",
+                    "device": "drive1", "type": "backup" },
+          "event": "BLOCK_JOB_COMPLETED" }
+
+* In the above example, "d0-incr-1.qcow2" is valid and must be kept,
+  but "d1-incr-1.qcow2" is invalid and should be deleted. If a VM-wide
+  incremental backup of all drives at a point-in-time is to be made,
+  new backups for both drives will need to be made, taking into account
+  that a new incremental backup for drive0 needs to be based on top of
+  "d0-incr-1.qcow2."
+
+### Grouped Completion Mode
+
+* While jobs launched by transactions normally complete or fail on their own,
+  it is possible to instruct them to complete or fail together as a group.
+
+* QMP transactions take an optional properties structure that can affect
+  the semantics of the transaction.
+
+* The "completion-mode" transaction property can be either "individual"
+  which is the default, legacy behavior described above, or "grouped,"
+  a new behavior detailed below.
+
+* Delayed Completion: In grouped completion mode, no jobs will report
+  success until all jobs are ready to report success.
+
+* Grouped failure: If any job fails in grouped completion mode, all remaining
+  jobs will be cancelled. Any incremental backups will restore their dirty
+  bitmap objects as if no backup command was ever issued.
+
+    * Regardless of if QEMU reports a particular incremental backup job as
+      CANCELLED or as an ERROR, the in-memory bitmap will be restored.
+
+#### Example
+
+* Here's the same example scenario from above with the new property:
+
+    ```json
+    { "execute": "transaction",
+      "arguments": {
+        "actions": [
+          { "type": "drive-backup",
+            "data": { "device": "drive0", "bitmap": "bitmap0",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d0-incr-1.qcow2" } },
+          { "type": "drive-backup",
+            "data": { "device": "drive1", "bitmap": "bitmap1",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d1-incr-1.qcow2" } },
+        ],
+        "properties": {
+          "completion-mode": "grouped"
+        }
+      }
+    }
+    ```
+
+* QMP example response, highlighting a failure for drive2:
+    * Acknowledgement that the Transaction was accepted and jobs were launched:
+        ```json
+        { "return": {} }
+        ```
+
+    * Later, QEMU sends notice that the second job has errored out,
+      but that the first job was also cancelled:
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 632377 },
+          "data": { "device": "drive1", "action": "report",
+                    "operation": "read" },
+          "event": "BLOCK_JOB_ERROR" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 640074 },
+          "data": { "speed": 0, "offset": 0, "len": 67108864,
+                    "error": "Input/output error",
+                    "device": "drive1", "type": "backup" },
+          "event": "BLOCK_JOB_COMPLETED" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 640163 },
+          "data": { "device": "drive0", "type": "backup", "speed": 0,
+                    "len": 67108864, "offset": 16777216 },
+          "event": "BLOCK_JOB_CANCELLED" }
+        ```
+
 <!--
 The FreeBSD Documentation License
 
diff --git a/docs/build-system.txt b/docs/build-system.txt
new file mode 100644
index 000000000..5ddddeaaf
--- /dev/null
+++ b/docs/build-system.txt
@@ -0,0 +1,507 @@
+    The QEMU build system architecture
+    ==================================
+
+This document aims to help developers understand the architecture of the
+QEMU build system. As with projects using GNU autotools, the QEMU build
+system has two stages, first the developer runs the "configure" script
+to determine the local build environment characteristics, then they run
+"make" to build the project. There is about where the similarities with
+GNU autotools end, so try to forget what you know about them.
+
+
+Stage 1: configure
+==================
+
+The QEMU configure script is written directly in shell, and should be
+compatible with any POSIX shell, hence it uses #!/bin/sh. An important
+implication of this is that it is important to avoid using bash-isms on
+development platforms where bash is the primary host.
+
+In contrast to autoconf scripts, QEMU's configure is expected to be
+silent while it is checking for features. It will only display output
+when an error occurs, or to show the final feature enablement summary
+on completion.
+
+Adding new checks to the configure script usually comprises the
+following tasks:
+
+ - Initialize one or more variables with the default feature state.
+
+   Ideally features should auto-detect whether they are present,
+   so try to avoid hardcoding the initial state to either enabled
+   or disabled, as that forces the user to pass a --enable-XXX
+   / --disable-XXX flag on every invocation of configure.
+
+ - Add support to the command line arg parser to handle any new
+   --enable-XXX / --disable-XXX flags required by the feature XXX.
+
+ - Add information to the help output message to report on the new
+   feature flag.
+
+ - Add code to perform the actual feature check. As noted above, try to
+   be fully dynamic in checking enablement/disablement.
+
+ - Add code to print out the feature status in the configure summary
+   upon completion.
+
+ - Add any new makefile variables to $config_host_mak on completion.
+
+
+Taking (a simplified version of) the probe for gnutls from configure,
+we have the following pieces:
+
+  # Initial variable state
+  gnutls=""
+
+  ..snip..
+
+  # Configure flag processing
+  --disable-gnutls) gnutls="no"
+  ;;
+  --enable-gnutls) gnutls="yes"
+  ;;
+
+  ..snip..
+
+  # Help output feature message
+  gnutls          GNUTLS cryptography support
+
+  ..snip..
+
+  # Test for gnutls
+  if test "$gnutls" != "no"; then
+     if ! $pkg_config --exists "gnutls"; then
+        gnutls_cflags=`$pkg_config --cflags gnutls`
+        gnutls_libs=`$pkg_config --libs gnutls`
+        libs_softmmu="$gnutls_libs $libs_softmmu"
+        libs_tools="$gnutls_libs $libs_tools"
+        QEMU_CFLAGS="$QEMU_CFLAGS $gnutls_cflags"
+        gnutls="yes"
+     elif test "$gnutls" = "yes"; then
+        feature_not_found "gnutls" "Install gnutls devel"
+     else
+        gnutls="no"
+     fi
+  fi
+
+  ..snip..
+
+  # Completion feature summary
+  echo "GNUTLS support    $gnutls"
+
+  ..snip..
+
+  # Define make variables
+  if test "$gnutls" = "yes" ; then
+     echo "CONFIG_GNUTLS=y" >> $config_host_mak
+  fi
+
+
+Helper functions
+----------------
+
+The configure script provides a variety of helper functions to assist
+developers in checking for system features:
+
+ - do_cc $ARGS...
+
+   Attempt to run the system C compiler passing it $ARGS...
+
+ - do_cxx $ARGS...
+
+   Attempt to run the system C++ compiler passing it $ARGS...
+
+ - compile_object $CFLAGS
+
+   Attempt to compile a test program with the system C compiler using
+   $CFLAGS. The test program must have been previously written to a file
+   called $TMPC.
+
+ - compile_prog $CFLAGS $LDFLAGS
+
+   Attempt to compile a test program with the system C compiler using
+   $CFLAGS and link it with the system linker using $LDFLAGS. The test
+   program must have been previously written to a file called $TMPC.
+
+ - has $COMMAND
+
+   Determine if $COMMAND exists in the current environment, either as a
+   shell builtin, or executable binary, returning 0 on success.
+
+ - path_of $COMMAND
+
+   Return the fully qualified path of $COMMAND, printing it to stdout,
+   and returning 0 on success.
+
+ - check_define $NAME
+
+   Determine if the macro $NAME is defined by the system C compiler
+
+ - check_include $NAME
+
+   Determine if the include $NAME file is available to the system C
+   compiler
+
+ - write_c_skeleton
+
+   Write a minimal C program main() function to the temporary file
+   indicated by $TMPC
+
+ - feature_not_found $NAME $REMEDY
+
+   Print a message to stderr that the feature $NAME was not available
+   on the system, suggesting the user try $REMEDY to address the
+   problem.
+
+ - error_exit $MESSAGE $MORE...
+
+   Print $MESSAGE to stderr, followed by $MORE... and then exit from the
+   configure script with non-zero status
+
+ - query_pkg_config $ARGS...
+
+   Run pkg-config passing it $ARGS. If QEMU is doing a static build,
+   then --static will be automatically added to $ARGS
+
+
+Stage 2: makefiles
+==================
+
+The use of GNU make is required with the QEMU build system.
+
+Although the source code is spread across multiple subdirectories, the
+build system should be considered largely non-recursive in nature, in
+contrast to common practices seen with automake. There is some recursive
+invocation of make, but this is related to the things being built,
+rather than the source directory structure.
+
+QEMU currently supports both VPATH and non-VPATH builds, so there are
+three general ways to invoke configure & perform a build.
+
+ - VPATH, build artifacts outside of QEMU source tree entirely
+
+     cd ../
+     mkdir build
+     cd build
+     ../qemu/configure
+     make
+
+ - VPATH, build artifacts in a subdir of QEMU source tree
+
+     mkdir build
+     cd build
+     ../configure
+     make
+
+ - non-VPATH, build artifacts everywhere
+
+     ./configure
+     make
+
+The QEMU maintainers generally recommend that a VPATH build is used by
+developers. Patches to QEMU are expected to ensure VPATH build still
+works.
+
+
+Module structure
+----------------
+
+There are a number of key outputs of the QEMU build system:
+
+ - Tools - qemu-img, qemu-nbd, qga (guest agent), etc
+ - System emulators - qemu-system-$ARCH
+ - Userspace emulators - qemu-$ARCH
+ - Unit tests
+
+The source code is highly modularized, split across many files to
+facilitate building of all of these components with as little duplicated
+compilation as possible. There can be considered to be two distinct
+groups of files, those which are independent of the QEMU emulation
+target and those which are dependent on the QEMU emulation target.
+
+In the target-independent set lives various general purpose helper code,
+such as error handling infrastructure, standard data structures,
+platform portability wrapper functions, etc. This code can be compiled
+once only and the .o files linked into all output binaries.
+
+In the target-dependent set lives CPU emulation, device emulation and
+much glue code. This sometimes also has to be compiled multiple times,
+once for each target being built.
+
+The utility code that is used by all binaries is built into a
+static archive called libqemuutil.a, which is then linked to all the
+binaries. In order to provide hooks that are only needed by some of the
+binaries, code in libqemuutil.a may depend on other functions that are
+not fully implemented by all QEMU binaries. To deal with this there is a
+second library called libqemustub.a which provides dummy stubs for all
+these functions. These will get lazy linked into the binary if the real
+implementation is not present. In this way, the libqemustub.a static
+library can be thought of as a portable implementation of the weak
+symbols concept. All binaries should link to both libqemuutil.a and
+libqemustub.a. e.g.
+
+ qemu-img$(EXESUF): qemu-img.o ..snip.. libqemuutil.a libqemustub.a
+
+
+Windows platform portability
+----------------------------
+
+On Windows, all binaries have the suffix '.exe', so all Makefile rules
+which create binaries must include the $(EXESUF) variable on the binary
+name. e.g.
+
+ qemu-img$(EXESUF): qemu-img.o ..snip..
+
+This expands to '.exe' on Windows, or '' on other platforms.
+
+A further complication for the system emulator binaries is that
+two separate binaries need to be generated.
+
+The main binary (e.g. qemu-system-x86_64.exe) is linked against the
+Windows console runtime subsystem. These are expected to be run from a
+command prompt window, and so will print stderr to the console that
+launched them.
+
+The second binary generated has a 'w' on the end of its name (e.g.
+qemu-system-x86_64w.exe) and is linked against the Windows graphical
+runtime subsystem. These are expected to be run directly from the
+desktop and will open up a dedicated console window for stderr output.
+
+The Makefile.target will generate the binary for the graphical subsystem
+first, and then use objcopy to relink it against the console subsystem
+to generate the second binary.
+
+
+Object variable naming
+----------------------
+
+The QEMU convention is to define variables to list different groups of
+object files. These are named with the convention $PREFIX-obj-y. For
+example the libqemuutil.a file will be linked with all objects listed
+in a variable 'util-obj-y'. So, for example, util/Makefile.obj will
+contain a set of definitions looking like
+
+  util-obj-y += bitmap.o bitops.o hbitmap.o
+  util-obj-y += fifo8.o
+  util-obj-y += acl.o
+  util-obj-y += error.o qemu-error.o
+
+When there is an object file which needs to be conditionally built based
+on some characteristic of the host system, the configure script will
+define a variable for the conditional. For example, on Windows it will
+define $(CONFIG_POSIX) with a value of 'n' and $(CONFIG_WIN32) with a
+value of 'y'. It is now possible to use the config variables when
+listing object files. For example,
+
+  util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o
+  util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o
+
+On Windows this expands to
+
+  util-obj-y += oslib-win32.o qemu-thread-win32.o
+  util-obj-n += oslib-posix.o qemu-thread-posix.o
+
+Since libqemutil.a links in $(util-obj-y), the POSIX specific files
+listed against $(util-obj-n) are ignored on the Windows platform builds.
+
+
+CFLAGS / LDFLAGS / LIBS handling
+--------------------------------
+
+There are many different binaries being built with differing purposes,
+and some of them might even be 3rd party libraries pulled in via git
+submodules. As such the use of the global CFLAGS variable is generally
+avoided in QEMU, since it would apply to too many build targets.
+
+Flags that are needed by any QEMU code (i.e. everything *except* GIT
+submodule projects) are put in $(QEMU_CFLAGS) variable. For linker
+flags the $(LIBS) variable is sometimes used, but a couple of more
+targeted variables are preferred. $(libs_softmmu) is used for
+libraries that must be linked to system emulator targets, $(LIBS_TOOLS)
+is used for tools like qemu-img, qemu-nbd, etc and $(LIBS_QGA) is used
+for the QEMU guest agent. There is currently no specific variable for
+the userspace emulator targets as the global $(LIBS), or more targeted
+variables shown below, are sufficient.
+
+In addition to these variables, it is possible to provide cflags and
+libs against individual source code files, by defining variables of the
+form $FILENAME-cflags and $FILENAME-libs. For example, the curl block
+driver needs to link to the libcurl library, so block/Makefile defines
+some variables:
+
+  curl.o-cflags      := $(CURL_CFLAGS)
+  curl.o-libs        := $(CURL_LIBS)
+
+The scope is a little different between the two variables. The libs get
+used when linking any target binary that includes the curl.o object
+file, while the cflags get used when compiling the curl.c file only.
+
+
+Statically defined files
+------------------------
+
+The following key files are statically defined in the source tree, with
+the rules needed to build QEMU. Their behaviour is influenced by a
+number of dynamically created files listed later.
+
+- Makefile
+
+The main entry point used when invoking make to build all the components
+of QEMU. The default 'all' target will naturally result in the build of
+every component. The various tools and helper binaries are built
+directly via a non-recursive set of rules.
+
+Each system/userspace emulation target needs to have a slightly
+different set of make rules / variables. Thus, make will be recursively
+invoked for each of the emulation targets.
+
+The recursive invocation will end up processing the toplevel
+Makefile.target file (more on that later).
+
+
+- */Makefile.objs
+
+Since the source code is spread across multiple directories, the rules
+for each file are similarly modularized. Thus each subdirectory
+containing .c files will usually also contain a Makefile.objs file.
+These files are not directly invoked by a recursive make, but instead
+they are imported by the top level Makefile and/or Makefile.target
+
+Each Makefile.objs usually just declares a set of variables listing the
+.o files that need building from the source files in the directory. They
+will also define any custom linker or compiler flags. For example in
+block/Makefile.objs
+
+  block-obj-$(CONFIG_LIBISCSI) += iscsi.o
+  block-obj-$(CONFIG_CURL) += curl.o
+
+  ..snip...
+
+  iscsi.o-cflags     := $(LIBISCSI_CFLAGS)
+  iscsi.o-libs       := $(LIBISCSI_LIBS)
+  curl.o-cflags      := $(CURL_CFLAGS)
+  curl.o-libs        := $(CURL_LIBS)
+
+If there are any rules defined in the Makefile.objs file, they should
+all use $(obj) as a prefix to the target, e.g.
+
+  $(obj)/generated-tcg-tracers.h: $(obj)/generated-tcg-tracers.h-timestamp
+
+
+- Makefile.target
+
+This file provides the entry point used to build each individual system
+or userspace emulator target. Each enabled target has its own
+subdirectory. For example if configure is run with the argument
+'--target-list=x86_64-softmmu', then a sub-directory 'x86_64-softmu'
+will be created, containing a 'Makefile' which symlinks back to
+Makefile.target
+
+So when the recursive '$(MAKE) -C x86_64-softmmu' is invoked, it ends up
+using Makefile.target for the build rules.
+
+
+- rules.mak
+
+This file provides the generic helper rules for invoking build tools, in
+particular the compiler and linker. This also contains the magic (hairy)
+'unnest-vars' function which is used to merge the variable definitions
+from all Makefile.objs in the source tree down into the main Makefile
+context.
+
+
+- default-configs/*.mak
+
+The files under default-configs/ control what emulated hardware is built
+into each QEMU system and userspace emulator targets. They merely
+contain a long list of config variable definitions. For example,
+default-configs/x86_64-softmmu.mak has:
+
+  include pci.mak
+  include sound.mak
+  include usb.mak
+  CONFIG_QXL=$(CONFIG_SPICE)
+  CONFIG_VGA_ISA=y
+  CONFIG_VGA_CIRRUS=y
+  CONFIG_VMWARE_VGA=y
+  CONFIG_VIRTIO_VGA=y
+  ...snip...
+
+These files rarely need changing unless new devices / hardware need to
+be enabled for a particular system/userspace emulation target
+
+
+- tests/Makefile
+
+Rules for building the unit tests. This file is included directly by the
+top level Makefile, so anything defined in this file will influence the
+entire build system. Care needs to be taken when writing rules for tests
+to ensure they only apply to the unit test execution / build.
+
+
+- po/Makefile
+
+Rules for building and installing the binary message catalogs from the
+text .po file sources. This almost never needs changing for any reason.
+
+
+Dynamically created files
+-------------------------
+
+The following files are generated dynamically by configure in order to
+control the behaviour of the statically defined makefiles. This avoids
+the need for QEMU makefiles to go through any pre-processing as seen
+with autotools, where Makefile.am generates Makefile.in which generates
+Makefile.
+
+
+- config-host.mak
+
+When configure has determined the characteristics of the build host it
+will write a long list of variables to config-host.mak file. This
+provides the various install directories, compiler / linker flags and a
+variety of CONFIG_* variables related to optionally enabled features.
+This is imported by the top level Makefile in order to tailor the build
+output.
+
+The variables defined here are those which are applicable to all QEMU
+build outputs. Variables which are potentially different for each
+emulator target are defined by the next file...
+
+It is also used as a dependency checking mechanism. If make sees that
+the modification timestamp on configure is newer than that on
+config-host.mak, then configure will be re-run.
+
+
+- config-host.h
+
+The config-host.h file is used by source code to determine what features
+are enabled. It is generated from the contents of config-host.mak using
+the scripts/create_config program. This extracts all the CONFIG_* variables,
+most of the HOST_* variables and a few other misc variables from
+config-host.mak, formatting them as C preprocessor macros.
+
+
+- $TARGET-NAME/config-target.mak
+
+TARGET-NAME is the name of a system or userspace emulator, for example,
+x86_64-softmmu denotes the system emulator for the x86_64 architecture.
+This file contains the variables which need to vary on a per-target
+basis. For example, it will indicate whether KVM or Xen are enabled for
+the target and any other potential custom libraries needed for linking
+the target.
+
+
+- $TARGET-NAME/config-devices.mak
+
+TARGET-NAME is again the name of a system or userspace emulator. The
+config-devices.mak file is automatically generated by make using the
+scripts/make_device_config.sh program, feeding it the
+default-configs/$TARGET-NAME file as input.
+
+
+- $TARGET-NAME/Makefile
+
+This is the entrypoint used when make recurses to build a single system
+or userspace emulator target. It is merely a symlink back to the
+Makefile.target in the top level.
diff --git a/docs/libcacard.txt b/docs/libcacard.txt
deleted file mode 100644
index 8db421d3a..000000000
--- a/docs/libcacard.txt
+++ /dev/null
@@ -1,483 +0,0 @@
-This file documents the CAC (Common Access Card) library in the libcacard
-subdirectory.
-
-Virtual Smart Card Emulator
-
-This emulator is designed to provide emulation of actual smart cards to a
-virtual card reader running in a guest virtual machine. The emulated smart
-cards can be representations of real smart cards, where the necessary functions
-such as signing, card removal/insertion, etc. are mapped to real, physical
-cards which are shared with the client machine the emulator is running on, or
-the cards could be pure software constructs.
-
-The emulator is structured to allow multiple replaceable or additional pieces,
-so it can be easily modified for future requirements. The primary envisioned
-modifications are:
-
-1) The socket connection to the virtual card reader (presumably a CCID reader,
-but other ISO-7816 compatible readers could be used). The code that handles
-this is in vscclient.c.
-
-2) The virtual card low level emulation. This is currently supplied by using
-NSS. This emulation could be replaced by implementations based on other
-security libraries, including but not limitted to openssl+pkcs#11 library,
-raw pkcs#11, Microsoft CAPI, direct opensc calls, etc. The code that handles
-this is in vcard_emul_nss.c.
-
-3) Emulation for new types of cards. The current implementation emulates the
-original DoD CAC standard with separate pki containers. This emulator lives in
-cac.c. More than one card type emulator could be included. Other cards could
-be emulated as well, including PIV, newer versions of CAC, PKCS #15, etc.
-
---------------------
-Replacing the Socket Based Virtual Reader Interface.
-
-The current implementation contains a replaceable module vscclient.c. The
-current vscclient.c implements a sockets interface to the virtual ccid reader
-on the guest. CCID commands that are pertinent to emulation are passed
-across the socket, and their responses are passed back along that same socket.
-The protocol that vscclient uses is defined in vscard_common.h and connects
-to a qemu ccid usb device. Since this socket runs as a client, vscclient.c
-implements a program with a main entry. It also handles argument parsing for
-the emulator.
-
-An application that wants to use the virtual reader can replace vscclient.c
-with its own implementation that connects to its own CCID reader.  The calls
-that the CCID reader can call are:
-
-      VReaderList * vreader_get_reader_list();
-
-  This function returns a list of virtual readers.  These readers may map to
-  physical devices, or simulated devices depending on vcard the back end. Each
-  reader in the list should represent a reader to the virtual machine. Virtual
-  USB address mapping is left to the CCID reader front end. This call can be
-  made any time to get an updated list. The returned list is a copy of the
-  internal list that can be referenced by the caller without locking. This copy
-  must be freed by the caller with vreader_list_delete when it is no longer
-  needed.
-
-      VReaderListEntry *vreader_list_get_first(VReaderList *);
-
-  This function gets the first entry on the reader list. Along with
-  vreader_list_get_next(), vreader_list_get_first() can be used to walk the
-  reader list returned from vreader_get_reader_list(). VReaderListEntries are
-  part of the list themselves and do not need to be freed separately from the
-  list. If there are no entries on the list, it will return NULL.
-
-      VReaderListEntry *vreader_list_get_next(VReaderListEntry *);
-
-  This function gets the next entry in the list. If there are no more entries
-  it will return NULL.
-
-      VReader * vreader_list_get_reader(VReaderListEntry *)
-
-  This function returns the reader stored in the reader List entry. Caller gets
-  a new reference to a reader. The caller must free its reference when it is
-  finished with vreader_free().
-
-      void vreader_free(VReader *reader);
-
-   This function frees a reference to a reader. Readers are reference counted
-   and are automatically deleted when the last reference is freed.
-
-      void vreader_list_delete(VReaderList *list);
-
-   This function frees the list, all the elements on the list, and all the
-   reader references held by the list.
-
-      VReaderStatus vreader_power_on(VReader *reader, char *atr, int *len);
-
-  This function simulates a card power on. A virtual card does not care about
-  the actual voltage and other physical parameters, but it does care that the
-  card is actually on or off. Cycling the card causes the card to reset. If
-  the caller provides enough space, vreader_power_on will return the ATR of
-  the virtual card. The amount of space provided in atr should be indicated
-  in *len. The function modifies *len to be the actual length of of the
-  returned ATR.
-
-      VReaderStatus vreader_power_off(VReader *reader);
-
-  This function simulates a power off of a virtual card.
-
-      VReaderStatus vreader_xfer_bytes(VReader *reader, unsigne char *send_buf,
-                                       int send_buf_len,
-                                       unsigned char *receive_buf,
-                                       int receive_buf_len);
-
-  This function sends a raw apdu to a card and returns the card's response.
-  The CCID front end should return the response back. Most of the emulation
-  is driven from these APDUs.
-
-      VReaderStatus vreader_card_is_present(VReader *reader);
-
-  This function returns whether or not the reader has a card inserted. The
-  vreader_power_on, vreader_power_off, and vreader_xfer_bytes will return
-  VREADER_NO_CARD.
-
-       const char *vreader_get_name(VReader *reader);
-
-  This function returns the name of the reader. The name comes from the card
-  emulator level and is usually related to the name of the physical reader.
-
-       VReaderID vreader_get_id(VReader *reader);
-
-  This function returns the id of a reader. All readers start out with an id
-  of -1. The application can set the id with vreader_set_id.
-
-       VReaderStatus vreader_get_id(VReader *reader, VReaderID id);
-
-  This function sets the reader id. The application is responsible for making
-  sure that the id is unique for all readers it is actively using.
-
-       VReader *vreader_find_reader_by_id(VReaderID id);
-
-  This function returns the reader which matches the id. If two readers match,
-  only one is returned. The function returns NULL if the id is -1.
-
-       Event *vevent_wait_next_vevent();
-
-  This function blocks waiting for reader and card insertion events. There
-  will be one event for each card insertion, each card removal, each reader
-  insertion and each reader removal. At start up, events are created for all
-  the initial readers found, as well as all the cards that are inserted.
-
-       Event *vevent_get_next_vevent();
-
-  This function returns a pending event if it exists, otherwise it returns
-  NULL. It does not block.
-
-----------------
-Card Type Emulator: Adding a New Virtual Card Type
-
-The ISO 7816 card spec describes 2 types of cards:
- 1) File system cards, where the smartcard is managed by reading and writing
-data to files in a file system. There is currently only boiler plate
-implemented for file system cards.
- 2) VM cards, where the card has loadable applets which perform the card
-functions. The current implementation supports VM cards.
-
-In the case of VM cards, the difference between various types of cards is
-really what applets have been installed in that card. This structure is
-mirrored in card type emulators. The 7816 emulator already handles the basic
-ISO 7186 commands. Card type emulators simply need to add the virtual applets
-which emulate the real card applets. Card type emulators have exactly one
-public entry point:
-
-       VCARDStatus xxx_card_init(VCard *card, const char *flags,
-                               const unsigned char *cert[],
-                               int cert_len[],
-                               VCardKey *key[],
-                               int cert_count);
-
-  The parameters for this are:
-  card       - the virtual card structure which will represent this card.
-  flags      - option flags that may be specific to this card type.
-  cert       - array of binary certificates.
-  cert_len   - array of lengths of each of the certificates specified in cert.
-  key        - array of opaque key structures representing the private keys on
-               the card.
-  cert_count - number of entries in cert, cert_len, and key arrays.
-
-  Any cert, cert_len, or key with the same index are matching sets. That is
-  cert[0] is cert_len[0] long and has the corresponding private key of key[0].
-
-The card type emulator is expected to own the VCardKeys, but it should copy
-any raw cert data it wants to save. It can create new applets and add them to
-the card using the following functions:
-
-       VCardApplet *vcard_new_applet(VCardProcessAPDU apdu_func,
-                                     VCardResetApplet reset_func,
-                                     const unsigned char *aid,
-                                     int aid_len);
-
-  This function creates a new applet. Applet structures store the following
-  information:
-     1) the AID of the applet (set by aid and aid_len).
-     2) a function to handle APDUs for this applet. (set by apdu_func, more on
-        this below).
-     3) a function to reset the applet state when the applet is selected.
-        (set by reset_func, more on this below).
-     3) applet private data, a data pointer used by the card type emulator to
-        store any data or state it needs to complete requests. (set by a
-        separate call).
-     4) applet private data free, a function used to free the applet private
-        data when the applet itself is destroyed.
-  The created applet can be added to the card with vcard_add_applet below.
-
-        void vcard_set_applet_private(VCardApplet *applet,
-                                      VCardAppletPrivate *private,
-                                      VCardAppletPrivateFree private_free);
-  This function sets the private data and the corresponding free function.
-  VCardAppletPrivate is an opaque data structure to the rest of the emulator.
-  The card type emulator can define it any way it wants by defining
-  struct VCardAppletPrivateStruct {};. If there is already a private data
-  structure on the applet, the old one is freed before the new one is set up.
-  passing two NULL clear any existing private data.
-
-         VCardStatus vcard_add_applet(VCard *card, VCardApplet *applet);
-
-  Add an applet onto the list of applets attached to the card. Once an applet
-  has been added, it can be selected by its AID, and then commands will be
-  routed to it VCardProcessAPDU function. This function adopts the applet that
-  is passed into it. Note: 2 applets with the same AID should not be added to
-  the same card. It is permissible to add more than one applet. Multiple applets
-  may have the same VCardPRocessAPDU entry point.
-
-The certs and keys should be attached to private data associated with one or
-more appropriate applets for that card. Control will come to the card type
-emulators once one of its applets are selected through the VCardProcessAPDU
-function it specified when it created the applet.
-
-The signature of VCardResetApplet is:
-        VCardStatus (*VCardResetApplet) (VCard *card, int channel);
-  This function will reset the any internal applet state that needs to be
-  cleared after a select applet call. It should return VCARD_DONE;
-
-The signature of VCardProcessAPDU is:
-        VCardStatus (*VCardProcessAPDU)(VCard *card, VCardAPDU *apdu,
-                                         VCardResponse **response);
-  This function examines the APDU and determines whether it should process
-  the apdu directly, reject the apdu as invalid, or pass the apdu on to
-  the basic 7816 emulator for processing.
-      If the 7816 emulator should process the apdu, then the VCardProcessAPDU
-  should return VCARD_NEXT.
-      If there is an error, then VCardProcessAPDU should return an error
-  response using vcard_make_response and the appropriate 7816 error code
-  (see card_7816t.h) or vcard_make_response with a card type specific error
-  code. It should then return VCARD_DONE.
-      If the apdu can be processed correctly, VCardProcessAPDU should do so,
-  set the response value appropriately for that APDU, and return VCARD_DONE.
-  VCardProcessAPDU should always set the response if it returns VCARD_DONE.
-  It should always either return VCARD_DONE or VCARD_NEXT.
-
-Parsing the APDU --
-
-Prior to processing calling the card type emulator's VCardProcessAPDU function, the emulator has already decoded the APDU header and set several fields:
-
-   apdu->a_data - The raw apdu data bytes.
-   apdu->a_len  - The len of the raw apdu data.
-   apdu->a_body - The start of any post header parameter data.
-   apdu->a_Lc   - The parameter length value.
-   apdu->a_Le   - The expected length of any returned data.
-   apdu->a_cla  - The raw apdu class.
-   apdu->a_channel - The channel (decoded from the class).
-   apdu->a_secure_messaging_type - The decoded secure messaging type
-                                   (from class).
-   apdu->a_type - The decode class type.
-   apdu->a_gen_type - the generic class type (7816, PROPRIETARY, RFU, PTS).
-   apdu->a_ins  - The instruction byte.
-   apdu->a_p1   - Parameter 1.
-   apdu->a_p2   - Parameter 2.
-
-Creating a Response --
-
-The expected result of any APDU call is a response. The card type emulator must
-set *response with an appropriate VCardResponse value if it returns VCARD_DONE.
-Responses could be as simple as returning a 2 byte status word response, to as
-complex as returning a block of data along with a 2 byte response. Which is
-returned will depend on the semantics of the APDU. The following functions will
-create card responses.
-
-        VCardResponse *vcard_make_response(VCard7816Status status);
-
-    This is the most basic function to get a response. This function will
-    return a response the consists solely one 2 byte status code. If that status
-    code is defined in card_7816t.h, then this function is guaranteed to
-    return a response with that status. If a cart type specific status code
-    is passed and vcard_make_response fails to allocate the appropriate memory
-    for that response, then vcard_make_response will return a VCardResponse
-    of VCARD7816_STATUS_EXC_ERROR_MEMORY. In any case, this function is
-    guaranteed to return a valid VCardResponse.
-
-        VCardResponse *vcard_response_new(unsigned char *buf, int len,
-                                          VCard7816Status status);
-
-    This function is similar to vcard_make_response except it includes some
-    returned data with the response. It could also fail to allocate enough
-    memory, in which case it will return NULL.
-
-        VCardResponse *vcard_response_new_status_bytes(unsigned char sw1,
-                                                       unsigned char sw2);
-
-    Sometimes in 7816 the response bytes are treated as two separate bytes with
-    split meanings. This function allows you to create a response based on
-    two separate bytes. This function could fail, in which case it will return
-    NULL.
-
-       VCardResponse *vcard_response_new_bytes(unsigned char *buf, int len,
-                                               unsigned char sw1,
-                                               unsigned char sw2);
-
-    This function is the same as vcard_response_new except you may specify
-    the status as two separate bytes like vcard_response_new_status_bytes.
-
-
-Implementing functionality ---
-
-The following helper functions access information about the current card
-and applet.
-
-        VCARDAppletPrivate *vcard_get_current_applet_private(VCard *card,
-                                                             int channel);
-
-    This function returns any private data set by the card type emulator on
-    the currently selected applet. The card type emulator keeps track of the
-    current applet state in this data structure. Any certs and keys associated
-    with a particular applet is also stored here.
-
-        int vcard_emul_get_login_count(VCard *card);
-
-    This function returns the the number of remaining login attempts for this
-    card. If the card emulator does not know, or the card does not have a
-    way of giving this information, this function returns -1.
-
-
-         VCard7816Status vcard_emul_login(VCard *card, unsigned char *pin,
-                                          int pin_len);
-
-    This function logs into the card and returns the standard 7816 status
-    word depending on the success or failure of the call.
-
-         void vcard_emul_delete_key(VCardKey *key);
-
-     This function frees the VCardKey passed in to xxxx_card_init. The card
-     type emulator is responsible for freeing this key when it no longer needs
-     it.
-
-         VCard7816Status vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                                           unsigned char *buffer,
-                                           int buffer_size);
-
-     This function does a raw rsa op on the buffer with the given key.
-
-The sample card type emulator is found in cac.c. It implements the cac specific
-applets.  Only those applets needed by the coolkey pkcs#11 driver on the guest
-have been implemented. To support the full range CAC middleware, a complete CAC
-card according to the CAC specs should be implemented here.
-
-------------------------------
-Virtual Card Emulator
-
-This code accesses both real smart cards and simulated smart cards through
-services provided on the client. The current implementation uses NSS, which
-already knows how to talk to various PKCS #11 modules on the client, and is
-portable to most operating systems. A particular emulator can have only one
-virtual card implementation at a time.
-
-The virtual card emulator consists of a series of virtual card services. In
-addition to the services describe above (services starting with
-vcard_emul_xxxx), the virtual card emulator also provides the following
-functions:
-
-    VCardEmulError vcard_emul_init(cont VCardEmulOptions *options);
-
-  The options structure is built by another function in the virtual card
-  interface where a string of virtual card emulator specific strings are
-  mapped to the options. The actual structure is defined by the virtual card
-  emulator and is used to determine the configuration of soft cards, or to
-  determine which physical cards to present to the guest.
-
-  The vcard_emul_init function will build up sets of readers, create any
-  threads that are needed to watch for changes in the reader state. If readers
-  have cards present in them, they are also initialized.
-
-  Readers are created with the function.
-
-          VReader *vreader_new(VReaderEmul *reader_emul,
-                               VReaderEmulFree reader_emul_free);
-
-      The freeFunc is used to free the VReaderEmul * when the reader is
-      destroyed.  The VReaderEmul structure is an opaque structure to the
-      rest of the code, but defined by the virtual card emulator, which can
-      use it to store any reader specific state.
-
-  Once the reader has been created, it can be added to the front end with the
-  call:
-
-           VReaderStatus vreader_add_reader(VReader *reader);
-
-      This function will automatically generate the appropriate new reader
-      events and add the reader to the list.
-
-  To create a new card, the virtual card emulator will call a similar
-  function.
-
-           VCard *vcard_new(VCardEmul *card_emul,
-                            VCardEmulFree card_emul_free);
-
-      Like vreader_new, this function takes a virtual card emulator specific
-      structure which it uses to keep track of the card state.
-
-  Once the card is created, it is attached to a card type emulator with the
-  following function:
-
-            VCardStatus vcard_init(VCard *vcard, VCardEmulType type,
-                                   const char *flags,
-                                   unsigned char *const *certs,
-                                   int *cert_len,
-                                   VCardKey *key[],
-                                   int cert_count);
-
-      The vcard is the value returned from vcard_new. The type is the
-      card type emulator that this card should presented to the guest as.
-      The flags are card type emulator specific options. The certs,
-      cert_len, and keys are all arrays of length cert_count. These are the
-      the same of the parameters xxxx_card_init() accepts.
-
-   Finally the card is associated with its reader by the call:
-
-            VReaderStatus vreader_insert_card(VReader *vreader, VCard *vcard);
-
-      This function, like vreader_add_reader, will take care of any event
-      notification for the card insert.
-
-
-    VCardEmulError vcard_emul_force_card_remove(VReader *vreader);
-
-  Force a card that is present to appear to be removed to the guest, even if
-  that card is a physical card and is present.
-
-
-    VCardEmulError vcard_emul_force_card_insert(VReader *reader);
-
-  Force a card that has been removed by vcard_emul_force_card_remove to be
-  reinserted from the point of view of the guest. This will only work if the
-  card is physically present (which is always true fro a soft card).
-
-     void vcard_emul_get_atr(Vcard *card, unsigned char *atr, int *atr_len);
-
-  Return the virtual ATR for the card. By convention this should be the value
-  VCARD_ATR_PREFIX(size) followed by several ascii bytes related to this
-  particular emulator. For instance the NSS emulator returns
-  {VCARD_ATR_PREFIX(3), 'N', 'S', 'S' }. Do ot return more data then *atr_len;
-
-     void vcard_emul_reset(VCard *card, VCardPower power)
-
-   Set the state of 'card' to the current power level and reset its internal
-   state (logout, etc).
-
--------------------------------------------------------
-List of files and their function:
-README - This file
-card_7816.c - emulate basic 7816 functionality. Parse APDUs.
-card_7816.h - apdu and response services definitions.
-card_7816t.h - 7816 specific structures, types and definitions.
-event.c - event handling code.
-event.h - event handling services definitions.
-eventt.h - event handling structures and types
-vcard.c - handle common virtual card services like creation, destruction, and
-          applet management.
-vcard.h - common virtual card services function definitions.
-vcardt.h - comon virtual card types
-vreader.c - common virtual reader services.
-vreader.h - common virtual reader services definitions.
-vreadert.h - comon virtual reader types.
-vcard_emul_type.c - manage the card type emulators.
-vcard_emul_type.h - definitions for card type emulators.
-cac.c - card type emulator for CAC cards
-vcard_emul.h - virtual card emulator service definitions.
-vcard_emul_nss.c - virtual card emulator implementation for nss.
-vscclient.c - socket connection to guest qemu usb driver.
-vscard_common.h - common header with the guest qemu usb driver.
-mutex.h - header file for machine independent mutexes.
-link_test.c - static test to make sure all the symbols are properly defined.
diff --git a/docs/migration.txt b/docs/migration.txt
index f6df4beb2..fda8d61d6 100644
--- a/docs/migration.txt
+++ b/docs/migration.txt
@@ -291,3 +291,194 @@ save/send this state when we are in the middle of a pio operation
 (that is what ide_drive_pio_state_needed() checks).  If DRQ_STAT is
 not enabled, the values on that fields are garbage and don't need to
 be sent.
+
+= Return path =
+
+In most migration scenarios there is only a single data path that runs
+from the source VM to the destination, typically along a single fd (although
+possibly with another fd or similar for some fast way of throwing pages across).
+
+However, some uses need two way communication; in particular the Postcopy
+destination needs to be able to request pages on demand from the source.
+
+For these scenarios there is a 'return path' from the destination to the source;
+qemu_file_get_return_path(QEMUFile* fwdpath) gives the QEMUFile* for the return
+path.
+
+  Source side
+     Forward path - written by migration thread
+     Return path  - opened by main thread, read by return-path thread
+
+  Destination side
+     Forward path - read by main thread
+     Return path  - opened by main thread, written by main thread AND postcopy
+                    thread (protected by rp_mutex)
+
+= Postcopy =
+'Postcopy' migration is a way to deal with migrations that refuse to converge
+(or take too long to converge) its plus side is that there is an upper bound on
+the amount of migration traffic and time it takes, the down side is that during
+the postcopy phase, a failure of *either* side or the network connection causes
+the guest to be lost.
+
+In postcopy the destination CPUs are started before all the memory has been
+transferred, and accesses to pages that are yet to be transferred cause
+a fault that's translated by QEMU into a request to the source QEMU.
+
+Postcopy can be combined with precopy (i.e. normal migration) so that if precopy
+doesn't finish in a given time the switch is made to postcopy.
+
+=== Enabling postcopy ===
+
+To enable postcopy, issue this command on the monitor prior to the
+start of migration:
+
+migrate_set_capability x-postcopy-ram on
+
+The normal commands are then used to start a migration, which is still
+started in precopy mode.  Issuing:
+
+migrate_start_postcopy
+
+will now cause the transition from precopy to postcopy.
+It can be issued immediately after migration is started or any
+time later on.  Issuing it after the end of a migration is harmless.
+
+Note: During the postcopy phase, the bandwidth limits set using
+migrate_set_speed is ignored (to avoid delaying requested pages that
+the destination is waiting for).
+
+=== Postcopy device transfer ===
+
+Loading of device data may cause the device emulation to access guest RAM
+that may trigger faults that have to be resolved by the source, as such
+the migration stream has to be able to respond with page data *during* the
+device load, and hence the device data has to be read from the stream completely
+before the device load begins to free the stream up.  This is achieved by
+'packaging' the device data into a blob that's read in one go.
+
+Source behaviour
+
+Until postcopy is entered the migration stream is identical to normal
+precopy, except for the addition of a 'postcopy advise' command at
+the beginning, to tell the destination that postcopy might happen.
+When postcopy starts the source sends the page discard data and then
+forms the 'package' containing:
+
+   Command: 'postcopy listen'
+   The device state
+      A series of sections, identical to the precopy streams device state stream
+      containing everything except postcopiable devices (i.e. RAM)
+   Command: 'postcopy run'
+
+The 'package' is sent as the data part of a Command: 'CMD_PACKAGED', and the
+contents are formatted in the same way as the main migration stream.
+
+During postcopy the source scans the list of dirty pages and sends them
+to the destination without being requested (in much the same way as precopy),
+however when a page request is received from the destination, the dirty page
+scanning restarts from the requested location.  This causes requested pages
+to be sent quickly, and also causes pages directly after the requested page
+to be sent quickly in the hope that those pages are likely to be used
+by the destination soon.
+
+Destination behaviour
+
+Initially the destination looks the same as precopy, with a single thread
+reading the migration stream; the 'postcopy advise' and 'discard' commands
+are processed to change the way RAM is managed, but don't affect the stream
+processing.
+
+------------------------------------------------------------------------------
+                        1      2   3     4 5                      6   7
+main -----DISCARD-CMD_PACKAGED ( LISTEN  DEVICE     DEVICE DEVICE RUN )
+thread                             |       |
+                                   |     (page request)
+                                   |        \___
+                                   v            \
+listen thread:                     --- page -- page -- page -- page -- page --
+
+                                   a   b        c
+------------------------------------------------------------------------------
+
+On receipt of CMD_PACKAGED (1)
+   All the data associated with the package - the ( ... ) section in the
+diagram - is read into memory (into a QEMUSizedBuffer), and the main thread
+recurses into qemu_loadvm_state_main to process the contents of the package (2)
+which contains commands (3,6) and devices (4...)
+
+On receipt of 'postcopy listen' - 3 -(i.e. the 1st command in the package)
+a new thread (a) is started that takes over servicing the migration stream,
+while the main thread carries on loading the package.   It loads normal
+background page data (b) but if during a device load a fault happens (5) the
+returned page (c) is loaded by the listen thread allowing the main threads
+device load to carry on.
+
+The last thing in the CMD_PACKAGED is a 'RUN' command (6) letting the destination
+CPUs start running.
+At the end of the CMD_PACKAGED (7) the main thread returns to normal running behaviour
+and is no longer used by migration, while the listen thread carries
+on servicing page data until the end of migration.
+
+=== Postcopy states ===
+
+Postcopy moves through a series of states (see postcopy_state) from
+ADVISE->DISCARD->LISTEN->RUNNING->END
+
+  Advise:  Set at the start of migration if postcopy is enabled, even
+           if it hasn't had the start command; here the destination
+           checks that its OS has the support needed for postcopy, and performs
+           setup to ensure the RAM mappings are suitable for later postcopy.
+           The destination will fail early in migration at this point if the
+           required OS support is not present.
+           (Triggered by reception of POSTCOPY_ADVISE command)
+
+  Discard: Entered on receipt of the first 'discard' command; prior to
+           the first Discard being performed, hugepages are switched off
+           (using madvise) to ensure that no new huge pages are created
+           during the postcopy phase, and to cause any huge pages that
+           have discards on them to be broken.
+
+  Listen:  The first command in the package, POSTCOPY_LISTEN, switches
+           the destination state to Listen, and starts a new thread
+           (the 'listen thread') which takes over the job of receiving
+           pages off the migration stream, while the main thread carries
+           on processing the blob.  With this thread able to process page
+           reception, the destination now 'sensitises' the RAM to detect
+           any access to missing pages (on Linux using the 'userfault'
+           system).
+
+  Running: POSTCOPY_RUN causes the destination to synchronise all
+           state and start the CPUs and IO devices running.  The main
+           thread now finishes processing the migration package and
+           now carries on as it would for normal precopy migration
+           (although it can't do the cleanup it would do as it
+           finishes a normal migration).
+
+  End:     The listen thread can now quit, and perform the cleanup of migration
+           state, the migration is now complete.
+
+=== Source side page maps ===
+
+The source side keeps two bitmaps during postcopy; 'the migration bitmap'
+and 'unsent map'.  The 'migration bitmap' is basically the same as in
+the precopy case, and holds a bit to indicate that page is 'dirty' -
+i.e. needs sending.  During the precopy phase this is updated as the CPU
+dirties pages, however during postcopy the CPUs are stopped and nothing
+should dirty anything any more.
+
+The 'unsent map' is used for the transition to postcopy. It is a bitmap that
+has a bit cleared whenever a page is sent to the destination, however during
+the transition to postcopy mode it is combined with the migration bitmap
+to form a set of pages that:
+   a) Have been sent but then redirtied (which must be discarded)
+   b) Have not yet been sent - which also must be discarded to cause any
+      transparent huge pages built during precopy to be broken.
+
+Note that the contents of the unsentmap are sacrificed during the calculation
+of the discard set and thus aren't valid once in postcopy.  The dirtymap
+is still valid and is used to ensure that no page is sent more than once.  Any
+request for a page that has already been sent is ignored.  Duplicate requests
+such as this can happen as a page is sent at about the same time the
+destination accesses it.
+
diff --git a/docs/multiseat.txt b/docs/multiseat.txt
index ebf244693..807518c8a 100644
--- a/docs/multiseat.txt
+++ b/docs/multiseat.txt
@@ -135,7 +135,7 @@ configuration:
             TAG+="seat", ENV{ID_AUTOSEAT}="1"
 
 Patch with this rule has been submitted to upstream udev/systemd, was
-accepted and and should be included in the next systemd release (222).
+accepted and should be included in the next systemd release (222).
 So, if your guest has this or a newer version, multiseat will work just
 fine without any manual guest configuration.
 
diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index 61b5be47f..ceb9a782d 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -106,15 +106,15 @@ Types, commands, and events share a common namespace.  Therefore,
 generally speaking, type definitions should always use CamelCase for
 user-defined type names, while built-in types are lowercase. Type
 definitions should not end in 'Kind', as this namespace is used for
-creating implicit C enums for visiting union types.  Command names,
+creating implicit C enums for visiting union types, or in 'List', as
+this namespace is used for creating array types.  Command names,
 and field names within a type, should be all lower case with words
 separated by a hyphen.  However, some existing older commands and
 complex types use underscore; when extending such expressions,
 consistency is preferred over blindly avoiding underscore.  Event
-names should be ALL_CAPS with words separated by underscore.  The
-special string '**' appears for some commands that manually perform
-their own type checking rather than relying on the type-safe code
-produced by the qapi code generators.
+names should be ALL_CAPS with words separated by underscore.  Field
+names cannot start with 'has-' or 'has_', as this is reserved for
+tracking optional fields.
 
 Any name (command, event, type, field, or enum value) beginning with
 "x-" is marked experimental, and may be withdrawn or changed
@@ -125,9 +125,10 @@ vendor), even if the rest of the name uses dash (example:
 __com.redhat_drive-mirror).  Other than downstream extensions (with
 leading underscore and the use of dots), all names should begin with a
 letter, and contain only ASCII letters, digits, dash, and underscore.
-It is okay to reuse names that match C keywords; the generator will
-rename a field named "default" in the QAPI to "q_default" in the
-generated C code.
+Names beginning with 'q_' are reserved for the generator: QMP names
+that resemble C keywords or other problematic strings will be munged
+in C to use this prefix.  For example, a field named "default" in
+qapi becomes "q_default" in the generated C code.
 
 In the rest of this document, usage lines are given for each
 expression type, with literal strings written in lower case and
@@ -140,17 +141,25 @@ must have a value that forms a struct name.
 
 === Built-in Types ===
 
-The following types are built-in to the parser:
-  'str' - arbitrary UTF-8 string
-  'int' - 64-bit signed integer (although the C code may place further
-          restrictions on acceptable range)
-  'number' - floating point number
-  'bool' - JSON value of true or false
-  'int8', 'int16', 'int32', 'int64' - like 'int', but enforce maximum
-                                      bit size
-  'uint8', 'uint16', 'uint32', 'uint64' - unsigned counterparts
-  'size' - like 'uint64', but allows scaled suffix from command line
-           visitor
+The following types are predefined, and map to C as follows:
+
+  Schema    C          JSON
+  str       char *     any JSON string, UTF-8
+  number    double     any JSON number
+  int       int64_t    a JSON number without fractional part
+                       that fits into the C integer type
+  int8      int8_t     likewise
+  int16     int16_t    likewise
+  int32     int32_t    likewise
+  int64     int64_t    likewise
+  uint8     uint8_t    likewise
+  uint16    uint16_t   likewise
+  uint32    uint32_t   likewise
+  uint64    uint64_t   likewise
+  size      uint64_t   like uint64_t, except StringInputVisitor
+                       accepts size suffixes
+  bool      bool       JSON true or false
+  any       QObject *  any JSON value
 
 
 === Includes ===
@@ -163,7 +172,7 @@ The QAPI schema definitions can be modularized using the 'include' directive:
 
 The directive is evaluated recursively, and include paths are relative to the
 file using the directive. Multiple includes of the same file are
-safe.  No other keys should appear in the expression, and the include
+idempotent.  No other keys should appear in the expression, and the include
 value should be a string.
 
 As a matter of style, it is a good idea to have all files be
@@ -236,6 +245,7 @@ both fields like this:
 === Enumeration types ===
 
 Usage: { 'enum': STRING, 'data': ARRAY-OF-STRING }
+       { 'enum': STRING, '*prefix': STRING, 'data': ARRAY-OF-STRING }
 
 An enumeration type is a dictionary containing a single 'data' key
 whose value is a list of strings.  An example enumeration is:
@@ -247,6 +257,13 @@ useful.  The list of strings should be lower case; if an enum name
 represents multiple words, use '-' between words.  The string 'max' is
 not allowed as an enum value, and values should not be repeated.
 
+The enum constants will be named by using a heuristic to turn the
+type name into a set of underscore separated words. For the example
+above, 'MyEnum' will turn into 'MY_ENUM' giving a constant name
+of 'MY_ENUM_VALUE1' for the first value. If the default heuristic
+does not result in a desirable name, the optional 'prefix' field
+can be used when defining the enum.
+
 The enumeration values are passed as strings over the Client JSON
 Protocol, but are encoded as C enum integral values in generated code.
 While the C code starts numbering at 0, it is better to use explicit
@@ -300,7 +317,6 @@ an implicit C enum 'NameKind' is created, corresponding to the union
 the union can be named 'max', as this would collide with the implicit
 enum.  The value for each branch can be of any type.
 
-
 A flat union definition specifies a struct as its base, and
 avoids nesting on the wire.  All branches of the union must be
 complex types, and the top-level fields of the union dictionary on
@@ -314,7 +330,7 @@ adding a common field 'readonly', renaming the discriminator to
 something more applicable, and reducing the number of {} required on
 the wire:
 
- { 'enum': 'BlockdevDriver', 'data': [ 'raw', 'qcow2' ] }
+ { 'enum': 'BlockdevDriver', 'data': [ 'file', 'qcow2' ] }
  { 'struct': 'BlockdevCommonOptions',
    'data': { 'driver': 'BlockdevDriver', 'readonly': 'bool' } }
  { 'union': 'BlockdevOptions',
@@ -350,7 +366,7 @@ is identical on the wire to:
  { 'struct': 'Base', 'data': { 'type': 'Enum' } }
  { 'struct': 'Branch1', 'data': { 'data': 'str' } }
  { 'struct': 'Branch2', 'data': { 'data': 'int' } }
- { 'union': 'Flat': 'base': 'Base', 'discriminator': 'type',
+ { 'union': 'Flat', 'base': 'Base', 'discriminator': 'type',
    'data': { 'one': 'Branch1', 'two': 'Branch2' } }
 
 
@@ -394,7 +410,7 @@ following example objects:
 === Commands ===
 
 Usage: { 'command': STRING, '*data': COMPLEX-TYPE-NAME-OR-DICT,
-         '*returns': TYPE-NAME-OR-DICT,
+         '*returns': TYPE-NAME,
          '*gen': false, '*success-response': false }
 
 Commands are defined by using a dictionary containing several members,
@@ -405,10 +421,9 @@ Client JSON Protocol command exchange.
 The 'data' argument maps to the "arguments" dictionary passed in as
 part of a Client JSON Protocol command.  The 'data' member is optional
 and defaults to {} (an empty dictionary).  If present, it must be the
-string name of a complex type, a one-element array containing the name
-of a complex type, or a dictionary that declares an anonymous type
-with the same semantics as a 'struct' expression, with one exception
-noted below when 'gen' is used.
+string name of a complex type, or a dictionary that declares an
+anonymous type with the same semantics as a 'struct' expression, with
+one exception noted below when 'gen' is used.
 
 The 'returns' member describes what will appear in the "return" field
 of a Client JSON Protocol reply on successful completion of a command.
@@ -416,14 +431,13 @@ The member is optional from the command declaration; if absent, the
 "return" field will be an empty dictionary.  If 'returns' is present,
 it must be the string name of a complex or built-in type, a
 one-element array containing the name of a complex or built-in type,
-or a dictionary that declares an anonymous type with the same
-semantics as a 'struct' expression, with one exception noted below
-when 'gen' is used.  Although it is permitted to have the 'returns'
-member name a built-in type or an array of built-in types, any command
-that does this cannot be extended to return additional information in
-the future; thus, new commands should strongly consider returning a
-dictionary-based type or an array of dictionaries, even if the
-dictionary only contains one field at the present.
+with one exception noted below when 'gen' is used.  Although it is
+permitted to have the 'returns' member name a built-in type or an
+array of built-in types, any command that does this cannot be extended
+to return additional information in the future; thus, new commands
+should strongly consider returning a dictionary-based type or an array
+of dictionaries, even if the dictionary only contains one field at the
+present.
 
 All commands in Client JSON Protocol use a dictionary to report
 failure, with no way to specify that in QAPI.  Where the error return
@@ -448,17 +462,14 @@ which would validate this Client JSON Protocol transaction:
  <= { "return": [ { "value": "one" }, { } ] }
 
 In rare cases, QAPI cannot express a type-safe representation of a
-corresponding Client JSON Protocol command.  In these cases, if the
-command expression includes the key 'gen' with boolean value false,
-then the 'data' or 'returns' member that intends to bypass generated
-type-safety and do its own manual validation should use an inline
-dictionary definition, with a value of '**' rather than a valid type
-name for the keys that the generated code will not validate.  Please
-try to avoid adding new commands that rely on this, and instead use
-type-safe unions.  For an example of bypass usage:
+corresponding Client JSON Protocol command.  You then have to suppress
+generation of a marshalling function by including a key 'gen' with
+boolean value false, and instead write your own function.  Please try
+to avoid adding new commands that rely on this, and instead use
+type-safe unions.  For an example of this usage:
 
  { 'command': 'netdev_add',
-   'data': {'type': 'str', 'id': 'str', '*props': '**'},
+   'data': {'type': 'str', 'id': 'str'},
    'gen': false }
 
 Normally, the QAPI schema is used to describe synchronous exchanges,
@@ -495,13 +506,229 @@ Resulting in this JSON object:
   "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
 
+== Client JSON Protocol introspection ==
+
+Clients of a Client JSON Protocol commonly need to figure out what
+exactly the server (QEMU) supports.
+
+For this purpose, QMP provides introspection via command
+query-qmp-schema.  QGA currently doesn't support introspection.
+
+While Client JSON Protocol wire compatibility should be maintained
+between qemu versions, we cannot make the same guarantees for
+introspection stability.  For example, one version of qemu may provide
+a non-variant optional member of a struct, and a later version rework
+the member to instead be non-optional and associated with a variant.
+Likewise, one version of qemu may list a member with open-ended type
+'str', and a later version could convert it to a finite set of strings
+via an enum type; or a member may be converted from a specific type to
+an alternate that represents a choice between the original type and
+something else.
+
+query-qmp-schema returns a JSON array of SchemaInfo objects.  These
+objects together describe the wire ABI, as defined in the QAPI schema.
+There is no specified order to the SchemaInfo objects returned; a
+client must search for a particular name throughout the entire array
+to learn more about that name, but is at least guaranteed that there
+will be no collisions between type, command, and event names.
+
+However, the SchemaInfo can't reflect all the rules and restrictions
+that apply to QMP.  It's interface introspection (figuring out what's
+there), not interface specification.  The specification is in the QAPI
+schema.  To understand how QMP is to be used, you need to study the
+QAPI schema.
+
+Like any other command, query-qmp-schema is itself defined in the QAPI
+schema, along with the SchemaInfo type.  This text attempts to give an
+overview how things work.  For details you need to consult the QAPI
+schema.
+
+SchemaInfo objects have common members "name" and "meta-type", and
+additional variant members depending on the value of meta-type.
+
+Each SchemaInfo object describes a wire ABI entity of a certain
+meta-type: a command, event or one of several kinds of type.
+
+SchemaInfo for commands and events have the same name as in the QAPI
+schema.
+
+Command and event names are part of the wire ABI, but type names are
+not.  Therefore, the SchemaInfo for types have auto-generated
+meaningless names.  For readability, the examples in this section use
+meaningful type names instead.
+
+To examine a type, start with a command or event using it, then follow
+references by name.
+
+QAPI schema definitions not reachable that way are omitted.
+
+The SchemaInfo for a command has meta-type "command", and variant
+members "arg-type" and "ret-type".  On the wire, the "arguments"
+member of a client's "execute" command must conform to the object type
+named by "arg-type".  The "return" member that the server passes in a
+success response conforms to the type named by "ret-type".
+
+If the command takes no arguments, "arg-type" names an object type
+without members.  Likewise, if the command returns nothing, "ret-type"
+names an object type without members.
+
+Example: the SchemaInfo for command query-qmp-schema
+
+    { "name": "query-qmp-schema", "meta-type": "command",
+      "arg-type": ":empty", "ret-type": "SchemaInfoList" }
+
+    Type ":empty" is an object type without members, and type
+    "SchemaInfoList" is the array of SchemaInfo type.
+
+The SchemaInfo for an event has meta-type "event", and variant member
+"arg-type".  On the wire, a "data" member that the server passes in an
+event conforms to the object type named by "arg-type".
+
+If the event carries no additional information, "arg-type" names an
+object type without members.  The event may not have a data member on
+the wire then.
+
+Each command or event defined with dictionary-valued 'data' in the
+QAPI schema implicitly defines an object type.
+
+Example: the SchemaInfo for EVENT_C from section Events
+
+    { "name": "EVENT_C", "meta-type": "event",
+      "arg-type": ":obj-EVENT_C-arg" }
+
+    Type ":obj-EVENT_C-arg" is an implicitly defined object type with
+    the two members from the event's definition.
+
+The SchemaInfo for struct and union types has meta-type "object".
+
+The SchemaInfo for a struct type has variant member "members".
+
+The SchemaInfo for a union type additionally has variant members "tag"
+and "variants".
+
+"members" is a JSON array describing the object's common members, if
+any.  Each element is a JSON object with members "name" (the member's
+name), "type" (the name of its type), and optionally "default".  The
+member is optional if "default" is present.  Currently, "default" can
+only have value null.  Other values are reserved for future
+extensions.  The "members" array is in no particular order; clients
+must search the entire object when learning whether a particular
+member is supported.
+
+Example: the SchemaInfo for MyType from section Struct types
+
+    { "name": "MyType", "meta-type": "object",
+      "members": [
+          { "name": "member1", "type": "str" },
+          { "name": "member2", "type": "int" },
+          { "name": "member3", "type": "str", "default": null } ] }
+
+"tag" is the name of the common member serving as type tag.
+"variants" is a JSON array describing the object's variant members.
+Each element is a JSON object with members "case" (the value of type
+tag this element applies to) and "type" (the name of an object type
+that provides the variant members for this type tag value).  The
+"variants" array is in no particular order, and is not guaranteed to
+list cases in the same order as the corresponding "tag" enum type.
+
+Example: the SchemaInfo for flat union BlockdevOptions from section
+Union types
+
+    { "name": "BlockdevOptions", "meta-type": "object",
+      "members": [
+          { "name": "driver", "type": "BlockdevDriver" },
+          { "name": "readonly", "type": "bool"} ],
+      "tag": "driver",
+      "variants": [
+          { "case": "file", "type": "FileOptions" },
+          { "case": "qcow2", "type": "Qcow2Options" } ] }
+
+Note that base types are "flattened": its members are included in the
+"members" array.
+
+A simple union implicitly defines an enumeration type for its implicit
+discriminator (called "type" on the wire, see section Union types).
+
+A simple union implicitly defines an object type for each of its
+variants.
+
+Example: the SchemaInfo for simple union BlockdevOptions from section
+Union types
+
+    { "name": "BlockdevOptions", "meta-type": "object",
+      "members": [
+          { "name": "kind", "type": "BlockdevOptionsKind" } ],
+      "tag": "type",
+      "variants": [
+          { "case": "file", "type": ":obj-FileOptions-wrapper" },
+          { "case": "qcow2", "type": ":obj-Qcow2Options-wrapper" } ] }
+
+    Enumeration type "BlockdevOptionsKind" and the object types
+    ":obj-FileOptions-wrapper", ":obj-Qcow2Options-wrapper" are
+    implicitly defined.
+
+The SchemaInfo for an alternate type has meta-type "alternate", and
+variant member "members".  "members" is a JSON array.  Each element is
+a JSON object with member "type", which names a type.  Values of the
+alternate type conform to exactly one of its member types.  There is
+no guarantee on the order in which "members" will be listed.
+
+Example: the SchemaInfo for BlockRef from section Alternate types
+
+    { "name": "BlockRef", "meta-type": "alternate",
+      "members": [
+          { "type": "BlockdevOptions" },
+          { "type": "str" } ] }
+
+The SchemaInfo for an array type has meta-type "array", and variant
+member "element-type", which names the array's element type.  Array
+types are implicitly defined.  For convenience, the array's name may
+resemble the element type; however, clients should examine member
+"element-type" instead of making assumptions based on parsing member
+"name".
+
+Example: the SchemaInfo for ['str']
+
+    { "name": "[str]", "meta-type": "array",
+      "element-type": "str" }
+
+The SchemaInfo for an enumeration type has meta-type "enum" and
+variant member "values".  The values are listed in no particular
+order; clients must search the entire enum when learning whether a
+particular value is supported.
+
+Example: the SchemaInfo for MyEnum from section Enumeration types
+
+    { "name": "MyEnum", "meta-type": "enum",
+      "values": [ "value1", "value2", "value3" ] }
+
+The SchemaInfo for a built-in type has the same name as the type in
+the QAPI schema (see section Built-in Types), with one exception
+detailed below.  It has variant member "json-type" that shows how
+values of this type are encoded on the wire.
+
+Example: the SchemaInfo for str
+
+    { "name": "str", "meta-type": "builtin", "json-type": "string" }
+
+The QAPI schema supports a number of integer types that only differ in
+how they map to C.  They are identical as far as SchemaInfo is
+concerned.  Therefore, they get all mapped to a single type "int" in
+SchemaInfo.
+
+As explained above, type names are not part of the wire ABI.  Not even
+the names of built-in types.  Clients should examine member
+"json-type" instead of hard-coding names of built-in types.
+
+
 == Code generation ==
 
-Schemas are fed into 3 scripts to generate all the code/files that, paired
-with the core QAPI libraries, comprise everything required to take JSON
-commands read in by a Client JSON Protocol server, unmarshal the arguments into
-the underlying C types, call into the corresponding C function, and map the
-response back to a Client JSON Protocol response to be returned to the user.
+Schemas are fed into four scripts to generate all the code/files that,
+paired with the core QAPI libraries, comprise everything required to
+take JSON commands read in by a Client JSON Protocol server, unmarshal
+the arguments into the underlying C types, call into the corresponding
+C function, and map the response back to a Client JSON Protocol
+response to be returned to the user.
 
 As an example, we'll use the following schema, which describes a single
 complex user-defined type (which will produce a C struct, along with a list
@@ -540,36 +767,35 @@ Example:
     $ cat qapi-generated/example-qapi-types.c
 [Uninteresting stuff omitted...]
 
-    void qapi_free_UserDefOneList(UserDefOneList *obj)
+    void qapi_free_UserDefOne(UserDefOne *obj)
     {
-        QapiDeallocVisitor *md;
+        QapiDeallocVisitor *qdv;
         Visitor *v;
 
         if (!obj) {
             return;
         }
 
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOneList(v, &obj, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(md);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
+        visit_type_UserDefOne(v, &obj, NULL, NULL);
+        qapi_dealloc_visitor_cleanup(qdv);
     }
 
-    void qapi_free_UserDefOne(UserDefOne *obj)
+    void qapi_free_UserDefOneList(UserDefOneList *obj)
     {
-        QapiDeallocVisitor *md;
+        QapiDeallocVisitor *qdv;
         Visitor *v;
 
         if (!obj) {
             return;
         }
 
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOne(v, &obj, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(md);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
+        visit_type_UserDefOneList(v, &obj, NULL, NULL);
+        qapi_dealloc_visitor_cleanup(qdv);
     }
-
     $ cat qapi-generated/example-qapi-types.h
 [Uninteresting stuff omitted...]
 
@@ -580,25 +806,24 @@ Example:
 
     typedef struct UserDefOne UserDefOne;
 
-    typedef struct UserDefOneList
-    {
+    typedef struct UserDefOneList UserDefOneList;
+
+    struct UserDefOne {
+        int64_t integer;
+        char *string;
+    };
+
+    void qapi_free_UserDefOne(UserDefOne *obj);
+
+    struct UserDefOneList {
         union {
             UserDefOne *value;
             uint64_t padding;
         };
-        struct UserDefOneList *next;
-    } UserDefOneList;
-
-[Functions on built-in types omitted...]
-
-    struct UserDefOne
-    {
-        int64_t integer;
-        char *string;
+        UserDefOneList *next;
     };
 
     void qapi_free_UserDefOneList(UserDefOneList *obj);
-    void qapi_free_UserDefOne(UserDefOne *obj);
 
     #endif
 
@@ -627,14 +852,15 @@ Example:
     $ cat qapi-generated/example-qapi-visit.c
 [Uninteresting stuff omitted...]
 
-    static void visit_type_UserDefOne_fields(Visitor *m, UserDefOne **obj, Error **errp)
+    static void visit_type_UserDefOne_fields(Visitor *v, UserDefOne **obj, Error **errp)
     {
         Error *err = NULL;
-        visit_type_int(m, &(*obj)->integer, "integer", &err);
+
+        visit_type_int(v, &(*obj)->integer, "integer", &err);
         if (err) {
             goto out;
         }
-        visit_type_str(m, &(*obj)->string, "string", &err);
+        visit_type_str(v, &(*obj)->string, "string", &err);
         if (err) {
             goto out;
         }
@@ -643,40 +869,40 @@ Example:
         error_propagate(errp, err);
     }
 
-    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp)
+    void visit_type_UserDefOne(Visitor *v, UserDefOne **obj, const char *name, Error **errp)
     {
         Error *err = NULL;
 
-        visit_start_struct(m, (void **)obj, "UserDefOne", name, sizeof(UserDefOne), &err);
+        visit_start_struct(v, (void **)obj, "UserDefOne", name, sizeof(UserDefOne), &err);
         if (!err) {
             if (*obj) {
-                visit_type_UserDefOne_fields(m, obj, errp);
+                visit_type_UserDefOne_fields(v, obj, errp);
             }
-            visit_end_struct(m, &err);
+            visit_end_struct(v, &err);
         }
         error_propagate(errp, err);
     }
 
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp)
+    void visit_type_UserDefOneList(Visitor *v, UserDefOneList **obj, const char *name, Error **errp)
     {
         Error *err = NULL;
         GenericList *i, **prev;
 
-        visit_start_list(m, name, &err);
+        visit_start_list(v, name, &err);
         if (err) {
             goto out;
         }
 
         for (prev = (GenericList **)obj;
-             !err && (i = visit_next_list(m, prev, &err)) != NULL;
+             !err && (i = visit_next_list(v, prev, &err)) != NULL;
              prev = &i) {
             UserDefOneList *native_i = (UserDefOneList *)i;
-            visit_type_UserDefOne(m, &native_i->value, NULL, &err);
+            visit_type_UserDefOne(v, &native_i->value, NULL, &err);
         }
 
         error_propagate(errp, err);
         err = NULL;
-        visit_end_list(m, &err);
+        visit_end_list(v, &err);
     out:
         error_propagate(errp, err);
     }
@@ -688,8 +914,8 @@ Example:
 
 [Visitors for built-in types omitted...]
 
-    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp);
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp);
+    void visit_type_UserDefOne(Visitor *v, UserDefOne **obj, const char *name, Error **errp);
+    void visit_type_UserDefOneList(Visitor *v, UserDefOneList **obj, const char *name, Error **errp);
 
     #endif
 
@@ -717,64 +943,63 @@ Example:
     $ cat qapi-generated/example-qmp-marshal.c
 [Uninteresting stuff omitted...]
 
-    static void qmp_marshal_output_my_command(UserDefOne *ret_in, QObject **ret_out, Error **errp)
+    static void qmp_marshal_output_UserDefOne(UserDefOne *ret_in, QObject **ret_out, Error **errp)
     {
-        Error *local_err = NULL;
-        QmpOutputVisitor *mo = qmp_output_visitor_new();
-        QapiDeallocVisitor *md;
+        Error *err = NULL;
+        QmpOutputVisitor *qov = qmp_output_visitor_new();
+        QapiDeallocVisitor *qdv;
         Visitor *v;
 
-        v = qmp_output_get_visitor(mo);
-        visit_type_UserDefOne(v, &ret_in, "unused", &local_err);
-        if (local_err) {
+        v = qmp_output_get_visitor(qov);
+        visit_type_UserDefOne(v, &ret_in, "unused", &err);
+        if (err) {
             goto out;
         }
-        *ret_out = qmp_output_get_qobject(mo);
+        *ret_out = qmp_output_get_qobject(qov);
 
     out:
-        error_propagate(errp, local_err);
-        qmp_output_visitor_cleanup(mo);
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
+        error_propagate(errp, err);
+        qmp_output_visitor_cleanup(qov);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
         visit_type_UserDefOne(v, &ret_in, "unused", NULL);
-        qapi_dealloc_visitor_cleanup(md);
+        qapi_dealloc_visitor_cleanup(qdv);
     }
 
-    static void qmp_marshal_input_my_command(QDict *args, QObject **ret, Error **errp)
+    static void qmp_marshal_my_command(QDict *args, QObject **ret, Error **errp)
     {
-        Error *local_err = NULL;
-        UserDefOne *retval = NULL;
-        QmpInputVisitor *mi = qmp_input_visitor_new_strict(QOBJECT(args));
-        QapiDeallocVisitor *md;
+        Error *err = NULL;
+        UserDefOne *retval;
+        QmpInputVisitor *qiv = qmp_input_visitor_new_strict(QOBJECT(args));
+        QapiDeallocVisitor *qdv;
         Visitor *v;
         UserDefOne *arg1 = NULL;
 
-        v = qmp_input_get_visitor(mi);
-        visit_type_UserDefOne(v, &arg1, "arg1", &local_err);
-        if (local_err) {
+        v = qmp_input_get_visitor(qiv);
+        visit_type_UserDefOne(v, &arg1, "arg1", &err);
+        if (err) {
             goto out;
         }
 
-        retval = qmp_my_command(arg1, &local_err);
-        if (local_err) {
+        retval = qmp_my_command(arg1, &err);
+        if (err) {
             goto out;
         }
 
-        qmp_marshal_output_my_command(retval, ret, &local_err);
+        qmp_marshal_output_UserDefOne(retval, ret, &err);
 
     out:
-        error_propagate(errp, local_err);
-        qmp_input_visitor_cleanup(mi);
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
+        error_propagate(errp, err);
+        qmp_input_visitor_cleanup(qiv);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
         visit_type_UserDefOne(v, &arg1, "arg1", NULL);
-        qapi_dealloc_visitor_cleanup(md);
-        return;
+        qapi_dealloc_visitor_cleanup(qdv);
     }
 
     static void qmp_init_marshal(void)
     {
-        qmp_register_command("my-command", qmp_marshal_input_my_command, QCO_NO_OPTIONS);
+        qmp_register_command("my-command", qmp_marshal_my_command, QCO_NO_OPTIONS);
     }
 
     qapi_init(qmp_init_marshal);
@@ -811,7 +1036,7 @@ Example:
     void qapi_event_send_my_event(Error **errp)
     {
         QDict *qmp;
-        Error *local_err = NULL;
+        Error *err = NULL;
         QMPEventFuncEmit emit;
         emit = qmp_event_get_func_emit();
         if (!emit) {
@@ -820,15 +1045,15 @@ Example:
 
         qmp = qmp_event_build_dict("MY_EVENT");
 
-        emit(EXAMPLE_QAPI_EVENT_MY_EVENT, qmp, &local_err);
+        emit(EXAMPLE_QAPI_EVENT_MY_EVENT, qmp, &err);
 
-        error_propagate(errp, local_err);
+        error_propagate(errp, err);
         QDECREF(qmp);
     }
 
-    const char *EXAMPLE_QAPIEvent_lookup[] = {
-        "MY_EVENT",
-        NULL,
+    const char *const example_QAPIEvent_lookup[] = {
+        [EXAMPLE_QAPI_EVENT_MY_EVENT] = "MY_EVENT",
+        [EXAMPLE_QAPI_EVENT_MAX] = NULL,
     };
     $ cat qapi-generated/example-qapi-event.h
 [Uninteresting stuff omitted...]
@@ -843,11 +1068,45 @@ Example:
 
     void qapi_event_send_my_event(Error **errp);
 
-    extern const char *EXAMPLE_QAPIEvent_lookup[];
-    typedef enum EXAMPLE_QAPIEvent
-    {
+    typedef enum example_QAPIEvent {
         EXAMPLE_QAPI_EVENT_MY_EVENT = 0,
         EXAMPLE_QAPI_EVENT_MAX = 1,
-    } EXAMPLE_QAPIEvent;
+    } example_QAPIEvent;
+
+    extern const char *const example_QAPIEvent_lookup[];
+
+    #endif
+
+=== scripts/qapi-introspect.py ===
+
+Used to generate the introspection C code for a schema. The following
+files are created:
+
+$(prefix)qmp-introspect.c - Defines a string holding a JSON
+                            description of the schema.
+$(prefix)qmp-introspect.h - Declares the above string.
+
+Example:
+
+    $ python scripts/qapi-introspect.py --output-dir="qapi-generated"
+    --prefix="example-" example-schema.json
+    $ cat qapi-generated/example-qmp-introspect.c
+[Uninteresting stuff omitted...]
+
+    const char example_qmp_schema_json[] = "["
+        "{\"arg-type\": \"0\", \"meta-type\": \"event\", \"name\": \"MY_EVENT\"}, "
+        "{\"arg-type\": \"1\", \"meta-type\": \"command\", \"name\": \"my-command\", \"ret-type\": \"2\"}, "
+        "{\"members\": [], \"meta-type\": \"object\", \"name\": \"0\"}, "
+        "{\"members\": [{\"name\": \"arg1\", \"type\": \"2\"}], \"meta-type\": \"object\", \"name\": \"1\"}, "
+        "{\"members\": [{\"name\": \"integer\", \"type\": \"int\"}, {\"name\": \"string\", \"type\": \"str\"}], \"meta-type\": \"object\", \"name\": \"2\"}, "
+        "{\"json-type\": \"int\", \"meta-type\": \"builtin\", \"name\": \"int\"}, "
+        "{\"json-type\": \"string\", \"meta-type\": \"builtin\", \"name\": \"str\"}]";
+    $ cat qapi-generated/example-qmp-introspect.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QMP_INTROSPECT_H
+    #define EXAMPLE_QMP_INTROSPECT_H
+
+    extern const char example_qmp_schema_json[];
 
     #endif
diff --git a/docs/qcow2-cache.txt b/docs/qcow2-cache.txt
new file mode 100644
index 000000000..5bb06072d
--- /dev/null
+++ b/docs/qcow2-cache.txt
@@ -0,0 +1,164 @@
+qcow2 L2/refcount cache configuration
+=====================================
+Copyright (C) 2015 Igalia, S.L.
+Author: Alberto Garcia <berto@igalia.com>
+
+This work is licensed under the terms of the GNU GPL, version 2 or
+later. See the COPYING file in the top-level directory.
+
+Introduction
+------------
+The QEMU qcow2 driver has two caches that can improve the I/O
+performance significantly. However, setting the right cache sizes is
+not a straightforward operation.
+
+This document attempts to give an overview of the L2 and refcount
+caches, and how to configure them.
+
+Please refer to the docs/specs/qcow2.txt file for an in-depth
+technical description of the qcow2 file format.
+
+
+Clusters
+--------
+A qcow2 file is organized in units of constant size called clusters.
+
+The cluster size is configurable, but it must be a power of two and
+its value 512 bytes or higher. QEMU currently defaults to 64 KB
+clusters, and it does not support sizes larger than 2MB.
+
+The 'qemu-img create' command supports specifying the size using the
+cluster_size option:
+
+   qemu-img create -f qcow2 -o cluster_size=128K hd.qcow2 4G
+
+
+The L2 tables
+-------------
+The qcow2 format uses a two-level structure to map the virtual disk as
+seen by the guest to the disk image in the host. These structures are
+called the L1 and L2 tables.
+
+There is one single L1 table per disk image. The table is small and is
+always kept in memory.
+
+There can be many L2 tables, depending on how much space has been
+allocated in the image. Each table is one cluster in size. In order to
+read or write data from the virtual disk, QEMU needs to read its
+corresponding L2 table to find out where that data is located. Since
+reading the table for each I/O operation can be expensive, QEMU keeps
+an L2 cache in memory to speed up disk access.
+
+The size of the L2 cache can be configured, and setting the right
+value can improve the I/O performance significantly.
+
+
+The refcount blocks
+-------------------
+The qcow2 format also mantains a reference count for each cluster.
+Reference counts are used for cluster allocation and internal
+snapshots. The data is stored in a two-level structure similar to the
+L1/L2 tables described above.
+
+The second level structures are called refcount blocks, are also one
+cluster in size and the number is also variable and dependent on the
+amount of allocated space.
+
+Each block contains a number of refcount entries. Their size (in bits)
+is a power of two and must not be higher than 64. It defaults to 16
+bits, but a different value can be set using the refcount_bits option:
+
+   qemu-img create -f qcow2 -o refcount_bits=8 hd.qcow2 4G
+
+QEMU keeps a refcount cache to speed up I/O much like the
+aforementioned L2 cache, and its size can also be configured.
+
+
+Choosing the right cache sizes
+------------------------------
+In order to choose the cache sizes we need to know how they relate to
+the amount of allocated space.
+
+The amount of virtual disk that can be mapped by the L2 and refcount
+caches (in bytes) is:
+
+   disk_size = l2_cache_size * cluster_size / 8
+   disk_size = refcount_cache_size * cluster_size * 8 / refcount_bits
+
+With the default values for cluster_size (64KB) and refcount_bits
+(16), that is
+
+   disk_size = l2_cache_size * 8192
+   disk_size = refcount_cache_size * 32768
+
+So in order to cover n GB of disk space with the default values we
+need:
+
+   l2_cache_size = disk_size_GB * 131072
+   refcount_cache_size = disk_size_GB * 32768
+
+QEMU has a default L2 cache of 1MB (1048576 bytes) and a refcount
+cache of 256KB (262144 bytes), so using the formulas we've just seen
+we have
+
+   1048576 / 131072 = 8 GB of virtual disk covered by that cache
+    262144 /  32768 = 8 GB
+
+
+How to configure the cache sizes
+--------------------------------
+Cache sizes can be configured using the -drive option in the
+command-line, or the 'blockdev-add' QMP command.
+
+There are three options available, and all of them take bytes:
+
+"l2-cache-size":         maximum size of the L2 table cache
+"refcount-cache-size":   maximum size of the refcount block cache
+"cache-size":            maximum size of both caches combined
+
+There are two things that need to be taken into account:
+
+ - Both caches must have a size that is a multiple of the cluster
+   size.
+
+ - If you only set one of the options above, QEMU will automatically
+   adjust the others so that the L2 cache is 4 times bigger than the
+   refcount cache.
+
+This means that these options are equivalent:
+
+   -drive file=hd.qcow2,l2-cache-size=2097152
+   -drive file=hd.qcow2,refcount-cache-size=524288
+   -drive file=hd.qcow2,cache-size=2621440
+
+The reason for this 1/4 ratio is to ensure that both caches cover the
+same amount of disk space. Note however that this is only valid with
+the default value of refcount_bits (16). If you are using a different
+value you might want to calculate both cache sizes yourself since QEMU
+will always use the same 1/4 ratio.
+
+It's also worth mentioning that there's no strict need for both caches
+to cover the same amount of disk space. The refcount cache is used
+much less often than the L2 cache, so it's perfectly reasonable to
+keep it small.
+
+
+Reducing the memory usage
+-------------------------
+It is possible to clean unused cache entries in order to reduce the
+memory usage during periods of low I/O activity.
+
+The parameter "cache-clean-interval" defines an interval (in seconds).
+All cache entries that haven't been accessed during that interval are
+removed from memory.
+
+This example removes all unused cache entries every 15 minutes:
+
+   -drive file=hd.qcow2,cache-clean-interval=900
+
+If unset, the default value for this parameter is 0 and it disables
+this feature.
+
+Note that this functionality currently relies on the MADV_DONTNEED
+argument for madvise() to actually free the memory, so it is not
+useful in systems that don't follow that behavior.
diff --git a/docs/qmp/qmp-events.txt b/docs/qmp-events.txt
index d92cc4833..d2f1ce497 100644
--- a/docs/qmp/qmp-events.txt
+++ b/docs/qmp-events.txt
@@ -28,6 +28,8 @@ Example:
     "data": { "actual": 944766976 },
     "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
+Note: this event is rate-limited.
+
 BLOCK_IMAGE_CORRUPTED
 ---------------------
 
@@ -296,6 +298,8 @@ Example:
      "data": { "reference": "usr1", "sector-num": 345435, "sectors-count": 5 },
      "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
 
+Note: this event is rate-limited.
+
 QUORUM_REPORT_BAD
 -----------------
 
@@ -318,6 +322,8 @@ Example:
      "data": { "node-name": "1.raw", "sector-num": 345435, "sectors-count": 5 },
      "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
 
+Note: this event is rate-limited.
+
 RESET
 -----
 
@@ -358,6 +364,8 @@ Example:
     "data": { "offset": 78 },
     "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
+Note: this event is rate-limited.
+
 SHUTDOWN
 --------
 
@@ -632,6 +640,8 @@ Example:
     "data": { "id": "channel0", "open": true },
     "timestamp": { "seconds": 1401385907, "microseconds": 422329 } }
 
+Note: this event is rate-limited separately for each "id".
+
 WAKEUP
 ------
 
@@ -662,3 +672,5 @@ Example:
 
 Note: If action is "reset", "shutdown", or "pause" the WATCHDOG event is
 followed respectively by the RESET, SHUTDOWN, or STOP events.
+
+Note: this event is rate-limited.
diff --git a/docs/qmp/README b/docs/qmp-intro.txt
index f6a3a031e..f6a3a031e 100644
--- a/docs/qmp/README
+++ b/docs/qmp-intro.txt
diff --git a/docs/qmp/qmp-spec.txt b/docs/qmp-spec.txt
index 4c28cd943..4fb10a5d6 100644
--- a/docs/qmp/qmp-spec.txt
+++ b/docs/qmp-spec.txt
@@ -175,6 +175,11 @@ The format of asynchronous events is:
 For a listing of supported asynchronous events, please, refer to the
 qmp-events.txt file.
 
+Some events are rate-limited to at most one per second.  If additional
+"similar" events arrive within one second, all but the last one are
+dropped, and the last one is delayed.  "Similar" normally means same
+event type.  See qmp-events.txt for details.
+
 2.5 QGA Synchronization
 -----------------------
 
diff --git a/docs/rcu.txt b/docs/rcu.txt
index 21ecb8106..2f70954e8 100644
--- a/docs/rcu.txt
+++ b/docs/rcu.txt
@@ -128,7 +128,7 @@ The core RCU API is small:
         the callback function is g_free, in particular, g_free_rcu can be
         used.  In the above case, one could have written simply:
 
-            g_free_rcu(foo_reclaim, rcu);
+            g_free_rcu(&foo, rcu);
 
      typeof(*p) atomic_rcu_read(p);
 
diff --git a/docs/replay.txt b/docs/replay.txt
new file mode 100644
index 000000000..149727e2a
--- /dev/null
+++ b/docs/replay.txt
@@ -0,0 +1,168 @@
+Copyright (c) 2010-2015 Institute for System Programming
+                        of the Russian Academy of Sciences.
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.
+See the COPYING file in the top-level directory.
+
+Record/replay
+-------------
+
+Record/replay functions are used for the reverse execution and deterministic
+replay of qemu execution. This implementation of deterministic replay can
+be used for deterministic debugging of guest code through a gdb remote
+interface.
+
+Execution recording writes a non-deterministic events log, which can be later
+used for replaying the execution anywhere and for unlimited number of times.
+It also supports checkpointing for faster rewinding during reverse debugging.
+Execution replaying reads the log and replays all non-deterministic events
+including external input, hardware clocks, and interrupts.
+
+Deterministic replay has the following features:
+ * Deterministically replays whole system execution and all contents of
+   the memory, state of the hardware devices, clocks, and screen of the VM.
+ * Writes execution log into the file for later replaying for multiple times
+   on different machines.
+ * Supports i386, x86_64, and ARM hardware platforms.
+ * Performs deterministic replay of all operations with keyboard and mouse
+   input devices.
+
+Usage of the record/replay:
+ * First, record the execution, by adding the following arguments to the command line:
+   '-icount shift=7,rr=record,rrfile=replay.bin -net none'.
+   Block devices' images are not actually changed in the recording mode,
+   because all of the changes are written to the temporary overlay file.
+ * Then you can replay it by using another command
+   line option: '-icount shift=7,rr=replay,rrfile=replay.bin -net none'
+ * '-net none' option should also be specified if network replay patches
+   are not applied.
+
+Papers with description of deterministic replay implementation:
+http://www.computer.org/csdl/proceedings/csmr/2012/4666/00/4666a553-abs.html
+http://dl.acm.org/citation.cfm?id=2786805.2803179
+
+Modifications of qemu include:
+ * wrappers for clock and time functions to save their return values in the log
+ * saving different asynchronous events (e.g. system shutdown) into the log
+ * synchronization of the bottom halves execution
+ * synchronization of the threads from thread pool
+ * recording/replaying user input (mouse and keyboard)
+ * adding internal checkpoints for cpu and io synchronization
+
+Non-deterministic events
+------------------------
+
+Our record/replay system is based on saving and replaying non-deterministic
+events (e.g. keyboard input) and simulating deterministic ones (e.g. reading
+from HDD or memory of the VM). Saving only non-deterministic events makes
+log file smaller, simulation faster, and allows using reverse debugging even
+for realtime applications.
+
+The following non-deterministic data from peripheral devices is saved into
+the log: mouse and keyboard input, network packets, audio controller input,
+USB packets, serial port input, and hardware clocks (they are non-deterministic
+too, because their values are taken from the host machine). Inputs from
+simulated hardware, memory of VM, software interrupts, and execution of
+instructions are not saved into the log, because they are deterministic and
+can be replayed by simulating the behavior of virtual machine starting from
+initial state.
+
+We had to solve three tasks to implement deterministic replay: recording
+non-deterministic events, replaying non-deterministic events, and checking
+that there is no divergence between record and replay modes.
+
+We changed several parts of QEMU to make event log recording and replaying.
+Devices' models that have non-deterministic input from external devices were
+changed to write every external event into the execution log immediately.
+E.g. network packets are written into the log when they arrive into the virtual
+network adapter.
+
+All non-deterministic events are coming from these devices. But to
+replay them we need to know at which moments they occur. We specify
+these moments by counting the number of instructions executed between
+every pair of consecutive events.
+
+Instruction counting
+--------------------
+
+QEMU should work in icount mode to use record/replay feature. icount was
+designed to allow deterministic execution in absence of external inputs
+of the virtual machine. We also use icount to control the occurrence of the
+non-deterministic events. The number of instructions elapsed from the last event
+is written to the log while recording the execution. In replay mode we
+can predict when to inject that event using the instruction counter.
+
+Timers
+------
+
+Timers are used to execute callbacks from different subsystems of QEMU
+at the specified moments of time. There are several kinds of timers:
+ * Real time clock. Based on host time and used only for callbacks that
+   do not change the virtual machine state. For this reason real time
+   clock and timers does not affect deterministic replay at all.
+ * Virtual clock. These timers run only during the emulation. In icount
+   mode virtual clock value is calculated using executed instructions counter.
+   That is why it is completely deterministic and does not have to be recorded.
+ * Host clock. This clock is used by device models that simulate real time
+   sources (e.g. real time clock chip). Host clock is the one of the sources
+   of non-determinism. Host clock read operations should be logged to
+   make the execution deterministic.
+ * Real time clock for icount. This clock is similar to real time clock but
+   it is used only for increasing virtual clock while virtual machine is
+   sleeping. Due to its nature it is also non-deterministic as the host clock
+   and has to be logged too.
+
+Checkpoints
+-----------
+
+Replaying of the execution of virtual machine is bound by sources of
+non-determinism. These are inputs from clock and peripheral devices,
+and QEMU thread scheduling. Thread scheduling affect on processing events
+from timers, asynchronous input-output, and bottom halves.
+
+Invocations of timers are coupled with clock reads and changing the state
+of the virtual machine. Reads produce non-deterministic data taken from
+host clock. And VM state changes should preserve their order. Their relative
+order in replay mode must replicate the order of callbacks in record mode.
+To preserve this order we use checkpoints. When a specific clock is processed
+in record mode we save to the log special "checkpoint" event.
+Checkpoints here do not refer to virtual machine snapshots. They are just
+record/replay events used for synchronization.
+
+QEMU in replay mode will try to invoke timers processing in random moment
+of time. That's why we do not process a group of timers until the checkpoint
+event will be read from the log. Such an event allows synchronizing CPU
+execution and timer events.
+
+Another checkpoints application in record/replay is instruction counting
+while the virtual machine is idle. This function (qemu_clock_warp) is called
+from the wait loop. It changes virtual machine state and must be deterministic
+then. That is why we added checkpoint to this function to prevent its
+operation in replay mode when it does not correspond to record mode.
+
+Bottom halves
+-------------
+
+Disk I/O events are completely deterministic in our model, because
+in both record and replay modes we start virtual machine from the same
+disk state. But callbacks that virtual disk controller uses for reading and
+writing the disk may occur at different moments of time in record and replay
+modes.
+
+Reading and writing requests are created by CPU thread of QEMU. Later these
+requests proceed to block layer which creates "bottom halves". Bottom
+halves consist of callback and its parameters. They are processed when
+main loop locks the global mutex. These locks are not synchronized with
+replaying process because main loop also processes the events that do not
+affect the virtual machine state (like user interaction with monitor).
+
+That is why we had to implement saving and replaying bottom halves callbacks
+synchronously to the CPU execution. When the callback is about to execute
+it is added to the queue in the replay module. This queue is written to the
+log when its callbacks are executed. In replay mode callbacks are not processed
+until the corresponding event is read from the events log file.
+
+Sometimes the block layer uses asynchronous callbacks for its internal purposes
+(like reading or writing VM snapshots or disk image cluster tables). In this
+case bottom halves are not marked as "replayable" and do not saved
+into the log.
diff --git a/docs/specs/fw_cfg.txt b/docs/specs/fw_cfg.txt
index 74351dd18..b8c794f54 100644
--- a/docs/specs/fw_cfg.txt
+++ b/docs/specs/fw_cfg.txt
@@ -76,6 +76,13 @@ increasing address order, similar to memcpy().
 
 Selector Register IOport: 0x510
 Data Register IOport:     0x511
+DMA Address IOport:       0x514
+
+=== ARM Register Locations ===
+
+Selector Register address: Base + 8 (2 bytes)
+Data Register address:     Base + 0 (8 bytes)
+DMA Address address:       Base + 16 (8 bytes)
 
 == Firmware Configuration Items ==
 
@@ -86,11 +93,15 @@ by selecting the "signature" item using key 0x0000 (FW_CFG_SIGNATURE),
 and reading four bytes from the data register. If the fw_cfg device is
 present, the four bytes read will contain the characters "QEMU".
 
-=== Revision (Key 0x0001, FW_CFG_ID) ===
+If the DMA interface is available, then reading the DMA Address
+Register returns 0x51454d5520434647 ("QEMU CFG" in big-endian format).
+
+=== Revision / feature bitmap (Key 0x0001, FW_CFG_ID) ===
 
-A 32-bit little-endian unsigned int, this item is used as an interface
-revision number, and is currently set to 1 by QEMU when fw_cfg is
-initialized.
+A 32-bit little-endian unsigned int, this item is used to check for enabled
+features.
+ - Bit 0: traditional interface. Always set.
+ - Bit 1: DMA interface.
 
 === File Directory (Key 0x0019, FW_CFG_FILE_DIR) ===
 
@@ -132,6 +143,55 @@ Selector Reg.    Range Usage
 In practice, the number of allowed firmware configuration items is given
 by the value of FW_CFG_MAX_ENTRY (see fw_cfg.h).
 
+= Guest-side DMA Interface =
+
+If bit 1 of the feature bitmap is set, the DMA interface is present. This does
+not replace the existing fw_cfg interface, it is an add-on. This interface
+can be used through the 64-bit wide address register.
+
+The address register is in big-endian format. The value for the register is 0
+at startup and after an operation. A write to the least significant half (at
+offset 4) triggers an operation. This means that operations with 32-bit
+addresses can be triggered with just one write, whereas operations with
+64-bit addresses can be triggered with one 64-bit write or two 32-bit writes,
+starting with the most significant half (at offset 0).
+
+In this register, the physical address of a FWCfgDmaAccess structure in RAM
+should be written. This is the format of the FWCfgDmaAccess structure:
+
+typedef struct FWCfgDmaAccess {
+    uint32_t control;
+    uint32_t length;
+    uint64_t address;
+} FWCfgDmaAccess;
+
+The fields of the structure are in big endian mode, and the field at the lowest
+address is the "control" field.
+
+The "control" field has the following bits:
+ - Bit 0: Error
+ - Bit 1: Read
+ - Bit 2: Skip
+ - Bit 3: Select. The upper 16 bits are the selected index.
+
+When an operation is triggered, if the "control" field has bit 3 set, the
+upper 16 bits are interpreted as an index of a firmware configuration item.
+This has the same effect as writing the selector register.
+
+If the "control" field has bit 1 set, a read operation will be performed.
+"length" bytes for the current selector and offset will be copied into the
+physical RAM address specified by the "address" field.
+
+If the "control" field has bit 2 set (and not bit 1), a skip operation will be
+performed. The offset for the current selector will be advanced "length" bytes.
+
+To check the result, read the "control" field:
+   error bit set        ->  something went wrong.
+   all bits cleared     ->  transfer finished successfully.
+   otherwise            ->  transfer still in progress (doesn't happen
+                            today due to implementation not being async,
+                            but may in the future).
+
 = Host-side API =
 
 The following functions are available to the QEMU programmer for adding
@@ -159,6 +219,17 @@ will convert a 16-, 32-, or 64-bit integer to little-endian, then add
 a dynamically allocated copy of the appropriately sized item to fw_cfg
 under the given selector key value.
 
+== fw_cfg_modify_iXX() ==
+
+Modify the value of an XX-bit item (where XX may be 16, 32, or 64).
+Similarly to the corresponding fw_cfg_add_iXX() function set, convert
+a 16-, 32-, or 64-bit integer to little endian, create a dynamically
+allocated copy of the required size, and replace the existing item at
+the given selector key value with the newly allocated one. The previous
+item, assumed to have been allocated during an earlier call to
+fw_cfg_add_iXX() or fw_cfg_modify_iXX() (of the same width XX), is freed
+before the function returns.
+
 == fw_cfg_add_file() ==
 
 Given a filename (i.e., fw_cfg item name), starting pointer, and size,
@@ -216,6 +287,21 @@ the following syntax:
 where <item_name> is the fw_cfg item name, and <path> is the location
 on the host file system of a file containing the data to be inserted.
 
+Small enough items may be provided directly as strings on the command
+line, using the syntax:
+
+    -fw_cfg [name=]<item_name>,string=<string>
+
+The terminating NUL character of the content <string> will NOT be
+included as part of the fw_cfg item data, which is consistent with
+the absence of a NUL terminator for items inserted via the file option.
+
+Both <item_name> and, if applicable, the content <string> are passed
+through by QEMU without any interpretation, expansion, or further
+processing. Any such processing (potentially performed e.g., by the shell)
+is outside of QEMU's responsibility; as such, using plain ASCII characters
+is recommended.
+
 NOTE: Users *SHOULD* choose item names beginning with the prefix "opt/"
 when using the "-fw_cfg" command line option, to avoid conflicting with
 item names used internally by QEMU. For instance:
diff --git a/docs/specs/ivshmem_device_spec.txt b/docs/specs/ivshmem_device_spec.txt
index 667a8628f..d318d65c3 100644
--- a/docs/specs/ivshmem_device_spec.txt
+++ b/docs/specs/ivshmem_device_spec.txt
@@ -2,30 +2,106 @@
 Device Specification for Inter-VM shared memory device
 ------------------------------------------------------
 
-The Inter-VM shared memory device is designed to share a region of memory to
-userspace in multiple virtual guests.  The memory region does not belong to any
-guest, but is a POSIX memory object on the host.  Optionally, the device may
-support sending interrupts to other guests sharing the same memory region.
+The Inter-VM shared memory device is designed to share a memory region (created
+on the host via the POSIX shared memory API) between multiple QEMU processes
+running different guests. In order for all guests to be able to pick up the
+shared memory area, it is modeled by QEMU as a PCI device exposing said memory
+to the guest as a PCI BAR.
+The memory region does not belong to any guest, but is a POSIX memory object on
+the host. The host can access this shared memory if needed.
+
+The device also provides an optional communication mechanism between guests
+sharing the same memory object. More details about that in the section 'Guest to
+guest communication' section.
 
 
 The Inter-VM PCI device
 -----------------------
 
-*BARs*
+From the VM point of view, the ivshmem PCI device supports three BARs.
+
+- BAR0 is a 1 Kbyte MMIO region to support registers and interrupts when MSI is
+  not used.
+- BAR1 is used for MSI-X when it is enabled in the device.
+- BAR2 is used to access the shared memory object.
+
+It is your choice how to use the device but you must choose between two
+behaviors :
+
+- basically, if you only need the shared memory part, you will map BAR2.
+  This way, you have access to the shared memory in guest and can use it as you
+  see fit (memnic, for example, uses it in userland
+  http://dpdk.org/browse/memnic).
+
+- BAR0 and BAR1 are used to implement an optional communication mechanism
+  through interrupts in the guests. If you need an event mechanism between the
+  guests accessing the shared memory, you will most likely want to write a
+  kernel driver that will handle interrupts. See details in the section 'Guest
+  to guest communication' section.
+
+The behavior is chosen when starting your QEMU processes:
+- no communication mechanism needed, the first QEMU to start creates the shared
+  memory on the host, subsequent QEMU processes will use it.
+
+- communication mechanism needed, an ivshmem server must be started before any
+  QEMU processes, then each QEMU process connects to the server unix socket.
+
+For more details on the QEMU ivshmem parameters, see qemu-doc documentation.
+
+
+Guest to guest communication
+----------------------------
+
+This section details the communication mechanism between the guests accessing
+the ivhsmem shared memory.
 
-The device supports three BARs.  BAR0 is a 1 Kbyte MMIO region to support
-registers.  BAR1 is used for MSI-X when it is enabled in the device.  BAR2 is
-used to map the shared memory object from the host.  The size of BAR2 is
-specified when the guest is started and must be a power of 2 in size.
+*ivshmem server*
 
-*Registers*
+This server code is available in qemu.git/contrib/ivshmem-server.
 
-The device currently supports 4 registers of 32-bits each.  Registers
-are used for synchronization between guests sharing the same memory object when
-interrupts are supported (this requires using the shared memory server).
+The server must be started on the host before any guest.
+It creates a shared memory object then waits for clients to connect on a unix
+socket. All the messages are little-endian int64_t integer.
 
-The server assigns each VM an ID number and sends this ID number to the QEMU
-process when the guest starts.
+For each client (QEMU process) that connects to the server:
+- the server sends a protocol version, if client does not support it, the client
+  closes the communication,
+- the server assigns an ID for this client and sends this ID to him as the first
+  message,
+- the server sends a fd to the shared memory object to this client,
+- the server creates a new set of host eventfds associated to the new client and
+  sends this set to all already connected clients,
+- finally, the server sends all the eventfds sets for all clients to the new
+  client.
+
+The server signals all clients when one of them disconnects.
+
+The client IDs are limited to 16 bits because of the current implementation (see
+Doorbell register in 'PCI device registers' subsection). Hence only 65536
+clients are supported.
+
+All the file descriptors (fd to the shared memory, eventfds for each client)
+are passed to clients using SCM_RIGHTS over the server unix socket.
+
+Apart from the current ivshmem implementation in QEMU, an ivshmem client has
+been provided in qemu.git/contrib/ivshmem-client for debug.
+
+*QEMU as an ivshmem client*
+
+At initialisation, when creating the ivshmem device, QEMU first receives a
+protocol version and closes communication with server if it does not match.
+Then, QEMU gets its ID from the server then makes it available through BAR0
+IVPosition register for the VM to use (see 'PCI device registers' subsection).
+QEMU then uses the fd to the shared memory to map it to BAR2.
+eventfds for all other clients received from the server are stored to implement
+BAR0 Doorbell register (see 'PCI device registers' subsection).
+Finally, eventfds assigned to this QEMU process are used to send interrupts in
+this VM.
+
+*PCI device registers*
+
+From the VM point of view, the ivshmem PCI device supports 4 registers of
+32-bits each.
 
 enum ivshmem_registers {
     IntrMask = 0,
@@ -49,8 +125,8 @@ bit to 0 and unmasked by setting the first bit to 1.
 IVPosition Register: The IVPosition register is read-only and reports the
 guest's ID number.  The guest IDs are non-negative integers.  When using the
 server, since the server is a separate process, the VM ID will only be set when
-the device is ready (shared memory is received from the server and accessible via
-the device).  If the device is not ready, the IVPosition will return -1.
+the device is ready (shared memory is received from the server and accessible
+via the device).  If the device is not ready, the IVPosition will return -1.
 Applications should ensure that they have a valid VM ID before accessing the
 shared memory.
 
@@ -59,8 +135,8 @@ Doorbell register.  The doorbell register is 32-bits, logically divided into
 two 16-bit fields.  The high 16-bits are the guest ID to interrupt and the low
 16-bits are the interrupt vector to trigger.  The semantics of the value
 written to the doorbell depends on whether the device is using MSI or a regular
-pin-based interrupt.  In short, MSI uses vectors while regular interrupts set the
-status register.
+pin-based interrupt.  In short, MSI uses vectors while regular interrupts set
+the status register.
 
 Regular Interrupts
 
@@ -71,7 +147,7 @@ interrupt in the destination guest.
 
 Message Signalled Interrupts
 
-A ivshmem device may support multiple MSI vectors.  If so, the lower 16-bits
+An ivshmem device may support multiple MSI vectors.  If so, the lower 16-bits
 written to the Doorbell register must be between 0 and the maximum number of
 vectors the guest supports.  The lower 16 bits written to the doorbell is the
 MSI vector that will be raised in the destination guest.  The number of MSI
@@ -83,14 +159,3 @@ interrupt itself should be communicated via the shared memory region.  Devices
 supporting multiple MSI vectors can use different vectors to indicate different
 events have occurred.  The semantics of interrupt vectors are left to the
 user's discretion.
-
-
-Usage in the Guest
-------------------
-
-The shared memory device is intended to be used with the provided UIO driver.
-Very little configuration is needed.  The guest should map BAR0 to access the
-registers (an array of 32-bit ints allows simple writing) and map BAR2 to
-access the shared memory region itself.  The size of the shared memory region
-is specified when the guest (or shared memory server) is started.  A guest may
-map the whole shared memory region or only part of it.
diff --git a/docs/specs/ppc-spapr-hcalls.txt b/docs/specs/ppc-spapr-hcalls.txt
index 667b3fa00..5bd8eab78 100644
--- a/docs/specs/ppc-spapr-hcalls.txt
+++ b/docs/specs/ppc-spapr-hcalls.txt
@@ -41,8 +41,8 @@ When the guest runs in "real mode" (in powerpc lingua this means
 with MMU disabled, ie guest effective == guest physical), it only
 has access to a subset of memory and no IOs.
 
-PAPR provides a set of hypervisor calls to perform cachable or
-non-cachable accesses to any guest physical addresses that the
+PAPR provides a set of hypervisor calls to perform cacheable or
+non-cacheable accesses to any guest physical addresses that the
 guest can use in order to access IO devices while in real mode.
 
 This is typically used by the firmware running in the guest.
diff --git a/docs/specs/ppc-spapr-hotplug.txt b/docs/specs/ppc-spapr-hotplug.txt
index 46e07196b..631b0cada 100644
--- a/docs/specs/ppc-spapr-hotplug.txt
+++ b/docs/specs/ppc-spapr-hotplug.txt
@@ -302,4 +302,52 @@ consisting of <phys>, <size> and <maxcpus>.
 pseries guests use this property to note the maximum allowed CPUs for the
 guest.
 
+== ibm,dynamic-reconfiguration-memory ==
+
+ibm,dynamic-reconfiguration-memory is a device tree node that represents
+dynamically reconfigurable logical memory blocks (LMB). This node
+is generated only when the guest advertises the support for it via
+ibm,client-architecture-support call. Memory that is not dynamically
+reconfigurable is represented by /memory nodes. The properties of this
+node that are of interest to the sPAPR memory hotplug implementation
+in QEMU are described here.
+
+ibm,lmb-size
+
+This 64bit integer defines the size of each dynamically reconfigurable LMB.
+
+ibm,associativity-lookup-arrays
+
+This property defines a lookup array in which the NUMA associativity
+information for each LMB can be found. It is a property encoded array
+that begins with an integer M, the number of associativity lists followed
+by an integer N, the number of entries per associativity list and terminated
+by M associativity lists each of length N integers.
+
+This property provides the same information as given by ibm,associativity
+property in a /memory node. Each assigned LMB has an index value between
+0 and M-1 which is used as an index into this table to select which
+associativity list to use for the LMB. This index value for each LMB
+is defined in ibm,dynamic-memory property.
+
+ibm,dynamic-memory
+
+This property describes the dynamically reconfigurable memory. It is a
+property encoded array that has an integer N, the number of LMBs followed
+by N LMB list entires.
+
+Each LMB list entry consists of the following elements:
+
+- Logical address of the start of the LMB encoded as a 64bit integer. This
+  corresponds to reg property in /memory node.
+- DRC index of the LMB that corresponds to ibm,my-drc-index property
+  in a /memory node.
+- Four bytes reserved for expansion.
+- Associativity list index for the LMB that is used as an index into
+  ibm,associativity-lookup-arrays property described earlier. This
+  is used to retrieve the right associativity list to be used for this
+  LMB.
+- A 32bit flags word. The bit at bit position 0x00000008 defines whether
+  the LMB is assigned to the the partition as of boot time.
+
 [1] http://thread.gmane.org/gmane.linux.ports.ppc.embedded/75350/focus=106867
diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt
index 121dfc8cc..f236d8c6d 100644
--- a/docs/specs/qcow2.txt
+++ b/docs/specs/qcow2.txt
@@ -257,7 +257,7 @@ L2 table entry:
 
               63:   0 for a cluster that is unused or requires COW, 1 if its
                     refcount is exactly one. This information is only accurate
-                    in L2 tables that are reachable from the the active L1
+                    in L2 tables that are reachable from the active L1
                     table.
 
 Standard Cluster Descriptor:
diff --git a/docs/specs/rocker.txt b/docs/specs/rocker.txt
index 1c743515c..d2a82624f 100644
--- a/docs/specs/rocker.txt
+++ b/docs/specs/rocker.txt
@@ -297,7 +297,7 @@ but not fired.  If only partial credits are returned, the interrupt remains
 masked but the device generates an interrupt, signaling the driver that more
 outstanding work is available.
 
-(* this masking is unrelated to to the MSI-X interrupt mask register)
+(* this masking is unrelated to the MSI-X interrupt mask register)
 
 Endianness
 ----------
diff --git a/docs/specs/vhost-user.txt b/docs/specs/vhost-user.txt
index 650bb1818..0312d40af 100644
--- a/docs/specs/vhost-user.txt
+++ b/docs/specs/vhost-user.txt
@@ -87,6 +87,14 @@ Depending on the request type, payload can be:
    User address: a 64-bit user address
    mmap offset: 64-bit offset where region starts in the mapped memory
 
+* Log description
+   ---------------------------
+   | log size | log offset |
+   ---------------------------
+   log size: size of area used for logging
+   log offset: offset from start of supplied file descriptor
+       where logging starts (i.e. where guest address 0 would be logged)
+
 In QEMU the vhost-user message is implemented with the following struct:
 
 typedef struct VhostUserMsg {
@@ -98,6 +106,7 @@ typedef struct VhostUserMsg {
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
+        VhostUserLog log;
     };
 } QEMU_PACKED VhostUserMsg;
 
@@ -113,12 +122,15 @@ message replies. Most of the requests don't require replies. Here is a list of
 the ones that do:
 
  * VHOST_GET_FEATURES
+ * VHOST_GET_PROTOCOL_FEATURES
  * VHOST_GET_VRING_BASE
+ * VHOST_SET_LOG_BASE (if VHOST_USER_PROTOCOL_F_LOG_SHMFD)
 
 There are several messages that the master sends with file descriptors passed
 in the ancillary data:
 
  * VHOST_SET_MEM_TABLE
+ * VHOST_SET_LOG_BASE (if VHOST_USER_PROTOCOL_F_LOG_SHMFD)
  * VHOST_SET_LOG_FD
  * VHOST_SET_VRING_KICK
  * VHOST_SET_VRING_CALL
@@ -127,6 +139,122 @@ in the ancillary data:
 If Master is unable to send the full message or receives a wrong reply it will
 close the connection. An optional reconnection mechanism can be implemented.
 
+Any protocol extensions are gated by protocol feature bits,
+which allows full backwards compatibility on both master
+and slave.
+As older slaves don't support negotiating protocol features,
+a feature bit was dedicated for this purpose:
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+
+Starting and stopping rings
+----------------------
+Client must only process each ring when it is started.
+
+Client must only pass data between the ring and the
+backend, when the ring is enabled.
+
+If ring is started but disabled, client must process the
+ring without talking to the backend.
+
+For example, for a networking device, in the disabled state
+client must not supply any new RX packets, but must process
+and discard any TX packets.
+
+If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated, the ring is initialized
+in an enabled state.
+
+If VHOST_USER_F_PROTOCOL_FEATURES has been negotiated, the ring is initialized
+in a disabled state. Client must not pass data to/from the backend until ring is enabled by
+VHOST_USER_SET_VRING_ENABLE with parameter 1, or after it has been disabled by
+VHOST_USER_SET_VRING_ENABLE with parameter 0.
+
+Each ring is initialized in a stopped state, client must not process it until
+ring is started, or after it has been stopped.
+
+Client must start ring upon receiving a kick (that is, detecting that file
+descriptor is readable) on the descriptor specified by
+VHOST_USER_SET_VRING_KICK, and stop ring upon receiving
+VHOST_USER_GET_VRING_BASE.
+
+While processing the rings (whether they are enabled or not), client must
+support changing some configuration aspects on the fly.
+
+Multiple queue support
+----------------------
+
+Multiple queue is treated as a protocol extension, hence the slave has to
+implement protocol features first. The multiple queues feature is supported
+only when the protocol feature VHOST_USER_PROTOCOL_F_MQ (bit 0) is set.
+
+The max number of queues the slave supports can be queried with message
+VHOST_USER_GET_PROTOCOL_FEATURES. Master should stop when the number of
+requested queues is bigger than that.
+
+As all queues share one connection, the master uses a unique index for each
+queue in the sent message to identify a specified queue. One queue pair
+is enabled initially. More queues are enabled dynamically, by sending
+message VHOST_USER_SET_VRING_ENABLE.
+
+Migration
+---------
+
+During live migration, the master may need to track the modifications
+the slave makes to the memory mapped regions. The client should mark
+the dirty pages in a log. Once it complies to this logging, it may
+declare the VHOST_F_LOG_ALL vhost feature.
+
+To start/stop logging of data/used ring writes, server may send messages
+VHOST_USER_SET_FEATURES with VHOST_F_LOG_ALL and VHOST_USER_SET_VRING_ADDR with
+VHOST_VRING_F_LOG in ring's flags set to 1/0, respectively.
+
+All the modifications to memory pointed by vring "descriptor" should
+be marked. Modifications to "used" vring should be marked if
+VHOST_VRING_F_LOG is part of ring's flags.
+
+Dirty pages are of size:
+#define VHOST_LOG_PAGE 0x1000
+
+The log memory fd is provided in the ancillary data of
+VHOST_USER_SET_LOG_BASE message when the slave has
+VHOST_USER_PROTOCOL_F_LOG_SHMFD protocol feature.
+
+The size of the log is supplied as part of VhostUserMsg
+which should be large enough to cover all known guest
+addresses. Log starts at the supplied offset in the
+supplied file descriptor.
+The log covers from address 0 to the maximum of guest
+regions. In pseudo-code, to mark page at "addr" as dirty:
+
+page = addr / VHOST_LOG_PAGE
+log[page / 8] |= 1 << page % 8
+
+Where addr is the guest physical address.
+
+Use atomic operations, as the log may be concurrently manipulated.
+
+Note that when logging modifications to the used ring (when VHOST_VRING_F_LOG
+is set for this ring), log_guest_addr should be used to calculate the log
+offset: the write to first byte of the used ring is logged at this offset from
+log start. Also note that this value might be outside the legal guest physical
+address range (i.e. does not have to be covered by the VhostUserMemory table),
+but the bit offset of the last byte of the ring must fall within
+the size supplied by VhostUserLog.
+
+VHOST_USER_SET_LOG_FD is an optional message with an eventfd in
+ancillary data, it may be used to inform the master that the log has
+been modified.
+
+Once the source has finished migration, rings will be stopped by
+the source. No further update must be done before rings are
+restarted.
+
+Protocol features
+-----------------
+
+#define VHOST_USER_PROTOCOL_F_MQ             0
+#define VHOST_USER_PROTOCOL_F_LOG_SHMFD      1
+#define VHOST_USER_PROTOCOL_F_RARP           2
+
 Message types
 -------------
 
@@ -138,6 +266,8 @@ Message types
       Slave payload: u64
 
       Get from the underlying vhost implementation the features bitmask.
+      Feature bit VHOST_USER_F_PROTOCOL_FEATURES signals slave support for
+      VHOST_USER_GET_PROTOCOL_FEATURES and VHOST_USER_SET_PROTOCOL_FEATURES.
 
  * VHOST_USER_SET_FEATURES
 
@@ -146,6 +276,33 @@ Message types
       Master payload: u64
 
       Enable features in the underlying vhost implementation using a bitmask.
+      Feature bit VHOST_USER_F_PROTOCOL_FEATURES signals slave support for
+      VHOST_USER_GET_PROTOCOL_FEATURES and VHOST_USER_SET_PROTOCOL_FEATURES.
+
+ * VHOST_USER_GET_PROTOCOL_FEATURES
+
+      Id: 15
+      Equivalent ioctl: VHOST_GET_FEATURES
+      Master payload: N/A
+      Slave payload: u64
+
+      Get the protocol feature bitmask from the underlying vhost implementation.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES.
+      Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must support
+      this message even before VHOST_USER_SET_FEATURES was called.
+
+ * VHOST_USER_SET_PROTOCOL_FEATURES
+
+      Id: 16
+      Ioctl: VHOST_SET_FEATURES
+      Master payload: u64
+
+      Enable protocol features in the underlying vhost implementation.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES.
+      Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must support
+      this message even before VHOST_USER_SET_FEATURES was called.
 
  * VHOST_USER_SET_OWNER
 
@@ -160,11 +317,13 @@ Message types
  * VHOST_USER_RESET_OWNER
 
       Id: 4
-      Equivalent ioctl: VHOST_RESET_OWNER
       Master payload: N/A
 
-      Issued when a new connection is about to be closed. The Master will no
-      longer own this connection (and will usually close it).
+      This is no longer used. Used to be sent to request disabling
+      all rings, but some clients interpreted it to also discard
+      connection state (this interpretation would lead to bugs).
+      It is recommended that clients either ignore this message,
+      or use it to disable all rings.
 
  * VHOST_USER_SET_MEM_TABLE
 
@@ -182,8 +341,14 @@ Message types
       Id: 6
       Equivalent ioctl: VHOST_SET_LOG_BASE
       Master payload: u64
+      Slave payload: N/A
+
+      Sets logging shared memory space.
+      When slave has VHOST_USER_PROTOCOL_F_LOG_SHMFD protocol
+      feature, the log memory fd is provided in the ancillary data of
+      VHOST_USER_SET_LOG_BASE message, the size and offset of shared
+      memory area provided in the message.
 
-      Sets the logging base address.
 
  * VHOST_USER_SET_LOG_FD
 
@@ -264,3 +429,38 @@ Message types
       Bits (0-7) of the payload contain the vring index. Bit 8 is the
       invalid FD flag. This flag is set when there is no file descriptor
       in the ancillary data.
+
+ * VHOST_USER_GET_QUEUE_NUM
+
+      Id: 17
+      Equivalent ioctl: N/A
+      Master payload: N/A
+      Slave payload: u64
+
+      Query how many queues the backend supports. This request should be
+      sent only when VHOST_USER_PROTOCOL_F_MQ is set in quried protocol
+      features by VHOST_USER_GET_PROTOCOL_FEATURES.
+
+ * VHOST_USER_SET_VRING_ENABLE
+
+      Id: 18
+      Equivalent ioctl: N/A
+      Master payload: vring state description
+
+      Signal slave to enable or disable corresponding vring.
+      This request should be sent only when VHOST_USER_F_PROTOCOL_FEATURES
+      has been negotiated.
+
+ * VHOST_USER_SEND_RARP
+
+      Id: 19
+      Equivalent ioctl: N/A
+      Master payload: u64
+
+      Ask vhost user backend to broadcast a fake RARP to notify the migration
+      is terminated for guest that does not support GUEST_ANNOUNCE.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES and protocol feature bit VHOST_USER_PROTOCOL_F_RARP
+      is present in VHOST_USER_GET_PROTOCOL_FEATURES.
+      The first 6 bytes of the payload contain the mac address of the guest to
+      allow the vhost user backend to construct and broadcast the fake RARP.
diff --git a/docs/tracing.txt b/docs/tracing.txt
index 7117c5e7d..3853a6ad8 100644
--- a/docs/tracing.txt
+++ b/docs/tracing.txt
@@ -258,11 +258,11 @@ is generated to make use in scripts more convenient.  This step can also be
 performed manually after a build in order to change the binary name in the .stp
 probes:
 
-    scripts/tracetool --dtrace --stap \
-                      --binary path/to/qemu-binary \
-                      --target-type system \
-                      --target-name x86_64 \
-                      <trace-events >qemu.stp
+    scripts/tracetool.py --backends=dtrace --format=stap \
+                         --binary path/to/qemu-binary \
+                         --target-type system \
+                         --target-name x86_64 \
+                         <trace-events >qemu.stp
 
 == Trace event properties ==
 
diff --git a/docs/virtio-migration.txt b/docs/virtio-migration.txt
new file mode 100644
index 000000000..cf66458b9
--- /dev/null
+++ b/docs/virtio-migration.txt
@@ -0,0 +1,106 @@
+Virtio devices and migration
+============================
+
+Copyright 2015 IBM Corp.
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.  See
+the COPYING file in the top-level directory.
+
+Saving and restoring the state of virtio devices is a bit of a twisty maze,
+for several reasons:
+- state is distributed between several parts:
+  - virtio core, for common fields like features, number of queues, ...
+  - virtio transport (pci, ccw, ...), for the different proxy devices and
+    transport specific state (msix vectors, indicators, ...)
+  - virtio device (net, blk, ...), for the different device types and their
+    state (mac address, request queue, ...)
+- most fields are saved via the stream interface; subsequently, subsections
+  have been added to make cross-version migration possible
+
+This file attempts to document the current procedure and point out some
+caveats.
+
+
+Save state procedure
+====================
+
+virtio core               virtio transport          virtio device
+-----------               ----------------          -------------
+
+                                                    save() function registered
+                                                    via register_savevm()
+virtio_save()                                       <----------
+             ------>      save_config()
+                          - save proxy device
+                          - save transport-specific
+                            device fields
+- save common device
+  fields
+- save common virtqueue
+  fields
+             ------>      save_queue()
+                          - save transport-specific
+                            virtqueue fields
+             ------>                               save_device()
+                                                   - save device-specific
+                                                     fields
+- save subsections
+  - device endianness,
+    if changed from
+    default endianness
+  - 64 bit features, if
+    any high feature bit
+    is set
+  - virtio-1 virtqueue
+    fields, if VERSION_1
+    is set
+
+
+Load state procedure
+====================
+
+virtio core               virtio transport          virtio device
+-----------               ----------------          -------------
+
+                                                    load() function registered
+                                                    via register_savevm()
+virtio_load()                                       <----------
+             ------>      load_config()
+                          - load proxy device
+                          - load transport-specific
+                            device fields
+- load common device
+  fields
+- load common virtqueue
+  fields
+             ------>      load_queue()
+                          - load transport-specific
+                            virtqueue fields
+- notify guest
+             ------>                               load_device()
+                                                   - load device-specific
+                                                     fields
+- load subsections
+  - device endianness
+  - 64 bit features
+  - virtio-1 virtqueue
+    fields
+- sanitize endianness
+- sanitize features
+- virtqueue index sanity
+  check
+                                                   - feature-dependent setup
+
+
+Implications of this setup
+==========================
+
+Devices need to be careful in their state processing during load: The
+load_device() procedure is invoked by the core before subsections have
+been loaded. Any code that depends on information transmitted in subsections
+therefore has to be invoked in the device's load() function _after_
+virtio_load() returned (like e.g. code depending on features).
+
+Any extension of the state being migrated should be done in subsections
+added to the core for compatibility reasons. If transport or device specific
+state is added, core needs to invoke a callback from the new subsection.
diff --git a/docs/win32-qemu-event.promela b/docs/win32-qemu-event.promela
new file mode 100644
index 000000000..c446a7155
--- /dev/null
+++ b/docs/win32-qemu-event.promela
@@ -0,0 +1,98 @@
+/*
+ * This model describes the implementation of QemuEvent in
+ * util/qemu-thread-win32.c.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This file is in the public domain.  If you really want a license,
+ * the WTFPL will do.
+ *
+ * To verify it:
+ *     spin -a docs/event.promela
+ *     gcc -O2 pan.c -DSAFETY
+ *     ./a.out
+ */
+
+bool event;
+int value;
+
+/* Primitives for a Win32 event */
+#define RAW_RESET event = false
+#define RAW_SET   event = true
+#define RAW_WAIT  do :: event -> break; od
+
+#if 0
+/* Basic sanity checking: test the Win32 event primitives */
+#define RESET     RAW_RESET
+#define SET       RAW_SET
+#define WAIT      RAW_WAIT
+#else
+/* Full model: layer a userspace-only fast path on top of the RAW_*
+ * primitives.  SET/RESET/WAIT have exactly the same semantics as
+ * RAW_SET/RAW_RESET/RAW_WAIT, but try to avoid invoking them.
+ */
+#define EV_SET 0
+#define EV_FREE 1
+#define EV_BUSY -1
+
+int state = EV_FREE;
+
+int xchg_result;
+#define SET   if :: state != EV_SET ->                                      \
+                    atomic { /* xchg_result=xchg(state, EV_SET) */          \
+                        xchg_result = state;                                \
+                        state = EV_SET;                                     \
+                    }                                                       \
+                    if :: xchg_result == EV_BUSY -> RAW_SET;                \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                 :: else -> skip;                                           \
+              fi
+
+#define RESET if :: state == EV_SET -> atomic { state = state | EV_FREE; }  \
+                 :: else            -> skip;                                \
+              fi
+
+int tmp1, tmp2;
+#define WAIT  tmp1 = state;                                                 \
+              if :: tmp1 != EV_SET ->                                       \
+                    if :: tmp1 == EV_FREE ->                                \
+                          RAW_RESET;                                        \
+                          atomic { /* tmp2=cas(state, EV_FREE, EV_BUSY) */  \
+                              tmp2 = state;                                 \
+                              if :: tmp2 == EV_FREE -> state = EV_BUSY;     \
+                                 :: else            -> skip;                \
+                              fi;                                           \
+                          }                                                 \
+                          if :: tmp2 == EV_SET -> tmp1 = EV_SET;            \
+                             :: else           -> tmp1 = EV_BUSY;           \
+                          fi;                                               \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                    assert(tmp1 != EV_FREE);                                \
+                    if :: tmp1 == EV_BUSY -> RAW_WAIT;                      \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                 :: else -> skip;                                           \
+              fi
+#endif
+
+active proctype waiter()
+{
+     if
+         :: !value ->
+             RESET;
+             if
+                 :: !value -> WAIT;
+                 :: else   -> skip;
+             fi;
+        :: else -> skip;
+    fi;
+    assert(value);
+}
+
+active proctype notifier()
+{
+    value = true;
+    SET;
+}
diff --git a/docs/writing-qmp-commands.txt b/docs/writing-qmp-commands.txt
index ab1fdd36b..59aa77ae2 100644
--- a/docs/writing-qmp-commands.txt
+++ b/docs/writing-qmp-commands.txt
@@ -122,12 +122,12 @@ There are a few things to be noticed:
 Now a little hack is needed. As we're still using the old QMP server we need
 to add the new command to its internal dispatch table. This step won't be
 required in the near future. Open the qmp-commands.hx file and add the
-following in the botton:
+following at the bottom:
 
     {
         .name       = "hello-world",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_hello_world,
+        .mhandler.cmd_new = qmp_marshal_hello_world,
     },
 
 You're done. Now build qemu, run it as suggested in the "Testing" section,
@@ -179,7 +179,7 @@ The last step is to update the qmp-commands.hx file:
     {
         .name       = "hello-world",
         .args_type  = "message:s?",
-        .mhandler.cmd_new = qmp_marshal_input_hello_world,
+        .mhandler.cmd_new = qmp_marshal_hello_world,
     },
 
 Notice that the "args_type" member got our "message" argument. The character
@@ -210,7 +210,7 @@ if you don't see these strings, then something went wrong.
 === Errors ===
 
 QMP commands should use the error interface exported by the error.h header
-file. Basically, errors are set by calling the error_set() function.
+file. Basically, most errors are set by calling the error_setg() function.
 
 Let's say we don't accept the string "message" to contain the word "love". If
 it does contain it, we want the "hello-world" command to return an error:
@@ -219,8 +219,7 @@ void qmp_hello_world(bool has_message, const char *message, Error **errp)
 {
     if (has_message) {
         if (strstr(message, "love")) {
-            error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                      "the word 'love' is not allowed");
+            error_setg(errp, "the word 'love' is not allowed");
             return;
         }
         printf("%s\n", message);
@@ -229,10 +228,8 @@ void qmp_hello_world(bool has_message, const char *message, Error **errp)
     }
 }
 
-The first argument to the error_set() function is the Error pointer to pointer,
-which is passed to all QMP functions. The second argument is a ErrorClass
-value, which should be ERROR_CLASS_GENERIC_ERROR most of the time (more
-details about error classes are given below). The third argument is a human
+The first argument to the error_setg() function is the Error pointer
+to pointer, which is passed to all QMP functions. The next argument is a human
 description of the error, this is a free-form printf-like string.
 
 Let's test the example above. Build qemu, run it as defined in the "Testing"
@@ -249,8 +246,9 @@ The QMP server's response should be:
     }
 }
 
-As a general rule, all QMP errors should use ERROR_CLASS_GENERIC_ERROR. There
-are two exceptions to this rule:
+As a general rule, all QMP errors should use ERROR_CLASS_GENERIC_ERROR
+(done by default when using error_setg()). There are two exceptions to
+this rule:
 
  1. A non-generic ErrorClass value exists* for the failure you want to report
     (eg. DeviceNotFound)
@@ -259,8 +257,8 @@ are two exceptions to this rule:
     want to report, hence you have to add a new ErrorClass value so that they
     can check for it
 
-If the failure you want to report doesn't fall in one of the two cases above,
-just report ERROR_CLASS_GENERIC_ERROR.
+If the failure you want to report falls into one of the two cases above,
+use error_set() with a second argument of an ErrorClass value.
 
  * All existing ErrorClass values are defined in the qapi-schema.json file
 
@@ -461,7 +459,7 @@ The last step is to add the correspoding entry in the qmp-commands.hx file:
     {
         .name       = "query-alarm-clock",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_alarm_clock,
+        .mhandler.cmd_new = qmp_marshal_query_alarm_clock,
     },
 
 Time to test the new command. Build qemu, run it as described in the "Testing"
@@ -607,7 +605,7 @@ To test this you have to add the corresponding qmp-commands.hx entry:
     {
         .name       = "query-alarm-methods",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_alarm_methods,
+        .mhandler.cmd_new = qmp_marshal_query_alarm_methods,
     },
 
 Now Build qemu, run it as explained in the "Testing" section and try our new
diff --git a/exec.c b/exec.c
index 0a4a0c5af..0bf0a6e7e 100644
--- a/exec.c
+++ b/exec.c
@@ -49,13 +49,16 @@
 #include "exec/cpu-all.h"
 #include "qemu/rcu_queue.h"
 #include "qemu/main-loop.h"
-#include "exec/cputlb.h"
 #include "translate-all.h"
+#include "sysemu/replay.h"
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
 
 #include "qemu/range.h"
+#ifndef _WIN32
+#include "qemu/mmap-alloc.h"
+#endif
 
 //#define DEBUG_SUBPAGE
 
@@ -85,12 +88,15 @@ static MemoryRegion io_mem_unassigned;
  */
 #define RAM_RESIZEABLE (1 << 2)
 
+/* RAM is backed by an mmapped file.
+ */
+#define RAM_FILE (1 << 3)
 #endif
 
 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
 /* current CPU in the current thread. It is only valid inside
    cpu_exec() */
-DEFINE_TLS(CPUState *, current_cpu);
+__thread CPUState *current_cpu;
 /* 0 = Do not count executed instructions.
    1 = Precise instruction counting.
    2 = Adaptive rate instruction counting.  */
@@ -159,6 +165,21 @@ static void memory_map_init(void);
 static void tcg_commit(MemoryListener *listener);
 
 static MemoryRegion io_mem_watch;
+
+/**
+ * CPUAddressSpace: all the information a CPU needs about an AddressSpace
+ * @cpu: the CPU whose AddressSpace this is
+ * @as: the AddressSpace itself
+ * @memory_dispatch: its dispatch pointer (cached, RCU protected)
+ * @tcg_as_listener: listener for tracking changes to the AddressSpace
+ */
+struct CPUAddressSpace {
+    CPUState *cpu;
+    AddressSpace *as;
+    struct AddressSpaceDispatch *memory_dispatch;
+    MemoryListener tcg_as_listener;
+};
+
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
@@ -429,7 +450,7 @@ address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
                                   hwaddr *xlat, hwaddr *plen)
 {
     MemoryRegionSection *section;
-    section = address_space_translate_internal(cpu->memory_dispatch,
+    section = address_space_translate_internal(cpu->cpu_ases[0].memory_dispatch,
                                                addr, xlat, plen, false);
 
     assert(!section->mr->iommu_ops);
@@ -478,6 +499,24 @@ static const VMStateDescription vmstate_cpu_common_exception_index = {
     }
 };
 
+static bool cpu_common_crash_occurred_needed(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    return cpu->crash_occurred;
+}
+
+static const VMStateDescription vmstate_cpu_common_crash_occurred = {
+    .name = "cpu_common/crash_occurred",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = cpu_common_crash_occurred_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(crash_occurred, CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_cpu_common = {
     .name = "cpu_common",
     .version_id = 1,
@@ -491,6 +530,7 @@ const VMStateDescription vmstate_cpu_common = {
     },
     .subsections = (const VMStateDescription*[]) {
         &vmstate_cpu_common_exception_index,
+        &vmstate_cpu_common_crash_occurred,
         NULL
     }
 };
@@ -516,13 +556,16 @@ void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
     /* We only support one address space per cpu at the moment.  */
     assert(cpu->as == as);
 
-    if (cpu->tcg_as_listener) {
-        memory_listener_unregister(cpu->tcg_as_listener);
-    } else {
-        cpu->tcg_as_listener = g_new0(MemoryListener, 1);
+    if (cpu->cpu_ases) {
+        /* We've already registered the listener for our only AS */
+        return;
     }
-    cpu->tcg_as_listener->commit = tcg_commit;
-    memory_listener_register(cpu->tcg_as_listener, as);
+
+    cpu->cpu_ases = g_new0(CPUAddressSpace, 1);
+    cpu->cpu_ases[0].cpu = cpu;
+    cpu->cpu_ases[0].as = as;
+    cpu->cpu_ases[0].tcg_as_listener.commit = tcg_commit;
+    memory_listener_register(&cpu->cpu_ases[0].tcg_as_listener, as);
 }
 #endif
 
@@ -580,7 +623,6 @@ void cpu_exec_init(CPUState *cpu, Error **errp)
 #ifndef CONFIG_USER_ONLY
     cpu->as = &address_space_memory;
     cpu->thread_id = qemu_get_thread_id();
-    cpu_reload_memory_map(cpu);
 #endif
 
 #if defined(CONFIG_USER_ONLY)
@@ -841,6 +883,7 @@ void cpu_abort(CPUState *cpu, const char *fmt, ...)
     }
     va_end(ap2);
     va_end(ap);
+    replay_finish();
 #if defined(CONFIG_USER_ONLY)
     {
         struct sigaction act;
@@ -860,7 +903,7 @@ static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
 
     block = atomic_rcu_read(&ram_list.mru_block);
     if (block && addr - block->offset < block->max_length) {
-        goto found;
+        return block;
     }
     QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
         if (addr - block->offset < block->max_length) {
@@ -894,6 +937,7 @@ found:
 
 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
 {
+    CPUState *cpu;
     ram_addr_t start1;
     RAMBlock *block;
     ram_addr_t end;
@@ -905,7 +949,9 @@ static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
     block = qemu_get_ram_block(start);
     assert(block == qemu_get_ram_block(end - 1));
     start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
-    cpu_tlb_reset_dirty_all(start1, length);
+    CPU_FOREACH(cpu) {
+        tlb_reset_dirty(cpu, start1, length);
+    }
     rcu_read_unlock();
 }
 
@@ -1018,9 +1064,11 @@ static uint16_t phys_section_add(PhysPageMap *map,
 
 static void phys_section_destroy(MemoryRegion *mr)
 {
+    bool have_sub_page = mr->subpage;
+
     memory_region_unref(mr);
 
-    if (mr->subpage) {
+    if (have_sub_page) {
         subpage_t *subpage = container_of(mr, subpage_t, iomem);
         object_unref(OBJECT(&subpage->iomem));
         g_free(subpage);
@@ -1150,9 +1198,6 @@ static long gethugepagesize(const char *path, Error **errp)
         return 0;
     }
 
-    if (fs.f_type != HUGETLBFS_MAGIC)
-        fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
-
     return fs.f_bsize;
 }
 
@@ -1161,10 +1206,11 @@ static void *file_ram_alloc(RAMBlock *block,
                             const char *path,
                             Error **errp)
 {
+    struct stat st;
     char *filename;
     char *sanitized_name;
     char *c;
-    void *area = NULL;
+    void *area;
     int fd;
     uint64_t hpagesize;
     Error *local_err = NULL;
@@ -1189,28 +1235,35 @@ static void *file_ram_alloc(RAMBlock *block,
         goto error;
     }
 
-    /* Make name safe to use with mkstemp by replacing '/' with '_'. */
-    sanitized_name = g_strdup(memory_region_name(block->mr));
-    for (c = sanitized_name; *c != '\0'; c++) {
-        if (*c == '/')
-            *c = '_';
-    }
+    if (!stat(path, &st) && S_ISDIR(st.st_mode)) {
+        /* Make name safe to use with mkstemp by replacing '/' with '_'. */
+        sanitized_name = g_strdup(memory_region_name(block->mr));
+        for (c = sanitized_name; *c != '\0'; c++) {
+            if (*c == '/') {
+                *c = '_';
+            }
+        }
+
+        filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
+                                   sanitized_name);
+        g_free(sanitized_name);
 
-    filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
-                               sanitized_name);
-    g_free(sanitized_name);
+        fd = mkstemp(filename);
+        if (fd >= 0) {
+            unlink(filename);
+        }
+        g_free(filename);
+    } else {
+        fd = open(path, O_RDWR | O_CREAT, 0644);
+    }
 
-    fd = mkstemp(filename);
     if (fd < 0) {
         error_setg_errno(errp, errno,
                          "unable to create backing store for hugepages");
-        g_free(filename);
         goto error;
     }
-    unlink(filename);
-    g_free(filename);
 
-    memory = (memory+hpagesize-1) & ~(hpagesize-1);
+    memory = ROUND_UP(memory, hpagesize);
 
     /*
      * ftruncate is not supported by hugetlbfs in older
@@ -1222,9 +1275,7 @@ static void *file_ram_alloc(RAMBlock *block,
         perror("ftruncate");
     }
 
-    area = mmap(0, memory, PROT_READ | PROT_WRITE,
-                (block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE),
-                fd, 0);
+    area = qemu_ram_mmap(fd, memory, hpagesize, block->flags & RAM_SHARED);
     if (area == MAP_FAILED) {
         error_setg_errno(errp, errno,
                          "unable to map backing store for hugepages");
@@ -1240,10 +1291,6 @@ static void *file_ram_alloc(RAMBlock *block,
     return area;
 
 error:
-    if (mem_prealloc) {
-        error_report("%s", error_get_pretty(*errp));
-        exit(1);
-    }
     return NULL;
 }
 #endif
@@ -1329,6 +1376,11 @@ static RAMBlock *find_ram_block(ram_addr_t addr)
     return NULL;
 }
 
+const char *qemu_ram_get_idstr(RAMBlock *rb)
+{
+    return rb->idstr;
+}
+
 /* Called with iothread lock held.  */
 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
 {
@@ -1399,7 +1451,7 @@ int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
 
     assert(block);
 
-    newsize = TARGET_PAGE_ALIGN(newsize);
+    newsize = HOST_PAGE_ALIGN(newsize);
 
     if (block->used_length == newsize) {
         return 0;
@@ -1543,12 +1595,13 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
         return -1;
     }
 
-    size = TARGET_PAGE_ALIGN(size);
+    size = HOST_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
     new_block->used_length = size;
     new_block->max_length = size;
     new_block->flags = share ? RAM_SHARED : 0;
+    new_block->flags |= RAM_FILE;
     new_block->host = file_ram_alloc(new_block, size,
                                      mem_path, errp);
     if (!new_block->host) {
@@ -1578,8 +1631,8 @@ ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
     ram_addr_t addr;
     Error *local_err = NULL;
 
-    size = TARGET_PAGE_ALIGN(size);
-    max_size = TARGET_PAGE_ALIGN(max_size);
+    size = HOST_PAGE_ALIGN(size);
+    max_size = HOST_PAGE_ALIGN(max_size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
     new_block->resized = resized;
@@ -1650,7 +1703,11 @@ static void reclaim_ramblock(RAMBlock *block)
         xen_invalidate_map_cache_entry(block->host);
 #ifndef _WIN32
     } else if (block->fd >= 0) {
-        munmap(block->host, block->max_length);
+        if (block->flags & RAM_FILE) {
+            qemu_ram_munmap(block->host, block->max_length);
+        } else {
+            munmap(block->host, block->max_length);
+        }
         close(block->fd);
 #endif
     } else {
@@ -1824,8 +1881,16 @@ static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
     }
 }
 
-/* Some of the softmmu routines need to translate from a host pointer
- * (typically a TLB entry) back to a ram offset.
+/*
+ * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
+ * in that RAMBlock.
+ *
+ * ptr: Host pointer to look up
+ * round_offset: If true round the result offset down to a page boundary
+ * *ram_addr: set to result ram_addr
+ * *offset: set to result offset within the RAMBlock
+ *
+ * Returns: RAMBlock (or NULL if not found)
  *
  * By the time this function returns, the returned pointer is not protected
  * by RCU anymore.  If the caller is not within an RCU critical section and
@@ -1833,18 +1898,22 @@ static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
  * pointer, such as a reference to the region that includes the incoming
  * ram_addr_t.
  */
-MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
+RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
+                                   ram_addr_t *ram_addr,
+                                   ram_addr_t *offset)
 {
     RAMBlock *block;
     uint8_t *host = ptr;
-    MemoryRegion *mr;
 
     if (xen_enabled()) {
         rcu_read_lock();
         *ram_addr = xen_ram_addr_from_mapcache(ptr);
-        mr = qemu_get_ram_block(*ram_addr)->mr;
+        block = qemu_get_ram_block(*ram_addr);
+        if (block) {
+            *offset = (host - block->host);
+        }
         rcu_read_unlock();
-        return mr;
+        return block;
     }
 
     rcu_read_lock();
@@ -1867,10 +1936,49 @@ MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
     return NULL;
 
 found:
-    *ram_addr = block->offset + (host - block->host);
-    mr = block->mr;
+    *offset = (host - block->host);
+    if (round_offset) {
+        *offset &= TARGET_PAGE_MASK;
+    }
+    *ram_addr = block->offset + *offset;
     rcu_read_unlock();
-    return mr;
+    return block;
+}
+
+/*
+ * Finds the named RAMBlock
+ *
+ * name: The name of RAMBlock to find
+ *
+ * Returns: RAMBlock (or NULL if not found)
+ */
+RAMBlock *qemu_ram_block_by_name(const char *name)
+{
+    RAMBlock *block;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (!strcmp(name, block->idstr)) {
+            return block;
+        }
+    }
+
+    return NULL;
+}
+
+/* Some of the softmmu routines need to translate from a host pointer
+   (typically a TLB entry) back to a ram offset.  */
+MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
+{
+    RAMBlock *block;
+    ram_addr_t offset; /* Not used */
+
+    block = qemu_ram_block_from_host(ptr, false, ram_addr, &offset);
+
+    if (!block) {
+        return NULL;
+    }
+
+    return block->mr;
 }
 
 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
@@ -1900,8 +2008,7 @@ static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
     /* we remove the notdirty callback only if the code has been
        flushed */
     if (!cpu_physical_memory_is_clean(ram_addr)) {
-        CPUArchState *env = current_cpu->env_ptr;
-        tlb_set_dirty(env, current_cpu->mem_io_vaddr);
+        tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
     }
 }
 
@@ -2163,7 +2270,8 @@ static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
 
 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
 {
-    AddressSpaceDispatch *d = atomic_rcu_read(&cpu->memory_dispatch);
+    CPUAddressSpace *cpuas = &cpu->cpu_ases[0];
+    AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
     MemoryRegionSection *sections = d->map.sections;
 
     return sections[index & ~TARGET_PAGE_MASK].mr;
@@ -2222,19 +2330,20 @@ static void mem_commit(MemoryListener *listener)
 
 static void tcg_commit(MemoryListener *listener)
 {
-    CPUState *cpu;
+    CPUAddressSpace *cpuas;
+    AddressSpaceDispatch *d;
 
     /* since each CPU stores ram addresses in its TLB cache, we must
        reset the modified entries */
-    /* XXX: slow ! */
-    CPU_FOREACH(cpu) {
-        /* FIXME: Disentangle the cpu.h circular files deps so we can
-           directly get the right CPU from listener.  */
-        if (cpu->tcg_as_listener != listener) {
-            continue;
-        }
-        cpu_reload_memory_map(cpu);
-    }
+    cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
+    cpu_reloading_memory_map();
+    /* The CPU and TLB are protected by the iothread lock.
+     * We reload the dispatch pointer now because cpu_reloading_memory_map()
+     * may have split the RCU critical section.
+     */
+    d = atomic_rcu_read(&cpuas->as->dispatch);
+    cpuas->memory_dispatch = d;
+    tlb_flush(cpuas->cpu, 1);
 }
 
 void address_space_init_dispatch(AddressSpace *as)
@@ -2374,9 +2483,7 @@ static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
     if (l > access_size_max) {
         l = access_size_max;
     }
-    if (l & (l - 1)) {
-        l = 1 << (qemu_fls(l) - 1);
-    }
+    l = pow2floor(l);
 
     return l;
 }
@@ -2652,8 +2759,8 @@ void cpu_register_map_client(QEMUBH *bh)
 void cpu_exec_init_all(void)
 {
     qemu_mutex_init(&ram_list.mutex);
-    memory_map_init();
     io_mem_init();
+    memory_map_init();
     qemu_mutex_init(&map_client_list_lock);
 }
 
@@ -3450,6 +3557,16 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
     }
     return 0;
 }
+
+/*
+ * Allows code that needs to deal with migration bitmaps etc to still be built
+ * target independent.
+ */
+size_t qemu_target_page_bits(void)
+{
+    return TARGET_PAGE_BITS;
+}
+
 #endif
 
 /*
diff --git a/fsdev/virtfs-proxy-helper.c b/fsdev/virtfs-proxy-helper.c
index 9097d15c9..ad1da0d6f 100644
--- a/fsdev/virtfs-proxy-helper.c
+++ b/fsdev/virtfs-proxy-helper.c
@@ -1128,10 +1128,19 @@ int main(int argc, char **argv)
         }
     }
 
+    if (chdir("/") < 0) {
+        do_perror("chdir");
+        goto error;
+    }
+    if (chroot(rpath) < 0) {
+        do_perror("chroot");
+        goto error;
+    }
+
     get_version = false;
 #ifdef FS_IOC_GETVERSION
     /* check whether underlying FS support IOC_GETVERSION */
-    retval = statfs(rpath, &st_fs);
+    retval = statfs("/", &st_fs);
     if (!retval) {
         switch (st_fs.f_type) {
         case EXT2_SUPER_MAGIC:
@@ -1144,16 +1153,7 @@ int main(int argc, char **argv)
     }
 #endif
 
-    if (chdir("/") < 0) {
-        do_perror("chdir");
-        goto error;
-    }
-    if (chroot(rpath) < 0) {
-        do_perror("chroot");
-        goto error;
-    }
     umask(0);
-
     if (init_capabilities() < 0) {
         goto error;
     }
diff --git a/fsdev/virtio-9p-marshal.c b/fsdev/virtio-9p-marshal.c
index 20f308b76..7748d3207 100644
--- a/fsdev/virtio-9p-marshal.c
+++ b/fsdev/virtio-9p-marshal.c
@@ -14,7 +14,6 @@
 #include <glib.h>
 #include <glib/gprintf.h>
 #include <sys/types.h>
-#include <dirent.h>
 #include <sys/time.h>
 #include <utime.h>
 #include <sys/uio.h>
diff --git a/gdb-xml/s390-cr.xml b/gdb-xml/s390-cr.xml
new file mode 100644
index 000000000..5246beaab
--- /dev/null
+++ b/gdb-xml/s390-cr.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!-- Copyright 2015 IBM Corp.
+
+     This work is licensed under the terms of the GNU GPL, version 2 or
+     (at your option) any later version. See the COPYING file in the
+     top-level directory. -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.cr">
+  <reg name="cr0" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr1" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr2" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr3" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr4" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr5" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr6" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr7" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr8" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr9" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr10" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr11" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr12" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr13" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr14" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr15" bitsize="64" type="uint64" group="control"/>
+</feature>
diff --git a/gdb-xml/s390-virt.xml b/gdb-xml/s390-virt.xml
new file mode 100644
index 000000000..e2e9a7ad3
--- /dev/null
+++ b/gdb-xml/s390-virt.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!-- Copyright 2015 IBM Corp.
+
+     This work is licensed under the terms of the GNU GPL, version 2 or
+     (at your option) any later version. See the COPYING file in the
+     top-level directory. -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.virt">
+  <reg name="ckc" bitsize="64" type="uint64" group="system"/>
+  <reg name="cputm" bitsize="64" type="uint64" group="system"/>
+  <reg name="last_break" bitsize="64" type="code_ptr" group="system"/>
+  <reg name="prefix" bitsize="64" type="data_ptr" group="system"/>
+  <reg name="pp" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_token" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_select" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_compare" bitsize="64" type="uint64" group="system"/>
+</feature>
diff --git a/gdbstub.c b/gdbstub.c
index ffe7e6efb..9c29aa0e8 100644
--- a/gdbstub.c
+++ b/gdbstub.c
@@ -956,6 +956,13 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
         if (*p == ',')
             p++;
         len = strtoull(p, NULL, 16);
+
+        /* memtohex() doubles the required space */
+        if (len > MAX_PACKET_LENGTH / 2) {
+            put_packet (s, "E22");
+            break;
+        }
+
         if (target_memory_rw_debug(s->g_cpu, addr, mem_buf, len, false) != 0) {
             put_packet (s, "E14");
         } else {
@@ -970,6 +977,12 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
         len = strtoull(p, (char **)&p, 16);
         if (*p == ':')
             p++;
+
+        /* hextomem() reads 2*len bytes */
+        if (len > strlen(p) / 2) {
+            put_packet (s, "E22");
+            break;
+        }
         hextomem(mem_buf, p, len);
         if (target_memory_rw_debug(s->g_cpu, addr, mem_buf, len,
                                    true) != 0) {
@@ -1107,7 +1120,8 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
             cpu = find_cpu(thread);
             if (cpu != NULL) {
                 cpu_synchronize_state(cpu);
-                len = snprintf((char *)mem_buf, sizeof(mem_buf),
+                /* memtohex() doubles the required space */
+                len = snprintf((char *)mem_buf, sizeof(buf) / 2,
                                "CPU#%d [%s]", cpu->cpu_index,
                                cpu->halted ? "halted " : "running");
                 memtohex(buf, mem_buf, len);
@@ -1136,8 +1150,8 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
                 put_packet(s, "E01");
                 break;
             }
-            hextomem(mem_buf, p + 5, len);
             len = len / 2;
+            hextomem(mem_buf, p + 5, len);
             mem_buf[len++] = 0;
             qemu_chr_be_write(s->mon_chr, mem_buf, len);
             put_packet(s, "OK");
@@ -1301,9 +1315,8 @@ send_packet:
     %x  - target_ulong argument printed in hex.
     %lx - 64-bit argument printed in hex.
     %s  - string pointer (target_ulong) and length (int) pair.  */
-void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
+void gdb_do_syscallv(gdb_syscall_complete_cb cb, const char *fmt, va_list va)
 {
-    va_list va;
     char *p;
     char *p_end;
     target_ulong addr;
@@ -1317,7 +1330,6 @@ void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
 #ifndef CONFIG_USER_ONLY
     vm_stop(RUN_STATE_DEBUG);
 #endif
-    va_start(va, fmt);
     p = s->syscall_buf;
     p_end = &s->syscall_buf[sizeof(s->syscall_buf)];
     *(p++) = 'F';
@@ -1351,7 +1363,6 @@ void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
         }
     }
     *p = 0;
-    va_end(va);
 #ifdef CONFIG_USER_ONLY
     put_packet(s, s->syscall_buf);
     gdb_handlesig(s->c_cpu, 0);
@@ -1362,10 +1373,19 @@ void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
        is still in the running state, which can cause packets to be dropped
        and state transition 'T' packets to be sent while the syscall is still
        being processed.  */
-    cpu_exit(s->c_cpu);
+    qemu_cpu_kick(s->c_cpu);
 #endif
 }
 
+void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
+{
+    va_list va;
+
+    va_start(va, fmt);
+    gdb_do_syscallv(cb, fmt, va);
+    va_end(va);
+}
+
 static void gdb_read_byte(GDBState *s, int ch)
 {
     int i, csum;
diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx
new file mode 100644
index 000000000..9b71351d0
--- /dev/null
+++ b/hmp-commands-info.hx
@@ -0,0 +1,795 @@
+HXCOMM Use DEFHEADING() to define headings in both help text and texi
+HXCOMM Text between STEXI and ETEXI are copied to texi version and
+HXCOMM discarded from C version
+HXCOMM DEF(command, args, callback, arg_string, help) is used to construct
+HXCOMM monitor info commands
+HXCOMM HXCOMM can be used for comments, discarded from both texi and C
+
+STEXI
+@table @option
+@item info @var{subcommand}
+@findex info
+Show various information about the system state.
+@table @option
+ETEXI
+
+    {
+        .name       = "version",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the version of QEMU",
+        .mhandler.cmd = hmp_info_version,
+    },
+
+STEXI
+@item info version
+@findex version
+Show the version of QEMU.
+ETEXI
+
+    {
+        .name       = "network",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the network state",
+        .mhandler.cmd = hmp_info_network,
+    },
+
+STEXI
+@item info network
+@findex network
+Show the network state.
+ETEXI
+
+    {
+        .name       = "chardev",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the character devices",
+        .mhandler.cmd = hmp_info_chardev,
+    },
+
+STEXI
+@item info chardev
+@findex chardev
+Show the character devices.
+ETEXI
+
+    {
+        .name       = "block",
+        .args_type  = "nodes:-n,verbose:-v,device:B?",
+        .params     = "[-n] [-v] [device]",
+        .help       = "show info of one block device or all block devices "
+                      "(-n: show named nodes; -v: show details)",
+        .mhandler.cmd = hmp_info_block,
+    },
+
+STEXI
+@item info block
+@findex block
+Show info of one block device or all block devices.
+ETEXI
+
+    {
+        .name       = "blockstats",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show block device statistics",
+        .mhandler.cmd = hmp_info_blockstats,
+    },
+
+STEXI
+@item info blockstats
+@findex blockstats
+Show block device statistics.
+ETEXI
+
+    {
+        .name       = "block-jobs",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show progress of ongoing block device operations",
+        .mhandler.cmd = hmp_info_block_jobs,
+    },
+
+STEXI
+@item info block-jobs
+@findex block-jobs
+Show progress of ongoing block device operations.
+ETEXI
+
+    {
+        .name       = "registers",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the cpu registers",
+        .mhandler.cmd = hmp_info_registers,
+    },
+
+STEXI
+@item info registers
+@findex registers
+Show the cpu registers.
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "lapic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show local apic state",
+        .mhandler.cmd = hmp_info_local_apic,
+    },
+#endif
+
+STEXI
+@item info lapic
+@findex lapic
+Show local APIC state
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "ioapic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show io apic state",
+        .mhandler.cmd = hmp_info_io_apic,
+    },
+#endif
+
+STEXI
+@item info ioapic
+@findex ioapic
+Show io APIC state
+ETEXI
+
+    {
+        .name       = "cpus",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show infos for each CPU",
+        .mhandler.cmd = hmp_info_cpus,
+    },
+
+STEXI
+@item info cpus
+@findex cpus
+Show infos for each CPU.
+ETEXI
+
+    {
+        .name       = "history",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the command line history",
+        .mhandler.cmd = hmp_info_history,
+    },
+
+STEXI
+@item info history
+@findex history
+Show the command line history.
+ETEXI
+
+#if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
+    defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
+    {
+        .name       = "irq",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the interrupts statistics (if available)",
+#ifdef TARGET_SPARC
+        .mhandler.cmd = sun4m_hmp_info_irq,
+#elif defined(TARGET_LM32)
+        .mhandler.cmd = lm32_hmp_info_irq,
+#else
+        .mhandler.cmd = hmp_info_irq,
+#endif
+    },
+
+STEXI
+@item info irq
+@findex irq
+Show the interrupts statistics (if available).
+ETEXI
+
+    {
+        .name       = "pic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show i8259 (PIC) state",
+#ifdef TARGET_SPARC
+        .mhandler.cmd = sun4m_hmp_info_pic,
+#elif defined(TARGET_LM32)
+        .mhandler.cmd = lm32_hmp_info_pic,
+#else
+        .mhandler.cmd = hmp_info_pic,
+#endif
+    },
+#endif
+
+STEXI
+@item info pic
+@findex pic
+Show i8259 (PIC) state.
+ETEXI
+
+    {
+        .name       = "pci",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show PCI info",
+        .mhandler.cmd = hmp_info_pci,
+    },
+
+STEXI
+@item info pci
+@findex pci
+Show PCI information.
+ETEXI
+
+#if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
+    defined(TARGET_PPC) || defined(TARGET_XTENSA)
+    {
+        .name       = "tlb",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show virtual to physical memory mappings",
+        .mhandler.cmd = hmp_info_tlb,
+    },
+#endif
+
+STEXI
+@item info tlb
+@findex tlb
+Show virtual to physical memory mappings.
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "mem",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the active virtual memory mappings",
+        .mhandler.cmd = hmp_info_mem,
+    },
+#endif
+
+STEXI
+@item info mem
+@findex mem
+Show the active virtual memory mappings.
+ETEXI
+
+    {
+        .name       = "mtree",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory tree",
+        .mhandler.cmd = hmp_info_mtree,
+    },
+
+STEXI
+@item info mtree
+@findex mtree
+Show memory tree.
+ETEXI
+
+    {
+        .name       = "jit",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show dynamic compiler info",
+        .mhandler.cmd = hmp_info_jit,
+    },
+
+STEXI
+@item info jit
+@findex jit
+Show dynamic compiler info.
+ETEXI
+
+    {
+        .name       = "opcount",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show dynamic compiler opcode counters",
+        .mhandler.cmd = hmp_info_opcount,
+    },
+
+STEXI
+@item info opcount
+@findex opcount
+Show dynamic compiler opcode counters
+ETEXI
+
+    {
+        .name       = "kvm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show KVM information",
+        .mhandler.cmd = hmp_info_kvm,
+    },
+
+STEXI
+@item info kvm
+@findex kvm
+Show KVM information.
+ETEXI
+
+    {
+        .name       = "numa",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show NUMA information",
+        .mhandler.cmd = hmp_info_numa,
+    },
+
+STEXI
+@item info numa
+@findex numa
+Show NUMA information.
+ETEXI
+
+    {
+        .name       = "usb",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show guest USB devices",
+        .mhandler.cmd = hmp_info_usb,
+    },
+
+STEXI
+@item info usb
+@findex usb
+Show guest USB devices.
+ETEXI
+
+    {
+        .name       = "usbhost",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show host USB devices",
+        .mhandler.cmd = hmp_info_usbhost,
+    },
+
+STEXI
+@item info usbhost
+@findex usbhost
+Show host USB devices.
+ETEXI
+
+    {
+        .name       = "profile",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show profiling information",
+        .mhandler.cmd = hmp_info_profile,
+    },
+
+STEXI
+@item info profile
+@findex profile
+Show profiling information.
+ETEXI
+
+    {
+        .name       = "capture",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show capture information",
+        .mhandler.cmd = hmp_info_capture,
+    },
+
+STEXI
+@item info capture
+@findex capture
+Show capture information.
+ETEXI
+
+    {
+        .name       = "snapshots",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the currently saved VM snapshots",
+        .mhandler.cmd = hmp_info_snapshots,
+    },
+
+STEXI
+@item info snapshots
+@findex snapshots
+Show the currently saved VM snapshots.
+ETEXI
+
+    {
+        .name       = "status",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM status (running|paused)",
+        .mhandler.cmd = hmp_info_status,
+    },
+
+STEXI
+@item info status
+@findex status
+Show the current VM status (running|paused).
+ETEXI
+
+    {
+        .name       = "mice",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show which guest mouse is receiving events",
+        .mhandler.cmd = hmp_info_mice,
+    },
+
+STEXI
+@item info mice
+@findex mice
+Show which guest mouse is receiving events.
+ETEXI
+
+    {
+        .name       = "vnc",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the vnc server status",
+        .mhandler.cmd = hmp_info_vnc,
+    },
+
+STEXI
+@item info vnc
+@findex vnc
+Show the vnc server status.
+ETEXI
+
+#if defined(CONFIG_SPICE)
+    {
+        .name       = "spice",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the spice server status",
+        .mhandler.cmd = hmp_info_spice,
+    },
+#endif
+
+STEXI
+@item info spice
+@findex spice
+Show the spice server status.
+ETEXI
+
+    {
+        .name       = "name",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM name",
+        .mhandler.cmd = hmp_info_name,
+    },
+
+STEXI
+@item info name
+@findex name
+Show the current VM name.
+ETEXI
+
+    {
+        .name       = "uuid",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM UUID",
+        .mhandler.cmd = hmp_info_uuid,
+    },
+
+STEXI
+@item info uuid
+@findex uuid
+Show the current VM UUID.
+ETEXI
+
+    {
+        .name       = "cpustats",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show CPU statistics",
+        .mhandler.cmd = hmp_info_cpustats,
+    },
+
+STEXI
+@item info cpustats
+@findex cpustats
+Show CPU statistics.
+ETEXI
+
+#if defined(CONFIG_SLIRP)
+    {
+        .name       = "usernet",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show user network stack connection states",
+        .mhandler.cmd = hmp_info_usernet,
+    },
+#endif
+
+STEXI
+@item info usernet
+@findex usernet
+Show user network stack connection states.
+ETEXI
+
+    {
+        .name       = "migrate",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show migration status",
+        .mhandler.cmd = hmp_info_migrate,
+    },
+
+STEXI
+@item info migrate
+@findex migrate
+Show migration status.
+ETEXI
+
+    {
+        .name       = "migrate_capabilities",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration capabilities",
+        .mhandler.cmd = hmp_info_migrate_capabilities,
+    },
+
+STEXI
+@item info migrate_capabilities
+@findex migrate_capabilities
+Show current migration capabilities.
+ETEXI
+
+    {
+        .name       = "migrate_parameters",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration parameters",
+        .mhandler.cmd = hmp_info_migrate_parameters,
+    },
+
+STEXI
+@item info migrate_parameters
+@findex migrate_parameters
+Show current migration parameters.
+ETEXI
+
+    {
+        .name       = "migrate_cache_size",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration xbzrle cache size",
+        .mhandler.cmd = hmp_info_migrate_cache_size,
+    },
+
+STEXI
+@item info migrate_cache_size
+@findex migrate_cache_size
+Show current migration xbzrle cache size.
+ETEXI
+
+    {
+        .name       = "balloon",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show balloon information",
+        .mhandler.cmd = hmp_info_balloon,
+    },
+
+STEXI
+@item info balloon
+@findex balloon
+Show balloon information.
+ETEXI
+
+    {
+        .name       = "qtree",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show device tree",
+        .mhandler.cmd = hmp_info_qtree,
+    },
+
+STEXI
+@item info qtree
+@findex qtree
+Show device tree.
+ETEXI
+
+    {
+        .name       = "qdm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show qdev device model list",
+        .mhandler.cmd = hmp_info_qdm,
+    },
+
+STEXI
+@item info qdm
+@findex qdm
+Show qdev device model list.
+ETEXI
+
+    {
+        .name       = "qom-tree",
+        .args_type  = "path:s?",
+        .params     = "[path]",
+        .help       = "show QOM composition tree",
+        .mhandler.cmd = hmp_info_qom_tree,
+    },
+
+STEXI
+@item info qom-tree
+@findex qom-tree
+Show QOM composition tree.
+ETEXI
+
+    {
+        .name       = "roms",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show roms",
+        .mhandler.cmd = hmp_info_roms,
+    },
+
+STEXI
+@item info roms
+@findex roms
+Show roms.
+ETEXI
+
+    {
+        .name       = "trace-events",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show available trace-events & their state",
+        .mhandler.cmd = hmp_info_trace_events,
+    },
+
+STEXI
+@item info trace-events
+@findex trace-events
+Show available trace-events & their state.
+ETEXI
+
+    {
+        .name       = "tpm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the TPM device",
+        .mhandler.cmd = hmp_info_tpm,
+    },
+
+STEXI
+@item info tpm
+@findex tpm
+Show the TPM device.
+ETEXI
+
+    {
+        .name       = "memdev",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory backends",
+        .mhandler.cmd = hmp_info_memdev,
+    },
+
+STEXI
+@item info memdev
+@findex memdev
+Show memory backends
+ETEXI
+
+    {
+        .name       = "memory-devices",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory devices",
+        .mhandler.cmd = hmp_info_memory_devices,
+    },
+
+STEXI
+@item info memory-devices
+@findex memory-devices
+Show memory devices.
+ETEXI
+
+    {
+        .name       = "iothreads",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show iothreads",
+        .mhandler.cmd = hmp_info_iothreads,
+    },
+
+STEXI
+@item info iothreads
+@findex iothreads
+Show iothread's identifiers.
+ETEXI
+
+    {
+        .name       = "rocker",
+        .args_type  = "name:s",
+        .params     = "name",
+        .help       = "Show rocker switch",
+        .mhandler.cmd = hmp_rocker,
+    },
+
+STEXI
+@item info rocker @var{name}
+@findex rocker
+Show rocker switch.
+ETEXI
+
+    {
+        .name       = "rocker-ports",
+        .args_type  = "name:s",
+        .params     = "name",
+        .help       = "Show rocker ports",
+        .mhandler.cmd = hmp_rocker_ports,
+    },
+
+STEXI
+@item info rocker_ports @var{name}-ports
+@findex ocker-ports
+Show rocker ports.
+ETEXI
+
+    {
+        .name       = "rocker-of-dpa-flows",
+        .args_type  = "name:s,tbl_id:i?",
+        .params     = "name [tbl_id]",
+        .help       = "Show rocker OF-DPA flow tables",
+        .mhandler.cmd = hmp_rocker_of_dpa_flows,
+    },
+
+STEXI
+@item info rocker_of_dpa_flows @var{name} [@var{tbl_id}]
+@findex rocker-of-dpa-flows
+Show rocker OF-DPA flow tables.
+ETEXI
+
+    {
+        .name       = "rocker-of-dpa-groups",
+        .args_type  = "name:s,type:i?",
+        .params     = "name [type]",
+        .help       = "Show rocker OF-DPA groups",
+        .mhandler.cmd = hmp_rocker_of_dpa_groups,
+    },
+
+STEXI
+@item info rocker-of-dpa-groups @var{name} [@var{type}]
+@findex rocker-of-dpa-groups
+Show rocker OF-DPA groups.
+ETEXI
+
+#if defined(TARGET_S390X)
+    {
+        .name       = "skeys",
+        .args_type  = "addr:l",
+        .params     = "address",
+        .help       = "Display the value of a storage key",
+        .mhandler.cmd = hmp_info_skeys,
+    },
+#endif
+
+STEXI
+@item info skeys @var{address}
+@findex skeys
+Display the value of a storage key (s390 only)
+ETEXI
+
+STEXI
+@end table
+ETEXI
+
+STEXI
+@end table
+ETEXI
diff --git a/hmp-commands.hx b/hmp-commands.hx
index d3b7932ff..bb52e4d3b 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -194,8 +194,8 @@ ETEXI
 
     {
         .name       = "change",
-        .args_type  = "device:B,target:F,arg:s?",
-        .params     = "device filename [format]",
+        .args_type  = "device:B,target:F,arg:s?,read-only-mode:s?",
+        .params     = "device filename [format [read-only-mode]]",
         .help       = "change a removable medium, optional format",
         .mhandler.cmd = hmp_change,
     },
@@ -206,7 +206,7 @@ STEXI
 Change the configuration of a device.
 
 @table @option
-@item change @var{diskdevice} @var{filename} [@var{format}]
+@item change @var{diskdevice} @var{filename} [@var{format} [@var{read-only-mode}]]
 Change the medium for a removable disk device to point to @var{filename}. eg
 
 @example
@@ -215,6 +215,20 @@ Change the medium for a removable disk device to point to @var{filename}. eg
 
 @var{format} is optional.
 
+@var{read-only-mode} may be used to change the read-only status of the device.
+It accepts the following values:
+
+@table @var
+@item retain
+Retains the current status; this is the default.
+
+@item read-only
+Makes the device read-only.
+
+@item read-write
+Makes the device writable.
+@end table
+
 @item change vnc @var{display},@var{options}
 Change the configuration of the VNC server. The valid syntax for @var{display}
 and @var{options} are described at @ref{sec_invocation}. eg
@@ -271,6 +285,7 @@ ETEXI
         .params     = "name on|off",
         .help       = "changes status of a specific trace event",
         .mhandler.cmd = hmp_trace_event,
+        .command_completion = trace_event_completion,
     },
 
 STEXI
@@ -675,7 +690,8 @@ ETEXI
 STEXI
 @item device_del @var{id}
 @findex device_del
-Remove device @var{id}.
+Remove device @var{id}. @var{id} may be a short ID
+or a QOM object path.
 ETEXI
 
     {
@@ -1006,6 +1022,23 @@ Set the parameter @var{parameter} for migration.
 ETEXI
 
     {
+        .name       = "migrate_start_postcopy",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Followup to a migration command to switch the migration"
+                      " to postcopy mode. The x-postcopy-ram capability must "
+                      "be set before the original migration command.",
+        .mhandler.cmd = hmp_migrate_start_postcopy,
+    },
+
+STEXI
+@item migrate_start_postcopy
+@findex migrate_start_postcopy
+Switch in-progress migration to postcopy mode. Ignored after the end of
+migration (or once already in postcopy).
+ETEXI
+
+    {
         .name       = "client_migrate_info",
         .args_type  = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
         .params     = "protocol hostname port tls-port cert-subject",
@@ -1053,6 +1086,22 @@ gdb. Without -z|-l|-s, the dump format is ELF.
             together with begin.
 ETEXI
 
+#if defined(TARGET_S390X)
+    {
+        .name       = "dump-skeys",
+        .args_type  = "filename:F",
+        .params     = "",
+        .help       = "Save guest storage keys into file 'filename'.\n",
+        .mhandler.cmd = hmp_dump_skeys,
+    },
+#endif
+
+STEXI
+@item dump-skeys @var{filename}
+@findex dump-skeys
+Save guest storage keys to a file.
+ETEXI
+
     {
         .name       = "snapshot_blkdev",
         .args_type  = "reuse:-n,device:B,snapshot-file:s?,format:s?",
@@ -1707,121 +1756,5 @@ ETEXI
     },
 
 STEXI
-@item info @var{subcommand}
-@findex info
-Show various information about the system state.
-
-@table @option
-@item info version
-show the version of QEMU
-@item info network
-show the various VLANs and the associated devices
-@item info chardev
-show the character devices
-@item info block
-show the block devices
-@item info blockstats
-show block device statistics
-@item info registers
-show the cpu registers
-@item info cpus
-show infos for each CPU
-@item info history
-show the command line history
-@item info irq
-show the interrupts statistics (if available)
-@item info pic
-show i8259 (PIC) state
-@item info pci
-show emulated PCI device info
-@item info tlb
-show virtual to physical memory mappings (i386, SH4, SPARC, PPC, and Xtensa only)
-@item info mem
-show the active virtual memory mappings (i386 only)
-@item info jit
-show dynamic compiler info
-@item info numa
-show NUMA information
-@item info kvm
-show KVM information
-@item info usb
-show USB devices plugged on the virtual USB hub
-@item info usbhost
-show all USB host devices
-@item info profile
-show profiling information
-@item info capture
-show information about active capturing
-@item info snapshots
-show list of VM snapshots
-@item info status
-show the current VM status (running|paused)
-@item info mice
-show which guest mouse is receiving events
-@item info vnc
-show the vnc server status
-@item info name
-show the current VM name
-@item info uuid
-show the current VM UUID
-@item info cpustats
-show CPU statistics
-@item info usernet
-show user network stack connection states
-@item info migrate
-show migration status
-@item info migrate_capabilities
-show current migration capabilities
-@item info migrate_parameters
-show current migration parameters
-@item info migrate_cache_size
-show current migration XBZRLE cache size
-@item info balloon
-show balloon information
-@item info qtree
-show device tree
-@item info qdm
-show qdev device model list
-@item info qom-tree
-show object composition tree
-@item info roms
-show roms
-@item info tpm
-show the TPM device
-@item info memory-devices
-show the memory devices
-@end table
-ETEXI
-
-STEXI
-@item info trace-events
-show available trace events and their state
-ETEXI
-
-STEXI
-@item rocker @var{name}
-@findex rocker
-Show Rocker(s)
-ETEXI
-
-STEXI
-@item rocker_ports @var{name}
-@findex rocker_ports
-Show Rocker ports
-ETEXI
-
-STEXI
-@item rocker_of_dpa_flows @var{name} [@var{tbl_id}]
-@findex rocker_of_dpa_flows
-Show Rocker OF-DPA flow tables
-ETEXI
-
-STEXI
-@item rocker_of_dpa_groups @var{name} [@var{type}]
-@findex rocker_of_dpa_groups
-Show Rocker OF-DPA groups
-ETEXI
-
-STEXI
 @end table
 ETEXI
diff --git a/hmp.c b/hmp.c
index dcc66f159..1904203d3 100644
--- a/hmp.c
+++ b/hmp.c
@@ -27,6 +27,7 @@
 #include "qapi/opts-visitor.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/string-output-visitor.h"
+#include "qapi/util.h"
 #include "qapi-visit.h"
 #include "ui/console.h"
 #include "block/qapi.h"
@@ -232,6 +233,11 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict)
                        info->xbzrle_cache->overflow);
     }
 
+    if (info->has_x_cpu_throttle_percentage) {
+        monitor_printf(mon, "cpu throttle percentage: %" PRIu64 "\n",
+                       info->x_cpu_throttle_percentage);
+    }
+
     qapi_free_MigrationInfo(info);
     qapi_free_MigrationCapabilityStatusList(caps);
 }
@@ -272,6 +278,12 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
         monitor_printf(mon, " %s: %" PRId64,
             MigrationParameter_lookup[MIGRATION_PARAMETER_DECOMPRESS_THREADS],
             params->decompress_threads);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL],
+            params->x_cpu_throttle_initial);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT],
+            params->x_cpu_throttle_increment);
         monitor_printf(mon, "\n");
     }
 
@@ -510,6 +522,7 @@ void hmp_info_blockstats(Monitor *mon, const QDict *qdict)
                        " flush_total_time_ns=%" PRId64
                        " rd_merged=%" PRId64
                        " wr_merged=%" PRId64
+                       " idle_time_ns=%" PRId64
                        "\n",
                        stats->value->stats->rd_bytes,
                        stats->value->stats->wr_bytes,
@@ -520,7 +533,8 @@ void hmp_info_blockstats(Monitor *mon, const QDict *qdict)
                        stats->value->stats->rd_total_time_ns,
                        stats->value->stats->flush_total_time_ns,
                        stats->value->stats->rd_merged,
-                       stats->value->stats->wr_merged);
+                       stats->value->stats->wr_merged,
+                       stats->value->stats->idle_time_ns);
     }
 
     qapi_free_BlockStatsList(stats_list);
@@ -558,8 +572,8 @@ void hmp_info_vnc(Monitor *mon, const QDict *qdict)
         for (client = info->clients; client; client = client->next) {
             monitor_printf(mon, "Client:\n");
             monitor_printf(mon, "     address: %s:%s\n",
-                           client->value->base->host,
-                           client->value->base->service);
+                           client->value->host,
+                           client->value->service);
             monitor_printf(mon, "  x509_dname: %s\n",
                            client->value->x509_dname ?
                            client->value->x509_dname : "none");
@@ -627,7 +641,7 @@ void hmp_info_spice(Monitor *mon, const QDict *qdict)
         for (chan = info->channels; chan; chan = chan->next) {
             monitor_printf(mon, "Channel:\n");
             monitor_printf(mon, "     address: %s:%s%s\n",
-                           chan->value->base->host, chan->value->base->port,
+                           chan->value->host, chan->value->port,
                            chan->value->tls ? " [tls]" : "");
             monitor_printf(mon, "     session: %" PRId64 "\n",
                            chan->value->connection_id);
@@ -830,11 +844,11 @@ void hmp_info_tpm(Monitor *mon, const QDict *qdict)
                        c, TpmModel_lookup[ti->model]);
 
         monitor_printf(mon, "  \\ %s: type=%s",
-                       ti->id, TpmTypeOptionsKind_lookup[ti->options->kind]);
+                       ti->id, TpmTypeOptionsKind_lookup[ti->options->type]);
 
-        switch (ti->options->kind) {
+        switch (ti->options->type) {
         case TPM_TYPE_OPTIONS_KIND_PASSTHROUGH:
-            tpo = ti->options->passthrough;
+            tpo = ti->options->u.passthrough;
             monitor_printf(mon, "%s%s%s%s",
                            tpo->has_path ? ",path=" : "",
                            tpo->has_path ? tpo->path : "",
@@ -1221,6 +1235,8 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
     bool has_compress_level = false;
     bool has_compress_threads = false;
     bool has_decompress_threads = false;
+    bool has_x_cpu_throttle_initial = false;
+    bool has_x_cpu_throttle_increment = false;
     int i;
 
     for (i = 0; i < MIGRATION_PARAMETER_MAX; i++) {
@@ -1235,10 +1251,18 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
             case MIGRATION_PARAMETER_DECOMPRESS_THREADS:
                 has_decompress_threads = true;
                 break;
+            case MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL:
+                has_x_cpu_throttle_initial = true;
+                break;
+            case MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT:
+                has_x_cpu_throttle_increment = true;
+                break;
             }
             qmp_migrate_set_parameters(has_compress_level, value,
                                        has_compress_threads, value,
                                        has_decompress_threads, value,
+                                       has_x_cpu_throttle_initial, value,
+                                       has_x_cpu_throttle_increment, value,
                                        &err);
             break;
         }
@@ -1272,6 +1296,13 @@ void hmp_client_migrate_info(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &err);
 }
 
+void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    qmp_migrate_start_postcopy(&err);
+    hmp_handle_error(mon, &err);
+}
+
 void hmp_set_password(Monitor *mon, const QDict *qdict)
 {
     const char *protocol  = qdict_get_str(qdict, "protocol");
@@ -1315,24 +1346,46 @@ void hmp_change(Monitor *mon, const QDict *qdict)
     const char *device = qdict_get_str(qdict, "device");
     const char *target = qdict_get_str(qdict, "target");
     const char *arg = qdict_get_try_str(qdict, "arg");
+    const char *read_only = qdict_get_try_str(qdict, "read-only-mode");
+    BlockdevChangeReadOnlyMode read_only_mode = 0;
     Error *err = NULL;
 
-    if (strcmp(device, "vnc") == 0 &&
-            (strcmp(target, "passwd") == 0 ||
-             strcmp(target, "password") == 0)) {
-        if (!arg) {
-            monitor_read_password(mon, hmp_change_read_arg, NULL);
+    if (strcmp(device, "vnc") == 0) {
+        if (read_only) {
+            monitor_printf(mon,
+                           "Parameter 'read-only-mode' is invalid for VNC\n");
             return;
         }
-    }
+        if (strcmp(target, "passwd") == 0 ||
+            strcmp(target, "password") == 0) {
+            if (!arg) {
+                monitor_read_password(mon, hmp_change_read_arg, NULL);
+                return;
+            }
+        }
+        qmp_change("vnc", target, !!arg, arg, &err);
+    } else {
+        if (read_only) {
+            read_only_mode =
+                qapi_enum_parse(BlockdevChangeReadOnlyMode_lookup,
+                                read_only, BLOCKDEV_CHANGE_READ_ONLY_MODE_MAX,
+                                BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN, &err);
+            if (err) {
+                hmp_handle_error(mon, &err);
+                return;
+            }
+        }
 
-    qmp_change(device, target, !!arg, arg, &err);
-    if (err &&
-        error_get_class(err) == ERROR_CLASS_DEVICE_ENCRYPTED) {
-        error_free(err);
-        monitor_read_block_device_key(mon, device, NULL, NULL);
-        return;
+        qmp_blockdev_change_medium(device, target, !!arg, arg,
+                                   !!read_only, read_only_mode, &err);
+        if (err &&
+            error_get_class(err) == ERROR_CLASS_DEVICE_ENCRYPTED) {
+            error_free(err);
+            monitor_read_block_device_key(mon, device, NULL, NULL);
+            return;
+        }
     }
+
     hmp_handle_error(mon, &err);
 }
 
@@ -1681,21 +1734,18 @@ void hmp_sendkey(Monitor *mon, const QDict *qdict)
     int has_hold_time = qdict_haskey(qdict, "hold-time");
     int hold_time = qdict_get_try_int(qdict, "hold-time", -1);
     Error *err = NULL;
-    char keyname_buf[16];
     char *separator;
     int keyname_len;
 
     while (1) {
         separator = strchr(keys, '-');
         keyname_len = separator ? separator - keys : strlen(keys);
-        pstrcpy(keyname_buf, sizeof(keyname_buf), keys);
 
         /* Be compatible with old interface, convert user inputted "<" */
-        if (!strncmp(keyname_buf, "<", 1) && keyname_len == 1) {
-            pstrcpy(keyname_buf, sizeof(keyname_buf), "less");
+        if (keys[0] == '<' && keyname_len == 1) {
+            keys = "less";
             keyname_len = 4;
         }
-        keyname_buf[keyname_len] = 0;
 
         keylist = g_malloc0(sizeof(*keylist));
         keylist->value = g_malloc0(sizeof(*keylist->value));
@@ -1708,21 +1758,22 @@ void hmp_sendkey(Monitor *mon, const QDict *qdict)
         }
         tmp = keylist;
 
-        if (strstart(keyname_buf, "0x", NULL)) {
+        if (strstart(keys, "0x", NULL)) {
             char *endp;
-            int value = strtoul(keyname_buf, &endp, 0);
-            if (*endp != '\0') {
+            int value = strtoul(keys, &endp, 0);
+            assert(endp <= keys + keyname_len);
+            if (endp != keys + keyname_len) {
                 goto err_out;
             }
-            keylist->value->kind = KEY_VALUE_KIND_NUMBER;
-            keylist->value->number = value;
+            keylist->value->type = KEY_VALUE_KIND_NUMBER;
+            keylist->value->u.number = value;
         } else {
-            int idx = index_from_key(keyname_buf);
+            int idx = index_from_key(keys, keyname_len);
             if (idx == Q_KEY_CODE_MAX) {
                 goto err_out;
             }
-            keylist->value->kind = KEY_VALUE_KIND_QCODE;
-            keylist->value->qcode = idx;
+            keylist->value->type = KEY_VALUE_KIND_QCODE;
+            keylist->value->u.qcode = idx;
         }
 
         if (!separator) {
@@ -1739,7 +1790,7 @@ out:
     return;
 
 err_out:
-    monitor_printf(mon, "invalid parameter: %s\n", keyname_buf);
+    monitor_printf(mon, "invalid parameter: %.*s\n", keyname_len, keys);
     goto out;
 }
 
@@ -1937,12 +1988,12 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
         value = info->value;
 
         if (value) {
-            switch (value->kind) {
+            switch (value->type) {
             case MEMORY_DEVICE_INFO_KIND_DIMM:
-                di = value->dimm;
+                di = value->u.dimm;
 
                 monitor_printf(mon, "Memory device [%s]: \"%s\"\n",
-                               MemoryDeviceInfoKind_lookup[value->kind],
+                               MemoryDeviceInfoKind_lookup[value->type],
                                di->id ? di->id : "");
                 monitor_printf(mon, "  addr: 0x%" PRIx64 "\n", di->addr);
                 monitor_printf(mon, "  slot: %" PRId64 "\n", di->slot);
@@ -1963,6 +2014,19 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
     qapi_free_MemoryDeviceInfoList(info_list);
 }
 
+void hmp_info_iothreads(Monitor *mon, const QDict *qdict)
+{
+    IOThreadInfoList *info_list = qmp_query_iothreads(NULL);
+    IOThreadInfoList *info;
+
+    for (info = info_list; info; info = info->next) {
+        monitor_printf(mon, "%s: thread_id=%" PRId64 "\n",
+                       info->value->id, info->value->thread_id);
+    }
+
+    qapi_free_IOThreadInfoList(info_list);
+}
+
 void hmp_qom_list(Monitor *mon, const QDict *qdict)
 {
     const char *path = qdict_get_try_str(qdict, "path");
diff --git a/hmp.h b/hmp.h
index 0cf4f2a3d..a8c5b5a9a 100644
--- a/hmp.h
+++ b/hmp.h
@@ -39,6 +39,7 @@ void hmp_info_balloon(Monitor *mon, const QDict *qdict);
 void hmp_info_pci(Monitor *mon, const QDict *qdict);
 void hmp_info_block_jobs(Monitor *mon, const QDict *qdict);
 void hmp_info_tpm(Monitor *mon, const QDict *qdict);
+void hmp_info_iothreads(Monitor *mon, const QDict *qdict);
 void hmp_quit(Monitor *mon, const QDict *qdict);
 void hmp_stop(Monitor *mon, const QDict *qdict);
 void hmp_system_reset(Monitor *mon, const QDict *qdict);
@@ -68,6 +69,7 @@ void hmp_migrate_set_capability(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict);
 void hmp_client_migrate_info(Monitor *mon, const QDict *qdict);
+void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict);
 void hmp_set_password(Monitor *mon, const QDict *qdict);
 void hmp_expire_password(Monitor *mon, const QDict *qdict);
 void hmp_eject(Monitor *mon, const QDict *qdict);
@@ -113,6 +115,7 @@ void set_link_completion(ReadLineState *rs, int nb_args, const char *str);
 void netdev_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void netdev_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void ringbuf_write_completion(ReadLineState *rs, int nb_args, const char *str);
+void trace_event_completion(ReadLineState *rs, int nb_args, const char *str);
 void watchdog_action_completion(ReadLineState *rs, int nb_args,
                                 const char *str);
 void migrate_set_capability_completion(ReadLineState *rs, int nb_args,
diff --git a/hw/9pfs/codir.c b/hw/9pfs/codir.c
index 65ad3298b..ec9cc7fb2 100644
--- a/hw/9pfs/codir.c
+++ b/hw/9pfs/codir.c
@@ -14,7 +14,7 @@
 
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_readdir_r(V9fsPDU *pdu, V9fsFidState *fidp, struct dirent *dent,
diff --git a/hw/9pfs/cofile.c b/hw/9pfs/cofile.c
index 2efebf357..7cb55ee93 100644
--- a/hw/9pfs/cofile.c
+++ b/hw/9pfs/cofile.c
@@ -14,7 +14,7 @@
 
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t st_mode,
diff --git a/hw/9pfs/cofs.c b/hw/9pfs/cofs.c
index 42ee614e2..e1953a9aa 100644
--- a/hw/9pfs/cofs.c
+++ b/hw/9pfs/cofs.c
@@ -14,7 +14,7 @@
 
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "virtio-9p-coth.h"
 
 static ssize_t __readlink(V9fsState *s, V9fsPath *path, V9fsString *buf)
diff --git a/hw/9pfs/coxattr.c b/hw/9pfs/coxattr.c
index 18ee08df0..55c0d231c 100644
--- a/hw/9pfs/coxattr.c
+++ b/hw/9pfs/coxattr.c
@@ -14,7 +14,7 @@
 
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_llistxattr(V9fsPDU *pdu, V9fsPath *path, void *value, size_t size)
diff --git a/hw/9pfs/virtio-9p-coth.c b/hw/9pfs/virtio-9p-coth.c
index 8185c533c..ab9425c60 100644
--- a/hw/9pfs/virtio-9p-coth.c
+++ b/hw/9pfs/virtio-9p-coth.c
@@ -12,71 +12,30 @@
  *
  */
 
-#include "fsdev/qemu-fsdev.h"
-#include "qemu/thread.h"
-#include "qemu/event_notifier.h"
-#include "block/coroutine.h"
+#include "qemu-common.h"
+#include "block/thread-pool.h"
+#include "qemu/coroutine.h"
+#include "qemu/main-loop.h"
 #include "virtio-9p-coth.h"
 
-/* v9fs glib thread pool */
-static V9fsThPool v9fs_pool;
-
-void co_run_in_worker_bh(void *opaque)
+/* Called from QEMU I/O thread.  */
+static void coroutine_enter_cb(void *opaque, int ret)
 {
     Coroutine *co = opaque;
-    g_thread_pool_push(v9fs_pool.pool, co, NULL);
-}
-
-static void v9fs_qemu_process_req_done(EventNotifier *e)
-{
-    Coroutine *co;
-
-    event_notifier_test_and_clear(e);
-
-    while ((co = g_async_queue_try_pop(v9fs_pool.completed)) != NULL) {
-        qemu_coroutine_enter(co, NULL);
-    }
+    qemu_coroutine_enter(co, NULL);
 }
 
-static void v9fs_thread_routine(gpointer data, gpointer user_data)
+/* Called from worker thread.  */
+static int coroutine_enter_func(void *arg)
 {
-    Coroutine *co = data;
-
+    Coroutine *co = arg;
     qemu_coroutine_enter(co, NULL);
-
-    g_async_queue_push(v9fs_pool.completed, co);
-
-    event_notifier_set(&v9fs_pool.e);
+    return 0;
 }
 
-int v9fs_init_worker_threads(void)
+void co_run_in_worker_bh(void *opaque)
 {
-    int ret = 0;
-    V9fsThPool *p = &v9fs_pool;
-    sigset_t set, oldset;
-
-    sigfillset(&set);
-    /* Leave signal handling to the iothread.  */
-    pthread_sigmask(SIG_SETMASK, &set, &oldset);
-
-    p->pool = g_thread_pool_new(v9fs_thread_routine, p, -1, FALSE, NULL);
-    if (!p->pool) {
-        ret = -1;
-        goto err_out;
-    }
-    p->completed = g_async_queue_new();
-    if (!p->completed) {
-        /*
-         * We are going to terminate.
-         * So don't worry about cleanup
-         */
-        ret = -1;
-        goto err_out;
-    }
-    event_notifier_init(&p->e, 0);
-
-    event_notifier_set_handler(&p->e, v9fs_qemu_process_req_done);
-err_out:
-    pthread_sigmask(SIG_SETMASK, &oldset, NULL);
-    return ret;
+    Coroutine *co = opaque;
+    thread_pool_submit_aio(aio_get_thread_pool(qemu_get_aio_context()),
+                           coroutine_enter_func, co, coroutine_enter_cb, co);
 }
diff --git a/hw/9pfs/virtio-9p-coth.h b/hw/9pfs/virtio-9p-coth.h
index 4f51b250d..4ac1aaf90 100644
--- a/hw/9pfs/virtio-9p-coth.h
+++ b/hw/9pfs/virtio-9p-coth.h
@@ -16,16 +16,8 @@
 #define _QEMU_VIRTIO_9P_COTH_H
 
 #include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "virtio-9p.h"
-#include <glib.h>
-
-typedef struct V9fsThPool {
-    EventNotifier e;
-
-    GThreadPool *pool;
-    GAsyncQueue *completed;
-} V9fsThPool;
 
 /*
  * we want to use bottom half because we want to make sure the below
@@ -45,7 +37,7 @@ typedef struct V9fsThPool {
         qemu_bh_schedule(co_bh);                                        \
         /*                                                              \
          * yield in qemu thread and re-enter back                       \
-         * in glib worker thread                                        \
+         * in worker thread                                             \
          */                                                             \
         qemu_coroutine_yield();                                         \
         qemu_bh_delete(co_bh);                                          \
diff --git a/hw/9pfs/virtio-9p-device.c b/hw/9pfs/virtio-9p-device.c
index 93a407c45..b42d3b30a 100644
--- a/hw/9pfs/virtio-9p-device.c
+++ b/hw/9pfs/virtio-9p-device.c
@@ -43,6 +43,16 @@ static void virtio_9p_get_config(VirtIODevice *vdev, uint8_t *config)
     g_free(cfg);
 }
 
+static void virtio_9p_save(QEMUFile *f, void *opaque)
+{
+    virtio_save(VIRTIO_DEVICE(opaque), f);
+}
+
+static int virtio_9p_load(QEMUFile *f, void *opaque, int version_id)
+{
+    return virtio_load(VIRTIO_DEVICE(opaque), f, version_id);
+}
+
 static void virtio_9p_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
@@ -106,10 +116,6 @@ static void virtio_9p_device_realize(DeviceState *dev, Error **errp)
                    " and export path:%s", s->fsconf.fsdev_id, s->ctx.fs_root);
         goto out;
     }
-    if (v9fs_init_worker_threads() < 0) {
-        error_setg(errp, "worker thread initialization failed");
-        goto out;
-    }
 
     /*
      * Check details of export path, We need to use fs driver
@@ -130,6 +136,7 @@ static void virtio_9p_device_realize(DeviceState *dev, Error **errp)
     }
     v9fs_path_free(&path);
 
+    register_savevm(dev, "virtio-9p", -1, 1, virtio_9p_save, virtio_9p_load, s);
     return;
 out:
     g_free(s->ctx.fs_root);
@@ -138,6 +145,17 @@ out:
     v9fs_path_free(&path);
 }
 
+static void virtio_9p_device_unrealize(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    V9fsState *s = VIRTIO_9P(dev);
+
+    virtio_cleanup(vdev);
+    unregister_savevm(dev, "virtio-9p", s);
+    g_free(s->ctx.fs_root);
+    g_free(s->tag);
+}
+
 /* virtio-9p device */
 
 static Property virtio_9p_properties[] = {
@@ -154,6 +172,7 @@ static void virtio_9p_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_9p_properties;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     vdc->realize = virtio_9p_device_realize;
+    vdc->unrealize = virtio_9p_device_unrealize;
     vdc->get_features = virtio_9p_get_features;
     vdc->get_config = virtio_9p_get_config;
 }
diff --git a/hw/9pfs/virtio-9p.h b/hw/9pfs/virtio-9p.h
index 2e7d48857..d7a4dc1e9 100644
--- a/hw/9pfs/virtio-9p.h
+++ b/hw/9pfs/virtio-9p.h
@@ -13,7 +13,7 @@
 #include "fsdev/file-op-9p.h"
 #include "fsdev/virtio-9p-marshal.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 enum {
     P9_TLERROR = 6,
diff --git a/hw/Makefile.objs b/hw/Makefile.objs
index 73afa41b3..7e7c24110 100644
--- a/hw/Makefile.objs
+++ b/hw/Makefile.objs
@@ -31,6 +31,7 @@ devices-dirs-$(CONFIG_VIRTIO) += virtio/
 devices-dirs-$(CONFIG_SOFTMMU) += watchdog/
 devices-dirs-$(CONFIG_SOFTMMU) += xen/
 devices-dirs-$(CONFIG_MEM_HOTPLUG) += mem/
+devices-dirs-$(CONFIG_SMBIOS) += smbios/
 devices-dirs-y += core/
 common-obj-y += $(devices-dirs-y)
 obj-y += $(devices-dirs-y)
diff --git a/hw/acpi/aml-build.c b/hw/acpi/aml-build.c
index 0d4b3247b..a00a0abe8 100644
--- a/hw/acpi/aml-build.c
+++ b/hw/acpi/aml-build.c
@@ -1163,9 +1163,7 @@ void *acpi_data_push(GArray *table_data, unsigned size)
 
 unsigned acpi_data_len(GArray *table)
 {
-#if GLIB_CHECK_VERSION(2, 22, 0)
     assert(g_array_get_element_size(table) == 1);
-#endif
     return table->len;
 }
 
diff --git a/hw/acpi/core.c b/hw/acpi/core.c
index fe6215af4..21e113d71 100644
--- a/hw/acpi/core.c
+++ b/hw/acpi/core.c
@@ -625,8 +625,12 @@ void acpi_pm1_cnt_reset(ACPIREGS *ar)
 void acpi_gpe_init(ACPIREGS *ar, uint8_t len)
 {
     ar->gpe.len = len;
-    ar->gpe.sts = g_malloc0(len / 2);
-    ar->gpe.en = g_malloc0(len / 2);
+    /* Only first len / 2 bytes are ever used,
+     * but the caller in ich9.c migrates full len bytes.
+     * TODO: fix ich9.c and drop the extra allocation.
+     */
+    ar->gpe.sts = g_malloc0(len);
+    ar->gpe.en = g_malloc0(len);
 }
 
 void acpi_gpe_reset(ACPIREGS *ar)
diff --git a/hw/acpi/memory_hotplug.c b/hw/acpi/memory_hotplug.c
index 2ff0d5ce1..e4b9a01f8 100644
--- a/hw/acpi/memory_hotplug.c
+++ b/hw/acpi/memory_hotplug.c
@@ -155,6 +155,7 @@ static void acpi_memory_hotplug_write(void *opaque, hwaddr addr, uint64_t data,
                 qapi_event_send_mem_unplug_error(dev->id,
                                                  error_get_pretty(local_err),
                                                  &error_abort);
+                error_free(local_err);
                 break;
             }
             trace_mhp_acpi_pc_dimm_deleted(mem_st->selector);
@@ -238,10 +239,12 @@ void acpi_memory_plug_cb(ACPIREGS *ar, qemu_irq irq, MemHotplugState *mem_st,
 
     mdev->dimm = dev;
     mdev->is_enabled = true;
-    mdev->is_inserting = true;
+    if (dev->hotplugged) {
+        mdev->is_inserting = true;
 
-    /* do ACPI magic */
-    acpi_send_gpe_event(ar, irq, ACPI_MEMORY_HOTPLUG_STATUS);
+        /* do ACPI magic */
+        acpi_send_gpe_event(ar, irq, ACPI_MEMORY_HOTPLUG_STATUS);
+    }
     return;
 }
 
diff --git a/hw/alpha/dp264.c b/hw/alpha/dp264.c
index f86e7bb83..27bdaa1ad 100644
--- a/hw/alpha/dp264.c
+++ b/hw/alpha/dp264.c
@@ -168,17 +168,12 @@ static void clipper_init(MachineState *machine)
     }
 }
 
-static QEMUMachine clipper_machine = {
-    .name = "clipper",
-    .desc = "Alpha DP264/CLIPPER",
-    .init = clipper_init,
-    .max_cpus = 4,
-    .is_default = 1,
-};
-
-static void clipper_machine_init(void)
+static void clipper_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&clipper_machine);
+    mc->desc = "Alpha DP264/CLIPPER";
+    mc->init = clipper_init;
+    mc->max_cpus = 4;
+    mc->is_default = 1;
 }
 
-machine_init(clipper_machine_init);
+DEFINE_MACHINE("clipper", clipper_machine_init)
diff --git a/hw/arm/Makefile.objs b/hw/arm/Makefile.objs
index cf346c1d0..2195b60fa 100644
--- a/hw/arm/Makefile.objs
+++ b/hw/arm/Makefile.objs
@@ -1,6 +1,6 @@
 obj-y += boot.o collie.o exynos4_boards.o gumstix.o highbank.o
 obj-$(CONFIG_DIGIC) += digic_boards.o
-obj-y += integratorcp.o kzm.o mainstone.o musicpal.o nseries.o
+obj-y += integratorcp.o mainstone.o musicpal.o nseries.o
 obj-y += omap_sx1.o palm.o realview.o spitz.o stellaris.o
 obj-y += tosa.o versatilepb.o vexpress.o virt.o xilinx_zynq.o z2.o
 obj-$(CONFIG_ACPI) += virt-acpi-build.o
@@ -13,3 +13,5 @@ obj-y += omap1.o omap2.o strongarm.o
 obj-$(CONFIG_ALLWINNER_A10) += allwinner-a10.o cubieboard.o
 obj-$(CONFIG_STM32F205_SOC) += stm32f205_soc.o
 obj-$(CONFIG_XLNX_ZYNQMP) += xlnx-zynqmp.o xlnx-ep108.o
+obj-$(CONFIG_FSL_IMX25) += fsl-imx25.o imx25_pdk.o
+obj-$(CONFIG_FSL_IMX31) += fsl-imx31.o kzm.o
diff --git a/hw/arm/allwinner-a10.c b/hw/arm/allwinner-a10.c
index 43dc0a12d..b0ca81cea 100644
--- a/hw/arm/allwinner-a10.c
+++ b/hw/arm/allwinner-a10.c
@@ -39,6 +39,9 @@ static void aw_a10_init(Object *obj)
         qemu_check_nic_model(&nd_table[0], TYPE_AW_EMAC);
         qdev_set_nic_properties(DEVICE(&s->emac), &nd_table[0]);
     }
+
+    object_initialize(&s->sata, sizeof(s->sata), TYPE_ALLWINNER_AHCI);
+    qdev_set_parent_bus(DEVICE(&s->sata), sysbus_get_default());
 }
 
 static void aw_a10_realize(DeviceState *dev, Error **errp)
@@ -93,6 +96,14 @@ static void aw_a10_realize(DeviceState *dev, Error **errp)
     sysbus_mmio_map(sysbusdev, 0, AW_A10_EMAC_BASE);
     sysbus_connect_irq(sysbusdev, 0, s->irq[55]);
 
+    object_property_set_bool(OBJECT(&s->sata), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->sata), 0, AW_A10_SATA_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sata), 0, s->irq[56]);
+
     /* FIXME use a qdev chardev prop instead of serial_hds[] */
     serial_mm_init(get_system_memory(), AW_A10_UART0_REG_BASE, 2, s->irq[1],
                    115200, serial_hds[0], DEVICE_NATIVE_ENDIAN);
diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c
index c6eab6de3..a80d2ad29 100644
--- a/hw/arm/armv7m.c
+++ b/hw/arm/armv7m.c
@@ -166,17 +166,15 @@ static void armv7m_reset(void *opaque)
    mem_size is in bytes.
    Returns the NVIC array.  */
 
-qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
+DeviceState *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
                       const char *kernel_filename, const char *cpu_model)
 {
     ARMCPU *cpu;
     CPUARMState *env;
     DeviceState *nvic;
-    qemu_irq *pic = g_new(qemu_irq, num_irq);
     int image_size;
     uint64_t entry;
     uint64_t lowaddr;
-    int i;
     int big_endian;
     MemoryRegion *hack = g_new(MemoryRegion, 1);
 
@@ -198,9 +196,6 @@ qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
     qdev_init_nofail(nvic);
     sysbus_connect_irq(SYS_BUS_DEVICE(nvic), 0,
                        qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ));
-    for (i = 0; i < num_irq; i++) {
-        pic[i] = qdev_get_gpio_in(nvic, i);
-    }
 
 #ifdef TARGET_WORDS_BIGENDIAN
     big_endian = 1;
@@ -215,7 +210,7 @@ qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
 
     if (kernel_filename) {
         image_size = load_elf(kernel_filename, NULL, NULL, &entry, &lowaddr,
-                              NULL, big_endian, ELF_MACHINE, 1);
+                              NULL, big_endian, EM_ARM, 1);
         if (image_size < 0) {
             image_size = load_image_targphys(kernel_filename, 0, mem_size);
             lowaddr = 0;
@@ -229,12 +224,12 @@ qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
     /* Hack to map an additional page of ram at the top of the address
        space.  This stops qemu complaining about executing code outside RAM
        when returning from an exception.  */
-    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000, &error_abort);
+    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000, &error_fatal);
     vmstate_register_ram_global(hack);
     memory_region_add_subregion(system_memory, 0xfffff000, hack);
 
     qemu_register_reset(armv7m_reset, cpu);
-    return pic;
+    return nvic;
 }
 
 static Property bitband_properties[] = {
diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index 5b969cda1..75f69bfe0 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -10,6 +10,8 @@
 #include "config.h"
 #include "hw/hw.h"
 #include "hw/arm/arm.h"
+#include "hw/arm/linux-boot-if.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -27,14 +29,15 @@
 #define KERNEL64_LOAD_ADDR 0x00080000
 
 typedef enum {
-    FIXUP_NONE = 0,   /* do nothing */
-    FIXUP_TERMINATOR, /* end of insns */
-    FIXUP_BOARDID,    /* overwrite with board ID number */
-    FIXUP_ARGPTR,     /* overwrite with pointer to kernel args */
-    FIXUP_ENTRYPOINT, /* overwrite with kernel entry point */
-    FIXUP_GIC_CPU_IF, /* overwrite with GIC CPU interface address */
-    FIXUP_BOOTREG,    /* overwrite with boot register address */
-    FIXUP_DSB,        /* overwrite with correct DSB insn for cpu */
+    FIXUP_NONE = 0,     /* do nothing */
+    FIXUP_TERMINATOR,   /* end of insns */
+    FIXUP_BOARDID,      /* overwrite with board ID number */
+    FIXUP_BOARD_SETUP,  /* overwrite with board specific setup code address */
+    FIXUP_ARGPTR,       /* overwrite with pointer to kernel args */
+    FIXUP_ENTRYPOINT,   /* overwrite with kernel entry point */
+    FIXUP_GIC_CPU_IF,   /* overwrite with GIC CPU interface address */
+    FIXUP_BOOTREG,      /* overwrite with boot register address */
+    FIXUP_DSB,          /* overwrite with correct DSB insn for cpu */
     FIXUP_MAX,
 } FixupType;
 
@@ -57,8 +60,17 @@ static const ARMInsnFixup bootloader_aarch64[] = {
     { 0, FIXUP_TERMINATOR }
 };
 
-/* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
+/* A very small bootloader: call the board-setup code (if needed),
+ * set r0-r2, then jump to the kernel.
+ * If we're not calling boot setup code then we don't copy across
+ * the first BOOTLOADER_NO_BOARD_SETUP_OFFSET insns in this array.
+ */
+
 static const ARMInsnFixup bootloader[] = {
+    { 0xe28fe008 }, /* add     lr, pc, #8 */
+    { 0xe51ff004 }, /* ldr     pc, [pc, #-4] */
+    { 0, FIXUP_BOARD_SETUP },
+#define BOOTLOADER_NO_BOARD_SETUP_OFFSET 3
     { 0xe3a00000 }, /* mov     r0, #0 */
     { 0xe59f1004 }, /* ldr     r1, [pc, #4] */
     { 0xe59f2004 }, /* ldr     r2, [pc, #4] */
@@ -130,6 +142,7 @@ static void write_bootloader(const char *name, hwaddr addr,
         case FIXUP_NONE:
             break;
         case FIXUP_BOARDID:
+        case FIXUP_BOARD_SETUP:
         case FIXUP_ARGPTR:
         case FIXUP_ENTRYPOINT:
         case FIXUP_GIC_CPU_IF:
@@ -483,7 +496,8 @@ static void do_cpu_reset(void *opaque)
                 }
 
                 /* Set to non-secure if not a secure boot */
-                if (!info->secure_boot) {
+                if (!info->secure_boot &&
+                    (cs != first_cpu || !info->secure_board_setup)) {
                     /* Linux expects non-secure state */
                     env->cp15.scr_el3 |= SCR_NS;
                 }
@@ -555,6 +569,20 @@ static void load_image_to_fw_cfg(FWCfgState *fw_cfg, uint16_t size_key,
     fw_cfg_add_bytes(fw_cfg, data_key, data, size);
 }
 
+static int do_arm_linux_init(Object *obj, void *opaque)
+{
+    if (object_dynamic_cast(obj, TYPE_ARM_LINUX_BOOT_IF)) {
+        ARMLinuxBootIf *albif = ARM_LINUX_BOOT_IF(obj);
+        ARMLinuxBootIfClass *albifc = ARM_LINUX_BOOT_IF_GET_CLASS(obj);
+        struct arm_boot_info *info = opaque;
+
+        if (albifc->arm_linux_init) {
+            albifc->arm_linux_init(albif, info->secure_boot);
+        }
+    }
+    return 0;
+}
+
 static void arm_load_kernel_notify(Notifier *notifier, void *data)
 {
     CPUState *cs;
@@ -572,6 +600,12 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
     struct arm_boot_info *info =
         container_of(n, struct arm_boot_info, load_kernel_notifier);
 
+    /* The board code is not supposed to set secure_board_setup unless
+     * running its code in secure mode is actually possible, and KVM
+     * doesn't support secure.
+     */
+    assert(!(info->secure_board_setup && kvm_enabled()));
+
     /* Load the kernel.  */
     if (!info->kernel_filename || info->firmware_loaded) {
 
@@ -625,6 +659,9 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
         elf_machine = EM_AARCH64;
     } else {
         primary_loader = bootloader;
+        if (!info->write_board_setup) {
+            primary_loader += BOOTLOADER_NO_BOARD_SETUP_OFFSET;
+        }
         kernel_load_offset = KERNEL_LOAD_ADDR;
         elf_machine = EM_ARM;
     }
@@ -730,6 +767,7 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
         info->initrd_size = initrd_size;
 
         fixupcontext[FIXUP_BOARDID] = info->board_id;
+        fixupcontext[FIXUP_BOARD_SETUP] = info->board_setup_addr;
 
         /* for device tree boot, we pass the DTB directly in r2. Otherwise
          * we point to the kernel args.
@@ -778,6 +816,15 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
         if (info->nb_cpus > 1) {
             info->write_secondary_boot(cpu, info);
         }
+        if (info->write_board_setup) {
+            info->write_board_setup(cpu, info);
+        }
+
+        /* Notify devices which need to fake up firmware initialization
+         * that we're doing a direct kernel boot.
+         */
+        object_child_foreach_recursive(object_get_root(),
+                                       do_arm_linux_init, info);
     }
     info->is_linux = is_linux;
 
@@ -803,3 +850,16 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
         qemu_register_reset(do_cpu_reset, ARM_CPU(cs));
     }
 }
+
+static const TypeInfo arm_linux_boot_if_info = {
+    .name = TYPE_ARM_LINUX_BOOT_IF,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(ARMLinuxBootIfClass),
+};
+
+static void arm_linux_boot_register_types(void)
+{
+    type_register_static(&arm_linux_boot_if_info);
+}
+
+type_init(arm_linux_boot_register_types)
diff --git a/hw/arm/collie.c b/hw/arm/collie.c
index 6c9b82fc5..9991c0c4a 100644
--- a/hw/arm/collie.c
+++ b/hw/arm/collie.c
@@ -58,15 +58,10 @@ static void collie_init(MachineState *machine)
     arm_load_kernel(s->cpu, &collie_binfo);
 }
 
-static QEMUMachine collie_machine = {
-    .name = "collie",
-    .desc = "Collie PDA (SA-1110)",
-    .init = collie_init,
-};
-
-static void collie_machine_init(void)
+static void collie_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&collie_machine);
+    mc->desc = "Sharp SL-5500 (Collie) PDA (SA-1110)";
+    mc->init = collie_init;
 }
 
-machine_init(collie_machine_init)
+DEFINE_MACHINE("collie", collie_machine_init)
diff --git a/hw/arm/cubieboard.c b/hw/arm/cubieboard.c
index 1582250eb..bf068cd3c 100644
--- a/hw/arm/cubieboard.c
+++ b/hw/arm/cubieboard.c
@@ -74,16 +74,10 @@ static void cubieboard_init(MachineState *machine)
     arm_load_kernel(&s->a10->cpu, &cubieboard_binfo);
 }
 
-static QEMUMachine cubieboard_machine = {
-    .name = "cubieboard",
-    .desc = "cubietech cubieboard",
-    .init = cubieboard_init,
-};
-
-
-static void cubieboard_machine_init(void)
+static void cubieboard_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&cubieboard_machine);
+    mc->desc = "cubietech cubieboard";
+    mc->init = cubieboard_init;
 }
 
-machine_init(cubieboard_machine_init)
+DEFINE_MACHINE("cubieboard", cubieboard_machine_init)
diff --git a/hw/arm/digic_boards.c b/hw/arm/digic_boards.c
index f8ba9e595..710045adc 100644
--- a/hw/arm/digic_boards.c
+++ b/hw/arm/digic_boards.c
@@ -148,15 +148,10 @@ static void canon_a1100_init(MachineState *machine)
     digic4_board_init(&digic4_board_canon_a1100);
 }
 
-static QEMUMachine canon_a1100 = {
-    .name = "canon-a1100",
-    .desc = "Canon PowerShot A1100 IS",
-    .init = &canon_a1100_init,
-};
-
-static void digic_register_machines(void)
+static void canon_a1100_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&canon_a1100);
+    mc->desc = "Canon PowerShot A1100 IS";
+    mc->init = &canon_a1100_init;
 }
 
-machine_init(digic_register_machines)
+DEFINE_MACHINE("canon-a1100", canon_a1100_machine_init)
diff --git a/hw/arm/exynos4210.c b/hw/arm/exynos4210.c
index c55fab813..d934980ef 100644
--- a/hw/arm/exynos4210.c
+++ b/hw/arm/exynos4210.c
@@ -259,7 +259,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
     /* Internal ROM */
     memory_region_init_ram(&s->irom_mem, NULL, "exynos4210.irom",
-                           EXYNOS4210_IROM_SIZE, &error_abort);
+                           EXYNOS4210_IROM_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->irom_mem);
     memory_region_set_readonly(&s->irom_mem, true);
     memory_region_add_subregion(system_mem, EXYNOS4210_IROM_BASE_ADDR,
@@ -275,7 +275,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
     /* Internal RAM */
     memory_region_init_ram(&s->iram_mem, NULL, "exynos4210.iram",
-                           EXYNOS4210_IRAM_SIZE, &error_abort);
+                           EXYNOS4210_IRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->iram_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_IRAM_BASE_ADDR,
                                 &s->iram_mem);
@@ -284,14 +284,14 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     mem_size = ram_size;
     if (mem_size > EXYNOS4210_DRAM_MAX_SIZE) {
         memory_region_init_ram(&s->dram1_mem, NULL, "exynos4210.dram1",
-                mem_size - EXYNOS4210_DRAM_MAX_SIZE, &error_abort);
+                mem_size - EXYNOS4210_DRAM_MAX_SIZE, &error_fatal);
         vmstate_register_ram_global(&s->dram1_mem);
         memory_region_add_subregion(system_mem, EXYNOS4210_DRAM1_BASE_ADDR,
                 &s->dram1_mem);
         mem_size = EXYNOS4210_DRAM_MAX_SIZE;
     }
     memory_region_init_ram(&s->dram0_mem, NULL, "exynos4210.dram0", mem_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->dram0_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_DRAM0_BASE_ADDR,
             &s->dram0_mem);
diff --git a/hw/arm/exynos4_boards.c b/hw/arm/exynos4_boards.c
index d644db1ef..da82b27ba 100644
--- a/hw/arm/exynos4_boards.c
+++ b/hw/arm/exynos4_boards.c
@@ -74,8 +74,6 @@ static struct arm_boot_info exynos4_board_binfo = {
     .write_secondary_boot = exynos4210_write_secondary,
 };
 
-static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS];
-
 static void lan9215_init(uint32_t base, qemu_irq irq)
 {
     DeviceState *dev;
@@ -97,11 +95,12 @@ static void lan9215_init(uint32_t base, qemu_irq irq)
 static Exynos4210State *exynos4_boards_init_common(MachineState *machine,
                                                    Exynos4BoardType board_type)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+
     if (smp_cpus != EXYNOS4210_NCPUS && !qtest_enabled()) {
         fprintf(stderr, "%s board supports only %d CPU cores. Ignoring smp_cpus"
                 " value.\n",
-                exynos4_machines[board_type].name,
-                exynos4_machines[board_type].max_cpus);
+                mc->name, EXYNOS4210_NCPUS);
     }
 
     exynos4_board_binfo.ram_size = exynos4_board_ram_size[board_type];
@@ -145,25 +144,40 @@ static void smdkc210_init(MachineState *machine)
     arm_load_kernel(ARM_CPU(first_cpu), &exynos4_board_binfo);
 }
 
-static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS] = {
-    [EXYNOS4_BOARD_NURI] = {
-        .name = "nuri",
-        .desc = "Samsung NURI board (Exynos4210)",
-        .init = nuri_init,
-        .max_cpus = EXYNOS4210_NCPUS,
-    },
-    [EXYNOS4_BOARD_SMDKC210] = {
-        .name = "smdkc210",
-        .desc = "Samsung SMDKC210 board (Exynos4210)",
-        .init = smdkc210_init,
-        .max_cpus = EXYNOS4210_NCPUS,
-    },
+static void nuri_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Samsung NURI board (Exynos4210)";
+    mc->init = nuri_init;
+    mc->max_cpus = EXYNOS4210_NCPUS;
+}
+
+static const TypeInfo nuri_type = {
+    .name = MACHINE_TYPE_NAME("nuri"),
+    .parent = TYPE_MACHINE,
+    .class_init = nuri_class_init,
+};
+
+static void smdkc210_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Samsung SMDKC210 board (Exynos4210)";
+    mc->init = smdkc210_init;
+    mc->max_cpus = EXYNOS4210_NCPUS;
+}
+
+static const TypeInfo smdkc210_type = {
+    .name = MACHINE_TYPE_NAME("smdkc210"),
+    .parent = TYPE_MACHINE,
+    .class_init = smdkc210_class_init,
 };
 
-static void exynos4_machine_init(void)
+static void exynos4_machines_init(void)
 {
-    qemu_register_machine(&exynos4_machines[EXYNOS4_BOARD_NURI]);
-    qemu_register_machine(&exynos4_machines[EXYNOS4_BOARD_SMDKC210]);
+    type_register_static(&nuri_type);
+    type_register_static(&smdkc210_type);
 }
 
-machine_init(exynos4_machine_init);
+machine_init(exynos4_machines_init)
diff --git a/hw/arm/fsl-imx25.c b/hw/arm/fsl-imx25.c
new file mode 100644
index 000000000..e1cadac99
--- /dev/null
+++ b/hw/arm/fsl-imx25.c
@@ -0,0 +1,308 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * i.MX25 SOC emulation.
+ *
+ * Based on hw/arm/xlnx-zynqmp.c
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/arm/fsl-imx25.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "sysemu/char.h"
+
+static void fsl_imx25_init(Object *obj)
+{
+    FslIMX25State *s = FSL_IMX25(obj);
+    int i;
+
+    object_initialize(&s->cpu, sizeof(s->cpu), "arm926-" TYPE_ARM_CPU);
+
+    object_initialize(&s->avic, sizeof(s->avic), TYPE_IMX_AVIC);
+    qdev_set_parent_bus(DEVICE(&s->avic), sysbus_get_default());
+
+    object_initialize(&s->ccm, sizeof(s->ccm), TYPE_IMX_CCM);
+    qdev_set_parent_bus(DEVICE(&s->ccm), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX25_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_IMX_SERIAL);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_GPTS; i++) {
+        object_initialize(&s->gpt[i], sizeof(s->gpt[i]), TYPE_IMX_GPT);
+        qdev_set_parent_bus(DEVICE(&s->gpt[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_EPITS; i++) {
+        object_initialize(&s->epit[i], sizeof(s->epit[i]), TYPE_IMX_EPIT);
+        qdev_set_parent_bus(DEVICE(&s->epit[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->fec, sizeof(s->fec), TYPE_IMX_FEC);
+    qdev_set_parent_bus(DEVICE(&s->fec), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX25_NUM_I2CS; i++) {
+        object_initialize(&s->i2c[i], sizeof(s->i2c[i]), TYPE_IMX_I2C);
+        qdev_set_parent_bus(DEVICE(&s->i2c[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_GPIOS; i++) {
+        object_initialize(&s->gpio[i], sizeof(s->gpio[i]), TYPE_IMX_GPIO);
+        qdev_set_parent_bus(DEVICE(&s->gpio[i]), sysbus_get_default());
+    }
+}
+
+static void fsl_imx25_realize(DeviceState *dev, Error **errp)
+{
+    FslIMX25State *s = FSL_IMX25(dev);
+    uint8_t i;
+    Error *err = NULL;
+
+    object_property_set_bool(OBJECT(&s->cpu), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->avic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->avic), 0, FSL_IMX25_AVIC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 0,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 1,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_FIQ));
+
+    object_property_set_bool(OBJECT(&s->ccm), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->ccm), 0, FSL_IMX25_CCM_ADDR);
+
+    /* Initialize all UARTs */
+    for (i = 0; i < FSL_IMX25_NUM_UARTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } serial_table[FSL_IMX25_NUM_UARTS] = {
+            { FSL_IMX25_UART1_ADDR, FSL_IMX25_UART1_IRQ },
+            { FSL_IMX25_UART2_ADDR, FSL_IMX25_UART2_IRQ },
+            { FSL_IMX25_UART3_ADDR, FSL_IMX25_UART3_IRQ },
+            { FSL_IMX25_UART4_ADDR, FSL_IMX25_UART4_IRQ },
+            { FSL_IMX25_UART5_ADDR, FSL_IMX25_UART5_IRQ }
+        };
+
+        if (i < MAX_SERIAL_PORTS) {
+            CharDriverState *chr;
+
+            chr = serial_hds[i];
+
+            if (!chr) {
+                char label[20];
+                snprintf(label, sizeof(label), "imx31.uart%d", i);
+                chr = qemu_chr_new(label, "null", NULL);
+            }
+
+            qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", chr);
+        }
+
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, serial_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            serial_table[i].irq));
+    }
+
+    /* Initialize all GPT timers */
+    for (i = 0; i < FSL_IMX25_NUM_GPTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpt_table[FSL_IMX25_NUM_GPTS] = {
+            { FSL_IMX25_GPT1_ADDR, FSL_IMX25_GPT1_IRQ },
+            { FSL_IMX25_GPT2_ADDR, FSL_IMX25_GPT2_IRQ },
+            { FSL_IMX25_GPT3_ADDR, FSL_IMX25_GPT3_IRQ },
+            { FSL_IMX25_GPT4_ADDR, FSL_IMX25_GPT4_IRQ }
+        };
+
+        s->gpt[i].ccm = DEVICE(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->gpt[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpt[i]), 0, gpt_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpt[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpt_table[i].irq));
+    }
+
+    /* Initialize all EPIT timers */
+    for (i = 0; i < FSL_IMX25_NUM_EPITS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } epit_table[FSL_IMX25_NUM_EPITS] = {
+            { FSL_IMX25_EPIT1_ADDR, FSL_IMX25_EPIT1_IRQ },
+            { FSL_IMX25_EPIT2_ADDR, FSL_IMX25_EPIT2_IRQ }
+        };
+
+        s->epit[i].ccm = DEVICE(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->epit[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->epit[i]), 0, epit_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->epit[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            epit_table[i].irq));
+    }
+
+    qdev_set_nic_properties(DEVICE(&s->fec), &nd_table[0]);
+    object_property_set_bool(OBJECT(&s->fec), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->fec), 0, FSL_IMX25_FEC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->fec), 0,
+                       qdev_get_gpio_in(DEVICE(&s->avic), FSL_IMX25_FEC_IRQ));
+
+
+    /* Initialize all I2C */
+    for (i = 0; i < FSL_IMX25_NUM_I2CS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } i2c_table[FSL_IMX25_NUM_I2CS] = {
+            { FSL_IMX25_I2C1_ADDR, FSL_IMX25_I2C1_IRQ },
+            { FSL_IMX25_I2C2_ADDR, FSL_IMX25_I2C2_IRQ },
+            { FSL_IMX25_I2C3_ADDR, FSL_IMX25_I2C3_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->i2c[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c[i]), 0, i2c_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            i2c_table[i].irq));
+    }
+
+    /* Initialize all GPIOs */
+    for (i = 0; i < FSL_IMX25_NUM_GPIOS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpio_table[FSL_IMX25_NUM_GPIOS] = {
+            { FSL_IMX25_GPIO1_ADDR, FSL_IMX25_GPIO1_IRQ },
+            { FSL_IMX25_GPIO2_ADDR, FSL_IMX25_GPIO2_IRQ },
+            { FSL_IMX25_GPIO3_ADDR, FSL_IMX25_GPIO3_IRQ },
+            { FSL_IMX25_GPIO4_ADDR, FSL_IMX25_GPIO4_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio[i]), 0, gpio_table[i].addr);
+        /* Connect GPIO IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpio_table[i].irq));
+    }
+
+    /* initialize 2 x 16 KB ROM */
+    memory_region_init_rom_device(&s->rom[0], NULL, NULL, NULL,
+                                  "imx25.rom0", FSL_IMX25_ROM0_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_ROM0_ADDR,
+                                &s->rom[0]);
+    memory_region_init_rom_device(&s->rom[1], NULL, NULL, NULL,
+                                  "imx25.rom1", FSL_IMX25_ROM1_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_ROM1_ADDR,
+                                &s->rom[1]);
+
+    /* initialize internal RAM (128 KB) */
+    memory_region_init_ram(&s->iram, NULL, "imx25.iram", FSL_IMX25_IRAM_SIZE,
+                           &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_IRAM_ADDR,
+                                &s->iram);
+    vmstate_register_ram_global(&s->iram);
+
+    /* internal RAM (128 KB) is aliased over 128 MB - 128 KB */
+    memory_region_init_alias(&s->iram_alias, NULL, "imx25.iram_alias",
+                             &s->iram, 0, FSL_IMX25_IRAM_ALIAS_SIZE);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_IRAM_ALIAS_ADDR,
+                                &s->iram_alias);
+}
+
+static void fsl_imx25_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = fsl_imx25_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo fsl_imx25_type_info = {
+    .name = TYPE_FSL_IMX25,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(FslIMX25State),
+    .instance_init = fsl_imx25_init,
+    .class_init = fsl_imx25_class_init,
+};
+
+static void fsl_imx25_register_types(void)
+{
+    type_register_static(&fsl_imx25_type_info);
+}
+
+type_init(fsl_imx25_register_types)
diff --git a/hw/arm/fsl-imx31.c b/hw/arm/fsl-imx31.c
new file mode 100644
index 000000000..53d447325
--- /dev/null
+++ b/hw/arm/fsl-imx31.c
@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * i.MX31 SOC emulation.
+ *
+ * Based on hw/arm/fsl-imx31.c
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/arm/fsl-imx31.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "sysemu/char.h"
+
+static void fsl_imx31_init(Object *obj)
+{
+    FslIMX31State *s = FSL_IMX31(obj);
+    int i;
+
+    object_initialize(&s->cpu, sizeof(s->cpu), "arm1136-" TYPE_ARM_CPU);
+
+    object_initialize(&s->avic, sizeof(s->avic), TYPE_IMX_AVIC);
+    qdev_set_parent_bus(DEVICE(&s->avic), sysbus_get_default());
+
+    object_initialize(&s->ccm, sizeof(s->ccm), TYPE_IMX_CCM);
+    qdev_set_parent_bus(DEVICE(&s->ccm), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX31_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_IMX_SERIAL);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->gpt, sizeof(s->gpt), TYPE_IMX_GPT);
+    qdev_set_parent_bus(DEVICE(&s->gpt), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX31_NUM_EPITS; i++) {
+        object_initialize(&s->epit[i], sizeof(s->epit[i]), TYPE_IMX_EPIT);
+        qdev_set_parent_bus(DEVICE(&s->epit[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX31_NUM_I2CS; i++) {
+        object_initialize(&s->i2c[i], sizeof(s->i2c[i]), TYPE_IMX_I2C);
+        qdev_set_parent_bus(DEVICE(&s->i2c[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX31_NUM_GPIOS; i++) {
+        object_initialize(&s->gpio[i], sizeof(s->gpio[i]), TYPE_IMX_GPIO);
+        qdev_set_parent_bus(DEVICE(&s->gpio[i]), sysbus_get_default());
+    }
+}
+
+static void fsl_imx31_realize(DeviceState *dev, Error **errp)
+{
+    FslIMX31State *s = FSL_IMX31(dev);
+    uint16_t i;
+    Error *err = NULL;
+
+    object_property_set_bool(OBJECT(&s->cpu), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->avic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->avic), 0, FSL_IMX31_AVIC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 0,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 1,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_FIQ));
+
+    object_property_set_bool(OBJECT(&s->ccm), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->ccm), 0, FSL_IMX31_CCM_ADDR);
+
+    /* Initialize all UARTS */
+    for (i = 0; i < FSL_IMX31_NUM_UARTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } serial_table[FSL_IMX31_NUM_UARTS] = {
+            { FSL_IMX31_UART1_ADDR, FSL_IMX31_UART1_IRQ },
+            { FSL_IMX31_UART2_ADDR, FSL_IMX31_UART2_IRQ },
+        };
+
+        if (i < MAX_SERIAL_PORTS) {
+            CharDriverState *chr;
+
+            chr = serial_hds[i];
+
+            if (!chr) {
+                char label[20];
+                snprintf(label, sizeof(label), "imx31.uart%d", i);
+                chr = qemu_chr_new(label, "null", NULL);
+            }
+
+            qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", chr);
+        }
+
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, serial_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            serial_table[i].irq));
+    }
+
+    s->gpt.ccm = DEVICE(&s->ccm);
+
+    object_property_set_bool(OBJECT(&s->gpt), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpt), 0, FSL_IMX31_GPT_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpt), 0,
+                       qdev_get_gpio_in(DEVICE(&s->avic), FSL_IMX31_GPT_IRQ));
+
+    /* Initialize all EPIT timers */
+    for (i = 0; i < FSL_IMX31_NUM_EPITS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } epit_table[FSL_IMX31_NUM_EPITS] = {
+            { FSL_IMX31_EPIT1_ADDR, FSL_IMX31_EPIT1_IRQ },
+            { FSL_IMX31_EPIT2_ADDR, FSL_IMX31_EPIT2_IRQ },
+        };
+
+        s->epit[i].ccm = DEVICE(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->epit[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->epit[i]), 0, epit_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->epit[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            epit_table[i].irq));
+    }
+
+    /* Initialize all I2C */
+    for (i = 0; i < FSL_IMX31_NUM_I2CS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } i2c_table[FSL_IMX31_NUM_I2CS] = {
+            { FSL_IMX31_I2C1_ADDR, FSL_IMX31_I2C1_IRQ },
+            { FSL_IMX31_I2C2_ADDR, FSL_IMX31_I2C2_IRQ },
+            { FSL_IMX31_I2C3_ADDR, FSL_IMX31_I2C3_IRQ }
+        };
+
+        /* Initialize the I2C */
+        object_property_set_bool(OBJECT(&s->i2c[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        /* Map I2C memory */
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c[i]), 0, i2c_table[i].addr);
+        /* Connect I2C IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            i2c_table[i].irq));
+    }
+
+    /* Initialize all GPIOs */
+    for (i = 0; i < FSL_IMX31_NUM_GPIOS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpio_table[FSL_IMX31_NUM_GPIOS] = {
+            { FSL_IMX31_GPIO1_ADDR, FSL_IMX31_GPIO1_IRQ },
+            { FSL_IMX31_GPIO2_ADDR, FSL_IMX31_GPIO2_IRQ },
+            { FSL_IMX31_GPIO3_ADDR, FSL_IMX31_GPIO3_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->gpio[i]), false, "has-edge-sel",
+                                 &error_abort);
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio[i]), 0, gpio_table[i].addr);
+        /* Connect GPIO IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpio_table[i].irq));
+    }
+
+    /* On a real system, the first 16k is a `secure boot rom' */
+    memory_region_init_rom_device(&s->secure_rom, NULL, NULL, NULL,
+                                  "imx31.secure_rom",
+                                  FSL_IMX31_SECURE_ROM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_SECURE_ROM_ADDR,
+                                &s->secure_rom);
+
+    /* There is also a 16k ROM */
+    memory_region_init_rom_device(&s->rom, NULL, NULL, NULL, "imx31.rom",
+                                  FSL_IMX31_ROM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_ROM_ADDR,
+                                &s->rom);
+
+    /* initialize internal RAM (16 KB) */
+    memory_region_init_ram(&s->iram, NULL, "imx31.iram", FSL_IMX31_IRAM_SIZE,
+                           &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_IRAM_ADDR,
+                                &s->iram);
+    vmstate_register_ram_global(&s->iram);
+
+    /* internal RAM (16 KB) is aliased over 256 MB - 16 KB */
+    memory_region_init_alias(&s->iram_alias, NULL, "imx31.iram_alias",
+                             &s->iram, 0, FSL_IMX31_IRAM_ALIAS_SIZE);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_IRAM_ALIAS_ADDR,
+                                &s->iram_alias);
+}
+
+static void fsl_imx31_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = fsl_imx31_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo fsl_imx31_type_info = {
+    .name = TYPE_FSL_IMX31,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(FslIMX31State),
+    .instance_init = fsl_imx31_init,
+    .class_init = fsl_imx31_class_init,
+};
+
+static void fsl_imx31_register_types(void)
+{
+    type_register_static(&fsl_imx31_type_info);
+}
+
+type_init(fsl_imx31_register_types)
diff --git a/hw/arm/gumstix.c b/hw/arm/gumstix.c
index 8103278b1..32ad041b2 100644
--- a/hw/arm/gumstix.c
+++ b/hw/arm/gumstix.c
@@ -121,22 +121,38 @@ static void verdex_init(MachineState *machine)
                     qdev_get_gpio_in(cpu->gpio, 99));
 }
 
-static QEMUMachine connex_machine = {
-    .name = "connex",
-    .desc = "Gumstix Connex (PXA255)",
-    .init = connex_init,
+static void connex_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Gumstix Connex (PXA255)";
+    mc->init = connex_init;
+}
+
+static const TypeInfo connex_type = {
+    .name = MACHINE_TYPE_NAME("connex"),
+    .parent = TYPE_MACHINE,
+    .class_init = connex_class_init,
 };
 
-static QEMUMachine verdex_machine = {
-    .name = "verdex",
-    .desc = "Gumstix Verdex (PXA270)",
-    .init = verdex_init,
+static void verdex_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Gumstix Verdex (PXA270)";
+    mc->init = verdex_init;
+}
+
+static const TypeInfo verdex_type = {
+    .name = MACHINE_TYPE_NAME("verdex"),
+    .parent = TYPE_MACHINE,
+    .class_init = verdex_class_init,
 };
 
 static void gumstix_machine_init(void)
 {
-    qemu_register_machine(&connex_machine);
-    qemu_register_machine(&verdex_machine);
+    type_register_static(&connex_type);
+    type_register_static(&verdex_type);
 }
 
-machine_init(gumstix_machine_init);
+machine_init(gumstix_machine_init)
diff --git a/hw/arm/highbank.c b/hw/arm/highbank.c
index f8353a787..85ae69efd 100644
--- a/hw/arm/highbank.c
+++ b/hw/arm/highbank.c
@@ -22,6 +22,7 @@
 #include "hw/devices.h"
 #include "hw/loader.h"
 #include "net/net.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "sysemu/block-backend.h"
@@ -32,10 +33,52 @@
 #define SMP_BOOT_REG            0x40
 #define MPCORE_PERIPHBASE       0xfff10000
 
+#define MVBAR_ADDR              0x200
+
 #define NIRQ_GIC                160
 
 /* Board init.  */
 
+/* MVBAR_ADDR is limited by precision of movw */
+
+QEMU_BUILD_BUG_ON(MVBAR_ADDR >= (1 << 16));
+
+#define ARMV7_IMM16(x) (extract32((x),  0, 12) | \
+                        extract32((x), 12,  4) << 16)
+
+static void hb_write_board_setup(ARMCPU *cpu,
+                                 const struct arm_boot_info *info)
+{
+    int n;
+    uint32_t board_setup_blob[] = {
+        /* MVBAR_ADDR */
+        /* Default unimplemented and unused vectors to spin. Makes it
+         * easier to debug (as opposed to the CPU running away).
+         */
+        0xeafffffe, /* notused1: b notused */
+        0xeafffffe, /* notused2: b notused */
+        0xe1b0f00e, /* smc: movs pc, lr - exception return */
+        0xeafffffe, /* prefetch_abort: b prefetch_abort */
+        0xeafffffe, /* data_abort: b data_abort */
+        0xeafffffe, /* notused3: b notused3 */
+        0xeafffffe, /* irq: b irq */
+        0xeafffffe, /* fiq: b fiq */
+#define BOARD_SETUP_ADDR (MVBAR_ADDR + 8 * sizeof(uint32_t))
+        0xe3000000 + ARMV7_IMM16(MVBAR_ADDR), /* movw r0, MVBAR_ADDR */
+        0xee0c0f30, /* mcr p15, 0, r0, c12, c0, 1 - set MVBAR */
+        0xee110f11, /* mrc p15, 0, r0, c1 , c1, 0 - get SCR */
+        0xe3810001, /* orr r0, #1 - set NS */
+        0xee010f11, /* mcr p15, 0, r0, c1 , c1, 0 - set SCR */
+        0xe1600070, /* smc - go to monitor mode to flush NS change */
+        0xe12fff1e, /* bx lr - return to caller */
+    };
+    for (n = 0; n < ARRAY_SIZE(board_setup_blob); n++) {
+        board_setup_blob[n] = tswap32(board_setup_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup", board_setup_blob,
+                       sizeof(board_setup_blob), MVBAR_ADDR);
+}
+
 static void hb_write_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
 {
     int n;
@@ -223,15 +266,13 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
     MemoryRegion *sysmem;
     char *sysboot_filename;
 
-    if (!cpu_model) {
-        switch (machine_id) {
-        case CALXEDA_HIGHBANK:
-            cpu_model = "cortex-a9";
-            break;
-        case CALXEDA_MIDWAY:
-            cpu_model = "cortex-a15";
-            break;
-        }
+    switch (machine_id) {
+    case CALXEDA_HIGHBANK:
+        cpu_model = "cortex-a9";
+        break;
+    case CALXEDA_MIDWAY:
+        cpu_model = "cortex-a15";
+        break;
     }
 
     for (n = 0; n < smp_cpus; n++) {
@@ -240,24 +281,16 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
         ARMCPU *cpu;
         Error *err = NULL;
 
-        if (!oc) {
-            error_report("Unable to find CPU definition");
-            exit(1);
-        }
-
         cpuobj = object_new(object_class_get_name(oc));
         cpu = ARM_CPU(cpuobj);
 
-        /* By default A9 and A15 CPUs have EL3 enabled.  This board does not
-         * currently support EL3 so the CPU EL3 property is disabled before
-         * realization.
-         */
-        if (object_property_find(cpuobj, "has_el3", NULL)) {
-            object_property_set_bool(cpuobj, false, "has_el3", &err);
-            if (err) {
-                error_report_err(err);
-                exit(1);
-            }
+        object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_SMC,
+                                "psci-conduit", &error_abort);
+
+        if (n) {
+            /* Secondary CPUs start in PSCI powered-down state */
+            object_property_set_bool(cpuobj, true,
+                                     "start-powered-off", &error_abort);
         }
 
         if (object_property_find(cpuobj, "reset-cbar", NULL)) {
@@ -281,7 +314,7 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
 
     sysram = g_new(MemoryRegion, 1);
     memory_region_init_ram(sysram, NULL, "highbank.sysram", 0x8000,
-                           &error_abort);
+                           &error_fatal);
     memory_region_add_subregion(sysmem, 0xfff88000, sysram);
     if (bios_name != NULL) {
         sysboot_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
@@ -378,6 +411,16 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
     highbank_binfo.loader_start = 0;
     highbank_binfo.write_secondary_boot = hb_write_secondary;
     highbank_binfo.secondary_cpu_reset_hook = hb_reset_secondary;
+    if (!kvm_enabled()) {
+        highbank_binfo.board_setup_addr = BOARD_SETUP_ADDR;
+        highbank_binfo.write_board_setup = hb_write_board_setup;
+        highbank_binfo.secure_board_setup = true;
+    } else {
+        error_report("WARNING: cannot load built-in Monitor support "
+                     "if KVM is enabled. Some guests (such as Linux) "
+                     "may not boot.");
+    }
+
     arm_load_kernel(ARM_CPU(first_cpu), &highbank_binfo);
 }
 
@@ -391,26 +434,42 @@ static void midway_init(MachineState *machine)
     calxeda_init(machine, CALXEDA_MIDWAY);
 }
 
-static QEMUMachine highbank_machine = {
-    .name = "highbank",
-    .desc = "Calxeda Highbank (ECX-1000)",
-    .init = highbank_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void highbank_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Calxeda Highbank (ECX-1000)";
+    mc->init = highbank_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo highbank_type = {
+    .name = MACHINE_TYPE_NAME("highbank"),
+    .parent = TYPE_MACHINE,
+    .class_init = highbank_class_init,
 };
 
-static QEMUMachine midway_machine = {
-    .name = "midway",
-    .desc = "Calxeda Midway (ECX-2000)",
-    .init = midway_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void midway_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Calxeda Midway (ECX-2000)";
+    mc->init = midway_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo midway_type = {
+    .name = MACHINE_TYPE_NAME("midway"),
+    .parent = TYPE_MACHINE,
+    .class_init = midway_class_init,
 };
 
 static void calxeda_machines_init(void)
 {
-    qemu_register_machine(&highbank_machine);
-    qemu_register_machine(&midway_machine);
+    type_register_static(&highbank_type);
+    type_register_static(&midway_type);
 }
 
-machine_init(calxeda_machines_init);
+machine_init(calxeda_machines_init)
diff --git a/hw/arm/imx25_pdk.c b/hw/arm/imx25_pdk.c
new file mode 100644
index 000000000..59a4c1127
--- /dev/null
+++ b/hw/arm/imx25_pdk.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * PDK Board System emulation.
+ *
+ * Based on hw/arm/kzm.c
+ *
+ * Copyright (c) 2008 OKL and 2011 NICTA
+ * Written by Hans at OK-Labs
+ * Updated by Peter Chubb.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/arm/fsl-imx25.h"
+#include "hw/boards.h"
+#include "qemu/error-report.h"
+#include "exec/address-spaces.h"
+#include "sysemu/qtest.h"
+#include "hw/i2c/i2c.h"
+
+/* Memory map for PDK Emulation Baseboard:
+ * 0x00000000-0x7fffffff See i.MX25 SOC fr support
+ * 0x80000000-0x87ffffff RAM + Alias          EMULATED
+ * 0x90000000-0x9fffffff RAM + Alias          EMULATED
+ * 0xa0000000-0xa7ffffff Flash                IGNORED
+ * 0xa8000000-0xafffffff Flash                IGNORED
+ * 0xb0000000-0xb1ffffff SRAM                 IGNORED
+ * 0xb2000000-0xb3ffffff SRAM                 IGNORED
+ * 0xb4000000-0xb5ffffff CS4                  IGNORED
+ * 0xb6000000-0xb8000fff Reserved             IGNORED
+ * 0xb8001000-0xb8001fff SDRAM CTRL reg       IGNORED
+ * 0xb8002000-0xb8002fff WEIM CTRL reg        IGNORED
+ * 0xb8003000-0xb8003fff M3IF CTRL reg        IGNORED
+ * 0xb8004000-0xb8004fff EMI CTRL reg         IGNORED
+ * 0xb8005000-0xbaffffff Reserved             IGNORED
+ * 0xbb000000-0xbb000fff NAND flash area buf  IGNORED
+ * 0xbb001000-0xbb0011ff NAND flash reserved  IGNORED
+ * 0xbb001200-0xbb001dff Reserved             IGNORED
+ * 0xbb001e00-0xbb001fff NAN flash CTRL reg   IGNORED
+ * 0xbb012000-0xbfffffff Reserved             IGNORED
+ * 0xc0000000-0xffffffff Reserved             IGNORED
+ */
+
+typedef struct IMX25PDK {
+    FslIMX25State soc;
+    MemoryRegion ram;
+    MemoryRegion ram_alias;
+} IMX25PDK;
+
+static struct arm_boot_info imx25_pdk_binfo;
+
+static void imx25_pdk_init(MachineState *machine)
+{
+    IMX25PDK *s = g_new0(IMX25PDK, 1);
+    Error *err = NULL;
+    unsigned int ram_size;
+    unsigned int alias_offset;
+    int i;
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_FSL_IMX25);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &err);
+    if (err != NULL) {
+        error_report("%s", error_get_pretty(err));
+        exit(1);
+    }
+
+    /* We need to initialize our memory */
+    if (machine->ram_size > (FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE)) {
+        error_report("WARNING: RAM size " RAM_ADDR_FMT " above max supported, "
+                     "reduced to %x", machine->ram_size,
+                     FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE);
+        machine->ram_size = FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE;
+    }
+
+    memory_region_allocate_system_memory(&s->ram, NULL, "imx25.ram",
+                                         machine->ram_size);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_SDRAM0_ADDR,
+                                &s->ram);
+
+    /* initialize the alias memory if any */
+    for (i = 0, ram_size = machine->ram_size, alias_offset = 0;
+         (i < 2) && ram_size; i++) {
+        unsigned int size;
+        static const struct {
+            hwaddr addr;
+            unsigned int size;
+        } ram[2] = {
+            { FSL_IMX25_SDRAM0_ADDR, FSL_IMX25_SDRAM0_SIZE },
+            { FSL_IMX25_SDRAM1_ADDR, FSL_IMX25_SDRAM1_SIZE },
+        };
+
+        size = MIN(ram_size, ram[i].size);
+
+        ram_size -= size;
+
+        if (size < ram[i].size) {
+            memory_region_init_alias(&s->ram_alias, NULL, "ram.alias",
+                                     &s->ram, alias_offset, ram[i].size - size);
+            memory_region_add_subregion(get_system_memory(),
+                                        ram[i].addr + size, &s->ram_alias);
+        }
+
+        alias_offset += ram[i].size;
+    }
+
+    imx25_pdk_binfo.ram_size = machine->ram_size;
+    imx25_pdk_binfo.kernel_filename = machine->kernel_filename;
+    imx25_pdk_binfo.kernel_cmdline = machine->kernel_cmdline;
+    imx25_pdk_binfo.initrd_filename = machine->initrd_filename;
+    imx25_pdk_binfo.loader_start = FSL_IMX25_SDRAM0_ADDR;
+    imx25_pdk_binfo.board_id = 1771,
+    imx25_pdk_binfo.nb_cpus = 1;
+
+    /*
+     * We test explicitly for qtest here as it is not done (yet?) in
+     * arm_load_kernel(). Without this the "make check" command would
+     * fail.
+     */
+    if (!qtest_enabled()) {
+        arm_load_kernel(&s->soc.cpu, &imx25_pdk_binfo);
+    } else {
+        /*
+         * This I2C device doesn't exist on the real board.
+         * We add it here (only on qtest usage) to be able to do a bit
+         * of simple qtest. See "make check" for details.
+         */
+        i2c_create_slave((I2CBus *)qdev_get_child_bus(DEVICE(&s->soc.i2c[0]),
+                                                      "i2c"),
+                         "ds1338", 0x68);
+    }
+}
+
+static void imx25_pdk_machine_init(MachineClass *mc)
+{
+    mc->desc = "ARM i.MX25 PDK board (ARM926)";
+    mc->init = imx25_pdk_init;
+}
+
+DEFINE_MACHINE("imx25-pdk", imx25_pdk_machine_init)
diff --git a/hw/arm/integratorcp.c b/hw/arm/integratorcp.c
index 0fbbf997e..421bde9a1 100644
--- a/hw/arm/integratorcp.c
+++ b/hw/arm/integratorcp.c
@@ -266,7 +266,7 @@ static int integratorcm_init(SysBusDevice *dev)
     s->cm_refcnt_offset = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24,
                                    1000);
     memory_region_init_ram(&s->flash, OBJECT(s), "integrator.flash", 0x100000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->flash);
 
     memory_region_init_io(&s->iomem, OBJECT(s), &integratorcm_ops, s,
@@ -619,18 +619,13 @@ static void integratorcp_init(MachineState *machine)
     arm_load_kernel(cpu, &integrator_binfo);
 }
 
-static QEMUMachine integratorcp_machine = {
-    .name = "integratorcp",
-    .desc = "ARM Integrator/CP (ARM926EJ-S)",
-    .init = integratorcp_init,
-};
-
-static void integratorcp_machine_init(void)
+static void integratorcp_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&integratorcp_machine);
+    mc->desc = "ARM Integrator/CP (ARM926EJ-S)";
+    mc->init = integratorcp_init;
 }
 
-machine_init(integratorcp_machine_init);
+DEFINE_MACHINE("integratorcp", integratorcp_machine_init)
 
 static Property core_properties[] = {
     DEFINE_PROP_UINT32("memsz", IntegratorCMState, memsz, 0),
diff --git a/hw/arm/kzm.c b/hw/arm/kzm.c
index 5be0369a5..eff6f4681 100644
--- a/hw/arm/kzm.c
+++ b/hw/arm/kzm.c
@@ -13,141 +13,136 @@
  * i.MX31 SoC
  */
 
-#include "hw/sysbus.h"
+#include "hw/arm/fsl-imx31.h"
+#include "hw/boards.h"
+#include "qemu/error-report.h"
 #include "exec/address-spaces.h"
-#include "hw/hw.h"
-#include "hw/arm/arm.h"
-#include "hw/devices.h"
 #include "net/net.h"
-#include "sysemu/sysemu.h"
-#include "hw/boards.h"
+#include "hw/devices.h"
 #include "hw/char/serial.h"
-#include "hw/arm/imx.h"
-
-    /* Memory map for Kzm Emulation Baseboard:
-     * 0x00000000-0x00003fff 16k secure ROM       IGNORED
-     * 0x00004000-0x00407fff Reserved             IGNORED
-     * 0x00404000-0x00407fff ROM                  IGNORED
-     * 0x00408000-0x0fffffff Reserved             IGNORED
-     * 0x10000000-0x1fffbfff RAM aliasing         IGNORED
-     * 0x1fffc000-0x1fffffff RAM                  EMULATED
-     * 0x20000000-0x2fffffff Reserved             IGNORED
-     * 0x30000000-0x7fffffff I.MX31 Internal Register Space
-     *   0x43f00000 IO_AREA0
-     *   0x43f90000 UART1                         EMULATED
-     *   0x43f94000 UART2                         EMULATED
-     *   0x68000000 AVIC                          EMULATED
-     *   0x53f80000 CCM                           EMULATED
-     *   0x53f94000 PIT 1                         EMULATED
-     *   0x53f98000 PIT 2                         EMULATED
-     *   0x53f90000 GPT                           EMULATED
-     * 0x80000000-0x87ffffff RAM                  EMULATED
-     * 0x88000000-0x8fffffff RAM Aliasing         EMULATED
-     * 0xa0000000-0xafffffff NAND Flash           IGNORED
-     * 0xb0000000-0xb3ffffff Unavailable          IGNORED
-     * 0xb4000000-0xb4000fff 8-bit free space     IGNORED
-     * 0xb4001000-0xb400100f Board control        IGNORED
-     *  0xb4001003           DIP switch
-     * 0xb4001010-0xb400101f 7-segment LED        IGNORED
-     * 0xb4001020-0xb400102f LED                  IGNORED
-     * 0xb4001030-0xb400103f LED                  IGNORED
-     * 0xb4001040-0xb400104f FPGA, UART           EMULATED
-     * 0xb4001050-0xb400105f FPGA, UART           EMULATED
-     * 0xb4001060-0xb40fffff FPGA                 IGNORED
-     * 0xb6000000-0xb61fffff LAN controller       EMULATED
-     * 0xb6200000-0xb62fffff FPGA NAND Controller IGNORED
-     * 0xb6300000-0xb7ffffff Free                 IGNORED
-     * 0xb8000000-0xb8004fff Memory control registers IGNORED
-     * 0xc0000000-0xc3ffffff PCMCIA/CF            IGNORED
-     * 0xc4000000-0xffffffff Reserved             IGNORED
-     */
-
-#define KZM_RAMADDRESS (0x80000000)
-#define KZM_FPGA       (0xb4001040)
+#include "sysemu/qtest.h"
+
+/* Memory map for Kzm Emulation Baseboard:
+ * 0x00000000-0x7fffffff See i.MX31 SOC for support
+ * 0x80000000-0x8fffffff RAM                  EMULATED
+ * 0x90000000-0x9fffffff RAM                  EMULATED
+ * 0xa0000000-0xafffffff Flash                IGNORED
+ * 0xb0000000-0xb3ffffff Unavailable          IGNORED
+ * 0xb4000000-0xb4000fff 8-bit free space     IGNORED
+ * 0xb4001000-0xb400100f Board control        IGNORED
+ *  0xb4001003           DIP switch
+ * 0xb4001010-0xb400101f 7-segment LED        IGNORED
+ * 0xb4001020-0xb400102f LED                  IGNORED
+ * 0xb4001030-0xb400103f LED                  IGNORED
+ * 0xb4001040-0xb400104f FPGA, UART           EMULATED
+ * 0xb4001050-0xb400105f FPGA, UART           EMULATED
+ * 0xb4001060-0xb40fffff FPGA                 IGNORED
+ * 0xb6000000-0xb61fffff LAN controller       EMULATED
+ * 0xb6200000-0xb62fffff FPGA NAND Controller IGNORED
+ * 0xb6300000-0xb7ffffff Free                 IGNORED
+ * 0xb8000000-0xb8004fff Memory control registers IGNORED
+ * 0xc0000000-0xc3ffffff PCMCIA/CF            IGNORED
+ * 0xc4000000-0xffffffff Reserved             IGNORED
+ */
+
+typedef struct IMX31KZM {
+    FslIMX31State soc;
+    MemoryRegion ram;
+    MemoryRegion ram_alias;
+} IMX31KZM;
+
+#define KZM_RAM_ADDR            (FSL_IMX31_SDRAM0_ADDR)
+#define KZM_FPGA_ADDR           (FSL_IMX31_CS4_ADDR + 0x1040)
+#define KZM_LAN9118_ADDR        (FSL_IMX31_CS5_ADDR)
 
 static struct arm_boot_info kzm_binfo = {
-    .loader_start = KZM_RAMADDRESS,
+    .loader_start = KZM_RAM_ADDR,
     .board_id = 1722,
 };
 
 static void kzm_init(MachineState *machine)
 {
-    ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
-    const char *kernel_filename = machine->kernel_filename;
-    const char *kernel_cmdline = machine->kernel_cmdline;
-    const char *initrd_filename = machine->initrd_filename;
-    ARMCPU *cpu;
-    MemoryRegion *address_space_mem = get_system_memory();
-    MemoryRegion *ram = g_new(MemoryRegion, 1);
-    MemoryRegion *sram = g_new(MemoryRegion, 1);
-    MemoryRegion *ram_alias = g_new(MemoryRegion, 1);
-    DeviceState *dev;
-    DeviceState *ccm;
-
-    if (!cpu_model) {
-        cpu_model = "arm1136";
-    }
-
-    cpu = cpu_arm_init(cpu_model);
-    if (!cpu) {
-        fprintf(stderr, "Unable to find CPU definition\n");
+    IMX31KZM *s = g_new0(IMX31KZM, 1);
+    Error *err = NULL;
+    unsigned int ram_size;
+    unsigned int alias_offset;
+    unsigned int i;
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_FSL_IMX31);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &err);
+    if (err != NULL) {
+        error_report("%s", error_get_pretty(err));
         exit(1);
     }
 
-    /* On a real system, the first 16k is a `secure boot rom' */
-
-    memory_region_allocate_system_memory(ram, NULL, "kzm.ram", ram_size);
-    memory_region_add_subregion(address_space_mem, KZM_RAMADDRESS, ram);
-
-    memory_region_init_alias(ram_alias, NULL, "ram.alias", ram, 0, ram_size);
-    memory_region_add_subregion(address_space_mem, 0x88000000, ram_alias);
-
-    memory_region_init_ram(sram, NULL, "kzm.sram", 0x4000, &error_abort);
-    memory_region_add_subregion(address_space_mem, 0x1FFFC000, sram);
-
-    dev = sysbus_create_varargs("imx_avic", 0x68000000,
-                                qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ),
-                                qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_FIQ),
-                                NULL);
-
-    imx_serial_create(0, 0x43f90000, qdev_get_gpio_in(dev, 45));
-    imx_serial_create(1, 0x43f94000, qdev_get_gpio_in(dev, 32));
-
-    ccm = sysbus_create_simple("imx_ccm", 0x53f80000, NULL);
+    /* Check the amount of memory is compatible with the SOC */
+    if (machine->ram_size > (FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE)) {
+        error_report("WARNING: RAM size " RAM_ADDR_FMT " above max supported, "
+                     "reduced to %x", machine->ram_size,
+                     FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE);
+        machine->ram_size = FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE;
+    }
 
-    imx_timerp_create(0x53f94000, qdev_get_gpio_in(dev, 28), ccm);
-    imx_timerp_create(0x53f98000, qdev_get_gpio_in(dev, 27), ccm);
-    imx_timerg_create(0x53f90000, qdev_get_gpio_in(dev, 29), ccm);
+    memory_region_allocate_system_memory(&s->ram, NULL, "kzm.ram",
+                                         machine->ram_size);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_SDRAM0_ADDR,
+                                &s->ram);
+
+    /* initialize the alias memory if any */
+    for (i = 0, ram_size = machine->ram_size, alias_offset = 0;
+         (i < 2) && ram_size; i++) {
+        unsigned int size;
+        static const struct {
+            hwaddr addr;
+            unsigned int size;
+        } ram[2] = {
+            { FSL_IMX31_SDRAM0_ADDR, FSL_IMX31_SDRAM0_SIZE },
+            { FSL_IMX31_SDRAM1_ADDR, FSL_IMX31_SDRAM1_SIZE },
+        };
+
+        size = MIN(ram_size, ram[i].size);
+
+        ram_size -= size;
+
+        if (size < ram[i].size) {
+            memory_region_init_alias(&s->ram_alias, NULL, "ram.alias",
+                                     &s->ram, alias_offset, ram[i].size - size);
+            memory_region_add_subregion(get_system_memory(),
+                                        ram[i].addr + size, &s->ram_alias);
+        }
+
+        alias_offset += ram[i].size;
+    }
 
     if (nd_table[0].used) {
-        lan9118_init(&nd_table[0], 0xb6000000, qdev_get_gpio_in(dev, 52));
+        lan9118_init(&nd_table[0], KZM_LAN9118_ADDR,
+                     qdev_get_gpio_in(DEVICE(&s->soc.avic), 52));
     }
 
     if (serial_hds[2]) { /* touchscreen */
-        serial_mm_init(address_space_mem, KZM_FPGA+0x10, 0,
-                       qdev_get_gpio_in(dev, 52),
-                       14745600, serial_hds[2],
-                       DEVICE_NATIVE_ENDIAN);
+        serial_mm_init(get_system_memory(), KZM_FPGA_ADDR+0x10, 0,
+                       qdev_get_gpio_in(DEVICE(&s->soc.avic), 52),
+                       14745600, serial_hds[2], DEVICE_NATIVE_ENDIAN);
     }
 
-    kzm_binfo.ram_size = ram_size;
-    kzm_binfo.kernel_filename = kernel_filename;
-    kzm_binfo.kernel_cmdline = kernel_cmdline;
-    kzm_binfo.initrd_filename = initrd_filename;
+    kzm_binfo.ram_size = machine->ram_size;
+    kzm_binfo.kernel_filename = machine->kernel_filename;
+    kzm_binfo.kernel_cmdline = machine->kernel_cmdline;
+    kzm_binfo.initrd_filename = machine->initrd_filename;
     kzm_binfo.nb_cpus = 1;
-    arm_load_kernel(cpu, &kzm_binfo);
-}
 
-static QEMUMachine kzm_machine = {
-    .name = "kzm",
-    .desc = "ARM KZM Emulation Baseboard (ARM1136)",
-    .init = kzm_init,
-};
+    if (!qtest_enabled()) {
+        arm_load_kernel(&s->soc.cpu, &kzm_binfo);
+    }
+}
 
-static void kzm_machine_init(void)
+static void kzm_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&kzm_machine);
+    mc->desc = "ARM KZM Emulation Baseboard (ARM1136)";
+    mc->init = kzm_init;
 }
 
-machine_init(kzm_machine_init)
+DEFINE_MACHINE("kzm", kzm_machine_init)
diff --git a/hw/arm/mainstone.c b/hw/arm/mainstone.c
index 0da02a67e..e434cb6ab 100644
--- a/hw/arm/mainstone.c
+++ b/hw/arm/mainstone.c
@@ -124,7 +124,7 @@ static void mainstone_common_init(MemoryRegion *address_space_mem,
     /* Setup CPU & memory */
     mpu = pxa270_init(address_space_mem, mainstone_binfo.ram_size, cpu_model);
     memory_region_init_ram(rom, NULL, "mainstone.rom", MAINSTONE_ROM,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -188,15 +188,10 @@ static void mainstone_init(MachineState *machine)
     mainstone_common_init(get_system_memory(), machine, mainstone, 0x196);
 }
 
-static QEMUMachine mainstone2_machine = {
-    .name = "mainstone",
-    .desc = "Mainstone II (PXA27x)",
-    .init = mainstone_init,
-};
-
-static void mainstone_machine_init(void)
+static void mainstone2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mainstone2_machine);
+    mc->desc = "Mainstone II (PXA27x)";
+    mc->init = mainstone_init;
 }
 
-machine_init(mainstone_machine_init);
+DEFINE_MACHINE("mainstone", mainstone2_machine_init)
diff --git a/hw/arm/musicpal.c b/hw/arm/musicpal.c
index 42f66b33e..b534bb904 100644
--- a/hw/arm/musicpal.c
+++ b/hw/arm/musicpal.c
@@ -1599,7 +1599,7 @@ static void musicpal_init(MachineState *machine)
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     memory_region_init_ram(sram, NULL, "musicpal.sram", MP_SRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, MP_SRAM_BASE, sram);
 
@@ -1709,18 +1709,13 @@ static void musicpal_init(MachineState *machine)
     arm_load_kernel(cpu, &musicpal_binfo);
 }
 
-static QEMUMachine musicpal_machine = {
-    .name = "musicpal",
-    .desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)",
-    .init = musicpal_init,
-};
-
-static void musicpal_machine_init(void)
+static void musicpal_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&musicpal_machine);
+    mc->desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)";
+    mc->init = musicpal_init;
 }
 
-machine_init(musicpal_machine_init);
+DEFINE_MACHINE("musicpal", musicpal_machine_init)
 
 static void mv88w8618_wlan_class_init(ObjectClass *klass, void *data)
 {
diff --git a/hw/arm/netduino2.c b/hw/arm/netduino2.c
index 8f26780ef..a3b9e82ff 100644
--- a/hw/arm/netduino2.c
+++ b/hw/arm/netduino2.c
@@ -43,15 +43,10 @@ static void netduino2_init(MachineState *machine)
     }
 }
 
-static QEMUMachine netduino2_machine = {
-    .name = "netduino2",
-    .desc = "Netduino 2 Machine",
-    .init = netduino2_init,
-};
-
-static void netduino2_machine_init(void)
+static void netduino2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&netduino2_machine);
+    mc->desc = "Netduino 2 Machine";
+    mc->init = netduino2_init;
 }
 
-machine_init(netduino2_machine_init);
+DEFINE_MACHINE("netduino2", netduino2_machine_init)
diff --git a/hw/arm/nseries.c b/hw/arm/nseries.c
index a659e8525..2a8835ec0 100644
--- a/hw/arm/nseries.c
+++ b/hw/arm/nseries.c
@@ -1275,7 +1275,7 @@ static int n8x0_atag_setup(void *p, int model)
     strcpy((void *) w, "hw-build");		/* char component[12] */
     w += 6;
     strcpy((void *) w, "QEMU ");
-    pstrcat((void *) w, 12, qemu_get_version()); /* char version[12] */
+    pstrcat((void *) w, 12, qemu_hw_version()); /* char version[12] */
     w += 6;
 
     tag = (model == 810) ? "1.1.10-qemu" : "1.1.6-qemu";
@@ -1413,24 +1413,40 @@ static void n810_init(MachineState *machine)
     n8x0_init(machine, &n810_binfo, 810);
 }
 
-static QEMUMachine n800_machine = {
-    .name = "n800",
-    .desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)",
-    .init = n800_init,
-    .default_boot_order = "",
+static void n800_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)";
+    mc->init = n800_init;
+    mc->default_boot_order = "";
+}
+
+static const TypeInfo n800_type = {
+    .name = MACHINE_TYPE_NAME("n800"),
+    .parent = TYPE_MACHINE,
+    .class_init = n800_class_init,
 };
 
-static QEMUMachine n810_machine = {
-    .name = "n810",
-    .desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)",
-    .init = n810_init,
-    .default_boot_order = "",
+static void n810_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)";
+    mc->init = n810_init;
+    mc->default_boot_order = "";
+}
+
+static const TypeInfo n810_type = {
+    .name = MACHINE_TYPE_NAME("n810"),
+    .parent = TYPE_MACHINE,
+    .class_init = n810_class_init,
 };
 
 static void nseries_machine_init(void)
 {
-    qemu_register_machine(&n800_machine);
-    qemu_register_machine(&n810_machine);
+    type_register_static(&n800_type);
+    type_register_static(&n810_type);
 }
 
-machine_init(nseries_machine_init);
+machine_init(nseries_machine_init)
diff --git a/hw/arm/omap1.c b/hw/arm/omap1.c
index de2b28925..6b1c07659 100644
--- a/hw/arm/omap1.c
+++ b/hw/arm/omap1.c
@@ -258,8 +258,7 @@ static struct omap_mpu_timer_s *omap_mpu_timer_init(MemoryRegion *system_memory,
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *)
-            g_malloc0(sizeof(struct omap_mpu_timer_s));
+    struct omap_mpu_timer_s *s = g_new0(struct omap_mpu_timer_s, 1);
 
     s->irq = irq;
     s->clk = clk;
@@ -388,8 +387,7 @@ static struct omap_watchdog_timer_s *omap_wd_timer_init(MemoryRegion *memory,
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *)
-            g_malloc0(sizeof(struct omap_watchdog_timer_s));
+    struct omap_watchdog_timer_s *s = g_new0(struct omap_watchdog_timer_s, 1);
 
     s->timer.irq = irq;
     s->timer.clk = clk;
@@ -495,8 +493,7 @@ static struct omap_32khz_timer_s *omap_os_timer_init(MemoryRegion *memory,
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *)
-            g_malloc0(sizeof(struct omap_32khz_timer_s));
+    struct omap_32khz_timer_s *s = g_new0(struct omap_32khz_timer_s, 1);
 
     s->timer.irq = irq;
     s->timer.clk = clk;
@@ -1236,8 +1233,7 @@ static struct omap_tipb_bridge_s *omap_tipb_bridge_init(
     MemoryRegion *memory, hwaddr base,
     qemu_irq abort_irq, omap_clk clk)
 {
-    struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *)
-            g_malloc0(sizeof(struct omap_tipb_bridge_s));
+    struct omap_tipb_bridge_s *s = g_new0(struct omap_tipb_bridge_s, 1);
 
     s->abort = abort_irq;
     omap_tipb_bridge_reset(s);
@@ -2099,8 +2095,7 @@ static struct omap_mpuio_s *omap_mpuio_init(MemoryRegion *memory,
                 qemu_irq kbd_int, qemu_irq gpio_int, qemu_irq wakeup,
                 omap_clk clk)
 {
-    struct omap_mpuio_s *s = (struct omap_mpuio_s *)
-            g_malloc0(sizeof(struct omap_mpuio_s));
+    struct omap_mpuio_s *s = g_new0(struct omap_mpuio_s, 1);
 
     s->irq = gpio_int;
     s->kbd_irq = kbd_int;
@@ -2292,8 +2287,7 @@ static struct omap_uwire_s *omap_uwire_init(MemoryRegion *system_memory,
                                             qemu_irq dma,
                                             omap_clk clk)
 {
-    struct omap_uwire_s *s = (struct omap_uwire_s *)
-            g_malloc0(sizeof(struct omap_uwire_s));
+    struct omap_uwire_s *s = g_new0(struct omap_uwire_s, 1);
 
     s->txirq = txirq;
     s->rxirq = rxirq;
@@ -2932,8 +2926,7 @@ static struct omap_rtc_s *omap_rtc_init(MemoryRegion *system_memory,
                                         qemu_irq timerirq, qemu_irq alarmirq,
                                         omap_clk clk)
 {
-    struct omap_rtc_s *s = (struct omap_rtc_s *)
-            g_malloc0(sizeof(struct omap_rtc_s));
+    struct omap_rtc_s *s = g_new0(struct omap_rtc_s, 1);
 
     s->irq = timerirq;
     s->alarm = alarmirq;
@@ -3468,8 +3461,7 @@ static struct omap_mcbsp_s *omap_mcbsp_init(MemoryRegion *system_memory,
                                             qemu_irq txirq, qemu_irq rxirq,
                                             qemu_irq *dma, omap_clk clk)
 {
-    struct omap_mcbsp_s *s = (struct omap_mcbsp_s *)
-            g_malloc0(sizeof(struct omap_mcbsp_s));
+    struct omap_mcbsp_s *s = g_new0(struct omap_mcbsp_s, 1);
 
     s->txirq = txirq;
     s->rxirq = rxirq;
@@ -3648,8 +3640,7 @@ static void omap_lpg_clk_update(void *opaque, int line, int on)
 static struct omap_lpg_s *omap_lpg_init(MemoryRegion *system_memory,
                                         hwaddr base, omap_clk clk)
 {
-    struct omap_lpg_s *s = (struct omap_lpg_s *)
-            g_malloc0(sizeof(struct omap_lpg_s));
+    struct omap_lpg_s *s = g_new0(struct omap_lpg_s, 1);
 
     s->tm = timer_new_ms(QEMU_CLOCK_VIRTUAL, omap_lpg_tick, s);
 
@@ -3853,8 +3844,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
                 const char *core)
 {
     int i;
-    struct omap_mpu_state_s *s = (struct omap_mpu_state_s *)
-            g_malloc0(sizeof(struct omap_mpu_state_s));
+    struct omap_mpu_state_s *s = g_new0(struct omap_mpu_state_s, 1);
     qemu_irq dma_irqs[6];
     DriveInfo *dinfo;
     SysBusDevice *busdev;
@@ -3882,7 +3872,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
                                          s->sdram_size);
     memory_region_add_subregion(system_memory, OMAP_EMIFF_BASE, &s->emiff_ram);
     memory_region_init_ram(&s->imif_ram, NULL, "omap1.sram", s->sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->imif_ram);
     memory_region_add_subregion(system_memory, OMAP_IMIF_BASE, &s->imif_ram);
 
diff --git a/hw/arm/omap2.c b/hw/arm/omap2.c
index e39b31729..98ee19f86 100644
--- a/hw/arm/omap2.c
+++ b/hw/arm/omap2.c
@@ -596,8 +596,7 @@ static const MemoryRegionOps omap_eac_ops = {
 static struct omap_eac_s *omap_eac_init(struct omap_target_agent_s *ta,
                 qemu_irq irq, qemu_irq *drq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_eac_s *s = (struct omap_eac_s *)
-            g_malloc0(sizeof(struct omap_eac_s));
+    struct omap_eac_s *s = g_new0(struct omap_eac_s, 1);
 
     s->irq = irq;
     s->codec.rxdrq = *drq ++;
@@ -788,8 +787,7 @@ static struct omap_sti_s *omap_sti_init(struct omap_target_agent_s *ta,
                 hwaddr channel_base, qemu_irq irq, omap_clk clk,
                 CharDriverState *chr)
 {
-    struct omap_sti_s *s = (struct omap_sti_s *)
-            g_malloc0(sizeof(struct omap_sti_s));
+    struct omap_sti_s *s = g_new0(struct omap_sti_s, 1);
 
     s->irq = irq;
     omap_sti_reset(s);
@@ -1806,8 +1804,7 @@ static struct omap_prcm_s *omap_prcm_init(struct omap_target_agent_s *ta,
                 qemu_irq mpu_int, qemu_irq dsp_int, qemu_irq iva_int,
                 struct omap_mpu_state_s *mpu)
 {
-    struct omap_prcm_s *s = (struct omap_prcm_s *)
-            g_malloc0(sizeof(struct omap_prcm_s));
+    struct omap_prcm_s *s = g_new0(struct omap_prcm_s, 1);
 
     s->irq[0] = mpu_int;
     s->irq[1] = dsp_int;
@@ -2185,8 +2182,7 @@ static void omap_sysctl_reset(struct omap_sysctl_s *s)
 static struct omap_sysctl_s *omap_sysctl_init(struct omap_target_agent_s *ta,
                 omap_clk iclk, struct omap_mpu_state_s *mpu)
 {
-    struct omap_sysctl_s *s = (struct omap_sysctl_s *)
-            g_malloc0(sizeof(struct omap_sysctl_s));
+    struct omap_sysctl_s *s = g_new0(struct omap_sysctl_s, 1);
 
     s->mpu = mpu;
     omap_sysctl_reset(s);
@@ -2248,8 +2244,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
                 unsigned long sdram_size,
                 const char *core)
 {
-    struct omap_mpu_state_s *s = (struct omap_mpu_state_s *)
-            g_malloc0(sizeof(struct omap_mpu_state_s));
+    struct omap_mpu_state_s *s = g_new0(struct omap_mpu_state_s, 1);
     qemu_irq dma_irqs[4];
     DriveInfo *dinfo;
     int i;
@@ -2276,7 +2271,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
                                          s->sdram_size);
     memory_region_add_subregion(sysmem, OMAP2_Q2_BASE, &s->sdram);
     memory_region_init_ram(&s->sram, NULL, "omap2.sram", s->sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sram);
     memory_region_add_subregion(sysmem, OMAP2_SRAM_BASE, &s->sram);
 
diff --git a/hw/arm/omap_sx1.c b/hw/arm/omap_sx1.c
index 4b0f7f9c4..8eaf8f331 100644
--- a/hw/arm/omap_sx1.c
+++ b/hw/arm/omap_sx1.c
@@ -122,7 +122,7 @@ static void sx1_init(MachineState *machine, const int version)
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, NULL, "omap_sx1.flash0-0", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space, OMAP_CS0_BASE, flash);
@@ -166,7 +166,7 @@ static void sx1_init(MachineState *machine, const int version)
             (dinfo = drive_get(IF_PFLASH, 0, fl_idx)) != NULL) {
         MemoryRegion *flash_1 = g_new(MemoryRegion, 1);
         memory_region_init_ram(flash_1, NULL, "omap_sx1.flash1-0", flash1_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(flash_1);
         memory_region_set_readonly(flash_1, true);
         memory_region_add_subregion(address_space, OMAP_CS1_BASE, flash_1);
@@ -217,22 +217,38 @@ static void sx1_init_v2(MachineState *machine)
     sx1_init(machine, 2);
 }
 
-static QEMUMachine sx1_machine_v2 = {
-    .name = "sx1",
-    .desc = "Siemens SX1 (OMAP310) V2",
-    .init = sx1_init_v2,
+static void sx1_machine_v2_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Siemens SX1 (OMAP310) V2";
+    mc->init = sx1_init_v2;
+}
+
+static const TypeInfo sx1_machine_v2_type = {
+    .name = MACHINE_TYPE_NAME("sx1"),
+    .parent = TYPE_MACHINE,
+    .class_init = sx1_machine_v2_class_init,
 };
 
-static QEMUMachine sx1_machine_v1 = {
-    .name = "sx1-v1",
-    .desc = "Siemens SX1 (OMAP310) V1",
-    .init = sx1_init_v1,
+static void sx1_machine_v1_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Siemens SX1 (OMAP310) V1";
+    mc->init = sx1_init_v1;
+}
+
+static const TypeInfo sx1_machine_v1_type = {
+    .name = MACHINE_TYPE_NAME("sx1-v1"),
+    .parent = TYPE_MACHINE,
+    .class_init = sx1_machine_v1_class_init,
 };
 
 static void sx1_machine_init(void)
 {
-    qemu_register_machine(&sx1_machine_v2);
-    qemu_register_machine(&sx1_machine_v1);
+    type_register_static(&sx1_machine_v1_type);
+    type_register_static(&sx1_machine_v2_type);
 }
 
-machine_init(sx1_machine_init);
+machine_init(sx1_machine_init)
diff --git a/hw/arm/palm.c b/hw/arm/palm.c
index 7f1cfb8f6..82ec99d93 100644
--- a/hw/arm/palm.c
+++ b/hw/arm/palm.c
@@ -213,7 +213,7 @@ static void palmte_init(MachineState *machine)
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, NULL, "palmte.flash", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space_mem, OMAP_CS0_BASE, flash);
@@ -269,15 +269,10 @@ static void palmte_init(MachineState *machine)
     arm_load_kernel(mpu->cpu, &palmte_binfo);
 }
 
-static QEMUMachine palmte_machine = {
-    .name = "cheetah",
-    .desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)",
-    .init = palmte_init,
-};
-
-static void palmte_machine_init(void)
+static void palmte_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&palmte_machine);
+    mc->desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)";
+    mc->init = palmte_init;
 }
 
-machine_init(palmte_machine_init);
+DEFINE_MACHINE("cheetah", palmte_machine_init)
diff --git a/hw/arm/pxa2xx.c b/hw/arm/pxa2xx.c
index d94e20777..79d22d91e 100644
--- a/hw/arm/pxa2xx.c
+++ b/hw/arm/pxa2xx.c
@@ -1731,8 +1731,7 @@ static PXA2xxI2SState *pxa2xx_i2s_init(MemoryRegion *sysmem,
                 hwaddr base,
                 qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma)
 {
-    PXA2xxI2SState *s = (PXA2xxI2SState *)
-            g_malloc0(sizeof(PXA2xxI2SState));
+    PXA2xxI2SState *s = g_new0(PXA2xxI2SState, 1);
 
     s->irq = irq;
     s->rx_dma = rx_dma;
@@ -2061,7 +2060,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
     PXA2xxState *s;
     int i;
     DriveInfo *dinfo;
-    s = (PXA2xxState *) g_malloc0(sizeof(PXA2xxState));
+    s = g_new0(PXA2xxState, 1);
 
     if (revision && strncmp(revision, "pxa27", 5)) {
         fprintf(stderr, "Machine requires a PXA27x processor.\n");
@@ -2079,11 +2078,11 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
 
     /* SDRAM & Internal Memory Storage */
     memory_region_init_ram(&s->sdram, NULL, "pxa270.sdram", sdram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
     memory_region_init_ram(&s->internal, NULL, "pxa270.internal", 0x40000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2157,7 +2156,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
     vmstate_register(NULL, 0, &vmstate_pxa2xx_pm, s);
 
     for (i = 0; pxa27x_ssp[i].io_base; i ++);
-    s->ssp = (SSIBus **)g_malloc0(sizeof(SSIBus *) * i);
+    s->ssp = g_new0(SSIBus *, i);
     for (i = 0; pxa27x_ssp[i].io_base; i ++) {
         DeviceState *dev;
         dev = sysbus_create_simple(TYPE_PXA2XX_SSP, pxa27x_ssp[i].io_base,
@@ -2202,7 +2201,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
     int i;
     DriveInfo *dinfo;
 
-    s = (PXA2xxState *) g_malloc0(sizeof(PXA2xxState));
+    s = g_new0(PXA2xxState, 1);
 
     s->cpu = cpu_arm_init("pxa255");
     if (s->cpu == NULL) {
@@ -2213,11 +2212,11 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
 
     /* SDRAM & Internal Memory Storage */
     memory_region_init_ram(&s->sdram, NULL, "pxa255.sdram", sdram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
     memory_region_init_ram(&s->internal, NULL, "pxa255.internal",
-                           PXA2XX_INTERNAL_SIZE, &error_abort);
+                           PXA2XX_INTERNAL_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2290,7 +2289,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
     vmstate_register(NULL, 0, &vmstate_pxa2xx_pm, s);
 
     for (i = 0; pxa255_ssp[i].io_base; i ++);
-    s->ssp = (SSIBus **)g_malloc0(sizeof(SSIBus *) * i);
+    s->ssp = g_new0(SSIBus *, i);
     for (i = 0; pxa255_ssp[i].io_base; i ++) {
         DeviceState *dev;
         dev = sysbus_create_simple(TYPE_PXA2XX_SSP, pxa255_ssp[i].io_base,
diff --git a/hw/arm/realview.c b/hw/arm/realview.c
index ef2788d3e..e14828db0 100644
--- a/hw/arm/realview.c
+++ b/hw/arm/realview.c
@@ -151,13 +151,13 @@ static void realview_init(MachineState *machine,
         low_ram_size = ram_size - 0x20000000;
         ram_size = 0x20000000;
         memory_region_init_ram(ram_lo, NULL, "realview.lowmem", low_ram_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(ram_lo);
         memory_region_add_subregion(sysmem, 0x20000000, ram_lo);
     }
 
     memory_region_init_ram(ram_hi, NULL, "realview.highmem", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_hi);
     low_ram_size = ram_size;
     if (low_ram_size > 0x10000000)
@@ -353,7 +353,7 @@ static void realview_init(MachineState *machine,
        BootROM happens to be in ROM/flash or in memory that isn't clobbered
        until after Linux boots the secondary CPUs.  */
     memory_region_init_ram(ram_hack, NULL, "realview.hack", 0x1000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_hack);
     memory_region_add_subregion(sysmem, SMP_BOOT_ADDR, ram_hack);
 
@@ -399,41 +399,73 @@ static void realview_pbx_a9_init(MachineState *machine)
     realview_init(machine, BOARD_PBX_A9);
 }
 
-static QEMUMachine realview_eb_machine = {
-    .name = "realview-eb",
-    .desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)",
-    .init = realview_eb_init,
-    .block_default_type = IF_SCSI,
+static void realview_eb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)";
+    mc->init = realview_eb_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo realview_eb_type = {
+    .name = MACHINE_TYPE_NAME("realview-eb"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_eb_class_init,
 };
 
-static QEMUMachine realview_eb_mpcore_machine = {
-    .name = "realview-eb-mpcore",
-    .desc = "ARM RealView Emulation Baseboard (ARM11MPCore)",
-    .init = realview_eb_mpcore_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void realview_eb_mpcore_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Emulation Baseboard (ARM11MPCore)";
+    mc->init = realview_eb_mpcore_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo realview_eb_mpcore_type = {
+    .name = MACHINE_TYPE_NAME("realview-eb-mpcore"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_eb_mpcore_class_init,
 };
 
-static QEMUMachine realview_pb_a8_machine = {
-    .name = "realview-pb-a8",
-    .desc = "ARM RealView Platform Baseboard for Cortex-A8",
-    .init = realview_pb_a8_init,
+static void realview_pb_a8_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Platform Baseboard for Cortex-A8";
+    mc->init = realview_pb_a8_init;
+}
+
+static const TypeInfo realview_pb_a8_type = {
+    .name = MACHINE_TYPE_NAME("realview-pb-a8"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_pb_a8_class_init,
 };
 
-static QEMUMachine realview_pbx_a9_machine = {
-    .name = "realview-pbx-a9",
-    .desc = "ARM RealView Platform Baseboard Explore for Cortex-A9",
-    .init = realview_pbx_a9_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void realview_pbx_a9_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Platform Baseboard Explore for Cortex-A9";
+    mc->init = realview_pbx_a9_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo realview_pbx_a9_type = {
+    .name = MACHINE_TYPE_NAME("realview-pbx-a9"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_pbx_a9_class_init,
 };
 
 static void realview_machine_init(void)
 {
-    qemu_register_machine(&realview_eb_machine);
-    qemu_register_machine(&realview_eb_mpcore_machine);
-    qemu_register_machine(&realview_pb_a8_machine);
-    qemu_register_machine(&realview_pbx_a9_machine);
+    type_register_static(&realview_eb_type);
+    type_register_static(&realview_eb_mpcore_type);
+    type_register_static(&realview_pb_a8_type);
+    type_register_static(&realview_pbx_a9_type);
 }
 
-machine_init(realview_machine_init);
+machine_init(realview_machine_init)
diff --git a/hw/arm/spitz.c b/hw/arm/spitz.c
index 5bf032a63..8d3cc0b6b 100644
--- a/hw/arm/spitz.c
+++ b/hw/arm/spitz.c
@@ -913,7 +913,7 @@ static void spitz_common_init(MachineState *machine,
 
     sl_flash_register(mpu, (model == spitz) ? FLASH_128M : FLASH_1024M);
 
-    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM, &error_abort);
+    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -972,39 +972,71 @@ static void terrier_init(MachineState *machine)
     spitz_common_init(machine, terrier, 0x33f);
 }
 
-static QEMUMachine akitapda_machine = {
-    .name = "akita",
-    .desc = "Akita PDA (PXA270)",
-    .init = akita_init,
+static void akitapda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C1000 (Akita) PDA (PXA270)";
+    mc->init = akita_init;
+}
+
+static const TypeInfo akitapda_type = {
+    .name = MACHINE_TYPE_NAME("akita"),
+    .parent = TYPE_MACHINE,
+    .class_init = akitapda_class_init,
 };
 
-static QEMUMachine spitzpda_machine = {
-    .name = "spitz",
-    .desc = "Spitz PDA (PXA270)",
-    .init = spitz_init,
+static void spitzpda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3000 (Spitz) PDA (PXA270)";
+    mc->init = spitz_init;
+}
+
+static const TypeInfo spitzpda_type = {
+    .name = MACHINE_TYPE_NAME("spitz"),
+    .parent = TYPE_MACHINE,
+    .class_init = spitzpda_class_init,
 };
 
-static QEMUMachine borzoipda_machine = {
-    .name = "borzoi",
-    .desc = "Borzoi PDA (PXA270)",
-    .init = borzoi_init,
+static void borzoipda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3100 (Borzoi) PDA (PXA270)";
+    mc->init = borzoi_init;
+}
+
+static const TypeInfo borzoipda_type = {
+    .name = MACHINE_TYPE_NAME("borzoi"),
+    .parent = TYPE_MACHINE,
+    .class_init = borzoipda_class_init,
 };
 
-static QEMUMachine terrierpda_machine = {
-    .name = "terrier",
-    .desc = "Terrier PDA (PXA270)",
-    .init = terrier_init,
+static void terrierpda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3200 (Terrier) PDA (PXA270)";
+    mc->init = terrier_init;
+}
+
+static const TypeInfo terrierpda_type = {
+    .name = MACHINE_TYPE_NAME("terrier"),
+    .parent = TYPE_MACHINE,
+    .class_init = terrierpda_class_init,
 };
 
 static void spitz_machine_init(void)
 {
-    qemu_register_machine(&akitapda_machine);
-    qemu_register_machine(&spitzpda_machine);
-    qemu_register_machine(&borzoipda_machine);
-    qemu_register_machine(&terrierpda_machine);
+    type_register_static(&akitapda_type);
+    type_register_static(&spitzpda_type);
+    type_register_static(&borzoipda_type);
+    type_register_static(&terrierpda_type);
 }
 
-machine_init(spitz_machine_init);
+machine_init(spitz_machine_init)
 
 static bool is_version_0(void *opaque, int version_id)
 {
@@ -1060,10 +1092,6 @@ static VMStateDescription vmstate_spitz_kbd = {
     },
 };
 
-static Property spitz_keyboard_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void spitz_keyboard_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -1071,7 +1099,6 @@ static void spitz_keyboard_class_init(ObjectClass *klass, void *data)
 
     k->init = spitz_keyboard_init;
     dc->vmsd = &vmstate_spitz_kbd;
-    dc->props = spitz_keyboard_properties;
 }
 
 static const TypeInfo spitz_keyboard_info = {
diff --git a/hw/arm/stellaris.c b/hw/arm/stellaris.c
index cb515ec76..0114e0a7f 100644
--- a/hw/arm/stellaris.c
+++ b/hw/arm/stellaris.c
@@ -16,6 +16,7 @@
 #include "net/net.h"
 #include "hw/boards.h"
 #include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 
 #define GPIO_A 0
 #define GPIO_B 1
@@ -675,7 +676,7 @@ static int stellaris_sys_init(uint32_t base, qemu_irq irq,
 {
     ssys_state *s;
 
-    s = (ssys_state *)g_malloc0(sizeof(ssys_state));
+    s = g_new0(ssys_state, 1);
     s->irq = irq;
     s->board = board;
     /* Most devices come preprogrammed with a MAC address in the user data. */
@@ -1176,6 +1177,14 @@ static int stellaris_adc_init(SysBusDevice *sbd)
     return 0;
 }
 
+static
+void do_sys_reset(void *opaque, int n, int level)
+{
+    if (level) {
+        qemu_system_reset_request();
+    }
+}
+
 /* Board init.  */
 static stellaris_board_info stellaris_boards[] = {
   { "LM3S811EVB",
@@ -1210,8 +1219,7 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         0x40024000, 0x40025000, 0x40026000};
     static const int gpio_irq[7] = {0, 1, 2, 3, 4, 30, 31};
 
-    qemu_irq *pic;
-    DeviceState *gpio_dev[7];
+    DeviceState *gpio_dev[7], *nvic;
     qemu_irq gpio_in[7][8];
     qemu_irq gpio_out[7][8];
     qemu_irq adc;
@@ -1231,22 +1239,29 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
 
     /* Flash programming is done via the SCU, so pretend it is ROM.  */
     memory_region_init_ram(flash, NULL, "stellaris.flash", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(system_memory, 0, flash);
 
     memory_region_init_ram(sram, NULL, "stellaris.sram", sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(system_memory, 0x20000000, sram);
 
-    pic = armv7m_init(system_memory, flash_size, NUM_IRQ_LINES,
+    nvic = armv7m_init(system_memory, flash_size, NUM_IRQ_LINES,
                       kernel_filename, cpu_model);
 
+    qdev_connect_gpio_out_named(nvic, "SYSRESETREQ", 0,
+                                qemu_allocate_irq(&do_sys_reset, NULL, 0));
+
     if (board->dc1 & (1 << 16)) {
         dev = sysbus_create_varargs(TYPE_STELLARIS_ADC, 0x40038000,
-                                    pic[14], pic[15], pic[16], pic[17], NULL);
+                                    qdev_get_gpio_in(nvic, 14),
+                                    qdev_get_gpio_in(nvic, 15),
+                                    qdev_get_gpio_in(nvic, 16),
+                                    qdev_get_gpio_in(nvic, 17),
+                                    NULL);
         adc = qdev_get_gpio_in(dev, 0);
     } else {
         adc = NULL;
@@ -1255,19 +1270,21 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         if (board->dc2 & (0x10000 << i)) {
             dev = sysbus_create_simple(TYPE_STELLARIS_GPTM,
                                        0x40030000 + i * 0x1000,
-                                       pic[timer_irq[i]]);
+                                       qdev_get_gpio_in(nvic, timer_irq[i]));
             /* TODO: This is incorrect, but we get away with it because
                the ADC output is only ever pulsed.  */
             qdev_connect_gpio_out(dev, 0, adc);
         }
     }
 
-    stellaris_sys_init(0x400fe000, pic[28], board, nd_table[0].macaddr.a);
+    stellaris_sys_init(0x400fe000, qdev_get_gpio_in(nvic, 28),
+                       board, nd_table[0].macaddr.a);
 
     for (i = 0; i < 7; i++) {
         if (board->dc4 & (1 << i)) {
             gpio_dev[i] = sysbus_create_simple("pl061_luminary", gpio_addr[i],
-                                               pic[gpio_irq[i]]);
+                                               qdev_get_gpio_in(nvic,
+                                                                gpio_irq[i]));
             for (j = 0; j < 8; j++) {
                 gpio_in[i][j] = qdev_get_gpio_in(gpio_dev[i], j);
                 gpio_out[i][j] = NULL;
@@ -1276,7 +1293,8 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
     }
 
     if (board->dc2 & (1 << 12)) {
-        dev = sysbus_create_simple(TYPE_STELLARIS_I2C, 0x40020000, pic[8]);
+        dev = sysbus_create_simple(TYPE_STELLARIS_I2C, 0x40020000,
+                                   qdev_get_gpio_in(nvic, 8));
         i2c = (I2CBus *)qdev_get_child_bus(dev, "i2c");
         if (board->peripherals & BP_OLED_I2C) {
             i2c_create_slave(i2c, "ssd0303", 0x3d);
@@ -1286,11 +1304,12 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
     for (i = 0; i < 4; i++) {
         if (board->dc2 & (1 << i)) {
             sysbus_create_simple("pl011_luminary", 0x4000c000 + i * 0x1000,
-                                 pic[uart_irq[i]]);
+                                 qdev_get_gpio_in(nvic, uart_irq[i]));
         }
     }
     if (board->dc2 & (1 << 4)) {
-        dev = sysbus_create_simple("pl022", 0x40008000, pic[7]);
+        dev = sysbus_create_simple("pl022", 0x40008000,
+                                   qdev_get_gpio_in(nvic, 7));
         if (board->peripherals & BP_OLED_SSI) {
             void *bus;
             DeviceState *sddev;
@@ -1326,7 +1345,7 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         qdev_set_nic_properties(enet, &nd_table[0]);
         qdev_init_nofail(enet);
         sysbus_mmio_map(SYS_BUS_DEVICE(enet), 0, 0x40048000);
-        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, pic[42]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, qdev_get_gpio_in(nvic, 42));
     }
     if (board->peripherals & BP_GAMEPAD) {
         qemu_irq gpad_irq[5];
@@ -1366,25 +1385,41 @@ static void lm3s6965evb_init(MachineState *machine)
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[1]);
 }
 
-static QEMUMachine lm3s811evb_machine = {
-    .name = "lm3s811evb",
-    .desc = "Stellaris LM3S811EVB",
-    .init = lm3s811evb_init,
+static void lm3s811evb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Stellaris LM3S811EVB";
+    mc->init = lm3s811evb_init;
+}
+
+static const TypeInfo lm3s811evb_type = {
+    .name = MACHINE_TYPE_NAME("lm3s811evb"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm3s811evb_class_init,
 };
 
-static QEMUMachine lm3s6965evb_machine = {
-    .name = "lm3s6965evb",
-    .desc = "Stellaris LM3S6965EVB",
-    .init = lm3s6965evb_init,
+static void lm3s6965evb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Stellaris LM3S6965EVB";
+    mc->init = lm3s6965evb_init;
+}
+
+static const TypeInfo lm3s6965evb_type = {
+    .name = MACHINE_TYPE_NAME("lm3s6965evb"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm3s6965evb_class_init,
 };
 
 static void stellaris_machine_init(void)
 {
-    qemu_register_machine(&lm3s811evb_machine);
-    qemu_register_machine(&lm3s6965evb_machine);
+    type_register_static(&lm3s811evb_type);
+    type_register_static(&lm3s6965evb_type);
 }
 
-machine_init(stellaris_machine_init);
+machine_init(stellaris_machine_init)
 
 static void stellaris_i2c_class_init(ObjectClass *klass, void *data)
 {
diff --git a/hw/arm/stm32f205_soc.c b/hw/arm/stm32f205_soc.c
index 0f3bdc77b..3f993406d 100644
--- a/hw/arm/stm32f205_soc.c
+++ b/hw/arm/stm32f205_soc.c
@@ -59,9 +59,8 @@ static void stm32f205_soc_initfn(Object *obj)
 static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
 {
     STM32F205State *s = STM32F205_SOC(dev_soc);
-    DeviceState *syscfgdev, *usartdev, *timerdev;
+    DeviceState *syscfgdev, *usartdev, *timerdev, *nvic;
     SysBusDevice *syscfgbusdev, *usartbusdev, *timerbusdev;
-    qemu_irq *pic;
     Error *err = NULL;
     int i;
 
@@ -71,7 +70,7 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
     MemoryRegion *flash_alias = g_new(MemoryRegion, 1);
 
     memory_region_init_ram(flash, NULL, "STM32F205.flash", FLASH_SIZE,
-                           &error_abort);
+                           &error_fatal);
     memory_region_init_alias(flash_alias, NULL, "STM32F205.flash.alias",
                              flash, 0, FLASH_SIZE);
 
@@ -84,12 +83,12 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
     memory_region_add_subregion(system_memory, 0, flash_alias);
 
     memory_region_init_ram(sram, NULL, "STM32F205.sram", SRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(system_memory, SRAM_BASE_ADDRESS, sram);
 
-    pic = armv7m_init(get_system_memory(), FLASH_SIZE, 96,
-                      s->kernel_filename, s->cpu_model);
+    nvic = armv7m_init(get_system_memory(), FLASH_SIZE, 96,
+                       s->kernel_filename, s->cpu_model);
 
     /* System configuration controller */
     syscfgdev = DEVICE(&s->syscfg);
@@ -100,7 +99,7 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
     }
     syscfgbusdev = SYS_BUS_DEVICE(syscfgdev);
     sysbus_mmio_map(syscfgbusdev, 0, 0x40013800);
-    sysbus_connect_irq(syscfgbusdev, 0, pic[71]);
+    sysbus_connect_irq(syscfgbusdev, 0, qdev_get_gpio_in(nvic, 71));
 
     /* Attach UART (uses USART registers) and USART controllers */
     for (i = 0; i < STM_NUM_USARTS; i++) {
@@ -112,7 +111,8 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
         }
         usartbusdev = SYS_BUS_DEVICE(usartdev);
         sysbus_mmio_map(usartbusdev, 0, usart_addr[i]);
-        sysbus_connect_irq(usartbusdev, 0, pic[usart_irq[i]]);
+        sysbus_connect_irq(usartbusdev, 0,
+                           qdev_get_gpio_in(nvic, usart_irq[i]));
     }
 
     /* Timer 2 to 5 */
@@ -126,7 +126,8 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
         }
         timerbusdev = SYS_BUS_DEVICE(timerdev);
         sysbus_mmio_map(timerbusdev, 0, timer_addr[i]);
-        sysbus_connect_irq(timerbusdev, 0, pic[timer_irq[i]]);
+        sysbus_connect_irq(timerbusdev, 0,
+                           qdev_get_gpio_in(nvic, timer_irq[i]));
     }
 }
 
diff --git a/hw/arm/strongarm.c b/hw/arm/strongarm.c
index da9fc1d51..9624ecb58 100644
--- a/hw/arm/strongarm.c
+++ b/hw/arm/strongarm.c
@@ -1588,7 +1588,7 @@ StrongARMState *sa1110_init(MemoryRegion *sysmem,
     StrongARMState *s;
     int i;
 
-    s = g_malloc0(sizeof(StrongARMState));
+    s = g_new0(StrongARMState, 1);
 
     if (!rev) {
         rev = "sa1110-b5";
diff --git a/hw/arm/tosa.c b/hw/arm/tosa.c
index 73572ebe0..02814d7ab 100644
--- a/hw/arm/tosa.c
+++ b/hw/arm/tosa.c
@@ -227,7 +227,7 @@ static void tosa_init(MachineState *machine)
 
     mpu = pxa255_init(address_space_mem, tosa_binfo.ram_size);
 
-    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM, &error_abort);
+    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -252,18 +252,13 @@ static void tosa_init(MachineState *machine)
     sl_bootparam_write(SL_PXA_PARAM_BASE);
 }
 
-static QEMUMachine tosapda_machine = {
-    .name = "tosa",
-    .desc = "Tosa PDA (PXA255)",
-    .init = tosa_init,
-};
-
-static void tosapda_machine_init(void)
+static void tosapda_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&tosapda_machine);
+    mc->desc = "Sharp SL-6000 (Tosa) PDA (PXA255)";
+    mc->init = tosa_init;
 }
 
-machine_init(tosapda_machine_init);
+DEFINE_MACHINE("tosa", tosapda_machine_init)
 
 static void tosa_dac_class_init(ObjectClass *klass, void *data)
 {
diff --git a/hw/arm/versatilepb.c b/hw/arm/versatilepb.c
index 6c69f4eaa..912c2908f 100644
--- a/hw/arm/versatilepb.c
+++ b/hw/arm/versatilepb.c
@@ -391,27 +391,43 @@ static void vab_init(MachineState *machine)
     versatile_init(machine, 0x25e);
 }
 
-static QEMUMachine versatilepb_machine = {
-    .name = "versatilepb",
-    .desc = "ARM Versatile/PB (ARM926EJ-S)",
-    .init = vpb_init,
-    .block_default_type = IF_SCSI,
+static void versatilepb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile/PB (ARM926EJ-S)";
+    mc->init = vpb_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo versatilepb_type = {
+    .name = MACHINE_TYPE_NAME("versatilepb"),
+    .parent = TYPE_MACHINE,
+    .class_init = versatilepb_class_init,
 };
 
-static QEMUMachine versatileab_machine = {
-    .name = "versatileab",
-    .desc = "ARM Versatile/AB (ARM926EJ-S)",
-    .init = vab_init,
-    .block_default_type = IF_SCSI,
+static void versatileab_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile/AB (ARM926EJ-S)";
+    mc->init = vab_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo versatileab_type = {
+    .name = MACHINE_TYPE_NAME("versatileab"),
+    .parent = TYPE_MACHINE,
+    .class_init = versatileab_class_init,
 };
 
 static void versatile_machine_init(void)
 {
-    qemu_register_machine(&versatilepb_machine);
-    qemu_register_machine(&versatileab_machine);
+    type_register_static(&versatilepb_type);
+    type_register_static(&versatileab_type);
 }
 
-machine_init(versatile_machine_init);
+machine_init(versatile_machine_init)
 
 static void vpb_sic_class_init(ObjectClass *klass, void *data)
 {
diff --git a/hw/arm/vexpress.c b/hw/arm/vexpress.c
index da217884e..058abbde3 100644
--- a/hw/arm/vexpress.c
+++ b/hw/arm/vexpress.c
@@ -168,8 +168,8 @@ typedef struct {
 } VexpressMachineState;
 
 #define TYPE_VEXPRESS_MACHINE   "vexpress"
-#define TYPE_VEXPRESS_A9_MACHINE   "vexpress-a9"
-#define TYPE_VEXPRESS_A15_MACHINE   "vexpress-a15"
+#define TYPE_VEXPRESS_A9_MACHINE   MACHINE_TYPE_NAME("vexpress-a9")
+#define TYPE_VEXPRESS_A15_MACHINE   MACHINE_TYPE_NAME("vexpress-a15")
 #define VEXPRESS_MACHINE(obj) \
     OBJECT_CHECK(VexpressMachineState, (obj), TYPE_VEXPRESS_MACHINE)
 #define VEXPRESS_MACHINE_GET_CLASS(obj) \
@@ -391,7 +391,7 @@ static void a15_daughterboard_init(const VexpressMachineState *vms,
     /* 0x2b0a0000: PL341 dynamic memory controller: not modelled */
     /* 0x2e000000: system SRAM */
     memory_region_init_ram(sram, NULL, "vexpress.a15sram", 0x10000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0x2e000000, sram);
 
@@ -541,7 +541,7 @@ static void vexpress_common_init(MachineState *machine)
 {
     VexpressMachineState *vms = VEXPRESS_MACHINE(machine);
     VexpressMachineClass *vmc = VEXPRESS_MACHINE_GET_CLASS(machine);
-    VEDBoardInfo *daughterboard = vmc->daughterboard;;
+    VEDBoardInfo *daughterboard = vmc->daughterboard;
     DeviceState *dev, *sysctl, *pl041;
     qemu_irq pic[64];
     uint32_t sys_id;
@@ -671,13 +671,13 @@ static void vexpress_common_init(MachineState *machine)
 
     sram_size = 0x2000000;
     memory_region_init_ram(sram, NULL, "vexpress.sram", sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, map[VE_SRAM], sram);
 
     vram_size = 0x800000;
     memory_region_init_ram(vram, NULL, "vexpress.vram", vram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(vram);
     memory_region_add_subregion(sysmem, map[VE_VIDEORAM], vram);
 
@@ -747,7 +747,6 @@ static void vexpress_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = TYPE_VEXPRESS_MACHINE;
     mc->desc = "ARM Versatile Express";
     mc->init = vexpress_common_init;
     mc->block_default_type = IF_SCSI;
@@ -759,10 +758,9 @@ static void vexpress_a9_class_init(ObjectClass *oc, void *data)
     MachineClass *mc = MACHINE_CLASS(oc);
     VexpressMachineClass *vmc = VEXPRESS_MACHINE_CLASS(oc);
 
-    mc->name = TYPE_VEXPRESS_A9_MACHINE;
     mc->desc = "ARM Versatile Express for Cortex-A9";
 
-    vmc->daughterboard = &a9_daughterboard;;
+    vmc->daughterboard = &a9_daughterboard;
 }
 
 static void vexpress_a15_class_init(ObjectClass *oc, void *data)
@@ -770,7 +768,6 @@ static void vexpress_a15_class_init(ObjectClass *oc, void *data)
     MachineClass *mc = MACHINE_CLASS(oc);
     VexpressMachineClass *vmc = VEXPRESS_MACHINE_CLASS(oc);
 
-    mc->name = TYPE_VEXPRESS_A15_MACHINE;
     mc->desc = "ARM Versatile Express for Cortex-A15";
 
     vmc->daughterboard = &a15_daughterboard;
diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
index f36514031..3c2c5d6bf 100644
--- a/hw/arm/virt-acpi-build.c
+++ b/hw/arm/virt-acpi-build.c
@@ -114,7 +114,7 @@ static void acpi_dsdt_add_flash(Aml *scope, const MemMapEntry *flash_memmap)
 {
     Aml *dev, *crs;
     hwaddr base = flash_memmap->base;
-    hwaddr size = flash_memmap->size;
+    hwaddr size = flash_memmap->size / 2;
 
     dev = aml_device("FLS0");
     aml_append(dev, aml_name_decl("_HID", aml_string("LNRO0015")));
@@ -159,7 +159,8 @@ static void acpi_dsdt_add_virtio(Aml *scope,
     }
 }
 
-static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
+static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq,
+                              bool use_highmem)
 {
     Aml *method, *crs, *ifctx, *UUID, *ifctx1, *elsectx, *buf;
     int i, bus_no;
@@ -179,6 +180,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
     aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
     aml_append(dev, aml_name_decl("_UID", aml_string("PCI0")));
     aml_append(dev, aml_name_decl("_STR", aml_unicode("PCIe 0 Device")));
+    aml_append(dev, aml_name_decl("_CCA", aml_int(1)));
 
     /* Declare the PCI Routing Table. */
     Aml *rt_pkg = aml_package(nr_pcie_buses * PCI_NUM_PINS);
@@ -234,6 +236,17 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
                      AML_ENTIRE_RANGE, 0x0000, 0x0000, size_pio - 1, base_pio,
                      size_pio));
 
+    if (use_highmem) {
+        hwaddr base_mmio_high = memmap[VIRT_PCIE_MMIO_HIGH].base;
+        hwaddr size_mmio_high = memmap[VIRT_PCIE_MMIO_HIGH].size;
+
+        aml_append(rbuf,
+            aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                             AML_NON_CACHEABLE, AML_READ_WRITE, 0x0000,
+                             base_mmio_high, base_mmio_high, 0x0000,
+                             size_mmio_high));
+    }
+
     aml_append(method, aml_name_decl("RBUF", rbuf));
     aml_append(method, aml_return(rbuf));
     aml_append(dev, method);
@@ -431,33 +444,47 @@ build_madt(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info,
 
     madt = acpi_data_push(table_data, sizeof *madt);
 
+    gicd = acpi_data_push(table_data, sizeof *gicd);
+    gicd->type = ACPI_APIC_GENERIC_DISTRIBUTOR;
+    gicd->length = sizeof(*gicd);
+    gicd->base_address = memmap[VIRT_GIC_DIST].base;
+
     for (i = 0; i < guest_info->smp_cpus; i++) {
         AcpiMadtGenericInterrupt *gicc = acpi_data_push(table_data,
                                                      sizeof *gicc);
+        ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(i));
+
         gicc->type = ACPI_APIC_GENERIC_INTERRUPT;
         gicc->length = sizeof(*gicc);
-        gicc->base_address = memmap[VIRT_GIC_CPU].base;
+        if (guest_info->gic_version == 2) {
+            gicc->base_address = memmap[VIRT_GIC_CPU].base;
+        }
         gicc->cpu_interface_number = i;
-        gicc->arm_mpidr = i;
+        gicc->arm_mpidr = armcpu->mp_affinity;
         gicc->uid = i;
         if (test_bit(i, cpuinfo->found_cpus)) {
             gicc->flags = cpu_to_le32(ACPI_GICC_ENABLED);
         }
     }
 
-    gicd = acpi_data_push(table_data, sizeof *gicd);
-    gicd->type = ACPI_APIC_GENERIC_DISTRIBUTOR;
-    gicd->length = sizeof(*gicd);
-    gicd->base_address = memmap[VIRT_GIC_DIST].base;
-
-    gic_msi = acpi_data_push(table_data, sizeof *gic_msi);
-    gic_msi->type = ACPI_APIC_GENERIC_MSI_FRAME;
-    gic_msi->length = sizeof(*gic_msi);
-    gic_msi->gic_msi_frame_id = 0;
-    gic_msi->base_address = cpu_to_le64(memmap[VIRT_GIC_V2M].base);
-    gic_msi->flags = cpu_to_le32(1);
-    gic_msi->spi_count = cpu_to_le16(NUM_GICV2M_SPIS);
-    gic_msi->spi_base = cpu_to_le16(irqmap[VIRT_GIC_V2M] + ARM_SPI_BASE);
+    if (guest_info->gic_version == 3) {
+        AcpiMadtGenericRedistributor *gicr = acpi_data_push(table_data,
+                                                         sizeof *gicr);
+
+        gicr->type = ACPI_APIC_GENERIC_REDISTRIBUTOR;
+        gicr->length = sizeof(*gicr);
+        gicr->base_address = cpu_to_le64(memmap[VIRT_GIC_REDIST].base);
+        gicr->range_length = cpu_to_le32(memmap[VIRT_GIC_REDIST].size);
+    } else {
+        gic_msi = acpi_data_push(table_data, sizeof *gic_msi);
+        gic_msi->type = ACPI_APIC_GENERIC_MSI_FRAME;
+        gic_msi->length = sizeof(*gic_msi);
+        gic_msi->gic_msi_frame_id = 0;
+        gic_msi->base_address = cpu_to_le64(memmap[VIRT_GIC_V2M].base);
+        gic_msi->flags = cpu_to_le32(1);
+        gic_msi->spi_count = cpu_to_le16(NUM_GICV2M_SPIS);
+        gic_msi->spi_base = cpu_to_le16(irqmap[VIRT_GIC_V2M] + ARM_SPI_BASE);
+    }
 
     build_header(linker, table_data,
                  (void *)(table_data->data + madt_start), "APIC",
@@ -510,7 +537,8 @@ build_dsdt(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info)
     acpi_dsdt_add_flash(scope, &memmap[VIRT_FLASH]);
     acpi_dsdt_add_virtio(scope, &memmap[VIRT_MMIO],
                     (irqmap[VIRT_MMIO] + ARM_SPI_BASE), NUM_VIRTIO_TRANSPORTS);
-    acpi_dsdt_add_pci(scope, memmap, (irqmap[VIRT_PCIE] + ARM_SPI_BASE));
+    acpi_dsdt_add_pci(scope, memmap, (irqmap[VIRT_PCIE] + ARM_SPI_BASE),
+                      guest_info->use_highmem);
 
     aml_append(dsdt, scope);
 
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 484689264..9c6792cea 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -48,6 +48,10 @@
 #include "hw/arm/sysbus-fdt.h"
 #include "hw/platform-bus.h"
 #include "hw/arm/fdt.h"
+#include "hw/intc/arm_gic_common.h"
+#include "kvm_arm.h"
+#include "hw/smbios/smbios.h"
+#include "qapi/visitor.h"
 
 /* Number of external interrupt lines to configure the GIC with */
 #define NUM_IRQS 256
@@ -77,9 +81,11 @@ typedef struct {
 typedef struct {
     MachineState parent;
     bool secure;
+    bool highmem;
+    int32_t gic_version;
 } VirtMachineState;
 
-#define TYPE_VIRT_MACHINE   "virt"
+#define TYPE_VIRT_MACHINE   MACHINE_TYPE_NAME("virt")
 #define VIRT_MACHINE(obj) \
     OBJECT_CHECK(VirtMachineState, (obj), TYPE_VIRT_MACHINE)
 #define VIRT_MACHINE_GET_CLASS(obj) \
@@ -107,9 +113,13 @@ static const MemMapEntry a15memmap[] = {
     [VIRT_GIC_DIST] =           { 0x08000000, 0x00010000 },
     [VIRT_GIC_CPU] =            { 0x08010000, 0x00010000 },
     [VIRT_GIC_V2M] =            { 0x08020000, 0x00001000 },
+    /* The space in between here is reserved for GICv3 CPU/vCPU/HYP */
+    [VIRT_GIC_ITS] =            { 0x08080000, 0x00020000 },
+    /* This redistributor space allows up to 2*64kB*123 CPUs */
+    [VIRT_GIC_REDIST] =         { 0x080A0000, 0x00F60000 },
     [VIRT_UART] =               { 0x09000000, 0x00001000 },
     [VIRT_RTC] =                { 0x09010000, 0x00001000 },
-    [VIRT_FW_CFG] =             { 0x09020000, 0x0000000a },
+    [VIRT_FW_CFG] =             { 0x09020000, 0x00000018 },
     [VIRT_MMIO] =               { 0x0a000000, 0x00000200 },
     /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */
     [VIRT_PLATFORM_BUS] =       { 0x0c000000, 0x02000000 },
@@ -117,6 +127,8 @@ static const MemMapEntry a15memmap[] = {
     [VIRT_PCIE_PIO] =           { 0x3eff0000, 0x00010000 },
     [VIRT_PCIE_ECAM] =          { 0x3f000000, 0x01000000 },
     [VIRT_MEM] =                { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
+    /* Second PCIe window, 512GB wide at the 512GB boundary */
+    [VIRT_PCIE_MMIO_HIGH] =   { 0x8000000000ULL, 0x8000000000ULL },
 };
 
 static const int a15irqmap[] = {
@@ -249,7 +261,7 @@ static void fdt_add_psci_node(const VirtBoardInfo *vbi)
     qemu_fdt_setprop_cell(fdt, "/psci", "migrate", migrate_fn);
 }
 
-static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
+static void fdt_add_timer_nodes(const VirtBoardInfo *vbi, int gictype)
 {
     /* Note that on A15 h/w these interrupts are level-triggered,
      * but for the GIC implementation provided by both QEMU and KVM
@@ -258,8 +270,11 @@ static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
     ARMCPU *armcpu;
     uint32_t irqflags = GIC_FDT_IRQ_FLAGS_EDGE_LO_HI;
 
-    irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
-                         GIC_FDT_IRQ_PPI_CPU_WIDTH, (1 << vbi->smp_cpus) - 1);
+    if (gictype == 2) {
+        irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
+                             GIC_FDT_IRQ_PPI_CPU_WIDTH,
+                             (1 << vbi->smp_cpus) - 1);
+    }
 
     qemu_fdt_add_subnode(vbi->fdt, "/timer");
 
@@ -282,9 +297,32 @@ static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
 static void fdt_add_cpu_nodes(const VirtBoardInfo *vbi)
 {
     int cpu;
+    int addr_cells = 1;
+
+    /*
+     * From Documentation/devicetree/bindings/arm/cpus.txt
+     *  On ARM v8 64-bit systems value should be set to 2,
+     *  that corresponds to the MPIDR_EL1 register size.
+     *  If MPIDR_EL1[63:32] value is equal to 0 on all CPUs
+     *  in the system, #address-cells can be set to 1, since
+     *  MPIDR_EL1[63:32] bits are not used for CPUs
+     *  identification.
+     *
+     *  Here we actually don't know whether our system is 32- or 64-bit one.
+     *  The simplest way to go is to examine affinity IDs of all our CPUs. If
+     *  at least one of them has Aff3 populated, we set #address-cells to 2.
+     */
+    for (cpu = 0; cpu < vbi->smp_cpus; cpu++) {
+        ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(cpu));
+
+        if (armcpu->mp_affinity & ARM_AFF3_MASK) {
+            addr_cells = 2;
+            break;
+        }
+    }
 
     qemu_fdt_add_subnode(vbi->fdt, "/cpus");
-    qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#address-cells", 0x1);
+    qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#address-cells", addr_cells);
     qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#size-cells", 0x0);
 
     for (cpu = vbi->smp_cpus - 1; cpu >= 0; cpu--) {
@@ -301,7 +339,14 @@ static void fdt_add_cpu_nodes(const VirtBoardInfo *vbi)
                                         "enable-method", "psci");
         }
 
-        qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg", armcpu->mp_affinity);
+        if (addr_cells == 2) {
+            qemu_fdt_setprop_u64(vbi->fdt, nodename, "reg",
+                                 armcpu->mp_affinity);
+        } else {
+            qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg",
+                                  armcpu->mp_affinity);
+        }
+
         g_free(nodename);
     }
 }
@@ -319,25 +364,36 @@ static void fdt_add_v2m_gic_node(VirtBoardInfo *vbi)
     qemu_fdt_setprop_cell(vbi->fdt, "/intc/v2m", "phandle", vbi->v2m_phandle);
 }
 
-static void fdt_add_gic_node(VirtBoardInfo *vbi)
+static void fdt_add_gic_node(VirtBoardInfo *vbi, int type)
 {
     vbi->gic_phandle = qemu_fdt_alloc_phandle(vbi->fdt);
     qemu_fdt_setprop_cell(vbi->fdt, "/", "interrupt-parent", vbi->gic_phandle);
 
     qemu_fdt_add_subnode(vbi->fdt, "/intc");
-    /* 'cortex-a15-gic' means 'GIC v2' */
-    qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
-                            "arm,cortex-a15-gic");
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#interrupt-cells", 3);
     qemu_fdt_setprop(vbi->fdt, "/intc", "interrupt-controller", NULL, 0);
-    qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
-                                     2, vbi->memmap[VIRT_GIC_DIST].base,
-                                     2, vbi->memmap[VIRT_GIC_DIST].size,
-                                     2, vbi->memmap[VIRT_GIC_CPU].base,
-                                     2, vbi->memmap[VIRT_GIC_CPU].size);
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#address-cells", 0x2);
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#size-cells", 0x2);
     qemu_fdt_setprop(vbi->fdt, "/intc", "ranges", NULL, 0);
+    if (type == 3) {
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,gic-v3");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+                                     2, vbi->memmap[VIRT_GIC_DIST].base,
+                                     2, vbi->memmap[VIRT_GIC_DIST].size,
+                                     2, vbi->memmap[VIRT_GIC_REDIST].base,
+                                     2, vbi->memmap[VIRT_GIC_REDIST].size);
+    } else {
+        /* 'cortex-a15-gic' means 'GIC v2' */
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,cortex-a15-gic");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+                                      2, vbi->memmap[VIRT_GIC_DIST].base,
+                                      2, vbi->memmap[VIRT_GIC_DIST].size,
+                                      2, vbi->memmap[VIRT_GIC_CPU].base,
+                                      2, vbi->memmap[VIRT_GIC_CPU].size);
+    }
+
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "phandle", vbi->gic_phandle);
 }
 
@@ -360,29 +416,34 @@ static void create_v2m(VirtBoardInfo *vbi, qemu_irq *pic)
     fdt_add_v2m_gic_node(vbi);
 }
 
-static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic)
+static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic, int type, bool secure)
 {
-    /* We create a standalone GIC v2 */
+    /* We create a standalone GIC */
     DeviceState *gicdev;
     SysBusDevice *gicbusdev;
-    const char *gictype = "arm_gic";
+    const char *gictype;
     int i;
 
-    if (kvm_irqchip_in_kernel()) {
-        gictype = "kvm-arm-gic";
-    }
+    gictype = (type == 3) ? gicv3_class_name() : gic_class_name();
 
     gicdev = qdev_create(NULL, gictype);
-    qdev_prop_set_uint32(gicdev, "revision", 2);
+    qdev_prop_set_uint32(gicdev, "revision", type);
     qdev_prop_set_uint32(gicdev, "num-cpu", smp_cpus);
     /* Note that the num-irq property counts both internal and external
      * interrupts; there are always 32 of the former (mandated by GIC spec).
      */
     qdev_prop_set_uint32(gicdev, "num-irq", NUM_IRQS + 32);
+    if (!kvm_irqchip_in_kernel()) {
+        qdev_prop_set_bit(gicdev, "has-security-extensions", secure);
+    }
     qdev_init_nofail(gicdev);
     gicbusdev = SYS_BUS_DEVICE(gicdev);
     sysbus_mmio_map(gicbusdev, 0, vbi->memmap[VIRT_GIC_DIST].base);
-    sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_CPU].base);
+    if (type == 3) {
+        sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_REDIST].base);
+    } else {
+        sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_CPU].base);
+    }
 
     /* Wire the outputs from each CPU's generic timer to the
      * appropriate GIC PPI inputs, and the GIC's IRQ output to
@@ -390,15 +451,23 @@ static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic)
      */
     for (i = 0; i < smp_cpus; i++) {
         DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
-        int ppibase = NUM_IRQS + i * 32;
-        /* physical timer; we wire it up to the non-secure timer's ID,
-         * since a real A15 always has TrustZone but QEMU doesn't.
+        int ppibase = NUM_IRQS + i * GIC_INTERNAL + GIC_NR_SGIS;
+        int irq;
+        /* Mapping from the output timer irq lines from the CPU to the
+         * GIC PPI inputs we use for the virt board.
          */
-        qdev_connect_gpio_out(cpudev, 0,
-                              qdev_get_gpio_in(gicdev, ppibase + 30));
-        /* virtual timer */
-        qdev_connect_gpio_out(cpudev, 1,
-                              qdev_get_gpio_in(gicdev, ppibase + 27));
+        const int timer_irq[] = {
+            [GTIMER_PHYS] = ARCH_TIMER_NS_EL1_IRQ,
+            [GTIMER_VIRT] = ARCH_TIMER_VIRT_IRQ,
+            [GTIMER_HYP]  = ARCH_TIMER_NS_EL2_IRQ,
+            [GTIMER_SEC]  = ARCH_TIMER_S_EL1_IRQ,
+        };
+
+        for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) {
+            qdev_connect_gpio_out(cpudev, irq,
+                                  qdev_get_gpio_in(gicdev,
+                                                   ppibase + timer_irq[irq]));
+        }
 
         sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ));
         sysbus_connect_irq(gicbusdev, i + smp_cpus,
@@ -409,9 +478,11 @@ static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic)
         pic[i] = qdev_get_gpio_in(gicdev, i);
     }
 
-    fdt_add_gic_node(vbi);
+    fdt_add_gic_node(vbi, type);
 
-    create_v2m(vbi, pic);
+    if (type == 2) {
+        create_v2m(vbi, pic);
+    }
 }
 
 static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic)
@@ -606,13 +677,13 @@ static void create_flash(const VirtBoardInfo *vbi)
     g_free(nodename);
 }
 
-static void create_fw_cfg(const VirtBoardInfo *vbi)
+static void create_fw_cfg(const VirtBoardInfo *vbi, AddressSpace *as)
 {
     hwaddr base = vbi->memmap[VIRT_FW_CFG].base;
     hwaddr size = vbi->memmap[VIRT_FW_CFG].size;
     char *nodename;
 
-    fw_cfg_init_mem_wide(base + 8, base, 8);
+    fw_cfg_init_mem_wide(base + 8, base, 8, base + 16, as);
 
     nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base);
     qemu_fdt_add_subnode(vbi->fdt, nodename);
@@ -658,10 +729,13 @@ static void create_pcie_irq_map(const VirtBoardInfo *vbi, uint32_t gic_phandle,
                            0x7           /* PCI irq */);
 }
 
-static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
+static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic,
+                        bool use_highmem)
 {
     hwaddr base_mmio = vbi->memmap[VIRT_PCIE_MMIO].base;
     hwaddr size_mmio = vbi->memmap[VIRT_PCIE_MMIO].size;
+    hwaddr base_mmio_high = vbi->memmap[VIRT_PCIE_MMIO_HIGH].base;
+    hwaddr size_mmio_high = vbi->memmap[VIRT_PCIE_MMIO_HIGH].size;
     hwaddr base_pio = vbi->memmap[VIRT_PCIE_PIO].base;
     hwaddr size_pio = vbi->memmap[VIRT_PCIE_PIO].size;
     hwaddr base_ecam = vbi->memmap[VIRT_PCIE_ECAM].base;
@@ -698,6 +772,16 @@ static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
                              mmio_reg, base_mmio, size_mmio);
     memory_region_add_subregion(get_system_memory(), base_mmio, mmio_alias);
 
+    if (use_highmem) {
+        /* Map high MMIO space */
+        MemoryRegion *high_mmio_alias = g_new0(MemoryRegion, 1);
+
+        memory_region_init_alias(high_mmio_alias, OBJECT(dev), "pcie-mmio-high",
+                                 mmio_reg, base_mmio_high, size_mmio_high);
+        memory_region_add_subregion(get_system_memory(), base_mmio_high,
+                                    high_mmio_alias);
+    }
+
     /* Map IO port space */
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 2, base_pio);
 
@@ -715,15 +799,30 @@ static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
     qemu_fdt_setprop_cells(vbi->fdt, nodename, "bus-range", 0,
                            nr_pcie_buses - 1);
 
-    qemu_fdt_setprop_cells(vbi->fdt, nodename, "msi-parent", vbi->v2m_phandle);
+    if (vbi->v2m_phandle) {
+        qemu_fdt_setprop_cells(vbi->fdt, nodename, "msi-parent",
+                               vbi->v2m_phandle);
+    }
 
     qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
                                  2, base_ecam, 2, size_ecam);
-    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
-                                 1, FDT_PCI_RANGE_IOPORT, 2, 0,
-                                 2, base_pio, 2, size_pio,
-                                 1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
-                                 2, base_mmio, 2, size_mmio);
+
+    if (use_highmem) {
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
+                                     1, FDT_PCI_RANGE_IOPORT, 2, 0,
+                                     2, base_pio, 2, size_pio,
+                                     1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
+                                     2, base_mmio, 2, size_mmio,
+                                     1, FDT_PCI_RANGE_MMIO_64BIT,
+                                     2, base_mmio_high,
+                                     2, base_mmio_high, 2, size_mmio_high);
+    } else {
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
+                                     1, FDT_PCI_RANGE_IOPORT, 2, 0,
+                                     2, base_pio, 2, size_pio,
+                                     1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
+                                     2, base_mmio, 2, size_mmio);
+    }
 
     qemu_fdt_setprop_cell(vbi->fdt, nodename, "#interrupt-cells", 1);
     create_pcie_irq_map(vbi, vbi->gic_phandle, irq, nodename);
@@ -780,12 +879,42 @@ static void *machvirt_dtb(const struct arm_boot_info *binfo, int *fdt_size)
     return board->fdt;
 }
 
+static void virt_build_smbios(VirtGuestInfo *guest_info)
+{
+    FWCfgState *fw_cfg = guest_info->fw_cfg;
+    uint8_t *smbios_tables, *smbios_anchor;
+    size_t smbios_tables_len, smbios_anchor_len;
+    const char *product = "QEMU Virtual Machine";
+
+    if (!fw_cfg) {
+        return;
+    }
+
+    if (kvm_enabled()) {
+        product = "KVM Virtual Machine";
+    }
+
+    smbios_set_defaults("QEMU", product,
+                        "1.0", false, true, SMBIOS_ENTRY_POINT_30);
+
+    smbios_get_tables(NULL, 0, &smbios_tables, &smbios_tables_len,
+                      &smbios_anchor, &smbios_anchor_len);
+
+    if (smbios_anchor) {
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
+                        smbios_tables, smbios_tables_len);
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
+                        smbios_anchor, smbios_anchor_len);
+    }
+}
+
 static
 void virt_guest_info_machine_done(Notifier *notifier, void *data)
 {
     VirtGuestInfoState *guest_info_state = container_of(notifier,
                                               VirtGuestInfoState, machine_done);
     virt_acpi_setup(&guest_info_state->info);
+    virt_build_smbios(&guest_info_state->info);
 }
 
 static void machvirt_init(MachineState *machine)
@@ -793,7 +922,8 @@ static void machvirt_init(MachineState *machine)
     VirtMachineState *vms = VIRT_MACHINE(machine);
     qemu_irq pic[NUM_IRQS];
     MemoryRegion *sysmem = get_system_memory();
-    int n;
+    int gic_version = vms->gic_version;
+    int n, max_cpus;
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     const char *cpu_model = machine->cpu_model;
     VirtBoardInfo *vbi;
@@ -805,6 +935,18 @@ static void machvirt_init(MachineState *machine)
         cpu_model = "cortex-a15";
     }
 
+    /* We can probe only here because during property set
+     * KVM is not available yet
+     */
+    if (!gic_version) {
+        gic_version = kvm_arm_vgic_probe();
+        if (!gic_version) {
+            error_report("Unable to determine GIC version supported by host");
+            error_printf("KVM acceleration is probably not supported\n");
+            exit(1);
+        }
+    }
+
     /* Separate the actual CPU model name from any appended features */
     cpustr = g_strsplit(cpu_model, ",", 2);
 
@@ -815,6 +957,22 @@ static void machvirt_init(MachineState *machine)
         exit(1);
     }
 
+    /* The maximum number of CPUs depends on the GIC version, or on how
+     * many redistributors we can fit into the memory map.
+     */
+    if (gic_version == 3) {
+        max_cpus = vbi->memmap[VIRT_GIC_REDIST].size / 0x20000;
+    } else {
+        max_cpus = GIC_NCPU;
+    }
+
+    if (smp_cpus > max_cpus) {
+        error_report("Number of SMP CPUs requested (%d) exceeds max CPUs "
+                     "supported by machine 'mach-virt' (%d)",
+                     smp_cpus, max_cpus);
+        exit(1);
+    }
+
     vbi->smp_cpus = smp_cpus;
 
     if (machine->ram_size > vbi->memmap[VIRT_MEM].size) {
@@ -832,7 +990,7 @@ static void machvirt_init(MachineState *machine)
         char *cpuopts = g_strdup(cpustr[1]);
 
         if (!oc) {
-            fprintf(stderr, "Unable to find CPU definition\n");
+            error_report("Unable to find CPU definition");
             exit(1);
         }
         cpuobj = object_new(object_class_get_name(oc));
@@ -865,7 +1023,7 @@ static void machvirt_init(MachineState *machine)
         object_property_set_bool(cpuobj, true, "realized", NULL);
     }
     g_strfreev(cpustr);
-    fdt_add_timer_nodes(vbi);
+    fdt_add_timer_nodes(vbi, gic_version);
     fdt_add_cpu_nodes(vbi);
     fdt_add_psci_node(vbi);
 
@@ -875,13 +1033,13 @@ static void machvirt_init(MachineState *machine)
 
     create_flash(vbi);
 
-    create_gic(vbi, pic);
+    create_gic(vbi, pic, gic_version, vms->secure);
 
     create_uart(vbi, pic);
 
     create_rtc(vbi, pic);
 
-    create_pcie(vbi, pic);
+    create_pcie(vbi, pic, vms->highmem);
 
     /* Create mmio transports, so the user can create virtio backends
      * (which will be automatically plugged in to the transports). If
@@ -889,13 +1047,15 @@ static void machvirt_init(MachineState *machine)
      */
     create_virtio_devices(vbi, pic);
 
-    create_fw_cfg(vbi);
+    create_fw_cfg(vbi, &address_space_memory);
     rom_set_fw(fw_cfg_find());
 
     guest_info->smp_cpus = smp_cpus;
     guest_info->fw_cfg = fw_cfg_find();
     guest_info->memmap = vbi->memmap;
     guest_info->irqmap = vbi->irqmap;
+    guest_info->use_highmem = vms->highmem;
+    guest_info->gic_version = gic_version;
     guest_info_state->machine_done.notify = virt_guest_info_machine_done;
     qemu_add_machine_init_done_notifier(&guest_info_state->machine_done);
 
@@ -933,31 +1093,93 @@ static void virt_set_secure(Object *obj, bool value, Error **errp)
     vms->secure = value;
 }
 
+static bool virt_get_highmem(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    return vms->highmem;
+}
+
+static void virt_set_highmem(Object *obj, bool value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    vms->highmem = value;
+}
+
+static char *virt_get_gic_version(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+    const char *val = vms->gic_version == 3 ? "3" : "2";
+
+    return g_strdup(val);
+}
+
+static void virt_set_gic_version(Object *obj, const char *value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    if (!strcmp(value, "3")) {
+        vms->gic_version = 3;
+    } else if (!strcmp(value, "2")) {
+        vms->gic_version = 2;
+    } else if (!strcmp(value, "host")) {
+        vms->gic_version = 0; /* Will probe later */
+    } else {
+        error_report("Invalid gic-version option value");
+        error_printf("Allowed gic-version values are: 3, 2, host\n");
+        exit(1);
+    }
+}
+
 static void virt_instance_init(Object *obj)
 {
     VirtMachineState *vms = VIRT_MACHINE(obj);
 
-    /* EL3 is enabled by default on virt */
-    vms->secure = true;
+    /* EL3 is disabled by default on virt: this makes us consistent
+     * between KVM and TCG for this board, and it also allows us to
+     * boot UEFI blobs which assume no TrustZone support.
+     */
+    vms->secure = false;
     object_property_add_bool(obj, "secure", virt_get_secure,
                              virt_set_secure, NULL);
     object_property_set_description(obj, "secure",
                                     "Set on/off to enable/disable the ARM "
                                     "Security Extensions (TrustZone)",
                                     NULL);
+
+    /* High memory is enabled by default */
+    vms->highmem = true;
+    object_property_add_bool(obj, "highmem", virt_get_highmem,
+                             virt_set_highmem, NULL);
+    object_property_set_description(obj, "highmem",
+                                    "Set on/off to enable/disable using "
+                                    "physical address space above 32 bits",
+                                    NULL);
+    /* Default GIC type is v2 */
+    vms->gic_version = 2;
+    object_property_add_str(obj, "gic-version", virt_get_gic_version,
+                        virt_set_gic_version, NULL);
+    object_property_set_description(obj, "gic-version",
+                                    "Set GIC version. "
+                                    "Valid values are 2, 3 and host", NULL);
 }
 
 static void virt_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = TYPE_VIRT_MACHINE;
     mc->desc = "ARM Virtual Machine",
     mc->init = machvirt_init;
-    mc->max_cpus = 8;
+    /* Start max_cpus at the maximum QEMU supports. We'll further restrict
+     * it later in machvirt_init, where we have more information about the
+     * configuration of the particular instance.
+     */
+    mc->max_cpus = MAX_CPUMASK_BITS;
     mc->has_dynamic_sysbus = true;
     mc->block_default_type = IF_VIRTIO;
     mc->no_cdrom = 1;
+    mc->pci_allow_0_address = true;
 }
 
 static const TypeInfo machvirt_info = {
diff --git a/hw/arm/xilinx_zynq.c b/hw/arm/xilinx_zynq.c
index a4e7b5c63..1c1a44547 100644
--- a/hw/arm/xilinx_zynq.c
+++ b/hw/arm/xilinx_zynq.c
@@ -24,6 +24,7 @@
 #include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
 #include "hw/loader.h"
+#include "hw/misc/zynq-xadc.h"
 #include "hw/ssi.h"
 #include "qemu/error-report.h"
 
@@ -43,6 +44,45 @@ static const int dma_irqs[8] = {
     46, 47, 48, 49, 72, 73, 74, 75
 };
 
+#define BOARD_SETUP_ADDR        0x100
+
+#define SLCR_LOCK_OFFSET        0x004
+#define SLCR_UNLOCK_OFFSET      0x008
+#define SLCR_ARM_PLL_OFFSET     0x100
+
+#define SLCR_XILINX_UNLOCK_KEY  0xdf0d
+#define SLCR_XILINX_LOCK_KEY    0x767b
+
+#define ARMV7_IMM16(x) (extract32((x),  0, 12) | \
+                        extract32((x), 12,  4) << 16)
+
+/* Write immediate val to address r0 + addr. r0 should contain base offset
+ * of the SLCR block. Clobbers r1.
+ */
+
+#define SLCR_WRITE(addr, val) \
+    0xe3001000 + ARMV7_IMM16(extract32((val),  0, 16)), /* movw r1 ... */ \
+    0xe3401000 + ARMV7_IMM16(extract32((val), 16, 16)), /* movt r1 ... */ \
+    0xe5801000 + (addr)
+
+static void zynq_write_board_setup(ARMCPU *cpu,
+                                   const struct arm_boot_info *info)
+{
+    int n;
+    uint32_t board_setup_blob[] = {
+        0xe3a004f8, /* mov r0, #0xf8000000 */
+        SLCR_WRITE(SLCR_UNLOCK_OFFSET, SLCR_XILINX_UNLOCK_KEY),
+        SLCR_WRITE(SLCR_ARM_PLL_OFFSET, 0x00014008),
+        SLCR_WRITE(SLCR_LOCK_OFFSET, SLCR_XILINX_LOCK_KEY),
+        0xe12fff1e, /* bx lr */
+    };
+    for (n = 0; n < ARRAY_SIZE(board_setup_blob); n++) {
+        board_setup_blob[n] = tswap32(board_setup_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup", board_setup_blob,
+                       sizeof(board_setup_blob), BOARD_SETUP_ADDR);
+}
+
 static struct arm_boot_info zynq_binfo = {};
 
 static void gem_init(NICInfo *nd, uint32_t base, qemu_irq irq)
@@ -167,7 +207,7 @@ static void zynq_init(MachineState *machine)
 
     /* 256K of on-chip memory */
     memory_region_init_ram(ocm_ram, NULL, "zynq.ocm_ram", 256 << 10,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ocm_ram);
     memory_region_add_subregion(address_space_mem, 0xFFFC0000, ocm_ram);
 
@@ -225,6 +265,11 @@ static void zynq_init(MachineState *machine)
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xE0101000);
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[79-IRQ_OFFSET]);
 
+    dev = qdev_create(NULL, TYPE_ZYNQ_XADC);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xF8007100);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[39-IRQ_OFFSET]);
+
     dev = qdev_create(NULL, "pl330");
     qdev_prop_set_uint8(dev, "num_chnls",  8);
     qdev_prop_set_uint8(dev, "num_periph_req",  4);
@@ -252,21 +297,19 @@ static void zynq_init(MachineState *machine)
     zynq_binfo.nb_cpus = 1;
     zynq_binfo.board_id = 0xd32;
     zynq_binfo.loader_start = 0;
+    zynq_binfo.board_setup_addr = BOARD_SETUP_ADDR;
+    zynq_binfo.write_board_setup = zynq_write_board_setup;
+
     arm_load_kernel(ARM_CPU(first_cpu), &zynq_binfo);
 }
 
-static QEMUMachine zynq_machine = {
-    .name = "xilinx-zynq-a9",
-    .desc = "Xilinx Zynq Platform Baseboard for Cortex-A9",
-    .init = zynq_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 1,
-    .no_sdcard = 1,
-};
-
-static void zynq_machine_init(void)
+static void zynq_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&zynq_machine);
+    mc->desc = "Xilinx Zynq Platform Baseboard for Cortex-A9";
+    mc->init = zynq_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 1;
+    mc->no_sdcard = 1;
 }
 
-machine_init(zynq_machine_init);
+DEFINE_MACHINE("xilinx-zynq-a9", zynq_machine_init)
diff --git a/hw/arm/xlnx-ep108.c b/hw/arm/xlnx-ep108.c
index f94da86cb..85b978fa7 100644
--- a/hw/arm/xlnx-ep108.c
+++ b/hw/arm/xlnx-ep108.c
@@ -51,7 +51,7 @@ static void xlnx_ep108_init(MachineState *machine)
         machine->ram_size = EP108_MAX_RAM_SIZE;
     }
 
-    if (machine->ram_size <= 0x08000000) {
+    if (machine->ram_size < 0x08000000) {
         qemu_log("WARNING: RAM size " RAM_ADDR_FMT " is small for EP108",
                  machine->ram_size);
     }
@@ -68,15 +68,10 @@ static void xlnx_ep108_init(MachineState *machine)
     arm_load_kernel(s->soc.boot_cpu_ptr, &xlnx_ep108_binfo);
 }
 
-static QEMUMachine xlnx_ep108_machine = {
-    .name = "xlnx-ep108",
-    .desc = "Xilinx ZynqMP EP108 board",
-    .init = xlnx_ep108_init,
-};
-
-static void xlnx_ep108_machine_init(void)
+static void xlnx_ep108_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xlnx_ep108_machine);
+    mc->desc = "Xilinx ZynqMP EP108 board";
+    mc->init = xlnx_ep108_init;
 }
 
-machine_init(xlnx_ep108_machine_init);
+DEFINE_MACHINE("xlnx-ep108", xlnx_ep108_machine_init)
diff --git a/hw/arm/xlnx-zynqmp.c b/hw/arm/xlnx-zynqmp.c
index 5157565f9..87553bbc6 100644
--- a/hw/arm/xlnx-zynqmp.c
+++ b/hw/arm/xlnx-zynqmp.c
@@ -28,6 +28,10 @@
 #define GIC_DIST_ADDR       0xf9010000
 #define GIC_CPU_ADDR        0xf9020000
 
+#define SATA_INTR           133
+#define SATA_ADDR           0xFD0C0000
+#define SATA_NUM_PORTS      2
+
 static const uint64_t gem_addr[XLNX_ZYNQMP_NUM_GEMS] = {
     0xFF0B0000, 0xFF0C0000, 0xFF0D0000, 0xFF0E0000,
 };
@@ -44,6 +48,14 @@ static const int uart_intr[XLNX_ZYNQMP_NUM_UARTS] = {
     21, 22,
 };
 
+static const uint64_t sdhci_addr[XLNX_ZYNQMP_NUM_SDHCI] = {
+    0xFF160000, 0xFF170000,
+};
+
+static const int sdhci_intr[XLNX_ZYNQMP_NUM_SDHCI] = {
+    48, 49,
+};
+
 typedef struct XlnxZynqMPGICRegion {
     int region_index;
     uint32_t address;
@@ -90,6 +102,16 @@ static void xlnx_zynqmp_init(Object *obj)
         object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_CADENCE_UART);
         qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
     }
+
+    object_initialize(&s->sata, sizeof(s->sata), TYPE_SYSBUS_AHCI);
+    qdev_set_parent_bus(DEVICE(&s->sata), sysbus_get_default());
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        object_initialize(&s->sdhci[i], sizeof(s->sdhci[i]),
+                          TYPE_SYSBUS_SDHCI);
+        qdev_set_parent_bus(DEVICE(&s->sdhci[i]),
+                            sysbus_get_default());
+    }
 }
 
 static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
@@ -101,12 +123,27 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
     qemu_irq gic_spi[GIC_NUM_SPI_INTR];
     Error *err = NULL;
 
+    /* Create the four OCM banks */
+    for (i = 0; i < XLNX_ZYNQMP_NUM_OCM_BANKS; i++) {
+        char *ocm_name = g_strdup_printf("zynqmp.ocm_ram_bank_%d", i);
+
+        memory_region_init_ram(&s->ocm_ram[i], NULL, ocm_name,
+                               XLNX_ZYNQMP_OCM_RAM_SIZE, &error_fatal);
+        vmstate_register_ram_global(&s->ocm_ram[i]);
+        memory_region_add_subregion(get_system_memory(),
+                                    XLNX_ZYNQMP_OCM_RAM_0_ADDRESS +
+                                        i * XLNX_ZYNQMP_OCM_RAM_SIZE,
+                                    &s->ocm_ram[i]);
+
+        g_free(ocm_name);
+    }
+
     qdev_prop_set_uint32(DEVICE(&s->gic), "num-irq", GIC_NUM_SPI_INTR + 32);
     qdev_prop_set_uint32(DEVICE(&s->gic), "revision", 2);
     qdev_prop_set_uint32(DEVICE(&s->gic), "num-cpu", XLNX_ZYNQMP_NUM_APU_CPUS);
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err) {
-        error_propagate((errp), (err));
+        error_propagate(errp, err);
         return;
     }
     assert(ARRAY_SIZE(xlnx_zynqmp_gic_regions) == XLNX_ZYNQMP_GIC_REGIONS);
@@ -147,16 +184,11 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
         g_free(name);
 
         object_property_set_int(OBJECT(&s->apu_cpu[i]), GIC_BASE_ADDR,
-                                "reset-cbar", &err);
-        if (err) {
-            error_propagate((errp), (err));
-            return;
-        }
-
+                                "reset-cbar", &error_abort);
         object_property_set_bool(OBJECT(&s->apu_cpu[i]), true, "realized",
                                  &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
 
@@ -185,16 +217,11 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
         g_free(name);
 
         object_property_set_bool(OBJECT(&s->rpu_cpu[i]), true, "reset-hivecs",
-                                 &err);
-        if (err != NULL) {
-            error_propagate(errp, err);
-            return;
-        }
-
+                                 &error_abort);
         object_property_set_bool(OBJECT(&s->rpu_cpu[i]), true, "realized",
                                  &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
     }
@@ -217,7 +244,7 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
         }
         object_property_set_bool(OBJECT(&s->gem[i]), true, "realized", &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
         sysbus_mmio_map(SYS_BUS_DEVICE(&s->gem[i]), 0, gem_addr[i]);
@@ -228,13 +255,37 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
     for (i = 0; i < XLNX_ZYNQMP_NUM_UARTS; i++) {
         object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
         sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, uart_addr[i]);
         sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
                            gic_spi[uart_intr[i]]);
     }
+
+    object_property_set_int(OBJECT(&s->sata), SATA_NUM_PORTS, "num-ports",
+                            &error_abort);
+    object_property_set_bool(OBJECT(&s->sata), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->sata), 0, SATA_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sata), 0, gic_spi[SATA_INTR]);
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        object_property_set_bool(OBJECT(&s->sdhci[i]), true,
+                                 "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->sdhci[i]), 0,
+                        sdhci_addr[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->sdhci[i]), 0,
+                           gic_spi[sdhci_intr[i]]);
+    }
 }
 
 static Property xlnx_zynqmp_props[] = {
diff --git a/hw/arm/z2.c b/hw/arm/z2.c
index 17355479a..b44eb76fc 100644
--- a/hw/arm/z2.c
+++ b/hw/arm/z2.c
@@ -372,15 +372,10 @@ static void z2_init(MachineState *machine)
     arm_load_kernel(mpu->cpu, &z2_binfo);
 }
 
-static QEMUMachine z2_machine = {
-    .name = "z2",
-    .desc = "Zipit Z2 (PXA27x)",
-    .init = z2_init,
-};
-
-static void z2_machine_init(void)
+static void z2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&z2_machine);
+    mc->desc = "Zipit Z2 (PXA27x)";
+    mc->init = z2_init;
 }
 
-machine_init(z2_machine_init);
+DEFINE_MACHINE("z2", z2_machine_init)
diff --git a/hw/audio/adlib.c b/hw/audio/adlib.c
index 656eb3773..334935f8d 100644
--- a/hw/audio/adlib.c
+++ b/hw/audio/adlib.c
@@ -57,11 +57,6 @@ void YMF262UpdateOneQEMU (int which, INT16 *dst, int length);
 #define SHIFT 1
 #endif
 
-#define IO_READ_PROTO(name) \
-    uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name) \
-    void name (void *opaque, uint32_t nport, uint32_t val)
-
 #define TYPE_ADLIB "adlib"
 #define ADLIB(obj) OBJECT_CHECK(AdlibState, (obj), TYPE_ADLIB)
 
@@ -124,7 +119,7 @@ static void adlib_kill_timers (AdlibState *s)
     }
 }
 
-static IO_WRITE_PROTO (adlib_write)
+static void adlib_write(void *opaque, uint32_t nport, uint32_t val)
 {
     AdlibState *s = opaque;
     int a = nport & 3;
@@ -141,7 +136,7 @@ static IO_WRITE_PROTO (adlib_write)
 #endif
 }
 
-static IO_READ_PROTO (adlib_read)
+static uint32_t adlib_read(void *opaque, uint32_t nport)
 {
     AdlibState *s = opaque;
     uint8_t data;
diff --git a/hw/audio/es1370.c b/hw/audio/es1370.c
index 8e7bcf503..592578b14 100644
--- a/hw/audio/es1370.c
+++ b/hw/audio/es1370.c
@@ -157,11 +157,6 @@ static const unsigned dac1_samplerate[] = { 5512, 11025, 22050, 44100 };
 #define DAC2_CHANNEL 1
 #define ADC_CHANNEL 2
 
-#define IO_READ_PROTO(n) \
-static uint32_t n (void *opaque, uint32_t addr)
-#define IO_WRITE_PROTO(n) \
-static void n (void *opaque, uint32_t addr, uint32_t val)
-
 static void es1370_dac1_callback (void *opaque, int free);
 static void es1370_dac2_callback (void *opaque, int free);
 static void es1370_adc_callback (void *opaque, int avail);
@@ -474,7 +469,7 @@ static inline uint32_t es1370_fixup (ES1370State *s, uint32_t addr)
     return addr;
 }
 
-IO_WRITE_PROTO (es1370_writeb)
+static void es1370_writeb(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     uint32_t shift, mask;
@@ -512,7 +507,7 @@ IO_WRITE_PROTO (es1370_writeb)
     }
 }
 
-IO_WRITE_PROTO (es1370_writew)
+static void es1370_writew(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     addr = es1370_fixup (s, addr);
@@ -549,7 +544,7 @@ IO_WRITE_PROTO (es1370_writew)
     }
 }
 
-IO_WRITE_PROTO (es1370_writel)
+static void es1370_writel(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     struct chan *d = &s->chan[0];
@@ -615,7 +610,7 @@ IO_WRITE_PROTO (es1370_writel)
     }
 }
 
-IO_READ_PROTO (es1370_readb)
+static uint32_t es1370_readb(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     uint32_t val;
@@ -650,7 +645,7 @@ IO_READ_PROTO (es1370_readb)
     return val;
 }
 
-IO_READ_PROTO (es1370_readw)
+static uint32_t es1370_readw(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     struct chan *d = &s->chan[0];
@@ -692,7 +687,7 @@ IO_READ_PROTO (es1370_readw)
     return val;
 }
 
-IO_READ_PROTO (es1370_readl)
+static uint32_t es1370_readl(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     uint32_t val;
diff --git a/hw/audio/fmopl.c b/hw/audio/fmopl.c
index adcef2d3b..81c0c1be2 100644
--- a/hw/audio/fmopl.c
+++ b/hw/audio/fmopl.c
@@ -1177,7 +1177,7 @@ void OPLResetChip(FM_OPL *OPL)
 	OPLWriteReg(OPL,0x03,0); /* Timer2 */
 	OPLWriteReg(OPL,0x04,0); /* IRQ mask clear */
 	for(i = 0xff ; i >= 0x20 ; i-- ) OPLWriteReg(OPL,i,0);
-	/* reset OPerator paramater */
+	/* reset operator parameter */
 	for( c = 0 ; c < OPL->max_ch ; c++ )
 	{
 		OPL_CH *CH = &OPL->P_CH[c];
diff --git a/hw/audio/gus.c b/hw/audio/gus.c
index 86223a954..e0c8a4ee1 100644
--- a/hw/audio/gus.c
+++ b/hw/audio/gus.c
@@ -41,11 +41,6 @@
 #define GUS_ENDIANNESS 0
 #endif
 
-#define IO_READ_PROTO(name) \
-    static uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name) \
-    static void name (void *opaque, uint32_t nport, uint32_t val)
-
 #define TYPE_GUS "gus"
 #define GUS(obj) OBJECT_CHECK (GUSState, (obj), TYPE_GUS)
 
@@ -64,14 +59,14 @@ typedef struct GUSState {
     qemu_irq pic;
 } GUSState;
 
-IO_READ_PROTO (gus_readb)
+static uint32_t gus_readb(void *opaque, uint32_t nport)
 {
     GUSState *s = opaque;
 
     return gus_read (&s->emu, nport, 1);
 }
 
-IO_WRITE_PROTO (gus_writeb)
+static void gus_writeb(void *opaque, uint32_t nport, uint32_t val)
 {
     GUSState *s = opaque;
 
diff --git a/hw/audio/sb16.c b/hw/audio/sb16.c
index b052de5f7..995435f24 100644
--- a/hw/audio/sb16.c
+++ b/hw/audio/sb16.c
@@ -40,11 +40,6 @@
 #define ldebug(...)
 #endif
 
-#define IO_READ_PROTO(name)                             \
-    uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name)                                    \
-    void name (void *opaque, uint32_t nport, uint32_t val)
-
 static const char e3[] = "COPYRIGHT (C) CREATIVE TECHNOLOGY LTD, 1992.";
 
 #define TYPE_SB16 "sb16"
@@ -881,7 +876,7 @@ static void reset (SB16State *s)
     legacy_reset (s);
 }
 
-static IO_WRITE_PROTO (dsp_write)
+static void dsp_write(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
     int iport;
@@ -959,7 +954,7 @@ static IO_WRITE_PROTO (dsp_write)
     }
 }
 
-static IO_READ_PROTO (dsp_read)
+static uint32_t dsp_read(void *opaque, uint32_t nport)
 {
     SB16State *s = opaque;
     int iport, retval, ack = 0;
@@ -1058,14 +1053,14 @@ static void reset_mixer (SB16State *s)
     }
 }
 
-static IO_WRITE_PROTO (mixer_write_indexb)
+static void mixer_write_indexb(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
     (void) nport;
     s->mixer_nreg = val;
 }
 
-static IO_WRITE_PROTO (mixer_write_datab)
+static void mixer_write_datab(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
 
@@ -1121,7 +1116,7 @@ static IO_WRITE_PROTO (mixer_write_datab)
     s->mixer_regs[s->mixer_nreg] = val;
 }
 
-static IO_READ_PROTO (mixer_read)
+static uint32_t mixer_read(void *opaque, uint32_t nport)
 {
     SB16State *s = opaque;
 
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index 6106e4615..c57f293cc 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -45,7 +45,6 @@ struct VirtIOBlockDataPlane {
      * use it).
      */
     IOThread *iothread;
-    IOThread internal_iothread_obj;
     AioContext *ctx;
     EventNotifier host_notifier;    /* doorbell */
 
@@ -149,14 +148,14 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
 
     *dataplane = NULL;
 
-    if (!conf->data_plane && !conf->iothread) {
+    if (!conf->iothread) {
         return;
     }
 
     /* Don't try if transport does not support notifiers. */
     if (!k->set_guest_notifiers || !k->set_host_notifier) {
         error_setg(errp,
-                   "device is incompatible with x-data-plane "
+                   "device is incompatible with dataplane "
                    "(transport does not support notifiers)");
         return;
     }
@@ -179,16 +178,6 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
     if (conf->iothread) {
         s->iothread = conf->iothread;
         object_ref(OBJECT(s->iothread));
-    } else {
-        /* Create per-device IOThread if none specified.  This is for
-         * x-data-plane option compatibility.  If x-data-plane is removed we
-         * can drop this.
-         */
-        object_initialize(&s->internal_iothread_obj,
-                          sizeof(s->internal_iothread_obj),
-                          TYPE_IOTHREAD);
-        user_creatable_complete(OBJECT(&s->internal_iothread_obj), &error_abort);
-        s->iothread = &s->internal_iothread_obj;
     }
     s->ctx = iothread_get_aio_context(s->iothread);
     s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
@@ -283,7 +272,8 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
 
     /* Get this show started by hooking up our callbacks */
     aio_context_acquire(s->ctx);
-    aio_set_event_notifier(s->ctx, &s->host_notifier, handle_notify);
+    aio_set_event_notifier(s->ctx, &s->host_notifier, true,
+                           handle_notify);
     aio_context_release(s->ctx);
     return;
 
@@ -319,7 +309,7 @@ void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
     aio_context_acquire(s->ctx);
 
     /* Stop notifications for new requests from guest */
-    aio_set_event_notifier(s->ctx, &s->host_notifier, NULL);
+    aio_set_event_notifier(s->ctx, &s->host_notifier, true, NULL);
 
     /* Drain and switch bs back to the QEMU main loop */
     blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());
diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index 5e1b67ee4..4292eced3 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -192,6 +192,8 @@ typedef struct FDrive {
     uint8_t ro;               /* Is read-only           */
     uint8_t media_changed;    /* Is media changed       */
     uint8_t media_rate;       /* Data rate of medium    */
+
+    bool media_inserted;      /* Is there a medium in the tray */
 } FDrive;
 
 static void fd_init(FDrive *drv)
@@ -261,7 +263,7 @@ static int fd_seek(FDrive *drv, uint8_t head, uint8_t track, uint8_t sect,
 #endif
         drv->head = head;
         if (drv->track != track) {
-            if (drv->blk != NULL && blk_is_inserted(drv->blk)) {
+            if (drv->media_inserted) {
                 drv->media_changed = 0;
             }
             ret = 1;
@@ -270,7 +272,7 @@ static int fd_seek(FDrive *drv, uint8_t head, uint8_t track, uint8_t sect,
         drv->sect = sect;
     }
 
-    if (drv->blk == NULL || !blk_is_inserted(drv->blk)) {
+    if (!drv->media_inserted) {
         ret = 2;
     }
 
@@ -296,7 +298,7 @@ static void fd_revalidate(FDrive *drv)
         ro = blk_is_read_only(drv->blk);
         pick_geometry(drv->blk, &nb_heads, &max_track,
                       &last_sect, drv->drive, &drive, &rate);
-        if (!blk_is_inserted(drv->blk)) {
+        if (!drv->media_inserted) {
             FLOPPY_DPRINTF("No disk in drive\n");
         } else {
             FLOPPY_DPRINTF("Floppy disk (%d h %d t %d s) %s\n", nb_heads,
@@ -692,7 +694,7 @@ static bool fdrive_media_changed_needed(void *opaque)
 {
     FDrive *drive = opaque;
 
-    return (drive->blk != NULL && drive->media_changed != 1);
+    return (drive->media_inserted && drive->media_changed != 1);
 }
 
 static const VMStateDescription vmstate_fdrive_media_changed = {
@@ -1417,7 +1419,7 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
                  * recall us...
                  */
                 DMA_hold_DREQ(fdctrl->dma_chann);
-                DMA_schedule(fdctrl->dma_chann);
+                DMA_schedule();
             } else {
                 /* Start transfer */
                 fdctrl_transfer_handler(fdctrl, fdctrl->dma_chann, 0,
@@ -2184,12 +2186,21 @@ static void fdctrl_change_cb(void *opaque, bool load)
 {
     FDrive *drive = opaque;
 
+    drive->media_inserted = load && drive->blk && blk_is_inserted(drive->blk);
+
     drive->media_changed = 1;
     fd_revalidate(drive);
 }
 
+static bool fdctrl_is_tray_open(void *opaque)
+{
+    FDrive *drive = opaque;
+    return !drive->media_inserted;
+}
+
 static const BlockDevOps fdctrl_block_ops = {
     .change_media_cb = fdctrl_change_cb,
+    .is_tray_open = fdctrl_is_tray_open,
 };
 
 /* Init functions */
@@ -2217,6 +2228,7 @@ static void fdctrl_connect_drives(FDCtrl *fdctrl, Error **errp)
         fdctrl_change_cb(drive, 0);
         if (drive->blk) {
             blk_set_dev_ops(drive->blk, &fdctrl_block_ops, drive);
+            drive->media_inserted = blk_is_inserted(drive->blk);
         }
     }
 }
diff --git a/hw/block/nand.c b/hw/block/nand.c
index 61d2cec03..f0e34139f 100644
--- a/hw/block/nand.c
+++ b/hw/block/nand.c
@@ -522,8 +522,8 @@ void nand_setio(DeviceState *dev, uint32_t value)
 
     if (s->ale) {
         unsigned int shift = s->addrlen * 8;
-        unsigned int mask = ~(0xff << shift);
-        unsigned int v = value << shift;
+        uint64_t mask = ~(0xffull << shift);
+        uint64_t v = (uint64_t)value << shift;
 
         s->addr = (s->addr & mask) | v;
         s->addrlen ++;
@@ -712,7 +712,7 @@ static void glue(nand_blk_erase_, PAGE_SIZE)(NANDFlashState *s)
         memset(s->storage + (PAGE(addr) << OOB_SHIFT),
                         0xff, OOB_SIZE << s->erase_shift);
         i = SECTOR(addr);
-        page = SECTOR(addr + (ADDR_SHIFT + s->erase_shift));
+        page = SECTOR(addr + (1 << (ADDR_SHIFT + s->erase_shift)));
         for (; i < page; i ++)
             if (blk_write(s->blk, i, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n", __func__, i);
diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 40d488032..169e4fa7a 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -201,10 +201,11 @@ static void nvme_rw_cb(void *opaque, int ret)
     NvmeCtrl *n = sq->ctrl;
     NvmeCQueue *cq = n->cq[sq->cqid];
 
-    block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
     if (!ret) {
+        block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
         req->status = NVME_SUCCESS;
     } else {
+        block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
         req->status = NVME_INTERNAL_DEV_ERROR;
     }
     if (req->has_sg) {
@@ -238,18 +239,22 @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
     uint64_t data_size = (uint64_t)nlb << data_shift;
     uint64_t aio_slba  = slba << (data_shift - BDRV_SECTOR_BITS);
     int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
+    enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
 
     if ((slba + nlb) > ns->id_ns.nsze) {
+        block_acct_invalid(blk_get_stats(n->conf.blk), acct);
         return NVME_LBA_RANGE | NVME_DNR;
     }
+
     if (nvme_map_prp(&req->qsg, prp1, prp2, data_size, n)) {
+        block_acct_invalid(blk_get_stats(n->conf.blk), acct);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
+
     assert((nlb << data_shift) == req->qsg.size);
 
     req->has_sg = true;
-    dma_acct_start(n->conf.blk, &req->acct, &req->qsg,
-                   is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
+    dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
     req->aiocb = is_write ?
         dma_blk_write(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req) :
         dma_blk_read(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req);
@@ -805,7 +810,7 @@ static int nvme_init(PCIDevice *pci_dev)
 
     n->num_namespaces = 1;
     n->num_queues = 64;
-    n->reg_size = 1 << qemu_fls(0x1004 + 2 * (n->num_queues + 1) * 4);
+    n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
     n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
     n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
diff --git a/hw/block/onenand.c b/hw/block/onenand.c
index 1b2c89375..58eff508b 100644
--- a/hw/block/onenand.c
+++ b/hw/block/onenand.c
@@ -786,7 +786,7 @@ static int onenand_initfn(SysBusDevice *sbd)
     s->otp = memset(g_malloc((64 + 2) << PAGE_SHIFT),
                     0xff, (64 + 2) << PAGE_SHIFT);
     memory_region_init_ram(&s->ram, OBJECT(s), "onenand.ram",
-                           0xc000 << s->shift, &error_abort);
+                           0xc000 << s->shift, &error_fatal);
     vmstate_register_ram_global(&s->ram);
     ram = memory_region_get_ram_ptr(&s->ram);
     s->boot[0] = ram + (0x0000 << s->shift);
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index 5625a9fa7..b88b726be 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -30,7 +30,7 @@
 
 VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s)
 {
-    VirtIOBlockReq *req = g_slice_new(VirtIOBlockReq);
+    VirtIOBlockReq *req = g_new(VirtIOBlockReq, 1);
     req->dev = s;
     req->qiov.size = 0;
     req->in_len = 0;
@@ -42,7 +42,7 @@ VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s)
 void virtio_blk_free_request(VirtIOBlockReq *req)
 {
     if (req) {
-        g_slice_free(VirtIOBlockReq, req);
+        g_free(req);
     }
 }
 
@@ -72,11 +72,14 @@ static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
     VirtIOBlock *s = req->dev;
 
     if (action == BLOCK_ERROR_ACTION_STOP) {
+        /* Break the link as the next request is going to be parsed from the
+         * ring again. Otherwise we may end up doing a double completion! */
+        req->mr_next = NULL;
         req->next = s->rq;
         s->rq = req;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
         virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
-        block_acct_done(blk_get_stats(s->blk), &req->acct);
+        block_acct_failed(blk_get_stats(s->blk), &req->acct);
         virtio_blk_free_request(req);
     }
 
@@ -536,6 +539,8 @@ void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
         if (!virtio_blk_sect_range_ok(req->dev, req->sector_num,
                                       req->qiov.size)) {
             virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
+            block_acct_invalid(blk_get_stats(req->dev->blk),
+                               is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
             virtio_blk_free_request(req);
             return;
         }
@@ -600,6 +605,8 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
         return;
     }
 
+    blk_io_plug(s->blk);
+
     while ((req = virtio_blk_get_request(s))) {
         virtio_blk_handle_request(req, &mrb);
     }
@@ -607,6 +614,8 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
     if (mrb.num_reqs) {
         virtio_blk_submit_multireq(s->blk, &mrb);
     }
+
+    blk_io_unplug(s->blk);
 }
 
 static void virtio_blk_dma_restart_bh(void *opaque)
@@ -840,10 +849,7 @@ static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f,
         req->next = s->rq;
         s->rq = req;
 
-        virtqueue_map_sg(req->elem.in_sg, req->elem.in_addr,
-            req->elem.in_num, 1);
-        virtqueue_map_sg(req->elem.out_sg, req->elem.out_addr,
-            req->elem.out_num, 0);
+        virtqueue_map(&req->elem);
     }
 
     return 0;
@@ -976,11 +982,10 @@ static Property virtio_blk_properties[] = {
     DEFINE_PROP_STRING("serial", VirtIOBlock, conf.serial),
     DEFINE_PROP_BIT("config-wce", VirtIOBlock, conf.config_wce, 0, true),
 #ifdef __linux__
-    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, true),
+    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, false),
 #endif
     DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0,
                     true),
-    DEFINE_PROP_BIT("x-data-plane", VirtIOBlock, conf.data_plane, 0, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/block/xen_blkif.h b/hw/block/xen_blkif.h
index 711b69274..c68487cb3 100644
--- a/hw/block/xen_blkif.h
+++ b/hw/block/xen_blkif.h
@@ -85,8 +85,10 @@ static inline void blkif_get_x86_32_req(blkif_request_t *dst, blkif_x86_32_reque
 		d->nr_sectors = s->nr_sectors;
 		return;
 	}
-	if (n > src->nr_segments)
-		n = src->nr_segments;
+	/* prevent the compiler from optimizing the code and using src->nr_segments instead */
+	barrier();
+	if (n > dst->nr_segments)
+		n = dst->nr_segments;
 	for (i = 0; i < n; i++)
 		dst->seg[i] = src->seg[i];
 }
@@ -106,8 +108,10 @@ static inline void blkif_get_x86_64_req(blkif_request_t *dst, blkif_x86_64_reque
 		d->nr_sectors = s->nr_sectors;
 		return;
 	}
-	if (n > src->nr_segments)
-		n = src->nr_segments;
+	/* prevent the compiler from optimizing the code and using src->nr_segments instead */
+	barrier();
+	if (n > dst->nr_segments)
+		n = dst->nr_segments;
 	for (i = 0; i < n; i++)
 		dst->seg[i] = src->seg[i];
 }
diff --git a/hw/block/xen_disk.c b/hw/block/xen_disk.c
index 267d8a8c7..814665034 100644
--- a/hw/block/xen_disk.c
+++ b/hw/block/xen_disk.c
@@ -24,7 +24,6 @@
 #include <stdarg.h>
 #include <string.h>
 #include <unistd.h>
-#include <signal.h>
 #include <inttypes.h>
 #include <time.h>
 #include <fcntl.h>
@@ -76,7 +75,6 @@ struct ioreq {
     off_t               start;
     QEMUIOVector        v;
     int                 presync;
-    int                 postsync;
     uint8_t             mapped;
 
     /* grant mapping */
@@ -145,7 +143,6 @@ static void ioreq_reset(struct ioreq *ioreq)
     ioreq->status = 0;
     ioreq->start = 0;
     ioreq->presync = 0;
-    ioreq->postsync = 0;
     ioreq->mapped = 0;
 
     memset(ioreq->domids, 0, sizeof(ioreq->domids));
@@ -521,12 +518,6 @@ static void qemu_aio_complete(void *opaque, int ret)
     if (ioreq->aio_inflight > 0) {
         return;
     }
-    if (ioreq->postsync) {
-        ioreq->postsync = 0;
-        ioreq->aio_inflight++;
-        blk_aio_flush(ioreq->blkdev->blk, qemu_aio_complete, ioreq);
-        return;
-    }
 
     ioreq->status = ioreq->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
     ioreq_unmap(ioreq);
@@ -538,7 +529,11 @@ static void qemu_aio_complete(void *opaque, int ret)
             break;
         }
     case BLKIF_OP_READ:
-        block_acct_done(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        if (ioreq->status == BLKIF_RSP_OKAY) {
+            block_acct_done(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        } else {
+            block_acct_failed(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        }
         break;
     case BLKIF_OP_DISCARD:
     default:
@@ -577,7 +572,9 @@ static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
         }
 
         block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
-                         ioreq->v.size, BLOCK_ACCT_WRITE);
+                         ioreq->v.size,
+                         ioreq->req.operation == BLKIF_OP_WRITE ?
+                         BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH);
         ioreq->aio_inflight++;
         blk_aio_writev(blkdev->blk, ioreq->start / BLOCK_SIZE,
                        &ioreq->v, ioreq->v.size / BLOCK_SIZE,
@@ -721,6 +718,23 @@ static void blk_handle_requests(struct XenBlkDev *blkdev)
 
         /* parse them */
         if (ioreq_parse(ioreq) != 0) {
+
+            switch (ioreq->req.operation) {
+            case BLKIF_OP_READ:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_READ);
+                break;
+            case BLKIF_OP_WRITE:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_WRITE);
+                break;
+            case BLKIF_OP_FLUSH_DISKCACHE:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_FLUSH);
+            default:
+                break;
+            };
+
             if (blk_send_response_one(ioreq)) {
                 xen_be_send_notify(&blkdev->xendev);
             }
@@ -932,9 +946,11 @@ static int blk_connect(struct XenDevice *xendev)
     blk_attach_dev_nofail(blkdev->blk, blkdev);
     blkdev->file_size = blk_getlength(blkdev->blk);
     if (blkdev->file_size < 0) {
+        BlockDriverState *bs = blk_bs(blkdev->blk);
+        const char *drv_name = bs ? bdrv_get_format_name(bs) : NULL;
         xen_be_printf(&blkdev->xendev, 1, "blk_getlength: %d (%s) | drv %s\n",
                       (int)blkdev->file_size, strerror(-blkdev->file_size),
-                      bdrv_get_format_name(blk_bs(blkdev->blk)) ?: "-");
+                      drv_name ?: "-");
         blkdev->file_size = 0;
     }
 
diff --git a/hw/bt/hci.c b/hw/bt/hci.c
index 7ea3dc6b7..2151d0128 100644
--- a/hw/bt/hci.c
+++ b/hw/bt/hci.c
@@ -23,6 +23,8 @@
 #include "hw/usb.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/replay.h"
 
 struct bt_hci_s {
     uint8_t *(*evt_packet)(void *opaque);
@@ -72,6 +74,8 @@ struct bt_hci_s {
 
     struct HCIInfo info;
     struct bt_device_s device;
+
+    Error *replay_blocker;
 };
 
 #define DEFAULT_RSSI_DBM	20
@@ -595,7 +599,7 @@ static void bt_hci_inquiry_result(struct bt_hci_s *hci,
 static void bt_hci_mod_timer_1280ms(QEMUTimer *timer, int period)
 {
     timer_mod(timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(period << 7, get_ticks_per_sec(), 100));
+                     (uint64_t)(period << 7) * 10000000);
 }
 
 static void bt_hci_inquiry_start(struct bt_hci_s *hci, int length)
@@ -1099,7 +1103,7 @@ static int bt_hci_mode_change(struct bt_hci_s *hci, uint16_t handle,
     bt_hci_event_status(hci, HCI_SUCCESS);
 
     timer_mod(link->acl_mode_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(interval * 625, get_ticks_per_sec(), 1000000));
+                                    ((uint64_t)interval * 625) * 1000);
     bt_hci_lmp_mode_change_master(hci, link->link, mode, interval);
 
     return 0;
@@ -1151,8 +1155,7 @@ static void bt_hci_reset(struct bt_hci_s *hci)
     hci->event_mask[7] = 0x00;
     hci->device.inquiry_scan = 0;
     hci->device.page_scan = 0;
-    if (hci->device.lmp_name)
-        g_free((void *) hci->device.lmp_name);
+    g_free((void *) hci->device.lmp_name);
     hci->device.lmp_name = NULL;
     hci->device.class[0] = 0x00;
     hci->device.class[1] = 0x00;
@@ -1829,8 +1832,7 @@ static void bt_submit_hci(struct HCIInfo *info,
     case cmd_opcode_pack(OGF_HOST_CTL, OCF_CHANGE_LOCAL_NAME):
         LENGTH_CHECK(change_local_name);
 
-        if (hci->device.lmp_name)
-            g_free((void *) hci->device.lmp_name);
+        g_free((void *) hci->device.lmp_name);
         hci->device.lmp_name = g_strndup(PARAM(change_local_name, name),
                         sizeof(PARAM(change_local_name, name)));
         bt_hci_event_complete_status(hci, HCI_SUCCESS);
@@ -2191,6 +2193,9 @@ struct HCIInfo *bt_new_hci(struct bt_scatternet_s *net)
 
     s->device.handle_destroy = bt_hci_destroy;
 
+    error_setg(&s->replay_blocker, QERR_REPLAY_NOT_SUPPORTED, "-bt hci");
+    replay_add_blocker(s->replay_blocker);
+
     return &s->info;
 }
 
@@ -2231,8 +2236,7 @@ static void bt_hci_done(struct HCIInfo *info)
 
     bt_device_done(&hci->device);
 
-    if (hci->device.lmp_name)
-        g_free((void *) hci->device.lmp_name);
+    g_free((void *) hci->device.lmp_name);
 
     /* Be gentle and send DISCONNECT to all connected peers and those
      * currently waiting for us to accept or reject a connection request.
diff --git a/hw/bt/sdp.c b/hw/bt/sdp.c
index c90374795..04eaecae6 100644
--- a/hw/bt/sdp.c
+++ b/hw/bt/sdp.c
@@ -42,7 +42,7 @@ struct bt_l2cap_sdp_state_s {
 
 static ssize_t sdp_datalen(const uint8_t **element, ssize_t *left)
 {
-    size_t len = *(*element) ++ & SDP_DSIZE_MASK;
+    uint32_t len = *(*element) ++ & SDP_DSIZE_MASK;
 
     if (!*left)
         return -1;
@@ -150,12 +150,14 @@ static ssize_t sdp_svc_search(struct bt_l2cap_sdp_state_s *sdp,
         if (seqlen < 3 || len < seqlen)
             return -SDP_INVALID_SYNTAX;
         len -= seqlen;
-
         while (seqlen)
             if (sdp_svc_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 3)
         return -SDP_INVALID_SYNTAX;
@@ -278,8 +280,11 @@ static ssize_t sdp_attr_get(struct bt_l2cap_sdp_state_s *sdp,
         while (seqlen)
             if (sdp_attr_match(record, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_attr_match(record, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_attr_match(record, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 1)
         return -SDP_INVALID_SYNTAX;
@@ -393,8 +398,11 @@ static ssize_t sdp_svc_search_attr_get(struct bt_l2cap_sdp_state_s *sdp,
         while (seqlen)
             if (sdp_svc_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 3)
         return -SDP_INVALID_SYNTAX;
@@ -413,8 +421,11 @@ static ssize_t sdp_svc_search_attr_get(struct bt_l2cap_sdp_state_s *sdp,
         while (seqlen)
             if (sdp_svc_attr_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_attr_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_attr_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 1)
         return -SDP_INVALID_SYNTAX;
diff --git a/hw/char/cadence_uart.c b/hw/char/cadence_uart.c
index 9d379e5b1..a217271be 100644
--- a/hw/char/cadence_uart.c
+++ b/hw/char/cadence_uart.c
@@ -374,6 +374,9 @@ static void uart_write(void *opaque, hwaddr offset,
 
     DB_PRINT(" offset:%x data:%08x\n", (unsigned)offset, (unsigned)value);
     offset >>= 2;
+    if (offset >= CADENCE_UART_R_MAX) {
+        return;
+    }
     switch (offset) {
     case R_IER: /* ier (wts imr) */
         s->r[R_IMR] |= value;
diff --git a/hw/char/escc.c b/hw/char/escc.c
index ba653efd6..c9840e11d 100644
--- a/hw/char/escc.c
+++ b/hw/char/escc.c
@@ -842,13 +842,13 @@ static void sunkbd_handle_event(DeviceState *dev, QemuConsole *src,
     ChannelState *s = (ChannelState *)dev;
     int qcode, keycode;
 
-    assert(evt->kind == INPUT_EVENT_KIND_KEY);
-    qcode = qemu_input_key_value_to_qcode(evt->key->key);
+    assert(evt->type == INPUT_EVENT_KIND_KEY);
+    qcode = qemu_input_key_value_to_qcode(evt->u.key->key);
     trace_escc_sunkbd_event_in(qcode, QKeyCode_lookup[qcode],
-                               evt->key->down);
+                               evt->u.key->down);
 
     if (qcode == Q_KEY_CODE_CAPS_LOCK) {
-        if (evt->key->down) {
+        if (evt->u.key->down) {
             s->caps_lock_mode ^= 1;
             if (s->caps_lock_mode == 2) {
                 return; /* Drop second press */
@@ -862,7 +862,7 @@ static void sunkbd_handle_event(DeviceState *dev, QemuConsole *src,
     }
 
     if (qcode == Q_KEY_CODE_NUM_LOCK) {
-        if (evt->key->down) {
+        if (evt->u.key->down) {
             s->num_lock_mode ^= 1;
             if (s->num_lock_mode == 2) {
                 return; /* Drop second press */
@@ -876,7 +876,7 @@ static void sunkbd_handle_event(DeviceState *dev, QemuConsole *src,
     }
 
     keycode = qcode_to_keycode[qcode];
-    if (!evt->key->down) {
+    if (!evt->u.key->down) {
         keycode |= 0x80;
     }
     trace_escc_sunkbd_event_out(keycode);
@@ -1035,6 +1035,7 @@ static void escc_class_init(ObjectClass *klass, void *data)
     dc->reset = escc_reset;
     dc->vmsd = &vmstate_escc;
     dc->props = escc_properties;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo escc_info = {
diff --git a/hw/char/etraxfs_ser.c b/hw/char/etraxfs_ser.c
index 857c13621..562021e28 100644
--- a/hw/char/etraxfs_ser.c
+++ b/hw/char/etraxfs_ser.c
@@ -182,15 +182,13 @@ static void serial_receive(void *opaque, const uint8_t *buf, int size)
 static int serial_can_receive(void *opaque)
 {
     ETRAXSerial *s = opaque;
-    int r;
 
     /* Is the receiver enabled?  */
     if (!(s->regs[RW_REC_CTRL] & (1 << 3))) {
         return 0;
     }
 
-    r = sizeof(s->rx_fifo) - s->rx_fifo_len;
-    return r;
+    return sizeof(s->rx_fifo) - s->rx_fifo_len;
 }
 
 static void serial_event(void *opaque, int event)
diff --git a/hw/char/exynos4210_uart.c b/hw/char/exynos4210_uart.c
index 7614e5860..215f9622c 100644
--- a/hw/char/exynos4210_uart.c
+++ b/hw/char/exynos4210_uart.c
@@ -234,10 +234,8 @@ static int fifo_empty_elements_number(Exynos4210UartFIFO *q)
 
 static void fifo_reset(Exynos4210UartFIFO *q)
 {
-    if (q->data != NULL) {
-        g_free(q->data);
-        q->data = NULL;
-    }
+    g_free(q->data);
+    q->data = NULL;
 
     q->data = (uint8_t *)g_malloc0(q->size);
 
diff --git a/hw/char/imx_serial.c b/hw/char/imx_serial.c
index f3fbc776b..f30f9c24b 100644
--- a/hw/char/imx_serial.c
+++ b/hw/char/imx_serial.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2008 OKL
  * Originally Written by Hans Jiang
  * Copyright (c) 2011 NICTA Pty Ltd.
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -17,63 +18,24 @@
  *     is a real serial device.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
+#include "hw/char/imx_serial.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
-#include "hw/arm/imx.h"
 
-//#define DEBUG_SERIAL 1
-#ifdef DEBUG_SERIAL
-#define DPRINTF(fmt, args...) \
-do { printf("imx_serial: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-//#define DEBUG_IMPLEMENTATION 1
-#ifdef DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-    do  { fprintf(stderr, "imx_serial: " fmt, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
+#ifndef DEBUG_IMX_UART
+#define DEBUG_IMX_UART 0
 #endif
 
-#define TYPE_IMX_SERIAL "imx-serial"
-#define IMX_SERIAL(obj) OBJECT_CHECK(IMXSerialState, (obj), TYPE_IMX_SERIAL)
-
-typedef struct IMXSerialState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    int32_t readbuff;
-
-    uint32_t usr1;
-    uint32_t usr2;
-    uint32_t ucr1;
-    uint32_t ucr2;
-    uint32_t uts1;
-
-    /*
-     * The registers below are implemented just so that the
-     * guest OS sees what it has written
-     */
-    uint32_t onems;
-    uint32_t ufcr;
-    uint32_t ubmr;
-    uint32_t ubrc;
-    uint32_t ucr3;
-
-    qemu_irq irq;
-    CharDriverState *chr;
-} IMXSerialState;
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_UART) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_SERIAL, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static const VMStateDescription vmstate_imx_serial = {
-    .name = "imx-serial",
+    .name = TYPE_IMX_SERIAL,
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
@@ -91,61 +53,14 @@ static const VMStateDescription vmstate_imx_serial = {
     },
 };
 
-
-#define URXD_CHARRDY    (1<<15)   /* character read is valid */
-#define URXD_ERR        (1<<14)   /* Character has error */
-#define URXD_BRK        (1<<11)   /* Break received */
-
-#define USR1_PARTYER    (1<<15)   /* Parity Error */
-#define USR1_RTSS       (1<<14)   /* RTS pin status */
-#define USR1_TRDY       (1<<13)   /* Tx ready */
-#define USR1_RTSD       (1<<12)   /* RTS delta: pin changed state */
-#define USR1_ESCF       (1<<11)   /* Escape sequence interrupt */
-#define USR1_FRAMERR    (1<<10)   /* Framing error  */
-#define USR1_RRDY       (1<<9)    /* receiver ready */
-#define USR1_AGTIM      (1<<8)    /* Aging timer interrupt */
-#define USR1_DTRD       (1<<7)    /* DTR changed */
-#define USR1_RXDS       (1<<6)    /* Receiver is idle */
-#define USR1_AIRINT     (1<<5)    /* Aysnch IR interrupt */
-#define USR1_AWAKE      (1<<4)    /* Falling edge detected on RXd pin */
-
-#define USR2_ADET       (1<<15)   /* Autobaud complete */
-#define USR2_TXFE       (1<<14)   /* Transmit FIFO empty */
-#define USR2_DTRF       (1<<13)   /* DTR/DSR transition */
-#define USR2_IDLE       (1<<12)   /* UART has been idle for too long */
-#define USR2_ACST       (1<<11)   /* Autobaud counter stopped */
-#define USR2_RIDELT     (1<<10)   /* Ring Indicator delta */
-#define USR2_RIIN       (1<<9)    /* Ring Indicator Input */
-#define USR2_IRINT      (1<<8)    /* Serial Infrared Interrupt */
-#define USR2_WAKE       (1<<7)    /* Start bit detected */
-#define USR2_DCDDELT    (1<<6)    /* Data Carrier Detect delta */
-#define USR2_DCDIN      (1<<5)    /* Data Carrier Detect Input */
-#define USR2_RTSF       (1<<4)    /* RTS transition */
-#define USR2_TXDC       (1<<3)    /* Transmission complete */
-#define USR2_BRCD       (1<<2)    /* Break condition detected */
-#define USR2_ORE        (1<<1)    /* Overrun error */
-#define USR2_RDR        (1<<0)    /* Receive data ready */
-
-#define UCR1_TRDYEN     (1<<13)   /* Tx Ready Interrupt Enable */
-#define UCR1_RRDYEN     (1<<9)    /* Rx Ready Interrupt Enable */
-#define UCR1_TXMPTYEN   (1<<6)    /* Tx Empty Interrupt Enable */
-#define UCR1_UARTEN     (1<<0)    /* UART Enable */
-
-#define UCR2_TXEN       (1<<2)    /* Transmitter enable */
-#define UCR2_RXEN       (1<<1)    /* Receiver enable */
-#define UCR2_SRST       (1<<0)    /* Reset complete */
-
-#define UTS1_TXEMPTY    (1<<6)
-#define UTS1_RXEMPTY    (1<<5)
-#define UTS1_TXFULL     (1<<4)
-#define UTS1_RXFULL     (1<<3)
-
 static void imx_update(IMXSerialState *s)
 {
     uint32_t flags;
 
     flags = (s->usr1 & s->ucr1) & (USR1_TRDY|USR1_RRDY);
-    if (!(s->ucr1 & UCR1_TXMPTYEN)) {
+    if (s->ucr1 & UCR1_TXMPTYEN) {
+        flags |= (s->uts1 & UTS1_TXEMPTY);
+    } else {
         flags &= ~USR1_TRDY;
     }
 
@@ -192,7 +107,8 @@ static uint64_t imx_serial_read(void *opaque, hwaddr offset,
     IMXSerialState *s = (IMXSerialState *)opaque;
     uint32_t c;
 
-    DPRINTF("read(offset=%x)\n", offset >> 2);
+    DPRINTF("read(offset=0x%" HWADDR_PRIx ")\n", offset);
+
     switch (offset >> 2) {
     case 0x0: /* URXD */
         c = s->readbuff;
@@ -203,7 +119,9 @@ static uint64_t imx_serial_read(void *opaque, hwaddr offset,
             s->usr2 &= ~USR2_RDR;
             s->uts1 |= UTS1_RXEMPTY;
             imx_update(s);
-            qemu_chr_accept_input(s->chr);
+            if (s->chr) {
+                qemu_chr_accept_input(s->chr);
+            }
         }
         return c;
 
@@ -242,20 +160,20 @@ static uint64_t imx_serial_read(void *opaque, hwaddr offset,
         return 0x0; /* TODO */
 
     default:
-        IPRINTF("imx_serial_read: bad offset: 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
         return 0;
     }
 }
 
 static void imx_serial_write(void *opaque, hwaddr offset,
-                      uint64_t value, unsigned size)
+                             uint64_t value, unsigned size)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
     unsigned char ch;
 
-    DPRINTF("write(offset=%x, value = %x) to %s\n",
-            offset >> 2,
-            (unsigned int)value, s->chr ? s->chr->label : "NODEV");
+    DPRINTF("write(offset=0x%" HWADDR_PRIx ", value = 0x%x) to %s\n",
+            offset, (unsigned int)value, s->chr ? s->chr->label : "NODEV");
 
     switch (offset >> 2) {
     case 0x10: /* UTXD */
@@ -273,7 +191,9 @@ static void imx_serial_write(void *opaque, hwaddr offset,
 
     case 0x20: /* UCR1 */
         s->ucr1 = value & 0xffff;
+
         DPRINTF("write(ucr1=%x)\n", (unsigned int)value);
+
         imx_update(s);
         break;
 
@@ -290,7 +210,9 @@ static void imx_serial_write(void *opaque, hwaddr offset,
         }
         if (value & UCR2_RXEN) {
             if (!(s->ucr2 & UCR2_RXEN)) {
-                qemu_chr_accept_input(s->chr);
+                if (s->chr) {
+                    qemu_chr_accept_input(s->chr);
+                }
             }
         }
         s->ucr2 = value & 0xffff;
@@ -298,25 +220,25 @@ static void imx_serial_write(void *opaque, hwaddr offset,
 
     case 0x25: /* USR1 */
         value &= USR1_AWAKE | USR1_AIRINT | USR1_DTRD | USR1_AGTIM |
-            USR1_FRAMERR | USR1_ESCF | USR1_RTSD | USR1_PARTYER;
+                 USR1_FRAMERR | USR1_ESCF | USR1_RTSD | USR1_PARTYER;
         s->usr1 &= ~value;
         break;
 
     case 0x26: /* USR2 */
-       /*
-        * Writing 1 to some bits clears them; all other
-        * values are ignored
-        */
+        /*
+         * Writing 1 to some bits clears them; all other
+         * values are ignored
+         */
         value &= USR2_ADET | USR2_DTRF | USR2_IDLE | USR2_ACST |
-            USR2_RIDELT | USR2_IRINT | USR2_WAKE |
-            USR2_DCDDELT | USR2_RTSF | USR2_BRCD | USR2_ORE;
+                 USR2_RIDELT | USR2_IRINT | USR2_WAKE |
+                 USR2_DCDDELT | USR2_RTSF | USR2_BRCD | USR2_ORE;
         s->usr2 &= ~value;
         break;
 
-        /*
-         * Linux expects to see what it writes to these registers
-         * We don't currently alter the baud rate
-         */
+    /*
+     * Linux expects to see what it writes to these registers
+     * We don't currently alter the baud rate
+     */
     case 0x29: /* UBIR */
         s->ubrc = value & 0xffff;
         break;
@@ -339,12 +261,14 @@ static void imx_serial_write(void *opaque, hwaddr offset,
 
     case 0x2d: /* UTS1 */
     case 0x23: /* UCR4 */
-        IPRINTF("Unimplemented Register %x written to\n", offset >> 2);
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: Unimplemented reg 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
         /* TODO */
         break;
 
     default:
-        IPRINTF("imx_serial_write: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
     }
 }
 
@@ -357,7 +281,9 @@ static int imx_can_receive(void *opaque)
 static void imx_put_data(void *opaque, uint32_t value)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
+
     DPRINTF("received char\n");
+
     s->usr1 |= USR1_RRDY;
     s->usr2 |= USR2_RDR;
     s->uts1 &= ~UTS1_RXEMPTY;
@@ -384,62 +310,30 @@ static const struct MemoryRegionOps imx_serial_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int imx_serial_init(SysBusDevice *dev)
+static void imx_serial_realize(DeviceState *dev, Error **errp)
 {
     IMXSerialState *s = IMX_SERIAL(dev);
 
-
-    memory_region_init_io(&s->iomem, OBJECT(s), &imx_serial_ops, s,
-                          "imx-serial", 0x1000);
-    sysbus_init_mmio(dev, &s->iomem);
-    sysbus_init_irq(dev, &s->irq);
-
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, imx_can_receive, imx_receive,
                               imx_event, s);
     } else {
-        DPRINTF("No char dev for uart at 0x%lx\n",
-                (unsigned long)s->iomem.ram_addr);
+        DPRINTF("No char dev for uart\n");
     }
-
-    return 0;
 }
 
-void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq)
+static void imx_serial_init(Object *obj)
 {
-    DeviceState *dev;
-    SysBusDevice *bus;
-    CharDriverState *chr;
-    const char chr_name[] = "serial";
-    char label[ARRAY_SIZE(chr_name) + 1];
-
-    dev = qdev_create(NULL, TYPE_IMX_SERIAL);
-
-    if (uart >= MAX_SERIAL_PORTS) {
-        hw_error("Cannot assign uart %d: QEMU supports only %d ports\n",
-                 uart, MAX_SERIAL_PORTS);
-    }
-    chr = serial_hds[uart];
-    if (!chr) {
-        snprintf(label, ARRAY_SIZE(label), "%s%d", chr_name, uart);
-        chr = qemu_chr_new(label, "null", NULL);
-        if (!(chr)) {
-            hw_error("Can't assign serial port to imx-uart%d.\n", uart);
-        }
-    }
-
-    qdev_prop_set_chr(dev, "chardev", chr);
-    bus = SYS_BUS_DEVICE(dev);
-    qdev_init_nofail(dev);
-    if (addr != (hwaddr)-1) {
-        sysbus_mmio_map(bus, 0, addr);
-    }
-    sysbus_connect_irq(bus, 0, irq);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    IMXSerialState *s = IMX_SERIAL(obj);
 
+    memory_region_init_io(&s->iomem, obj, &imx_serial_ops, s,
+                          TYPE_IMX_SERIAL, 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
 }
 
-
-static Property imx32_serial_properties[] = {
+static Property imx_serial_properties[] = {
     DEFINE_PROP_CHR("chardev", IMXSerialState, chr),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -447,21 +341,21 @@ static Property imx32_serial_properties[] = {
 static void imx_serial_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = imx_serial_init;
+    dc->realize = imx_serial_realize;
     dc->vmsd = &vmstate_imx_serial;
     dc->reset = imx_serial_reset_at_boot;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
     dc->desc = "i.MX series UART";
-    dc->props = imx32_serial_properties;
+    dc->props = imx_serial_properties;
 }
 
 static const TypeInfo imx_serial_info = {
-    .name = TYPE_IMX_SERIAL,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(IMXSerialState),
-    .class_init = imx_serial_class_init,
+    .name           = TYPE_IMX_SERIAL,
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(IMXSerialState),
+    .instance_init  = imx_serial_init,
+    .class_init     = imx_serial_class_init,
 };
 
 static void imx_serial_register_types(void)
diff --git a/hw/char/mcf_uart.c b/hw/char/mcf_uart.c
index 98fd44e66..cda22eea5 100644
--- a/hw/char/mcf_uart.c
+++ b/hw/char/mcf_uart.c
@@ -126,7 +126,7 @@ static void mcf_uart_do_tx(mcf_uart_state *s)
 static void mcf_do_command(mcf_uart_state *s, uint8_t cmd)
 {
     /* Misc command.  */
-    switch ((cmd >> 4) & 3) {
+    switch ((cmd >> 4) & 7) {
     case 0: /* No-op.  */
         break;
     case 1: /* Reset mode register pointer.  */
diff --git a/hw/char/omap_uart.c b/hw/char/omap_uart.c
index 88f20943e..278ce36cb 100644
--- a/hw/char/omap_uart.c
+++ b/hw/char/omap_uart.c
@@ -55,8 +55,7 @@ struct omap_uart_s *omap_uart_init(hwaddr base,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr)
 {
-    struct omap_uart_s *s = (struct omap_uart_s *)
-            g_malloc0(sizeof(struct omap_uart_s));
+    struct omap_uart_s *s = g_new0(struct omap_uart_s, 1);
 
     s->base = base;
     s->fclk = fclk;
diff --git a/hw/char/virtio-serial-bus.c b/hw/char/virtio-serial-bus.c
index be9705871..497b0afd9 100644
--- a/hw/char/virtio-serial-bus.c
+++ b/hw/char/virtio-serial-bus.c
@@ -705,10 +705,7 @@ static int fetch_active_ports_list(QEMUFile *f, int version_id,
 
                 qemu_get_buffer(f, (unsigned char *)&port->elem,
                                 sizeof(port->elem));
-                virtqueue_map_sg(port->elem.in_sg, port->elem.in_addr,
-                                 port->elem.in_num, 1);
-                virtqueue_map_sg(port->elem.out_sg, port->elem.out_addr,
-                                 port->elem.out_num, 1);
+                virtqueue_map(&port->elem);
 
                 /*
                  *  Port was throttled on source machine.  Let's
diff --git a/hw/core/loader.c b/hw/core/loader.c
index 216eeeb91..eb67f05ee 100644
--- a/hw/core/loader.c
+++ b/hw/core/loader.c
@@ -594,8 +594,7 @@ static int load_uboot_image(const char *filename, hwaddr *ep, hwaddr *loadaddr,
     ret = hdr->ih_size;
 
 out:
-    if (data)
-        g_free(data);
+    g_free(data);
     close(fd);
     return ret;
 }
@@ -741,7 +740,7 @@ static void *rom_set_mr(Rom *rom, Object *owner, const char *name)
     memory_region_init_resizeable_ram(rom->mr, owner, name,
                                       rom->datasize, rom->romsize,
                                       fw_cfg_resized,
-                                      &error_abort);
+                                      &error_fatal);
     memory_region_set_readonly(rom->mr, true);
     vmstate_register_ram_global(rom->mr);
 
diff --git a/hw/core/machine.c b/hw/core/machine.c
index ac4654e9d..f4d317088 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -226,6 +226,20 @@ static void machine_set_usb(Object *obj, bool value, Error **errp)
     ms->usb_disabled = !value;
 }
 
+static bool machine_get_igd_gfx_passthru(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->igd_gfx_passthru;
+}
+
+static void machine_set_igd_gfx_passthru(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->igd_gfx_passthru = value;
+}
+
 static char *machine_get_firmware(Object *obj, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
@@ -269,6 +283,21 @@ static bool machine_get_suppress_vmdesc(Object *obj, Error **errp)
     return ms->suppress_vmdesc;
 }
 
+static void machine_set_enforce_config_section(Object *obj, bool value,
+                                             Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->enforce_config_section = value;
+}
+
+static bool machine_get_enforce_config_section(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->enforce_config_section;
+}
+
 static int error_on_sysbus_device(SysBusDevice *sbdev, void *opaque)
 {
     error_report("Option '-device %s' cannot be handled by this machine",
@@ -302,6 +331,17 @@ static void machine_class_init(ObjectClass *oc, void *data)
     mc->default_ram_size = 128 * M_BYTE;
 }
 
+static void machine_class_base_init(ObjectClass *oc, void *data)
+{
+    if (!object_class_is_abstract(oc)) {
+        MachineClass *mc = MACHINE_CLASS(oc);
+        const char *cname = object_class_get_name(oc);
+        assert(g_str_has_suffix(cname, TYPE_MACHINE_SUFFIX));
+        mc->name = g_strndup(cname,
+                            strlen(cname) - strlen(TYPE_MACHINE_SUFFIX));
+    }
+}
+
 static void machine_initfn(Object *obj)
 {
     MachineState *ms = MACHINE(obj);
@@ -388,6 +428,12 @@ static void machine_initfn(Object *obj)
     object_property_set_description(obj, "usb",
                                     "Set on/off to enable/disable usb",
                                     NULL);
+    object_property_add_bool(obj, "igd-passthru",
+                             machine_get_igd_gfx_passthru,
+                             machine_set_igd_gfx_passthru, NULL);
+    object_property_set_description(obj, "igd-passthru",
+                                    "Set on/off to enable/disable igd passthrou",
+                                    NULL);
     object_property_add_str(obj, "firmware",
                             machine_get_firmware,
                             machine_set_firmware, NULL);
@@ -406,6 +452,12 @@ static void machine_initfn(Object *obj)
     object_property_set_description(obj, "suppress-vmdesc",
                                     "Set on to disable self-describing migration",
                                     NULL);
+    object_property_add_bool(obj, "enforce-config-section",
+                             machine_get_enforce_config_section,
+                             machine_set_enforce_config_section, NULL);
+    object_property_set_description(obj, "enforce-config-section",
+                                    "Set on to enforce configuration section migration",
+                                    NULL);
 
     /* Register notifier when init is done for sysbus sanity checks */
     ms->sysbus_notifier.notify = machine_init_notify;
@@ -431,11 +483,6 @@ bool machine_usb(MachineState *machine)
     return machine->usb;
 }
 
-bool machine_iommu(MachineState *machine)
-{
-    return machine->iommu;
-}
-
 bool machine_kernel_irqchip_allowed(MachineState *machine)
 {
     return machine->kernel_irqchip_allowed;
@@ -472,6 +519,7 @@ static const TypeInfo machine_info = {
     .abstract = true,
     .class_size = sizeof(MachineClass),
     .class_init    = machine_class_init,
+    .class_base_init = machine_class_base_init,
     .instance_size = sizeof(MachineState),
     .instance_init = machine_initfn,
     .instance_finalize = machine_finalize,
diff --git a/hw/core/null-machine.c b/hw/core/null-machine.c
index 1ec7c3bbe..f36fbf231 100644
--- a/hw/core/null-machine.c
+++ b/hw/core/null-machine.c
@@ -19,17 +19,11 @@ static void machine_none_init(MachineState *machine)
 {
 }
 
-static QEMUMachine machine_none = {
-    .name = "none",
-    .desc = "empty machine",
-    .init = machine_none_init,
-    .max_cpus = 0,
-};
-
-static void register_machines(void)
+static void machine_none_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&machine_none);
+    mc->desc = "empty machine";
+    mc->init = machine_none_init;
+    mc->max_cpus = 0;
 }
 
-machine_init(register_machines);
-
+DEFINE_MACHINE("none", machine_none_machine_init)
diff --git a/hw/core/ptimer.c b/hw/core/ptimer.c
index 8437bd6e8..edf077cfd 100644
--- a/hw/core/ptimer.c
+++ b/hw/core/ptimer.c
@@ -9,6 +9,7 @@
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
 #include "qemu/host-utils.h"
+#include "sysemu/replay.h"
 
 struct ptimer_state
 {
@@ -27,7 +28,7 @@ struct ptimer_state
 static void ptimer_trigger(ptimer_state *s)
 {
     if (s->bh) {
-        qemu_bh_schedule(s->bh);
+        replay_bh_schedule_event(s->bh);
     }
 }
 
diff --git a/hw/core/qdev-properties.c b/hw/core/qdev-properties.c
index 04fd80a4d..33e245e12 100644
--- a/hw/core/qdev-properties.c
+++ b/hw/core/qdev-properties.c
@@ -422,9 +422,7 @@ static void set_string(Object *obj, Visitor *v, void *opaque,
         error_propagate(errp, local_err);
         return;
     }
-    if (*ptr) {
-        g_free(*ptr);
-    }
+    g_free(*ptr);
     *ptr = str;
 }
 
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index b2f404a76..b3ad46775 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -325,6 +325,11 @@ void qdev_reset_all(DeviceState *dev)
     qdev_walk_children(dev, NULL, NULL, qdev_reset_one, qbus_reset_one, NULL);
 }
 
+void qdev_reset_all_fn(void *opaque)
+{
+    qdev_reset_all(DEVICE(opaque));
+}
+
 void qbus_reset_all(BusState *bus)
 {
     qbus_walk_children(bus, NULL, NULL, qdev_reset_one, qbus_reset_one, NULL);
@@ -417,17 +422,21 @@ void qdev_init_gpio_in_named(DeviceState *dev, qemu_irq_handler handler,
 {
     int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
-    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-in");
 
     assert(gpio_list->num_out == 0 || !name);
     gpio_list->in = qemu_extend_irqs(gpio_list->in, gpio_list->num_in, handler,
                                      dev, n);
 
+    if (!name) {
+        name = "unnamed-gpio-in";
+    }
     for (i = gpio_list->num_in; i < gpio_list->num_in + n; i++) {
+        gchar *propname = g_strdup_printf("%s[%u]", name, i);
+
         object_property_add_child(OBJECT(dev), propname,
                                   OBJECT(gpio_list->in[i]), &error_abort);
+        g_free(propname);
     }
-    g_free(propname);
 
     gpio_list->num_in += n;
 }
@@ -442,20 +451,25 @@ void qdev_init_gpio_out_named(DeviceState *dev, qemu_irq *pins,
 {
     int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
-    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-out");
 
     assert(gpio_list->num_in == 0 || !name);
-    gpio_list->num_out += n;
 
+    if (!name) {
+        name = "unnamed-gpio-out";
+    }
+    memset(pins, 0, sizeof(*pins) * n);
     for (i = 0; i < n; ++i) {
-        memset(&pins[i], 0, sizeof(*pins));
+        gchar *propname = g_strdup_printf("%s[%u]", name,
+                                          gpio_list->num_out + i);
+
         object_property_add_link(OBJECT(dev), propname, TYPE_IRQ,
                                  (Object **)&pins[i],
                                  object_property_allow_set_link,
                                  OBJ_PROP_LINK_UNREF_ON_RELEASE,
                                  &error_abort);
+        g_free(propname);
     }
-    g_free(propname);
+    gpio_list->num_out += n;
 }
 
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n)
@@ -506,7 +520,7 @@ qemu_irq qdev_get_gpio_out_connector(DeviceState *dev, const char *name, int n)
     return ret;
 }
 
-/* disconnect a GPIO ouput, returning the disconnected input (if any) */
+/* disconnect a GPIO output, returning the disconnected input (if any) */
 
 static qemu_irq qdev_disconnect_gpio_out_named(DeviceState *dev,
                                                const char *name, int n)
diff --git a/hw/cpu/Makefile.objs b/hw/cpu/Makefile.objs
index 6381238cc..0954a1872 100644
--- a/hw/cpu/Makefile.objs
+++ b/hw/cpu/Makefile.objs
@@ -2,5 +2,4 @@ obj-$(CONFIG_ARM11MPCORE) += arm11mpcore.o
 obj-$(CONFIG_REALVIEW) += realview_mpcore.o
 obj-$(CONFIG_A9MPCORE) += a9mpcore.o
 obj-$(CONFIG_A15MPCORE) += a15mpcore.o
-obj-$(CONFIG_ICC_BUS) += icc_bus.o
 
diff --git a/hw/cpu/a15mpcore.c b/hw/cpu/a15mpcore.c
index acc419e11..94e8cc1a6 100644
--- a/hw/cpu/a15mpcore.c
+++ b/hw/cpu/a15mpcore.c
@@ -20,6 +20,7 @@
 
 #include "hw/cpu/a15mpcore.h"
 #include "sysemu/kvm.h"
+#include "kvm_arm.h"
 
 static void a15mp_priv_set_irq(void *opaque, int irq, int level)
 {
@@ -33,16 +34,11 @@ static void a15mp_priv_initfn(Object *obj)
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     A15MPPrivState *s = A15MPCORE_PRIV(obj);
     DeviceState *gicdev;
-    const char *gictype = "arm_gic";
-
-    if (kvm_irqchip_in_kernel()) {
-        gictype = "kvm-arm-gic";
-    }
 
     memory_region_init(&s->container, obj, "a15mp-priv-container", 0x8000);
     sysbus_init_mmio(sbd, &s->container);
 
-    object_initialize(&s->gic, sizeof(s->gic), gictype);
+    object_initialize(&s->gic, sizeof(s->gic), gic_class_name());
     gicdev = DEVICE(&s->gic);
     qdev_set_parent_bus(gicdev, sysbus_get_default());
     qdev_prop_set_uint32(gicdev, "revision", 2);
@@ -56,10 +52,23 @@ static void a15mp_priv_realize(DeviceState *dev, Error **errp)
     SysBusDevice *busdev;
     int i;
     Error *err = NULL;
+    bool has_el3;
+    Object *cpuobj;
 
     gicdev = DEVICE(&s->gic);
     qdev_prop_set_uint32(gicdev, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(gicdev, "num-irq", s->num_irq);
+
+    if (!kvm_irqchip_in_kernel()) {
+        /* Make the GIC's TZ support match the CPUs. We assume that
+         * either all the CPUs have TZ, or none do.
+         */
+        cpuobj = OBJECT(qemu_get_cpu(0));
+        has_el3 = object_property_find(cpuobj, "has_el3", NULL) &&
+            object_property_get_bool(cpuobj, "has_el3", &error_abort);
+        qdev_prop_set_bit(gicdev, "has-security-extensions", has_el3);
+    }
+
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err != NULL) {
         error_propagate(errp, err);
@@ -79,14 +88,21 @@ static void a15mp_priv_realize(DeviceState *dev, Error **errp)
     for (i = 0; i < s->num_cpu; i++) {
         DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
         int ppibase = s->num_irq - 32 + i * 32;
-        /* physical timer; we wire it up to the non-secure timer's ID,
-         * since a real A15 always has TrustZone but QEMU doesn't.
+        int irq;
+        /* Mapping from the output timer irq lines from the CPU to the
+         * GIC PPI inputs used on the A15:
          */
-        qdev_connect_gpio_out(cpudev, 0,
-                              qdev_get_gpio_in(gicdev, ppibase + 30));
-        /* virtual timer */
-        qdev_connect_gpio_out(cpudev, 1,
-                              qdev_get_gpio_in(gicdev, ppibase + 27));
+        const int timer_irq[] = {
+            [GTIMER_PHYS] = 30,
+            [GTIMER_VIRT] = 27,
+            [GTIMER_HYP]  = 26,
+            [GTIMER_SEC]  = 29,
+        };
+        for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) {
+            qdev_connect_gpio_out(cpudev, irq,
+                                  qdev_get_gpio_in(gicdev,
+                                                   ppibase + timer_irq[irq]));
+        }
     }
 
     /* Memory map (addresses are offsets from PERIPHBASE):
diff --git a/hw/cpu/a9mpcore.c b/hw/cpu/a9mpcore.c
index c09358c6e..869818cd8 100644
--- a/hw/cpu/a9mpcore.c
+++ b/hw/cpu/a9mpcore.c
@@ -49,6 +49,8 @@ static void a9mp_priv_realize(DeviceState *dev, Error **errp)
                  *wdtbusdev;
     Error *err = NULL;
     int i;
+    bool has_el3;
+    Object *cpuobj;
 
     scudev = DEVICE(&s->scu);
     qdev_prop_set_uint32(scudev, "num-cpu", s->num_cpu);
@@ -62,6 +64,15 @@ static void a9mp_priv_realize(DeviceState *dev, Error **errp)
     gicdev = DEVICE(&s->gic);
     qdev_prop_set_uint32(gicdev, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(gicdev, "num-irq", s->num_irq);
+
+    /* Make the GIC's TZ support match the CPUs. We assume that
+     * either all the CPUs have TZ, or none do.
+     */
+    cpuobj = OBJECT(qemu_get_cpu(0));
+    has_el3 = object_property_find(cpuobj, "has_el3", NULL) &&
+        object_property_get_bool(cpuobj, "has_el3", &error_abort);
+    qdev_prop_set_bit(gicdev, "has-security-extensions", has_el3);
+
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err != NULL) {
         error_propagate(errp, err);
diff --git a/hw/cpu/icc_bus.c b/hw/cpu/icc_bus.c
deleted file mode 100644
index 6646ea2b3..000000000
--- a/hw/cpu/icc_bus.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/* icc_bus.c
- * emulate x86 ICC (Interrupt Controller Communications) bus
- *
- * Copyright (c) 2013 Red Hat, Inc
- *
- * Authors:
- *     Igor Mammedov <imammedo@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-#include "hw/cpu/icc_bus.h"
-#include "hw/sysbus.h"
-
-/* icc-bridge implementation */
-
-static const TypeInfo icc_bus_info = {
-    .name = TYPE_ICC_BUS,
-    .parent = TYPE_BUS,
-    .instance_size = sizeof(ICCBus),
-};
-
-
-/* icc-device implementation */
-
-static void icc_device_realize(DeviceState *dev, Error **errp)
-{
-    ICCDeviceClass *idc = ICC_DEVICE_GET_CLASS(dev);
-
-    /* convert to QOM */
-    if (idc->realize) {
-        idc->realize(dev, errp);
-    }
-
-}
-
-static void icc_device_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    dc->realize = icc_device_realize;
-    dc->bus_type = TYPE_ICC_BUS;
-}
-
-static const TypeInfo icc_device_info = {
-    .name = TYPE_ICC_DEVICE,
-    .parent = TYPE_DEVICE,
-    .abstract = true,
-    .instance_size = sizeof(ICCDevice),
-    .class_size = sizeof(ICCDeviceClass),
-    .class_init = icc_device_class_init,
-};
-
-
-/*  icc-bridge implementation */
-
-typedef struct ICCBridgeState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    /*< public >*/
-
-    ICCBus icc_bus;
-    MemoryRegion apic_container;
-} ICCBridgeState;
-
-#define ICC_BRIDGE(obj) OBJECT_CHECK(ICCBridgeState, (obj), TYPE_ICC_BRIDGE)
-
-static void icc_bridge_init(Object *obj)
-{
-    ICCBridgeState *s = ICC_BRIDGE(obj);
-    SysBusDevice *sb = SYS_BUS_DEVICE(obj);
-
-    qbus_create_inplace(&s->icc_bus, sizeof(s->icc_bus), TYPE_ICC_BUS,
-                        DEVICE(s), "icc");
-
-    /* Do not change order of registering regions,
-     * APIC must be first registered region, board maps it by 0 index
-     */
-    memory_region_init(&s->apic_container, obj, "icc-apic-container",
-                       APIC_SPACE_SIZE);
-    sysbus_init_mmio(sb, &s->apic_container);
-    s->icc_bus.apic_address_space = &s->apic_container;
-}
-
-static void icc_bridge_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
-}
-
-static const TypeInfo icc_bridge_info = {
-    .name  = TYPE_ICC_BRIDGE,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_init  = icc_bridge_init,
-    .instance_size  = sizeof(ICCBridgeState),
-    .class_init = icc_bridge_class_init,
-};
-
-
-static void icc_bus_register_types(void)
-{
-    type_register_static(&icc_bus_info);
-    type_register_static(&icc_device_info);
-    type_register_static(&icc_bridge_info);
-}
-
-type_init(icc_bus_register_types)
diff --git a/hw/cris/axis_dev88.c b/hw/cris/axis_dev88.c
index 3cae480fa..de880d173 100644
--- a/hw/cris/axis_dev88.c
+++ b/hw/cris/axis_dev88.c
@@ -138,7 +138,7 @@ static void tempsensor_clkedge(struct tempsensor_t *s,
                     s->count = 16;
 
                     if ((s->regs[0] & 0xff) == 0) {
-                        /* 25 degrees celcius.  */
+                        /* 25 degrees celsius.  */
                         s->shiftreg = 0x0b9f;
                     } else if ((s->regs[0] & 0xff) == 0xff) {
                         /* Sensor ID, 0x8100 LM70.  */
@@ -277,7 +277,7 @@ void axisdev88_init(MachineState *machine)
     /* The ETRAX-FS has 128Kb on chip ram, the docs refer to it as the 
        internal memory.  */
     memory_region_init_ram(phys_intmem, NULL, "axisdev88.chipram", INTMEM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_intmem);
     memory_region_add_subregion(address_space_mem, 0x38000000, phys_intmem);
 
@@ -351,16 +351,11 @@ void axisdev88_init(MachineState *machine)
     }
 }
 
-static QEMUMachine axisdev88_machine = {
-    .name = "axis-dev88",
-    .desc = "AXIS devboard 88",
-    .init = axisdev88_init,
-    .is_default = 1,
-};
-
-static void axisdev88_machine_init(void)
+static void axisdev88_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&axisdev88_machine);
+    mc->desc = "AXIS devboard 88";
+    mc->init = axisdev88_init;
+    mc->is_default = 1;
 }
 
-machine_init(axisdev88_machine_init);
+DEFINE_MACHINE("axis-dev88", axisdev88_machine_init)
diff --git a/hw/cris/boot.c b/hw/cris/boot.c
index 622f353c9..1cfa339c2 100644
--- a/hw/cris/boot.c
+++ b/hw/cris/boot.c
@@ -72,7 +72,7 @@ void cris_load_image(CRISCPU *cpu, struct cris_load_info *li)
     /* Boots a kernel elf binary, os/linux-2.6/vmlinux from the axis 
        devboard SDK.  */
     image_size = load_elf(li->image_filename, translate_kernel_address, NULL,
-                          &entry, NULL, &high, 0, ELF_MACHINE, 0);
+                          &entry, NULL, &high, 0, EM_CRIS, 0);
     li->entry = entry;
     if (image_size < 0) {
         /* Takes a kimage from the axis devboard SDK.  */
diff --git a/hw/display/Makefile.objs b/hw/display/Makefile.objs
index dd8ea76d1..f0cf431a0 100644
--- a/hw/display/Makefile.objs
+++ b/hw/display/Makefile.objs
@@ -35,6 +35,10 @@ obj-$(CONFIG_VGA) += vga.o
 
 common-obj-$(CONFIG_QXL) += qxl.o qxl-logger.o qxl-render.o
 
-obj-$(CONFIG_VIRTIO) += virtio-gpu.o
+obj-$(CONFIG_VIRTIO) += virtio-gpu.o virtio-gpu-3d.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio-gpu-pci.o
 obj-$(CONFIG_VIRTIO_VGA) += virtio-vga.o
+virtio-gpu.o-cflags := $(VIRGL_CFLAGS)
+virtio-gpu.o-libs += $(VIRGL_LIBS)
+virtio-gpu-3d.o-cflags := $(VIRGL_CFLAGS)
+virtio-gpu-3d.o-libs += $(VIRGL_LIBS)
diff --git a/hw/display/cg3.c b/hw/display/cg3.c
index 2d3bd7081..e309fbe92 100644
--- a/hw/display/cg3.c
+++ b/hw/display/cg3.c
@@ -281,7 +281,7 @@ static void cg3_initfn(Object *obj)
     CG3State *s = CG3(obj);
 
     memory_region_init_ram(&s->rom, obj, "cg3.prom", FCODE_MAX_ROM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
@@ -310,7 +310,7 @@ static void cg3_realizefn(DeviceState *dev, Error **errp)
     }
 
     memory_region_init_ram(&s->vram_mem, NULL, "cg3.vram", s->vram_size,
-                           &error_abort);
+                           &error_fatal);
     memory_region_set_log(&s->vram_mem, true, DIRTY_MEMORY_VGA);
     vmstate_register_ram_global(&s->vram_mem);
     sysbus_init_mmio(sbd, &s->vram_mem);
diff --git a/hw/display/exynos4210_fimd.c b/hw/display/exynos4210_fimd.c
index 603ef5056..45239e836 100644
--- a/hw/display/exynos4210_fimd.c
+++ b/hw/display/exynos4210_fimd.c
@@ -1354,9 +1354,7 @@ static void exynos4210_fimd_reset(DeviceState *d)
         fimd_update_get_alpha(s, w);
     }
 
-    if (s->ifb != NULL) {
-        g_free(s->ifb);
-    }
+    g_free(s->ifb);
     s->ifb = NULL;
 
     exynos4210_fimd_invalidate(s);
diff --git a/hw/display/milkymist-tmu2.c b/hw/display/milkymist-tmu2.c
index 3e1d0b9c2..e2de28176 100644
--- a/hw/display/milkymist-tmu2.c
+++ b/hw/display/milkymist-tmu2.c
@@ -30,8 +30,8 @@
 #include "qemu/error-report.h"
 
 #include <X11/Xlib.h>
-#include <GL/gl.h>
-#include <GL/glx.h>
+#include <epoxy/gl.h>
+#include <epoxy/glx.h>
 
 enum {
     R_CTL = 0,
diff --git a/hw/display/omap_dss.c b/hw/display/omap_dss.c
index f1fef2767..b1c7af581 100644
--- a/hw/display/omap_dss.c
+++ b/hw/display/omap_dss.c
@@ -1051,8 +1051,7 @@ struct omap_dss_s *omap_dss_init(struct omap_target_agent_s *ta,
                 omap_clk fck1, omap_clk fck2, omap_clk ck54m,
                 omap_clk ick1, omap_clk ick2)
 {
-    struct omap_dss_s *s = (struct omap_dss_s *)
-            g_malloc0(sizeof(struct omap_dss_s));
+    struct omap_dss_s *s = g_new0(struct omap_dss_s, 1);
 
     s->irq = irq;
     s->drq = drq;
diff --git a/hw/display/omap_lcdc.c b/hw/display/omap_lcdc.c
index a7c6cd79b..678f9a1b4 100644
--- a/hw/display/omap_lcdc.c
+++ b/hw/display/omap_lcdc.c
@@ -403,8 +403,7 @@ struct omap_lcd_panel_s *omap_lcdc_init(MemoryRegion *sysmem,
                                         struct omap_dma_lcd_channel_s *dma,
                                         omap_clk clk)
 {
-    struct omap_lcd_panel_s *s = (struct omap_lcd_panel_s *)
-            g_malloc0(sizeof(struct omap_lcd_panel_s));
+    struct omap_lcd_panel_s *s = g_new0(struct omap_lcd_panel_s, 1);
 
     s->irq = irq;
     s->dma = dma;
diff --git a/hw/display/qxl-render.c b/hw/display/qxl-render.c
index a542087fc..7e4ef1ed0 100644
--- a/hw/display/qxl-render.c
+++ b/hw/display/qxl-render.c
@@ -159,7 +159,7 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
 /*
  * use ssd.lock to protect render_update_cookie_num.
  * qxl_render_update is called by io thread or vcpu thread, and the completion
- * callbacks are called by spice_server thread, defering to bh called from the
+ * callbacks are called by spice_server thread, deferring to bh called from the
  * io thread.
  */
 void qxl_render_update(PCIQXLDevice *qxl)
diff --git a/hw/display/qxl.c b/hw/display/qxl.c
index 2288238d0..8a3040cbb 100644
--- a/hw/display/qxl.c
+++ b/hw/display/qxl.c
@@ -1970,14 +1970,14 @@ static void qxl_realize_common(PCIQXLDevice *qxl, Error **errp)
 
     qxl->rom_size = qxl_rom_size();
     memory_region_init_ram(&qxl->rom_bar, OBJECT(qxl), "qxl.vrom",
-                           qxl->rom_size, &error_abort);
+                           qxl->rom_size, &error_fatal);
     vmstate_register_ram(&qxl->rom_bar, &qxl->pci.qdev);
     init_qxl_rom(qxl);
     init_qxl_ram(qxl);
 
     qxl->guest_surfaces.cmds = g_new0(QXLPHYSICAL, qxl->ssd.num_surfaces);
     memory_region_init_ram(&qxl->vram_bar, OBJECT(qxl), "qxl.vram",
-                           qxl->vram_size, &error_abort);
+                           qxl->vram_size, &error_fatal);
     vmstate_register_ram(&qxl->vram_bar, &qxl->pci.qdev);
     memory_region_init_alias(&qxl->vram32_bar, OBJECT(qxl), "qxl.vram32",
                              &qxl->vram_bar, 0, qxl->vram32_size);
@@ -2079,7 +2079,7 @@ static void qxl_realize_secondary(PCIDevice *dev, Error **errp)
     qxl->id = device_id++;
     qxl_init_ramsize(qxl);
     memory_region_init_ram(&qxl->vga.vram, OBJECT(dev), "qxl.vgavram",
-                           qxl->vga.vram_size, &error_abort);
+                           qxl->vga.vram_size, &error_fatal);
     vmstate_register_ram(&qxl->vga.vram, &qxl->pci.qdev);
     qxl->vga.vram_ptr = memory_region_get_ram_ptr(&qxl->vga.vram);
     qxl->vga.con = graphic_console_init(DEVICE(dev), 0, &qxl_ops, qxl);
@@ -2156,7 +2156,7 @@ static int qxl_post_load(void *opaque, int version)
         qxl_create_guest_primary(d, 1, QXL_SYNC);
 
         /* replay surface-create and cursor-set commands */
-        cmds = g_malloc0(sizeof(QXLCommandExt) * (d->ssd.num_surfaces + 1));
+        cmds = g_new0(QXLCommandExt, d->ssd.num_surfaces + 1);
         for (in = 0, out = 0; in < d->ssd.num_surfaces; in++) {
             if (d->guest_surfaces.cmds[in] == 0) {
                 continue;
diff --git a/hw/display/sm501.c b/hw/display/sm501.c
index 15a5ba800..3c3f97824 100644
--- a/hw/display/sm501.c
+++ b/hw/display/sm501.c
@@ -1411,7 +1411,7 @@ void sm501_init(MemoryRegion *address_space_mem, uint32_t base,
 
     /* allocate local memory */
     memory_region_init_ram(&s->local_mem_region, NULL, "sm501.local",
-                           local_mem_bytes, &error_abort);
+                           local_mem_bytes, &error_fatal);
     vmstate_register_ram_global(&s->local_mem_region);
     memory_region_set_log(&s->local_mem_region, true, DIRTY_MEMORY_VGA);
     s->local_mem = memory_region_get_ram_ptr(&s->local_mem_region);
diff --git a/hw/display/tc6393xb.c b/hw/display/tc6393xb.c
index f5f3f3e69..516af1a5c 100644
--- a/hw/display/tc6393xb.c
+++ b/hw/display/tc6393xb.c
@@ -584,7 +584,7 @@ TC6393xbState *tc6393xb_init(MemoryRegion *sysmem, uint32_t base, qemu_irq irq)
     memory_region_add_subregion(sysmem, base, &s->iomem);
 
     memory_region_init_ram(&s->vram, NULL, "tc6393xb.vram", 0x100000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
     memory_region_add_subregion(sysmem, base + 0x100000, &s->vram);
diff --git a/hw/display/tcx.c b/hw/display/tcx.c
index 327ce3019..d720ea666 100644
--- a/hw/display/tcx.c
+++ b/hw/display/tcx.c
@@ -944,57 +944,55 @@ static void tcx_initfn(Object *obj)
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     TCXState *s = TCX(obj);
 
-    memory_region_init_ram(&s->rom, OBJECT(s), "tcx.prom", FCODE_MAX_ROM_SIZE,
-                           &error_abort);
+    memory_region_init_ram(&s->rom, obj, "tcx.prom", FCODE_MAX_ROM_SIZE,
+                           &error_fatal);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
     /* 2/STIP : Stippler */
-    memory_region_init_io(&s->stip, OBJECT(s), &tcx_stip_ops, s, "tcx.stip",
+    memory_region_init_io(&s->stip, obj, &tcx_stip_ops, s, "tcx.stip",
                           TCX_STIP_NREGS);
     sysbus_init_mmio(sbd, &s->stip);
 
     /* 3/BLIT : Blitter */
-    memory_region_init_io(&s->blit, OBJECT(s), &tcx_blit_ops, s, "tcx.blit",
+    memory_region_init_io(&s->blit, obj, &tcx_blit_ops, s, "tcx.blit",
                           TCX_BLIT_NREGS);
     sysbus_init_mmio(sbd, &s->blit);
 
     /* 5/RSTIP : Raw Stippler */
-    memory_region_init_io(&s->rstip, OBJECT(s), &tcx_rstip_ops, s, "tcx.rstip",
+    memory_region_init_io(&s->rstip, obj, &tcx_rstip_ops, s, "tcx.rstip",
                           TCX_RSTIP_NREGS);
     sysbus_init_mmio(sbd, &s->rstip);
 
     /* 6/RBLIT : Raw Blitter */
-    memory_region_init_io(&s->rblit, OBJECT(s), &tcx_rblit_ops, s, "tcx.rblit",
+    memory_region_init_io(&s->rblit, obj, &tcx_rblit_ops, s, "tcx.rblit",
                           TCX_RBLIT_NREGS);
     sysbus_init_mmio(sbd, &s->rblit);
 
     /* 7/TEC : ??? */
-    memory_region_init_io(&s->tec, OBJECT(s), &tcx_dummy_ops, s,
-                          "tcx.tec", TCX_TEC_NREGS);
+    memory_region_init_io(&s->tec, obj, &tcx_dummy_ops, s, "tcx.tec",
+                          TCX_TEC_NREGS);
     sysbus_init_mmio(sbd, &s->tec);
 
     /* 8/CMAP : DAC */
-    memory_region_init_io(&s->dac, OBJECT(s), &tcx_dac_ops, s,
-                          "tcx.dac", TCX_DAC_NREGS);
+    memory_region_init_io(&s->dac, obj, &tcx_dac_ops, s, "tcx.dac",
+                          TCX_DAC_NREGS);
     sysbus_init_mmio(sbd, &s->dac);
 
     /* 9/THC : Cursor */
-    memory_region_init_io(&s->thc, OBJECT(s), &tcx_thc_ops, s, "tcx.thc",
+    memory_region_init_io(&s->thc, obj, &tcx_thc_ops, s, "tcx.thc",
                           TCX_THC_NREGS);
     sysbus_init_mmio(sbd, &s->thc);
 
     /* 11/DHC : ??? */
-    memory_region_init_io(&s->dhc, OBJECT(s), &tcx_dummy_ops, s, "tcx.dhc",
+    memory_region_init_io(&s->dhc, obj, &tcx_dummy_ops, s, "tcx.dhc",
                           TCX_DHC_NREGS);
     sysbus_init_mmio(sbd, &s->dhc);
 
     /* 12/ALT : ??? */
-    memory_region_init_io(&s->alt, OBJECT(s), &tcx_dummy_ops, s, "tcx.alt",
+    memory_region_init_io(&s->alt, obj, &tcx_dummy_ops, s, "tcx.alt",
                           TCX_ALT_NREGS);
     sysbus_init_mmio(sbd, &s->alt);
-
-    return;
 }
 
 static void tcx_realizefn(DeviceState *dev, Error **errp)
@@ -1007,7 +1005,7 @@ static void tcx_realizefn(DeviceState *dev, Error **errp)
     char *fcode_filename;
 
     memory_region_init_ram(&s->vram_mem, OBJECT(s), "tcx.vram",
-                           s->vram_size * (1 + 4 + 4), &error_abort);
+                           s->vram_size * (1 + 4 + 4), &error_fatal);
     vmstate_register_ram_global(&s->vram_mem);
     memory_region_set_log(&s->vram_mem, true, DIRTY_MEMORY_VGA);
     vram_base = memory_region_get_ram_ptr(&s->vram_mem);
diff --git a/hw/display/vga.c b/hw/display/vga.c
index b35d523e6..679070e64 100644
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -140,6 +140,13 @@ static uint32_t expand4[256];
 static uint16_t expand2[256];
 static uint8_t expand4to8[16];
 
+static void vbe_update_vgaregs(VGACommonState *s);
+
+static inline bool vbe_enabled(VGACommonState *s)
+{
+    return s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED;
+}
+
 static void vga_update_memory_access(VGACommonState *s)
 {
     hwaddr base, offset, size;
@@ -177,6 +184,7 @@ static void vga_update_memory_access(VGACommonState *s)
             size = 0x8000;
             break;
         }
+        assert(offset + size <= s->vram_size);
         memory_region_init_alias(&s->chain4_alias, memory_region_owner(&s->vram),
                                  "vga.chain4", &s->vram, offset, size);
         memory_region_add_subregion_overlap(s->legacy_address_space, base,
@@ -476,6 +484,7 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
         printf("vga: write SR%x = 0x%02x\n", s->sr_index, val);
 #endif
         s->sr[s->sr_index] = val & sr_mask[s->sr_index];
+        vbe_update_vgaregs(s);
         if (s->sr_index == VGA_SEQ_CLOCK_MODE) {
             s->update_retrace_info(s);
         }
@@ -507,6 +516,7 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
         printf("vga: write GR%x = 0x%02x\n", s->gr_index, val);
 #endif
         s->gr[s->gr_index] = val & gr_mask[s->gr_index];
+        vbe_update_vgaregs(s);
         vga_update_memory_access(s);
         break;
     case VGA_CRT_IM:
@@ -525,10 +535,12 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
             if (s->cr_index == VGA_CRTC_OVERFLOW) {
                 s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x10) |
                     (val & 0x10);
+                vbe_update_vgaregs(s);
             }
             return;
         }
         s->cr[s->cr_index] = val;
+        vbe_update_vgaregs(s);
 
         switch(s->cr_index) {
         case VGA_CRTC_H_TOTAL:
@@ -561,7 +573,7 @@ static void vbe_fixup_regs(VGACommonState *s)
     uint16_t *r = s->vbe_regs;
     uint32_t bits, linelength, maxy, offset;
 
-    if (!(r[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
+    if (!vbe_enabled(s)) {
         /* vbe is turned off -- nothing to do */
         return;
     }
@@ -636,6 +648,49 @@ static void vbe_fixup_regs(VGACommonState *s)
     s->vbe_start_addr  = offset / 4;
 }
 
+/* we initialize the VGA graphic mode */
+static void vbe_update_vgaregs(VGACommonState *s)
+{
+    int h, shift_control;
+
+    if (!vbe_enabled(s)) {
+        /* vbe is turned off -- nothing to do */
+        return;
+    }
+
+    /* graphic mode + memory map 1 */
+    s->gr[VGA_GFX_MISC] = (s->gr[VGA_GFX_MISC] & ~0x0c) | 0x04 |
+        VGA_GR06_GRAPHICS_MODE;
+    s->cr[VGA_CRTC_MODE] |= 3; /* no CGA modes */
+    s->cr[VGA_CRTC_OFFSET] = s->vbe_line_offset >> 3;
+    /* width */
+    s->cr[VGA_CRTC_H_DISP] =
+        (s->vbe_regs[VBE_DISPI_INDEX_XRES] >> 3) - 1;
+    /* height (only meaningful if < 1024) */
+    h = s->vbe_regs[VBE_DISPI_INDEX_YRES] - 1;
+    s->cr[VGA_CRTC_V_DISP_END] = h;
+    s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x42) |
+        ((h >> 7) & 0x02) | ((h >> 3) & 0x40);
+    /* line compare to 1023 */
+    s->cr[VGA_CRTC_LINE_COMPARE] = 0xff;
+    s->cr[VGA_CRTC_OVERFLOW] |= 0x10;
+    s->cr[VGA_CRTC_MAX_SCAN] |= 0x40;
+
+    if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
+        shift_control = 0;
+        s->sr[VGA_SEQ_CLOCK_MODE] &= ~8; /* no double line */
+    } else {
+        shift_control = 2;
+        /* set chain 4 mode */
+        s->sr[VGA_SEQ_MEMORY_MODE] |= VGA_SR04_CHN_4M;
+        /* activate all planes */
+        s->sr[VGA_SEQ_PLANE_WRITE] |= VGA_SR02_ALL_PLANES;
+    }
+    s->gr[VGA_GFX_MODE] = (s->gr[VGA_GFX_MODE] & ~0x60) |
+        (shift_control << 5);
+    s->cr[VGA_CRTC_MAX_SCAN] &= ~0x9f; /* no double scan */
+}
+
 static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
@@ -712,13 +767,10 @@ void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
         case VBE_DISPI_INDEX_Y_OFFSET:
             s->vbe_regs[s->vbe_index] = val;
             vbe_fixup_regs(s);
+            vbe_update_vgaregs(s);
             break;
         case VBE_DISPI_INDEX_BANK:
-            if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
-              val &= (s->vbe_bank_mask >> 2);
-            } else {
-              val &= s->vbe_bank_mask;
-            }
+            val &= s->vbe_bank_mask;
             s->vbe_regs[s->vbe_index] = val;
             s->bank_offset = (val << 16);
             vga_update_memory_access(s);
@@ -726,52 +778,19 @@ void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
         case VBE_DISPI_INDEX_ENABLE:
             if ((val & VBE_DISPI_ENABLED) &&
                 !(s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
-                int h, shift_control;
 
                 s->vbe_regs[VBE_DISPI_INDEX_VIRT_WIDTH] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_X_OFFSET] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_Y_OFFSET] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_ENABLE] |= VBE_DISPI_ENABLED;
                 vbe_fixup_regs(s);
+                vbe_update_vgaregs(s);
 
                 /* clear the screen */
                 if (!(val & VBE_DISPI_NOCLEARMEM)) {
                     memset(s->vram_ptr, 0,
                            s->vbe_regs[VBE_DISPI_INDEX_YRES] * s->vbe_line_offset);
                 }
-
-                /* we initialize the VGA graphic mode */
-                /* graphic mode + memory map 1 */
-                s->gr[VGA_GFX_MISC] = (s->gr[VGA_GFX_MISC] & ~0x0c) | 0x04 |
-                    VGA_GR06_GRAPHICS_MODE;
-                s->cr[VGA_CRTC_MODE] |= 3; /* no CGA modes */
-                s->cr[VGA_CRTC_OFFSET] = s->vbe_line_offset >> 3;
-                /* width */
-                s->cr[VGA_CRTC_H_DISP] =
-                    (s->vbe_regs[VBE_DISPI_INDEX_XRES] >> 3) - 1;
-                /* height (only meaningful if < 1024) */
-                h = s->vbe_regs[VBE_DISPI_INDEX_YRES] - 1;
-                s->cr[VGA_CRTC_V_DISP_END] = h;
-                s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x42) |
-                    ((h >> 7) & 0x02) | ((h >> 3) & 0x40);
-                /* line compare to 1023 */
-                s->cr[VGA_CRTC_LINE_COMPARE] = 0xff;
-                s->cr[VGA_CRTC_OVERFLOW] |= 0x10;
-                s->cr[VGA_CRTC_MAX_SCAN] |= 0x40;
-
-                if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
-                    shift_control = 0;
-                    s->sr[VGA_SEQ_CLOCK_MODE] &= ~8; /* no double line */
-                } else {
-                    shift_control = 2;
-                    /* set chain 4 mode */
-                    s->sr[VGA_SEQ_MEMORY_MODE] |= VGA_SR04_CHN_4M;
-                    /* activate all planes */
-                    s->sr[VGA_SEQ_PLANE_WRITE] |= VGA_SR02_ALL_PLANES;
-                }
-                s->gr[VGA_GFX_MODE] = (s->gr[VGA_GFX_MODE] & ~0x60) |
-                    (shift_control << 5);
-                s->cr[VGA_CRTC_MAX_SCAN] &= ~0x9f; /* no double scan */
             } else {
                 s->bank_offset = 0;
             }
@@ -817,13 +836,21 @@ uint32_t vga_mem_readb(VGACommonState *s, hwaddr addr)
 
     if (s->sr[VGA_SEQ_MEMORY_MODE] & VGA_SR04_CHN_4M) {
         /* chain 4 mode : simplest access */
+        assert(addr < s->vram_size);
         ret = s->vram_ptr[addr];
     } else if (s->gr[VGA_GFX_MODE] & 0x10) {
         /* odd/even mode (aka text mode mapping) */
         plane = (s->gr[VGA_GFX_PLANE_READ] & 2) | (addr & 1);
-        ret = s->vram_ptr[((addr & ~1) << 1) | plane];
+        addr = ((addr & ~1) << 1) | plane;
+        if (addr >= s->vram_size) {
+            return 0xff;
+        }
+        ret = s->vram_ptr[addr];
     } else {
         /* standard VGA latched access */
+        if (addr * sizeof(uint32_t) >= s->vram_size) {
+            return 0xff;
+        }
         s->latch = ((uint32_t *)s->vram_ptr)[addr];
 
         if (!(s->gr[VGA_GFX_MODE] & 0x08)) {
@@ -880,6 +907,7 @@ void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
         plane = addr & 3;
         mask = (1 << plane);
         if (s->sr[VGA_SEQ_PLANE_WRITE] & mask) {
+            assert(addr < s->vram_size);
             s->vram_ptr[addr] = val;
 #ifdef DEBUG_VGA_MEM
             printf("vga: chain4: [0x" TARGET_FMT_plx "]\n", addr);
@@ -893,6 +921,9 @@ void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
         mask = (1 << plane);
         if (s->sr[VGA_SEQ_PLANE_WRITE] & mask) {
             addr = ((addr & ~1) << 1) | plane;
+            if (addr >= s->vram_size) {
+                return;
+            }
             s->vram_ptr[addr] = val;
 #ifdef DEBUG_VGA_MEM
             printf("vga: odd/even: [0x" TARGET_FMT_plx "]\n", addr);
@@ -966,6 +997,9 @@ void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
         mask = s->sr[VGA_SEQ_PLANE_WRITE];
         s->plane_updated |= mask; /* only used to detect font change */
         write_mask = mask16[mask];
+        if (addr * sizeof(uint32_t) >= s->vram_size) {
+            return;
+        }
         ((uint32_t *)s->vram_ptr)[addr] =
             (((uint32_t *)s->vram_ptr)[addr] & ~write_mask) |
             (val & write_mask);
@@ -1044,7 +1078,7 @@ static void vga_get_offsets(VGACommonState *s,
 {
     uint32_t start_addr, line_offset, line_compare;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         line_offset = s->vbe_line_offset;
         start_addr = s->vbe_start_addr;
         line_compare = 65535;
@@ -1369,7 +1403,7 @@ static int vga_get_bpp(VGACommonState *s)
 {
     int ret;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         ret = s->vbe_regs[VBE_DISPI_INDEX_BPP];
     } else {
         ret = 0;
@@ -1381,7 +1415,7 @@ static void vga_get_resolution(VGACommonState *s, int *pwidth, int *pheight)
 {
     int width, height;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         width = s->vbe_regs[VBE_DISPI_INDEX_XRES];
         height = s->vbe_regs[VBE_DISPI_INDEX_YRES];
     } else {
@@ -2139,7 +2173,7 @@ void vga_common_init(VGACommonState *s, Object *obj, bool global_vmstate)
 
     s->is_vbe_vmstate = 1;
     memory_region_init_ram(&s->vram, obj, "vga.vram", s->vram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram(&s->vram, global_vmstate ? NULL : DEVICE(obj));
     xen_register_framebuffer(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
diff --git a/hw/display/virtio-gpu-3d.c b/hw/display/virtio-gpu-3d.c
new file mode 100644
index 000000000..28dccfdec
--- /dev/null
+++ b/hw/display/virtio-gpu-3d.c
@@ -0,0 +1,598 @@
+/*
+ * Virtio GPU Device
+ *
+ * Copyright Red Hat, Inc. 2013-2014
+ *
+ * Authors:
+ *     Dave Airlie <airlied@redhat.com>
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+#include "qemu/iov.h"
+#include "trace.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-gpu.h"
+
+#ifdef CONFIG_VIRGL
+
+#include "virglrenderer.h"
+
+static struct virgl_renderer_callbacks virtio_gpu_3d_cbs;
+
+static void virgl_cmd_create_resource_2d(VirtIOGPU *g,
+                                         struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_create_2d c2d;
+    struct virgl_renderer_resource_create_args args;
+
+    VIRTIO_GPU_FILL_CMD(c2d);
+    trace_virtio_gpu_cmd_res_create_2d(c2d.resource_id, c2d.format,
+                                       c2d.width, c2d.height);
+
+    args.handle = c2d.resource_id;
+    args.target = 2;
+    args.format = c2d.format;
+    args.bind = (1 << 1);
+    args.width = c2d.width;
+    args.height = c2d.height;
+    args.depth = 1;
+    args.array_size = 1;
+    args.last_level = 0;
+    args.nr_samples = 0;
+    args.flags = VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP;
+    virgl_renderer_resource_create(&args, NULL, 0);
+}
+
+static void virgl_cmd_create_resource_3d(VirtIOGPU *g,
+                                         struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_create_3d c3d;
+    struct virgl_renderer_resource_create_args args;
+
+    VIRTIO_GPU_FILL_CMD(c3d);
+    trace_virtio_gpu_cmd_res_create_3d(c3d.resource_id, c3d.format,
+                                       c3d.width, c3d.height, c3d.depth);
+
+    args.handle = c3d.resource_id;
+    args.target = c3d.target;
+    args.format = c3d.format;
+    args.bind = c3d.bind;
+    args.width = c3d.width;
+    args.height = c3d.height;
+    args.depth = c3d.depth;
+    args.array_size = c3d.array_size;
+    args.last_level = c3d.last_level;
+    args.nr_samples = c3d.nr_samples;
+    args.flags = c3d.flags;
+    virgl_renderer_resource_create(&args, NULL, 0);
+}
+
+static void virgl_cmd_resource_unref(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_unref unref;
+
+    VIRTIO_GPU_FILL_CMD(unref);
+    trace_virtio_gpu_cmd_res_unref(unref.resource_id);
+
+    virgl_renderer_resource_unref(unref.resource_id);
+}
+
+static void virgl_cmd_context_create(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_create cc;
+
+    VIRTIO_GPU_FILL_CMD(cc);
+    trace_virtio_gpu_cmd_ctx_create(cc.hdr.ctx_id,
+                                    cc.debug_name);
+
+    virgl_renderer_context_create(cc.hdr.ctx_id, cc.nlen,
+                                  cc.debug_name);
+}
+
+static void virgl_cmd_context_destroy(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_destroy cd;
+
+    VIRTIO_GPU_FILL_CMD(cd);
+    trace_virtio_gpu_cmd_ctx_destroy(cd.hdr.ctx_id);
+
+    virgl_renderer_context_destroy(cd.hdr.ctx_id);
+}
+
+static void virtio_gpu_rect_update(VirtIOGPU *g, int idx, int x, int y,
+                                int width, int height)
+{
+    if (!g->scanout[idx].con) {
+        return;
+    }
+
+    dpy_gl_update(g->scanout[idx].con, x, y, width, height);
+}
+
+static void virgl_cmd_resource_flush(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_flush rf;
+    int i;
+
+    VIRTIO_GPU_FILL_CMD(rf);
+    trace_virtio_gpu_cmd_res_flush(rf.resource_id,
+                                   rf.r.width, rf.r.height, rf.r.x, rf.r.y);
+
+    for (i = 0; i < VIRTIO_GPU_MAX_SCANOUT; i++) {
+        if (g->scanout[i].resource_id != rf.resource_id) {
+            continue;
+        }
+        virtio_gpu_rect_update(g, i, rf.r.x, rf.r.y, rf.r.width, rf.r.height);
+    }
+}
+
+static void virgl_cmd_set_scanout(VirtIOGPU *g,
+                                  struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_set_scanout ss;
+    struct virgl_renderer_resource_info info;
+    int ret;
+
+    VIRTIO_GPU_FILL_CMD(ss);
+    trace_virtio_gpu_cmd_set_scanout(ss.scanout_id, ss.resource_id,
+                                     ss.r.width, ss.r.height, ss.r.x, ss.r.y);
+
+    if (ss.scanout_id >= VIRTIO_GPU_MAX_SCANOUT) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal scanout id specified %d",
+                      __func__, ss.scanout_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_SCANOUT_ID;
+        return;
+    }
+    g->enable = 1;
+
+    memset(&info, 0, sizeof(info));
+
+    if (ss.resource_id && ss.r.width && ss.r.height) {
+        ret = virgl_renderer_resource_get_info(ss.resource_id, &info);
+        if (ret == -1) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "%s: illegal resource specified %d\n",
+                          __func__, ss.resource_id);
+            cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+            return;
+        }
+        qemu_console_resize(g->scanout[ss.scanout_id].con,
+                            ss.r.width, ss.r.height);
+        virgl_renderer_force_ctx_0();
+        dpy_gl_scanout(g->scanout[ss.scanout_id].con, info.tex_id,
+                       info.flags & 1 /* FIXME: Y_0_TOP */,
+                       ss.r.x, ss.r.y, ss.r.width, ss.r.height);
+    } else {
+        if (ss.scanout_id != 0) {
+            dpy_gfx_replace_surface(g->scanout[ss.scanout_id].con, NULL);
+        }
+        dpy_gl_scanout(g->scanout[ss.scanout_id].con, 0, false,
+                       0, 0, 0, 0);
+    }
+    g->scanout[ss.scanout_id].resource_id = ss.resource_id;
+}
+
+static void virgl_cmd_submit_3d(VirtIOGPU *g,
+                                struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_cmd_submit cs;
+    void *buf;
+    size_t s;
+
+    VIRTIO_GPU_FILL_CMD(cs);
+    trace_virtio_gpu_cmd_ctx_submit(cs.hdr.ctx_id, cs.size);
+
+    buf = g_malloc(cs.size);
+    s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
+                   sizeof(cs), buf, cs.size);
+    if (s != cs.size) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: size mismatch (%zd/%d)",
+                      __func__, s, cs.size);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+        return;
+    }
+
+    if (virtio_gpu_stats_enabled(g->conf)) {
+        g->stats.req_3d++;
+        g->stats.bytes_3d += cs.size;
+    }
+
+    virgl_renderer_submit_cmd(buf, cs.hdr.ctx_id, cs.size / 4);
+
+    g_free(buf);
+}
+
+static void virgl_cmd_transfer_to_host_2d(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_to_host_2d t2d;
+    struct virtio_gpu_box box;
+
+    VIRTIO_GPU_FILL_CMD(t2d);
+    trace_virtio_gpu_cmd_res_xfer_toh_2d(t2d.resource_id);
+
+    box.x = t2d.r.x;
+    box.y = t2d.r.y;
+    box.z = 0;
+    box.w = t2d.r.width;
+    box.h = t2d.r.height;
+    box.d = 1;
+
+    virgl_renderer_transfer_write_iov(t2d.resource_id,
+                                      0,
+                                      0,
+                                      0,
+                                      0,
+                                      (struct virgl_box *)&box,
+                                      t2d.offset, NULL, 0);
+}
+
+static void virgl_cmd_transfer_to_host_3d(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_host_3d t3d;
+
+    VIRTIO_GPU_FILL_CMD(t3d);
+    trace_virtio_gpu_cmd_res_xfer_toh_3d(t3d.resource_id);
+
+    virgl_renderer_transfer_write_iov(t3d.resource_id,
+                                      t3d.hdr.ctx_id,
+                                      t3d.level,
+                                      t3d.stride,
+                                      t3d.layer_stride,
+                                      (struct virgl_box *)&t3d.box,
+                                      t3d.offset, NULL, 0);
+}
+
+static void
+virgl_cmd_transfer_from_host_3d(VirtIOGPU *g,
+                                struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_host_3d tf3d;
+
+    VIRTIO_GPU_FILL_CMD(tf3d);
+    trace_virtio_gpu_cmd_res_xfer_fromh_3d(tf3d.resource_id);
+
+    virgl_renderer_transfer_read_iov(tf3d.resource_id,
+                                     tf3d.hdr.ctx_id,
+                                     tf3d.level,
+                                     tf3d.stride,
+                                     tf3d.layer_stride,
+                                     (struct virgl_box *)&tf3d.box,
+                                     tf3d.offset, NULL, 0);
+}
+
+
+static void virgl_resource_attach_backing(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_attach_backing att_rb;
+    struct iovec *res_iovs;
+    int ret;
+
+    VIRTIO_GPU_FILL_CMD(att_rb);
+    trace_virtio_gpu_cmd_res_back_attach(att_rb.resource_id);
+
+    ret = virtio_gpu_create_mapping_iov(&att_rb, cmd, &res_iovs);
+    if (ret != 0) {
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        return;
+    }
+
+    virgl_renderer_resource_attach_iov(att_rb.resource_id,
+                                       res_iovs, att_rb.nr_entries);
+}
+
+static void virgl_resource_detach_backing(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_detach_backing detach_rb;
+    struct iovec *res_iovs = NULL;
+    int num_iovs = 0;
+
+    VIRTIO_GPU_FILL_CMD(detach_rb);
+    trace_virtio_gpu_cmd_res_back_detach(detach_rb.resource_id);
+
+    virgl_renderer_resource_detach_iov(detach_rb.resource_id,
+                                       &res_iovs,
+                                       &num_iovs);
+    if (res_iovs == NULL || num_iovs == 0) {
+        return;
+    }
+    virtio_gpu_cleanup_mapping_iov(res_iovs, num_iovs);
+}
+
+
+static void virgl_cmd_ctx_attach_resource(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_resource att_res;
+
+    VIRTIO_GPU_FILL_CMD(att_res);
+    trace_virtio_gpu_cmd_ctx_res_attach(att_res.hdr.ctx_id,
+                                        att_res.resource_id);
+
+    virgl_renderer_ctx_attach_resource(att_res.hdr.ctx_id, att_res.resource_id);
+}
+
+static void virgl_cmd_ctx_detach_resource(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_resource det_res;
+
+    VIRTIO_GPU_FILL_CMD(det_res);
+    trace_virtio_gpu_cmd_ctx_res_detach(det_res.hdr.ctx_id,
+                                        det_res.resource_id);
+
+    virgl_renderer_ctx_detach_resource(det_res.hdr.ctx_id, det_res.resource_id);
+}
+
+static void virgl_cmd_get_capset_info(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_get_capset_info info;
+    struct virtio_gpu_resp_capset_info resp;
+
+    VIRTIO_GPU_FILL_CMD(info);
+
+    if (info.capset_index == 0) {
+        resp.capset_id = VIRTIO_GPU_CAPSET_VIRGL;
+        virgl_renderer_get_cap_set(resp.capset_id,
+                                   &resp.capset_max_version,
+                                   &resp.capset_max_size);
+    } else {
+        resp.capset_max_version = 0;
+        resp.capset_max_size = 0;
+    }
+    resp.hdr.type = VIRTIO_GPU_RESP_OK_CAPSET_INFO;
+    virtio_gpu_ctrl_response(g, cmd, &resp.hdr, sizeof(resp));
+}
+
+static void virgl_cmd_get_capset(VirtIOGPU *g,
+                                 struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_get_capset gc;
+    struct virtio_gpu_resp_capset *resp;
+    uint32_t max_ver, max_size;
+    VIRTIO_GPU_FILL_CMD(gc);
+
+    virgl_renderer_get_cap_set(gc.capset_id, &max_ver,
+                               &max_size);
+    resp = g_malloc(sizeof(*resp) + max_size);
+
+    resp->hdr.type = VIRTIO_GPU_RESP_OK_CAPSET;
+    virgl_renderer_fill_caps(gc.capset_id,
+                             gc.capset_version,
+                             (void *)resp->capset_data);
+    virtio_gpu_ctrl_response(g, cmd, &resp->hdr, sizeof(*resp) + max_size);
+    g_free(resp);
+}
+
+void virtio_gpu_virgl_process_cmd(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    VIRTIO_GPU_FILL_CMD(cmd->cmd_hdr);
+
+    virgl_renderer_force_ctx_0();
+    switch (cmd->cmd_hdr.type) {
+    case VIRTIO_GPU_CMD_CTX_CREATE:
+        virgl_cmd_context_create(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_DESTROY:
+        virgl_cmd_context_destroy(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_CREATE_2D:
+        virgl_cmd_create_resource_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_CREATE_3D:
+        virgl_cmd_create_resource_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_SUBMIT_3D:
+        virgl_cmd_submit_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D:
+        virgl_cmd_transfer_to_host_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D:
+        virgl_cmd_transfer_to_host_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D:
+        virgl_cmd_transfer_from_host_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING:
+        virgl_resource_attach_backing(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING:
+        virgl_resource_detach_backing(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_SET_SCANOUT:
+        virgl_cmd_set_scanout(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_FLUSH:
+        virgl_cmd_resource_flush(g, cmd);
+       break;
+    case VIRTIO_GPU_CMD_RESOURCE_UNREF:
+        virgl_cmd_resource_unref(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE:
+        /* TODO add security */
+        virgl_cmd_ctx_attach_resource(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE:
+        /* TODO add security */
+        virgl_cmd_ctx_detach_resource(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_GET_CAPSET_INFO:
+        virgl_cmd_get_capset_info(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_GET_CAPSET:
+        virgl_cmd_get_capset(g, cmd);
+        break;
+
+    case VIRTIO_GPU_CMD_GET_DISPLAY_INFO:
+        virtio_gpu_get_display_info(g, cmd);
+        break;
+    default:
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        break;
+    }
+
+    if (cmd->finished) {
+        return;
+    }
+    if (cmd->error) {
+        fprintf(stderr, "%s: ctrl 0x%x, error 0x%x\n", __func__,
+                cmd->cmd_hdr.type, cmd->error);
+        virtio_gpu_ctrl_response_nodata(g, cmd, cmd->error);
+        return;
+    }
+    if (!(cmd->cmd_hdr.flags & VIRTIO_GPU_FLAG_FENCE)) {
+        virtio_gpu_ctrl_response_nodata(g, cmd, VIRTIO_GPU_RESP_OK_NODATA);
+        return;
+    }
+
+    trace_virtio_gpu_fence_ctrl(cmd->cmd_hdr.fence_id, cmd->cmd_hdr.type);
+    virgl_renderer_create_fence(cmd->cmd_hdr.fence_id, cmd->cmd_hdr.type);
+}
+
+static void virgl_write_fence(void *opaque, uint32_t fence)
+{
+    VirtIOGPU *g = opaque;
+    struct virtio_gpu_ctrl_command *cmd, *tmp;
+
+    QTAILQ_FOREACH_SAFE(cmd, &g->fenceq, next, tmp) {
+        /*
+	 * the guest can end up emitting fences out of order
+	 * so we should check all fenced cmds not just the first one.
+	 */
+        if (cmd->cmd_hdr.fence_id > fence) {
+            continue;
+        }
+        trace_virtio_gpu_fence_resp(cmd->cmd_hdr.fence_id);
+        virtio_gpu_ctrl_response_nodata(g, cmd, VIRTIO_GPU_RESP_OK_NODATA);
+        QTAILQ_REMOVE(&g->fenceq, cmd, next);
+        g_free(cmd);
+        g->inflight--;
+        if (virtio_gpu_stats_enabled(g->conf)) {
+            fprintf(stderr, "inflight: %3d (-)\r", g->inflight);
+        }
+    }
+}
+
+static virgl_renderer_gl_context
+virgl_create_context(void *opaque, int scanout_idx,
+                     struct virgl_renderer_gl_ctx_param *params)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext ctx;
+    QEMUGLParams qparams;
+
+    qparams.major_ver = params->major_ver;
+    qparams.minor_ver = params->minor_ver;
+
+    ctx = dpy_gl_ctx_create(g->scanout[scanout_idx].con, &qparams);
+    return (virgl_renderer_gl_context)ctx;
+}
+
+static void virgl_destroy_context(void *opaque, virgl_renderer_gl_context ctx)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext qctx = (QEMUGLContext)ctx;
+
+    dpy_gl_ctx_destroy(g->scanout[0].con, qctx);
+}
+
+static int virgl_make_context_current(void *opaque, int scanout_idx,
+                                      virgl_renderer_gl_context ctx)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext qctx = (QEMUGLContext)ctx;
+
+    return dpy_gl_ctx_make_current(g->scanout[scanout_idx].con, qctx);
+}
+
+static struct virgl_renderer_callbacks virtio_gpu_3d_cbs = {
+    .version             = 1,
+    .write_fence         = virgl_write_fence,
+    .create_gl_context   = virgl_create_context,
+    .destroy_gl_context  = virgl_destroy_context,
+    .make_current        = virgl_make_context_current,
+};
+
+static void virtio_gpu_print_stats(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+
+    if (g->stats.requests) {
+        fprintf(stderr, "stats: vq req %4d, %3d -- 3D %4d (%5d)\n",
+                g->stats.requests,
+                g->stats.max_inflight,
+                g->stats.req_3d,
+                g->stats.bytes_3d);
+        g->stats.requests     = 0;
+        g->stats.max_inflight = 0;
+        g->stats.req_3d       = 0;
+        g->stats.bytes_3d     = 0;
+    } else {
+        fprintf(stderr, "stats: idle\r");
+    }
+    timer_mod(g->print_stats, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 1000);
+}
+
+static void virtio_gpu_fence_poll(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+
+    virgl_renderer_poll();
+    if (g->inflight) {
+        timer_mod(g->fence_poll, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 10);
+    }
+}
+
+void virtio_gpu_virgl_fence_poll(VirtIOGPU *g)
+{
+    virtio_gpu_fence_poll(g);
+}
+
+void virtio_gpu_virgl_reset(VirtIOGPU *g)
+{
+    int i;
+
+    /* virgl_renderer_reset() ??? */
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        if (i != 0) {
+            dpy_gfx_replace_surface(g->scanout[i].con, NULL);
+        }
+        dpy_gl_scanout(g->scanout[i].con, 0, false, 0, 0, 0, 0);
+    }
+}
+
+int virtio_gpu_virgl_init(VirtIOGPU *g)
+{
+    int ret;
+
+    ret = virgl_renderer_init(g, 0, &virtio_gpu_3d_cbs);
+    if (ret != 0) {
+        return ret;
+    }
+
+    g->fence_poll = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                 virtio_gpu_fence_poll, g);
+
+    if (virtio_gpu_stats_enabled(g->conf)) {
+        g->print_stats = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                      virtio_gpu_print_stats, g);
+        timer_mod(g->print_stats, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 1000);
+    }
+    return 0;
+}
+
+#endif /* CONFIG_VIRGL */
diff --git a/hw/display/virtio-gpu-pci.c b/hw/display/virtio-gpu-pci.c
index 5bc62cf34..eef137f81 100644
--- a/hw/display/virtio-gpu-pci.c
+++ b/hw/display/virtio-gpu-pci.c
@@ -6,8 +6,8 @@
  * Authors:
  *  Dave Airlie
  *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
  *
  */
 #include "hw/pci/pci.h"
diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
index a67d927f5..a836ce385 100644
--- a/hw/display/virtio-gpu.c
+++ b/hw/display/virtio-gpu.c
@@ -7,7 +7,7 @@
  *     Dave Airlie <airlied@redhat.com>
  *     Gerd Hoffmann <kraxel@redhat.com>
  *
- * This work is licensed under the terms of the GNU GPL, version 2.
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
@@ -22,6 +22,23 @@
 static struct virtio_gpu_simple_resource*
 virtio_gpu_find_resource(VirtIOGPU *g, uint32_t resource_id);
 
+#ifdef CONFIG_VIRGL
+#include "virglrenderer.h"
+#define VIRGL(_g, _virgl, _simple, ...)                     \
+    do {                                                    \
+        if (_g->use_virgl_renderer) {                       \
+            _virgl(__VA_ARGS__);                            \
+        } else {                                            \
+            _simple(__VA_ARGS__);                           \
+        }                                                   \
+    } while (0)
+#else
+#define VIRGL(_g, _virgl, _simple, ...)                 \
+    do {                                                \
+        _simple(__VA_ARGS__);                           \
+    } while (0)
+#endif
+
 static void update_cursor_data_simple(VirtIOGPU *g,
                                       struct virtio_gpu_scanout *s,
                                       uint32_t resource_id)
@@ -45,16 +62,49 @@ static void update_cursor_data_simple(VirtIOGPU *g,
            pixels * sizeof(uint32_t));
 }
 
+#ifdef CONFIG_VIRGL
+
+static void update_cursor_data_virgl(VirtIOGPU *g,
+                                     struct virtio_gpu_scanout *s,
+                                     uint32_t resource_id)
+{
+    uint32_t width, height;
+    uint32_t pixels, *data;
+
+    data = virgl_renderer_get_cursor_data(resource_id, &width, &height);
+    if (!data) {
+        return;
+    }
+
+    if (width != s->current_cursor->width ||
+        height != s->current_cursor->height) {
+        return;
+    }
+
+    pixels = s->current_cursor->width * s->current_cursor->height;
+    memcpy(s->current_cursor->data, data, pixels * sizeof(uint32_t));
+    free(data);
+}
+
+#endif
+
 static void update_cursor(VirtIOGPU *g, struct virtio_gpu_update_cursor *cursor)
 {
     struct virtio_gpu_scanout *s;
+    bool move = cursor->hdr.type != VIRTIO_GPU_CMD_MOVE_CURSOR;
 
     if (cursor->pos.scanout_id >= g->conf.max_outputs) {
         return;
     }
     s = &g->scanout[cursor->pos.scanout_id];
 
-    if (cursor->hdr.type != VIRTIO_GPU_CMD_MOVE_CURSOR) {
+    trace_virtio_gpu_update_cursor(cursor->pos.scanout_id,
+                                   cursor->pos.x,
+                                   cursor->pos.y,
+                                   move ? "move" : "update",
+                                   cursor->resource_id);
+
+    if (move) {
         if (!s->current_cursor) {
             s->current_cursor = cursor_alloc(64, 64);
         }
@@ -63,7 +113,8 @@ static void update_cursor(VirtIOGPU *g, struct virtio_gpu_update_cursor *cursor)
         s->current_cursor->hot_y = cursor->hot_y;
 
         if (cursor->resource_id > 0) {
-            update_cursor_data_simple(g, s, cursor->resource_id);
+            VIRGL(g, update_cursor_data_virgl, update_cursor_data_simple,
+                  g, s, cursor->resource_id);
         }
         dpy_cursor_define(s->con, s->current_cursor);
     }
@@ -92,9 +143,23 @@ static void virtio_gpu_set_config(VirtIODevice *vdev, const uint8_t *config)
 static uint64_t virtio_gpu_get_features(VirtIODevice *vdev, uint64_t features,
                                         Error **errp)
 {
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+
+    if (virtio_gpu_virgl_enabled(g->conf)) {
+        features |= (1 << VIRTIO_GPU_FEATURE_VIRGL);
+    }
     return features;
 }
 
+static void virtio_gpu_set_features(VirtIODevice *vdev, uint64_t features)
+{
+    static const uint32_t virgl = (1 << VIRTIO_GPU_FEATURE_VIRGL);
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+
+    g->use_virgl_renderer = ((features & virgl) == virgl);
+    trace_virtio_gpu_features(g->use_virgl_renderer);
+}
+
 static void virtio_gpu_notify_event(VirtIOGPU *g, uint32_t event_type)
 {
     g->virtio_config.events_read |= event_type;
@@ -563,7 +628,6 @@ int virtio_gpu_create_mapping_iov(struct virtio_gpu_resource_attach_backing *ab,
                           __func__, ab->resource_id, i);
             virtio_gpu_cleanup_mapping_iov(*iov, i);
             g_free(ents);
-            g_free(*iov);
             *iov = NULL;
             return -1;
         }
@@ -580,12 +644,12 @@ void virtio_gpu_cleanup_mapping_iov(struct iovec *iov, uint32_t count)
         cpu_physical_memory_unmap(iov[i].iov_base, iov[i].iov_len, 1,
                                   iov[i].iov_len);
     }
+    g_free(iov);
 }
 
 static void virtio_gpu_cleanup_mapping(struct virtio_gpu_simple_resource *res)
 {
     virtio_gpu_cleanup_mapping_iov(res->iov, res->iov_cnt);
-    g_free(res->iov);
     res->iov = NULL;
     res->iov_cnt = 0;
 }
@@ -699,25 +763,43 @@ static void virtio_gpu_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
         return;
     }
 
+#ifdef CONFIG_VIRGL
+    if (!g->renderer_inited && g->use_virgl_renderer) {
+        virtio_gpu_virgl_init(g);
+        g->renderer_inited = true;
+    }
+#endif
+
     cmd = g_new(struct virtio_gpu_ctrl_command, 1);
     while (virtqueue_pop(vq, &cmd->elem)) {
         cmd->vq = vq;
         cmd->error = 0;
         cmd->finished = false;
-        g->stats.requests++;
+        if (virtio_gpu_stats_enabled(g->conf)) {
+            g->stats.requests++;
+        }
 
-        virtio_gpu_simple_process_cmd(g, cmd);
+        VIRGL(g, virtio_gpu_virgl_process_cmd, virtio_gpu_simple_process_cmd,
+              g, cmd);
         if (!cmd->finished) {
             QTAILQ_INSERT_TAIL(&g->fenceq, cmd, next);
-            g->stats.inflight++;
-            if (g->stats.max_inflight < g->stats.inflight) {
-                g->stats.max_inflight = g->stats.inflight;
+            g->inflight++;
+            if (virtio_gpu_stats_enabled(g->conf)) {
+                if (g->stats.max_inflight < g->inflight) {
+                    g->stats.max_inflight = g->inflight;
+                }
+                fprintf(stderr, "inflight: %3d (+)\r", g->inflight);
             }
-            fprintf(stderr, "inflight: %3d (+)\r", g->stats.inflight);
             cmd = g_new(struct virtio_gpu_ctrl_command, 1);
         }
     }
     g_free(cmd);
+
+#ifdef CONFIG_VIRGL
+    if (g->use_virgl_renderer) {
+        virtio_gpu_virgl_fence_poll(g);
+    }
+#endif
 }
 
 static void virtio_gpu_ctrl_bh(void *opaque)
@@ -804,6 +886,7 @@ static void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(qdev);
     VirtIOGPU *g = VIRTIO_GPU(qdev);
+    bool have_virgl;
     int i;
 
     g->config_size = sizeof(struct virtio_gpu_config);
@@ -814,8 +897,25 @@ static void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
     g->req_state[0].width = 1024;
     g->req_state[0].height = 768;
 
-    g->ctrl_vq   = virtio_add_queue(vdev, 64, virtio_gpu_handle_ctrl_cb);
-    g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+    g->use_virgl_renderer = false;
+#if !defined(CONFIG_VIRGL) || defined(HOST_WORDS_BIGENDIAN)
+    have_virgl = false;
+#else
+    have_virgl = display_opengl;
+#endif
+    if (!have_virgl) {
+        g->conf.flags &= ~(1 << VIRTIO_GPU_FLAG_VIRGL_ENABLED);
+    }
+
+    if (virtio_gpu_virgl_enabled(g->conf)) {
+        /* use larger control queue in 3d mode */
+        g->ctrl_vq   = virtio_add_queue(vdev, 256, virtio_gpu_handle_ctrl_cb);
+        g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+        g->virtio_config.num_capsets = 1;
+    } else {
+        g->ctrl_vq   = virtio_add_queue(vdev, 64, virtio_gpu_handle_ctrl_cb);
+        g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+    }
 
     g->ctrl_bh = qemu_bh_new(virtio_gpu_ctrl_bh, g);
     g->cursor_bh = qemu_bh_new(virtio_gpu_cursor_bh, g);
@@ -869,10 +969,23 @@ static void virtio_gpu_reset(VirtIODevice *vdev)
         g->scanout[i].ds = NULL;
     }
     g->enabled_output_bitmask = 1;
+
+#ifdef CONFIG_VIRGL
+    if (g->use_virgl_renderer) {
+        virtio_gpu_virgl_reset(g);
+        g->use_virgl_renderer = 0;
+    }
+#endif
 }
 
 static Property virtio_gpu_properties[] = {
     DEFINE_PROP_UINT32("max_outputs", VirtIOGPU, conf.max_outputs, 1),
+#ifdef CONFIG_VIRGL
+    DEFINE_PROP_BIT("virgl", VirtIOGPU, conf.flags,
+                    VIRTIO_GPU_FLAG_VIRGL_ENABLED, true),
+    DEFINE_PROP_BIT("stats", VirtIOGPU, conf.flags,
+                    VIRTIO_GPU_FLAG_STATS_ENABLED, false),
+#endif
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -885,6 +998,7 @@ static void virtio_gpu_class_init(ObjectClass *klass, void *data)
     vdc->get_config = virtio_gpu_get_config;
     vdc->set_config = virtio_gpu_set_config;
     vdc->get_features = virtio_gpu_get_features;
+    vdc->set_features = virtio_gpu_set_features;
 
     vdc->reset = virtio_gpu_reset;
 
@@ -917,3 +1031,14 @@ QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_mem_entry)               != 16);
 QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_attach_backing) != 32);
 QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_detach_backing) != 32);
 QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_display_info)       != 408);
+
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_transfer_host_3d)        != 72);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_create_3d)      != 72);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_create)              != 96);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_destroy)             != 24);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_resource)            != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_cmd_submit)              != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_get_capset_info)         != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_capset_info)        != 40);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_get_capset)              != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_capset)             != 24);
diff --git a/hw/display/vmware_vga.c b/hw/display/vmware_vga.c
index 7f397d3c2..935403785 100644
--- a/hw/display/vmware_vga.c
+++ b/hw/display/vmware_vga.c
@@ -488,10 +488,10 @@ static inline int vmsvga_fill_rect(struct vmsvga_state_s *s,
 #endif
 
 struct vmsvga_cursor_definition_s {
-    int width;
-    int height;
+    uint32_t width;
+    uint32_t height;
     int id;
-    int bpp;
+    uint32_t bpp;
     int hot_x;
     int hot_y;
     uint32_t mask[1024];
@@ -658,7 +658,10 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             cursor.bpp = vmsvga_fifo_read(s);
 
             args = SVGA_BITMAP_SIZE(x, y) + SVGA_PIXMAP_SIZE(x, y, cursor.bpp);
-            if (SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
+            if (cursor.width > 256 ||
+                cursor.height > 256 ||
+                cursor.bpp > 32 ||
+                SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
                 SVGA_PIXMAP_SIZE(x, y, cursor.bpp) > sizeof cursor.image) {
                     goto badcmd;
             }
@@ -1244,7 +1247,7 @@ static void vmsvga_init(DeviceState *dev, struct vmsvga_state_s *s,
 
     s->fifo_size = SVGA_FIFO_SIZE;
     memory_region_init_ram(&s->fifo_ram, NULL, "vmsvga.fifo", s->fifo_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->fifo_ram);
     s->fifo_ptr = memory_region_get_ram_ptr(&s->fifo_ram);
 
diff --git a/hw/display/xenfb.c b/hw/display/xenfb.c
index 5e324ef62..4e2a27a3d 100644
--- a/hw/display/xenfb.c
+++ b/hw/display/xenfb.c
@@ -784,18 +784,20 @@ static void xenfb_invalidate(void *opaque)
 
 static void xenfb_handle_events(struct XenFB *xenfb)
 {
-    uint32_t prod, cons;
+    uint32_t prod, cons, out_cons;
     struct xenfb_page *page = xenfb->c.page;
 
     prod = page->out_prod;
-    if (prod == page->out_cons)
+    out_cons = page->out_cons;
+    if (prod == out_cons)
 	return;
     xen_rmb();		/* ensure we see ring contents up to prod */
-    for (cons = page->out_cons; cons != prod; cons++) {
+    for (cons = out_cons; cons != prod; cons++) {
 	union xenfb_out_event *event = &XENFB_OUT_RING_REF(page, cons);
+        uint8_t type = event->type;
 	int x, y, w, h;
 
-	switch (event->type) {
+	switch (type) {
 	case XENFB_TYPE_UPDATE:
 	    if (xenfb->up_count == UP_QUEUE)
 		xenfb->up_fullscreen = 1;
diff --git a/hw/dma/i82374.c b/hw/dma/i82374.c
index b8ad2e64e..f63097154 100644
--- a/hw/dma/i82374.c
+++ b/hw/dma/i82374.c
@@ -38,7 +38,6 @@ do { fprintf(stderr, "i82374 ERROR: " fmt , ## __VA_ARGS__); } while (0)
 
 typedef struct I82374State {
     uint8_t commands[8];
-    qemu_irq out;
     PortioList port_list;
 } I82374State;
 
@@ -101,7 +100,7 @@ static uint32_t i82374_read_descriptor(void *opaque, uint32_t nport)
 
 static void i82374_realize(I82374State *s, Error **errp)
 {
-    DMA_init(1, &s->out);
+    DMA_init(1);
     memset(s->commands, 0, sizeof(s->commands));
 }
 
@@ -145,8 +144,6 @@ static void i82374_isa_realize(DeviceState *dev, Error **errp)
                     isa->iobase);
 
     i82374_realize(s, errp);
-
-    qdev_init_gpio_out(dev, &s->out, 1);
 }
 
 static Property i82374_properties[] = {
diff --git a/hw/dma/i8257.c b/hw/dma/i8257.c
index a414029be..13984244a 100644
--- a/hw/dma/i8257.c
+++ b/hw/dma/i8257.c
@@ -59,7 +59,6 @@ static struct dma_cont {
     uint8_t flip_flop;
     int dshift;
     struct dma_regs regs[4];
-    qemu_irq *cpu_request_exit;
     MemoryRegion channel_io;
     MemoryRegion cont_io;
 } dma_controllers[2];
@@ -358,6 +357,7 @@ static void channel_run (int ncont, int ichan)
 }
 
 static QEMUBH *dma_bh;
+static bool dma_bh_scheduled;
 
 static void DMA_run (void)
 {
@@ -390,12 +390,15 @@ static void DMA_run (void)
 
     running = 0;
 out:
-    if (rearm)
+    if (rearm) {
         qemu_bh_schedule_idle(dma_bh);
+        dma_bh_scheduled = true;
+    }
 }
 
 static void DMA_run_bh(void *unused)
 {
+    dma_bh_scheduled = false;
     DMA_run();
 }
 
@@ -458,12 +461,14 @@ int DMA_write_memory (int nchan, void *buf, int pos, int len)
     return len;
 }
 
-/* request the emulator to transfer a new DMA memory block ASAP */
-void DMA_schedule(int nchan)
+/* request the emulator to transfer a new DMA memory block ASAP (even
+ * if the idle bottom half would not have exited the iothread yet).
+ */
+void DMA_schedule(void)
 {
-    struct dma_cont *d = &dma_controllers[nchan > 3];
-
-    qemu_irq_pulse(*d->cpu_request_exit);
+    if (dma_bh_scheduled) {
+        qemu_notify_event();
+    }
 }
 
 static void dma_reset(void *opaque)
@@ -515,13 +520,11 @@ static const MemoryRegionOps cont_io_ops = {
 
 /* dshift = 0: 8 bit DMA, 1 = 16 bit DMA */
 static void dma_init2(struct dma_cont *d, int base, int dshift,
-                      int page_base, int pageh_base,
-                      qemu_irq *cpu_request_exit)
+                      int page_base, int pageh_base)
 {
     int i;
 
     d->dshift = dshift;
-    d->cpu_request_exit = cpu_request_exit;
 
     memory_region_init_io(&d->channel_io, NULL, &channel_io_ops, d,
                           "dma-chan", 8 << d->dshift);
@@ -585,12 +588,10 @@ static const VMStateDescription vmstate_dma = {
     }
 };
 
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
+void DMA_init(int high_page_enable)
 {
-    dma_init2(&dma_controllers[0], 0x00, 0, 0x80,
-              high_page_enable ? 0x480 : -1, cpu_request_exit);
-    dma_init2(&dma_controllers[1], 0xc0, 1, 0x88,
-              high_page_enable ? 0x488 : -1, cpu_request_exit);
+    dma_init2(&dma_controllers[0], 0x00, 0, 0x80, high_page_enable ? 0x480 : -1);
+    dma_init2(&dma_controllers[1], 0xc0, 1, 0x88, high_page_enable ? 0x488 : -1);
     vmstate_register (NULL, 0, &vmstate_dma, &dma_controllers[0]);
     vmstate_register (NULL, 1, &vmstate_dma, &dma_controllers[1]);
 
diff --git a/hw/dma/omap_dma.c b/hw/dma/omap_dma.c
index 97c57a03c..db6873099 100644
--- a/hw/dma/omap_dma.c
+++ b/hw/dma/omap_dma.c
@@ -1626,8 +1626,7 @@ struct soc_dma_s *omap_dma_init(hwaddr base, qemu_irq *irqs,
                 enum omap_dma_model model)
 {
     int num_irqs, memsize, i;
-    struct omap_dma_s *s = (struct omap_dma_s *)
-            g_malloc0(sizeof(struct omap_dma_s));
+    struct omap_dma_s *s = g_new0(struct omap_dma_s, 1);
 
     if (model <= omap_dma_3_1) {
         num_irqs = 6;
@@ -2061,8 +2060,7 @@ struct soc_dma_s *omap_dma4_init(hwaddr base, qemu_irq *irqs,
                 int chans, omap_clk iclk, omap_clk fclk)
 {
     int i;
-    struct omap_dma_s *s = (struct omap_dma_s *)
-            g_malloc0(sizeof(struct omap_dma_s));
+    struct omap_dma_s *s = g_new0(struct omap_dma_s, 1);
 
     s->model = omap_dma_4;
     s->chans = chans;
diff --git a/hw/dma/pxa2xx_dma.c b/hw/dma/pxa2xx_dma.c
index d4501fb4c..54cdb25a3 100644
--- a/hw/dma/pxa2xx_dma.c
+++ b/hw/dma/pxa2xx_dma.c
@@ -459,9 +459,8 @@ static int pxa2xx_dma_init(SysBusDevice *sbd)
         return -1;
     }
 
-    s->chan = g_malloc0(sizeof(PXA2xxDMAChannel) * s->channels);
+    s->chan = g_new0(PXA2xxDMAChannel, s->channels);
 
-    memset(s->chan, 0, sizeof(PXA2xxDMAChannel) * s->channels);
     for (i = 0; i < s->channels; i ++)
         s->chan[i].state = DCSR_STOPINTR;
 
diff --git a/hw/dma/xilinx_axidma.c b/hw/dma/xilinx_axidma.c
index cf842a3cc..b1cfa1135 100644
--- a/hw/dma/xilinx_axidma.c
+++ b/hw/dma/xilinx_axidma.c
@@ -133,7 +133,7 @@ struct XilinxAXIDMA {
 };
 
 /*
- * Helper calls to extract info from desriptors and other trivial
+ * Helper calls to extract info from descriptors and other trivial
  * state from regs.
  */
 static inline int stream_desc_sof(struct SDesc *d)
diff --git a/hw/gpio/Makefile.objs b/hw/gpio/Makefile.objs
index 1abcf1798..52233f7e2 100644
--- a/hw/gpio/Makefile.objs
+++ b/hw/gpio/Makefile.objs
@@ -5,3 +5,4 @@ common-obj-$(CONFIG_ZAURUS) += zaurus.o
 common-obj-$(CONFIG_E500) += mpc8xxx.o
 
 obj-$(CONFIG_OMAP) += omap_gpio.o
+obj-$(CONFIG_IMX) += imx_gpio.o
diff --git a/hw/gpio/imx_gpio.c b/hw/gpio/imx_gpio.c
new file mode 100644
index 000000000..3170585a2
--- /dev/null
+++ b/hw/gpio/imx_gpio.c
@@ -0,0 +1,341 @@
+/*
+ * i.MX processors GPIO emulation.
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/gpio/imx_gpio.h"
+
+#ifndef DEBUG_IMX_GPIO
+#define DEBUG_IMX_GPIO 0
+#endif
+
+typedef enum IMXGPIOLevel {
+    IMX_GPIO_LEVEL_LOW = 0,
+    IMX_GPIO_LEVEL_HIGH = 1,
+} IMXGPIOLevel;
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_GPIO) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_GPIO, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static const char *imx_gpio_reg_name(uint32_t reg)
+{
+    switch (reg) {
+    case DR_ADDR:
+        return "DR";
+    case GDIR_ADDR:
+        return "GDIR";
+    case PSR_ADDR:
+        return "PSR";
+    case ICR1_ADDR:
+        return "ICR1";
+    case ICR2_ADDR:
+        return "ICR2";
+    case IMR_ADDR:
+        return "IMR";
+    case ISR_ADDR:
+        return "ISR";
+    case EDGE_SEL_ADDR:
+        return "EDGE_SEL";
+    default:
+        return "[?]";
+    }
+}
+
+static void imx_gpio_update_int(IMXGPIOState *s)
+{
+    qemu_set_irq(s->irq, (s->isr & s->imr) ? 1 : 0);
+}
+
+static void imx_gpio_set_int_line(IMXGPIOState *s, int line, IMXGPIOLevel level)
+{
+    /* if this signal isn't configured as an input signal, nothing to do */
+    if (!extract32(s->gdir, line, 1)) {
+        return;
+    }
+
+    /* When set, EDGE_SEL overrides the ICR config */
+    if (extract32(s->edge_sel, line, 1)) {
+        /* we detect interrupt on rising and falling edge */
+        if (extract32(s->psr, line, 1) != level) {
+            /* level changed */
+            s->isr = deposit32(s->isr, line, 1, 1);
+        }
+    } else if (extract64(s->icr, 2*line + 1, 1)) {
+        /* interrupt is edge sensitive */
+        if (extract32(s->psr, line, 1) != level) {
+            /* level changed */
+            if (extract64(s->icr, 2*line, 1) != level) {
+                s->isr = deposit32(s->isr, line, 1, 1);
+            }
+        }
+    } else {
+        /* interrupt is level sensitive */
+        if (extract64(s->icr, 2*line, 1) == level) {
+            s->isr = deposit32(s->isr, line, 1, 1);
+        }
+    }
+}
+
+static void imx_gpio_set(void *opaque, int line, int level)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+    IMXGPIOLevel imx_level = level ? IMX_GPIO_LEVEL_HIGH : IMX_GPIO_LEVEL_LOW;
+
+    imx_gpio_set_int_line(s, line, imx_level);
+
+    /* this is an input signal, so set PSR */
+    s->psr = deposit32(s->psr, line, 1, imx_level);
+
+    imx_gpio_update_int(s);
+}
+
+static void imx_gpio_set_all_int_lines(IMXGPIOState *s)
+{
+    int i;
+
+    for (i = 0; i < IMX_GPIO_PIN_COUNT; i++) {
+        IMXGPIOLevel imx_level = extract32(s->psr, i, 1);
+        imx_gpio_set_int_line(s, i, imx_level);
+    }
+
+    imx_gpio_update_int(s);
+}
+
+static inline void imx_gpio_set_all_output_lines(IMXGPIOState *s)
+{
+    int i;
+
+    for (i = 0; i < IMX_GPIO_PIN_COUNT; i++) {
+        /*
+         * if the line is set as output, then forward the line
+         * level to its user.
+         */
+        if (extract32(s->gdir, i, 1) && s->output[i]) {
+            qemu_set_irq(s->output[i], extract32(s->dr, i, 1));
+        }
+    }
+}
+
+static uint64_t imx_gpio_read(void *opaque, hwaddr offset, unsigned size)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+    uint32_t reg_value = 0;
+
+    switch (offset) {
+    case DR_ADDR:
+        /*
+         * depending on the "line" configuration, the bit values
+         * are coming either from DR or PSR
+         */
+        reg_value = (s->dr & s->gdir) | (s->psr & ~s->gdir);
+        break;
+
+    case GDIR_ADDR:
+        reg_value = s->gdir;
+        break;
+
+    case PSR_ADDR:
+        reg_value = s->psr & ~s->gdir;
+        break;
+
+    case ICR1_ADDR:
+        reg_value = extract64(s->icr, 0, 32);
+        break;
+
+    case ICR2_ADDR:
+        reg_value = extract64(s->icr, 32, 32);
+        break;
+
+    case IMR_ADDR:
+        reg_value = s->imr;
+        break;
+
+    case ISR_ADDR:
+        reg_value = s->isr;
+        break;
+
+    case EDGE_SEL_ADDR:
+        if (s->has_edge_sel) {
+            reg_value = s->edge_sel;
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: EDGE_SEL register not "
+                          "present on this version of GPIO device\n",
+                          TYPE_IMX_GPIO, __func__);
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPIO, __func__, offset);
+        break;
+    }
+
+    DPRINTF("(%s) = 0x%" PRIx32 "\n", imx_gpio_reg_name(offset), reg_value);
+
+    return reg_value;
+}
+
+static void imx_gpio_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+
+    DPRINTF("(%s, value = 0x%" PRIx32 ")\n", imx_gpio_reg_name(offset),
+            (uint32_t)value);
+
+    switch (offset) {
+    case DR_ADDR:
+        s->dr = value;
+        imx_gpio_set_all_output_lines(s);
+        break;
+
+    case GDIR_ADDR:
+        s->gdir = value;
+        imx_gpio_set_all_output_lines(s);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case ICR1_ADDR:
+        s->icr = deposit64(s->icr, 0, 32, value);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case ICR2_ADDR:
+        s->icr = deposit64(s->icr, 32, 32, value);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case IMR_ADDR:
+        s->imr = value;
+        imx_gpio_update_int(s);
+        break;
+
+    case ISR_ADDR:
+        s->isr |= ~value;
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case EDGE_SEL_ADDR:
+        if (s->has_edge_sel) {
+            s->edge_sel = value;
+            imx_gpio_set_all_int_lines(s);
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: EDGE_SEL register not "
+                          "present on this version of GPIO device\n",
+                          TYPE_IMX_GPIO, __func__);
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPIO, __func__, offset);
+        break;
+    }
+
+    return;
+}
+
+static const MemoryRegionOps imx_gpio_ops = {
+    .read = imx_gpio_read,
+    .write = imx_gpio_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static const VMStateDescription vmstate_imx_gpio = {
+    .name = TYPE_IMX_GPIO,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(dr, IMXGPIOState),
+        VMSTATE_UINT32(gdir, IMXGPIOState),
+        VMSTATE_UINT32(psr, IMXGPIOState),
+        VMSTATE_UINT64(icr, IMXGPIOState),
+        VMSTATE_UINT32(imr, IMXGPIOState),
+        VMSTATE_UINT32(isr, IMXGPIOState),
+        VMSTATE_BOOL(has_edge_sel, IMXGPIOState),
+        VMSTATE_UINT32(edge_sel, IMXGPIOState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property imx_gpio_properties[] = {
+    DEFINE_PROP_BOOL("has-edge-sel", IMXGPIOState, has_edge_sel, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void imx_gpio_reset(DeviceState *dev)
+{
+    IMXGPIOState *s = IMX_GPIO(dev);
+
+    s->dr       = 0;
+    s->gdir     = 0;
+    s->psr      = 0;
+    s->icr      = 0;
+    s->imr      = 0;
+    s->isr      = 0;
+    s->edge_sel = 0;
+
+    imx_gpio_set_all_output_lines(s);
+    imx_gpio_update_int(s);
+}
+
+static void imx_gpio_realize(DeviceState *dev, Error **errp)
+{
+    IMXGPIOState *s = IMX_GPIO(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &imx_gpio_ops, s,
+                          TYPE_IMX_GPIO, IMX_GPIO_MEM_SIZE);
+
+    qdev_init_gpio_in(DEVICE(s), imx_gpio_set, IMX_GPIO_PIN_COUNT);
+    qdev_init_gpio_out(DEVICE(s), s->output, IMX_GPIO_PIN_COUNT);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+}
+
+static void imx_gpio_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = imx_gpio_realize;
+    dc->reset = imx_gpio_reset;
+    dc->props = imx_gpio_properties;
+    dc->vmsd = &vmstate_imx_gpio;
+    dc->desc = "i.MX GPIO controller";
+}
+
+static const TypeInfo imx_gpio_info = {
+    .name = TYPE_IMX_GPIO,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXGPIOState),
+    .class_init = imx_gpio_class_init,
+};
+
+static void imx_gpio_register_types(void)
+{
+    type_register_static(&imx_gpio_info);
+}
+
+type_init(imx_gpio_register_types)
diff --git a/hw/gpio/omap_gpio.c b/hw/gpio/omap_gpio.c
index d92f8cfba..3c538985e 100644
--- a/hw/gpio/omap_gpio.c
+++ b/hw/gpio/omap_gpio.c
@@ -710,8 +710,8 @@ static int omap2_gpio_init(SysBusDevice *sbd)
     } else {
         s->modulecount = 6;
     }
-    s->modules = g_malloc0(s->modulecount * sizeof(struct omap2_gpio_s));
-    s->handler = g_malloc0(s->modulecount * 32 * sizeof(qemu_irq));
+    s->modules = g_new0(struct omap2_gpio_s, s->modulecount);
+    s->handler = g_new0(qemu_irq, s->modulecount * 32);
     qdev_init_gpio_in(dev, omap2_gpio_set, s->modulecount * 32);
     qdev_init_gpio_out(dev, s->handler, s->modulecount * 32);
     for (i = 0; i < s->modulecount; i++) {
diff --git a/hw/gpio/zaurus.c b/hw/gpio/zaurus.c
index 24a77272d..aa8ee5fa0 100644
--- a/hw/gpio/zaurus.c
+++ b/hw/gpio/zaurus.c
@@ -235,10 +235,6 @@ static const VMStateDescription vmstate_scoop_regs = {
     },
 };
 
-static Property scoop_sysbus_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void scoop_sysbus_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -247,7 +243,6 @@ static void scoop_sysbus_class_init(ObjectClass *klass, void *data)
     k->init = scoop_init;
     dc->desc = "Scoop2 Sharp custom ASIC";
     dc->vmsd = &vmstate_scoop_regs;
-    dc->props = scoop_sysbus_properties;
 }
 
 static const TypeInfo scoop_sysbus_info = {
diff --git a/hw/i2c/Makefile.objs b/hw/i2c/Makefile.objs
index 0f130608c..aeb8f38d7 100644
--- a/hw/i2c/Makefile.objs
+++ b/hw/i2c/Makefile.objs
@@ -4,4 +4,5 @@ common-obj-$(CONFIG_ACPI_X86) += smbus_ich9.o
 common-obj-$(CONFIG_APM) += pm_smbus.o
 common-obj-$(CONFIG_BITBANG_I2C) += bitbang_i2c.o
 common-obj-$(CONFIG_EXYNOS4) += exynos4210_i2c.o
+common-obj-$(CONFIG_IMX_I2C) += imx_i2c.o
 obj-$(CONFIG_OMAP) += omap_i2c.o
diff --git a/hw/i2c/imx_i2c.c b/hw/i2c/imx_i2c.c
new file mode 100644
index 000000000..cb62c7a2c
--- /dev/null
+++ b/hw/i2c/imx_i2c.c
@@ -0,0 +1,335 @@
+/*
+ *  i.MX I2C Bus Serial Interface Emulation
+ *
+ *  Copyright (C) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "hw/i2c/imx_i2c.h"
+#include "hw/i2c/i2c.h"
+
+#ifndef DEBUG_IMX_I2C
+#define DEBUG_IMX_I2C 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_I2C) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_I2C, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static const char *imx_i2c_get_regname(unsigned offset)
+{
+    switch (offset) {
+    case IADR_ADDR:
+        return "IADR";
+    case IFDR_ADDR:
+        return "IFDR";
+    case I2CR_ADDR:
+        return "I2CR";
+    case I2SR_ADDR:
+        return "I2SR";
+    case I2DR_ADDR:
+        return "I2DR";
+    default:
+        return "[?]";
+    }
+}
+
+static inline bool imx_i2c_is_enabled(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_IEN;
+}
+
+static inline bool imx_i2c_interrupt_is_enabled(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_IIEN;
+}
+
+static inline bool imx_i2c_is_master(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_MSTA;
+}
+
+static void imx_i2c_reset(DeviceState *dev)
+{
+    IMXI2CState *s = IMX_I2C(dev);
+
+    if (s->address != ADDR_RESET) {
+        i2c_end_transfer(s->bus);
+    }
+
+    s->address    = ADDR_RESET;
+    s->iadr       = IADR_RESET;
+    s->ifdr       = IFDR_RESET;
+    s->i2cr       = I2CR_RESET;
+    s->i2sr       = I2SR_RESET;
+    s->i2dr_read  = I2DR_RESET;
+    s->i2dr_write = I2DR_RESET;
+}
+
+static inline void imx_i2c_raise_interrupt(IMXI2CState *s)
+{
+    /*
+     * raise an interrupt if the device is enabled and it is configured
+     * to generate some interrupts.
+     */
+    if (imx_i2c_is_enabled(s) && imx_i2c_interrupt_is_enabled(s)) {
+        s->i2sr |= I2SR_IIF;
+        qemu_irq_raise(s->irq);
+    }
+}
+
+static uint64_t imx_i2c_read(void *opaque, hwaddr offset,
+                             unsigned size)
+{
+    uint16_t value;
+    IMXI2CState *s = IMX_I2C(opaque);
+
+    switch (offset) {
+    case IADR_ADDR:
+        value = s->iadr;
+        break;
+    case IFDR_ADDR:
+        value = s->ifdr;
+        break;
+    case I2CR_ADDR:
+        value = s->i2cr;
+        break;
+    case I2SR_ADDR:
+        value = s->i2sr;
+        break;
+    case I2DR_ADDR:
+        value = s->i2dr_read;
+
+        if (imx_i2c_is_master(s)) {
+            int ret = 0xff;
+
+            if (s->address == ADDR_RESET) {
+                /* something is wrong as the address is not set */
+                qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read "
+                              "without specifying the slave address\n",
+                              TYPE_IMX_I2C, __func__);
+            } else if (s->i2cr & I2CR_MTX) {
+                qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read "
+                              "but MTX is set\n", TYPE_IMX_I2C, __func__);
+            } else {
+                /* get the next byte */
+                ret = i2c_recv(s->bus);
+
+                if (ret >= 0) {
+                    imx_i2c_raise_interrupt(s);
+                } else {
+                    qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: read failed "
+                                  "for device 0x%02x\n", TYPE_IMX_I2C,
+                                  __func__, s->address);
+                    ret = 0xff;
+                }
+            }
+
+            s->i2dr_read = ret;
+        } else {
+            qemu_log_mask(LOG_UNIMP, "[%s]%s: slave mode not implemented\n",
+                          TYPE_IMX_I2C, __func__);
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_I2C, __func__, offset);
+        value = 0;
+        break;
+    }
+
+    DPRINTF("read %s [0x%" HWADDR_PRIx "] -> 0x%02x\n",
+            imx_i2c_get_regname(offset), offset, value);
+
+    return (uint64_t)value;
+}
+
+static void imx_i2c_write(void *opaque, hwaddr offset,
+                          uint64_t value, unsigned size)
+{
+    IMXI2CState *s = IMX_I2C(opaque);
+
+    DPRINTF("write %s [0x%" HWADDR_PRIx "] <- 0x%02x\n",
+            imx_i2c_get_regname(offset), offset, (int)value);
+
+    value &= 0xff;
+
+    switch (offset) {
+    case IADR_ADDR:
+        s->iadr = value & IADR_MASK;
+        /* i2c_set_slave_address(s->bus, (uint8_t)s->iadr); */
+        break;
+    case IFDR_ADDR:
+        s->ifdr = value & IFDR_MASK;
+        break;
+    case I2CR_ADDR:
+        if (imx_i2c_is_enabled(s) && ((value & I2CR_IEN) == 0)) {
+            /* This is a soft reset. IADR is preserved during soft resets */
+            uint16_t iadr = s->iadr;
+            imx_i2c_reset(DEVICE(s));
+            s->iadr = iadr;
+        } else { /* normal write */
+            s->i2cr = value & I2CR_MASK;
+
+            if (imx_i2c_is_master(s)) {
+                /* set the bus to busy */
+                s->i2sr |= I2SR_IBB;
+            } else { /* slave mode */
+                /* bus is not busy anymore */
+                s->i2sr &= ~I2SR_IBB;
+
+                /*
+                 * if we unset the master mode then it ends the ongoing
+                 * transfer if any
+                 */
+                if (s->address != ADDR_RESET) {
+                    i2c_end_transfer(s->bus);
+                    s->address = ADDR_RESET;
+                }
+            }
+
+            if (s->i2cr & I2CR_RSTA) { /* Restart */
+                /* if this is a restart then it ends the ongoing transfer */
+                if (s->address != ADDR_RESET) {
+                    i2c_end_transfer(s->bus);
+                    s->address = ADDR_RESET;
+                    s->i2cr &= ~I2CR_RSTA;
+                }
+            }
+        }
+        break;
+    case I2SR_ADDR:
+        /*
+         * if the user writes 0 to IIF then lower the interrupt and
+         * reset the bit
+         */
+        if ((s->i2sr & I2SR_IIF) && !(value & I2SR_IIF)) {
+            s->i2sr &= ~I2SR_IIF;
+            qemu_irq_lower(s->irq);
+        }
+
+        /*
+         * if the user writes 0 to IAL, reset the bit
+         */
+        if ((s->i2sr & I2SR_IAL) && !(value & I2SR_IAL)) {
+            s->i2sr &= ~I2SR_IAL;
+        }
+
+        break;
+    case I2DR_ADDR:
+        /* if the device is not enabled, nothing to do */
+        if (!imx_i2c_is_enabled(s)) {
+            break;
+        }
+
+        s->i2dr_write = value & I2DR_MASK;
+
+        if (imx_i2c_is_master(s)) {
+            /* If this is the first write cycle then it is the slave addr */
+            if (s->address == ADDR_RESET) {
+                if (i2c_start_transfer(s->bus, extract32(s->i2dr_write, 1, 7),
+                                       extract32(s->i2dr_write, 0, 1))) {
+                    /* if non zero is returned, the adress is not valid */
+                    s->i2sr |= I2SR_RXAK;
+                } else {
+                    s->address = s->i2dr_write;
+                    s->i2sr &= ~I2SR_RXAK;
+                    imx_i2c_raise_interrupt(s);
+                }
+            } else { /* This is a normal data write */
+                if (i2c_send(s->bus, s->i2dr_write)) {
+                    /* if the target return non zero then end the transfer */
+                    s->i2sr |= I2SR_RXAK;
+                    s->address = ADDR_RESET;
+                    i2c_end_transfer(s->bus);
+                } else {
+                    s->i2sr &= ~I2SR_RXAK;
+                    imx_i2c_raise_interrupt(s);
+                }
+            }
+        } else {
+            qemu_log_mask(LOG_UNIMP, "[%s]%s: slave mode not implemented\n",
+                          TYPE_IMX_I2C, __func__);
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_I2C, __func__, offset);
+        break;
+    }
+}
+
+static const MemoryRegionOps imx_i2c_ops = {
+    .read = imx_i2c_read,
+    .write = imx_i2c_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 2,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static const VMStateDescription imx_i2c_vmstate = {
+    .name = TYPE_IMX_I2C,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(address, IMXI2CState),
+        VMSTATE_UINT16(iadr, IMXI2CState),
+        VMSTATE_UINT16(ifdr, IMXI2CState),
+        VMSTATE_UINT16(i2cr, IMXI2CState),
+        VMSTATE_UINT16(i2sr, IMXI2CState),
+        VMSTATE_UINT16(i2dr_read, IMXI2CState),
+        VMSTATE_UINT16(i2dr_write, IMXI2CState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void imx_i2c_realize(DeviceState *dev, Error **errp)
+{
+    IMXI2CState *s = IMX_I2C(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &imx_i2c_ops, s, TYPE_IMX_I2C,
+                          IMX_I2C_MEM_SIZE);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq);
+    s->bus = i2c_init_bus(DEVICE(dev), "i2c");
+}
+
+static void imx_i2c_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &imx_i2c_vmstate;
+    dc->reset = imx_i2c_reset;
+    dc->realize = imx_i2c_realize;
+}
+
+static const TypeInfo imx_i2c_type_info = {
+    .name = TYPE_IMX_I2C,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXI2CState),
+    .class_init = imx_i2c_class_init,
+};
+
+static void imx_i2c_register_types(void)
+{
+    type_register_static(&imx_i2c_type_info);
+}
+
+type_init(imx_i2c_register_types)
diff --git a/hw/i386/Makefile.objs b/hw/i386/Makefile.objs
index bd4f147f9..c250deb84 100644
--- a/hw/i386/Makefile.objs
+++ b/hw/i386/Makefile.objs
@@ -1,5 +1,5 @@
 obj-$(CONFIG_KVM) += kvm/
-obj-y += multiboot.o smbios.o
+obj-y += multiboot.o
 obj-y += pc.o pc_piix.o pc_q35.o
 obj-y += pc_sysfw.o
 obj-y += intel_iommu.o
@@ -7,8 +7,15 @@ obj-$(CONFIG_XEN) += ../xenpv/ xen/
 
 obj-y += kvmvapic.o
 obj-y += acpi-build.o
+obj-y += pci-assign-load-rom.o
+
+gen-hex-y += hw/i386/acpi-dsdt.hex
+gen-hex-y += hw/i386/q35-acpi-dsdt.hex
+
 hw/i386/acpi-build.o: hw/i386/acpi-build.c \
-	hw/i386/acpi-dsdt.hex hw/i386/q35-acpi-dsdt.hex
+	$(gen-hex-y)
+
+-include $(gen-hex-y:.hex=.d)
 
 iasl-option=$(shell if test -z "`$(1) $(2) 2>&1 > /dev/null`" \
     ; then echo "$(2)"; else echo "$(3)"; fi ;)
diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
index 46eddb8e4..95e0c657a 100644
--- a/hw/i386/acpi-build.c
+++ b/hw/i386/acpi-build.c
@@ -169,6 +169,7 @@ static void acpi_get_pm_info(AcpiPmInfo *pm)
     Object *obj = NULL;
     QObject *o;
 
+    pm->cpu_hp_io_base = 0;
     pm->pcihp_io_base = 0;
     pm->pcihp_io_len = 0;
     if (piix) {
diff --git a/hw/i386/acpi-dsdt.dsl b/hw/i386/acpi-dsdt.dsl
index a2d84ecf8..8dba096dd 100644
--- a/hw/i386/acpi-dsdt.dsl
+++ b/hw/i386/acpi-dsdt.dsl
@@ -43,7 +43,6 @@ DefinitionBlock (
 
 #include "acpi-dsdt-hpet.dsl"
 
-
 /****************************************************************
  * PIIX4 PM
  ****************************************************************/
diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
index 08055a8d8..3fe27fa51 100644
--- a/hw/i386/intel_iommu.c
+++ b/hw/i386/intel_iommu.c
@@ -22,6 +22,7 @@
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 #include "intel_iommu_internal.h"
+#include "hw/pci/pci.h"
 
 /*#define DEBUG_INTEL_IOMMU*/
 #ifdef DEBUG_INTEL_IOMMU
@@ -166,19 +167,17 @@ static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
  */
 static void vtd_reset_context_cache(IntelIOMMUState *s)
 {
-    VTDAddressSpace **pvtd_as;
     VTDAddressSpace *vtd_as;
-    uint32_t bus_it;
+    VTDBus *vtd_bus;
+    GHashTableIter bus_it;
     uint32_t devfn_it;
 
+    g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
+
     VTD_DPRINTF(CACHE, "global context_cache_gen=1");
-    for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) {
-        pvtd_as = s->address_spaces[bus_it];
-        if (!pvtd_as) {
-            continue;
-        }
+    while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
         for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
-            vtd_as = pvtd_as[devfn_it];
+            vtd_as = vtd_bus->dev_as[devfn_it];
             if (!vtd_as) {
                 continue;
             }
@@ -754,12 +753,13 @@ static inline bool vtd_is_interrupt_addr(hwaddr addr)
  * @is_write: The access is a write operation
  * @entry: IOMMUTLBEntry that contain the addr to be translated and result
  */
-static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,
+static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
                                    uint8_t devfn, hwaddr addr, bool is_write,
                                    IOMMUTLBEntry *entry)
 {
     IntelIOMMUState *s = vtd_as->iommu_state;
     VTDContextEntry ce;
+    uint8_t bus_num = pci_bus_num(bus);
     VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
     uint64_t slpte;
     uint32_t level;
@@ -874,6 +874,29 @@ static void vtd_context_global_invalidate(IntelIOMMUState *s)
     }
 }
 
+
+/* Find the VTD address space currently associated with a given bus number,
+ */
+static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
+{
+    VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
+    if (!vtd_bus) {
+        /* Iterate over the registered buses to find the one
+         * which currently hold this bus number, and update the bus_num lookup table:
+         */
+        GHashTableIter iter;
+
+        g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
+        while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
+            if (pci_bus_num(vtd_bus->bus) == bus_num) {
+                s->vtd_as_by_bus_num[bus_num] = vtd_bus;
+                return vtd_bus;
+            }
+        }
+    }
+    return vtd_bus;
+}
+
 /* Do a context-cache device-selective invalidation.
  * @func_mask: FM field after shifting
  */
@@ -882,7 +905,7 @@ static void vtd_context_device_invalidate(IntelIOMMUState *s,
                                           uint16_t func_mask)
 {
     uint16_t mask;
-    VTDAddressSpace **pvtd_as;
+    VTDBus *vtd_bus;
     VTDAddressSpace *vtd_as;
     uint16_t devfn;
     uint16_t devfn_it;
@@ -903,11 +926,11 @@ static void vtd_context_device_invalidate(IntelIOMMUState *s,
     }
     VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
                     " mask %"PRIu16, source_id, mask);
-    pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)];
-    if (pvtd_as) {
+    vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
+    if (vtd_bus) {
         devfn = VTD_SID_TO_DEVFN(source_id);
         for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
-            vtd_as = pvtd_as[devfn_it];
+            vtd_as = vtd_bus->dev_as[devfn_it];
             if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
                 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
                             devfn_it);
@@ -1805,11 +1828,11 @@ static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
         return ret;
     }
 
-    vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr,
+    vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr,
                            is_write, &ret);
     VTD_DPRINTF(MMU,
                 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
-                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num,
+                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus),
                 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
                 vtd_as->devfn, addr, ret.translated_addr);
     return ret;
@@ -1839,6 +1862,38 @@ static Property vtd_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
+
+VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
+{
+    uintptr_t key = (uintptr_t)bus;
+    VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
+    VTDAddressSpace *vtd_dev_as;
+
+    if (!vtd_bus) {
+        /* No corresponding free() */
+        vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX);
+        vtd_bus->bus = bus;
+        key = (uintptr_t)bus;
+        g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus);
+    }
+
+    vtd_dev_as = vtd_bus->dev_as[devfn];
+
+    if (!vtd_dev_as) {
+        vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
+
+        vtd_dev_as->bus = bus;
+        vtd_dev_as->devfn = (uint8_t)devfn;
+        vtd_dev_as->iommu_state = s;
+        vtd_dev_as->context_cache_entry.context_cache_gen = 0;
+        memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s),
+                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);
+        address_space_init(&vtd_dev_as->as,
+                           &vtd_dev_as->iommu, "intel_iommu");
+    }
+    return vtd_dev_as;
+}
+
 /* Do the initialization. It will also be called when reset, so pay
  * attention when adding new initialization stuff.
  */
@@ -1931,13 +1986,15 @@ static void vtd_realize(DeviceState *dev, Error **errp)
     IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
 
     VTD_DPRINTF(GENERAL, "");
-    memset(s->address_spaces, 0, sizeof(s->address_spaces));
+    memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
     memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
                           "intel_iommu", DMAR_REG_SIZE);
     sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
     /* No corresponding destroy */
     s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
                                      g_free, g_free);
+    s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
+                                              g_free, g_free);
     vtd_init(s);
 }
 
diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c
index efdf16584..0593a3f1f 100644
--- a/hw/i386/kvm/clock.c
+++ b/hw/i386/kvm/clock.c
@@ -17,7 +17,7 @@
 #include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
-#include "sysemu/cpus.h"
+#include "kvm_i386.h"
 #include "hw/sysbus.h"
 #include "hw/kvm/clock.h"
 
@@ -125,21 +125,7 @@ static void kvmclock_vm_state_change(void *opaque, int running,
             return;
         }
 
-        cpu_synchronize_all_states();
-        /* In theory, the cpu_synchronize_all_states() call above wouldn't
-         * affect the rest of the code, as the VCPU state inside CPUState
-         * is supposed to always match the VCPU state on the kernel side.
-         *
-         * In practice, calling cpu_synchronize_state() too soon will load the
-         * kernel-side APIC state into X86CPU.apic_state too early, APIC state
-         * won't be reloaded later because CPUState.vcpu_dirty==true, and
-         * outdated APIC state may be migrated to another host.
-         *
-         * The real fix would be to make sure outdated APIC state is read
-         * from the kernel again when necessary. While this is not fixed, we
-         * need the cpu_clean_all_dirty() call below.
-         */
-        cpu_clean_all_dirty();
+        kvm_synchronize_all_tsc();
 
         ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
         if (ret < 0) {
diff --git a/hw/i386/kvm/ioapic.c b/hw/i386/kvm/ioapic.c
index d2a6c4cf6..b7390ca0d 100644
--- a/hw/i386/kvm/ioapic.c
+++ b/hw/i386/kvm/ioapic.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "monitor/monitor.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/ioapic_internal.h"
 #include "hw/i386/apic_internal.h"
@@ -110,6 +111,15 @@ static void kvm_ioapic_put(IOAPICCommonState *s)
     }
 }
 
+void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict)
+{
+    IOAPICCommonState s;
+
+    kvm_ioapic_get(&s);
+
+    ioapic_print_redtbl(mon, &s);
+}
+
 static void kvm_ioapic_reset(DeviceState *dev)
 {
     IOAPICCommonState *s = IOAPIC_COMMON(dev);
diff --git a/hw/i386/kvm/pci-assign.c b/hw/i386/kvm/pci-assign.c
index 74d22f4fd..0fd69230a 100644
--- a/hw/i386/kvm/pci-assign.c
+++ b/hw/i386/kvm/pci-assign.c
@@ -22,7 +22,6 @@
  */
 #include <stdio.h>
 #include <unistd.h>
-#include <sys/io.h>
 #include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -37,6 +36,7 @@
 #include "hw/pci/pci.h"
 #include "hw/pci/msi.h"
 #include "kvm_i386.h"
+#include "hw/pci/pci-assign.h"
 
 #define MSIX_PAGE_SIZE 0x1000
 
@@ -48,17 +48,6 @@
 #define IORESOURCE_PREFETCH 0x00002000  /* No side effects */
 #define IORESOURCE_MEM_64   0x00100000
 
-//#define DEVICE_ASSIGNMENT_DEBUG
-
-#ifdef DEVICE_ASSIGNMENT_DEBUG
-#define DEBUG(fmt, ...)                                       \
-    do {                                                      \
-        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
-    } while (0)
-#else
-#define DEBUG(fmt, ...)
-#endif
-
 typedef struct PCIRegion {
     int type;           /* Memory or port I/O */
     int valid;
@@ -990,7 +979,7 @@ static void assigned_dev_update_msi(PCIDevice *pci_dev)
         MSIMessage msg = msi_get_message(pci_dev, 0);
         int virq;
 
-        virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        virq = kvm_irqchip_add_msi_route(kvm_state, msg, pci_dev);
         if (virq < 0) {
             perror("assigned_dev_update_msi: kvm_irqchip_add_msi_route");
             return;
@@ -1028,7 +1017,7 @@ static void assigned_dev_update_msi_msg(PCIDevice *pci_dev)
     }
 
     kvm_irqchip_update_msi_route(kvm_state, assigned_dev->msi_virq[0],
-                                 msi_get_message(pci_dev, 0));
+                                 msi_get_message(pci_dev, 0), pci_dev);
 }
 
 static bool assigned_dev_msix_masked(MSIXTableEntry *entry)
@@ -1094,7 +1083,7 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
 
         msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32);
         msg.data = entry->data;
-        r = kvm_irqchip_add_msi_route(kvm_state, msg);
+        r = kvm_irqchip_add_msi_route(kvm_state, msg, pci_dev);
         if (r < 0) {
             return r;
         }
@@ -1494,7 +1483,7 @@ static int assigned_device_pci_cap_init(PCIDevice *pci_dev, Error **errp)
          * error bits, leave the rest. */
         status = pci_get_long(pci_dev->config + pos + PCI_X_STATUS);
         status &= ~(PCI_X_STATUS_BUS | PCI_X_STATUS_DEVFN);
-        status |= (pci_bus_num(pci_dev->bus) << 8) | pci_dev->devfn;
+        status |= pci_requester_id(pci_dev);
         status &= ~(PCI_X_STATUS_SPL_DISC | PCI_X_STATUS_UNX_SPL |
                     PCI_X_STATUS_SPL_ERR);
         pci_set_long(pci_dev->config + pos + PCI_X_STATUS, status);
@@ -1613,7 +1602,8 @@ static void assigned_dev_msix_mmio_write(void *opaque, hwaddr addr,
                 msg.data = entry->data;
 
                 ret = kvm_irqchip_update_msi_route(kvm_state,
-                                                   adev->msi_virq[i], msg);
+                                                   adev->msi_virq[i], msg,
+                                                   pdev);
                 if (ret) {
                     error_report("Error updating irq routing entry (%d)", ret);
                 }
@@ -1896,73 +1886,15 @@ static void assign_register_types(void)
 
 type_init(assign_register_types)
 
-/*
- * Scan the assigned devices for the devices that have an option ROM, and then
- * load the corresponding ROM data to RAM. If an error occurs while loading an
- * option ROM, we just ignore that option ROM and continue with the next one.
- */
 static void assigned_dev_load_option_rom(AssignedDevice *dev)
 {
-    char name[32], rom_file[64];
-    FILE *fp;
-    uint8_t val;
-    struct stat st;
-    void *ptr;
-
-    /* If loading ROM from file, pci handles it */
-    if (dev->dev.romfile || !dev->dev.rom_bar) {
-        return;
-    }
+    int size = 0;
 
-    snprintf(rom_file, sizeof(rom_file),
-             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
-             dev->host.domain, dev->host.bus, dev->host.slot,
-             dev->host.function);
+    pci_assign_dev_load_option_rom(&dev->dev, OBJECT(dev), &size,
+                                   dev->host.domain, dev->host.bus,
+                                   dev->host.slot, dev->host.function);
 
-    if (stat(rom_file, &st)) {
-        return;
-    }
-
-    if (access(rom_file, F_OK)) {
-        error_report("pci-assign: Insufficient privileges for %s", rom_file);
-        return;
-    }
-
-    /* Write "1" to the ROM file to enable it */
-    fp = fopen(rom_file, "r+");
-    if (fp == NULL) {
-        return;
+    if (!size) {
+        error_report("pci-assign: Invalid ROM.");
     }
-    val = 1;
-    if (fwrite(&val, 1, 1, fp) != 1) {
-        goto close_rom;
-    }
-    fseek(fp, 0, SEEK_SET);
-
-    snprintf(name, sizeof(name), "%s.rom",
-            object_get_typename(OBJECT(dev)));
-    memory_region_init_ram(&dev->dev.rom, OBJECT(dev), name, st.st_size,
-                           &error_abort);
-    vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev);
-    ptr = memory_region_get_ram_ptr(&dev->dev.rom);
-    memset(ptr, 0xff, st.st_size);
-
-    if (!fread(ptr, 1, st.st_size, fp)) {
-        error_report("pci-assign: Cannot read from host %s", rom_file);
-        error_printf("Device option ROM contents are probably invalid "
-                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
-                     "or load from file with romfile=\n");
-        goto close_rom;
-    }
-
-    pci_register_bar(&dev->dev, PCI_ROM_SLOT, 0, &dev->dev.rom);
-    dev->dev.has_rom = true;
-close_rom:
-    /* Write "0" to disable ROM */
-    fseek(fp, 0, SEEK_SET);
-    val = 0;
-    if (!fwrite(&val, 1, 1, fp)) {
-        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
-    }
-    fclose(fp);
 }
diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c
index c6d34b254..f0922da68 100644
--- a/hw/i386/kvmvapic.c
+++ b/hw/i386/kvmvapic.c
@@ -634,13 +634,18 @@ static int vapic_prepare(VAPICROMState *s)
 static void vapic_write(void *opaque, hwaddr addr, uint64_t data,
                         unsigned int size)
 {
-    CPUState *cs = current_cpu;
-    X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
-    hwaddr rom_paddr;
     VAPICROMState *s = opaque;
+    X86CPU *cpu;
+    CPUX86State *env;
+    hwaddr rom_paddr;
 
-    cpu_synchronize_state(cs);
+    if (!current_cpu) {
+        return;
+    }
+
+    cpu_synchronize_state(current_cpu);
+    cpu = X86_CPU(current_cpu);
+    env = &cpu->env;
 
     /*
      * The VAPIC supports two PIO-based hypercalls, both via port 0x7E.
diff --git a/hw/i386/multiboot.c b/hw/i386/multiboot.c
index 1adbe9e25..6774a1932 100644
--- a/hw/i386/multiboot.c
+++ b/hw/i386/multiboot.c
@@ -195,7 +195,7 @@ int load_multiboot(FWCfgState *fw_cfg,
         }
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               &elf_low, &elf_high, 0, ELF_MACHINE, 0);
+                               &elf_low, &elf_high, 0, I386_ELF_MACHINE, 0);
         if (kernel_size < 0) {
             fprintf(stderr, "Error while loading elf kernel\n");
             exit(1);
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index 7661ea9cd..5e20e07b6 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -33,7 +33,7 @@
 #include "hw/pci/pci_bus.h"
 #include "hw/nvram/fw_cfg.h"
 #include "hw/timer/hpet.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "multiboot.h"
@@ -59,7 +59,6 @@
 #include "qemu/error-report.h"
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/cpu_hotplug.h"
-#include "hw/cpu/icc_bus.h"
 #include "hw/boards.h"
 #include "hw/pci/pci_host.h"
 #include "acpi-build.h"
@@ -428,26 +427,24 @@ static void pc_cmos_init_late(void *opaque)
     qemu_unregister_reset(pc_cmos_init_late, opaque);
 }
 
-void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device, MachineState *machine,
+void pc_cmos_init(PCMachineState *pcms,
                   BusState *idebus0, BusState *idebus1,
                   ISADevice *s)
 {
     int val;
     static pc_cmos_init_late_arg arg;
-    PCMachineState *pc_machine = PC_MACHINE(machine);
     Error *local_err = NULL;
 
     /* various important CMOS locations needed by PC/Bochs bios */
 
     /* memory size */
     /* base memory (first MiB) */
-    val = MIN(ram_size / 1024, 640);
+    val = MIN(pcms->below_4g_mem_size / 1024, 640);
     rtc_set_memory(s, 0x15, val);
     rtc_set_memory(s, 0x16, val >> 8);
     /* extended memory (next 64MiB) */
-    if (ram_size > 1024 * 1024) {
-        val = (ram_size - 1024 * 1024) / 1024;
+    if (pcms->below_4g_mem_size > 1024 * 1024) {
+        val = (pcms->below_4g_mem_size - 1024 * 1024) / 1024;
     } else {
         val = 0;
     }
@@ -458,8 +455,8 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
     rtc_set_memory(s, 0x30, val);
     rtc_set_memory(s, 0x31, val >> 8);
     /* memory between 16MiB and 4GiB */
-    if (ram_size > 16 * 1024 * 1024) {
-        val = (ram_size - 16 * 1024 * 1024) / 65536;
+    if (pcms->below_4g_mem_size > 16 * 1024 * 1024) {
+        val = (pcms->below_4g_mem_size - 16 * 1024 * 1024) / 65536;
     } else {
         val = 0;
     }
@@ -468,7 +465,7 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
     rtc_set_memory(s, 0x34, val);
     rtc_set_memory(s, 0x35, val >> 8);
     /* memory above 4GiB */
-    val = above_4g_mem_size / 65536;
+    val = pcms->above_4g_mem_size / 65536;
     rtc_set_memory(s, 0x5b, val);
     rtc_set_memory(s, 0x5c, val >> 8);
     rtc_set_memory(s, 0x5d, val >> 16);
@@ -476,15 +473,15 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
     /* set the number of CPU */
     rtc_set_memory(s, 0x5f, smp_cpus - 1);
 
-    object_property_add_link(OBJECT(machine), "rtc_state",
+    object_property_add_link(OBJECT(pcms), "rtc_state",
                              TYPE_ISA_DEVICE,
-                             (Object **)&pc_machine->rtc,
+                             (Object **)&pcms->rtc,
                              object_property_allow_set_link,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
-    object_property_set_link(OBJECT(machine), OBJECT(s),
+    object_property_set_link(OBJECT(pcms), OBJECT(s),
                              "rtc_state", &error_abort);
 
-    set_boot_dev(s, boot_device, &local_err);
+    set_boot_dev(s, MACHINE(pcms)->boot_order, &local_err);
     if (local_err) {
         error_report_err(local_err);
         exit(1);
@@ -718,16 +715,52 @@ static unsigned int pc_apic_id_limit(unsigned int max_cpus)
     return x86_cpu_apic_id_from_index(max_cpus - 1) + 1;
 }
 
-static FWCfgState *bochs_bios_init(void)
+static void pc_build_smbios(FWCfgState *fw_cfg)
 {
-    FWCfgState *fw_cfg;
     uint8_t *smbios_tables, *smbios_anchor;
     size_t smbios_tables_len, smbios_anchor_len;
+    struct smbios_phys_mem_area *mem_array;
+    unsigned i, array_count;
+
+    smbios_tables = smbios_get_table_legacy(&smbios_tables_len);
+    if (smbios_tables) {
+        fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
+                         smbios_tables, smbios_tables_len);
+    }
+
+    /* build the array of physical mem area from e820 table */
+    mem_array = g_malloc0(sizeof(*mem_array) * e820_get_num_entries());
+    for (i = 0, array_count = 0; i < e820_get_num_entries(); i++) {
+        uint64_t addr, len;
+
+        if (e820_get_entry(i, E820_RAM, &addr, &len)) {
+            mem_array[array_count].address = addr;
+            mem_array[array_count].length = len;
+            array_count++;
+        }
+    }
+    smbios_get_tables(mem_array, array_count,
+                      &smbios_tables, &smbios_tables_len,
+                      &smbios_anchor, &smbios_anchor_len);
+    g_free(mem_array);
+
+    if (smbios_anchor) {
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
+                        smbios_tables, smbios_tables_len);
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
+                        smbios_anchor, smbios_anchor_len);
+    }
+}
+
+static FWCfgState *bochs_bios_init(AddressSpace *as)
+{
+    FWCfgState *fw_cfg;
     uint64_t *numa_fw_cfg;
     int i, j;
     unsigned int apic_id_limit = pc_apic_id_limit(max_cpus);
 
-    fw_cfg = fw_cfg_init_io(BIOS_CFG_IOPORT);
+    fw_cfg = fw_cfg_init_io_dma(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 4, as);
+
     /* FW_CFG_MAX_CPUS is a bit confusing/problematic on x86:
      *
      * SeaBIOS needs FW_CFG_MAX_CPUS for CPU hotplug, but the CPU hotplug
@@ -748,20 +781,7 @@ static FWCfgState *bochs_bios_init(void)
                      acpi_tables, acpi_tables_len);
     fw_cfg_add_i32(fw_cfg, FW_CFG_IRQ0_OVERRIDE, kvm_allows_irq0_override());
 
-    smbios_tables = smbios_get_table_legacy(&smbios_tables_len);
-    if (smbios_tables) {
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
-                         smbios_tables, smbios_tables_len);
-    }
-
-    smbios_get_tables(&smbios_tables, &smbios_tables_len,
-                      &smbios_anchor, &smbios_anchor_len);
-    if (smbios_anchor) {
-        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
-                        smbios_tables, smbios_tables_len);
-        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
-                        smbios_anchor, smbios_anchor_len);
-    }
+    pc_build_smbios(fw_cfg);
 
     fw_cfg_add_bytes(fw_cfg, FW_CFG_E820_TABLE,
                      &e820_reserve, sizeof(e820_reserve));
@@ -809,11 +829,8 @@ static long get_file_size(FILE *f)
     return size;
 }
 
-static void load_linux(FWCfgState *fw_cfg,
-                       const char *kernel_filename,
-                       const char *initrd_filename,
-                       const char *kernel_cmdline,
-                       hwaddr max_ram_size)
+static void load_linux(PCMachineState *pcms,
+                       FWCfgState *fw_cfg)
 {
     uint16_t protocol;
     int setup_size, kernel_size, initrd_size = 0, cmdline_size;
@@ -822,6 +839,10 @@ static void load_linux(FWCfgState *fw_cfg,
     hwaddr real_addr, prot_addr, cmdline_addr, initrd_addr = 0;
     FILE *f;
     char *vmode;
+    MachineState *machine = MACHINE(pcms);
+    const char *kernel_filename = machine->kernel_filename;
+    const char *initrd_filename = machine->initrd_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
 
     /* Align to 16 bytes as a paranoia measure */
     cmdline_size = (strlen(kernel_cmdline)+16) & ~15;
@@ -886,8 +907,8 @@ static void load_linux(FWCfgState *fw_cfg,
         initrd_max = 0x37ffffff;
     }
 
-    if (initrd_max >= max_ram_size - acpi_data_size) {
-        initrd_max = max_ram_size - acpi_data_size - 1;
+    if (initrd_max >= pcms->below_4g_mem_size - acpi_data_size) {
+        initrd_max = pcms->below_4g_mem_size - acpi_data_size - 1;
     }
 
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_ADDR, cmdline_addr);
@@ -965,6 +986,10 @@ static void load_linux(FWCfgState *fw_cfg,
         setup_size = 4;
     }
     setup_size = (setup_size+1)*512;
+    if (setup_size > kernel_size) {
+        fprintf(stderr, "qemu: invalid kernel header\n");
+        exit(1);
+    }
     kernel_size -= setup_size;
 
     setup  = g_malloc(setup_size);
@@ -1031,23 +1056,16 @@ void pc_acpi_smi_interrupt(void *opaque, int irq, int level)
 }
 
 static X86CPU *pc_new_cpu(const char *cpu_model, int64_t apic_id,
-                          DeviceState *icc_bridge, Error **errp)
+                          Error **errp)
 {
     X86CPU *cpu = NULL;
     Error *local_err = NULL;
 
-    if (icc_bridge == NULL) {
-        error_setg(&local_err, "Invalid icc-bridge value");
-        goto out;
-    }
-
     cpu = cpu_x86_create(cpu_model, &local_err);
     if (local_err != NULL) {
         goto out;
     }
 
-    qdev_set_parent_bus(DEVICE(cpu), qdev_get_child_bus(icc_bridge, "icc"));
-
     object_property_set_int(OBJECT(cpu), apic_id, "apic-id", &local_err);
     object_property_set_bool(OBJECT(cpu), true, "realized", &local_err);
 
@@ -1060,12 +1078,10 @@ out:
     return cpu;
 }
 
-static const char *current_cpu_model;
-
 void pc_hot_add_cpu(const int64_t id, Error **errp)
 {
-    DeviceState *icc_bridge;
     X86CPU *cpu;
+    MachineState *machine = MACHINE(qdev_get_machine());
     int64_t apic_id = x86_cpu_apic_id_from_index(id);
     Error *local_err = NULL;
 
@@ -1093,9 +1109,7 @@ void pc_hot_add_cpu(const int64_t id, Error **errp)
         return;
     }
 
-    icc_bridge = DEVICE(object_resolve_path_type("icc-bridge",
-                                                 TYPE_ICC_BRIDGE, NULL));
-    cpu = pc_new_cpu(current_cpu_model, apic_id, icc_bridge, &local_err);
+    cpu = pc_new_cpu(machine->cpu_model, apic_id, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1103,22 +1117,22 @@ void pc_hot_add_cpu(const int64_t id, Error **errp)
     object_unref(OBJECT(cpu));
 }
 
-void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge)
+void pc_cpus_init(PCMachineState *pcms)
 {
     int i;
     X86CPU *cpu = NULL;
+    MachineState *machine = MACHINE(pcms);
     Error *error = NULL;
     unsigned long apic_id_limit;
 
     /* init CPUs */
-    if (cpu_model == NULL) {
+    if (machine->cpu_model == NULL) {
 #ifdef TARGET_X86_64
-        cpu_model = "qemu64";
+        machine->cpu_model = "qemu64";
 #else
-        cpu_model = "qemu32";
+        machine->cpu_model = "qemu32";
 #endif
     }
-    current_cpu_model = cpu_model;
 
     apic_id_limit = pc_apic_id_limit(max_cpus);
     if (apic_id_limit > ACPI_CPU_HOTPLUG_ID_LIMIT) {
@@ -1128,8 +1142,8 @@ void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge)
     }
 
     for (i = 0; i < smp_cpus; i++) {
-        cpu = pc_new_cpu(cpu_model, x86_cpu_apic_id_from_index(i),
-                         icc_bridge, &error);
+        cpu = pc_new_cpu(machine->cpu_model, x86_cpu_apic_id_from_index(i),
+                         &error);
         if (error) {
             error_report_err(error);
             exit(1);
@@ -1137,13 +1151,6 @@ void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge)
         object_unref(OBJECT(cpu));
     }
 
-    /* map APIC MMIO area if CPU has APIC */
-    if (cpu && cpu->apic_state) {
-        /* XXX: what if the base changes? */
-        sysbus_mmio_map_overlap(SYS_BUS_DEVICE(icc_bridge), 0,
-                                APIC_DEFAULT_ADDRESS, 0x1000);
-    }
-
     /* tell smbios about cpuid version and features */
     smbios_set_cpuid(cpu->env.cpuid_version, cpu->env.features[FEAT_1_EDX]);
 }
@@ -1189,15 +1196,14 @@ void pc_guest_info_machine_done(Notifier *notifier, void *data)
     acpi_setup(&guest_info_state->info);
 }
 
-PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
-                                ram_addr_t above_4g_mem_size)
+PcGuestInfo *pc_guest_info_init(PCMachineState *pcms)
 {
     PcGuestInfoState *guest_info_state = g_malloc0(sizeof *guest_info_state);
     PcGuestInfo *guest_info = &guest_info_state->info;
     int i, j;
 
-    guest_info->ram_size_below_4g = below_4g_mem_size;
-    guest_info->ram_size = below_4g_mem_size + above_4g_mem_size;
+    guest_info->ram_size_below_4g = pcms->below_4g_mem_size;
+    guest_info->ram_size = pcms->below_4g_mem_size + pcms->above_4g_mem_size;
     guest_info->apic_id_limit = pc_apic_id_limit(max_cpus);
     guest_info->apic_xrupt_override = kvm_allows_irq0_override();
     guest_info->numa_nodes = nb_numa_nodes;
@@ -1264,22 +1270,18 @@ void pc_acpi_init(const char *default_dsdt)
     }
 }
 
-FWCfgState *xen_load_linux(const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           ram_addr_t below_4g_mem_size,
+FWCfgState *xen_load_linux(PCMachineState *pcms,
                            PcGuestInfo *guest_info)
 {
     int i;
     FWCfgState *fw_cfg;
 
-    assert(kernel_filename != NULL);
+    assert(MACHINE(pcms)->kernel_filename != NULL);
 
     fw_cfg = fw_cfg_init_io(BIOS_CFG_IOPORT);
     rom_set_fw(fw_cfg);
 
-    load_linux(fw_cfg, kernel_filename, initrd_filename,
-               kernel_cmdline, below_4g_mem_size);
+    load_linux(pcms, fw_cfg);
     for (i = 0; i < nb_option_roms; i++) {
         assert(!strcmp(option_rom[i].name, "linuxboot.bin") ||
                !strcmp(option_rom[i].name, "multiboot.bin"));
@@ -1289,10 +1291,8 @@ FWCfgState *xen_load_linux(const char *kernel_filename,
     return fw_cfg;
 }
 
-FWCfgState *pc_memory_init(MachineState *machine,
+FWCfgState *pc_memory_init(PCMachineState *pcms,
                            MemoryRegion *system_memory,
-                           ram_addr_t below_4g_mem_size,
-                           ram_addr_t above_4g_mem_size,
                            MemoryRegion *rom_memory,
                            MemoryRegion **ram_memory,
                            PcGuestInfo *guest_info)
@@ -1301,9 +1301,10 @@ FWCfgState *pc_memory_init(MachineState *machine,
     MemoryRegion *ram, *option_rom_mr;
     MemoryRegion *ram_below_4g, *ram_above_4g;
     FWCfgState *fw_cfg;
-    PCMachineState *pcms = PC_MACHINE(machine);
+    MachineState *machine = MACHINE(pcms);
 
-    assert(machine->ram_size == below_4g_mem_size + above_4g_mem_size);
+    assert(machine->ram_size == pcms->below_4g_mem_size +
+                                pcms->above_4g_mem_size);
 
     linux_boot = (machine->kernel_filename != NULL);
 
@@ -1317,16 +1318,17 @@ FWCfgState *pc_memory_init(MachineState *machine,
     *ram_memory = ram;
     ram_below_4g = g_malloc(sizeof(*ram_below_4g));
     memory_region_init_alias(ram_below_4g, NULL, "ram-below-4g", ram,
-                             0, below_4g_mem_size);
+                             0, pcms->below_4g_mem_size);
     memory_region_add_subregion(system_memory, 0, ram_below_4g);
-    e820_add_entry(0, below_4g_mem_size, E820_RAM);
-    if (above_4g_mem_size > 0) {
+    e820_add_entry(0, pcms->below_4g_mem_size, E820_RAM);
+    if (pcms->above_4g_mem_size > 0) {
         ram_above_4g = g_malloc(sizeof(*ram_above_4g));
         memory_region_init_alias(ram_above_4g, NULL, "ram-above-4g", ram,
-                                 below_4g_mem_size, above_4g_mem_size);
+                                 pcms->below_4g_mem_size,
+                                 pcms->above_4g_mem_size);
         memory_region_add_subregion(system_memory, 0x100000000ULL,
                                     ram_above_4g);
-        e820_add_entry(0x100000000ULL, above_4g_mem_size, E820_RAM);
+        e820_add_entry(0x100000000ULL, pcms->above_4g_mem_size, E820_RAM);
     }
 
     if (!guest_info->has_reserved_memory &&
@@ -1359,7 +1361,7 @@ FWCfgState *pc_memory_init(MachineState *machine,
         }
 
         pcms->hotplug_memory.base =
-            ROUND_UP(0x100000000ULL + above_4g_mem_size, 1ULL << 30);
+            ROUND_UP(0x100000000ULL + pcms->above_4g_mem_size, 1ULL << 30);
 
         if (pcms->enforce_aligned_dimm) {
             /* size hotplug region assuming 1G page max alignment per slot */
@@ -1384,25 +1386,31 @@ FWCfgState *pc_memory_init(MachineState *machine,
 
     option_rom_mr = g_malloc(sizeof(*option_rom_mr));
     memory_region_init_ram(option_rom_mr, NULL, "pc.rom", PC_ROM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(option_rom_mr);
     memory_region_add_subregion_overlap(rom_memory,
                                         PC_ROM_MIN_VGA,
                                         option_rom_mr,
                                         1);
 
-    fw_cfg = bochs_bios_init();
+    fw_cfg = bochs_bios_init(&address_space_memory);
+
     rom_set_fw(fw_cfg);
 
     if (guest_info->has_reserved_memory && pcms->hotplug_memory.base) {
         uint64_t *val = g_malloc(sizeof(*val));
-        *val = cpu_to_le64(ROUND_UP(pcms->hotplug_memory.base, 0x1ULL << 30));
+        PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
+        uint64_t res_mem_end = pcms->hotplug_memory.base;
+
+        if (!pcmc->broken_reserved_end) {
+            res_mem_end += memory_region_size(&pcms->hotplug_memory.mr);
+        }
+        *val = cpu_to_le64(ROUND_UP(res_mem_end, 0x1ULL << 30));
         fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
     }
 
     if (linux_boot) {
-        load_linux(fw_cfg, machine->kernel_filename, machine->initrd_filename,
-                   machine->kernel_cmdline, below_4g_mem_size);
+        load_linux(pcms, fw_cfg);
     }
 
     for (i = 0; i < nb_option_roms; i++) {
@@ -1431,15 +1439,6 @@ DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus)
     return dev;
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static const MemoryRegionOps ioport80_io_ops = {
     .write = ioport80_write,
     .read = ioport80_read,
@@ -1474,7 +1473,6 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     qemu_irq rtc_irq = NULL;
     qemu_irq *a20_line;
     ISADevice *i8042, *port92, *vmmouse, *pit = NULL;
-    qemu_irq *cpu_exit_irq;
     MemoryRegion *ioport80_io = g_new(MemoryRegion, 1);
     MemoryRegion *ioportF0_io = g_new(MemoryRegion, 1);
 
@@ -1551,8 +1549,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     port92 = isa_create_simple(isa_bus, "port92");
     port92_init(port92, &a20_line[1]);
 
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(0);
 
     for(i = 0; i < MAX_FD; i++) {
         fd[i] = drive_get(IF_FLOPPY, 0, i);
@@ -1798,9 +1795,9 @@ static void pc_machine_set_max_ram_below_4g(Object *obj, Visitor *v,
         return;
     }
     if (value > (1ULL << 32)) {
-        error_set(&error, ERROR_CLASS_GENERIC_ERROR,
-                  "Machine option 'max-ram-below-4g=%"PRIu64
-                  "' expects size less than or equal to 4G", value);
+        error_setg(&error,
+                   "Machine option 'max-ram-below-4g=%"PRIu64
+                   "' expects size less than or equal to 4G", value);
         error_propagate(errp, error);
         return;
     }
@@ -1886,47 +1883,66 @@ static void pc_machine_initfn(Object *obj)
 
     object_property_add(obj, PC_MACHINE_MEMHP_REGION_SIZE, "int",
                         pc_machine_get_hotplug_memory_region_size,
-                        NULL, NULL, NULL, NULL);
+                        NULL, NULL, NULL, &error_abort);
 
     pcms->max_ram_below_4g = 1ULL << 32; /* 4G */
     object_property_add(obj, PC_MACHINE_MAX_RAM_BELOW_4G, "size",
                         pc_machine_get_max_ram_below_4g,
                         pc_machine_set_max_ram_below_4g,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
     object_property_set_description(obj, PC_MACHINE_MAX_RAM_BELOW_4G,
                                     "Maximum ram below the 4G boundary (32bit boundary)",
-                                    NULL);
+                                    &error_abort);
 
     pcms->smm = ON_OFF_AUTO_AUTO;
     object_property_add(obj, PC_MACHINE_SMM, "OnOffAuto",
                         pc_machine_get_smm,
                         pc_machine_set_smm,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
     object_property_set_description(obj, PC_MACHINE_SMM,
                                     "Enable SMM (pc & q35)",
-                                    NULL);
+                                    &error_abort);
 
     pcms->vmport = ON_OFF_AUTO_AUTO;
     object_property_add(obj, PC_MACHINE_VMPORT, "OnOffAuto",
                         pc_machine_get_vmport,
                         pc_machine_set_vmport,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
     object_property_set_description(obj, PC_MACHINE_VMPORT,
                                     "Enable vmport (pc & q35)",
-                                    NULL);
+                                    &error_abort);
 
     pcms->enforce_aligned_dimm = true;
     object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
                              pc_machine_get_aligned_dimm,
-                             NULL, NULL);
+                             NULL, &error_abort);
+}
+
+static void pc_machine_reset(void)
+{
+    CPUState *cs;
+    X86CPU *cpu;
+
+    qemu_devices_reset();
+
+    /* Reset APIC after devices have been reset to cancel
+     * any changes that qemu_devices_reset() might have done.
+     */
+    CPU_FOREACH(cs) {
+        cpu = X86_CPU(cs);
+
+        if (cpu->apic_state) {
+            device_reset(cpu->apic_state);
+        }
+    }
 }
 
 static unsigned pc_cpu_index_to_socket_id(unsigned cpu_index)
 {
-    unsigned pkg_id, core_id, smt_id;
+    X86CPUTopoInfo topo;
     x86_topo_ids_from_idx(smp_cores, smp_threads, cpu_index,
-                          &pkg_id, &core_id, &smt_id);
-    return pkg_id;
+                          &topo);
+    return topo.pkg_id;
 }
 
 static void pc_machine_class_init(ObjectClass *oc, void *data)
@@ -1938,6 +1954,10 @@ static void pc_machine_class_init(ObjectClass *oc, void *data)
     pcmc->get_hotplug_handler = mc->get_hotplug_handler;
     mc->get_hotplug_handler = pc_get_hotpug_handler;
     mc->cpu_index_to_socket_id = pc_cpu_index_to_socket_id;
+    mc->default_boot_order = "cad";
+    mc->hot_add_cpu = pc_hot_add_cpu;
+    mc->max_cpus = 255;
+    mc->reset = pc_machine_reset;
     hc->plug = pc_machine_device_plug_cb;
     hc->unplug_request = pc_machine_device_unplug_request_cb;
     hc->unplug = pc_machine_device_unplug_cb;
diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index a896624f8..2e41efe1b 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -28,7 +28,7 @@
 #include "hw/loader.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/apic.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/usb.h"
@@ -39,7 +39,6 @@
 #include "hw/kvm/clock.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
-#include "hw/cpu/icc_bus.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/block-backend.h"
 #include "hw/i2c/smbus.h"
@@ -50,7 +49,8 @@
 #include "cpu.h"
 #include "qemu/error-report.h"
 #ifdef CONFIG_XEN
-#  include <xen/hvm/hvm_info_table.h>
+#include <xen/hvm/hvm_info_table.h>
+#include "hw/xen/xen_pt.h"
 #endif
 #include "migration/migration.h"
 
@@ -76,13 +76,13 @@ static bool has_reserved_memory = true;
 static bool kvmclock_enabled = true;
 
 /* PC hardware initialisation */
-static void pc_init1(MachineState *machine)
+static void pc_init1(MachineState *machine,
+                     const char *host_type, const char *pci_type)
 {
-    PCMachineState *pc_machine = PC_MACHINE(machine);
+    PCMachineState *pcms = PC_MACHINE(machine);
     MemoryRegion *system_memory = get_system_memory();
     MemoryRegion *system_io = get_system_io();
     int i;
-    ram_addr_t below_4g_mem_size, above_4g_mem_size;
     PCIBus *pci_bus;
     ISABus *isa_bus;
     PCII440FXState *i440fx_state;
@@ -97,7 +97,6 @@ static void pc_init1(MachineState *machine)
     MemoryRegion *ram_memory;
     MemoryRegion *pci_memory;
     MemoryRegion *rom_memory;
-    DeviceState *icc_bridge;
     PcGuestInfo *guest_info;
     ram_addr_t lowmem;
 
@@ -117,35 +116,30 @@ static void pc_init1(MachineState *machine)
     /* Handle the machine opt max-ram-below-4g.  It is basically doing
      * min(qemu limit, user limit).
      */
-    if (lowmem > pc_machine->max_ram_below_4g) {
-        lowmem = pc_machine->max_ram_below_4g;
+    if (lowmem > pcms->max_ram_below_4g) {
+        lowmem = pcms->max_ram_below_4g;
         if (machine->ram_size - lowmem > lowmem &&
             lowmem & ((1ULL << 30) - 1)) {
             error_report("Warning: Large machine and max_ram_below_4g(%"PRIu64
                          ") not a multiple of 1G; possible bad performance.",
-                         pc_machine->max_ram_below_4g);
+                         pcms->max_ram_below_4g);
         }
     }
 
     if (machine->ram_size >= lowmem) {
-        above_4g_mem_size = machine->ram_size - lowmem;
-        below_4g_mem_size = lowmem;
+        pcms->above_4g_mem_size = machine->ram_size - lowmem;
+        pcms->below_4g_mem_size = lowmem;
     } else {
-        above_4g_mem_size = 0;
-        below_4g_mem_size = machine->ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = machine->ram_size;
     }
 
-    if (xen_enabled() && xen_hvm_init(&below_4g_mem_size, &above_4g_mem_size,
-                                      &ram_memory) != 0) {
+    if (xen_enabled() && xen_hvm_init(pcms, &ram_memory) != 0) {
         fprintf(stderr, "xen hardware virtual machine initialisation failed\n");
         exit(1);
     }
 
-    icc_bridge = qdev_create(NULL, TYPE_ICC_BRIDGE);
-    object_property_add_child(qdev_get_machine(), "icc-bridge",
-                              OBJECT(icc_bridge), NULL);
-
-    pc_cpus_init(machine->cpu_model, icc_bridge);
+    pc_cpus_init(pcms);
 
     if (kvm_enabled() && kvmclock_enabled) {
         kvmclock_create();
@@ -160,7 +154,7 @@ static void pc_init1(MachineState *machine)
         rom_memory = system_memory;
     }
 
-    guest_info = pc_guest_info_init(below_4g_mem_size, above_4g_mem_size);
+    guest_info = pc_guest_info_init(pcms);
 
     guest_info->has_acpi_build = has_acpi_build;
     guest_info->legacy_acpi_table_size = legacy_acpi_table_size;
@@ -173,21 +167,17 @@ static void pc_init1(MachineState *machine)
         MachineClass *mc = MACHINE_GET_CLASS(machine);
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (i440FX + PIIX, 1996)",
-                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
+                            mc->name, smbios_legacy_mode, smbios_uuid_encoded,
+                            SMBIOS_ENTRY_POINT_21);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
-        pc_memory_init(machine, system_memory,
-                       below_4g_mem_size, above_4g_mem_size,
+        pc_memory_init(pcms, system_memory,
                        rom_memory, &ram_memory, guest_info);
     } else if (machine->kernel_filename != NULL) {
         /* For xen HVM direct kernel boot, load linux here */
-        xen_load_linux(machine->kernel_filename,
-                       machine->kernel_cmdline,
-                       machine->initrd_filename,
-                       below_4g_mem_size,
-                       guest_info);
+        xen_load_linux(pcms, guest_info);
     }
 
     gsi_state = g_malloc0(sizeof(*gsi_state));
@@ -200,10 +190,12 @@ static void pc_init1(MachineState *machine)
     }
 
     if (pci_enabled) {
-        pci_bus = i440fx_init(&i440fx_state, &piix3_devfn, &isa_bus, gsi,
+        pci_bus = i440fx_init(host_type,
+                              pci_type,
+                              &i440fx_state, &piix3_devfn, &isa_bus, gsi,
                               system_memory, system_io, machine->ram_size,
-                              below_4g_mem_size,
-                              above_4g_mem_size,
+                              pcms->below_4g_mem_size,
+                              pcms->above_4g_mem_size,
                               pci_memory, ram_memory);
     } else {
         pci_bus = NULL;
@@ -228,20 +220,19 @@ static void pc_init1(MachineState *machine)
     if (pci_enabled) {
         ioapic_init_gsi(gsi_state, "i440fx");
     }
-    qdev_init_nofail(icc_bridge);
 
     pc_register_ferr_irq(gsi[13]);
 
     pc_vga_init(isa_bus, pci_enabled ? pci_bus : NULL);
 
-    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
-    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
-        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    assert(pcms->vmport != ON_OFF_AUTO_MAX);
+    if (pcms->vmport == ON_OFF_AUTO_AUTO) {
+        pcms->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
     }
 
     /* init basic PC hardware */
     pc_basic_device_init(isa_bus, gsi, &rtc_state, true,
-                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0x4);
+                         (pcms->vmport != ON_OFF_AUTO_ON), 0x4);
 
     pc_nic_init(isa_bus, pci_bus);
 
@@ -271,8 +262,7 @@ static void pc_init1(MachineState *machine)
         }
     }
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 machine, idebus[0], idebus[1], rtc_state);
+    pc_cmos_init(pcms, idebus[0], idebus[1], rtc_state);
 
     if (pci_enabled && usb_enabled()) {
         pci_create_simple(pci_bus, piix3_devfn + 2, "piix3-usb-uhci");
@@ -286,13 +276,13 @@ static void pc_init1(MachineState *machine)
         /* TODO: Populate SPD eeprom data.  */
         smbus = piix4_pm_init(pci_bus, piix3_devfn + 3, 0xb100,
                               gsi[9], smi_irq,
-                              pc_machine_is_smm_enabled(pc_machine),
+                              pc_machine_is_smm_enabled(pcms),
                               &piix4_pm);
         smbus_eeprom_init(smbus, 8, NULL, 0);
 
         object_property_add_link(OBJECT(machine), PC_MACHINE_ACPI_DEVICE_PROP,
                                  TYPE_HOTPLUG_HANDLER,
-                                 (Object **)&pc_machine->acpi_dev,
+                                 (Object **)&pcms->acpi_dev,
                                  object_property_allow_set_link,
                                  OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
         object_property_set_link(OBJECT(machine), OBJECT(piix4_pm),
@@ -304,6 +294,13 @@ static void pc_init1(MachineState *machine)
     }
 }
 
+/* Looking for a pc_compat_2_4() function? It doesn't exist.
+ * pc_compat_*() functions that run on machine-init time and
+ * change global QEMU state are deprecated. Please don't create
+ * one, and implement any pc-*-2.4 (and newer) compat code in
+ * HW_COMPAT_*, PC_COMPAT_*, or * pc_*_machine_options().
+ */
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
@@ -319,24 +316,6 @@ static void pc_compat_2_2(MachineState *machine)
 {
     pc_compat_2_3(machine);
     rsdp_in_ram = false;
-    x86_cpu_compat_set_features("kvm64", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("kvm32", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Conroe", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Penryn", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Nehalem", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("SandyBridge", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G1", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G2", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G3", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G4", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G5", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
     machine->suppress_vmdesc = true;
 }
 
@@ -346,9 +325,7 @@ static void pc_compat_2_1(MachineState *machine)
 
     pc_compat_2_2(machine);
     smbios_uuid_encoded = false;
-    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
+    x86_cpu_change_kvm_default("svm", NULL);
     pcms->enforce_aligned_dimm = false;
 }
 
@@ -384,7 +361,7 @@ static void pc_compat_1_7(MachineState *machine)
     gigabyte_align = false;
     option_rom_has_mr = true;
     legacy_acpi_table_size = 6414;
-    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
+    x86_cpu_change_kvm_default("x2apic", NULL);
 }
 
 static void pc_compat_1_6(MachineState *machine)
@@ -402,8 +379,6 @@ static void pc_compat_1_5(MachineState *machine)
 static void pc_compat_1_4(MachineState *machine)
 {
     pc_compat_1_5(machine);
-    x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
 }
 
 static void pc_compat_1_3(MachineState *machine)
@@ -416,7 +391,7 @@ static void pc_compat_1_3(MachineState *machine)
 static void pc_compat_1_2(MachineState *machine)
 {
     pc_compat_1_3(machine);
-    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, 1 << KVM_FEATURE_PV_EOI);
+    x86_cpu_change_kvm_default("kvm-pv-eoi", NULL);
 }
 
 /* PC compat function for pc-0.10 to pc-0.13 */
@@ -439,17 +414,32 @@ static void pc_init_isa(MachineState *machine)
     if (!machine->cpu_model) {
         machine->cpu_model = "486";
     }
-    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, 1 << KVM_FEATURE_PV_EOI);
+    x86_cpu_change_kvm_default("kvm-pv-eoi", NULL);
     enable_compat_apic_id_mode();
-    pc_init1(machine);
+    pc_init1(machine, TYPE_I440FX_PCI_HOST_BRIDGE, TYPE_I440FX_PCI_DEVICE);
 }
 
 #ifdef CONFIG_XEN
+static void pc_xen_hvm_init_pci(MachineState *machine)
+{
+    const char *pci_type = has_igd_gfx_passthru ?
+                TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE : TYPE_I440FX_PCI_DEVICE;
+
+    pc_init1(machine,
+             TYPE_I440FX_PCI_HOST_BRIDGE,
+             pci_type);
+}
+
 static void pc_xen_hvm_init(MachineState *machine)
 {
     PCIBus *bus;
 
-    pc_init1(machine);
+    if (!xen_enabled()) {
+        error_report("xenfv machine requires the xen accelerator");
+        exit(1);
+    }
+
+    pc_xen_hvm_init_pci(machine);
 
     bus = pci_find_primary_bus();
     if (bus != NULL) {
@@ -465,27 +455,42 @@ static void pc_xen_hvm_init(MachineState *machine)
         if (compat) { \
             compat(machine); \
         } \
-        pc_init1(machine); \
+        pc_init1(machine, TYPE_I440FX_PCI_HOST_BRIDGE, \
+                 TYPE_I440FX_PCI_DEVICE); \
     } \
     DEFINE_PC_MACHINE(suffix, name, pc_init_##suffix, optionfn)
 
 static void pc_i440fx_machine_options(MachineClass *m)
 {
-    pc_default_machine_options(m);
     m->family = "pc_piix";
     m->desc = "Standard PC (i440FX + PIIX, 1996)";
     m->hot_add_cpu = pc_hot_add_cpu;
+    m->default_machine_opts = "firmware=bios-256k.bin";
+    m->default_display = "std";
 }
 
-static void pc_i440fx_2_4_machine_options(MachineClass *m)
+static void pc_i440fx_2_5_machine_options(MachineClass *m)
 {
     pc_i440fx_machine_options(m);
-    m->default_machine_opts = "firmware=bios-256k.bin";
-    m->default_display = "std";
     m->alias = "pc";
     m->is_default = 1;
 }
 
+DEFINE_I440FX_MACHINE(v2_5, "pc-i440fx-2.5", NULL,
+                      pc_i440fx_2_5_machine_options);
+
+
+static void pc_i440fx_2_4_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_5_machine_options(m);
+    m->hw_version = "2.4.0";
+    m->alias = NULL;
+    m->is_default = 0;
+    pcmc->broken_reserved_end = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
+}
+
 DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
                       pc_i440fx_2_4_machine_options)
 
@@ -493,6 +498,7 @@ DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
 static void pc_i440fx_2_3_machine_options(MachineClass *m)
 {
     pc_i440fx_2_4_machine_options(m);
+    m->hw_version = "2.3.0";
     m->alias = NULL;
     m->is_default = 0;
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_3);
@@ -505,6 +511,7 @@ DEFINE_I440FX_MACHINE(v2_3, "pc-i440fx-2.3", pc_compat_2_3,
 static void pc_i440fx_2_2_machine_options(MachineClass *m)
 {
     pc_i440fx_2_3_machine_options(m);
+    m->hw_version = "2.2.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_2);
 }
 
@@ -515,6 +522,7 @@ DEFINE_I440FX_MACHINE(v2_2, "pc-i440fx-2.2", pc_compat_2_2,
 static void pc_i440fx_2_1_machine_options(MachineClass *m)
 {
     pc_i440fx_2_2_machine_options(m);
+    m->hw_version = "2.1.0";
     m->default_display = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_1);
 }
@@ -527,6 +535,7 @@ DEFINE_I440FX_MACHINE(v2_1, "pc-i440fx-2.1", pc_compat_2_1,
 static void pc_i440fx_2_0_machine_options(MachineClass *m)
 {
     pc_i440fx_2_1_machine_options(m);
+    m->hw_version = "2.0.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_0);
 }
 
@@ -537,6 +546,7 @@ DEFINE_I440FX_MACHINE(v2_0, "pc-i440fx-2.0", pc_compat_2_0,
 static void pc_i440fx_1_7_machine_options(MachineClass *m)
 {
     pc_i440fx_2_0_machine_options(m);
+    m->hw_version = "1.7.0";
     m->default_machine_opts = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_7);
 }
@@ -548,6 +558,7 @@ DEFINE_I440FX_MACHINE(v1_7, "pc-i440fx-1.7", pc_compat_1_7,
 static void pc_i440fx_1_6_machine_options(MachineClass *m)
 {
     pc_i440fx_1_7_machine_options(m);
+    m->hw_version = "1.6.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_6);
 }
 
@@ -558,6 +569,7 @@ DEFINE_I440FX_MACHINE(v1_6, "pc-i440fx-1.6", pc_compat_1_6,
 static void pc_i440fx_1_5_machine_options(MachineClass *m)
 {
     pc_i440fx_1_6_machine_options(m);
+    m->hw_version = "1.5.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_5);
 }
 
@@ -568,6 +580,7 @@ DEFINE_I440FX_MACHINE(v1_5, "pc-i440fx-1.5", pc_compat_1_5,
 static void pc_i440fx_1_4_machine_options(MachineClass *m)
 {
     pc_i440fx_1_5_machine_options(m);
+    m->hw_version = "1.4.0";
     m->hot_add_cpu = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_4);
 }
@@ -600,6 +613,7 @@ DEFINE_I440FX_MACHINE(v1_4, "pc-i440fx-1.4", pc_compat_1_4,
 static void pc_i440fx_1_3_machine_options(MachineClass *m)
 {
     pc_i440fx_1_4_machine_options(m);
+    m->hw_version = "1.3.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_3);
 }
 
@@ -638,6 +652,7 @@ DEFINE_I440FX_MACHINE(v1_3, "pc-1.3", pc_compat_1_3,
 static void pc_i440fx_1_2_machine_options(MachineClass *m)
 {
     pc_i440fx_1_3_machine_options(m);
+    m->hw_version = "1.2.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_2);
 }
 
@@ -680,6 +695,7 @@ DEFINE_I440FX_MACHINE(v1_2, "pc-1.2", pc_compat_1_2,
 static void pc_i440fx_1_1_machine_options(MachineClass *m)
 {
     pc_i440fx_1_2_machine_options(m);
+    m->hw_version = "1.1.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_1);
 }
 
@@ -906,10 +922,121 @@ static void pc_i440fx_0_10_machine_options(MachineClass *m)
 DEFINE_I440FX_MACHINE(v0_10, "pc-0.10", pc_compat_0_13,
                       pc_i440fx_0_10_machine_options);
 
+typedef struct {
+    uint16_t gpu_device_id;
+    uint16_t pch_device_id;
+    uint8_t pch_revision_id;
+} IGDDeviceIDInfo;
+
+/* In real world different GPU should have different PCH. But actually
+ * the different PCH DIDs likely map to different PCH SKUs. We do the
+ * same thing for the GPU. For PCH, the different SKUs are going to be
+ * all the same silicon design and implementation, just different
+ * features turn on and off with fuses. The SW interfaces should be
+ * consistent across all SKUs in a given family (eg LPT). But just same
+ * features may not be supported.
+ *
+ * Most of these different PCH features probably don't matter to the
+ * Gfx driver, but obviously any difference in display port connections
+ * will so it should be fine with any PCH in case of passthrough.
+ *
+ * So currently use one PCH version, 0x8c4e, to cover all HSW(Haswell)
+ * scenarios, 0x9cc3 for BDW(Broadwell).
+ */
+static const IGDDeviceIDInfo igd_combo_id_infos[] = {
+    /* HSW Classic */
+    {0x0402, 0x8c4e, 0x04}, /* HSWGT1D, HSWD_w7 */
+    {0x0406, 0x8c4e, 0x04}, /* HSWGT1M, HSWM_w7 */
+    {0x0412, 0x8c4e, 0x04}, /* HSWGT2D, HSWD_w7 */
+    {0x0416, 0x8c4e, 0x04}, /* HSWGT2M, HSWM_w7 */
+    {0x041E, 0x8c4e, 0x04}, /* HSWGT15D, HSWD_w7 */
+    /* HSW ULT */
+    {0x0A06, 0x8c4e, 0x04}, /* HSWGT1UT, HSWM_w7 */
+    {0x0A16, 0x8c4e, 0x04}, /* HSWGT2UT, HSWM_w7 */
+    {0x0A26, 0x8c4e, 0x06}, /* HSWGT3UT, HSWM_w7 */
+    {0x0A2E, 0x8c4e, 0x04}, /* HSWGT3UT28W, HSWM_w7 */
+    {0x0A1E, 0x8c4e, 0x04}, /* HSWGT2UX, HSWM_w7 */
+    {0x0A0E, 0x8c4e, 0x04}, /* HSWGT1ULX, HSWM_w7 */
+    /* HSW CRW */
+    {0x0D26, 0x8c4e, 0x04}, /* HSWGT3CW, HSWM_w7 */
+    {0x0D22, 0x8c4e, 0x04}, /* HSWGT3CWDT, HSWD_w7 */
+    /* HSW Server */
+    {0x041A, 0x8c4e, 0x04}, /* HSWSVGT2, HSWD_w7 */
+    /* HSW SRVR */
+    {0x040A, 0x8c4e, 0x04}, /* HSWSVGT1, HSWD_w7 */
+    /* BSW */
+    {0x1606, 0x9cc3, 0x03}, /* BDWULTGT1, BDWM_w7 */
+    {0x1616, 0x9cc3, 0x03}, /* BDWULTGT2, BDWM_w7 */
+    {0x1626, 0x9cc3, 0x03}, /* BDWULTGT3, BDWM_w7 */
+    {0x160E, 0x9cc3, 0x03}, /* BDWULXGT1, BDWM_w7 */
+    {0x161E, 0x9cc3, 0x03}, /* BDWULXGT2, BDWM_w7 */
+    {0x1602, 0x9cc3, 0x03}, /* BDWHALOGT1, BDWM_w7 */
+    {0x1612, 0x9cc3, 0x03}, /* BDWHALOGT2, BDWM_w7 */
+    {0x1622, 0x9cc3, 0x03}, /* BDWHALOGT3, BDWM_w7 */
+    {0x162B, 0x9cc3, 0x03}, /* BDWHALO28W, BDWM_w7 */
+    {0x162A, 0x9cc3, 0x03}, /* BDWGT3WRKS, BDWM_w7 */
+    {0x162D, 0x9cc3, 0x03}, /* BDWGT3SRVR, BDWM_w7 */
+};
+
+static void isa_bridge_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    dc->desc        = "ISA bridge faked to support IGD PT";
+    k->vendor_id    = PCI_VENDOR_ID_INTEL;
+    k->class_id     = PCI_CLASS_BRIDGE_ISA;
+};
+
+static TypeInfo isa_bridge_info = {
+    .name          = "igd-passthrough-isa-bridge",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIDevice),
+    .class_init = isa_bridge_class_init,
+};
+
+static void pt_graphics_register_types(void)
+{
+    type_register_static(&isa_bridge_info);
+}
+type_init(pt_graphics_register_types)
+
+void igd_passthrough_isa_bridge_create(PCIBus *bus, uint16_t gpu_dev_id)
+{
+    struct PCIDevice *bridge_dev;
+    int i, num;
+    uint16_t pch_dev_id = 0xffff;
+    uint8_t pch_rev_id;
+
+    num = ARRAY_SIZE(igd_combo_id_infos);
+    for (i = 0; i < num; i++) {
+        if (gpu_dev_id == igd_combo_id_infos[i].gpu_device_id) {
+            pch_dev_id = igd_combo_id_infos[i].pch_device_id;
+            pch_rev_id = igd_combo_id_infos[i].pch_revision_id;
+        }
+    }
+
+    if (pch_dev_id == 0xffff) {
+        return;
+    }
+
+    /* Currently IGD drivers always need to access PCH by 1f.0. */
+    bridge_dev = pci_create_simple(bus, PCI_DEVFN(0x1f, 0),
+                                   "igd-passthrough-isa-bridge");
+
+    /*
+     * Note that vendor id is always PCI_VENDOR_ID_INTEL.
+     */
+    if (!bridge_dev) {
+        fprintf(stderr, "set igd-passthrough-isa-bridge failed!\n");
+        return;
+    }
+    pci_config_set_device_id(bridge_dev->config, pch_dev_id);
+    pci_config_set_revision(bridge_dev->config, pch_rev_id);
+}
 
 static void isapc_machine_options(MachineClass *m)
 {
-    pc_common_machine_options(m);
     m->desc = "ISA-only PC";
     m->max_cpus = 1;
 }
@@ -921,7 +1048,6 @@ DEFINE_PC_MACHINE(isapc, "isapc", pc_init_isa,
 #ifdef CONFIG_XEN
 static void xenfv_machine_options(MachineClass *m)
 {
-    pc_common_machine_options(m);
     m->desc = "Xen Fully-virtualized PC";
     m->max_cpus = HVM_MAX_VCPUS;
     m->default_machine_opts = "accel=xen";
diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
index 974aead5a..133bc68fa 100644
--- a/hw/i386/pc_q35.c
+++ b/hw/i386/pc_q35.c
@@ -39,11 +39,10 @@
 #include "hw/pci-host/q35.h"
 #include "exec/address-spaces.h"
 #include "hw/i386/ich9.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/ide/pci.h"
 #include "hw/ide/ahci.h"
 #include "hw/usb.h"
-#include "hw/cpu/icc_bus.h"
 #include "qemu/error-report.h"
 #include "migration/migration.h"
 
@@ -65,8 +64,7 @@ static bool has_reserved_memory = true;
 /* PC hardware initialisation */
 static void pc_q35_init(MachineState *machine)
 {
-    PCMachineState *pc_machine = PC_MACHINE(machine);
-    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+    PCMachineState *pcms = PC_MACHINE(machine);
     Q35PCIHost *q35_host;
     PCIHostState *phb;
     PCIBus *host_bus;
@@ -84,7 +82,6 @@ static void pc_q35_init(MachineState *machine)
     int i;
     ICH9LPCState *ich9_lpc;
     PCIDevice *ahci;
-    DeviceState *icc_bridge;
     PcGuestInfo *guest_info;
     ram_addr_t lowmem;
     DriveInfo *hd[MAX_SATA_PORTS];
@@ -108,35 +105,30 @@ static void pc_q35_init(MachineState *machine)
     /* Handle the machine opt max-ram-below-4g.  It is basically doing
      * min(qemu limit, user limit).
      */
-    if (lowmem > pc_machine->max_ram_below_4g) {
-        lowmem = pc_machine->max_ram_below_4g;
+    if (lowmem > pcms->max_ram_below_4g) {
+        lowmem = pcms->max_ram_below_4g;
         if (machine->ram_size - lowmem > lowmem &&
             lowmem & ((1ULL << 30) - 1)) {
             error_report("Warning: Large machine and max_ram_below_4g(%"PRIu64
                          ") not a multiple of 1G; possible bad performance.",
-                         pc_machine->max_ram_below_4g);
+                         pcms->max_ram_below_4g);
         }
     }
 
     if (machine->ram_size >= lowmem) {
-        above_4g_mem_size = machine->ram_size - lowmem;
-        below_4g_mem_size = lowmem;
+        pcms->above_4g_mem_size = machine->ram_size - lowmem;
+        pcms->below_4g_mem_size = lowmem;
     } else {
-        above_4g_mem_size = 0;
-        below_4g_mem_size = machine->ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = machine->ram_size;
     }
 
-    if (xen_enabled() && xen_hvm_init(&below_4g_mem_size, &above_4g_mem_size,
-                                      &ram_memory) != 0) {
+    if (xen_enabled() && xen_hvm_init(pcms, &ram_memory) != 0) {
         fprintf(stderr, "xen hardware virtual machine initialisation failed\n");
         exit(1);
     }
 
-    icc_bridge = qdev_create(NULL, TYPE_ICC_BRIDGE);
-    object_property_add_child(qdev_get_machine(), "icc-bridge",
-                              OBJECT(icc_bridge), NULL);
-
-    pc_cpus_init(machine->cpu_model, icc_bridge);
+    pc_cpus_init(pcms);
     pc_acpi_init("q35-acpi-dsdt.aml");
 
     kvmclock_create();
@@ -151,7 +143,7 @@ static void pc_q35_init(MachineState *machine)
         rom_memory = get_system_memory();
     }
 
-    guest_info = pc_guest_info_init(below_4g_mem_size, above_4g_mem_size);
+    guest_info = pc_guest_info_init(pcms);
     guest_info->isapc_ram_fw = false;
     guest_info->has_acpi_build = has_acpi_build;
     guest_info->has_reserved_memory = has_reserved_memory;
@@ -165,13 +157,13 @@ static void pc_q35_init(MachineState *machine)
     if (smbios_defaults) {
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (Q35 + ICH9, 2009)",
-                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
+                            mc->name, smbios_legacy_mode, smbios_uuid_encoded,
+                            SMBIOS_ENTRY_POINT_21);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
-        pc_memory_init(machine, get_system_memory(),
-                       below_4g_mem_size, above_4g_mem_size,
+        pc_memory_init(pcms, get_system_memory(),
                        rom_memory, &ram_memory, guest_info);
     }
 
@@ -193,8 +185,8 @@ static void pc_q35_init(MachineState *machine)
     q35_host->mch.pci_address_space = pci_memory;
     q35_host->mch.system_memory = get_system_memory();
     q35_host->mch.address_space_io = get_system_io();
-    q35_host->mch.below_4g_mem_size = below_4g_mem_size;
-    q35_host->mch.above_4g_mem_size = above_4g_mem_size;
+    q35_host->mch.below_4g_mem_size = pcms->below_4g_mem_size;
+    q35_host->mch.above_4g_mem_size = pcms->above_4g_mem_size;
     q35_host->mch.guest_info = guest_info;
     /* pci */
     qdev_init_nofail(DEVICE(q35_host));
@@ -207,7 +199,7 @@ static void pc_q35_init(MachineState *machine)
 
     object_property_add_link(OBJECT(machine), PC_MACHINE_ACPI_DEVICE_PROP,
                              TYPE_HOTPLUG_HANDLER,
-                             (Object **)&pc_machine->acpi_dev,
+                             (Object **)&pcms->acpi_dev,
                              object_property_allow_set_link,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
     object_property_set_link(OBJECT(machine), OBJECT(lpc),
@@ -238,21 +230,20 @@ static void pc_q35_init(MachineState *machine)
     if (pci_enabled) {
         ioapic_init_gsi(gsi_state, "q35");
     }
-    qdev_init_nofail(icc_bridge);
 
     pc_register_ferr_irq(gsi[13]);
 
-    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
-    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
-        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    assert(pcms->vmport != ON_OFF_AUTO_MAX);
+    if (pcms->vmport == ON_OFF_AUTO_AUTO) {
+        pcms->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
     }
 
     /* init basic PC hardware */
     pc_basic_device_init(isa_bus, gsi, &rtc_state, !mc->no_floppy,
-                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0xff0104);
+                         (pcms->vmport != ON_OFF_AUTO_ON), 0xff0104);
 
     /* connect pm stuff to lpc */
-    ich9_lpc_pm_init(lpc, pc_machine_is_smm_enabled(pc_machine), !mc->no_tco);
+    ich9_lpc_pm_init(lpc, pc_machine_is_smm_enabled(pcms), !mc->no_tco);
 
     /* ahci and SATA device, for q35 1 ahci controller is built-in */
     ahci = pci_create_simple_multifunction(host_bus,
@@ -276,8 +267,7 @@ static void pc_q35_init(MachineState *machine)
                                     0xb100),
                       8, NULL, 0);
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 machine, idebus[0], idebus[1], rtc_state);
+    pc_cmos_init(pcms, idebus[0], idebus[1], rtc_state);
 
     /* the rest devices to which pci devfn is automatically assigned */
     pc_vga_init(isa_bus, host_bus);
@@ -287,6 +277,13 @@ static void pc_q35_init(MachineState *machine)
     }
 }
 
+/* Looking for a pc_compat_2_4() function? It doesn't exist.
+ * pc_compat_*() functions that run on machine-init time and
+ * change global QEMU state are deprecated. Please don't create
+ * one, and implement any pc-*-2.4 (and newer) compat code in
+ * HW_COMPAT_*, PC_COMPAT_*, or * pc_*_machine_options().
+ */
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
@@ -302,24 +299,6 @@ static void pc_compat_2_2(MachineState *machine)
 {
     pc_compat_2_3(machine);
     rsdp_in_ram = false;
-    x86_cpu_compat_set_features("kvm64", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("kvm32", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Conroe", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Penryn", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Nehalem", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("SandyBridge", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G1", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G2", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G3", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G4", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G5", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
     machine->suppress_vmdesc = true;
 }
 
@@ -330,9 +309,7 @@ static void pc_compat_2_1(MachineState *machine)
     pc_compat_2_2(machine);
     pcms->enforce_aligned_dimm = false;
     smbios_uuid_encoded = false;
-    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
+    x86_cpu_change_kvm_default("svm", NULL);
 }
 
 static void pc_compat_2_0(MachineState *machine)
@@ -349,7 +326,7 @@ static void pc_compat_1_7(MachineState *machine)
     smbios_defaults = false;
     gigabyte_align = false;
     option_rom_has_mr = true;
-    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
+    x86_cpu_change_kvm_default("x2apic", NULL);
 }
 
 static void pc_compat_1_6(MachineState *machine)
@@ -367,8 +344,6 @@ static void pc_compat_1_5(MachineState *machine)
 static void pc_compat_1_4(MachineState *machine)
 {
     pc_compat_1_5(machine);
-    x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
 }
 
 #define DEFINE_Q35_MACHINE(suffix, name, compatfn, optionfn) \
@@ -385,23 +360,35 @@ static void pc_compat_1_4(MachineState *machine)
 
 static void pc_q35_machine_options(MachineClass *m)
 {
-    pc_default_machine_options(m);
     m->family = "pc_q35";
     m->desc = "Standard PC (Q35 + ICH9, 2009)";
     m->hot_add_cpu = pc_hot_add_cpu;
     m->units_per_default_bus = 1;
-}
-
-static void pc_q35_2_4_machine_options(MachineClass *m)
-{
-    pc_q35_machine_options(m);
     m->default_machine_opts = "firmware=bios-256k.bin";
     m->default_display = "std";
     m->no_floppy = 1;
     m->no_tco = 0;
+}
+
+static void pc_q35_2_5_machine_options(MachineClass *m)
+{
+    pc_q35_machine_options(m);
     m->alias = "q35";
 }
 
+DEFINE_Q35_MACHINE(v2_5, "pc-q35-2.5", NULL,
+                   pc_q35_2_5_machine_options);
+
+static void pc_q35_2_4_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_q35_2_5_machine_options(m);
+    m->hw_version = "2.4.0";
+    m->alias = NULL;
+    pcmc->broken_reserved_end = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
+}
+
 DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
                    pc_q35_2_4_machine_options);
 
@@ -409,6 +396,7 @@ DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
 static void pc_q35_2_3_machine_options(MachineClass *m)
 {
     pc_q35_2_4_machine_options(m);
+    m->hw_version = "2.3.0";
     m->no_floppy = 0;
     m->no_tco = 1;
     m->alias = NULL;
@@ -422,6 +410,7 @@ DEFINE_Q35_MACHINE(v2_3, "pc-q35-2.3", pc_compat_2_3,
 static void pc_q35_2_2_machine_options(MachineClass *m)
 {
     pc_q35_2_3_machine_options(m);
+    m->hw_version = "2.2.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_2);
 }
 
@@ -432,6 +421,7 @@ DEFINE_Q35_MACHINE(v2_2, "pc-q35-2.2", pc_compat_2_2,
 static void pc_q35_2_1_machine_options(MachineClass *m)
 {
     pc_q35_2_2_machine_options(m);
+    m->hw_version = "2.1.0";
     m->default_display = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_1);
 }
@@ -443,6 +433,7 @@ DEFINE_Q35_MACHINE(v2_1, "pc-q35-2.1", pc_compat_2_1,
 static void pc_q35_2_0_machine_options(MachineClass *m)
 {
     pc_q35_2_1_machine_options(m);
+    m->hw_version = "2.0.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_0);
 }
 
@@ -453,6 +444,7 @@ DEFINE_Q35_MACHINE(v2_0, "pc-q35-2.0", pc_compat_2_0,
 static void pc_q35_1_7_machine_options(MachineClass *m)
 {
     pc_q35_2_0_machine_options(m);
+    m->hw_version = "1.7.0";
     m->default_machine_opts = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_7);
 }
@@ -464,6 +456,7 @@ DEFINE_Q35_MACHINE(v1_7, "pc-q35-1.7", pc_compat_1_7,
 static void pc_q35_1_6_machine_options(MachineClass *m)
 {
     pc_q35_machine_options(m);
+    m->hw_version = "1.6.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_6);
 }
 
@@ -474,6 +467,7 @@ DEFINE_Q35_MACHINE(v1_6, "pc-q35-1.6", pc_compat_1_6,
 static void pc_q35_1_5_machine_options(MachineClass *m)
 {
     pc_q35_1_6_machine_options(m);
+    m->hw_version = "1.5.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_5);
 }
 
@@ -484,6 +478,7 @@ DEFINE_Q35_MACHINE(v1_5, "pc-q35-1.5", pc_compat_1_5,
 static void pc_q35_1_4_machine_options(MachineClass *m)
 {
     pc_q35_1_5_machine_options(m);
+    m->hw_version = "1.4.0";
     m->hot_add_cpu = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_4);
 }
diff --git a/hw/i386/pc_sysfw.c b/hw/i386/pc_sysfw.c
index 662d99768..579461fa1 100644
--- a/hw/i386/pc_sysfw.c
+++ b/hw/i386/pc_sysfw.c
@@ -56,7 +56,7 @@ static void pc_isa_bios_init(MemoryRegion *rom_memory,
     isa_bios_size = MIN(flash_size, 128 * 1024);
     isa_bios = g_malloc(sizeof(*isa_bios));
     memory_region_init_ram(isa_bios, NULL, "isa-bios", isa_bios_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(isa_bios);
     memory_region_add_subregion_overlap(rom_memory,
                                         0x100000 - isa_bios_size,
@@ -193,7 +193,7 @@ static void old_pc_system_rom_init(MemoryRegion *rom_memory, bool isapc_ram_fw)
         goto bios_error;
     }
     bios = g_malloc(sizeof(*bios));
-    memory_region_init_ram(bios, NULL, "pc.bios", bios_size, &error_abort);
+    memory_region_init_ram(bios, NULL, "pc.bios", bios_size, &error_fatal);
     vmstate_register_ram_global(bios);
     if (!isapc_ram_fw) {
         memory_region_set_readonly(bios, true);
diff --git a/hw/i386/pci-assign-load-rom.c b/hw/i386/pci-assign-load-rom.c
new file mode 100644
index 000000000..e40b586b9
--- /dev/null
+++ b/hw/i386/pci-assign-load-rom.c
@@ -0,0 +1,87 @@
+/*
+ * This is splited from hw/i386/kvm/pci-assign.c
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "qemu/error-report.h"
+#include "ui/console.h"
+#include "hw/loader.h"
+#include "monitor/monitor.h"
+#include "qemu/range.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci-assign.h"
+
+/*
+ * Scan the assigned devices for the devices that have an option ROM, and then
+ * load the corresponding ROM data to RAM. If an error occurs while loading an
+ * option ROM, we just ignore that option ROM and continue with the next one.
+ */
+void *pci_assign_dev_load_option_rom(PCIDevice *dev, struct Object *owner,
+                                     int *size, unsigned int domain,
+                                     unsigned int bus, unsigned int slot,
+                                     unsigned int function)
+{
+    char name[32], rom_file[64];
+    FILE *fp;
+    uint8_t val;
+    struct stat st;
+    void *ptr = NULL;
+
+    /* If loading ROM from file, pci handles it */
+    if (dev->romfile || !dev->rom_bar) {
+        return NULL;
+    }
+
+    snprintf(rom_file, sizeof(rom_file),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
+             domain, bus, slot, function);
+
+    if (stat(rom_file, &st)) {
+        return NULL;
+    }
+
+    /* Write "1" to the ROM file to enable it */
+    fp = fopen(rom_file, "r+");
+    if (fp == NULL) {
+        error_report("pci-assign: Cannot open %s: %s", rom_file, strerror(errno));
+        return NULL;
+    }
+    val = 1;
+    if (fwrite(&val, 1, 1, fp) != 1) {
+        goto close_rom;
+    }
+    fseek(fp, 0, SEEK_SET);
+
+    snprintf(name, sizeof(name), "%s.rom", object_get_typename(owner));
+    memory_region_init_ram(&dev->rom, owner, name, st.st_size, &error_abort);
+    vmstate_register_ram(&dev->rom, &dev->qdev);
+    ptr = memory_region_get_ram_ptr(&dev->rom);
+    memset(ptr, 0xff, st.st_size);
+
+    if (!fread(ptr, 1, st.st_size, fp)) {
+        error_report("pci-assign: Cannot read from host %s", rom_file);
+        error_printf("Device option ROM contents are probably invalid "
+                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
+                     "or load from file with romfile=\n");
+        goto close_rom;
+    }
+
+    pci_register_bar(dev, PCI_ROM_SLOT, 0, &dev->rom);
+    dev->has_rom = true;
+    *size = st.st_size;
+close_rom:
+    /* Write "0" to disable ROM */
+    fseek(fp, 0, SEEK_SET);
+    val = 0;
+    if (!fwrite(&val, 1, 1, fp)) {
+        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
+    }
+    fclose(fp);
+
+    return ptr;
+}
diff --git a/hw/i386/q35-acpi-dsdt.dsl b/hw/i386/q35-acpi-dsdt.dsl
index 16eaca3fa..7be7b37b8 100644
--- a/hw/i386/q35-acpi-dsdt.dsl
+++ b/hw/i386/q35-acpi-dsdt.dsl
@@ -22,6 +22,7 @@
  * Based on acpi-dsdt.dsl, but heavily modified for q35 chipset.
  */
 
+
 ACPI_EXTRACT_ALL_CODE Q35AcpiDsdtAmlCode
 
 DefinitionBlock (
diff --git a/hw/i386/xen/xen_platform.c b/hw/i386/xen/xen_platform.c
index 28b324a6f..de83f4e3e 100644
--- a/hw/i386/xen/xen_platform.c
+++ b/hw/i386/xen/xen_platform.c
@@ -23,8 +23,6 @@
  * THE SOFTWARE.
  */
 
-#include <assert.h>
-
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/ide.h"
@@ -35,6 +33,7 @@
 #include "trace.h"
 #include "exec/address-spaces.h"
 #include "sysemu/block-backend.h"
+#include "qemu/error-report.h"
 
 #include <xenguest.h>
 
@@ -384,11 +383,17 @@ static const VMStateDescription vmstate_xen_platform = {
     }
 };
 
-static int xen_platform_initfn(PCIDevice *dev)
+static void xen_platform_realize(PCIDevice *dev, Error **errp)
 {
     PCIXenPlatformState *d = XEN_PLATFORM(dev);
     uint8_t *pci_conf;
 
+    /* Device will crash on reset if xen is not initialized */
+    if (!xen_enabled()) {
+        error_setg(errp, "xen-platform device requires the Xen accelerator");
+        return;
+    }
+
     pci_conf = dev->config;
 
     pci_set_word(pci_conf + PCI_COMMAND, PCI_COMMAND_IO | PCI_COMMAND_MEMORY);
@@ -406,8 +411,6 @@ static int xen_platform_initfn(PCIDevice *dev)
                      &d->mmio_bar);
 
     platform_fixed_ioport_init(d);
-
-    return 0;
 }
 
 static void platform_reset(DeviceState *dev)
@@ -422,7 +425,7 @@ static void xen_platform_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = xen_platform_initfn;
+    k->realize = xen_platform_realize;
     k->vendor_id = PCI_VENDOR_ID_XEN;
     k->device_id = PCI_DEVICE_ID_XEN_PLATFORM;
     k->class_id = PCI_CLASS_OTHERS << 8 | 0x80;
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index 378ad60c3..cdc92997b 100644
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -25,7 +25,6 @@
 #include <hw/pci/msi.h>
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
-#include <hw/sysbus.h>
 
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
@@ -47,7 +46,7 @@ do { \
 static void check_cmd(AHCIState *s, int port);
 static int handle_cmd(AHCIState *s, int port, uint8_t slot);
 static void ahci_reset_port(AHCIState *s, int port);
-static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis);
+static bool ahci_write_fis_d2h(AHCIDevice *ad);
 static void ahci_init_d2h(AHCIDevice *ad);
 static int ahci_dma_prepare_buf(IDEDMA *dma, int32_t limit);
 static bool ahci_map_clb_address(AHCIDevice *ad);
@@ -121,9 +120,9 @@ static uint32_t  ahci_port_read(AHCIState *s, int port, int offset)
 
 static void ahci_irq_raise(AHCIState *s, AHCIDevice *dev)
 {
-    AHCIPCIState *d = container_of(s, AHCIPCIState, ahci);
-    PCIDevice *pci_dev =
-        (PCIDevice *)object_dynamic_cast(OBJECT(d), TYPE_PCI_DEVICE);
+    DeviceState *dev_state = s->container;
+    PCIDevice *pci_dev = (PCIDevice *) object_dynamic_cast(OBJECT(dev_state),
+                                                           TYPE_PCI_DEVICE);
 
     DPRINTF(0, "raise irq\n");
 
@@ -136,9 +135,9 @@ static void ahci_irq_raise(AHCIState *s, AHCIDevice *dev)
 
 static void ahci_irq_lower(AHCIState *s, AHCIDevice *dev)
 {
-    AHCIPCIState *d = container_of(s, AHCIPCIState, ahci);
-    PCIDevice *pci_dev =
-        (PCIDevice *)object_dynamic_cast(OBJECT(d), TYPE_PCI_DEVICE);
+    DeviceState *dev_state = s->container;
+    PCIDevice *pci_dev = (PCIDevice *) object_dynamic_cast(OBJECT(dev_state),
+                                                           TYPE_PCI_DEVICE);
 
     DPRINTF(0, "lower irq\n");
 
@@ -296,7 +295,6 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
             if ((pr->cmd & PORT_CMD_FIS_ON) &&
                 !s->dev[port].init_d2h_sent) {
                 ahci_init_d2h(&s->dev[port]);
-                s->dev[port].init_d2h_sent = true;
             }
 
             check_cmd(s, port);
@@ -380,17 +378,23 @@ static uint64_t ahci_mem_read(void *opaque, hwaddr addr, unsigned size)
     int ofst = addr - aligned;
     uint64_t lo = ahci_mem_read_32(opaque, aligned);
     uint64_t hi;
+    uint64_t val;
 
     /* if < 8 byte read does not cross 4 byte boundary */
     if (ofst + size <= 4) {
-        return lo >> (ofst * 8);
+        val = lo >> (ofst * 8);
+    } else {
+        g_assert_cmpint(size, >, 1);
+
+        /* If the 64bit read is unaligned, we will produce undefined
+         * results. AHCI does not support unaligned 64bit reads. */
+        hi = ahci_mem_read_32(opaque, aligned + 4);
+        val = (hi << 32 | lo) >> (ofst * 8);
     }
-    g_assert_cmpint(size, >, 1);
 
-    /* If the 64bit read is unaligned, we will produce undefined
-     * results. AHCI does not support unaligned 64bit reads. */
-    hi = ahci_mem_read_32(opaque, aligned + 4);
-    return (hi << 32 | lo) >> (ofst * 8);
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    return val;
 }
 
 
@@ -399,6 +403,9 @@ static void ahci_mem_write(void *opaque, hwaddr addr,
 {
     AHCIState *s = opaque;
 
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+
     /* Only aligned reads are allowed on AHCI */
     if (addr & 3) {
         fprintf(stderr, "ahci: Mis-aligned write to addr 0x"
@@ -539,20 +546,33 @@ static void ahci_check_cmd_bh(void *opaque)
 
 static void ahci_init_d2h(AHCIDevice *ad)
 {
-    uint8_t init_fis[20];
     IDEState *ide_state = &ad->port.ifs[0];
+    AHCIPortRegs *pr = &ad->port_regs;
 
-    memset(init_fis, 0, sizeof(init_fis));
-
-    init_fis[4] = 1;
-    init_fis[12] = 1;
+    if (ad->init_d2h_sent) {
+        return;
+    }
 
-    if (ide_state->drive_kind == IDE_CD) {
-        init_fis[5] = ide_state->lcyl;
-        init_fis[6] = ide_state->hcyl;
+    if (ahci_write_fis_d2h(ad)) {
+        ad->init_d2h_sent = true;
+        /* We're emulating receiving the first Reg H2D Fis from the device;
+         * Update the SIG register, but otherwise proceed as normal. */
+        pr->sig = ((uint32_t)ide_state->hcyl << 24) |
+            (ide_state->lcyl << 16) |
+            (ide_state->sector << 8) |
+            (ide_state->nsector & 0xFF);
     }
+}
+
+static void ahci_set_signature(AHCIDevice *ad, uint32_t sig)
+{
+    IDEState *s = &ad->port.ifs[0];
+    s->hcyl = sig >> 24 & 0xFF;
+    s->lcyl = sig >> 16 & 0xFF;
+    s->sector = sig >> 8 & 0xFF;
+    s->nsector = sig & 0xFF;
 
-    ahci_write_fis_d2h(ad, init_fis);
+    DPRINTF(ad->port_no, "set hcyl:lcyl:sect:nsect = 0x%08x\n", sig);
 }
 
 static void ahci_reset_port(AHCIState *s, int port)
@@ -603,17 +623,11 @@ static void ahci_reset_port(AHCIState *s, int port)
     }
 
     s->dev[port].port_state = STATE_RUN;
-    if (!ide_state->blk) {
-        pr->sig = 0;
-        ide_state->status = SEEK_STAT | WRERR_STAT;
-    } else if (ide_state->drive_kind == IDE_CD) {
-        pr->sig = SATA_SIGNATURE_CDROM;
-        ide_state->lcyl = 0x14;
-        ide_state->hcyl = 0xeb;
-        DPRINTF(port, "set lcyl = %d\n", ide_state->lcyl);
+    if (ide_state->drive_kind == IDE_CD) {
+        ahci_set_signature(d, SATA_SIGNATURE_CDROM);\
         ide_state->status = SEEK_STAT | WRERR_STAT | READY_STAT;
     } else {
-        pr->sig = SATA_SIGNATURE_DISK;
+        ahci_set_signature(d, SATA_SIGNATURE_DISK);
         ide_state->status = SEEK_STAT | WRERR_STAT;
     }
 
@@ -647,6 +661,10 @@ static bool ahci_map_fis_address(AHCIDevice *ad)
 
 static void ahci_unmap_fis_address(AHCIDevice *ad)
 {
+    if (ad->res_fis == NULL) {
+        DPRINTF(ad->port_no, "Attempt to unmap NULL FIS address\n");
+        return;
+    }
     dma_memory_unmap(ad->hba->as, ad->res_fis, 256,
                      DMA_DIRECTION_FROM_DEVICE, 256);
     ad->res_fis = NULL;
@@ -663,6 +681,10 @@ static bool ahci_map_clb_address(AHCIDevice *ad)
 
 static void ahci_unmap_clb_address(AHCIDevice *ad)
 {
+    if (ad->lst == NULL) {
+        DPRINTF(ad->port_no, "Attempt to unmap NULL CLB address\n");
+        return;
+    }
     dma_memory_unmap(ad->hba->as, ad->lst, 1024,
                      DMA_DIRECTION_FROM_DEVICE, 1024);
     ad->lst = NULL;
@@ -749,7 +771,7 @@ static void ahci_write_fis_pio(AHCIDevice *ad, uint16_t len)
     ahci_trigger_irq(ad->hba, ad, PORT_IRQ_PIOS_FIS);
 }
 
-static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
+static bool ahci_write_fis_d2h(AHCIDevice *ad)
 {
     AHCIPortRegs *pr = &ad->port_regs;
     uint8_t *d2h_fis;
@@ -757,7 +779,7 @@ static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
     IDEState *s = &ad->port.ifs[0];
 
     if (!ad->res_fis || !(pr->cmd & PORT_CMD_FIS_RX)) {
-        return;
+        return false;
     }
 
     d2h_fis = &ad->res_fis[RES_FIS_RFIS];
@@ -790,6 +812,7 @@ static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
     }
 
     ahci_trigger_irq(ad->hba, ad, PORT_IRQ_D2H_REG_FIS);
+    return true;
 }
 
 static int prdt_tbl_entry_size(const AHCI_SG *tbl)
@@ -798,8 +821,21 @@ static int prdt_tbl_entry_size(const AHCI_SG *tbl)
     return (le32_to_cpu(tbl->flags_size) & AHCI_PRDT_SIZE_MASK) + 1;
 }
 
+/**
+ * Fetch entries in a guest-provided PRDT and convert it into a QEMU SGlist.
+ * @ad: The AHCIDevice for whom we are building the SGList.
+ * @sglist: The SGList target to add PRD entries to.
+ * @cmd: The AHCI Command Header that describes where the PRDT is.
+ * @limit: The remaining size of the S/ATA transaction, in bytes.
+ * @offset: The number of bytes already transferred, in bytes.
+ *
+ * The AHCI PRDT can describe up to 256GiB. S/ATA only support transactions of
+ * up to 32MiB as of ATA8-ACS3 rev 1b, assuming a 512 byte sector size. We stop
+ * building the sglist from the PRDT as soon as we hit @limit bytes,
+ * which is <= INT32_MAX/2GiB.
+ */
 static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
-                                AHCICmdHdr *cmd, int64_t limit, int32_t offset)
+                                AHCICmdHdr *cmd, int64_t limit, uint64_t offset)
 {
     uint16_t opts = le16_to_cpu(cmd->opts);
     uint16_t prdtl = le16_to_cpu(cmd->prdtl);
@@ -817,14 +853,6 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
     IDEBus *bus = &ad->port;
     BusState *qbus = BUS(bus);
 
-    /*
-     * Note: AHCI PRDT can describe up to 256GiB. SATA/ATA only support
-     * transactions of up to 32MiB as of ATA8-ACS3 rev 1b, assuming a
-     * 512 byte sector size. We limit the PRDT in this implementation to
-     * a reasonably large 2GiB, which can accommodate the maximum transfer
-     * request for sector sizes up to 32K.
-     */
-
     if (!prdtl) {
         DPRINTF(ad->port_no, "no sg list given by guest: 0x%08x\n", opts);
         return -1;
@@ -874,13 +902,6 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
             qemu_sglist_add(sglist, le64_to_cpu(tbl[i].addr),
                             MIN(prdt_tbl_entry_size(&tbl[i]),
                                 limit - sglist->size));
-            if (sglist->size > INT32_MAX) {
-                error_report("AHCI Physical Region Descriptor Table describes "
-                             "more than 2 GiB.\n");
-                qemu_sglist_destroy(sglist);
-                r = -1;
-                goto out;
-            }
         }
     }
 
@@ -897,6 +918,7 @@ static void ncq_err(NCQTransferState *ncq_tfs)
     ide_state->error = ABRT_ERR;
     ide_state->status = READY_STAT | ERR_STAT;
     ncq_tfs->drive->port_regs.scr_err |= (1 << ncq_tfs->tag);
+    ncq_tfs->used = 0;
 }
 
 static void ncq_finish(NCQTransferState *ncq_tfs)
@@ -1397,7 +1419,7 @@ static void ahci_cmd_done(IDEDMA *dma)
     DPRINTF(ad->port_no, "cmd done\n");
 
     /* update d2h status */
-    ahci_write_fis_d2h(ad, NULL);
+    ahci_write_fis_d2h(ad);
 
     if (!ad->check_bh) {
         /* maybe we still have something to process, check later */
@@ -1421,7 +1443,17 @@ static const IDEDMAOps ahci_dma_ops = {
     .cmd_done = ahci_cmd_done,
 };
 
-void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
+void ahci_init(AHCIState *s, DeviceState *qdev)
+{
+    s->container = qdev;
+    /* XXX BAR size should be 1k, but that breaks, so bump it to 4k for now */
+    memory_region_init_io(&s->mem, OBJECT(qdev), &ahci_mem_ops, s,
+                          "ahci", AHCI_MEM_BAR_SIZE);
+    memory_region_init_io(&s->idp, OBJECT(qdev), &ahci_idp_ops, s,
+                          "ahci-idp", 32);
+}
+
+void ahci_realize(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
 {
     qemu_irq *irqs;
     int i;
@@ -1430,14 +1462,7 @@ void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
     s->ports = ports;
     s->dev = g_new0(AHCIDevice, ports);
     ahci_reg_init(s);
-    /* XXX BAR size should be 1k, but that breaks, so bump it to 4k for now */
-    memory_region_init_io(&s->mem, OBJECT(qdev), &ahci_mem_ops, s,
-                          "ahci", AHCI_MEM_BAR_SIZE);
-    memory_region_init_io(&s->idp, OBJECT(qdev), &ahci_idp_ops, s,
-                          "ahci-idp", 32);
-
     irqs = qemu_allocate_irqs(ahci_irq_set, s, s->ports);
-
     for (i = 0; i < s->ports; i++) {
         AHCIDevice *ad = &s->dev[i];
 
@@ -1617,18 +1642,6 @@ const VMStateDescription vmstate_ahci = {
     },
 };
 
-#define TYPE_SYSBUS_AHCI "sysbus-ahci"
-#define SYSBUS_AHCI(obj) OBJECT_CHECK(SysbusAHCIState, (obj), TYPE_SYSBUS_AHCI)
-
-typedef struct SysbusAHCIState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    /*< public >*/
-
-    AHCIState ahci;
-    uint32_t num_ports;
-} SysbusAHCIState;
-
 static const VMStateDescription vmstate_sysbus_ahci = {
     .name = "sysbus-ahci",
     .fields = (VMStateField[]) {
@@ -1644,17 +1657,24 @@ static void sysbus_ahci_reset(DeviceState *dev)
     ahci_reset(&s->ahci);
 }
 
-static void sysbus_ahci_realize(DeviceState *dev, Error **errp)
+static void sysbus_ahci_init(Object *obj)
 {
-    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
-    SysbusAHCIState *s = SYSBUS_AHCI(dev);
+    SysbusAHCIState *s = SYSBUS_AHCI(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    ahci_init(&s->ahci, dev, &address_space_memory, s->num_ports);
+    ahci_init(&s->ahci, DEVICE(obj));
 
     sysbus_init_mmio(sbd, &s->ahci.mem);
     sysbus_init_irq(sbd, &s->ahci.irq);
 }
 
+static void sysbus_ahci_realize(DeviceState *dev, Error **errp)
+{
+    SysbusAHCIState *s = SYSBUS_AHCI(dev);
+
+    ahci_realize(&s->ahci, dev, &address_space_memory, s->num_ports);
+}
+
 static Property sysbus_ahci_properties[] = {
     DEFINE_PROP_UINT32("num-ports", SysbusAHCIState, num_ports, 1),
     DEFINE_PROP_END_OF_LIST(),
@@ -1675,12 +1695,108 @@ static const TypeInfo sysbus_ahci_info = {
     .name          = TYPE_SYSBUS_AHCI,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysbusAHCIState),
+    .instance_init = sysbus_ahci_init,
     .class_init    = sysbus_ahci_class_init,
 };
 
+#define ALLWINNER_AHCI_BISTAFR    ((0xa0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTCR     ((0xa4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTFCTR   ((0xa8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTSR     ((0xac - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTDECR   ((0xb0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_DIAGNR0    ((0xb4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_DIAGNR1    ((0xb8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_OOBR       ((0xbc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS0R    ((0xc0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS1R    ((0xc4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS2R    ((0xc8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_TIMER1MS   ((0xe0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_GPARAM1R   ((0xe8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_GPARAM2R   ((0xec - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PPARAMR    ((0xf0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_TESTR      ((0xf4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_VERSIONR   ((0xf8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_IDR        ((0xfc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_RWCR       ((0xfc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+
+static uint64_t allwinner_ahci_mem_read(void *opaque, hwaddr addr,
+                                        unsigned size)
+{
+    AllwinnerAHCIState *a = opaque;
+    uint64_t val = a->regs[addr/4];
+
+    switch (addr / 4) {
+    case ALLWINNER_AHCI_PHYCS0R:
+        val |= 0x2 << 28;
+        break;
+    case ALLWINNER_AHCI_PHYCS2R:
+        val &= ~(0x1 << 24);
+        break;
+    }
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    return  val;
+}
+
+static void allwinner_ahci_mem_write(void *opaque, hwaddr addr,
+                                     uint64_t val, unsigned size)
+{
+    AllwinnerAHCIState *a = opaque;
+
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    a->regs[addr/4] = val;
+}
+
+static const MemoryRegionOps allwinner_ahci_mem_ops = {
+    .read = allwinner_ahci_mem_read,
+    .write = allwinner_ahci_mem_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void allwinner_ahci_init(Object *obj)
+{
+    SysbusAHCIState *s = SYSBUS_AHCI(obj);
+    AllwinnerAHCIState *a = ALLWINNER_AHCI(obj);
+
+    memory_region_init_io(&a->mmio, OBJECT(obj), &allwinner_ahci_mem_ops, a,
+                          "allwinner-ahci", ALLWINNER_AHCI_MMIO_SIZE);
+    memory_region_add_subregion(&s->ahci.mem, ALLWINNER_AHCI_MMIO_OFF,
+                                &a->mmio);
+}
+
+static const VMStateDescription vmstate_allwinner_ahci = {
+    .name = "allwinner-ahci",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, AllwinnerAHCIState,
+                             ALLWINNER_AHCI_MMIO_SIZE/4),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void allwinner_ahci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_allwinner_ahci;
+}
+
+static const TypeInfo allwinner_ahci_info = {
+    .name          = TYPE_ALLWINNER_AHCI,
+    .parent        = TYPE_SYSBUS_AHCI,
+    .instance_size = sizeof(AllwinnerAHCIState),
+    .instance_init = allwinner_ahci_init,
+    .class_init    = allwinner_ahci_class_init,
+};
+
 static void sysbus_ahci_register_types(void)
 {
     type_register_static(&sysbus_ahci_info);
+    type_register_static(&allwinner_ahci_info);
 }
 
 type_init(sysbus_ahci_register_types)
diff --git a/hw/ide/ahci.h b/hw/ide/ahci.h
index 79a463d93..bc777ed5c 100644
--- a/hw/ide/ahci.h
+++ b/hw/ide/ahci.h
@@ -24,6 +24,8 @@
 #ifndef HW_IDE_AHCI_H
 #define HW_IDE_AHCI_H
 
+#include <hw/sysbus.h>
+
 #define AHCI_MEM_BAR_SIZE         0x1000
 #define AHCI_MAX_PORTS            32
 #define AHCI_MAX_SG               168 /* hardware max is 64K */
@@ -285,6 +287,8 @@ struct AHCIDevice {
 };
 
 typedef struct AHCIState {
+    DeviceState *container;
+
     AHCIDevice *dev;
     AHCIControlRegs control_regs;
     MemoryRegion mem;
@@ -362,11 +366,40 @@ typedef struct SDBFIS {
     uint32_t payload;
 } QEMU_PACKED SDBFIS;
 
-void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports);
+void ahci_realize(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports);
+void ahci_init(AHCIState *s, DeviceState *qdev);
 void ahci_uninit(AHCIState *s);
 
 void ahci_reset(AHCIState *s);
 
 void ahci_ide_create_devs(PCIDevice *dev, DriveInfo **hd);
 
+#define TYPE_SYSBUS_AHCI "sysbus-ahci"
+#define SYSBUS_AHCI(obj) OBJECT_CHECK(SysbusAHCIState, (obj), TYPE_SYSBUS_AHCI)
+
+typedef struct SysbusAHCIState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    AHCIState ahci;
+    uint32_t num_ports;
+} SysbusAHCIState;
+
+#define TYPE_ALLWINNER_AHCI "allwinner-ahci"
+#define ALLWINNER_AHCI(obj) OBJECT_CHECK(AllwinnerAHCIState, (obj), \
+                       TYPE_ALLWINNER_AHCI)
+
+#define ALLWINNER_AHCI_MMIO_OFF  0x80
+#define ALLWINNER_AHCI_MMIO_SIZE 0x80
+
+struct AllwinnerAHCIState {
+    /*< private >*/
+    SysbusAHCIState parent_obj;
+    /*< public >*/
+
+    MemoryRegion mmio;
+    uint32_t regs[ALLWINNER_AHCI_MMIO_SIZE/4];
+};
+
 #endif /* HW_IDE_AHCI_H */
diff --git a/hw/ide/atapi.c b/hw/ide/atapi.c
index 79dd16710..65f8dd457 100644
--- a/hw/ide/atapi.c
+++ b/hw/ide/atapi.c
@@ -105,33 +105,99 @@ static void cd_data_to_raw(uint8_t *buf, int lba)
     memset(buf, 0, 288);
 }
 
-static int cd_read_sector(IDEState *s, int lba, uint8_t *buf, int sector_size)
+static int
+cd_read_sector_sync(IDEState *s)
 {
     int ret;
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
 
-    switch(sector_size) {
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector_sync: lba=%d\n", s->lba);
+#endif
+
+    switch (s->cd_sector_size) {
     case 2048:
-        block_acct_start(blk_get_stats(s->blk), &s->acct,
-                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-        ret = blk_read(s->blk, (int64_t)lba << 2, buf, 4);
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        ret = blk_read(s->blk, (int64_t)s->lba << 2,
+                       s->io_buffer, 4);
         break;
     case 2352:
-        block_acct_start(blk_get_stats(s->blk), &s->acct,
-                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-        ret = blk_read(s->blk, (int64_t)lba << 2, buf + 16, 4);
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
-        if (ret < 0)
-            return ret;
-        cd_data_to_raw(buf, lba);
+        ret = blk_read(s->blk, (int64_t)s->lba << 2,
+                       s->io_buffer + 16, 4);
+        if (ret >= 0) {
+            cd_data_to_raw(s->io_buffer, s->lba);
+        }
         break;
     default:
-        ret = -EIO;
-        break;
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
+        return -EIO;
+    }
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        s->lba++;
+        s->io_buffer_index = 0;
     }
+
     return ret;
 }
 
+static void cd_read_sector_cb(void *opaque, int ret)
+{
+    IDEState *s = opaque;
+
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector_cb: lba=%d ret=%d\n", s->lba, ret);
+#endif
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        ide_atapi_io_error(s, ret);
+        return;
+    }
+
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
+    if (s->cd_sector_size == 2352) {
+        cd_data_to_raw(s->io_buffer, s->lba);
+    }
+
+    s->lba++;
+    s->io_buffer_index = 0;
+    s->status &= ~BUSY_STAT;
+
+    ide_atapi_cmd_reply_end(s);
+}
+
+static int cd_read_sector(IDEState *s)
+{
+    if (s->cd_sector_size != 2048 && s->cd_sector_size != 2352) {
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
+        return -EINVAL;
+    }
+
+    s->iov.iov_base = (s->cd_sector_size == 2352) ?
+                      s->io_buffer + 16 : s->io_buffer;
+
+    s->iov.iov_len = 4 * BDRV_SECTOR_SIZE;
+    qemu_iovec_init_external(&s->qiov, &s->iov, 1);
+
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector: lba=%d\n", s->lba);
+#endif
+
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+
+    ide_buffered_readv(s, (int64_t)s->lba << 2, &s->qiov, 4,
+                       cd_read_sector_cb, s);
+
+    s->status |= BUSY_STAT;
+    return 0;
+}
+
 void ide_atapi_cmd_ok(IDEState *s)
 {
     s->error = 0;
@@ -167,6 +233,17 @@ void ide_atapi_io_error(IDEState *s, int ret)
     }
 }
 
+static uint16_t atapi_byte_count_limit(IDEState *s)
+{
+    uint16_t bcl;
+
+    bcl = s->lcyl | (s->hcyl << 8);
+    if (bcl == 0xffff) {
+        return 0xfffe;
+    }
+    return bcl;
+}
+
 /* The whole ATAPI transfer logic is handled in this function */
 void ide_atapi_cmd_reply_end(IDEState *s)
 {
@@ -182,18 +259,27 @@ void ide_atapi_cmd_reply_end(IDEState *s)
         ide_atapi_cmd_ok(s);
         ide_set_irq(s->bus);
 #ifdef DEBUG_IDE_ATAPI
-        printf("status=0x%x\n", s->status);
+        printf("end of transfer, status=0x%x\n", s->status);
 #endif
     } else {
         /* see if a new sector must be read */
         if (s->lba != -1 && s->io_buffer_index >= s->cd_sector_size) {
-            ret = cd_read_sector(s, s->lba, s->io_buffer, s->cd_sector_size);
-            if (ret < 0) {
-                ide_atapi_io_error(s, ret);
+            if (!s->elementary_transfer_size) {
+                ret = cd_read_sector(s);
+                if (ret < 0) {
+                    ide_atapi_io_error(s, ret);
+                }
                 return;
+            } else {
+                /* rebuffering within an elementary transfer is
+                 * only possible with a sync request because we
+                 * end up with a race condition otherwise */
+                ret = cd_read_sector_sync(s);
+                if (ret < 0) {
+                    ide_atapi_io_error(s, ret);
+                    return;
+                }
             }
-            s->lba++;
-            s->io_buffer_index = 0;
         }
         if (s->elementary_transfer_size > 0) {
             /* there are some data left to transmit in this elementary
@@ -209,12 +295,10 @@ void ide_atapi_cmd_reply_end(IDEState *s)
         } else {
             /* a new transfer is needed */
             s->nsector = (s->nsector & ~7) | ATAPI_INT_REASON_IO;
-            byte_count_limit = s->lcyl | (s->hcyl << 8);
+            byte_count_limit = atapi_byte_count_limit(s);
 #ifdef DEBUG_IDE_ATAPI
             printf("byte_count_limit=%d\n", byte_count_limit);
 #endif
-            if (byte_count_limit == 0xffff)
-                byte_count_limit--;
             size = s->packet_transfer_size;
             if (size > byte_count_limit) {
                 /* byte count limit must be even if this case */
@@ -275,7 +359,6 @@ static void ide_atapi_cmd_read_pio(IDEState *s, int lba, int nb_sectors,
     s->io_buffer_index = sector_size;
     s->cd_sector_size = sector_size;
 
-    s->status = READY_STAT | SEEK_STAT;
     ide_atapi_cmd_reply_end(s);
 }
 
@@ -351,13 +434,17 @@ static void ide_atapi_cmd_read_dma_cb(void *opaque, int ret)
     s->bus->dma->iov.iov_len = n * 4 * 512;
     qemu_iovec_init_external(&s->bus->dma->qiov, &s->bus->dma->iov, 1);
 
-    s->bus->dma->aiocb = blk_aio_readv(s->blk, (int64_t)s->lba << 2,
-                                       &s->bus->dma->qiov, n * 4,
-                                       ide_atapi_cmd_read_dma_cb, s);
+    s->bus->dma->aiocb = ide_buffered_readv(s, (int64_t)s->lba << 2,
+                                            &s->bus->dma->qiov, n * 4,
+                                            ide_atapi_cmd_read_dma_cb, s);
     return;
 
 eot:
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+    }
     ide_set_inactive(s, false);
 }
 
@@ -1169,20 +1256,28 @@ enum {
      * 4.1.8)
      */
     CHECK_READY = 0x02,
+
+    /*
+     * Commands flagged with NONDATA do not in any circumstances return
+     * any data via ide_atapi_cmd_reply. These commands are exempt from
+     * the normal byte_count_limit constraints.
+     * See ATA8-ACS3 "7.21.5 Byte Count Limit"
+     */
+    NONDATA = 0x04,
 };
 
-static const struct {
+static const struct AtapiCmd {
     void (*handler)(IDEState *s, uint8_t *buf);
     int flags;
 } atapi_cmd_table[0x100] = {
-    [ 0x00 ] = { cmd_test_unit_ready,               CHECK_READY },
+    [ 0x00 ] = { cmd_test_unit_ready,               CHECK_READY | NONDATA },
     [ 0x03 ] = { cmd_request_sense,                 ALLOW_UA },
     [ 0x12 ] = { cmd_inquiry,                       ALLOW_UA },
-    [ 0x1b ] = { cmd_start_stop_unit,               0 }, /* [1] */
-    [ 0x1e ] = { cmd_prevent_allow_medium_removal,  0 },
+    [ 0x1b ] = { cmd_start_stop_unit,               NONDATA }, /* [1] */
+    [ 0x1e ] = { cmd_prevent_allow_medium_removal,  NONDATA },
     [ 0x25 ] = { cmd_read_cdvd_capacity,            CHECK_READY },
     [ 0x28 ] = { cmd_read, /* (10) */               CHECK_READY },
-    [ 0x2b ] = { cmd_seek,                          CHECK_READY },
+    [ 0x2b ] = { cmd_seek,                          CHECK_READY | NONDATA },
     [ 0x43 ] = { cmd_read_toc_pma_atip,             CHECK_READY },
     [ 0x46 ] = { cmd_get_configuration,             ALLOW_UA },
     [ 0x4a ] = { cmd_get_event_status_notification, ALLOW_UA },
@@ -1190,7 +1285,7 @@ static const struct {
     [ 0x5a ] = { cmd_mode_sense, /* (10) */         0 },
     [ 0xa8 ] = { cmd_read, /* (12) */               CHECK_READY },
     [ 0xad ] = { cmd_read_dvd_structure,            CHECK_READY },
-    [ 0xbb ] = { cmd_set_speed,                     0 },
+    [ 0xbb ] = { cmd_set_speed,                     NONDATA },
     [ 0xbd ] = { cmd_mechanism_status,              0 },
     [ 0xbe ] = { cmd_read_cd,                       CHECK_READY },
     /* [1] handler detects and reports not ready condition itself */
@@ -1198,9 +1293,9 @@ static const struct {
 
 void ide_atapi_cmd(IDEState *s)
 {
-    uint8_t *buf;
+    uint8_t *buf = s->io_buffer;
+    const struct AtapiCmd *cmd = &atapi_cmd_table[s->io_buffer[0]];
 
-    buf = s->io_buffer;
 #ifdef DEBUG_IDE_ATAPI
     {
         int i;
@@ -1211,14 +1306,14 @@ void ide_atapi_cmd(IDEState *s)
         printf("\n");
     }
 #endif
+
     /*
      * If there's a UNIT_ATTENTION condition pending, only command flagged with
      * ALLOW_UA are allowed to complete. with other commands getting a CHECK
      * condition response unless a higher priority status, defined by the drive
      * here, is pending.
      */
-    if (s->sense_key == UNIT_ATTENTION &&
-        !(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA)) {
+    if (s->sense_key == UNIT_ATTENTION && !(cmd->flags & ALLOW_UA)) {
         ide_atapi_cmd_check_status(s);
         return;
     }
@@ -1229,7 +1324,7 @@ void ide_atapi_cmd(IDEState *s)
      * GET_EVENT_STATUS_NOTIFICATION to detect such tray open/close
      * states rely on this behavior.
      */
-    if (!(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA) &&
+    if (!(cmd->flags & ALLOW_UA) &&
         !s->tray_open && blk_is_inserted(s->blk) && s->cdrom_changed) {
 
         if (s->cdrom_changed == 1) {
@@ -1244,16 +1339,29 @@ void ide_atapi_cmd(IDEState *s)
     }
 
     /* Report a Not Ready condition if appropriate for the command */
-    if ((atapi_cmd_table[s->io_buffer[0]].flags & CHECK_READY) &&
+    if ((cmd->flags & CHECK_READY) &&
         (!media_present(s) || !blk_is_inserted(s->blk)))
     {
         ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
         return;
     }
 
+    /* Nondata commands permit the byte_count_limit to be 0.
+     * If this is a data-transferring PIO command and BCL is 0,
+     * we abort at the /ATA/ level, not the ATAPI level.
+     * See ATA8 ACS3 section 7.17.6.49 and 7.21.5 */
+    if (cmd->handler && !(cmd->flags & NONDATA)) {
+        /* TODO: Check IDENTIFY data word 125 for default BCL (currently 0) */
+        if (!(atapi_byte_count_limit(s) || s->atapi_dma)) {
+            /* TODO: Move abort back into core.c and make static inline again */
+            ide_abort_command(s);
+            return;
+        }
+    }
+
     /* Execute the command */
-    if (atapi_cmd_table[s->io_buffer[0]].handler) {
-        atapi_cmd_table[s->io_buffer[0]].handler(s, buf);
+    if (cmd->handler) {
+        cmd->handler(s, buf);
         return;
     }
 
diff --git a/hw/ide/cmd646.c b/hw/ide/cmd646.c
index 66fb9d96d..27f3da21a 100644
--- a/hw/ide/cmd646.c
+++ b/hw/ide/cmd646.c
@@ -417,6 +417,7 @@ static void cmd646_ide_class_init(ObjectClass *klass, void *data)
     k->config_read = cmd646_pci_config_read;
     k->config_write = cmd646_pci_config_write;
     dc->props = cmd646_ide_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo cmd646_ide_info = {
diff --git a/hw/ide/core.c b/hw/ide/core.c
index 1cc6945d8..da3baab1e 100644
--- a/hw/ide/core.c
+++ b/hw/ide/core.c
@@ -457,7 +457,7 @@ BlockAIOCB *ide_issue_trim(BlockBackend *blk,
     return &iocb->common;
 }
 
-static inline void ide_abort_command(IDEState *s)
+void ide_abort_command(IDEState *s)
 {
     ide_transfer_stop(s);
     s->status = READY_STAT | ERR_STAT;
@@ -561,6 +561,53 @@ static bool ide_sect_range_ok(IDEState *s,
     return true;
 }
 
+static void ide_buffered_readv_cb(void *opaque, int ret)
+{
+    IDEBufferedRequest *req = opaque;
+    if (!req->orphaned) {
+        if (!ret) {
+            qemu_iovec_from_buf(req->original_qiov, 0, req->iov.iov_base,
+                                req->original_qiov->size);
+        }
+        req->original_cb(req->original_opaque, ret);
+    }
+    QLIST_REMOVE(req, list);
+    qemu_vfree(req->iov.iov_base);
+    g_free(req);
+}
+
+#define MAX_BUFFERED_REQS 16
+
+BlockAIOCB *ide_buffered_readv(IDEState *s, int64_t sector_num,
+                               QEMUIOVector *iov, int nb_sectors,
+                               BlockCompletionFunc *cb, void *opaque)
+{
+    BlockAIOCB *aioreq;
+    IDEBufferedRequest *req;
+    int c = 0;
+
+    QLIST_FOREACH(req, &s->buffered_requests, list) {
+        c++;
+    }
+    if (c > MAX_BUFFERED_REQS) {
+        return blk_abort_aio_request(s->blk, cb, opaque, -EIO);
+    }
+
+    req = g_new0(IDEBufferedRequest, 1);
+    req->original_qiov = iov;
+    req->original_cb = cb;
+    req->original_opaque = opaque;
+    req->iov.iov_base = qemu_blockalign(blk_bs(s->blk), iov->size);
+    req->iov.iov_len = iov->size;
+    qemu_iovec_init_external(&req->qiov, &req->iov, 1);
+
+    aioreq = blk_aio_readv(s->blk, sector_num, &req->qiov, nb_sectors,
+                           ide_buffered_readv_cb, req);
+
+    QLIST_INSERT_HEAD(&s->buffered_requests, req, list);
+    return aioreq;
+}
+
 static void ide_sector_read(IDEState *s);
 
 static void ide_sector_read_cb(void *opaque, int ret)
@@ -574,7 +621,6 @@ static void ide_sector_read_cb(void *opaque, int ret)
     if (ret == -ECANCELED) {
         return;
     }
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
     if (ret != 0) {
         if (ide_handle_rw_error(s, -ret, IDE_RETRY_PIO |
                                 IDE_RETRY_READ)) {
@@ -582,6 +628,8 @@ static void ide_sector_read_cb(void *opaque, int ret)
         }
     }
 
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
     n = s->nsector;
     if (n > s->req_nb_sectors) {
         n = s->req_nb_sectors;
@@ -621,6 +669,7 @@ static void ide_sector_read(IDEState *s)
 
     if (!ide_sect_range_ok(s, sector_num, n)) {
         ide_rw_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
         return;
     }
 
@@ -630,8 +679,8 @@ static void ide_sector_read(IDEState *s)
 
     block_acct_start(blk_get_stats(s->blk), &s->acct,
                      n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-    s->pio_aiocb = blk_aio_readv(s->blk, sector_num, &s->qiov, n,
-                                 ide_sector_read_cb, s);
+    s->pio_aiocb = ide_buffered_readv(s, sector_num, &s->qiov, n,
+                                      ide_sector_read_cb, s);
 }
 
 void dma_buf_commit(IDEState *s, uint32_t tx_bytes)
@@ -672,6 +721,7 @@ static int ide_handle_rw_error(IDEState *s, int error, int op)
         assert(s->bus->retry_unit == s->unit);
         s->bus->error_status = op;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
         if (op & IDE_RETRY_DMA) {
             ide_dma_error(s);
         } else {
@@ -750,6 +800,7 @@ static void ide_dma_cb(void *opaque, int ret)
     if ((s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) &&
         !ide_sect_range_ok(s, sector_num, n)) {
         ide_dma_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), s->acct.type);
         return;
     }
 
@@ -826,7 +877,6 @@ static void ide_sector_write_cb(void *opaque, int ret)
     if (ret == -ECANCELED) {
         return;
     }
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
 
     s->pio_aiocb = NULL;
     s->status &= ~BUSY_STAT;
@@ -837,6 +887,8 @@ static void ide_sector_write_cb(void *opaque, int ret)
         }
     }
 
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
     n = s->nsector;
     if (n > s->req_nb_sectors) {
         n = s->req_nb_sectors;
@@ -887,6 +939,7 @@ static void ide_sector_write(IDEState *s)
 
     if (!ide_sect_range_ok(s, sector_num, n)) {
         ide_rw_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_WRITE);
         return;
     }
 
@@ -895,7 +948,7 @@ static void ide_sector_write(IDEState *s)
     qemu_iovec_init_external(&s->qiov, &s->iov, 1);
 
     block_acct_start(blk_get_stats(s->blk), &s->acct,
-                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_WRITE);
     s->pio_aiocb = blk_aio_writev(s->blk, sector_num, &s->qiov, n,
                                   ide_sector_write_cb, s);
 }
@@ -2312,7 +2365,7 @@ int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
     if (version) {
         pstrcpy(s->version, sizeof(s->version), version);
     } else {
-        pstrcpy(s->version, sizeof(s->version), qemu_get_version());
+        pstrcpy(s->version, sizeof(s->version), qemu_hw_version());
     }
 
     ide_reset(s);
diff --git a/hw/ide/ich.c b/hw/ide/ich.c
index 350c7f1c7..16925fa25 100644
--- a/hw/ide/ich.c
+++ b/hw/ide/ich.c
@@ -97,6 +97,13 @@ static void pci_ich9_reset(DeviceState *dev)
     ahci_reset(&d->ahci);
 }
 
+static void pci_ich9_ahci_init(Object *obj)
+{
+    struct AHCIPCIState *d = ICH_AHCI(obj);
+
+    ahci_init(&d->ahci, DEVICE(obj));
+}
+
 static void pci_ich9_ahci_realize(PCIDevice *dev, Error **errp)
 {
     struct AHCIPCIState *d;
@@ -104,7 +111,7 @@ static void pci_ich9_ahci_realize(PCIDevice *dev, Error **errp)
     uint8_t *sata_cap;
     d = ICH_AHCI(dev);
 
-    ahci_init(&d->ahci, DEVICE(dev), pci_get_address_space(dev), 6);
+    ahci_realize(&d->ahci, DEVICE(dev), pci_get_address_space(dev), 6);
 
     pci_config_set_prog_interface(dev->config, AHCI_PROGMODE_MAJOR_REV_1);
 
@@ -171,6 +178,7 @@ static const TypeInfo ich_ahci_info = {
     .name          = TYPE_ICH9_AHCI,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(AHCIPCIState),
+    .instance_init = pci_ich9_ahci_init,
     .class_init    = ich_ahci_class_init,
 };
 
diff --git a/hw/ide/internal.h b/hw/ide/internal.h
index 7288a677a..2d1e2d2d2 100644
--- a/hw/ide/internal.h
+++ b/hw/ide/internal.h
@@ -343,6 +343,16 @@ enum ide_dma_cmd {
 #define ide_cmd_is_read(s) \
 	((s)->dma_cmd == IDE_DMA_READ)
 
+typedef struct IDEBufferedRequest {
+    QLIST_ENTRY(IDEBufferedRequest) list;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    QEMUIOVector *original_qiov;
+    BlockCompletionFunc *original_cb;
+    void *original_opaque;
+    bool orphaned;
+} IDEBufferedRequest;
+
 /* NOTE: IDEState represents in fact one drive */
 struct IDEState {
     IDEBus *bus;
@@ -396,8 +406,9 @@ struct IDEState {
     BlockAIOCB *pio_aiocb;
     struct iovec iov;
     QEMUIOVector qiov;
+    QLIST_HEAD(, IDEBufferedRequest) buffered_requests;
     /* ATA DMA state */
-    int32_t io_buffer_offset;
+    uint64_t io_buffer_offset;
     int32_t io_buffer_size;
     QEMUSGList sg;
     /* PIO transfer handling */
@@ -538,6 +549,7 @@ void ide_set_sector(IDEState *s, int64_t sector_num);
 void ide_start_dma(IDEState *s, BlockCompletionFunc *cb);
 void dma_buf_commit(IDEState *s, uint32_t tx_bytes);
 void ide_dma_error(IDEState *s);
+void ide_abort_command(IDEState *s);
 
 void ide_atapi_cmd_ok(IDEState *s);
 void ide_atapi_cmd_error(IDEState *s, int sense_key, int asc);
@@ -571,6 +583,9 @@ void ide_set_inactive(IDEState *s, bool more);
 BlockAIOCB *ide_issue_trim(BlockBackend *blk,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *ide_buffered_readv(IDEState *s, int64_t sector_num,
+                               QEMUIOVector *iov, int nb_sectors,
+                               BlockCompletionFunc *cb, void *opaque);
 
 /* hw/ide/atapi.c */
 void ide_atapi_cmd(IDEState *s);
diff --git a/hw/ide/macio.c b/hw/ide/macio.c
index 66ac2baa9..3ee962f83 100644
--- a/hw/ide/macio.c
+++ b/hw/ide/macio.c
@@ -286,7 +286,11 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
     return;
 
 done:
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+    }
     io->dma_end(opaque);
 
     return;
@@ -348,7 +352,11 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
 
 done:
     if (s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) {
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        if (ret < 0) {
+            block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        } else {
+            block_acct_done(blk_get_stats(s->blk), &s->acct);
+        }
     }
     io->dma_end(opaque);
 }
@@ -590,6 +598,7 @@ static void macio_ide_class_init(ObjectClass *oc, void *data)
     dc->realize = macio_ide_realizefn;
     dc->reset = macio_ide_reset;
     dc->vmsd = &vmstate_pmac;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo macio_ide_type_info = {
diff --git a/hw/ide/pci.c b/hw/ide/pci.c
index d31ff885b..37dbc291d 100644
--- a/hw/ide/pci.c
+++ b/hw/ide/pci.c
@@ -103,13 +103,6 @@ static int32_t bmdma_prepare_buf(IDEDMA *dma, int32_t limit)
                 qemu_sglist_add(&s->sg, bm->cur_prd_addr, sg_len);
             }
 
-            /* Note: We limit the max transfer to be 2GiB.
-             * This should accommodate the largest ATA transaction
-             * for LBA48 (65,536 sectors) and 32K sector sizes. */
-            if (s->sg.size > INT32_MAX) {
-                error_report("IDE: sglist describes more than 2GiB.");
-                break;
-            }
             bm->cur_prd_addr += l;
             bm->cur_prd_len -= l;
             s->io_buffer_size += l;
@@ -240,6 +233,22 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
     /* Ignore writes to SSBM if it keeps the old value */
     if ((val & BM_CMD_START) != (bm->cmd & BM_CMD_START)) {
         if (!(val & BM_CMD_START)) {
+            /* First invoke the callbacks of all buffered requests
+             * and flag those requests as orphaned. Ideally there
+             * are no unbuffered (Scatter Gather DMA Requests or
+             * write requests) pending and we can avoid to drain. */
+            IDEBufferedRequest *req;
+            IDEState *s = idebus_active_if(bm->bus);
+            QLIST_FOREACH(req, &s->buffered_requests, list) {
+                if (!req->orphaned) {
+#ifdef DEBUG_IDE
+                    printf("%s: invoking cb %p of buffered request %p with"
+                           " -ECANCELED\n", __func__, req->original_cb, req);
+#endif
+                    req->original_cb(req->original_opaque, -ECANCELED);
+                }
+                req->orphaned = true;
+            }
             /*
              * We can't cancel Scatter Gather DMA in the middle of the
              * operation or a partial (not full) DMA transfer would reach
@@ -253,6 +262,9 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
              * aio operation with preadv/pwritev.
              */
             if (bm->bus->dma->aiocb) {
+#ifdef DEBUG_IDE
+                printf("%s: draining all remaining requests", __func__);
+#endif
                 blk_drain_all();
                 assert(bm->bus->dma->aiocb == NULL);
             }
diff --git a/hw/input/Makefile.objs b/hw/input/Makefile.objs
index 624ba7ea4..7715d7230 100644
--- a/hw/input/Makefile.objs
+++ b/hw/input/Makefile.objs
@@ -8,9 +8,9 @@ common-obj-$(CONFIG_STELLARIS_INPUT) += stellaris_input.o
 common-obj-$(CONFIG_TSC2005) += tsc2005.o
 common-obj-$(CONFIG_VMMOUSE) += vmmouse.o
 
-ifeq ($(CONFIG_LINUX),y)
 common-obj-$(CONFIG_VIRTIO) += virtio-input.o
 common-obj-$(CONFIG_VIRTIO) += virtio-input-hid.o
+ifeq ($(CONFIG_LINUX),y)
 common-obj-$(CONFIG_VIRTIO) += virtio-input-host.o
 endif
 
diff --git a/hw/input/adb.c b/hw/input/adb.c
index a18eea265..09eead96b 100644
--- a/hw/input/adb.c
+++ b/hw/input/adb.c
@@ -362,6 +362,7 @@ static void adb_kbd_class_init(ObjectClass *oc, void *data)
 
     akc->parent_realize = dc->realize;
     dc->realize = adb_kbd_realizefn;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 
     adc->devreq = adb_kbd_request;
     dc->reset = adb_kbd_reset;
@@ -566,6 +567,7 @@ static void adb_mouse_class_init(ObjectClass *oc, void *data)
 
     amc->parent_realize = dc->realize;
     dc->realize = adb_mouse_realizefn;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 
     adc->devreq = adb_mouse_request;
     dc->reset = adb_mouse_reset;
diff --git a/hw/input/hid.c b/hw/input/hid.c
index 21ebd9e71..e39269fc7 100644
--- a/hw/input/hid.c
+++ b/hw/input/hid.c
@@ -119,33 +119,33 @@ static void hid_pointer_event(DeviceState *dev, QemuConsole *src,
     assert(hs->n < QUEUE_LENGTH);
     e = &hs->ptr.queue[(hs->head + hs->n) & QUEUE_MASK];
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_REL:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            e->xdx += evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            e->ydy += evt->rel->value;
+        if (evt->u.rel->axis == INPUT_AXIS_X) {
+            e->xdx += evt->u.rel->value;
+        } else if (evt->u.rel->axis == INPUT_AXIS_Y) {
+            e->ydy += evt->u.rel->value;
         }
         break;
 
     case INPUT_EVENT_KIND_ABS:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            e->xdx = evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            e->ydy = evt->rel->value;
+        if (evt->u.rel->axis == INPUT_AXIS_X) {
+            e->xdx = evt->u.rel->value;
+        } else if (evt->u.rel->axis == INPUT_AXIS_Y) {
+            e->ydy = evt->u.rel->value;
         }
         break;
 
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            e->buttons_state |= bmap[evt->btn->button];
-            if (evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        if (evt->u.btn->down) {
+            e->buttons_state |= bmap[evt->u.btn->button];
+            if (evt->u.btn->button == INPUT_BUTTON_WHEEL_UP) {
                 e->dz--;
-            } else if (evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+            } else if (evt->u.btn->button == INPUT_BUTTON_WHEEL_DOWN) {
                 e->dz++;
             }
         } else {
-            e->buttons_state &= ~bmap[evt->btn->button];
+            e->buttons_state &= ~bmap[evt->u.btn->button];
         }
         break;
 
@@ -223,8 +223,8 @@ static void hid_keyboard_event(DeviceState *dev, QemuConsole *src,
     int scancodes[3], i, count;
     int slot;
 
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(evt->u.key->key,
+                                             evt->u.key->down,
                                              scancodes);
     if (hs->n + count > QUEUE_LENGTH) {
         fprintf(stderr, "usb-kbd: warning: key event queue full\n");
diff --git a/hw/input/milkymist-softusb.c b/hw/input/milkymist-softusb.c
index 7b0f4db88..8a02d35ca 100644
--- a/hw/input/milkymist-softusb.c
+++ b/hw/input/milkymist-softusb.c
@@ -255,12 +255,12 @@ static int milkymist_softusb_init(SysBusDevice *dev)
 
     /* register pmem and dmem */
     memory_region_init_ram(&s->pmem, OBJECT(s), "milkymist-softusb.pmem",
-                           s->pmem_size, &error_abort);
+                           s->pmem_size, &error_fatal);
     vmstate_register_ram_global(&s->pmem);
     s->pmem_ptr = memory_region_get_ram_ptr(&s->pmem);
     sysbus_init_mmio(dev, &s->pmem);
     memory_region_init_ram(&s->dmem, OBJECT(s), "milkymist-softusb.dmem",
-                           s->dmem_size, &error_abort);
+                           s->dmem_size, &error_fatal);
     vmstate_register_ram_global(&s->dmem);
     s->dmem_ptr = memory_region_get_ram_ptr(&s->dmem);
     sysbus_init_mmio(dev, &s->dmem);
diff --git a/hw/input/ps2.c b/hw/input/ps2.c
index fdbe565e6..3d6d4961d 100644
--- a/hw/input/ps2.c
+++ b/hw/input/ps2.c
@@ -183,8 +183,8 @@ static void ps2_keyboard_event(DeviceState *dev, QemuConsole *src,
     int scancodes[3], i, count;
 
     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(evt->u.key->key,
+                                             evt->u.key->down,
                                              scancodes);
     for (i = 0; i < count; i++) {
         ps2_put_keycode(s, scancodes[i]);
@@ -393,25 +393,25 @@ static void ps2_mouse_event(DeviceState *dev, QemuConsole *src,
     if (!(s->mouse_status & MOUSE_STATUS_ENABLED))
         return;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_REL:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            s->mouse_dx += evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            s->mouse_dy -= evt->rel->value;
+        if (evt->u.rel->axis == INPUT_AXIS_X) {
+            s->mouse_dx += evt->u.rel->value;
+        } else if (evt->u.rel->axis == INPUT_AXIS_Y) {
+            s->mouse_dy -= evt->u.rel->value;
         }
         break;
 
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            s->mouse_buttons |= bmap[evt->btn->button];
-            if (evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        if (evt->u.btn->down) {
+            s->mouse_buttons |= bmap[evt->u.btn->button];
+            if (evt->u.btn->button == INPUT_BUTTON_WHEEL_UP) {
                 s->mouse_dz--;
-            } else if (evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+            } else if (evt->u.btn->button == INPUT_BUTTON_WHEEL_DOWN) {
                 s->mouse_dz++;
             }
         } else {
-            s->mouse_buttons &= ~bmap[evt->btn->button];
+            s->mouse_buttons &= ~bmap[evt->u.btn->button];
         }
         break;
 
diff --git a/hw/input/stellaris_input.c b/hw/input/stellaris_input.c
index 0609e8086..4d86c04e5 100644
--- a/hw/input/stellaris_input.c
+++ b/hw/input/stellaris_input.c
@@ -69,14 +69,14 @@ static const VMStateDescription vmstate_stellaris_gamepad = {
     }
 };
 
-/* Returns an array 5 ouput slots.  */
+/* Returns an array of 5 output slots.  */
 void stellaris_gamepad_init(int n, qemu_irq *irq, const int *keycode)
 {
     gamepad_state *s;
     int i;
 
-    s = (gamepad_state *)g_malloc0(sizeof (gamepad_state));
-    s->buttons = (gamepad_button *)g_malloc0(n * sizeof (gamepad_button));
+    s = g_new0(gamepad_state, 1);
+    s->buttons = g_new0(gamepad_button, n);
     for (i = 0; i < n; i++) {
         s->buttons[i].irq = irq[i];
         s->buttons[i].keycode = keycode[i];
diff --git a/hw/input/tsc210x.c b/hw/input/tsc210x.c
index fae338563..1073bbfec 100644
--- a/hw/input/tsc210x.c
+++ b/hw/input/tsc210x.c
@@ -1086,9 +1086,7 @@ uWireSlave *tsc2102_init(qemu_irq pint)
 {
     TSC210xState *s;
 
-    s = (TSC210xState *)
-            g_malloc0(sizeof(TSC210xState));
-    memset(s, 0, sizeof(TSC210xState));
+    s = g_new0(TSC210xState, 1);
     s->x = 160;
     s->y = 160;
     s->pressure = 0;
@@ -1135,9 +1133,7 @@ uWireSlave *tsc2301_init(qemu_irq penirq, qemu_irq kbirq, qemu_irq dav)
 {
     TSC210xState *s;
 
-    s = (TSC210xState *)
-            g_malloc0(sizeof(TSC210xState));
-    memset(s, 0, sizeof(TSC210xState));
+    s = g_new0(TSC210xState, 1);
     s->x = 400;
     s->y = 240;
     s->pressure = 0;
diff --git a/hw/input/virtio-input-hid.c b/hw/input/virtio-input-hid.c
index 4d85dad4d..bdd479cd0 100644
--- a/hw/input/virtio-input-hid.c
+++ b/hw/input/virtio-input-hid.c
@@ -191,44 +191,45 @@ static void virtio_input_handle_event(DeviceState *dev, QemuConsole *src,
     virtio_input_event event;
     int qcode;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_KEY:
-        qcode = qemu_input_key_value_to_qcode(evt->key->key);
+        qcode = qemu_input_key_value_to_qcode(evt->u.key->key);
         if (qcode && keymap_qcode[qcode]) {
             event.type  = cpu_to_le16(EV_KEY);
             event.code  = cpu_to_le16(keymap_qcode[qcode]);
-            event.value = cpu_to_le32(evt->key->down ? 1 : 0);
+            event.value = cpu_to_le32(evt->u.key->down ? 1 : 0);
             virtio_input_send(vinput, &event);
         } else {
-            if (evt->key->down) {
+            if (evt->u.key->down) {
                 fprintf(stderr, "%s: unmapped key: %d [%s]\n", __func__,
                         qcode, QKeyCode_lookup[qcode]);
             }
         }
         break;
     case INPUT_EVENT_KIND_BTN:
-        if (keymap_button[evt->btn->button]) {
+        if (keymap_button[evt->u.btn->button]) {
             event.type  = cpu_to_le16(EV_KEY);
-            event.code  = cpu_to_le16(keymap_button[evt->btn->button]);
-            event.value = cpu_to_le32(evt->btn->down ? 1 : 0);
+            event.code  = cpu_to_le16(keymap_button[evt->u.btn->button]);
+            event.value = cpu_to_le32(evt->u.btn->down ? 1 : 0);
             virtio_input_send(vinput, &event);
         } else {
-            if (evt->btn->down) {
+            if (evt->u.btn->down) {
                 fprintf(stderr, "%s: unmapped button: %d [%s]\n", __func__,
-                        evt->btn->button, InputButton_lookup[evt->btn->button]);
+                        evt->u.btn->button,
+                        InputButton_lookup[evt->u.btn->button]);
             }
         }
         break;
     case INPUT_EVENT_KIND_REL:
         event.type  = cpu_to_le16(EV_REL);
-        event.code  = cpu_to_le16(axismap_rel[evt->rel->axis]);
-        event.value = cpu_to_le32(evt->rel->value);
+        event.code  = cpu_to_le16(axismap_rel[evt->u.rel->axis]);
+        event.value = cpu_to_le32(evt->u.rel->value);
         virtio_input_send(vinput, &event);
         break;
     case INPUT_EVENT_KIND_ABS:
         event.type  = cpu_to_le16(EV_ABS);
-        event.code  = cpu_to_le16(axismap_abs[evt->abs->axis]);
-        event.value = cpu_to_le32(evt->abs->value);
+        event.code  = cpu_to_le16(axismap_abs[evt->u.abs->axis]);
+        event.value = cpu_to_le32(evt->u.abs->value);
         virtio_input_send(vinput, &event);
         break;
     default:
diff --git a/hw/intc/Makefile.objs b/hw/intc/Makefile.objs
index 092d8a80a..004b0c25e 100644
--- a/hw/intc/Makefile.objs
+++ b/hw/intc/Makefile.objs
@@ -12,10 +12,12 @@ common-obj-$(CONFIG_IOAPIC) += ioapic_common.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gic_common.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gic.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gicv2m.o
+common-obj-$(CONFIG_ARM_GIC) += arm_gicv3_common.o
 common-obj-$(CONFIG_OPENPIC) += openpic.o
 
 obj-$(CONFIG_APIC) += apic.o apic_common.o
 obj-$(CONFIG_ARM_GIC_KVM) += arm_gic_kvm.o
+obj-$(call land,$(CONFIG_ARM_GIC_KVM),$(TARGET_AARCH64)) += arm_gicv3_kvm.o
 obj-$(CONFIG_STELLARIS) += armv7m_nvic.o
 obj-$(CONFIG_EXYNOS4) += exynos4210_gic.o exynos4210_combiner.o
 obj-$(CONFIG_GRLIB) += grlib_irqmp.o
diff --git a/hw/intc/apic.c b/hw/intc/apic.c
index 77b639cce..0a840b851 100644
--- a/hw/intc/apic.c
+++ b/hw/intc/apic.c
@@ -51,14 +51,6 @@ static int apic_ffs_bit(uint32_t value)
     return ctz32(value);
 }
 
-static inline void apic_set_bit(uint32_t *tab, int index)
-{
-    int i, mask;
-    i = index >> 5;
-    mask = 1 << (index & 0x1f);
-    tab[i] |= mask;
-}
-
 static inline void apic_reset_bit(uint32_t *tab, int index)
 {
     int i, mask;
@@ -67,14 +59,6 @@ static inline void apic_reset_bit(uint32_t *tab, int index)
     tab[i] &= ~mask;
 }
 
-static inline int apic_get_bit(uint32_t *tab, int index)
-{
-    int i, mask;
-    i = index >> 5;
-    mask = 1 << (index & 0x1f);
-    return !!(tab[i] & mask);
-}
-
 /* return -1 if no bit is set */
 static int get_highest_priority_int(uint32_t *tab)
 {
@@ -318,7 +302,7 @@ static uint8_t apic_get_tpr(APICCommonState *s)
     return s->tpr >> 4;
 }
 
-static int apic_get_ppr(APICCommonState *s)
+int apic_get_ppr(APICCommonState *s)
 {
     int tpr, isrv, ppr;
 
@@ -739,7 +723,7 @@ static uint32_t apic_mem_readl(void *opaque, hwaddr addr)
         val = s->divide_conf;
         break;
     default:
-        s->esr |= ESR_ILLEGAL_ADDRESS;
+        s->esr |= APIC_ESR_ILLEGAL_ADDRESS;
         val = 0;
         break;
     }
@@ -852,7 +836,7 @@ static void apic_mem_writel(void *opaque, hwaddr addr, uint32_t val)
         }
         break;
     default:
-        s->esr |= ESR_ILLEGAL_ADDRESS;
+        s->esr |= APIC_ESR_ILLEGAL_ADDRESS;
         break;
     }
 }
diff --git a/hw/intc/apic_common.c b/hw/intc/apic_common.c
index 0032b97c5..ad959c4e7 100644
--- a/hw/intc/apic_common.c
+++ b/hw/intc/apic_common.c
@@ -296,7 +296,6 @@ static void apic_common_realize(DeviceState *dev, Error **errp)
     APICCommonClass *info;
     static DeviceState *vapic;
     static int apic_no;
-    static bool mmio_registered;
 
     if (apic_no >= MAX_APICS) {
         error_setg(errp, "%s initialization failed.",
@@ -307,11 +306,6 @@ static void apic_common_realize(DeviceState *dev, Error **errp)
 
     info = APIC_COMMON_GET_CLASS(s);
     info->realize(dev, errp);
-    if (!mmio_registered) {
-        ICCBus *b = ICC_BUS(qdev_get_parent_bus(dev));
-        memory_region_add_subregion(b->apic_address_space, 0, &s->io_memory);
-        mmio_registered = true;
-    }
 
     /* Note: We need at least 1M to map the VAPIC option ROM */
     if (!vapic && s->vapic_control & VAPIC_ENABLE_MASK &&
@@ -425,13 +419,12 @@ static Property apic_properties_common[] = {
 
 static void apic_common_class_init(ObjectClass *klass, void *data)
 {
-    ICCDeviceClass *idc = ICC_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     dc->vmsd = &vmstate_apic_common;
     dc->reset = apic_reset_common;
     dc->props = apic_properties_common;
-    idc->realize = apic_common_realize;
+    dc->realize = apic_common_realize;
     /*
      * Reason: APIC and CPU need to be wired up by
      * x86_cpu_apic_create()
@@ -441,7 +434,7 @@ static void apic_common_class_init(ObjectClass *klass, void *data)
 
 static const TypeInfo apic_common_type = {
     .name = TYPE_APIC_COMMON,
-    .parent = TYPE_ICC_DEVICE,
+    .parent = TYPE_DEVICE,
     .instance_size = sizeof(APICCommonState),
     .class_size = sizeof(APICCommonClass),
     .class_init = apic_common_class_init,
diff --git a/hw/intc/arm_gic.c b/hw/intc/arm_gic.c
index 454bfd7df..13e297d52 100644
--- a/hw/intc/arm_gic.c
+++ b/hw/intc/arm_gic.c
@@ -35,8 +35,6 @@ static const uint8_t gic_id[] = {
     0x90, 0x13, 0x04, 0x00, 0x0d, 0xf0, 0x05, 0xb1
 };
 
-#define NUM_CPU(s) ((s)->num_cpu)
-
 static inline int gic_get_current_cpu(GICState *s)
 {
     if (s->num_cpu > 1) {
@@ -64,7 +62,7 @@ void gic_update(GICState *s)
     int cpu;
     int cm;
 
-    for (cpu = 0; cpu < NUM_CPU(s); cpu++) {
+    for (cpu = 0; cpu < s->num_cpu; cpu++) {
         cm = 1 << cpu;
         s->current_pending[cpu] = 1023;
         if (!(s->ctlr & (GICD_CTLR_EN_GRP0 | GICD_CTLR_EN_GRP1))
@@ -219,15 +217,99 @@ static uint16_t gic_get_current_pending_irq(GICState *s, int cpu,
     return pending_irq;
 }
 
-static void gic_set_running_irq(GICState *s, int cpu, int irq)
+static int gic_get_group_priority(GICState *s, int cpu, int irq)
 {
-    s->running_irq[cpu] = irq;
-    if (irq == 1023) {
-        s->running_priority[cpu] = 0x100;
+    /* Return the group priority of the specified interrupt
+     * (which is the top bits of its priority, with the number
+     * of bits masked determined by the applicable binary point register).
+     */
+    int bpr;
+    uint32_t mask;
+
+    if (gic_has_groups(s) &&
+        !(s->cpu_ctlr[cpu] & GICC_CTLR_CBPR) &&
+        GIC_TEST_GROUP(irq, (1 << cpu))) {
+        bpr = s->abpr[cpu];
     } else {
-        s->running_priority[cpu] = GIC_GET_PRIORITY(irq, cpu);
+        bpr = s->bpr[cpu];
     }
-    gic_update(s);
+
+    /* a BPR of 0 means the group priority bits are [7:1];
+     * a BPR of 1 means they are [7:2], and so on down to
+     * a BPR of 7 meaning no group priority bits at all.
+     */
+    mask = ~0U << ((bpr & 7) + 1);
+
+    return GIC_GET_PRIORITY(irq, cpu) & mask;
+}
+
+static void gic_activate_irq(GICState *s, int cpu, int irq)
+{
+    /* Set the appropriate Active Priority Register bit for this IRQ,
+     * and update the running priority.
+     */
+    int prio = gic_get_group_priority(s, cpu, irq);
+    int preemption_level = prio >> (GIC_MIN_BPR + 1);
+    int regno = preemption_level / 32;
+    int bitno = preemption_level % 32;
+
+    if (gic_has_groups(s) && GIC_TEST_GROUP(irq, (1 << cpu))) {
+        s->nsapr[regno][cpu] |= (1 << bitno);
+    } else {
+        s->apr[regno][cpu] |= (1 << bitno);
+    }
+
+    s->running_priority[cpu] = prio;
+    GIC_SET_ACTIVE(irq, 1 << cpu);
+}
+
+static int gic_get_prio_from_apr_bits(GICState *s, int cpu)
+{
+    /* Recalculate the current running priority for this CPU based
+     * on the set bits in the Active Priority Registers.
+     */
+    int i;
+    for (i = 0; i < GIC_NR_APRS; i++) {
+        uint32_t apr = s->apr[i][cpu] | s->nsapr[i][cpu];
+        if (!apr) {
+            continue;
+        }
+        return (i * 32 + ctz32(apr)) << (GIC_MIN_BPR + 1);
+    }
+    return 0x100;
+}
+
+static void gic_drop_prio(GICState *s, int cpu, int group)
+{
+    /* Drop the priority of the currently active interrupt in the
+     * specified group.
+     *
+     * Note that we can guarantee (because of the requirement to nest
+     * GICC_IAR reads [which activate an interrupt and raise priority]
+     * with GICC_EOIR writes [which drop the priority for the interrupt])
+     * that the interrupt we're being called for is the highest priority
+     * active interrupt, meaning that it has the lowest set bit in the
+     * APR registers.
+     *
+     * If the guest does not honour the ordering constraints then the
+     * behaviour of the GIC is UNPREDICTABLE, which for us means that
+     * the values of the APR registers might become incorrect and the
+     * running priority will be wrong, so interrupts that should preempt
+     * might not do so, and interrupts that should not preempt might do so.
+     */
+    int i;
+
+    for (i = 0; i < GIC_NR_APRS; i++) {
+        uint32_t *papr = group ? &s->nsapr[i][cpu] : &s->apr[i][cpu];
+        if (!*papr) {
+            continue;
+        }
+        /* Clear lowest set bit */
+        *papr &= *papr - 1;
+        break;
+    }
+
+    s->running_priority[cpu] = gic_get_prio_from_apr_bits(s, cpu);
 }
 
 uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
@@ -239,7 +321,7 @@ uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
      * for the case where this GIC supports grouping and the pending interrupt
      * is in the wrong group.
      */
-    irq = gic_get_current_pending_irq(s, cpu, attrs);;
+    irq = gic_get_current_pending_irq(s, cpu, attrs);
 
     if (irq >= GIC_MAXIRQ) {
         DPRINTF("ACK, no pending interrupt or it is hidden: %d\n", irq);
@@ -250,7 +332,6 @@ uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
         DPRINTF("ACK, pending interrupt (%d) has insufficient priority\n", irq);
         return 1023;
     }
-    s->last_active[irq][cpu] = s->running_irq[cpu];
 
     if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
         /* Clear pending flags for both level and edge triggered interrupts.
@@ -281,7 +362,8 @@ uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
         }
     }
 
-    gic_set_running_irq(s, cpu, irq);
+    gic_activate_irq(s, cpu, irq);
+    gic_update(s);
     DPRINTF("ACK %d\n", irq);
     return ret;
 }
@@ -411,8 +493,9 @@ static uint8_t gic_get_running_priority(GICState *s, int cpu, MemTxAttrs attrs)
 
 void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
 {
-    int update = 0;
     int cm = 1 << cpu;
+    int group;
+
     DPRINTF("EOI %d\n", irq);
     if (irq >= s->num_irq) {
         /* This handles two cases:
@@ -425,8 +508,9 @@ void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
          */
         return;
     }
-    if (s->running_irq[cpu] == 1023)
+    if (s->running_priority[cpu] == 0x100) {
         return; /* No active IRQ.  */
+    }
 
     if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
         /* Mark level triggered interrupts as pending if they are still
@@ -435,11 +519,12 @@ void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
             && GIC_TEST_LEVEL(irq, cm) && (GIC_TARGET(irq) & cm) != 0) {
             DPRINTF("Set %d pending mask %x\n", irq, cm);
             GIC_SET_PENDING(irq, cm);
-            update = 1;
         }
     }
 
-    if (s->security_extn && !attrs.secure && !GIC_TEST_GROUP(irq, cm)) {
+    group = gic_has_groups(s) && GIC_TEST_GROUP(irq, cm);
+
+    if (s->security_extn && !attrs.secure && !group) {
         DPRINTF("Non-secure EOI for Group0 interrupt %d ignored\n", irq);
         return;
     }
@@ -449,23 +534,9 @@ void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
      * i.e. go ahead and complete the irq anyway.
      */
 
-    if (irq != s->running_irq[cpu]) {
-        /* Complete an IRQ that is not currently running.  */
-        int tmp = s->running_irq[cpu];
-        while (s->last_active[tmp][cpu] != 1023) {
-            if (s->last_active[tmp][cpu] == irq) {
-                s->last_active[tmp][cpu] = s->last_active[irq][cpu];
-                break;
-            }
-            tmp = s->last_active[tmp][cpu];
-        }
-        if (update) {
-            gic_update(s);
-        }
-    } else {
-        /* Complete the current running IRQ.  */
-        gic_set_running_irq(s, cpu, s->last_active[s->running_irq[cpu]][cpu]);
-    }
+    gic_drop_prio(s, cpu, group);
+    GIC_CLEAR_ACTIVE(irq, cm);
+    gic_update(s);
 }
 
 static uint32_t gic_dist_readb(void *opaque, hwaddr offset, MemTxAttrs attrs)
@@ -494,7 +565,7 @@ static uint32_t gic_dist_readb(void *opaque, hwaddr offset, MemTxAttrs attrs)
         if (offset == 4)
             /* Interrupt Controller Type Register */
             return ((s->num_irq / 32) - 1)
-                    | ((NUM_CPU(s) - 1) << 5)
+                    | ((s->num_cpu - 1) << 5)
                     | (s->security_extn << 10);
         if (offset < 0x08)
             return 0;
@@ -922,11 +993,67 @@ static MemTxResult gic_dist_write(void *opaque, hwaddr offset, uint64_t data,
     }
 }
 
-static const MemoryRegionOps gic_dist_ops = {
-    .read_with_attrs = gic_dist_read,
-    .write_with_attrs = gic_dist_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
+static inline uint32_t gic_apr_ns_view(GICState *s, int cpu, int regno)
+{
+    /* Return the Nonsecure view of GICC_APR<regno>. This is the
+     * second half of GICC_NSAPR.
+     */
+    switch (GIC_MIN_BPR) {
+    case 0:
+        if (regno < 2) {
+            return s->nsapr[regno + 2][cpu];
+        }
+        break;
+    case 1:
+        if (regno == 0) {
+            return s->nsapr[regno + 1][cpu];
+        }
+        break;
+    case 2:
+        if (regno == 0) {
+            return extract32(s->nsapr[0][cpu], 16, 16);
+        }
+        break;
+    case 3:
+        if (regno == 0) {
+            return extract32(s->nsapr[0][cpu], 8, 8);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    return 0;
+}
+
+static inline void gic_apr_write_ns_view(GICState *s, int cpu, int regno,
+                                         uint32_t value)
+{
+    /* Write the Nonsecure view of GICC_APR<regno>. */
+    switch (GIC_MIN_BPR) {
+    case 0:
+        if (regno < 2) {
+            s->nsapr[regno + 2][cpu] = value;
+        }
+        break;
+    case 1:
+        if (regno == 0) {
+            s->nsapr[regno + 1][cpu] = value;
+        }
+        break;
+    case 2:
+        if (regno == 0) {
+            s->nsapr[0][cpu] = deposit32(s->nsapr[0][cpu], 16, 16, value);
+        }
+        break;
+    case 3:
+        if (regno == 0) {
+            s->nsapr[0][cpu] = deposit32(s->nsapr[0][cpu], 8, 8, value);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
 
 static MemTxResult gic_cpu_read(GICState *s, int cpu, int offset,
                                 uint64_t *data, MemTxAttrs attrs)
@@ -968,8 +1095,31 @@ static MemTxResult gic_cpu_read(GICState *s, int cpu, int offset,
         }
         break;
     case 0xd0: case 0xd4: case 0xd8: case 0xdc:
-        *data = s->apr[(offset - 0xd0) / 4][cpu];
+    {
+        int regno = (offset - 0xd0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            *data = 0;
+        } else if (s->security_extn && !attrs.secure) {
+            /* NS view of GICC_APR<n> is the top half of GIC_NSAPR<n> */
+            *data = gic_apr_ns_view(s, regno, cpu);
+        } else {
+            *data = s->apr[regno][cpu];
+        }
+        break;
+    }
+    case 0xe0: case 0xe4: case 0xe8: case 0xec:
+    {
+        int regno = (offset - 0xe0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2 || !gic_has_groups(s) ||
+            (s->security_extn && !attrs.secure)) {
+            *data = 0;
+        } else {
+            *data = s->nsapr[regno][cpu];
+        }
         break;
+    }
     default:
         qemu_log_mask(LOG_GUEST_ERROR,
                       "gic_cpu_read: Bad offset %x\n", (int)offset);
@@ -1007,8 +1157,33 @@ static MemTxResult gic_cpu_write(GICState *s, int cpu, int offset,
         }
         break;
     case 0xd0: case 0xd4: case 0xd8: case 0xdc:
-        qemu_log_mask(LOG_UNIMP, "Writing APR not implemented\n");
+    {
+        int regno = (offset - 0xd0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            return MEMTX_OK;
+        }
+        if (s->security_extn && !attrs.secure) {
+            /* NS view of GICC_APR<n> is the top half of GIC_NSAPR<n> */
+            gic_apr_write_ns_view(s, regno, cpu, value);
+        } else {
+            s->apr[regno][cpu] = value;
+        }
+        break;
+    }
+    case 0xe0: case 0xe4: case 0xe8: case 0xec:
+    {
+        int regno = (offset - 0xe0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            return MEMTX_OK;
+        }
+        if (!gic_has_groups(s) || (s->security_extn && !attrs.secure)) {
+            return MEMTX_OK;
+        }
+        s->nsapr[regno][cpu] = value;
         break;
+    }
     default:
         qemu_log_mask(LOG_GUEST_ERROR,
                       "gic_cpu_write: Bad offset %x\n", (int)offset);
@@ -1056,10 +1231,17 @@ static MemTxResult gic_do_cpu_write(void *opaque, hwaddr addr,
     return gic_cpu_write(s, id, addr, value, attrs);
 }
 
-static const MemoryRegionOps gic_thiscpu_ops = {
-    .read_with_attrs = gic_thiscpu_read,
-    .write_with_attrs = gic_thiscpu_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+static const MemoryRegionOps gic_ops[2] = {
+    {
+        .read_with_attrs = gic_dist_read,
+        .write_with_attrs = gic_dist_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    },
+    {
+        .read_with_attrs = gic_thiscpu_read,
+        .write_with_attrs = gic_thiscpu_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    }
 };
 
 static const MemoryRegionOps gic_cpu_ops = {
@@ -1068,31 +1250,10 @@ static const MemoryRegionOps gic_cpu_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
+/* This function is used by nvic model */
 void gic_init_irqs_and_distributor(GICState *s)
 {
-    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
-    int i;
-
-    i = s->num_irq - GIC_INTERNAL;
-    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
-     * GPIO array layout is thus:
-     *  [0..N-1] SPIs
-     *  [N..N+31] PPIs for CPU 0
-     *  [N+32..N+63] PPIs for CPU 1
-     *   ...
-     */
-    if (s->revision != REV_NVIC) {
-        i += (GIC_INTERNAL * s->num_cpu);
-    }
-    qdev_init_gpio_in(DEVICE(s), gic_set_irq, i);
-    for (i = 0; i < NUM_CPU(s); i++) {
-        sysbus_init_irq(sbd, &s->parent_irq[i]);
-    }
-    for (i = 0; i < NUM_CPU(s); i++) {
-        sysbus_init_irq(sbd, &s->parent_fiq[i]);
-    }
-    memory_region_init_io(&s->iomem, OBJECT(s), &gic_dist_ops, s,
-                          "gic_dist", 0x1000);
+    gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
 }
 
 static void arm_gic_realize(DeviceState *dev, Error **errp)
@@ -1110,28 +1271,22 @@ static void arm_gic_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    gic_init_irqs_and_distributor(s);
+    /* This creates distributor and main CPU interface (s->cpuiomem[0]) */
+    gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
 
-    /* Memory regions for the CPU interfaces (NVIC doesn't have these):
-     * a region for "CPU interface for this core", then a region for
-     * "CPU interface for core 0", "for core 1", ...
+    /* Extra core-specific regions for the CPU interfaces. This is
+     * necessary for "franken-GIC" implementations, for example on
+     * Exynos 4.
      * NB that the memory region size of 0x100 applies for the 11MPCore
      * and also cores following the GIC v1 spec (ie A9).
      * GIC v2 defines a larger memory region (0x1000) so this will need
      * to be extended when we implement A15.
      */
-    memory_region_init_io(&s->cpuiomem[0], OBJECT(s), &gic_thiscpu_ops, s,
-                          "gic_cpu", 0x100);
-    for (i = 0; i < NUM_CPU(s); i++) {
+    for (i = 0; i < s->num_cpu; i++) {
         s->backref[i] = s;
         memory_region_init_io(&s->cpuiomem[i+1], OBJECT(s), &gic_cpu_ops,
                               &s->backref[i], "gic_cpu", 0x100);
-    }
-    /* Distributor */
-    sysbus_init_mmio(sbd, &s->iomem);
-    /* cpu interfaces (one for "current cpu" plus one per cpu) */
-    for (i = 0; i <= NUM_CPU(s); i++) {
-        sysbus_init_mmio(sbd, &s->cpuiomem[i]);
+        sysbus_init_mmio(sbd, &s->cpuiomem[i+1]);
     }
 }
 
diff --git a/hw/intc/arm_gic_common.c b/hw/intc/arm_gic_common.c
index a64d0714e..9c82b9729 100644
--- a/hw/intc/arm_gic_common.c
+++ b/hw/intc/arm_gic_common.c
@@ -19,6 +19,7 @@
  */
 
 #include "gic_internal.h"
+#include "hw/arm/linux-boot-if.h"
 
 static void gic_pre_save(void *opaque)
 {
@@ -59,8 +60,8 @@ static const VMStateDescription vmstate_gic_irq_state = {
 
 static const VMStateDescription vmstate_gic = {
     .name = "arm_gic",
-    .version_id = 10,
-    .minimum_version_id = 10,
+    .version_id = 12,
+    .minimum_version_id = 12,
     .pre_save = gic_pre_save,
     .post_load = gic_post_load,
     .fields = (VMStateField[]) {
@@ -71,19 +72,59 @@ static const VMStateDescription vmstate_gic = {
         VMSTATE_UINT8_ARRAY(irq_target, GICState, GIC_MAXIRQ),
         VMSTATE_UINT8_2DARRAY(priority1, GICState, GIC_INTERNAL, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(priority2, GICState, GIC_MAXIRQ - GIC_INTERNAL),
-        VMSTATE_UINT16_2DARRAY(last_active, GICState, GIC_MAXIRQ, GIC_NCPU),
         VMSTATE_UINT8_2DARRAY(sgi_pending, GICState, GIC_NR_SGIS, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(priority_mask, GICState, GIC_NCPU),
-        VMSTATE_UINT16_ARRAY(running_irq, GICState, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(running_priority, GICState, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(current_pending, GICState, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(bpr, GICState, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(abpr, GICState, GIC_NCPU),
         VMSTATE_UINT32_2DARRAY(apr, GICState, GIC_NR_APRS, GIC_NCPU),
+        VMSTATE_UINT32_2DARRAY(nsapr, GICState, GIC_NR_APRS, GIC_NCPU),
         VMSTATE_END_OF_LIST()
     }
 };
 
+void gic_init_irqs_and_mmio(GICState *s, qemu_irq_handler handler,
+                            const MemoryRegionOps *ops)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
+    int i = s->num_irq - GIC_INTERNAL;
+
+    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
+     * GPIO array layout is thus:
+     *  [0..N-1] SPIs
+     *  [N..N+31] PPIs for CPU 0
+     *  [N+32..N+63] PPIs for CPU 1
+     *   ...
+     */
+    if (s->revision != REV_NVIC) {
+        i += (GIC_INTERNAL * s->num_cpu);
+    }
+    qdev_init_gpio_in(DEVICE(s), handler, i);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_irq[i]);
+    }
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_fiq[i]);
+    }
+
+    /* Distributor */
+    memory_region_init_io(&s->iomem, OBJECT(s), ops, s, "gic_dist", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    if (s->revision != REV_NVIC) {
+        /* This is the main CPU interface "for this core". It is always
+         * present because it is required by both software emulation and KVM.
+         * NVIC is not handled here because its CPU interface is different,
+         * neither it can use KVM.
+         */
+        memory_region_init_io(&s->cpuiomem[0], OBJECT(s), ops ? &ops[1] : NULL,
+                              s, "gic_cpu", s->revision == 2 ? 0x1000 : 0x100);
+        sysbus_init_mmio(sbd, &s->cpuiomem[0]);
+    }
+}
+
 static void arm_gic_common_realize(DeviceState *dev, Error **errp)
 {
     GICState *s = ARM_GIC_COMMON(dev);
@@ -124,21 +165,35 @@ static void arm_gic_common_reset(DeviceState *dev)
 {
     GICState *s = ARM_GIC_COMMON(dev);
     int i, j;
+    int resetprio;
+
+    /* If we're resetting a TZ-aware GIC as if secure firmware
+     * had set it up ready to start a kernel in non-secure,
+     * we need to set interrupt priorities to a "zero for the
+     * NS view" value. This is particularly critical for the
+     * priority_mask[] values, because if they are zero then NS
+     * code cannot ever rewrite the priority to anything else.
+     */
+    if (s->security_extn && s->irq_reset_nonsecure) {
+        resetprio = 0x80;
+    } else {
+        resetprio = 0;
+    }
+
     memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
     for (i = 0 ; i < s->num_cpu; i++) {
         if (s->revision == REV_11MPCORE) {
             s->priority_mask[i] = 0xf0;
         } else {
-            s->priority_mask[i] = 0;
+            s->priority_mask[i] = resetprio;
         }
         s->current_pending[i] = 1023;
-        s->running_irq[i] = 1023;
         s->running_priority[i] = 0x100;
         s->cpu_ctlr[i] = 0;
         s->bpr[i] = GIC_MIN_BPR;
         s->abpr[i] = GIC_MIN_ABPR;
         for (j = 0; j < GIC_INTERNAL; j++) {
-            s->priority1[j][i] = 0;
+            s->priority1[j][i] = resetprio;
         }
         for (j = 0; j < GIC_NR_SGIS; j++) {
             s->sgi_pending[j][i] = 0;
@@ -150,7 +205,7 @@ static void arm_gic_common_reset(DeviceState *dev)
     }
 
     for (i = 0; i < ARRAY_SIZE(s->priority2); i++) {
-        s->priority2[i] = 0;
+        s->priority2[i] = resetprio;
     }
 
     for (i = 0; i < GIC_MAXIRQ; i++) {
@@ -161,9 +216,32 @@ static void arm_gic_common_reset(DeviceState *dev)
             s->irq_target[i] = 0;
         }
     }
+    if (s->security_extn && s->irq_reset_nonsecure) {
+        for (i = 0; i < GIC_MAXIRQ; i++) {
+            GIC_SET_GROUP(i, ALL_CPU_MASK);
+        }
+    }
+
     s->ctlr = 0;
 }
 
+static void arm_gic_common_linux_init(ARMLinuxBootIf *obj,
+                                      bool secure_boot)
+{
+    GICState *s = ARM_GIC_COMMON(obj);
+
+    if (s->security_extn && !secure_boot) {
+        /* We're directly booting a kernel into NonSecure. If this GIC
+         * implements the security extensions then we must configure it
+         * to have all the interrupts be NonSecure (this is a job that
+         * is done by the Secure boot firmware in real hardware, and in
+         * this mode QEMU is acting as a minimalist firmware-and-bootloader
+         * equivalent).
+         */
+        s->irq_reset_nonsecure = true;
+    }
+}
+
 static Property arm_gic_common_properties[] = {
     DEFINE_PROP_UINT32("num-cpu", GICState, num_cpu, 1),
     DEFINE_PROP_UINT32("num-irq", GICState, num_irq, 32),
@@ -180,11 +258,13 @@ static Property arm_gic_common_properties[] = {
 static void arm_gic_common_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMLinuxBootIfClass *albifc = ARM_LINUX_BOOT_IF_CLASS(klass);
 
     dc->reset = arm_gic_common_reset;
     dc->realize = arm_gic_common_realize;
     dc->props = arm_gic_common_properties;
     dc->vmsd = &vmstate_gic;
+    albifc->arm_linux_init = arm_gic_common_linux_init;
 }
 
 static const TypeInfo arm_gic_common_type = {
@@ -194,6 +274,10 @@ static const TypeInfo arm_gic_common_type = {
     .class_size = sizeof(ARMGICCommonClass),
     .class_init = arm_gic_common_class_init,
     .abstract = true,
+    .interfaces = (InterfaceInfo []) {
+        { TYPE_ARM_LINUX_BOOT_IF },
+        { },
+    },
 };
 
 static void register_types(void)
diff --git a/hw/intc/arm_gic_kvm.c b/hw/intc/arm_gic_kvm.c
index f56bff1af..0ceebbf87 100644
--- a/hw/intc/arm_gic_kvm.c
+++ b/hw/intc/arm_gic_kvm.c
@@ -20,9 +20,11 @@
  */
 
 #include "hw/sysbus.h"
+#include "migration/migration.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "gic_internal.h"
+#include "vgic_common.h"
 
 //#define DEBUG_GIC_KVM
 
@@ -52,7 +54,7 @@ typedef struct KVMARMGICClass {
     void (*parent_reset)(DeviceState *dev);
 } KVMARMGICClass;
 
-static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
+void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level)
 {
     /* Meaning of the 'irq' parameter:
      *  [0..N-1] : external interrupts
@@ -63,10 +65,9 @@ static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
      * has separate fields in the irq number for type,
      * CPU number and interrupt number.
      */
-    GICState *s = (GICState *)opaque;
     int kvm_irq, irqtype, cpu;
 
-    if (irq < (s->num_irq - GIC_INTERNAL)) {
+    if (irq < (num_irq - GIC_INTERNAL)) {
         /* External interrupt. The kernel numbers these like the GIC
          * hardware, with external interrupt IDs starting after the
          * internal ones.
@@ -77,7 +78,7 @@ static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
     } else {
         /* Internal interrupt: decode into (cpu, interrupt id) */
         irqtype = KVM_ARM_IRQ_TYPE_PPI;
-        irq -= (s->num_irq - GIC_INTERNAL);
+        irq -= (num_irq - GIC_INTERNAL);
         cpu = irq / GIC_INTERNAL;
         irq %= GIC_INTERNAL;
     }
@@ -87,69 +88,36 @@ static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
     kvm_set_irq(kvm_state, kvm_irq, !!level);
 }
 
-static bool kvm_arm_gic_can_save_restore(GICState *s)
-{
-    return s->dev_fd >= 0;
-}
-
-static bool kvm_gic_supports_attr(GICState *s, int group, int attrnum)
+static void kvm_arm_gicv2_set_irq(void *opaque, int irq, int level)
 {
-    struct kvm_device_attr attr = {
-        .group = group,
-        .attr = attrnum,
-        .flags = 0,
-    };
-
-    if (s->dev_fd == -1) {
-        return false;
-    }
+    GICState *s = (GICState *)opaque;
 
-    return kvm_device_ioctl(s->dev_fd, KVM_HAS_DEVICE_ATTR, &attr) == 0;
+    kvm_arm_gic_set_irq(s->num_irq, irq, level);
 }
 
-static void kvm_gic_access(GICState *s, int group, int offset,
-                                   int cpu, uint32_t *val, bool write)
+static bool kvm_arm_gic_can_save_restore(GICState *s)
 {
-    struct kvm_device_attr attr;
-    int type;
-    int err;
-
-    cpu = cpu & 0xff;
-
-    attr.flags = 0;
-    attr.group = group;
-    attr.attr = (((uint64_t)cpu << KVM_DEV_ARM_VGIC_CPUID_SHIFT) &
-                 KVM_DEV_ARM_VGIC_CPUID_MASK) |
-                (((uint64_t)offset << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) &
-                 KVM_DEV_ARM_VGIC_OFFSET_MASK);
-    attr.addr = (uintptr_t)val;
-
-    if (write) {
-        type = KVM_SET_DEVICE_ATTR;
-    } else {
-        type = KVM_GET_DEVICE_ATTR;
-    }
-
-    err = kvm_device_ioctl(s->dev_fd, type, &attr);
-    if (err < 0) {
-        fprintf(stderr, "KVM_{SET/GET}_DEVICE_ATTR failed: %s\n",
-                strerror(-err));
-        abort();
-    }
+    return s->dev_fd >= 0;
 }
 
+#define KVM_VGIC_ATTR(offset, cpu) \
+    ((((uint64_t)(cpu) << KVM_DEV_ARM_VGIC_CPUID_SHIFT) & \
+      KVM_DEV_ARM_VGIC_CPUID_MASK) | \
+     (((uint64_t)(offset) << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) & \
+      KVM_DEV_ARM_VGIC_OFFSET_MASK))
+
 static void kvm_gicd_access(GICState *s, int offset, int cpu,
                             uint32_t *val, bool write)
 {
-    kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
-                   offset, cpu, val, write);
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
+                      KVM_VGIC_ATTR(offset, cpu), val, write);
 }
 
 static void kvm_gicc_access(GICState *s, int offset, int cpu,
                             uint32_t *val, bool write)
 {
-    kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_CPU_REGS,
-                   offset, cpu, val, write);
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_REGS,
+                      KVM_VGIC_ATTR(offset, cpu), val, write);
 }
 
 #define for_each_irq_reg(_ctr, _max_irq, _field_width) \
@@ -340,11 +308,6 @@ static void kvm_arm_gic_put(GICState *s)
     int num_cpu;
     int num_irq;
 
-    if (!kvm_arm_gic_can_save_restore(s)) {
-            DPRINTF("Cannot put kernel gic state, no kernel interface");
-            return;
-    }
-
     /* Note: We do the restore in a slightly different order than the save
      * (where the order doesn't matter and is simply ordered according to the
      * register offset values */
@@ -444,11 +407,6 @@ static void kvm_arm_gic_get(GICState *s)
     int i;
     int cpu;
 
-    if (!kvm_arm_gic_can_save_restore(s)) {
-            DPRINTF("Cannot get kernel gic state, no kernel interface");
-            return;
-    }
-
     /*****************************************************************
      * Distributor State
      */
@@ -536,14 +494,16 @@ static void kvm_arm_gic_reset(DeviceState *dev)
     KVMARMGICClass *kgc = KVM_ARM_GIC_GET_CLASS(s);
 
     kgc->parent_reset(dev);
-    kvm_arm_gic_put(s);
+
+    if (kvm_arm_gic_can_save_restore(s)) {
+        kvm_arm_gic_put(s);
+    }
 }
 
 static void kvm_arm_gic_realize(DeviceState *dev, Error **errp)
 {
     int i;
     GICState *s = KVM_ARM_GIC(dev);
-    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
     KVMARMGICClass *kgc = KVM_ARM_GIC_GET_CLASS(s);
     Error *local_err = NULL;
     int ret;
@@ -560,58 +520,37 @@ static void kvm_arm_gic_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    i = s->num_irq - GIC_INTERNAL;
-    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
-     * GPIO array layout is thus:
-     *  [0..N-1] SPIs
-     *  [N..N+31] PPIs for CPU 0
-     *  [N+32..N+63] PPIs for CPU 1
-     *   ...
-     */
-    i += (GIC_INTERNAL * s->num_cpu);
-    qdev_init_gpio_in(dev, kvm_arm_gic_set_irq, i);
+    gic_init_irqs_and_mmio(s, kvm_arm_gicv2_set_irq, NULL);
 
     for (i = 0; i < s->num_irq - GIC_INTERNAL; i++) {
         qemu_irq irq = qdev_get_gpio_in(dev, i);
         kvm_irqchip_set_qemuirq_gsi(kvm_state, irq, i);
     }
 
-    /* We never use our outbound IRQ/FIQ lines but provide them so that
-     * we maintain the same interface as the non-KVM GIC.
-     */
-    for (i = 0; i < s->num_cpu; i++) {
-        sysbus_init_irq(sbd, &s->parent_irq[i]);
-    }
-    for (i = 0; i < s->num_cpu; i++) {
-        sysbus_init_irq(sbd, &s->parent_fiq[i]);
-    }
-
     /* Try to create the device via the device control API */
     s->dev_fd = -1;
     ret = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V2, false);
     if (ret >= 0) {
         s->dev_fd = ret;
+
+        /* Newstyle API is used, we may have attributes */
+        if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0)) {
+            uint32_t numirqs = s->num_irq;
+            kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0,
+                              &numirqs, true);
+        }
+        /* Tell the kernel to complete VGIC initialization now */
+        if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                                  KVM_DEV_ARM_VGIC_CTRL_INIT)) {
+            kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                              KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
+        }
     } else if (ret != -ENODEV && ret != -ENOTSUP) {
         error_setg_errno(errp, -ret, "error creating in-kernel VGIC");
         return;
     }
 
-    if (kvm_gic_supports_attr(s, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0)) {
-        uint32_t numirqs = s->num_irq;
-        kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0, 0, &numirqs, 1);
-    }
-
-    /* Tell the kernel to complete VGIC initialization now */
-    if (kvm_gic_supports_attr(s, KVM_DEV_ARM_VGIC_GRP_CTRL,
-                              KVM_DEV_ARM_VGIC_CTRL_INIT)) {
-        kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_CTRL,
-                          KVM_DEV_ARM_VGIC_CTRL_INIT, 0, 0, 1);
-    }
-
     /* Distributor */
-    memory_region_init_reservation(&s->iomem, OBJECT(s),
-                                   "kvm-gic_dist", 0x1000);
-    sysbus_init_mmio(sbd, &s->iomem);
     kvm_arm_register_device(&s->iomem,
                             (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
                             | KVM_VGIC_V2_ADDR_TYPE_DIST,
@@ -622,15 +561,18 @@ static void kvm_arm_gic_realize(DeviceState *dev, Error **errp)
      * provide the "interface for core #N" memory regions, because
      * cores with a VGIC don't have those.
      */
-    memory_region_init_reservation(&s->cpuiomem[0], OBJECT(s),
-                                   "kvm-gic_cpu", 0x1000);
-    sysbus_init_mmio(sbd, &s->cpuiomem[0]);
     kvm_arm_register_device(&s->cpuiomem[0],
                             (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
                             | KVM_VGIC_V2_ADDR_TYPE_CPU,
                             KVM_DEV_ARM_VGIC_GRP_ADDR,
                             KVM_VGIC_V2_ADDR_TYPE_CPU,
                             s->dev_fd);
+
+    if (!kvm_arm_gic_can_save_restore(s)) {
+        error_setg(&s->migration_blocker, "This operating system kernel does "
+                                          "not support vGICv2 migration");
+        migrate_add_blocker(s->migration_blocker);
+    }
 }
 
 static void kvm_arm_gic_class_init(ObjectClass *klass, void *data)
diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c
new file mode 100644
index 000000000..032ece216
--- /dev/null
+++ b/hw/intc/arm_gicv3_common.c
@@ -0,0 +1,140 @@
+/*
+ * ARM GICv3 support - common bits of emulated and KVM kernel model
+ *
+ * Copyright (c) 2012 Linaro Limited
+ * Copyright (c) 2015 Huawei.
+ * Written by Peter Maydell
+ * Extended to 64 cores by Shlomo Pongratz
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/intc/arm_gicv3_common.h"
+
+static void gicv3_pre_save(void *opaque)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    ARMGICv3CommonClass *c = ARM_GICV3_COMMON_GET_CLASS(s);
+
+    if (c->pre_save) {
+        c->pre_save(s);
+    }
+}
+
+static int gicv3_post_load(void *opaque, int version_id)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    ARMGICv3CommonClass *c = ARM_GICV3_COMMON_GET_CLASS(s);
+
+    if (c->post_load) {
+        c->post_load(s);
+    }
+    return 0;
+}
+
+static const VMStateDescription vmstate_gicv3 = {
+    .name = "arm_gicv3",
+    .unmigratable = 1,
+    .pre_save = gicv3_pre_save,
+    .post_load = gicv3_post_load,
+};
+
+void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler,
+                              const MemoryRegionOps *ops)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
+    int i;
+
+    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
+     * GPIO array layout is thus:
+     *  [0..N-1] spi
+     *  [N..N+31] PPIs for CPU 0
+     *  [N+32..N+63] PPIs for CPU 1
+     *   ...
+     */
+    i = s->num_irq - GIC_INTERNAL + GIC_INTERNAL * s->num_cpu;
+    qdev_init_gpio_in(DEVICE(s), handler, i);
+
+    s->parent_irq = g_malloc(s->num_cpu * sizeof(qemu_irq));
+    s->parent_fiq = g_malloc(s->num_cpu * sizeof(qemu_irq));
+
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_irq[i]);
+    }
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_fiq[i]);
+    }
+
+    memory_region_init_io(&s->iomem_dist, OBJECT(s), ops, s,
+                          "gicv3_dist", 0x10000);
+    memory_region_init_io(&s->iomem_redist, OBJECT(s), ops ? &ops[1] : NULL, s,
+                          "gicv3_redist", 0x20000 * s->num_cpu);
+
+    sysbus_init_mmio(sbd, &s->iomem_dist);
+    sysbus_init_mmio(sbd, &s->iomem_redist);
+}
+
+static void arm_gicv3_common_realize(DeviceState *dev, Error **errp)
+{
+    GICv3State *s = ARM_GICV3_COMMON(dev);
+
+    /* revision property is actually reserved and currently used only in order
+     * to keep the interface compatible with GICv2 code, avoiding extra
+     * conditions. However, in future it could be used, for example, if we
+     * implement GICv4.
+     */
+    if (s->revision != 3) {
+        error_setg(errp, "unsupported GIC revision %d", s->revision);
+        return;
+    }
+}
+
+static void arm_gicv3_common_reset(DeviceState *dev)
+{
+    /* TODO */
+}
+
+static Property arm_gicv3_common_properties[] = {
+    DEFINE_PROP_UINT32("num-cpu", GICv3State, num_cpu, 1),
+    DEFINE_PROP_UINT32("num-irq", GICv3State, num_irq, 32),
+    DEFINE_PROP_UINT32("revision", GICv3State, revision, 3),
+    DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void arm_gicv3_common_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = arm_gicv3_common_reset;
+    dc->realize = arm_gicv3_common_realize;
+    dc->props = arm_gicv3_common_properties;
+    dc->vmsd = &vmstate_gicv3;
+}
+
+static const TypeInfo arm_gicv3_common_type = {
+    .name = TYPE_ARM_GICV3_COMMON,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(GICv3State),
+    .class_size = sizeof(ARMGICv3CommonClass),
+    .class_init = arm_gicv3_common_class_init,
+    .abstract = true,
+};
+
+static void register_types(void)
+{
+    type_register_static(&arm_gicv3_common_type);
+}
+
+type_init(register_types)
diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
new file mode 100644
index 000000000..b48f78f13
--- /dev/null
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -0,0 +1,149 @@
+/*
+ * ARM Generic Interrupt Controller using KVM in-kernel support
+ *
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Written by Pavel Fedin
+ * Based on vGICv2 code by Peter Maydell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/intc/arm_gicv3_common.h"
+#include "hw/sysbus.h"
+#include "sysemu/kvm.h"
+#include "kvm_arm.h"
+#include "vgic_common.h"
+
+#ifdef DEBUG_GICV3_KVM
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "kvm_gicv3: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+#define TYPE_KVM_ARM_GICV3 "kvm-arm-gicv3"
+#define KVM_ARM_GICV3(obj) \
+     OBJECT_CHECK(GICv3State, (obj), TYPE_KVM_ARM_GICV3)
+#define KVM_ARM_GICV3_CLASS(klass) \
+     OBJECT_CLASS_CHECK(KVMARMGICv3Class, (klass), TYPE_KVM_ARM_GICV3)
+#define KVM_ARM_GICV3_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(KVMARMGICv3Class, (obj), TYPE_KVM_ARM_GICV3)
+
+typedef struct KVMARMGICv3Class {
+    ARMGICv3CommonClass parent_class;
+    DeviceRealize parent_realize;
+    void (*parent_reset)(DeviceState *dev);
+} KVMARMGICv3Class;
+
+static void kvm_arm_gicv3_set_irq(void *opaque, int irq, int level)
+{
+    GICv3State *s = (GICv3State *)opaque;
+
+    kvm_arm_gic_set_irq(s->num_irq, irq, level);
+}
+
+static void kvm_arm_gicv3_put(GICv3State *s)
+{
+    /* TODO */
+    DPRINTF("Cannot put kernel gic state, no kernel interface\n");
+}
+
+static void kvm_arm_gicv3_get(GICv3State *s)
+{
+    /* TODO */
+    DPRINTF("Cannot get kernel gic state, no kernel interface\n");
+}
+
+static void kvm_arm_gicv3_reset(DeviceState *dev)
+{
+    GICv3State *s = ARM_GICV3_COMMON(dev);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
+
+    DPRINTF("Reset\n");
+
+    kgc->parent_reset(dev);
+    kvm_arm_gicv3_put(s);
+}
+
+static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
+{
+    GICv3State *s = KVM_ARM_GICV3(dev);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
+    Error *local_err = NULL;
+
+    DPRINTF("kvm_arm_gicv3_realize\n");
+
+    kgc->parent_realize(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (s->security_extn) {
+        error_setg(errp, "the in-kernel VGICv3 does not implement the "
+                   "security extensions");
+        return;
+    }
+
+    gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL);
+
+    /* Try to create the device via the device control API */
+    s->dev_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V3, false);
+    if (s->dev_fd < 0) {
+        error_setg_errno(errp, -s->dev_fd, "error creating in-kernel VGIC");
+        return;
+    }
+
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
+                      0, &s->num_irq, true);
+
+    /* Tell the kernel to complete VGIC initialization now */
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                      KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
+
+    kvm_arm_register_device(&s->iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
+                            KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd);
+    kvm_arm_register_device(&s->iomem_redist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
+                            KVM_VGIC_V3_ADDR_TYPE_REDIST, s->dev_fd);
+}
+
+static void kvm_arm_gicv3_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_CLASS(klass);
+
+    agcc->pre_save = kvm_arm_gicv3_get;
+    agcc->post_load = kvm_arm_gicv3_put;
+    kgc->parent_realize = dc->realize;
+    kgc->parent_reset = dc->reset;
+    dc->realize = kvm_arm_gicv3_realize;
+    dc->reset = kvm_arm_gicv3_reset;
+}
+
+static const TypeInfo kvm_arm_gicv3_info = {
+    .name = TYPE_KVM_ARM_GICV3,
+    .parent = TYPE_ARM_GICV3_COMMON,
+    .instance_size = sizeof(GICv3State),
+    .class_init = kvm_arm_gicv3_class_init,
+    .class_size = sizeof(KVMARMGICv3Class),
+};
+
+static void kvm_arm_gicv3_register_types(void)
+{
+    type_register_static(&kvm_arm_gicv3_info);
+}
+
+type_init(kvm_arm_gicv3_register_types)
diff --git a/hw/intc/armv7m_nvic.c b/hw/intc/armv7m_nvic.c
index e13b729e1..6fc167e23 100644
--- a/hw/intc/armv7m_nvic.c
+++ b/hw/intc/armv7m_nvic.c
@@ -28,6 +28,7 @@ typedef struct {
     MemoryRegion gic_iomem_alias;
     MemoryRegion container;
     uint32_t num_irq;
+    qemu_irq sysresetreq;
 } nvic_state;
 
 #define TYPE_NVIC "armv7m_nvic"
@@ -185,26 +186,25 @@ static uint32_t nvic_readl(nvic_state *s, uint32_t offset)
         return cpu->midr;
     case 0xd04: /* Interrupt Control State.  */
         /* VECTACTIVE */
-        val = s->gic.running_irq[0];
+        cpu = ARM_CPU(current_cpu);
+        val = cpu->env.v7m.exception;
         if (val == 1023) {
             val = 0;
         } else if (val >= 32) {
             val -= 16;
         }
-        /* RETTOBASE */
-        if (s->gic.running_irq[0] == 1023
-                || s->gic.last_active[s->gic.running_irq[0]][0] == 1023) {
-            val |= (1 << 11);
-        }
         /* VECTPENDING */
         if (s->gic.current_pending[0] != 1023)
             val |= (s->gic.current_pending[0] << 12);
-        /* ISRPENDING */
+        /* ISRPENDING and RETTOBASE */
         for (irq = 32; irq < s->num_irq; irq++) {
             if (s->gic.irq_state[irq].pending) {
                 val |= (1 << 22);
                 break;
             }
+            if (irq != cpu->env.v7m.exception && s->gic.irq_state[irq].active) {
+                val |= (1 << 11);
+            }
         }
         /* PENDSTSET */
         if (s->gic.irq_state[ARMV7M_EXCP_SYSTICK].pending)
@@ -349,10 +349,13 @@ static void nvic_writel(nvic_state *s, uint32_t offset, uint32_t value)
         break;
     case 0xd0c: /* Application Interrupt/Reset Control.  */
         if ((value >> 16) == 0x05fa) {
+            if (value & 4) {
+                qemu_irq_pulse(s->sysresetreq);
+            }
             if (value & 2) {
                 qemu_log_mask(LOG_UNIMP, "VECTCLRACTIVE unimplemented\n");
             }
-            if (value & 5) {
+            if (value & 1) {
                 qemu_log_mask(LOG_UNIMP, "AIRCR system reset unimplemented\n");
             }
             if (value & 0x700) {
@@ -536,11 +539,14 @@ static void armv7m_nvic_instance_init(Object *obj)
      * value in the GICState struct.
      */
     GICState *s = ARM_GIC_COMMON(obj);
+    DeviceState *dev = DEVICE(obj);
+    nvic_state *nvic = NVIC(obj);
     /* The ARM v7m may have anything from 0 to 496 external interrupt
      * IRQ lines. We default to 64. Other boards may differ and should
      * set the num-irq property appropriately.
      */
     s->num_irq = 64;
+    qdev_init_gpio_out_named(dev, &nvic->sysresetreq, "SYSRESETREQ", 1);
 }
 
 static void armv7m_nvic_class_init(ObjectClass *klass, void *data)
diff --git a/hw/intc/imx_avic.c b/hw/intc/imx_avic.c
index e48f66c8f..e0535ffc5 100644
--- a/hw/intc/imx_avic.c
+++ b/hw/intc/imx_avic.c
@@ -7,6 +7,7 @@
  * Copyright (c) 2008 OKL
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under the GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
@@ -14,67 +15,22 @@
  * TODO: implement vectors.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
-#include "qemu/host-utils.h"
+#include "hw/intc/imx_avic.h"
 
-#define DEBUG_INT 1
-#undef DEBUG_INT /* comment out for debugging */
-
-#ifdef DEBUG_INT
-#define DPRINTF(fmt, args...) \
-do { printf("imx_avic: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
+#ifndef DEBUG_IMX_AVIC
+#define DEBUG_IMX_AVIC 0
 #endif
 
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-    do  { fprintf(stderr, "imx_avic: " fmt, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_AVIC_NUM_IRQS 64
-
-/* Interrupt Control Bits */
-#define ABFLAG (1<<25)
-#define ABFEN (1<<24)
-#define NIDIS (1<<22) /* Normal Interrupt disable */
-#define FIDIS (1<<21) /* Fast interrupt disable */
-#define NIAD  (1<<20) /* Normal Interrupt Arbiter Rise ARM level */
-#define FIAD  (1<<19) /* Fast Interrupt Arbiter Rise ARM level */
-#define NM    (1<<18) /* Normal interrupt mode */
-
-
-#define PRIO_PER_WORD (sizeof(uint32_t) * 8 / 4)
-#define PRIO_WORDS (IMX_AVIC_NUM_IRQS/PRIO_PER_WORD)
-
-#define TYPE_IMX_AVIC "imx_avic"
-#define IMX_AVIC(obj) \
-    OBJECT_CHECK(IMXAVICState, (obj), TYPE_IMX_AVIC)
-
-typedef struct IMXAVICState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    uint64_t pending;
-    uint64_t enabled;
-    uint64_t is_fiq;
-    uint32_t intcntl;
-    uint32_t intmask;
-    qemu_irq irq;
-    qemu_irq fiq;
-    uint32_t prio[PRIO_WORDS]; /* Priorities are 4-bits each */
-} IMXAVICState;
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_AVIC) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_AVIC, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static const VMStateDescription vmstate_imx_avic = {
-    .name = "imx-avic",
+    .name = TYPE_IMX_AVIC,
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
@@ -88,8 +44,6 @@ static const VMStateDescription vmstate_imx_avic = {
     },
 };
 
-
-
 static inline int imx_avic_prio(IMXAVICState *s, int irq)
 {
     uint32_t word = irq / PRIO_PER_WORD;
@@ -151,8 +105,8 @@ static uint64_t imx_avic_read(void *opaque,
 {
     IMXAVICState *s = (IMXAVICState *)opaque;
 
+    DPRINTF("read(offset = 0x%" HWADDR_PRIx ")\n", offset);
 
-    DPRINTF("read(offset = 0x%x)\n", offset >> 2);
     switch (offset >> 2) {
     case 0: /* INTCNTL */
         return s->intcntl;
@@ -249,7 +203,8 @@ static uint64_t imx_avic_read(void *opaque,
         return 0x4;
 
     default:
-        IPRINTF("imx_avic_read: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_AVIC, __func__, offset);
         return 0;
     }
 }
@@ -261,13 +216,13 @@ static void imx_avic_write(void *opaque, hwaddr offset,
 
     /* Vector Registers not yet supported */
     if (offset >= 0x100 && offset <= 0x2fc) {
-        IPRINTF("imx_avic_write to vector register %d ignored\n",
-                (unsigned int)((offset - 0x100) >> 2));
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: vector %d ignored\n",
+                      TYPE_IMX_AVIC, __func__, (int)((offset - 0x100) >> 2));
         return;
     }
 
-    DPRINTF("imx_avic_write(0x%x) = %x\n",
-            (unsigned int)offset>>2, (unsigned int)val);
+    DPRINTF("(0x%" HWADDR_PRIx ") = 0x%x\n", offset, (unsigned int)val);
+
     switch (offset >> 2) {
     case 0: /* Interrupt Control Register, INTCNTL */
         s->intcntl = val & (ABFEN | NIDIS | FIDIS | NIAD | FIAD | NM);
@@ -341,7 +296,8 @@ static void imx_avic_write(void *opaque, hwaddr offset,
         return;
 
     default:
-        IPRINTF("imx_avic_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_AVIC, __func__, offset);
     }
     imx_avic_update(s);
 }
@@ -370,7 +326,7 @@ static int imx_avic_init(SysBusDevice *sbd)
     IMXAVICState *s = IMX_AVIC(dev);
 
     memory_region_init_io(&s->iomem, OBJECT(s), &imx_avic_ops, s,
-                          "imx_avic", 0x1000);
+                          TYPE_IMX_AVIC, 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
 
     qdev_init_gpio_in(dev, imx_avic_set_irq, IMX_AVIC_NUM_IRQS);
diff --git a/hw/intc/ioapic.c b/hw/intc/ioapic.c
index b52793238..de2dd4ba8 100644
--- a/hw/intc/ioapic.c
+++ b/hw/intc/ioapic.c
@@ -20,6 +20,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "monitor/monitor.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/ioapic.h"
@@ -98,7 +99,9 @@ static void ioapic_set_irq(void *opaque, int vector, int level)
             /* level triggered */
             if (level) {
                 s->irr |= mask;
-                ioapic_service(s);
+                if (!(entry & IOAPIC_LVT_REMOTE_IRR)) {
+                    ioapic_service(s);
+                }
             } else {
                 s->irr &= ~mask;
             }
@@ -137,6 +140,17 @@ void ioapic_eoi_broadcast(int vector)
     }
 }
 
+void ioapic_dump_state(Monitor *mon, const QDict *qdict)
+{
+    int i;
+
+    for (i = 0; i < MAX_IOAPICS; i++) {
+        if (ioapics[i] != 0) {
+            ioapic_print_redtbl(mon, ioapics[i]);
+        }
+    }
+}
+
 static uint64_t
 ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
 {
@@ -154,15 +168,13 @@ ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
         }
         switch (s->ioregsel) {
         case IOAPIC_REG_ID:
+        case IOAPIC_REG_ARB:
             val = s->id << IOAPIC_ID_SHIFT;
             break;
         case IOAPIC_REG_VER:
             val = IOAPIC_VERSION |
                 ((IOAPIC_NUM_PINS - 1) << IOAPIC_VER_ENTRIES_SHIFT);
             break;
-        case IOAPIC_REG_ARB:
-            val = 0;
-            break;
         default:
             index = (s->ioregsel - IOAPIC_REG_REDTBL_BASE) >> 1;
             if (index >= 0 && index < IOAPIC_NUM_PINS) {
diff --git a/hw/intc/ioapic_common.c b/hw/intc/ioapic_common.c
index 8b7d11806..65f6877d7 100644
--- a/hw/intc/ioapic_common.c
+++ b/hw/intc/ioapic_common.c
@@ -19,6 +19,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "monitor/monitor.h"
 #include "hw/i386/ioapic.h"
 #include "hw/i386/ioapic_internal.h"
 #include "hw/sysbus.h"
@@ -31,6 +32,60 @@
  */
 int ioapic_no;
 
+static void ioapic_irr_dump(Monitor *mon, const char *name, uint32_t bitmap)
+{
+    int i;
+
+    monitor_printf(mon, "%-10s ", name);
+    if (bitmap == 0) {
+        monitor_printf(mon, "(none)\n");
+        return;
+    }
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        if (bitmap & (1 << i)) {
+            monitor_printf(mon, "%-2u ", i);
+        }
+    }
+    monitor_printf(mon, "\n");
+}
+
+void ioapic_print_redtbl(Monitor *mon, IOAPICCommonState *s)
+{
+    static const char *delm_str[] = {
+        "fixed", "lowest", "SMI", "...", "NMI", "INIT", "...", "extINT"};
+    uint32_t remote_irr = 0;
+    int i;
+
+    monitor_printf(mon, "ioapic id=0x%02x sel=0x%02x", s->id, s->ioregsel);
+    if (s->ioregsel) {
+        monitor_printf(mon, " (redir[%u])\n",
+                       (s->ioregsel - IOAPIC_REG_REDTBL_BASE) >> 1);
+    } else {
+        monitor_printf(mon, "\n");
+    }
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        uint64_t entry = s->ioredtbl[i];
+        uint32_t delm = (uint32_t)((entry & IOAPIC_LVT_DELIV_MODE) >>
+                                   IOAPIC_LVT_DELIV_MODE_SHIFT);
+        monitor_printf(mon, "pin %-2u 0x%016"PRIx64" dest=%"PRIx64
+                       " vec=%-3"PRIu64" %s %-5s %-6s %-6s %s\n",
+                       i, entry,
+                       (entry >> IOAPIC_LVT_DEST_SHIFT) &
+                            (entry & IOAPIC_LVT_DEST_MODE ? 0xff : 0xf),
+                       entry & IOAPIC_VECTOR_MASK,
+                       entry & IOAPIC_LVT_POLARITY ? "active-lo" : "active-hi",
+                       entry & IOAPIC_LVT_TRIGGER_MODE ? "level" : "edge",
+                       entry & IOAPIC_LVT_MASKED ? "masked" : "",
+                       delm_str[delm],
+                       entry & IOAPIC_LVT_DEST_MODE ? "logical" : "physical");
+
+        remote_irr |= entry & IOAPIC_LVT_TRIGGER_MODE ?
+                        (entry & IOAPIC_LVT_REMOTE_IRR ? (1 << i) : 0) : 0;
+    }
+    ioapic_irr_dump(mon, "IRR", s->irr);
+    ioapic_irr_dump(mon, "Remote IRR", remote_irr);
+}
+
 void ioapic_reset_common(DeviceState *dev)
 {
     IOAPICCommonState *s = IOAPIC_COMMON(dev);
diff --git a/hw/intc/openpic.c b/hw/intc/openpic.c
index 14ab0e31b..bfcf15535 100644
--- a/hw/intc/openpic.c
+++ b/hw/intc/openpic.c
@@ -1643,6 +1643,7 @@ static void openpic_class_init(ObjectClass *oc, void *data)
     dc->props = openpic_properties;
     dc->reset = openpic_reset;
     dc->vmsd = &vmstate_openpic;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo openpic_info = {
diff --git a/hw/intc/openpic_kvm.c b/hw/intc/openpic_kvm.c
index f7cac585a..649f476ac 100644
--- a/hw/intc/openpic_kvm.c
+++ b/hw/intc/openpic_kvm.c
@@ -275,6 +275,7 @@ static void kvm_openpic_class_init(ObjectClass *oc, void *data)
     dc->realize = kvm_openpic_realize;
     dc->props = kvm_openpic_properties;
     dc->reset = kvm_openpic_reset;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo kvm_openpic_info = {
diff --git a/hw/intc/s390_flic_kvm.c b/hw/intc/s390_flic_kvm.c
index b471e7a41..48714f96a 100644
--- a/hw/intc/s390_flic_kvm.c
+++ b/hw/intc/s390_flic_kvm.c
@@ -228,6 +228,8 @@ static int kvm_s390_add_adapter_routes(S390FLICState *fs,
         routes->gsi[i] = ret;
         routes->adapter.ind_offset++;
     }
+    kvm_irqchip_commit_routes(kvm_state);
+
     /* Restore passed-in structure to original state. */
     routes->adapter.ind_offset = ind_offset;
     return 0;
diff --git a/hw/intc/vgic_common.h b/hw/intc/vgic_common.h
new file mode 100644
index 000000000..80d919eb9
--- /dev/null
+++ b/hw/intc/vgic_common.h
@@ -0,0 +1,35 @@
+/*
+ * ARM KVM vGIC utility functions
+ *
+ * Copyright (c) 2015 Samsung Electronics
+ * Written by Pavel Fedin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef QEMU_ARM_VGIC_COMMON_H
+#define QEMU_ARM_VGIC_COMMON_H
+
+/**
+ * kvm_arm_gic_set_irq - Send an IRQ to the in-kernel vGIC
+ * @num_irq: Total number of IRQs configured for the GIC instance
+ * @irq: qemu internal IRQ line number:
+ *  [0..N-1] : external interrupts
+ *  [N..N+31] : PPI (internal) interrupts for CPU 0
+ *  [N+32..N+63] : PPI (internal interrupts for CPU 1
+ * @level: level of the IRQ line.
+ */
+void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level);
+
+#endif
diff --git a/hw/intc/xics.c b/hw/intc/xics.c
index 924b1ae3c..9ff579641 100644
--- a/hw/intc/xics.c
+++ b/hw/intc/xics.c
@@ -739,7 +739,7 @@ int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi)
 }
 
 /*
- * Allocate block of consequtive IRQs, returns a number of the first.
+ * Allocate block of consecutive IRQs, and return the number of the first IRQ in the block.
  * If align==true, aligns the first IRQ number to num.
  */
 int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align)
@@ -848,7 +848,7 @@ static target_ulong h_xirr_x(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     uint32_t xirr = icp_accept(ss);
 
     args[0] = xirr;
-    args[1] = cpu_get_real_ticks();
+    args[1] = cpu_get_host_ticks();
     return H_SUCCESS;
 }
 
diff --git a/hw/isa/i82378.c b/hw/isa/i82378.c
index fcf97d86a..d4c830684 100644
--- a/hw/isa/i82378.c
+++ b/hw/isa/i82378.c
@@ -100,7 +100,6 @@ static void i82378_realize(PCIDevice *pci, Error **errp)
 
     /* 2 82C37 (dma) */
     isa = isa_create_simple(isabus, "i82374");
-    qdev_connect_gpio_out(DEVICE(isa), 0, s->out[1]);
 
     /* timer */
     isa_create_simple(isabus, "mc146818rtc");
@@ -111,7 +110,7 @@ static void i82378_init(Object *obj)
     DeviceState *dev = DEVICE(obj);
     I82378State *s = I82378(obj);
 
-    qdev_init_gpio_out(dev, s->out, 2);
+    qdev_init_gpio_out(dev, s->out, 1);
     qdev_init_gpio_in(dev, i82378_request_pic_irq, 16);
 }
 
diff --git a/hw/isa/lpc_ich9.c b/hw/isa/lpc_ich9.c
index 360699f6f..1ffc80362 100644
--- a/hw/isa/lpc_ich9.c
+++ b/hw/isa/lpc_ich9.c
@@ -394,7 +394,7 @@ static void ich9_apm_ctrl_changed(uint32_t val, void *arg)
 
     /* SMI_EN = PMBASE + 30. SMI control and enable register */
     if (lpc->pm.smi_en & ICH9_PMIO_SMI_EN_APMC_EN) {
-        cpu_interrupt(first_cpu, CPU_INTERRUPT_SMI);
+        cpu_interrupt(current_cpu, CPU_INTERRUPT_SMI);
     }
 }
 
diff --git a/hw/lm32/lm32_boards.c b/hw/lm32/lm32_boards.c
index 70f48d3b1..eb553a174 100644
--- a/hw/lm32/lm32_boards.c
+++ b/hw/lm32/lm32_boards.c
@@ -142,7 +142,7 @@ static void lm32_evr_init(MachineState *machine)
         int kernel_size;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -244,7 +244,7 @@ static void lm32_uclinux_init(MachineState *machine)
         int kernel_size;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -292,24 +292,40 @@ static void lm32_uclinux_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static QEMUMachine lm32_evr_machine = {
-    .name = "lm32-evr",
-    .desc = "LatticeMico32 EVR32 eval system",
-    .init = lm32_evr_init,
-    .is_default = 1,
+static void lm32_evr_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "LatticeMico32 EVR32 eval system";
+    mc->init = lm32_evr_init;
+    mc->is_default = 1;
+}
+
+static const TypeInfo lm32_evr_type = {
+    .name = MACHINE_TYPE_NAME("lm32-evr"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm32_evr_class_init,
 };
 
-static QEMUMachine lm32_uclinux_machine = {
-    .name = "lm32-uclinux",
-    .desc = "lm32 platform for uClinux and u-boot by Theobroma Systems",
-    .init = lm32_uclinux_init,
-    .is_default = 0,
+static void lm32_uclinux_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lm32 platform for uClinux and u-boot by Theobroma Systems";
+    mc->init = lm32_uclinux_init;
+    mc->is_default = 0;
+}
+
+static const TypeInfo lm32_uclinux_type = {
+    .name = MACHINE_TYPE_NAME("lm32-uclinux"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm32_uclinux_class_init,
 };
 
 static void lm32_machine_init(void)
 {
-    qemu_register_machine(&lm32_uclinux_machine);
-    qemu_register_machine(&lm32_evr_machine);
+    type_register_static(&lm32_evr_type);
+    type_register_static(&lm32_uclinux_type);
 }
 
-machine_init(lm32_machine_init);
+machine_init(lm32_machine_init)
diff --git a/hw/lm32/milkymist-hw.h b/hw/lm32/milkymist-hw.h
index 8d20cac1d..c8dfb4d2d 100644
--- a/hw/lm32/milkymist-hw.h
+++ b/hw/lm32/milkymist-hw.h
@@ -88,7 +88,8 @@ static inline DeviceState *milkymist_pfpu_create(hwaddr base,
 
 #ifdef CONFIG_OPENGL
 #include <X11/Xlib.h>
-#include <GL/glx.h>
+#include <epoxy/gl.h>
+#include <epoxy/glx.h>
 static const int glx_fbconfig_attr[] = {
     GLX_GREEN_SIZE, 5,
     GLX_GREEN_SIZE, 6,
diff --git a/hw/lm32/milkymist.c b/hw/lm32/milkymist.c
index e755f5b24..13976b348 100644
--- a/hw/lm32/milkymist.c
+++ b/hw/lm32/milkymist.c
@@ -176,7 +176,7 @@ milkymist_init(MachineState *machine)
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -209,16 +209,11 @@ milkymist_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static QEMUMachine milkymist_machine = {
-    .name = "milkymist",
-    .desc = "Milkymist One",
-    .init = milkymist_init,
-    .is_default = 0,
-};
-
-static void milkymist_machine_init(void)
+static void milkymist_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&milkymist_machine);
+    mc->desc = "Milkymist One";
+    mc->init = milkymist_init;
+    mc->is_default = 0;
 }
 
-machine_init(milkymist_machine_init);
+DEFINE_MACHINE("milkymist", milkymist_machine_init)
diff --git a/hw/m68k/an5206.c b/hw/m68k/an5206.c
index f63ab2b94..c1dea17f3 100644
--- a/hw/m68k/an5206.c
+++ b/hw/m68k/an5206.c
@@ -54,7 +54,7 @@ static void an5206_init(MachineState *machine)
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "an5206.sram", 512, &error_abort);
+    memory_region_init_ram(sram, NULL, "an5206.sram", 512, &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, AN5206_RAMBAR_ADDR, sram);
 
@@ -70,7 +70,7 @@ static void an5206_init(MachineState *machine)
     }
 
     kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                           NULL, NULL, 1, ELF_MACHINE, 0);
+                           NULL, NULL, 1, EM_68K, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
         kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -89,15 +89,10 @@ static void an5206_init(MachineState *machine)
     env->pc = entry;
 }
 
-static QEMUMachine an5206_machine = {
-    .name = "an5206",
-    .desc = "Arnewsh 5206",
-    .init = an5206_init,
-};
-
-static void an5206_machine_init(void)
+static void an5206_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&an5206_machine);
+    mc->desc = "Arnewsh 5206";
+    mc->init = an5206_init;
 }
 
-machine_init(an5206_machine_init);
+DEFINE_MACHINE("an5206", an5206_machine_init)
diff --git a/hw/m68k/dummy_m68k.c b/hw/m68k/dummy_m68k.c
index 5b77d930e..8b3b77597 100644
--- a/hw/m68k/dummy_m68k.c
+++ b/hw/m68k/dummy_m68k.c
@@ -49,7 +49,7 @@ static void dummy_m68k_init(MachineState *machine)
     /* Load kernel.  */
     if (kernel_filename) {
         kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               NULL, NULL, 1, ELF_MACHINE, 0);
+                               NULL, NULL, 1, EM_68K, 0);
         entry = elf_entry;
         if (kernel_size < 0) {
             kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -72,15 +72,10 @@ static void dummy_m68k_init(MachineState *machine)
     env->pc = entry;
 }
 
-static QEMUMachine dummy_m68k_machine = {
-    .name = "dummy",
-    .desc = "Dummy board",
-    .init = dummy_m68k_init,
-};
-
-static void dummy_m68k_machine_init(void)
+static void dummy_m68k_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&dummy_m68k_machine);
+    mc->desc = "Dummy board";
+    mc->init = dummy_m68k_init;
 }
 
-machine_init(dummy_m68k_machine_init);
+DEFINE_MACHINE("dummy", dummy_m68k_machine_init)
diff --git a/hw/m68k/mcf5208.c b/hw/m68k/mcf5208.c
index 326a42d27..ddeccc506 100644
--- a/hw/m68k/mcf5208.c
+++ b/hw/m68k/mcf5208.c
@@ -222,7 +222,7 @@ static void mcf5208evb_init(MachineState *machine)
     memory_region_add_subregion(address_space_mem, 0x40000000, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384, &error_abort);
+    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384, &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, 0x80000000, sram);
 
@@ -275,7 +275,7 @@ static void mcf5208evb_init(MachineState *machine)
     }
 
     kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                           NULL, NULL, 1, ELF_MACHINE, 0);
+                           NULL, NULL, 1, EM_68K, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
         kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -294,16 +294,11 @@ static void mcf5208evb_init(MachineState *machine)
     env->pc = entry;
 }
 
-static QEMUMachine mcf5208evb_machine = {
-    .name = "mcf5208evb",
-    .desc = "MCF5206EVB",
-    .init = mcf5208evb_init,
-    .is_default = 1,
-};
-
-static void mcf5208evb_machine_init(void)
+static void mcf5208evb_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mcf5208evb_machine);
+    mc->desc = "MCF5206EVB";
+    mc->init = mcf5208evb_init;
+    mc->is_default = 1;
 }
 
-machine_init(mcf5208evb_machine_init);
+DEFINE_MACHINE("mcf5208evb", mcf5208evb_machine_init)
diff --git a/hw/mem/pc-dimm.c b/hw/mem/pc-dimm.c
index bb04862de..d5cdab270 100644
--- a/hw/mem/pc-dimm.c
+++ b/hw/mem/pc-dimm.c
@@ -25,6 +25,7 @@
 #include "sysemu/numa.h"
 #include "sysemu/kvm.h"
 #include "trace.h"
+#include "hw/virtio/vhost.h"
 
 typedef struct pc_dimms_capacity {
      uint64_t size;
@@ -95,6 +96,12 @@ void pc_dimm_memory_plug(DeviceState *dev, MemoryHotplugState *hpms,
         goto out;
     }
 
+    if (!vhost_has_free_slot()) {
+        error_setg(&local_err, "a used vhost backend has no free"
+                               " memory slots left");
+        goto out;
+    }
+
     memory_region_add_subregion(&hpms->mr, addr - hpms->base, mr);
     vmstate_register_ram(mr, dev);
     numa_set_mem_node_id(addr, memory_region_size(mr), dimm->node);
@@ -172,7 +179,7 @@ int qmp_pc_dimm_device_list(Object *obj, void *opaque)
                                                NULL);
             di->memdev = object_get_canonical_path(OBJECT(dimm->hostmem));
 
-            info->dimm = di;
+            info->u.dimm = di;
             elem->value = info;
             elem->next = NULL;
             **prev = elem;
@@ -196,9 +203,9 @@ ram_addr_t get_current_ram_size(void)
         MemoryDeviceInfo *value = info->value;
 
         if (value) {
-            switch (value->kind) {
+            switch (value->type) {
             case MEMORY_DEVICE_INFO_KIND_DIMM:
-                size += value->dimm->size;
+                size += value->u.dimm->size;
                 break;
             default:
                 break;
@@ -414,10 +421,11 @@ static void pc_dimm_realize(DeviceState *dev, Error **errp)
         error_setg(errp, "'" PC_DIMM_MEMDEV_PROP "' property is not set");
         return;
     }
-    if ((nb_numa_nodes > 0) && (dimm->node >= nb_numa_nodes)) {
+    if (((nb_numa_nodes > 0) && (dimm->node >= nb_numa_nodes)) ||
+        (!nb_numa_nodes && dimm->node)) {
         error_setg(errp, "'DIMM property " PC_DIMM_NODE_PROP " has value %"
                    PRIu32 "' which exceeds the number of numa nodes: %d",
-                   dimm->node, nb_numa_nodes);
+                   dimm->node, nb_numa_nodes ? nb_numa_nodes : 1);
         return;
     }
 }
diff --git a/hw/microblaze/boot.c b/hw/microblaze/boot.c
index 3e8820f36..d7eaa1f02 100644
--- a/hw/microblaze/boot.c
+++ b/hw/microblaze/boot.c
@@ -141,12 +141,12 @@ void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
                                &entry, &low, &high,
-                               big_endian, ELF_MACHINE, 0);
+                               big_endian, EM_MICROBLAZE, 0);
         base32 = entry;
         if (base32 == 0xc0000000) {
             kernel_size = load_elf(kernel_filename, translate_kernel_address,
                                    NULL, &entry, NULL, NULL,
-                                   big_endian, ELF_MACHINE, 0);
+                                   big_endian, EM_MICROBLAZE, 0);
         }
         /* Always boot into physical ram.  */
         boot_info.bootstrap_pc = (uint32_t)entry;
diff --git a/hw/microblaze/petalogix_ml605_mmu.c b/hw/microblaze/petalogix_ml605_mmu.c
index ed84a37e6..462060f09 100644
--- a/hw/microblaze/petalogix_ml605_mmu.c
+++ b/hw/microblaze/petalogix_ml605_mmu.c
@@ -92,12 +92,12 @@ petalogix_ml605_init(MachineState *machine)
 
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL, "petalogix_ml605.lmb_bram",
-                           LMB_BRAM_SIZE, &error_abort);
+                           LMB_BRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(address_space_mem, 0x00000000, phys_lmb_bram);
 
     memory_region_init_ram(phys_ram, NULL, "petalogix_ml605.ram", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(address_space_mem, MEMORY_BASEADDR, phys_ram);
 
@@ -206,16 +206,11 @@ petalogix_ml605_init(MachineState *machine)
 
 }
 
-static QEMUMachine petalogix_ml605_machine = {
-    .name = "petalogix-ml605",
-    .desc = "PetaLogix linux refdesign for xilinx ml605 little endian",
-    .init = petalogix_ml605_init,
-    .is_default = 0,
-};
-
-static void petalogix_ml605_machine_init(void)
+static void petalogix_ml605_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&petalogix_ml605_machine);
+    mc->desc = "PetaLogix linux refdesign for xilinx ml605 little endian";
+    mc->init = petalogix_ml605_init;
+    mc->is_default = 0;
 }
 
-machine_init(petalogix_ml605_machine_init);
+DEFINE_MACHINE("petalogix-ml605", petalogix_ml605_machine_init)
diff --git a/hw/microblaze/petalogix_s3adsp1800_mmu.c b/hw/microblaze/petalogix_s3adsp1800_mmu.c
index 0c2140c3f..33811561b 100644
--- a/hw/microblaze/petalogix_s3adsp1800_mmu.c
+++ b/hw/microblaze/petalogix_s3adsp1800_mmu.c
@@ -71,12 +71,12 @@ petalogix_s3adsp1800_init(MachineState *machine)
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL,
                            "petalogix_s3adsp1800.lmb_bram", LMB_BRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(sysmem, 0x00000000, phys_lmb_bram);
 
     memory_region_init_ram(phys_ram, NULL, "petalogix_s3adsp1800.ram",
-                           ram_size, &error_abort);
+                           ram_size, &error_fatal);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(sysmem, ddr_base, phys_ram);
 
@@ -124,16 +124,11 @@ petalogix_s3adsp1800_init(MachineState *machine)
                            NULL);
 }
 
-static QEMUMachine petalogix_s3adsp1800_machine = {
-    .name = "petalogix-s3adsp1800",
-    .desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800",
-    .init = petalogix_s3adsp1800_init,
-    .is_default = 1,
-};
-
-static void petalogix_s3adsp1800_machine_init(void)
+static void petalogix_s3adsp1800_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&petalogix_s3adsp1800_machine);
+    mc->desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800";
+    mc->init = petalogix_s3adsp1800_init;
+    mc->is_default = 1;
 }
 
-machine_init(petalogix_s3adsp1800_machine_init);
+DEFINE_MACHINE("petalogix-s3adsp1800", petalogix_s3adsp1800_machine_init)
diff --git a/hw/mips/cputimer.c b/hw/mips/cputimer.c
index 577c9aeab..f046588ad 100644
--- a/hw/mips/cputimer.c
+++ b/hw/mips/cputimer.c
@@ -25,18 +25,26 @@
 #include "qemu/timer.h"
 #include "sysemu/kvm.h"
 
-#define TIMER_FREQ	100 * 1000 * 1000
+#define TIMER_PERIOD 10 /* 10 ns period for 100 Mhz frequency */
 
 /* XXX: do not use a global */
 uint32_t cpu_mips_get_random (CPUMIPSState *env)
 {
-    static uint32_t lfsr = 1;
+    static uint32_t seed = 1;
     static uint32_t prev_idx = 0;
     uint32_t idx;
+    uint32_t nb_rand_tlb = env->tlb->nb_tlb - env->CP0_Wired;
+
+    if (nb_rand_tlb == 1) {
+        return env->tlb->nb_tlb - 1;
+    }
+
     /* Don't return same value twice, so get another value */
     do {
-        lfsr = (lfsr >> 1) ^ (-(lfsr & 1u) & 0xd0000001u);
-        idx = lfsr % (env->tlb->nb_tlb - env->CP0_Wired) + env->CP0_Wired;
+        /* Use a simple algorithm of Linear Congruential Generator
+         * from ISO/IEC 9899 standard. */
+        seed = 1103515245 * seed + 12345;
+        idx = (seed >> 16) % nb_rand_tlb + env->CP0_Wired;
     } while (idx == prev_idx);
     prev_idx = idx;
     return idx;
@@ -49,9 +57,8 @@ static void cpu_mips_timer_update(CPUMIPSState *env)
     uint32_t wait;
 
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    wait = env->CP0_Compare - env->CP0_Count -
-	    (uint32_t)muldiv64(now, TIMER_FREQ, get_ticks_per_sec());
-    next = now + muldiv64(wait, get_ticks_per_sec(), TIMER_FREQ);
+    wait = env->CP0_Compare - env->CP0_Count - (uint32_t)(now / TIMER_PERIOD);
+    next = now + (uint64_t)wait * TIMER_PERIOD;
     timer_mod(env->timer, next);
 }
 
@@ -79,8 +86,7 @@ uint32_t cpu_mips_get_count (CPUMIPSState *env)
             cpu_mips_timer_expire(env);
         }
 
-        return env->CP0_Count +
-            (uint32_t)muldiv64(now, TIMER_FREQ, get_ticks_per_sec());
+        return env->CP0_Count + (uint32_t)(now / TIMER_PERIOD);
     }
 }
 
@@ -95,9 +101,8 @@ void cpu_mips_store_count (CPUMIPSState *env, uint32_t count)
         env->CP0_Count = count;
     else {
         /* Store new count register */
-        env->CP0_Count =
-            count - (uint32_t)muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                                       TIMER_FREQ, get_ticks_per_sec());
+        env->CP0_Count = count -
+               (uint32_t)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / TIMER_PERIOD);
         /* Update timer timer */
         cpu_mips_timer_update(env);
     }
@@ -121,8 +126,8 @@ void cpu_mips_start_count(CPUMIPSState *env)
 void cpu_mips_stop_count(CPUMIPSState *env)
 {
     /* Store the current value */
-    env->CP0_Count += (uint32_t)muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                                         TIMER_FREQ, get_ticks_per_sec());
+    env->CP0_Count += (uint32_t)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) /
+                                 TIMER_PERIOD);
 }
 
 static void mips_timer_cb (void *opaque)
diff --git a/hw/mips/gt64xxx_pci.c b/hw/mips/gt64xxx_pci.c
index 10fcca33f..f76a9fd36 100644
--- a/hw/mips/gt64xxx_pci.c
+++ b/hw/mips/gt64xxx_pci.c
@@ -275,7 +275,8 @@ static void check_reserved_space (hwaddr *start,
 
 static void gt64120_isd_mapping(GT64120State *s)
 {
-    hwaddr start = s->regs[GT_ISD] << 21;
+    /* Bits 14:0 of ISD map to bits 35:21 of the start address.  */
+    hwaddr start = ((hwaddr)s->regs[GT_ISD] << 21) & 0xFFFE00000ull;
     hwaddr length = 0x1000;
 
     if (s->ISD_length) {
diff --git a/hw/mips/mips_fulong2e.c b/hw/mips/mips_fulong2e.c
index dea941ad8..5988a88c0 100644
--- a/hw/mips/mips_fulong2e.c
+++ b/hw/mips/mips_fulong2e.c
@@ -116,7 +116,7 @@ static int64_t load_kernel (CPUMIPSState *env)
 
     if (load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys, NULL,
                  (uint64_t *)&kernel_entry, (uint64_t *)&kernel_low,
-                 (uint64_t *)&kernel_high, 0, ELF_MACHINE, 1) < 0) {
+                 (uint64_t *)&kernel_high, 0, EM_MIPS, 1) < 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
                 loaderparams.kernel_filename);
         exit(1);
@@ -251,15 +251,6 @@ static void network_init (PCIBus *pci_bus)
     }
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static void mips_fulong2e_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
@@ -274,7 +265,6 @@ static void mips_fulong2e_init(MachineState *machine)
     long bios_size;
     int64_t kernel_entry;
     qemu_irq *i8259;
-    qemu_irq *cpu_exit_irq;
     PCIBus *pci_bus;
     ISABus *isa_bus;
     I2CBus *smbus;
@@ -304,7 +294,7 @@ static void mips_fulong2e_init(MachineState *machine)
     /* allocate RAM */
     memory_region_allocate_system_memory(ram, NULL, "fulong2e.ram", ram_size);
     memory_region_init_ram(bios, NULL, "fulong2e.bios", bios_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -375,8 +365,7 @@ static void mips_fulong2e_init(MachineState *machine)
 
     /* init other devices */
     pit = pit_init(isa_bus, 0x40, 0, NULL);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(0);
 
     /* Super I/O */
     isa_create_simple(isa_bus, "i8042");
@@ -392,15 +381,10 @@ static void mips_fulong2e_init(MachineState *machine)
     network_init(pci_bus);
 }
 
-static QEMUMachine mips_fulong2e_machine = {
-    .name = "fulong2e",
-    .desc = "Fulong 2e mini pc",
-    .init = mips_fulong2e_init,
-};
-
-static void mips_fulong2e_machine_init(void)
+static void mips_fulong2e_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_fulong2e_machine);
+    mc->desc = "Fulong 2e mini pc";
+    mc->init = mips_fulong2e_init;
 }
 
-machine_init(mips_fulong2e_machine_init);
+DEFINE_MACHINE("fulong2e", mips_fulong2e_machine_init)
diff --git a/hw/mips/mips_jazz.c b/hw/mips/mips_jazz.c
index 9d60633ef..1ab885bb3 100644
--- a/hw/mips/mips_jazz.c
+++ b/hw/mips/mips_jazz.c
@@ -104,15 +104,6 @@ static const MemoryRegionOps dma_dummy_ops = {
 #define MAGNUM_BIOS_SIZE_MAX 0x7e000
 #define MAGNUM_BIOS_SIZE (BIOS_SIZE < MAGNUM_BIOS_SIZE_MAX ? BIOS_SIZE : MAGNUM_BIOS_SIZE_MAX)
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static CPUUnassignedAccess real_do_unassigned_access;
 static void mips_jazz_do_unassigned_access(CPUState *cpu, hwaddr addr,
                                            bool is_write, bool is_exec,
@@ -150,7 +141,6 @@ static void mips_jazz_init(MachineState *machine,
     ISADevice *pit;
     DriveInfo *fds[MAX_FD];
     qemu_irq esp_reset, dma_enable;
-    qemu_irq *cpu_exit_irq;
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     MemoryRegion *bios = g_new(MemoryRegion, 1);
     MemoryRegion *bios2 = g_new(MemoryRegion, 1);
@@ -184,7 +174,7 @@ static void mips_jazz_init(MachineState *machine,
     memory_region_add_subregion(address_space, 0, ram);
 
     memory_region_init_ram(bios, NULL, "mips_jazz.bios", MAGNUM_BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
     memory_region_init_alias(bios2, NULL, "mips_jazz.bios", bios,
@@ -234,8 +224,7 @@ static void mips_jazz_init(MachineState *machine,
     /* ISA devices */
     i8259 = i8259_init(isa_bus, env->irq[4]);
     isa_bus_irqs(isa_bus, i8259);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(0);
     pit = pit_init(isa_bus, 0x40, 0, NULL);
     pcspk_init(isa_bus, pit);
 
@@ -252,7 +241,7 @@ static void mips_jazz_init(MachineState *machine,
             /* Simple ROM, so user doesn't have to provide one */
             MemoryRegion *rom_mr = g_new(MemoryRegion, 1);
             memory_region_init_ram(rom_mr, NULL, "g364fb.rom", 0x80000,
-                                   &error_abort);
+                                   &error_fatal);
             vmstate_register_ram_global(rom_mr);
             memory_region_set_readonly(rom_mr, true);
             uint8_t *rom = memory_region_get_ram_ptr(rom_mr);
@@ -360,24 +349,40 @@ void mips_pica61_init(MachineState *machine)
     mips_jazz_init(machine, JAZZ_PICA61);
 }
 
-static QEMUMachine mips_magnum_machine = {
-    .name = "magnum",
-    .desc = "MIPS Magnum",
-    .init = mips_magnum_init,
-    .block_default_type = IF_SCSI,
+static void mips_magnum_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "MIPS Magnum";
+    mc->init = mips_magnum_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo mips_magnum_type = {
+    .name = MACHINE_TYPE_NAME("magnum"),
+    .parent = TYPE_MACHINE,
+    .class_init = mips_magnum_class_init,
 };
 
-static QEMUMachine mips_pica61_machine = {
-    .name = "pica61",
-    .desc = "Acer Pica 61",
-    .init = mips_pica61_init,
-    .block_default_type = IF_SCSI,
+static void mips_pica61_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Acer Pica 61";
+    mc->init = mips_pica61_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo mips_pica61_type = {
+    .name = MACHINE_TYPE_NAME("pica61"),
+    .parent = TYPE_MACHINE,
+    .class_init = mips_pica61_class_init,
 };
 
 static void mips_jazz_machine_init(void)
 {
-    qemu_register_machine(&mips_magnum_machine);
-    qemu_register_machine(&mips_pica61_machine);
+    type_register_static(&mips_magnum_type);
+    type_register_static(&mips_pica61_type);
 }
 
-machine_init(mips_jazz_machine_init);
+machine_init(mips_jazz_machine_init)
diff --git a/hw/mips/mips_malta.c b/hw/mips/mips_malta.c
index 3082e7534..91c36baa5 100644
--- a/hw/mips/mips_malta.c
+++ b/hw/mips/mips_malta.c
@@ -795,7 +795,7 @@ static int64_t load_kernel (void)
 
     if (load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys, NULL,
                  (uint64_t *)&kernel_entry, NULL, (uint64_t *)&kernel_high,
-                 big_endian, ELF_MACHINE, 1) < 0) {
+                 big_endian, EM_MIPS, 1) < 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
                 loaderparams.kernel_filename);
         exit(1);
@@ -901,16 +901,7 @@ static void main_cpu_reset(void *opaque)
 
     if (kvm_enabled()) {
         /* Start running from the bootloader we wrote to end of RAM */
-        env->active_tc.PC = 0x40000000 + loaderparams.ram_size;
-    }
-}
-
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
+        env->active_tc.PC = 0x40000000 + loaderparams.ram_low_size;
     }
 }
 
@@ -939,7 +930,6 @@ void mips_malta_init(MachineState *machine)
     MIPSCPU *cpu;
     CPUMIPSState *env;
     qemu_irq *isa_irq;
-    qemu_irq *cpu_exit_irq;
     int piix4_devfn;
     I2CBus *smbus;
     int i;
@@ -1130,7 +1120,7 @@ void mips_malta_init(MachineState *machine)
      * regions are not executable.
      */
     memory_region_init_ram(bios_copy, NULL, "bios.1fc", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     if (!rom_copy(memory_region_get_ram_ptr(bios_copy),
                   FLASH_ADDRESS, BIOS_SIZE)) {
         memcpy(memory_region_get_ram_ptr(bios_copy),
@@ -1175,8 +1165,7 @@ void mips_malta_init(MachineState *machine)
     smbus_eeprom_init(smbus, 8, smbus_eeprom_buf, smbus_eeprom_size);
     g_free(smbus_eeprom_buf);
     pit = pit_init(isa_bus, 0x40, 0, NULL);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(0);
 
     /* Super I/O */
     isa_create_simple(isa_bus, "i8042");
@@ -1216,23 +1205,19 @@ static const TypeInfo mips_malta_device = {
     .class_init    = mips_malta_class_init,
 };
 
-static QEMUMachine mips_malta_machine = {
-    .name = "malta",
-    .desc = "MIPS Malta Core LV",
-    .init = mips_malta_init,
-    .max_cpus = 16,
-    .is_default = 1,
-};
-
-static void mips_malta_register_types(void)
+static void mips_malta_machine_init(MachineClass *mc)
 {
-    type_register_static(&mips_malta_device);
+    mc->desc = "MIPS Malta Core LV";
+    mc->init = mips_malta_init;
+    mc->max_cpus = 16;
+    mc->is_default = 1;
 }
 
-static void mips_malta_machine_init(void)
+DEFINE_MACHINE("malta", mips_malta_machine_init)
+
+static void mips_malta_register_types(void)
 {
-    qemu_register_machine(&mips_malta_machine);
+    type_register_static(&mips_malta_device);
 }
 
 type_init(mips_malta_register_types)
-machine_init(mips_malta_machine_init);
diff --git a/hw/mips/mips_mipssim.c b/hw/mips/mips_mipssim.c
index 61f74a631..23b35bea2 100644
--- a/hw/mips/mips_mipssim.c
+++ b/hw/mips/mips_mipssim.c
@@ -69,7 +69,7 @@ static int64_t load_kernel(void)
     kernel_size = load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys,
                            NULL, (uint64_t *)&entry, NULL,
                            (uint64_t *)&kernel_high, big_endian,
-                           ELF_MACHINE, 1);
+                           EM_MIPS, 1);
     if (kernel_size >= 0) {
         if ((entry & ~0x7fffffffULL) == 0x80000000)
             entry = (int32_t)entry;
@@ -174,7 +174,7 @@ mips_mipssim_init(MachineState *machine)
     memory_region_allocate_system_memory(ram, NULL, "mips_mipssim.ram",
                                          ram_size);
     memory_region_init_ram(bios, NULL, "mips_mipssim.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -231,15 +231,10 @@ mips_mipssim_init(MachineState *machine)
         mipsnet_init(0x4200, env->irq[2], &nd_table[0]);
 }
 
-static QEMUMachine mips_mipssim_machine = {
-    .name = "mipssim",
-    .desc = "MIPS MIPSsim platform",
-    .init = mips_mipssim_init,
-};
-
-static void mips_mipssim_machine_init(void)
+static void mips_mipssim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_mipssim_machine);
+    mc->desc = "MIPS MIPSsim platform";
+    mc->init = mips_mipssim_init;
 }
 
-machine_init(mips_mipssim_machine_init);
+DEFINE_MACHINE("mipssim", mips_mipssim_machine_init)
diff --git a/hw/mips/mips_r4k.c b/hw/mips/mips_r4k.c
index f4dcacd86..af10da1c5 100644
--- a/hw/mips/mips_r4k.c
+++ b/hw/mips/mips_r4k.c
@@ -87,7 +87,7 @@ static int64_t load_kernel(void)
     kernel_size = load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys,
                            NULL, (uint64_t *)&entry, NULL,
                            (uint64_t *)&kernel_high, big_endian,
-                           ELF_MACHINE, 1);
+                           EM_MIPS, 1);
     if (kernel_size >= 0) {
         if ((entry & ~0x7fffffffULL) == 0x80000000)
             entry = (int32_t)entry;
@@ -233,7 +233,7 @@ void mips_r4k_init(MachineState *machine)
     if ((bios_size > 0) && (bios_size <= BIOS_SIZE)) {
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "mips_r4k.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
         memory_region_set_readonly(bios, true);
         memory_region_add_subregion(get_system_memory(), 0x1fc00000, bios);
@@ -252,9 +252,7 @@ void mips_r4k_init(MachineState *machine)
         fprintf(stderr, "qemu: Warning, could not load MIPS bios '%s'\n",
 		bios_name);
     }
-    if (filename) {
-        g_free(filename);
-    }
+    g_free(filename);
 
     if (kernel_filename) {
         loaderparams.ram_size = ram_size;
@@ -300,15 +298,10 @@ void mips_r4k_init(MachineState *machine)
     isa_create_simple(isa_bus, "i8042");
 }
 
-static QEMUMachine mips_machine = {
-    .name = "mips",
-    .desc = "mips r4k platform",
-    .init = mips_r4k_init,
-};
-
-static void mips_machine_init(void)
+static void mips_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_machine);
+    mc->desc = "mips r4k platform";
+    mc->init = mips_r4k_init;
 }
 
-machine_init(mips_machine_init);
+DEFINE_MACHINE("mips", mips_machine_init)
diff --git a/hw/misc/Makefile.objs b/hw/misc/Makefile.objs
index 4aa76ffec..aeb6b7da7 100644
--- a/hw/misc/Makefile.objs
+++ b/hw/misc/Makefile.objs
@@ -36,6 +36,7 @@ obj-$(CONFIG_OMAP) += omap_sdrc.o
 obj-$(CONFIG_OMAP) += omap_tap.o
 obj-$(CONFIG_SLAVIO) += slavio_misc.o
 obj-$(CONFIG_ZYNQ) += zynq_slcr.o
+obj-$(CONFIG_ZYNQ) += zynq-xadc.o
 obj-$(CONFIG_STM32F2XX_SYSCFG) += stm32f2xx_syscfg.o
 
 obj-$(CONFIG_PVPANIC) += pvpanic.o
diff --git a/hw/misc/imx_ccm.c b/hw/misc/imx_ccm.c
index 092028863..4cc2bbc0e 100644
--- a/hw/misc/imx_ccm.c
+++ b/hw/misc/imx_ccm.c
@@ -2,6 +2,7 @@
  * IMX31 Clock Control Module
  *
  * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -10,51 +11,27 @@
  * the CCM.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
-#include "sysemu/sysemu.h"
-#include "hw/arm/imx.h"
+#include "hw/misc/imx_ccm.h"
 
 #define CKIH_FREQ 26000000 /* 26MHz crystal input */
 #define CKIL_FREQ    32768 /* nominal 32khz clock */
 
+#ifndef DEBUG_IMX_CCM
+#define DEBUG_IMX_CCM 0
+#endif
 
-//#define DEBUG_CCM 1
-#ifdef DEBUG_CCM
 #define DPRINTF(fmt, args...) \
-do { printf("imx_ccm: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
-#endif
+    do { \
+        if (DEBUG_IMX_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static int imx_ccm_post_load(void *opaque, int version_id);
 
-#define TYPE_IMX_CCM "imx_ccm"
-#define IMX_CCM(obj) OBJECT_CHECK(IMXCCMState, (obj), TYPE_IMX_CCM)
-
-typedef struct IMXCCMState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-
-    uint32_t ccmr;
-    uint32_t pdr0;
-    uint32_t pdr1;
-    uint32_t mpctl;
-    uint32_t spctl;
-    uint32_t cgr[3];
-    uint32_t pmcr0;
-    uint32_t pmcr1;
-
-    /* Frequencies precalculated on register changes */
-    uint32_t pll_refclk_freq;
-    uint32_t mcu_clk_freq;
-    uint32_t hsp_clk_freq;
-    uint32_t ipg_clk_freq;
-} IMXCCMState;
-
 static const VMStateDescription vmstate_imx_ccm = {
-    .name = "imx-ccm",
+    .name = TYPE_IMX_CCM,
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
@@ -72,44 +49,6 @@ static const VMStateDescription vmstate_imx_ccm = {
     .post_load = imx_ccm_post_load,
 };
 
-/* CCMR */
-#define CCMR_FPME (1<<0)
-#define CCMR_MPE  (1<<3)
-#define CCMR_MDS  (1<<7)
-#define CCMR_FPMF (1<<26)
-#define CCMR_PRCS (3<<1)
-
-/* PDR0 */
-#define PDR0_MCU_PODF_SHIFT (0)
-#define PDR0_MCU_PODF_MASK (0x7)
-#define PDR0_MAX_PODF_SHIFT (3)
-#define PDR0_MAX_PODF_MASK (0x7)
-#define PDR0_IPG_PODF_SHIFT (6)
-#define PDR0_IPG_PODF_MASK (0x3)
-#define PDR0_NFC_PODF_SHIFT (8)
-#define PDR0_NFC_PODF_MASK (0x7)
-#define PDR0_HSP_PODF_SHIFT (11)
-#define PDR0_HSP_PODF_MASK (0x7)
-#define PDR0_PER_PODF_SHIFT (16)
-#define PDR0_PER_PODF_MASK (0x1f)
-#define PDR0_CSI_PODF_SHIFT (23)
-#define PDR0_CSI_PODF_MASK (0x1ff)
-
-#define EXTRACT(value, name) (((value) >> PDR0_##name##_PODF_SHIFT) \
-                              & PDR0_##name##_PODF_MASK)
-#define INSERT(value, name) (((value) & PDR0_##name##_PODF_MASK) << \
-                             PDR0_##name##_PODF_SHIFT)
-/* PLL control registers */
-#define PD(v) (((v) >> 26) & 0xf)
-#define MFD(v) (((v) >> 16) & 0x3ff)
-#define MFI(v) (((v) >> 10) & 0xf);
-#define MFN(v) ((v) & 0x3ff)
-
-#define PLL_PD(x)               (((x) & 0xf) << 26)
-#define PLL_MFD(x)              (((x) & 0x3ff) << 16)
-#define PLL_MFI(x)              (((x) & 0xf) << 10)
-#define PLL_MFN(x)              (((x) & 0x3ff) << 0)
-
 uint32_t imx_clock_frequency(DeviceState *dev, IMXClk clock)
 {
     IMXCCMState *s = IMX_CCM(dev);
@@ -174,7 +113,7 @@ static void update_clocks(IMXCCMState *s)
     s->hsp_clk_freq = s->mcu_clk_freq / (1 + EXTRACT(s->pdr0, HSP));
     s->ipg_clk_freq = s->hsp_clk_freq / (1 + EXTRACT(s->pdr0, IPG));
 
-    DPRINTF("Clocks: mcu %uMHz, HSP %uMHz, IPG %uHz\n",
+    DPRINTF("mcu %uMHz, HSP %uMHz, IPG %uHz\n",
             s->mcu_clk_freq / 1000000,
             s->hsp_clk_freq / 1000000,
             s->ipg_clk_freq);
@@ -200,7 +139,8 @@ static uint64_t imx_ccm_read(void *opaque, hwaddr offset,
 {
     IMXCCMState *s = (IMXCCMState *)opaque;
 
-    DPRINTF("read(offset=%x)", offset >> 2);
+    DPRINTF("(offset=0x%" HWADDR_PRIx ")\n", offset);
+
     switch (offset >> 2) {
     case 0: /* CCMR */
         DPRINTF(" ccmr = 0x%x\n", s->ccmr);
@@ -231,9 +171,11 @@ static uint64_t imx_ccm_read(void *opaque, hwaddr offset,
     case 23:
         DPRINTF(" pcmr0 = 0x%x\n", s->pmcr0);
         return s->pmcr0;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_CCM, __func__, offset);
+        return 0;
     }
-    DPRINTF(" return 0\n");
-    return 0;
 }
 
 static void imx_ccm_write(void *opaque, hwaddr offset,
@@ -241,8 +183,9 @@ static void imx_ccm_write(void *opaque, hwaddr offset,
 {
     IMXCCMState *s = (IMXCCMState *)opaque;
 
-    DPRINTF("write(offset=%x, value = %x)\n",
-            offset >> 2, (unsigned int)value);
+    DPRINTF("(offset=0x%" HWADDR_PRIx ", value = 0x%x)\n",
+            offset, (unsigned int)value);
+
     switch (offset >> 2) {
     case 0:
         s->ccmr = CCMR_FPMF | (value & 0x3b6fdfff);
@@ -270,6 +213,8 @@ static void imx_ccm_write(void *opaque, hwaddr offset,
         return;
 
     default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_CCM, __func__, offset);
         return;
     }
     update_clocks(s);
@@ -286,7 +231,7 @@ static int imx_ccm_init(SysBusDevice *dev)
     IMXCCMState *s = IMX_CCM(dev);
 
     memory_region_init_io(&s->iomem, OBJECT(dev), &imx_ccm_ops, s,
-                          "imx_ccm", 0x1000);
+                          TYPE_IMX_CCM, 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
 
     return 0;
diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c
index cc76989a3..dcfc8cc5f 100644
--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -19,6 +19,7 @@
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "sysemu/kvm.h"
 #include "migration/migration.h"
@@ -26,6 +27,10 @@
 #include "qemu/event_notifier.h"
 #include "qemu/fifo8.h"
 #include "sysemu/char.h"
+#include "sysemu/hostmem.h"
+#include "qapi/visitor.h"
+
+#include "hw/misc/ivshmem.h"
 
 #include <sys/mman.h>
 #include <sys/types.h>
@@ -34,6 +39,7 @@
 #define PCI_VENDOR_ID_IVSHMEM   PCI_VENDOR_ID_REDHAT_QUMRANET
 #define PCI_DEVICE_ID_IVSHMEM   0x1110
 
+#define IVSHMEM_MAX_PEERS G_MAXUINT16
 #define IVSHMEM_IOEVENTFD   0
 #define IVSHMEM_MSI     1
 
@@ -59,19 +65,19 @@ typedef struct Peer {
     EventNotifier *eventfds;
 } Peer;
 
-typedef struct EventfdEntry {
+typedef struct MSIVector {
     PCIDevice *pdev;
-    int vector;
-} EventfdEntry;
+    int virq;
+} MSIVector;
 
 typedef struct IVShmemState {
     /*< private >*/
     PCIDevice parent_obj;
     /*< public >*/
 
+    HostMemoryBackend *hostmem;
     uint32_t intrmask;
     uint32_t intrstatus;
-    uint32_t doorbell;
 
     CharDriverState **eventfd_chr;
     CharDriverState *server_chr;
@@ -85,18 +91,15 @@ typedef struct IVShmemState {
     MemoryRegion bar;
     MemoryRegion ivshmem;
     uint64_t ivshmem_size; /* size of shared memory region */
-    uint32_t ivshmem_attr;
     uint32_t ivshmem_64bit;
-    int shm_fd; /* shared memory file descriptor */
 
     Peer *peers;
-    int nb_peers; /* how many guests we have space for */
-    int max_peer; /* maximum numbered peer */
+    int nb_peers; /* how many peers we have space for */
 
     int vm_id;
     uint32_t vectors;
     uint32_t features;
-    EventfdEntry *eventfd_table;
+    MSIVector *msi_vectors;
 
     Error *migration_blocker;
 
@@ -119,12 +122,8 @@ static inline uint32_t ivshmem_has_feature(IVShmemState *ivs,
     return (ivs->features & (1 << feature));
 }
 
-static inline bool is_power_of_two(uint64_t x) {
-    return (x & (x - 1)) == 0;
-}
-
 /* accessing registers - based on rtl8139 */
-static void ivshmem_update_irq(IVShmemState *s, int val)
+static void ivshmem_update_irq(IVShmemState *s)
 {
     PCIDevice *d = PCI_DEVICE(s);
     int isr;
@@ -145,7 +144,7 @@ static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val)
 
     s->intrmask = val;
 
-    ivshmem_update_irq(s, val);
+    ivshmem_update_irq(s);
 }
 
 static uint32_t ivshmem_IntrMask_read(IVShmemState *s)
@@ -163,7 +162,7 @@ static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val)
 
     s->intrstatus = val;
 
-    ivshmem_update_irq(s, val);
+    ivshmem_update_irq(s);
 }
 
 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
@@ -173,7 +172,7 @@ static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
     /* reading ISR clears all interrupts */
     s->intrstatus = 0;
 
-    ivshmem_update_irq(s, 0);
+    ivshmem_update_irq(s);
 
     return ret;
 }
@@ -201,7 +200,7 @@ static void ivshmem_io_write(void *opaque, hwaddr addr,
 
         case DOORBELL:
             /* check that dest VM ID is reasonable */
-            if (dest > s->max_peer) {
+            if (dest >= s->nb_peers) {
                 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest);
                 break;
             }
@@ -210,10 +209,13 @@ static void ivshmem_io_write(void *opaque, hwaddr addr,
             if (vector < s->peers[dest].nb_eventfds) {
                 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector);
                 event_notifier_set(&s->peers[dest].eventfds[vector]);
+            } else {
+                IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
+                                vector, dest);
             }
             break;
         default:
-            IVSHMEM_DPRINTF("Invalid VM Doorbell VM %d\n", dest);
+            IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr);
     }
 }
 
@@ -236,7 +238,7 @@ static uint64_t ivshmem_io_read(void *opaque, hwaddr addr,
 
         case IVPOSITION:
             /* return my VM ID if the memory is mapped */
-            if (s->shm_fd > 0) {
+            if (memory_region_is_mapped(&s->ivshmem)) {
                 ret = s->vm_id;
             } else {
                 ret = -1;
@@ -265,14 +267,14 @@ static void ivshmem_receive(void *opaque, const uint8_t *buf, int size)
 {
     IVShmemState *s = opaque;
 
-    ivshmem_IntrStatus_write(s, *buf);
+    IVSHMEM_DPRINTF("ivshmem_receive 0x%02x size: %d\n", *buf, size);
 
-    IVSHMEM_DPRINTF("ivshmem_receive 0x%02x\n", *buf);
+    ivshmem_IntrStatus_write(s, *buf);
 }
 
 static int ivshmem_can_receive(void * opaque)
 {
-    return 8;
+    return sizeof(int64_t);
 }
 
 static void ivshmem_event(void *opaque, int event)
@@ -282,36 +284,94 @@ static void ivshmem_event(void *opaque, int event)
 
 static void fake_irqfd(void *opaque, const uint8_t *buf, int size) {
 
-    EventfdEntry *entry = opaque;
+    MSIVector *entry = opaque;
     PCIDevice *pdev = entry->pdev;
+    IVShmemState *s = IVSHMEM(pdev);
+    int vector = entry - s->msi_vectors;
+
+    IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector);
+    msix_notify(pdev, vector);
+}
+
+static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
+                                 MSIMessage msg)
+{
+    IVShmemState *s = IVSHMEM(dev);
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    MSIVector *v = &s->msi_vectors[vector];
+    int ret;
+
+    IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector);
+
+    ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
+}
 
-    IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, entry->vector);
-    msix_notify(pdev, entry->vector);
+static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
+{
+    IVShmemState *s = IVSHMEM(dev);
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    int ret;
+
+    IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector);
+
+    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n,
+                                                s->msi_vectors[vector].virq);
+    if (ret != 0) {
+        error_report("remove_irqfd_notifier_gsi failed");
+    }
+}
+
+static void ivshmem_vector_poll(PCIDevice *dev,
+                                unsigned int vector_start,
+                                unsigned int vector_end)
+{
+    IVShmemState *s = IVSHMEM(dev);
+    unsigned int vector;
+
+    IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end);
+
+    vector_end = MIN(vector_end, s->vectors);
+
+    for (vector = vector_start; vector < vector_end; vector++) {
+        EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector];
+
+        if (!msix_is_masked(dev, vector)) {
+            continue;
+        }
+
+        if (event_notifier_test_and_clear(notifier)) {
+            msix_set_pending(dev, vector);
+        }
+    }
 }
 
-static CharDriverState* create_eventfd_chr_device(void * opaque, EventNotifier *n,
+static CharDriverState* create_eventfd_chr_device(IVShmemState *s,
+                                                  EventNotifier *n,
                                                   int vector)
 {
     /* create a event character device based on the passed eventfd */
-    IVShmemState *s = opaque;
-    CharDriverState * chr;
     int eventfd = event_notifier_get_fd(n);
+    CharDriverState *chr;
 
     chr = qemu_chr_open_eventfd(eventfd);
 
     if (chr == NULL) {
-        error_report("creating eventfd for eventfd %d failed", eventfd);
-        exit(1);
+        error_report("creating chardriver for eventfd %d failed", eventfd);
+        return NULL;
     }
     qemu_chr_fe_claim_no_fail(chr);
 
     /* if MSI is supported we need multiple interrupts */
     if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
-        s->eventfd_table[vector].pdev = PCI_DEVICE(s);
-        s->eventfd_table[vector].vector = vector;
+        s->msi_vectors[vector].pdev = PCI_DEVICE(s);
 
         qemu_chr_add_handlers(chr, ivshmem_can_receive, fake_irqfd,
-                      ivshmem_event, &s->eventfd_table[vector]);
+                      ivshmem_event, &s->msi_vectors[vector]);
     } else {
         qemu_chr_add_handlers(chr, ivshmem_can_receive, ivshmem_receive,
                       ivshmem_event, s);
@@ -321,22 +381,23 @@ static CharDriverState* create_eventfd_chr_device(void * opaque, EventNotifier *
 
 }
 
-static int check_shm_size(IVShmemState *s, int fd) {
+static int check_shm_size(IVShmemState *s, int fd, Error **errp)
+{
     /* check that the guest isn't going to try and map more memory than the
      * the object has allocated return -1 to indicate error */
 
     struct stat buf;
 
     if (fstat(fd, &buf) < 0) {
-        error_report("exiting: fstat on fd %d failed: %s",
-                     fd, strerror(errno));
+        error_setg(errp, "exiting: fstat on fd %d failed: %s",
+                   fd, strerror(errno));
         return -1;
     }
 
     if (s->ivshmem_size > buf.st_size) {
-        error_report("Requested memory size greater"
-                     " than shared object size (%" PRIu64 " > %" PRIu64")",
-                     s->ivshmem_size, (uint64_t)buf.st_size);
+        error_setg(errp, "Requested memory size greater"
+                   " than shared object size (%" PRIu64 " > %" PRIu64")",
+                   s->ivshmem_size, (uint64_t)buf.st_size);
         return -1;
     } else {
         return 0;
@@ -345,13 +406,16 @@ static int check_shm_size(IVShmemState *s, int fd) {
 
 /* create the shared memory BAR when we are not using the server, so we can
  * create the BAR and map the memory immediately */
-static void create_shared_memory_BAR(IVShmemState *s, int fd) {
-
+static int create_shared_memory_BAR(IVShmemState *s, int fd, uint8_t attr,
+                                    Error **errp)
+{
     void * ptr;
 
-    s->shm_fd = fd;
-
     ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+    if (ptr == MAP_FAILED) {
+        error_setg_errno(errp, errno, "Failed to mmap shared memory");
+        return -1;
+    }
 
     memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), "ivshmem.bar2",
                                s->ivshmem_size, ptr);
@@ -359,7 +423,9 @@ static void create_shared_memory_BAR(IVShmemState *s, int fd) {
     memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
 
     /* region for shared memory */
-    pci_register_bar(PCI_DEVICE(s), 2, s->ivshmem_attr, &s->bar);
+    pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar);
+
+    return 0;
 }
 
 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
@@ -382,25 +448,26 @@ static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i)
                               &s->peers[posn].eventfds[i]);
 }
 
-static void close_guest_eventfds(IVShmemState *s, int posn)
+static void close_peer_eventfds(IVShmemState *s, int posn)
 {
-    int i, guest_curr_max;
+    int i, n;
 
     if (!ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
         return;
     }
     if (posn < 0 || posn >= s->nb_peers) {
+        error_report("invalid peer %d", posn);
         return;
     }
 
-    guest_curr_max = s->peers[posn].nb_eventfds;
+    n = s->peers[posn].nb_eventfds;
 
     memory_region_transaction_begin();
-    for (i = 0; i < guest_curr_max; i++) {
+    for (i = 0; i < n; i++) {
         ivshmem_del_eventfd(s, posn, i);
     }
     memory_region_transaction_commit();
-    for (i = 0; i < guest_curr_max; i++) {
+    for (i = 0; i < n; i++) {
         event_notifier_cleanup(&s->peers[posn].eventfds[i]);
     }
 
@@ -409,125 +476,207 @@ static void close_guest_eventfds(IVShmemState *s, int posn)
 }
 
 /* this function increase the dynamic storage need to store data about other
- * guests */
-static int increase_dynamic_storage(IVShmemState *s, int new_min_size)
+ * peers */
+static int resize_peers(IVShmemState *s, int new_min_size)
 {
 
-    int j, old_nb_alloc;
+    int j, old_size;
 
-    /* check for integer overflow */
-    if (new_min_size >= INT_MAX / sizeof(Peer) - 1 || new_min_size <= 0) {
+    /* limit number of max peers */
+    if (new_min_size <= 0 || new_min_size > IVSHMEM_MAX_PEERS) {
         return -1;
     }
-
-    old_nb_alloc = s->nb_peers;
-
-    if (new_min_size >= s->nb_peers) {
-        /* +1 because #new_min_size is used as last array index */
-        s->nb_peers = new_min_size + 1;
-    } else {
+    if (new_min_size <= s->nb_peers) {
         return 0;
     }
 
-    IVSHMEM_DPRINTF("bumping storage to %d guests\n", s->nb_peers);
+    old_size = s->nb_peers;
+    s->nb_peers = new_min_size;
+
+    IVSHMEM_DPRINTF("bumping storage to %d peers\n", s->nb_peers);
+
     s->peers = g_realloc(s->peers, s->nb_peers * sizeof(Peer));
 
-    /* zero out new pointers */
-    for (j = old_nb_alloc; j < s->nb_peers; j++) {
-        s->peers[j].eventfds = NULL;
+    for (j = old_size; j < s->nb_peers; j++) {
+        s->peers[j].eventfds = g_new0(EventNotifier, s->vectors);
         s->peers[j].nb_eventfds = 0;
     }
 
     return 0;
 }
 
-static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
+static bool fifo_update_and_get(IVShmemState *s, const uint8_t *buf, int size,
+                                void *data, size_t len)
 {
-    IVShmemState *s = opaque;
-    int incoming_fd, tmp_fd;
-    int guest_max_eventfd;
-    long incoming_posn;
+    const uint8_t *p;
+    uint32_t num;
 
-    if (fifo8_is_empty(&s->incoming_fifo) && size == sizeof(incoming_posn)) {
-        memcpy(&incoming_posn, buf, size);
-    } else {
-        const uint8_t *p;
-        uint32_t num;
+    assert(len <= sizeof(int64_t)); /* limitation of the fifo */
+    if (fifo8_is_empty(&s->incoming_fifo) && size == len) {
+        memcpy(data, buf, size);
+        return true;
+    }
+
+    IVSHMEM_DPRINTF("short read of %d bytes\n", size);
+
+    num = MIN(size, sizeof(int64_t) - fifo8_num_used(&s->incoming_fifo));
+    fifo8_push_all(&s->incoming_fifo, buf, num);
+
+    if (fifo8_num_used(&s->incoming_fifo) < len) {
+        assert(num == 0);
+        return false;
+    }
+
+    size -= num;
+    buf += num;
+    p = fifo8_pop_buf(&s->incoming_fifo, len, &num);
+    assert(num == len);
+
+    memcpy(data, p, len);
+
+    if (size > 0) {
+        fifo8_push_all(&s->incoming_fifo, buf, size);
+    }
 
-        IVSHMEM_DPRINTF("short read of %d bytes\n", size);
-        num = MAX(size, sizeof(long) - fifo8_num_used(&s->incoming_fifo));
-        fifo8_push_all(&s->incoming_fifo, buf, num);
-        if (fifo8_num_used(&s->incoming_fifo) < sizeof(incoming_posn)) {
+    return true;
+}
+
+static bool fifo_update_and_get_i64(IVShmemState *s,
+                                    const uint8_t *buf, int size, int64_t *i64)
+{
+    if (fifo_update_and_get(s, buf, size, i64, sizeof(*i64))) {
+        *i64 = GINT64_FROM_LE(*i64);
+        return true;
+    }
+
+    return false;
+}
+
+static int ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector)
+{
+    PCIDevice *pdev = PCI_DEVICE(s);
+    MSIMessage msg = msix_get_message(pdev, vector);
+    int ret;
+
+    IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector);
+
+    if (s->msi_vectors[vector].pdev != NULL) {
+        return 0;
+    }
+
+    ret = kvm_irqchip_add_msi_route(kvm_state, msg, pdev);
+    if (ret < 0) {
+        error_report("ivshmem: kvm_irqchip_add_msi_route failed");
+        return -1;
+    }
+
+    s->msi_vectors[vector].virq = ret;
+    s->msi_vectors[vector].pdev = pdev;
+
+    return 0;
+}
+
+static void setup_interrupt(IVShmemState *s, int vector)
+{
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    bool with_irqfd = kvm_msi_via_irqfd_enabled() &&
+        ivshmem_has_feature(s, IVSHMEM_MSI);
+    PCIDevice *pdev = PCI_DEVICE(s);
+
+    IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector);
+
+    if (!with_irqfd) {
+        IVSHMEM_DPRINTF("with eventfd");
+        s->eventfd_chr[vector] = create_eventfd_chr_device(s, n, vector);
+    } else if (msix_enabled(pdev)) {
+        IVSHMEM_DPRINTF("with irqfd");
+        if (ivshmem_add_kvm_msi_virq(s, vector) < 0) {
             return;
         }
-        size -= num;
-        buf += num;
-        p = fifo8_pop_buf(&s->incoming_fifo, sizeof(incoming_posn), &num);
-        g_assert(num == sizeof(incoming_posn));
-        memcpy(&incoming_posn, p, sizeof(incoming_posn));
-        if (size > 0) {
-            fifo8_push_all(&s->incoming_fifo, buf, size);
+
+        if (!msix_is_masked(pdev, vector)) {
+            kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL,
+                                               s->msi_vectors[vector].virq);
         }
+    } else {
+        /* it will be delayed until msix is enabled, in write_config */
+        IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled");
+    }
+}
+
+static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
+{
+    IVShmemState *s = opaque;
+    int incoming_fd;
+    int new_eventfd;
+    int64_t incoming_posn;
+    Error *err = NULL;
+    Peer *peer;
+
+    if (!fifo_update_and_get_i64(s, buf, size, &incoming_posn)) {
+        return;
     }
 
     if (incoming_posn < -1) {
-        IVSHMEM_DPRINTF("invalid incoming_posn %ld\n", incoming_posn);
+        IVSHMEM_DPRINTF("invalid incoming_posn %" PRId64 "\n", incoming_posn);
         return;
     }
 
     /* pick off s->server_chr->msgfd and store it, posn should accompany msg */
-    tmp_fd = qemu_chr_fe_get_msgfd(s->server_chr);
-    IVSHMEM_DPRINTF("posn is %ld, fd is %d\n", incoming_posn, tmp_fd);
+    incoming_fd = qemu_chr_fe_get_msgfd(s->server_chr);
+    IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n",
+                    incoming_posn, incoming_fd);
 
-    /* make sure we have enough space for this guest */
+    /* make sure we have enough space for this peer */
     if (incoming_posn >= s->nb_peers) {
-        if (increase_dynamic_storage(s, incoming_posn) < 0) {
-            error_report("increase_dynamic_storage() failed");
-            if (tmp_fd != -1) {
-                close(tmp_fd);
+        if (resize_peers(s, incoming_posn + 1) < 0) {
+            error_report("failed to resize peers array");
+            if (incoming_fd != -1) {
+                close(incoming_fd);
             }
             return;
         }
     }
 
-    if (tmp_fd == -1) {
+    peer = &s->peers[incoming_posn];
+
+    if (incoming_fd == -1) {
         /* if posn is positive and unseen before then this is our posn*/
-        if ((incoming_posn >= 0) &&
-                            (s->peers[incoming_posn].eventfds == NULL)) {
+        if (incoming_posn >= 0 && s->vm_id == -1) {
             /* receive our posn */
             s->vm_id = incoming_posn;
-            return;
         } else {
-            /* otherwise an fd == -1 means an existing guest has gone away */
-            IVSHMEM_DPRINTF("posn %ld has gone away\n", incoming_posn);
-            close_guest_eventfds(s, incoming_posn);
-            return;
+            /* otherwise an fd == -1 means an existing peer has gone away */
+            IVSHMEM_DPRINTF("posn %" PRId64 " has gone away\n", incoming_posn);
+            close_peer_eventfds(s, incoming_posn);
         }
-    }
-
-    /* because of the implementation of get_msgfd, we need a dup */
-    incoming_fd = dup(tmp_fd);
-
-    if (incoming_fd == -1) {
-        error_report("could not allocate file descriptor %s", strerror(errno));
-        close(tmp_fd);
         return;
     }
 
     /* if the position is -1, then it's shared memory region fd */
     if (incoming_posn == -1) {
-
         void * map_ptr;
 
-        s->max_peer = 0;
+        if (memory_region_is_mapped(&s->ivshmem)) {
+            error_report("shm already initialized");
+            close(incoming_fd);
+            return;
+        }
 
-        if (check_shm_size(s, incoming_fd) == -1) {
-            exit(1);
+        if (check_shm_size(s, incoming_fd, &err) == -1) {
+            error_report_err(err);
+            close(incoming_fd);
+            return;
         }
 
         /* mmap the region and map into the BAR2 */
         map_ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED,
                                                             incoming_fd, 0);
+        if (map_ptr == MAP_FAILED) {
+            error_report("Failed to mmap shared memory %s", strerror(errno));
+            close(incoming_fd);
+            return;
+        }
         memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s),
                                    "ivshmem.bar2", s->ivshmem_size, map_ptr);
         vmstate_register_ram(&s->ivshmem, DEVICE(s));
@@ -537,44 +686,59 @@ static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
 
         memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
 
-        /* only store the fd if it is successfully mapped */
-        s->shm_fd = incoming_fd;
+        close(incoming_fd);
+        return;
+    }
 
+    /* each peer has an associated array of eventfds, and we keep
+     * track of how many eventfds received so far */
+    /* get a new eventfd: */
+    if (peer->nb_eventfds >= s->vectors) {
+        error_report("Too many eventfd received, device has %d vectors",
+                     s->vectors);
+        close(incoming_fd);
         return;
     }
 
-    /* each guest has an array of eventfds, and we keep track of how many
-     * guests for each VM */
-    guest_max_eventfd = s->peers[incoming_posn].nb_eventfds;
+    new_eventfd = peer->nb_eventfds++;
 
-    if (guest_max_eventfd == 0) {
-        /* one eventfd per MSI vector */
-        s->peers[incoming_posn].eventfds = g_new(EventNotifier, s->vectors);
+    /* this is an eventfd for a particular peer VM */
+    IVSHMEM_DPRINTF("eventfds[%" PRId64 "][%d] = %d\n", incoming_posn,
+                    new_eventfd, incoming_fd);
+    event_notifier_init_fd(&peer->eventfds[new_eventfd], incoming_fd);
+    fcntl_setfl(incoming_fd, O_NONBLOCK); /* msix/irqfd poll non block */
+
+    if (incoming_posn == s->vm_id) {
+        setup_interrupt(s, new_eventfd);
     }
 
-    /* this is an eventfd for a particular guest VM */
-    IVSHMEM_DPRINTF("eventfds[%ld][%d] = %d\n", incoming_posn,
-                    guest_max_eventfd, incoming_fd);
-    event_notifier_init_fd(&s->peers[incoming_posn].eventfds[guest_max_eventfd],
-                           incoming_fd);
+    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
+        ivshmem_add_eventfd(s, incoming_posn, new_eventfd);
+    }
+}
 
-    /* increment count for particular guest */
-    s->peers[incoming_posn].nb_eventfds++;
+static void ivshmem_check_version(void *opaque, const uint8_t * buf, int size)
+{
+    IVShmemState *s = opaque;
+    int tmp;
+    int64_t version;
 
-    /* keep track of the maximum VM ID */
-    if (incoming_posn > s->max_peer) {
-        s->max_peer = incoming_posn;
+    if (!fifo_update_and_get_i64(s, buf, size, &version)) {
+        return;
     }
 
-    if (incoming_posn == s->vm_id) {
-        s->eventfd_chr[guest_max_eventfd] = create_eventfd_chr_device(s,
-                   &s->peers[s->vm_id].eventfds[guest_max_eventfd],
-                   guest_max_eventfd);
+    tmp = qemu_chr_fe_get_msgfd(s->server_chr);
+    if (tmp != -1 || version != IVSHMEM_PROTOCOL_VERSION) {
+        fprintf(stderr, "incompatible version, you are connecting to a ivshmem-"
+                "server using a different protocol please check your setup\n");
+        qemu_chr_delete(s->server_chr);
+        s->server_chr = NULL;
+        return;
     }
 
-    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
-        ivshmem_add_eventfd(s, incoming_posn, guest_max_eventfd);
-    }
+    IVSHMEM_DPRINTF("version check ok, switch to real chardev handler\n");
+    qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, ivshmem_read,
+                          ivshmem_event, s);
 }
 
 /* Select the MSI-X vectors used by device.
@@ -585,6 +749,7 @@ static void ivshmem_use_msix(IVShmemState * s)
     PCIDevice *d = PCI_DEVICE(s);
     int i;
 
+    IVSHMEM_DPRINTF("%s, msix present: %d\n", __func__, msix_present(d));
     if (!msix_present(d)) {
         return;
     }
@@ -599,128 +764,127 @@ static void ivshmem_reset(DeviceState *d)
     IVShmemState *s = IVSHMEM(d);
 
     s->intrstatus = 0;
+    s->intrmask = 0;
     ivshmem_use_msix(s);
 }
 
-static uint64_t ivshmem_get_size(IVShmemState * s) {
-
-    uint64_t value;
-    char *ptr;
-
-    value = strtoull(s->sizearg, &ptr, 10);
-    switch (*ptr) {
-        case 0: case 'M': case 'm':
-            value <<= 20;
-            break;
-        case 'G': case 'g':
-            value <<= 30;
-            break;
-        default:
-            error_report("invalid ram size: %s", s->sizearg);
-            exit(1);
-    }
-
-    /* BARs must be a power of 2 */
-    if (!is_power_of_two(value)) {
-        error_report("size must be power of 2");
-        exit(1);
-    }
-
-    return value;
-}
-
-static void ivshmem_setup_msi(IVShmemState * s)
+static int ivshmem_setup_msi(IVShmemState * s)
 {
     if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) {
-        IVSHMEM_DPRINTF("msix initialization failed\n");
-        exit(1);
+        return -1;
     }
 
     IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
 
     /* allocate QEMU char devices for receiving interrupts */
-    s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
+    s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector));
 
     ivshmem_use_msix(s);
+    return 0;
 }
 
-static void ivshmem_save(QEMUFile* f, void *opaque)
+static void ivshmem_enable_irqfd(IVShmemState *s)
 {
-    IVShmemState *proxy = opaque;
-    PCIDevice *pci_dev = PCI_DEVICE(proxy);
-
-    IVSHMEM_DPRINTF("ivshmem_save\n");
-    pci_device_save(pci_dev, f);
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int i;
 
-    if (ivshmem_has_feature(proxy, IVSHMEM_MSI)) {
-        msix_save(pci_dev, f);
-    } else {
-        qemu_put_be32(f, proxy->intrstatus);
-        qemu_put_be32(f, proxy->intrmask);
+    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
+        ivshmem_add_kvm_msi_virq(s, i);
     }
 
+    if (msix_set_vector_notifiers(pdev,
+                                  ivshmem_vector_unmask,
+                                  ivshmem_vector_mask,
+                                  ivshmem_vector_poll)) {
+        error_report("ivshmem: msix_set_vector_notifiers failed");
+    }
 }
 
-static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
+static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
 {
-    IVSHMEM_DPRINTF("ivshmem_load\n");
+    IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
 
-    IVShmemState *proxy = opaque;
-    PCIDevice *pci_dev = PCI_DEVICE(proxy);
-    int ret;
-
-    if (version_id > 0) {
-        return -EINVAL;
+    if (s->msi_vectors[vector].pdev == NULL) {
+        return;
     }
 
-    if (proxy->role_val == IVSHMEM_PEER) {
-        error_report("'peer' devices are not migratable");
-        return -EINVAL;
-    }
+    /* it was cleaned when masked in the frontend. */
+    kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
 
-    ret = pci_device_load(pci_dev, f);
-    if (ret) {
-        return ret;
-    }
+    s->msi_vectors[vector].pdev = NULL;
+}
 
-    if (ivshmem_has_feature(proxy, IVSHMEM_MSI)) {
-        msix_load(pci_dev, f);
-	ivshmem_use_msix(proxy);
-    } else {
-        proxy->intrstatus = qemu_get_be32(f);
-        proxy->intrmask = qemu_get_be32(f);
+static void ivshmem_disable_irqfd(IVShmemState *s)
+{
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int i;
+
+    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
+        ivshmem_remove_kvm_msi_virq(s, i);
     }
 
-    return 0;
+    msix_unset_vector_notifiers(pdev);
 }
 
-static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
-				 uint32_t val, int len)
+static void ivshmem_write_config(PCIDevice *pdev, uint32_t address,
+                                 uint32_t val, int len)
 {
-    pci_default_write_config(pci_dev, address, val, len);
+    IVShmemState *s = IVSHMEM(pdev);
+    int is_enabled, was_enabled = msix_enabled(pdev);
+
+    pci_default_write_config(pdev, address, val, len);
+    is_enabled = msix_enabled(pdev);
+
+    if (kvm_msi_via_irqfd_enabled() && s->vm_id != -1) {
+        if (!was_enabled && is_enabled) {
+            ivshmem_enable_irqfd(s);
+        } else if (was_enabled && !is_enabled) {
+            ivshmem_disable_irqfd(s);
+        }
+    }
 }
 
-static int pci_ivshmem_init(PCIDevice *dev)
+static void pci_ivshmem_realize(PCIDevice *dev, Error **errp)
 {
     IVShmemState *s = IVSHMEM(dev);
     uint8_t *pci_conf;
+    uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
+        PCI_BASE_ADDRESS_MEM_PREFETCH;
 
-    if (s->sizearg == NULL)
-        s->ivshmem_size = 4 << 20; /* 4 MB default */
-    else {
-        s->ivshmem_size = ivshmem_get_size(s);
+    if (!!s->server_chr + !!s->shmobj + !!s->hostmem != 1) {
+        error_setg(errp,
+                   "You must specify either 'shm', 'chardev' or 'x-memdev'");
+        return;
     }
 
-    fifo8_create(&s->incoming_fifo, sizeof(long));
+    if (s->hostmem) {
+        MemoryRegion *mr;
 
-    register_savevm(DEVICE(dev), "ivshmem", 0, 0, ivshmem_save, ivshmem_load,
-                                                                        dev);
+        if (s->sizearg) {
+            g_warning("size argument ignored with hostmem");
+        }
+
+        mr = host_memory_backend_get_memory(s->hostmem, errp);
+        s->ivshmem_size = memory_region_size(mr);
+    } else if (s->sizearg == NULL) {
+        s->ivshmem_size = 4 << 20; /* 4 MB default */
+    } else {
+        char *end;
+        int64_t size = qemu_strtosz(s->sizearg, &end);
+        if (size < 0 || *end != '\0' || !is_power_of_2(size)) {
+            error_setg(errp, "Invalid size %s", s->sizearg);
+            return;
+        }
+        s->ivshmem_size = size;
+    }
+
+    fifo8_create(&s->incoming_fifo, sizeof(int64_t));
 
     /* IRQFD requires MSI */
     if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
         !ivshmem_has_feature(s, IVSHMEM_MSI)) {
-        error_report("ioeventfd/irqfd requires MSI");
-        exit(1);
+        error_setg(errp, "ioeventfd/irqfd requires MSI");
+        return;
     }
 
     /* check that role is reasonable */
@@ -730,8 +894,8 @@ static int pci_ivshmem_init(PCIDevice *dev)
         } else if (strncmp(s->role, "master", 7) == 0) {
             s->role_val = IVSHMEM_MASTER;
         } else {
-            error_report("'role' must be 'peer' or 'master'");
-            exit(1);
+            error_setg(errp, "'role' must be 'peer' or 'master'");
+            return;
         }
     } else {
         s->role_val = IVSHMEM_MASTER; /* default */
@@ -748,8 +912,6 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
     pci_config_set_interrupt_pin(pci_conf, 1);
 
-    s->shm_fd = 0;
-
     memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s,
                           "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE);
 
@@ -758,51 +920,52 @@ static int pci_ivshmem_init(PCIDevice *dev)
                      &s->ivshmem_mmio);
 
     memory_region_init(&s->bar, OBJECT(s), "ivshmem-bar2-container", s->ivshmem_size);
-    s->ivshmem_attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
-        PCI_BASE_ADDRESS_MEM_PREFETCH;
     if (s->ivshmem_64bit) {
-        s->ivshmem_attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+        attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
     }
 
-    if ((s->server_chr != NULL) &&
-                        (strncmp(s->server_chr->filename, "unix:", 5) == 0)) {
-        /* if we get a UNIX socket as the parameter we will talk
-         * to the ivshmem server to receive the memory region */
+    if (s->hostmem != NULL) {
+        MemoryRegion *mr;
+
+        IVSHMEM_DPRINTF("using hostmem\n");
 
-        if (s->shmobj != NULL) {
-            error_report("WARNING: do not specify both 'chardev' "
-                         "and 'shm' with ivshmem");
+        mr = host_memory_backend_get_memory(MEMORY_BACKEND(s->hostmem), errp);
+        vmstate_register_ram(mr, DEVICE(s));
+        memory_region_add_subregion(&s->bar, 0, mr);
+        pci_register_bar(PCI_DEVICE(s), 2, attr, &s->bar);
+    } else if (s->server_chr != NULL) {
+        /* FIXME do not rely on what chr drivers put into filename */
+        if (strncmp(s->server_chr->filename, "unix:", 5)) {
+            error_setg(errp, "chardev is not a unix client socket");
+            return;
         }
 
+        /* if we get a UNIX socket as the parameter we will talk
+         * to the ivshmem server to receive the memory region */
+
         IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
                         s->server_chr->filename);
 
-        if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
-            ivshmem_setup_msi(s);
+        if (ivshmem_has_feature(s, IVSHMEM_MSI) &&
+            ivshmem_setup_msi(s)) {
+            error_setg(errp, "msix initialization failed");
+            return;
         }
 
-        /* we allocate enough space for 16 guests and grow as needed */
-        s->nb_peers = 16;
+        /* we allocate enough space for 16 peers and grow as needed */
+        resize_peers(s, 16);
         s->vm_id = -1;
 
-        /* allocate/initialize space for interrupt handling */
-        s->peers = g_malloc0(s->nb_peers * sizeof(Peer));
-
-        pci_register_bar(dev, 2, s->ivshmem_attr, &s->bar);
+        pci_register_bar(dev, 2, attr, &s->bar);
 
         s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *));
 
-        qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, ivshmem_read,
-                     ivshmem_event, s);
+        qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive,
+                              ivshmem_check_version, ivshmem_event, s);
     } else {
         /* just map the file immediately, we're not using a server */
         int fd;
 
-        if (s->shmobj == NULL) {
-            error_report("Must specify 'chardev' or 'shm' to ivshmem");
-            exit(1);
-        }
-
         IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s->shmobj);
 
         /* try opening with O_EXCL and if it succeeds zero the memory
@@ -816,39 +979,155 @@ static int pci_ivshmem_init(PCIDevice *dev)
 
         } else if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR,
                         S_IRWXU|S_IRWXG|S_IRWXO)) < 0) {
-            error_report("could not open shared file");
-            exit(1);
+            error_setg(errp, "could not open shared file");
+            return;
+        }
+
+        if (check_shm_size(s, fd, errp) == -1) {
+            return;
+        }
+
+        create_shared_memory_BAR(s, fd, attr, errp);
+        close(fd);
+    }
+}
+
+static void pci_ivshmem_exit(PCIDevice *dev)
+{
+    IVShmemState *s = IVSHMEM(dev);
+    int i;
+
+    fifo8_destroy(&s->incoming_fifo);
+
+    if (s->migration_blocker) {
+        migrate_del_blocker(s->migration_blocker);
+        error_free(s->migration_blocker);
+    }
+
+    if (memory_region_is_mapped(&s->ivshmem)) {
+        if (!s->hostmem) {
+            void *addr = memory_region_get_ram_ptr(&s->ivshmem);
+
+            if (munmap(addr, s->ivshmem_size) == -1) {
+                error_report("Failed to munmap shared memory %s",
+                             strerror(errno));
+            }
+        }
 
+        vmstate_unregister_ram(&s->ivshmem, DEVICE(dev));
+        memory_region_del_subregion(&s->bar, &s->ivshmem);
+    }
+
+    if (s->eventfd_chr) {
+        for (i = 0; i < s->vectors; i++) {
+            if (s->eventfd_chr[i]) {
+                qemu_chr_free(s->eventfd_chr[i]);
+            }
         }
+        g_free(s->eventfd_chr);
+    }
 
-        if (check_shm_size(s, fd) == -1) {
-            exit(1);
+    if (s->peers) {
+        for (i = 0; i < s->nb_peers; i++) {
+            close_peer_eventfds(s, i);
         }
+        g_free(s->peers);
+    }
+
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        msix_uninit_exclusive_bar(dev);
+    }
+
+    g_free(s->msi_vectors);
+}
 
-        create_shared_memory_BAR(s, fd);
+static bool test_msix(void *opaque, int version_id)
+{
+    IVShmemState *s = opaque;
+
+    return ivshmem_has_feature(s, IVSHMEM_MSI);
+}
 
+static bool test_no_msix(void *opaque, int version_id)
+{
+    return !test_msix(opaque, version_id);
+}
+
+static int ivshmem_pre_load(void *opaque)
+{
+    IVShmemState *s = opaque;
+
+    if (s->role_val == IVSHMEM_PEER) {
+        error_report("'peer' devices are not migratable");
+        return -EINVAL;
     }
 
-    dev->config_write = ivshmem_write_config;
+    return 0;
+}
+
+static int ivshmem_post_load(void *opaque, int version_id)
+{
+    IVShmemState *s = opaque;
+
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        ivshmem_use_msix(s);
+    }
 
     return 0;
 }
 
-static void pci_ivshmem_uninit(PCIDevice *dev)
+static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id)
 {
-    IVShmemState *s = IVSHMEM(dev);
+    IVShmemState *s = opaque;
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int ret;
 
-    if (s->migration_blocker) {
-        migrate_del_blocker(s->migration_blocker);
-        error_free(s->migration_blocker);
+    IVSHMEM_DPRINTF("ivshmem_load_old\n");
+
+    if (version_id != 0) {
+        return -EINVAL;
     }
 
-    memory_region_del_subregion(&s->bar, &s->ivshmem);
-    vmstate_unregister_ram(&s->ivshmem, DEVICE(dev));
-    unregister_savevm(DEVICE(dev), "ivshmem", s);
-    fifo8_destroy(&s->incoming_fifo);
+    if (s->role_val == IVSHMEM_PEER) {
+        error_report("'peer' devices are not migratable");
+        return -EINVAL;
+    }
+
+    ret = pci_device_load(pdev, f);
+    if (ret) {
+        return ret;
+    }
+
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        msix_load(pdev, f);
+        ivshmem_use_msix(s);
+    } else {
+        s->intrstatus = qemu_get_be32(f);
+        s->intrmask = qemu_get_be32(f);
+    }
+
+    return 0;
 }
 
+static const VMStateDescription ivshmem_vmsd = {
+    .name = "ivshmem",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_load = ivshmem_pre_load,
+    .post_load = ivshmem_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
+
+        VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix),
+        VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix),
+        VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix),
+
+        VMSTATE_END_OF_LIST()
+    },
+    .load_state_old = ivshmem_load_old,
+    .minimum_version_id_old = 0
+};
+
 static Property ivshmem_properties[] = {
     DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
     DEFINE_PROP_STRING("size", IVShmemState, sizearg),
@@ -866,20 +1145,50 @@ static void ivshmem_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_ivshmem_init;
-    k->exit = pci_ivshmem_uninit;
+    k->realize = pci_ivshmem_realize;
+    k->exit = pci_ivshmem_exit;
+    k->config_write = ivshmem_write_config;
     k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
     k->device_id = PCI_DEVICE_ID_IVSHMEM;
     k->class_id = PCI_CLASS_MEMORY_RAM;
     dc->reset = ivshmem_reset;
     dc->props = ivshmem_properties;
+    dc->vmsd = &ivshmem_vmsd;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->desc = "Inter-VM shared memory";
+}
+
+static void ivshmem_check_memdev_is_busy(Object *obj, const char *name,
+                                         Object *val, Error **errp)
+{
+    MemoryRegion *mr;
+
+    mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), errp);
+    if (memory_region_is_mapped(mr)) {
+        char *path = object_get_canonical_path_component(val);
+        error_setg(errp, "can't use already busy memdev: %s", path);
+        g_free(path);
+    } else {
+        qdev_prop_allow_set_link_before_realize(obj, name, val, errp);
+    }
+}
+
+static void ivshmem_init(Object *obj)
+{
+    IVShmemState *s = IVSHMEM(obj);
+
+    object_property_add_link(obj, "x-memdev", TYPE_MEMORY_BACKEND,
+                             (Object **)&s->hostmem,
+                             ivshmem_check_memdev_is_busy,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
 }
 
 static const TypeInfo ivshmem_info = {
     .name          = TYPE_IVSHMEM,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(IVShmemState),
+    .instance_init = ivshmem_init,
     .class_init    = ivshmem_class_init,
 };
 
diff --git a/hw/misc/macio/cuda.c b/hw/misc/macio/cuda.c
index 5d7043e99..9db4c6415 100644
--- a/hw/misc/macio/cuda.c
+++ b/hw/misc/macio/cuda.c
@@ -57,6 +57,8 @@
 #define IER_SET		0x80		/* set bits in IER */
 #define IER_CLR		0		/* clear bits in IER */
 #define SR_INT		0x04		/* Shift register full/empty */
+#define SR_DATA_INT	0x08
+#define SR_CLOCK_INT	0x10
 #define T1_INT          0x40            /* Timer 1 interrupt */
 #define T2_INT          0x20            /* Timer 2 interrupt */
 
@@ -108,6 +110,24 @@
 /* CUDA returns time_t's offset from Jan 1, 1904, not 1970 */
 #define RTC_OFFSET                      2082844800
 
+/* CUDA registers */
+#define CUDA_REG_B       0x00
+#define CUDA_REG_A       0x01
+#define CUDA_REG_DIRB    0x02
+#define CUDA_REG_DIRA    0x03
+#define CUDA_REG_T1CL    0x04
+#define CUDA_REG_T1CH    0x05
+#define CUDA_REG_T1LL    0x06
+#define CUDA_REG_T1LH    0x07
+#define CUDA_REG_T2CL    0x08
+#define CUDA_REG_T2CH    0x09
+#define CUDA_REG_SR      0x0a
+#define CUDA_REG_ACR     0x0b
+#define CUDA_REG_PCR     0x0c
+#define CUDA_REG_IFR     0x0d
+#define CUDA_REG_IER     0x0e
+#define CUDA_REG_ANH     0x0f
+
 static void cuda_update(CUDAState *s);
 static void cuda_receive_packet_from_host(CUDAState *s,
                                           const uint8_t *data, int len);
@@ -116,47 +136,48 @@ static void cuda_timer_update(CUDAState *s, CUDATimer *ti,
 
 static void cuda_update_irq(CUDAState *s)
 {
-    if (s->ifr & s->ier & (SR_INT | T1_INT)) {
+    if (s->ifr & s->ier & (SR_INT | T1_INT | T2_INT)) {
         qemu_irq_raise(s->irq);
     } else {
         qemu_irq_lower(s->irq);
     }
 }
 
-static uint64_t get_tb(uint64_t freq)
+static uint64_t get_tb(uint64_t time, uint64_t freq)
 {
-    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                    freq, get_ticks_per_sec());
+    return muldiv64(time, freq, get_ticks_per_sec());
 }
 
-static unsigned int get_counter(CUDATimer *s)
+static unsigned int get_counter(CUDATimer *ti)
 {
     int64_t d;
     unsigned int counter;
     uint64_t tb_diff;
+    uint64_t current_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     /* Reverse of the tb calculation algorithm that Mac OS X uses on bootup. */
-    tb_diff = get_tb(s->frequency) - s->load_time;
-    d = (tb_diff * 0xBF401675E5DULL) / (s->frequency << 24);
+    tb_diff = get_tb(current_time, ti->frequency) - ti->load_time;
+    d = (tb_diff * 0xBF401675E5DULL) / (ti->frequency << 24);
 
-    if (s->index == 0) {
+    if (ti->index == 0) {
         /* the timer goes down from latch to -1 (period of latch + 2) */
-        if (d <= (s->counter_value + 1)) {
-            counter = (s->counter_value - d) & 0xffff;
+        if (d <= (ti->counter_value + 1)) {
+            counter = (ti->counter_value - d) & 0xffff;
         } else {
-            counter = (d - (s->counter_value + 1)) % (s->latch + 2);
-            counter = (s->latch - counter) & 0xffff;
+            counter = (d - (ti->counter_value + 1)) % (ti->latch + 2);
+            counter = (ti->latch - counter) & 0xffff;
         }
     } else {
-        counter = (s->counter_value - d) & 0xffff;
+        counter = (ti->counter_value - d) & 0xffff;
     }
     return counter;
 }
 
 static void set_counter(CUDAState *s, CUDATimer *ti, unsigned int val)
 {
-    CUDA_DPRINTF("T%d.counter=%d\n", 1 + (ti->timer == NULL), val);
-    ti->load_time = get_tb(s->frequency);
+    CUDA_DPRINTF("T%d.counter=%d\n", 1 + ti->index, val);
+    ti->load_time = get_tb(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           s->frequency);
     ti->counter_value = val;
     cuda_timer_update(s, ti, ti->load_time);
 }
@@ -199,7 +220,7 @@ static void cuda_timer_update(CUDAState *s, CUDATimer *ti,
 {
     if (!ti->timer)
         return;
-    if ((s->acr & T1MODE) != T1MODE_CONT) {
+    if (ti->index == 0 && (s->acr & T1MODE) != T1MODE_CONT) {
         timer_del(ti->timer);
     } else {
         ti->next_irq_time = get_next_irq_time(ti, current_time);
@@ -217,6 +238,41 @@ static void cuda_timer1(void *opaque)
     cuda_update_irq(s);
 }
 
+static void cuda_timer2(void *opaque)
+{
+    CUDAState *s = opaque;
+    CUDATimer *ti = &s->timers[1];
+
+    cuda_timer_update(s, ti, ti->next_irq_time);
+    s->ifr |= T2_INT;
+    cuda_update_irq(s);
+}
+
+static void cuda_set_sr_int(void *opaque)
+{
+    CUDAState *s = opaque;
+
+    CUDA_DPRINTF("CUDA: %s:%d\n", __func__, __LINE__);
+    s->ifr |= SR_INT;
+    cuda_update_irq(s);
+}
+
+static void cuda_delay_set_sr_int(CUDAState *s)
+{
+    int64_t expire;
+
+    if (s->dirb == 0xff) {
+        /* Not in Mac OS, fire the IRQ directly */
+        cuda_set_sr_int(s);
+        return;
+    }
+
+    CUDA_DPRINTF("CUDA: %s:%d\n", __func__, __LINE__);
+
+    expire = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 300 * SCALE_US;
+    timer_mod(s->sr_delay_timer, expire);
+}
+
 static uint32_t cuda_readb(void *opaque, hwaddr addr)
 {
     CUDAState *s = opaque;
@@ -224,66 +280,68 @@ static uint32_t cuda_readb(void *opaque, hwaddr addr)
 
     addr = (addr >> 9) & 0xf;
     switch(addr) {
-    case 0:
+    case CUDA_REG_B:
         val = s->b;
         break;
-    case 1:
+    case CUDA_REG_A:
         val = s->a;
         break;
-    case 2:
+    case CUDA_REG_DIRB:
         val = s->dirb;
         break;
-    case 3:
+    case CUDA_REG_DIRA:
         val = s->dira;
         break;
-    case 4:
+    case CUDA_REG_T1CL:
         val = get_counter(&s->timers[0]) & 0xff;
         s->ifr &= ~T1_INT;
         cuda_update_irq(s);
         break;
-    case 5:
+    case CUDA_REG_T1CH:
         val = get_counter(&s->timers[0]) >> 8;
         cuda_update_irq(s);
         break;
-    case 6:
+    case CUDA_REG_T1LL:
         val = s->timers[0].latch & 0xff;
         break;
-    case 7:
+    case CUDA_REG_T1LH:
         /* XXX: check this */
         val = (s->timers[0].latch >> 8) & 0xff;
         break;
-    case 8:
+    case CUDA_REG_T2CL:
         val = get_counter(&s->timers[1]) & 0xff;
         s->ifr &= ~T2_INT;
+        cuda_update_irq(s);
         break;
-    case 9:
+    case CUDA_REG_T2CH:
         val = get_counter(&s->timers[1]) >> 8;
         break;
-    case 10:
+    case CUDA_REG_SR:
         val = s->sr;
-        s->ifr &= ~SR_INT;
+        s->ifr &= ~(SR_INT | SR_CLOCK_INT | SR_DATA_INT);
         cuda_update_irq(s);
         break;
-    case 11:
+    case CUDA_REG_ACR:
         val = s->acr;
         break;
-    case 12:
+    case CUDA_REG_PCR:
         val = s->pcr;
         break;
-    case 13:
+    case CUDA_REG_IFR:
         val = s->ifr;
-        if (s->ifr & s->ier)
+        if (s->ifr & s->ier) {
             val |= 0x80;
+        }
         break;
-    case 14:
+    case CUDA_REG_IER:
         val = s->ier | 0x80;
         break;
     default:
-    case 15:
+    case CUDA_REG_ANH:
         val = s->anh;
         break;
     }
-    if (addr != 13 || val != 0) {
+    if (addr != CUDA_REG_IFR || val != 0) {
         CUDA_DPRINTF("read: reg=0x%x val=%02x\n", (int)addr, val);
     }
 
@@ -298,61 +356,65 @@ static void cuda_writeb(void *opaque, hwaddr addr, uint32_t val)
     CUDA_DPRINTF("write: reg=0x%x val=%02x\n", (int)addr, val);
 
     switch(addr) {
-    case 0:
+    case CUDA_REG_B:
         s->b = val;
         cuda_update(s);
         break;
-    case 1:
+    case CUDA_REG_A:
         s->a = val;
         break;
-    case 2:
+    case CUDA_REG_DIRB:
         s->dirb = val;
         break;
-    case 3:
+    case CUDA_REG_DIRA:
         s->dira = val;
         break;
-    case 4:
+    case CUDA_REG_T1CL:
         s->timers[0].latch = (s->timers[0].latch & 0xff00) | val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 5:
+    case CUDA_REG_T1CH:
         s->timers[0].latch = (s->timers[0].latch & 0xff) | (val << 8);
         s->ifr &= ~T1_INT;
         set_counter(s, &s->timers[0], s->timers[0].latch);
         break;
-    case 6:
+    case CUDA_REG_T1LL:
         s->timers[0].latch = (s->timers[0].latch & 0xff00) | val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 7:
+    case CUDA_REG_T1LH:
         s->timers[0].latch = (s->timers[0].latch & 0xff) | (val << 8);
         s->ifr &= ~T1_INT;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 8:
-        s->timers[1].latch = val;
-        set_counter(s, &s->timers[1], val);
+    case CUDA_REG_T2CL:
+        s->timers[1].latch = (s->timers[1].latch & 0xff00) | val;
         break;
-    case 9:
-        set_counter(s, &s->timers[1], (val << 8) | s->timers[1].latch);
+    case CUDA_REG_T2CH:
+        /* To ensure T2 generates an interrupt on zero crossing with the
+           common timer code, write the value directly from the latch to
+           the counter */
+        s->timers[1].latch = (s->timers[1].latch & 0xff) | (val << 8);
+        s->ifr &= ~T2_INT;
+        set_counter(s, &s->timers[1], s->timers[1].latch);
         break;
-    case 10:
+    case CUDA_REG_SR:
         s->sr = val;
         break;
-    case 11:
+    case CUDA_REG_ACR:
         s->acr = val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         cuda_update(s);
         break;
-    case 12:
+    case CUDA_REG_PCR:
         s->pcr = val;
         break;
-    case 13:
+    case CUDA_REG_IFR:
         /* reset bits */
         s->ifr &= ~val;
         cuda_update_irq(s);
         break;
-    case 14:
+    case CUDA_REG_IER:
         if (val & IER_SET) {
             /* set bits */
             s->ier |= val & 0x7f;
@@ -363,7 +425,7 @@ static void cuda_writeb(void *opaque, hwaddr addr, uint32_t val)
         cuda_update_irq(s);
         break;
     default:
-    case 15:
+    case CUDA_REG_ANH:
         s->anh = val;
         break;
     }
@@ -384,8 +446,7 @@ static void cuda_update(CUDAState *s)
                 if (s->data_out_index < sizeof(s->data_out)) {
                     CUDA_DPRINTF("send: %02x\n", s->sr);
                     s->data_out[s->data_out_index++] = s->sr;
-                    s->ifr |= SR_INT;
-                    cuda_update_irq(s);
+                    cuda_delay_set_sr_int(s);
                 }
             }
         } else {
@@ -398,8 +459,7 @@ static void cuda_update(CUDAState *s)
                     if (s->data_in_index >= s->data_in_size) {
                         s->b = (s->b | TREQ);
                     }
-                    s->ifr |= SR_INT;
-                    cuda_update_irq(s);
+                    cuda_delay_set_sr_int(s);
                 }
             }
         }
@@ -411,15 +471,13 @@ static void cuda_update(CUDAState *s)
                 s->b = (s->b | TREQ);
             else
                 s->b = (s->b & ~TREQ);
-            s->ifr |= SR_INT;
-            cuda_update_irq(s);
+            cuda_delay_set_sr_int(s);
         } else {
             if (!(s->last_b & TIP)) {
                 /* handle end of host to cuda transfer */
                 packet_received = (s->data_out_index > 0);
                 /* always an IRQ at the end of transfer */
-                s->ifr |= SR_INT;
-                cuda_update_irq(s);
+                cuda_delay_set_sr_int(s);
             }
             /* signal if there is data to read */
             if (s->data_in_index < s->data_in_size) {
@@ -456,8 +514,7 @@ static void cuda_send_packet_to_host(CUDAState *s,
     s->data_in_size = len;
     s->data_in_index = 0;
     cuda_update(s);
-    s->ifr |= SR_INT;
-    cuda_update_irq(s);
+    cuda_delay_set_sr_int(s);
 }
 
 static void cuda_adb_poll(void *opaque)
@@ -480,7 +537,7 @@ static void cuda_adb_poll(void *opaque)
 static void cuda_receive_packet(CUDAState *s,
                                 const uint8_t *data, int len)
 {
-    uint8_t obuf[16];
+    uint8_t obuf[16] = { CUDA_PACKET, 0, data[0] };
     int autopoll;
     uint32_t ti;
 
@@ -497,23 +554,18 @@ static void cuda_receive_packet(CUDAState *s,
                 timer_del(s->adb_poll_timer);
             }
         }
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = data[1];
-        cuda_send_packet_to_host(s, obuf, 2);
+        cuda_send_packet_to_host(s, obuf, 3);
+        break;
+    case CUDA_GET_6805_ADDR:
+        cuda_send_packet_to_host(s, obuf, 3);
         break;
     case CUDA_SET_TIME:
         ti = (((uint32_t)data[1]) << 24) + (((uint32_t)data[2]) << 16) + (((uint32_t)data[3]) << 8) + data[4];
         s->tick_offset = ti - (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / get_ticks_per_sec());
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        obuf[2] = 0;
         cuda_send_packet_to_host(s, obuf, 3);
         break;
     case CUDA_GET_TIME:
         ti = s->tick_offset + (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / get_ticks_per_sec());
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        obuf[2] = 0;
         obuf[3] = ti >> 24;
         obuf[4] = ti >> 16;
         obuf[5] = ti >> 8;
@@ -524,22 +576,34 @@ static void cuda_receive_packet(CUDAState *s,
     case CUDA_SET_DEVICE_LIST:
     case CUDA_SET_AUTO_RATE:
     case CUDA_SET_POWER_MESSAGES:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
+        cuda_send_packet_to_host(s, obuf, 3);
         break;
     case CUDA_POWERDOWN:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
+        cuda_send_packet_to_host(s, obuf, 3);
         qemu_system_shutdown_request();
         break;
     case CUDA_RESET_SYSTEM:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
+        cuda_send_packet_to_host(s, obuf, 3);
         qemu_system_reset_request();
         break;
+    case CUDA_COMBINED_FORMAT_IIC:
+        obuf[0] = ERROR_PACKET;
+        obuf[1] = 0x5;
+        obuf[2] = CUDA_PACKET;
+        obuf[3] = data[0];
+        cuda_send_packet_to_host(s, obuf, 4);
+        break;
+    case CUDA_GET_SET_IIC:
+        if (len == 4) {
+            cuda_send_packet_to_host(s, obuf, 3);
+        } else {
+            obuf[0] = ERROR_PACKET;
+            obuf[1] = 0x2;
+            obuf[2] = CUDA_PACKET;
+            obuf[3] = data[0];
+            cuda_send_packet_to_host(s, obuf, 4);
+        }
+        break;
     default:
         break;
     }
@@ -560,19 +624,21 @@ static void cuda_receive_packet_from_host(CUDAState *s,
     switch(data[0]) {
     case ADB_PACKET:
         {
-            uint8_t obuf[ADB_MAX_OUT_LEN + 2];
+            uint8_t obuf[ADB_MAX_OUT_LEN + 3];
             int olen;
             olen = adb_request(&s->adb_bus, obuf + 2, data + 1, len - 1);
             if (olen > 0) {
                 obuf[0] = ADB_PACKET;
                 obuf[1] = 0x00;
+                cuda_send_packet_to_host(s, obuf, olen + 2);
             } else {
                 /* error */
                 obuf[0] = ADB_PACKET;
                 obuf[1] = -olen;
+                obuf[2] = data[1];
                 olen = 0;
+                cuda_send_packet_to_host(s, obuf, olen + 3);
             }
-            cuda_send_packet_to_host(s, obuf, olen + 2);
         }
         break;
     case CUDA_PACKET:
@@ -671,7 +737,7 @@ static void cuda_reset(DeviceState *dev)
 
     s->b = 0;
     s->a = 0;
-    s->dirb = 0;
+    s->dirb = 0xff;
     s->dira = 0;
     s->sr = 0;
     s->acr = 0;
@@ -688,8 +754,9 @@ static void cuda_reset(DeviceState *dev)
     s->timers[0].latch = 0xffff;
     set_counter(s, &s->timers[0], 0xffff);
 
-    s->timers[1].latch = 0;
-    set_counter(s, &s->timers[1], 0xffff);
+    s->timers[1].latch = 0xffff;
+
+    s->sr_delay_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_set_sr_int, s);
 }
 
 static void cuda_realizefn(DeviceState *dev, Error **errp)
@@ -699,7 +766,8 @@ static void cuda_realizefn(DeviceState *dev, Error **errp)
 
     s->timers[0].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_timer1, s);
     s->timers[0].frequency = s->frequency;
-    s->timers[1].frequency = s->frequency;
+    s->timers[1].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_timer2, s);
+    s->timers[1].frequency = (SCALE_US * 6000) / 4700;
 
     qemu_get_timedate(&tm, 0);
     s->tick_offset = (uint32_t)mktimegm(&tm) + RTC_OFFSET;
@@ -738,6 +806,7 @@ static void cuda_class_init(ObjectClass *oc, void *data)
     dc->reset = cuda_reset;
     dc->vmsd = &vmstate_cuda;
     dc->props = cuda_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo cuda_type_info = {
diff --git a/hw/misc/macio/mac_dbdma.c b/hw/misc/macio/mac_dbdma.c
index 779683cc0..5ee8f022a 100644
--- a/hw/misc/macio/mac_dbdma.c
+++ b/hw/misc/macio/mac_dbdma.c
@@ -557,7 +557,6 @@ void DBDMA_register_channel(void *dbdma, int nchan, qemu_irq irq,
     DBDMA_DPRINTF("DBDMA_register_channel 0x%x\n", nchan);
 
     ch->irq = irq;
-    ch->channel = nchan;
     ch->rw = rw;
     ch->flush = flush;
     ch->io.opaque = opaque;
@@ -753,6 +752,7 @@ void* DBDMA_init (MemoryRegion **dbdma_mem)
     for (i = 0; i < DBDMA_CHANNELS; i++) {
         DBDMA_io *io = &s->channels[i].io;
         qemu_iovec_init(&io->iov, 1);
+        s->channels[i].channel = i;
     }
 
     memory_region_init_io(&s->mem, NULL, &dbdma_ops, s, "dbdma", 0x1000);
diff --git a/hw/misc/macio/macio.c b/hw/misc/macio/macio.c
index c661f86c2..adb990e56 100644
--- a/hw/misc/macio/macio.c
+++ b/hw/misc/macio/macio.c
@@ -393,6 +393,7 @@ static void macio_class_init(ObjectClass *klass, void *data)
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->class_id = PCI_CLASS_OTHERS << 8;
     dc->props = macio_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo macio_oldworld_type_info = {
diff --git a/hw/misc/omap_clk.c b/hw/misc/omap_clk.c
index 80a3c50e1..73d4f8bec 100644
--- a/hw/misc/omap_clk.c
+++ b/hw/misc/omap_clk.c
@@ -1239,7 +1239,7 @@ void omap_clk_init(struct omap_mpu_state_s *mpu)
     for (i = onchip_clks, count = 0; *i; i ++)
         if ((*i)->flags & flag)
             count ++;
-    mpu->clks = (struct clk *) g_malloc0(sizeof(struct clk) * (count + 1));
+    mpu->clks = g_new0(struct clk, count + 1);
     for (i = onchip_clks, j = mpu->clks; *i; i ++)
         if ((*i)->flags & flag) {
             memcpy(j, *i, sizeof(struct clk));
diff --git a/hw/misc/omap_gpmc.c b/hw/misc/omap_gpmc.c
index 74fc91c8e..8960f1bf1 100644
--- a/hw/misc/omap_gpmc.c
+++ b/hw/misc/omap_gpmc.c
@@ -826,8 +826,7 @@ struct omap_gpmc_s *omap_gpmc_init(struct omap_mpu_state_s *mpu,
                                    qemu_irq irq, qemu_irq drq)
 {
     int cs;
-    struct omap_gpmc_s *s = (struct omap_gpmc_s *)
-            g_malloc0(sizeof(struct omap_gpmc_s));
+    struct omap_gpmc_s *s = g_new0(struct omap_gpmc_s, 1);
 
     memory_region_init_io(&s->iomem, NULL, &omap_gpmc_ops, s, "omap-gpmc", 0x1000);
     memory_region_add_subregion(get_system_memory(), base, &s->iomem);
diff --git a/hw/misc/omap_sdrc.c b/hw/misc/omap_sdrc.c
index 3de0c0e9d..bca25307b 100644
--- a/hw/misc/omap_sdrc.c
+++ b/hw/misc/omap_sdrc.c
@@ -157,8 +157,7 @@ static const MemoryRegionOps omap_sdrc_ops = {
 struct omap_sdrc_s *omap_sdrc_init(MemoryRegion *sysmem,
                                    hwaddr base)
 {
-    struct omap_sdrc_s *s = (struct omap_sdrc_s *)
-            g_malloc0(sizeof(struct omap_sdrc_s));
+    struct omap_sdrc_s *s = g_new0(struct omap_sdrc_s, 1);
 
     omap_sdrc_reset(s);
 
diff --git a/hw/misc/pvpanic.c b/hw/misc/pvpanic.c
index 994f8af8e..370948872 100644
--- a/hw/misc/pvpanic.c
+++ b/hw/misc/pvpanic.c
@@ -41,8 +41,7 @@ static void handle_event(int event)
     }
 
     if (event & PVPANIC_PANICKED) {
-        qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE, &error_abort);
-        vm_stop(RUN_STATE_GUEST_PANICKED);
+        qemu_system_guest_panicked();
         return;
     }
 }
diff --git a/hw/misc/zynq-xadc.c b/hw/misc/zynq-xadc.c
new file mode 100644
index 000000000..1a3259545
--- /dev/null
+++ b/hw/misc/zynq-xadc.c
@@ -0,0 +1,302 @@
+/*
+ * ADC registers for Xilinx Zynq Platform
+ *
+ * Copyright (c) 2015 Guenter Roeck
+ * Based on hw/misc/zynq_slcr.c, written by Michal Simek
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/hw.h"
+#include "hw/misc/zynq-xadc.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+
+enum {
+    CFG                = 0x000 / 4,
+    INT_STS,
+    INT_MASK,
+    MSTS,
+    CMDFIFO,
+    RDFIFO,
+    MCTL,
+};
+
+#define CFG_ENABLE              BIT(31)
+#define CFG_CFIFOTH_SHIFT       20
+#define CFG_CFIFOTH_LENGTH      4
+#define CFG_DFIFOTH_SHIFT       16
+#define CFG_DFIFOTH_LENGTH      4
+#define CFG_WEDGE               BIT(13)
+#define CFG_REDGE               BIT(12)
+#define CFG_TCKRATE_SHIFT       8
+#define CFG_TCKRATE_LENGTH      2
+
+#define CFG_TCKRATE_DIV(x)      (0x1 << (x - 1))
+
+#define CFG_IGAP_SHIFT          0
+#define CFG_IGAP_LENGTH         5
+
+#define INT_CFIFO_LTH           BIT(9)
+#define INT_DFIFO_GTH           BIT(8)
+#define INT_OT                  BIT(7)
+#define INT_ALM_SHIFT           0
+#define INT_ALM_LENGTH          7
+#define INT_ALM_MASK            (((1 << INT_ALM_LENGTH) - 1) << INT_ALM_SHIFT)
+
+#define INT_ALL (INT_CFIFO_LTH | INT_DFIFO_GTH | INT_OT | INT_ALM_MASK)
+
+#define MSTS_CFIFO_LVL_SHIFT    16
+#define MSTS_CFIFO_LVL_LENGTH   4
+#define MSTS_DFIFO_LVL_SHIFT    12
+#define MSTS_DFIFO_LVL_LENGTH   4
+#define MSTS_CFIFOF             BIT(11)
+#define MSTS_CFIFOE             BIT(10)
+#define MSTS_DFIFOF             BIT(9)
+#define MSTS_DFIFOE             BIT(8)
+#define MSTS_OT                 BIT(7)
+#define MSTS_ALM_SHIFT          0
+#define MSTS_ALM_LENGTH         7
+
+#define MCTL_RESET              BIT(4)
+
+#define CMD_NOP                 0x00
+#define CMD_READ                0x01
+#define CMD_WRITE               0x02
+
+static void zynq_xadc_update_ints(ZynqXADCState *s)
+{
+
+    /* We are fast, commands are actioned instantly so the CFIFO is always
+     * empty (and below threshold).
+     */
+    s->regs[INT_STS] |= INT_CFIFO_LTH;
+
+    if (s->xadc_dfifo_entries >
+        extract32(s->regs[CFG], CFG_DFIFOTH_SHIFT, CFG_DFIFOTH_LENGTH)) {
+        s->regs[INT_STS] |= INT_DFIFO_GTH;
+    }
+
+    qemu_set_irq(s->qemu_irq, !!(s->regs[INT_STS] & ~s->regs[INT_MASK]));
+}
+
+static void zynq_xadc_reset(DeviceState *d)
+{
+    ZynqXADCState *s = ZYNQ_XADC(d);
+
+    s->regs[CFG] = 0x14 << CFG_IGAP_SHIFT |
+                   CFG_TCKRATE_DIV(4) << CFG_TCKRATE_SHIFT | CFG_REDGE;
+    s->regs[INT_STS] = INT_CFIFO_LTH;
+    s->regs[INT_MASK] = 0xffffffff;
+    s->regs[CMDFIFO] = 0;
+    s->regs[RDFIFO] = 0;
+    s->regs[MCTL] = MCTL_RESET;
+
+    memset(s->xadc_regs, 0, sizeof(s->xadc_regs));
+    memset(s->xadc_dfifo, 0, sizeof(s->xadc_dfifo));
+    s->xadc_dfifo_entries = 0;
+
+    zynq_xadc_update_ints(s);
+}
+
+static uint16_t xadc_pop_dfifo(ZynqXADCState *s)
+{
+    uint16_t rv = s->xadc_dfifo[0];
+    int i;
+
+    if (s->xadc_dfifo_entries > 0) {
+        s->xadc_dfifo_entries--;
+    }
+    for (i = 0; i < s->xadc_dfifo_entries; i++) {
+        s->xadc_dfifo[i] = s->xadc_dfifo[i + 1];
+    }
+    s->xadc_dfifo[s->xadc_dfifo_entries] = 0;
+    zynq_xadc_update_ints(s);
+    return rv;
+}
+
+static void xadc_push_dfifo(ZynqXADCState *s, uint16_t regval)
+{
+    if (s->xadc_dfifo_entries < ZYNQ_XADC_FIFO_DEPTH) {
+        s->xadc_dfifo[s->xadc_dfifo_entries++] = s->xadc_read_reg_previous;
+    }
+    s->xadc_read_reg_previous = regval;
+    zynq_xadc_update_ints(s);
+}
+
+static bool zynq_xadc_check_offset(hwaddr offset, bool rnw)
+{
+    switch (offset) {
+    case CFG:
+    case INT_MASK:
+    case INT_STS:
+    case MCTL:
+        return true;
+    case RDFIFO:
+    case MSTS:
+        return rnw;     /* read only */
+    case CMDFIFO:
+        return !rnw;    /* write only */
+    default:
+        return false;
+    }
+}
+
+static uint64_t zynq_xadc_read(void *opaque, hwaddr offset, unsigned size)
+{
+    ZynqXADCState *s = opaque;
+    int reg = offset / 4;
+    uint32_t rv = 0;
+
+    if (!zynq_xadc_check_offset(reg, true)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Invalid read access to "
+                      "addr %" HWADDR_PRIx "\n", offset);
+        return 0;
+    }
+
+    switch (reg) {
+    case CFG:
+    case INT_MASK:
+    case INT_STS:
+    case MCTL:
+        rv = s->regs[reg];
+        break;
+    case MSTS:
+        rv = MSTS_CFIFOE;
+        rv |= s->xadc_dfifo_entries << MSTS_DFIFO_LVL_SHIFT;
+        if (!s->xadc_dfifo_entries) {
+            rv |= MSTS_DFIFOE;
+        } else if (s->xadc_dfifo_entries == ZYNQ_XADC_FIFO_DEPTH) {
+            rv |= MSTS_DFIFOF;
+        }
+        break;
+    case RDFIFO:
+        rv = xadc_pop_dfifo(s);
+        break;
+    }
+    return rv;
+}
+
+static void zynq_xadc_write(void *opaque, hwaddr offset, uint64_t val,
+                            unsigned size)
+{
+    ZynqXADCState *s = (ZynqXADCState *)opaque;
+    int reg = offset / 4;
+    int xadc_reg;
+    int xadc_cmd;
+    int xadc_data;
+
+    if (!zynq_xadc_check_offset(reg, false)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Invalid write access "
+                      "to addr %" HWADDR_PRIx "\n", offset);
+        return;
+    }
+
+    switch (reg) {
+    case CFG:
+        s->regs[CFG] = val;
+        break;
+    case INT_STS:
+        s->regs[INT_STS] &= ~val;
+        break;
+    case INT_MASK:
+        s->regs[INT_MASK] = val & INT_ALL;
+        break;
+    case CMDFIFO:
+        xadc_cmd  = extract32(val, 26,  4);
+        xadc_reg  = extract32(val, 16, 10);
+        xadc_data = extract32(val,  0, 16);
+
+        if (s->regs[MCTL] & MCTL_RESET) {
+            qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Sending command "
+                          "while comm channel held in reset: %" PRIx32 "\n",
+                          (uint32_t) val);
+            break;
+        }
+
+        if (xadc_reg > ZYNQ_XADC_NUM_ADC_REGS && xadc_cmd != CMD_NOP) {
+            qemu_log_mask(LOG_GUEST_ERROR, "read/write op to invalid xadc "
+                          "reg 0x%x\n", xadc_reg);
+            break;
+        }
+
+        switch (xadc_cmd) {
+        case CMD_READ:
+            xadc_push_dfifo(s, s->xadc_regs[xadc_reg]);
+            break;
+        case CMD_WRITE:
+            s->xadc_regs[xadc_reg] = xadc_data;
+            /* fallthrough */
+        case CMD_NOP:
+            xadc_push_dfifo(s, 0);
+            break;
+        }
+        break;
+    case MCTL:
+        s->regs[MCTL] = val & 0x00fffeff;
+        break;
+    }
+    zynq_xadc_update_ints(s);
+}
+
+static const MemoryRegionOps xadc_ops = {
+    .read = zynq_xadc_read,
+    .write = zynq_xadc_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void zynq_xadc_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    ZynqXADCState *s = ZYNQ_XADC(obj);
+
+    memory_region_init_io(&s->iomem, obj, &xadc_ops, s, "zynq-xadc",
+                          ZYNQ_XADC_MMIO_SIZE);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->qemu_irq);
+}
+
+static const VMStateDescription vmstate_zynq_xadc = {
+    .name = "zynq-xadc",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, ZynqXADCState, ZYNQ_XADC_NUM_IO_REGS),
+        VMSTATE_UINT16_ARRAY(xadc_regs, ZynqXADCState,
+                             ZYNQ_XADC_NUM_ADC_REGS),
+        VMSTATE_UINT16_ARRAY(xadc_dfifo, ZynqXADCState,
+                             ZYNQ_XADC_FIFO_DEPTH),
+        VMSTATE_UINT16(xadc_read_reg_previous, ZynqXADCState),
+        VMSTATE_UINT16(xadc_dfifo_entries, ZynqXADCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void zynq_xadc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_zynq_xadc;
+    dc->reset = zynq_xadc_reset;
+}
+
+static const TypeInfo zynq_xadc_info = {
+    .class_init = zynq_xadc_class_init,
+    .name  = TYPE_ZYNQ_XADC,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(ZynqXADCState),
+    .instance_init = zynq_xadc_init,
+};
+
+static void zynq_xadc_register_types(void)
+{
+    type_register_static(&zynq_xadc_info);
+}
+
+type_init(zynq_xadc_register_types)
diff --git a/hw/moxie/moxiesim.c b/hw/moxie/moxiesim.c
index 80bcc5b4f..ada3d5882 100644
--- a/hw/moxie/moxiesim.c
+++ b/hw/moxie/moxiesim.c
@@ -34,6 +34,7 @@
 #include "hw/loader.h"
 #include "hw/char/serial.h"
 #include "exec/address-spaces.h"
+#include "elf.h"
 
 #define PHYS_MEM_BASE 0x80000000
 
@@ -52,8 +53,7 @@ static void load_kernel(MoxieCPU *cpu, LoaderParams *loader_params)
     ram_addr_t initrd_offset;
 
     kernel_size = load_elf(loader_params->kernel_filename,  NULL, NULL,
-                           &entry, &kernel_low, &kernel_high, 1,
-                           ELF_MACHINE, 0);
+                           &entry, &kernel_low, &kernel_high, 1, EM_MOXIE, 0);
 
     if (kernel_size <= 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
@@ -123,11 +123,11 @@ static void moxiesim_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, cpu);
 
     /* Allocate RAM. */
-    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, ram_base, ram);
 
-    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000, &error_abort);
+    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0x1000, rom);
 
@@ -146,16 +146,11 @@ static void moxiesim_init(MachineState *machine)
     }
 }
 
-static QEMUMachine moxiesim_machine = {
-    .name = "moxiesim",
-    .desc = "Moxie simulator platform",
-    .init = moxiesim_init,
-    .is_default = 1,
-};
-
-static void moxie_machine_init(void)
+static void moxiesim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&moxiesim_machine);
+    mc->desc = "Moxie simulator platform";
+    mc->init = moxiesim_init;
+    mc->is_default = 1;
 }
 
-machine_init(moxie_machine_init)
+DEFINE_MACHINE("moxiesim", moxiesim_machine_init)
diff --git a/hw/net/Makefile.objs b/hw/net/Makefile.objs
index 98801739e..64d044923 100644
--- a/hw/net/Makefile.objs
+++ b/hw/net/Makefile.objs
@@ -19,6 +19,7 @@ common-obj-$(CONFIG_XGMAC) += xgmac.o
 common-obj-$(CONFIG_MIPSNET) += mipsnet.o
 common-obj-$(CONFIG_XILINX_AXI) += xilinx_axienet.o
 common-obj-$(CONFIG_ALLWINNER_EMAC) += allwinner_emac.o
+common-obj-$(CONFIG_IMX_FEC) += imx_fec.o
 
 common-obj-$(CONFIG_CADENCE) += cadence_gem.o
 common-obj-$(CONFIG_STELLARIS_ENET) += stellaris_enet.o
diff --git a/hw/net/cadence_gem.c b/hw/net/cadence_gem.c
index 494a346cf..3639fc17f 100644
--- a/hw/net/cadence_gem.c
+++ b/hw/net/cadence_gem.c
@@ -951,7 +951,7 @@ static void gem_phy_reset(CadenceGEMState *s)
     s->phy_regs[PHY_REG_1000BTSTAT] = 0x7C00;
     s->phy_regs[PHY_REG_EXTSTAT] = 0x3000;
     s->phy_regs[PHY_REG_PHYSPCFC_CTL] = 0x0078;
-    s->phy_regs[PHY_REG_PHYSPCFC_ST] = 0xBC00;
+    s->phy_regs[PHY_REG_PHYSPCFC_ST] = 0x7C00;
     s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL] = 0x0C60;
     s->phy_regs[PHY_REG_LED] = 0x4100;
     s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL2] = 0x000A;
@@ -964,6 +964,7 @@ static void gem_reset(DeviceState *d)
 {
     int i;
     CadenceGEMState *s = CADENCE_GEM(d);
+    const uint8_t *a;
 
     DB_PRINT("\n");
 
@@ -982,6 +983,11 @@ static void gem_reset(DeviceState *d)
     s->regs[GEM_DESCONF5] = 0x002f2145;
     s->regs[GEM_DESCONF6] = 0x00000200;
 
+    /* Set MAC address */
+    a = &s->conf.macaddr.a[0];
+    s->regs[GEM_SPADDR1LO] = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24);
+    s->regs[GEM_SPADDR1HI] = a[4] | (a[5] << 8);
+
     for (i = 0; i < 4; i++) {
         s->sar_active[i] = false;
     }
diff --git a/hw/net/e1000.c b/hw/net/e1000.c
index 09c9e9d53..34d082354 100644
--- a/hw/net/e1000.c
+++ b/hw/net/e1000.c
@@ -37,24 +37,26 @@
 
 #include "e1000_regs.h"
 
+static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
 #define E1000_DEBUG
 
 #ifdef E1000_DEBUG
 enum {
-    DEBUG_GENERAL,	DEBUG_IO,	DEBUG_MMIO,	DEBUG_INTERRUPT,
-    DEBUG_RX,		DEBUG_TX,	DEBUG_MDIC,	DEBUG_EEPROM,
-    DEBUG_UNKNOWN,	DEBUG_TXSUM,	DEBUG_TXERR,	DEBUG_RXERR,
+    DEBUG_GENERAL,      DEBUG_IO,       DEBUG_MMIO,     DEBUG_INTERRUPT,
+    DEBUG_RX,           DEBUG_TX,       DEBUG_MDIC,     DEBUG_EEPROM,
+    DEBUG_UNKNOWN,      DEBUG_TXSUM,    DEBUG_TXERR,    DEBUG_RXERR,
     DEBUG_RXFILTER,     DEBUG_PHY,      DEBUG_NOTYET,
 };
-#define DBGBIT(x)	(1<<DEBUG_##x)
+#define DBGBIT(x)    (1<<DEBUG_##x)
 static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL);
 
-#define	DBGOUT(what, fmt, ...) do { \
+#define DBGOUT(what, fmt, ...) do { \
     if (debugflags & DBGBIT(what)) \
         fprintf(stderr, "e1000: " fmt, ## __VA_ARGS__); \
     } while (0)
 #else
-#define	DBGOUT(what, fmt, ...) do {} while (0)
+#define DBGOUT(what, fmt, ...) do {} while (0)
 #endif
 
 #define IOPORT_SIZE       0x40
@@ -118,7 +120,7 @@ typedef struct E1000State_st {
     } tx;
 
     struct {
-        uint32_t val_in;	// shifted in from guest driver
+        uint32_t val_in;    /* shifted in from guest driver */
         uint16_t bitnum_in;
         uint16_t bitnum_out;
         uint16_t reading;
@@ -135,11 +137,15 @@ typedef struct E1000State_st {
 /* Compatibility flags for migration to/from qemu 1.3.0 and older */
 #define E1000_FLAG_AUTONEG_BIT 0
 #define E1000_FLAG_MIT_BIT 1
+#define E1000_FLAG_MAC_BIT 2
 #define E1000_FLAG_AUTONEG (1 << E1000_FLAG_AUTONEG_BIT)
 #define E1000_FLAG_MIT (1 << E1000_FLAG_MIT_BIT)
+#define E1000_FLAG_MAC (1 << E1000_FLAG_MAC_BIT)
     uint32_t compat_flags;
 } E1000State;
 
+#define chkflag(x)     (s->compat_flags & E1000_FLAG_##x)
+
 typedef struct E1000BaseClass {
     PCIDeviceClass parent_class;
     uint16_t phy_id2;
@@ -155,20 +161,36 @@ typedef struct E1000BaseClass {
 #define E1000_DEVICE_GET_CLASS(obj) \
     OBJECT_GET_CLASS(E1000BaseClass, (obj), TYPE_E1000_BASE)
 
-#define	defreg(x)	x = (E1000_##x>>2)
+#define defreg(x)    x = (E1000_##x>>2)
 enum {
-    defreg(CTRL),	defreg(EECD),	defreg(EERD),	defreg(GPRC),
-    defreg(GPTC),	defreg(ICR),	defreg(ICS),	defreg(IMC),
-    defreg(IMS),	defreg(LEDCTL),	defreg(MANC),	defreg(MDIC),
-    defreg(MPC),	defreg(PBA),	defreg(RCTL),	defreg(RDBAH),
-    defreg(RDBAL),	defreg(RDH),	defreg(RDLEN),	defreg(RDT),
-    defreg(STATUS),	defreg(SWSM),	defreg(TCTL),	defreg(TDBAH),
-    defreg(TDBAL),	defreg(TDH),	defreg(TDLEN),	defreg(TDT),
-    defreg(TORH),	defreg(TORL),	defreg(TOTH),	defreg(TOTL),
-    defreg(TPR),	defreg(TPT),	defreg(TXDCTL),	defreg(WUFC),
-    defreg(RA),		defreg(MTA),	defreg(CRCERRS),defreg(VFTA),
-    defreg(VET),        defreg(RDTR),   defreg(RADV),   defreg(TADV),
-    defreg(ITR),
+    defreg(CTRL),    defreg(EECD),    defreg(EERD),    defreg(GPRC),
+    defreg(GPTC),    defreg(ICR),     defreg(ICS),     defreg(IMC),
+    defreg(IMS),     defreg(LEDCTL),  defreg(MANC),    defreg(MDIC),
+    defreg(MPC),     defreg(PBA),     defreg(RCTL),    defreg(RDBAH),
+    defreg(RDBAL),   defreg(RDH),     defreg(RDLEN),   defreg(RDT),
+    defreg(STATUS),  defreg(SWSM),    defreg(TCTL),    defreg(TDBAH),
+    defreg(TDBAL),   defreg(TDH),     defreg(TDLEN),   defreg(TDT),
+    defreg(TORH),    defreg(TORL),    defreg(TOTH),    defreg(TOTL),
+    defreg(TPR),     defreg(TPT),     defreg(TXDCTL),  defreg(WUFC),
+    defreg(RA),      defreg(MTA),     defreg(CRCERRS), defreg(VFTA),
+    defreg(VET),     defreg(RDTR),    defreg(RADV),    defreg(TADV),
+    defreg(ITR),     defreg(FCRUC),   defreg(TDFH),    defreg(TDFT),
+    defreg(TDFHS),   defreg(TDFTS),   defreg(TDFPC),   defreg(RDFH),
+    defreg(RDFT),    defreg(RDFHS),   defreg(RDFTS),   defreg(RDFPC),
+    defreg(IPAV),    defreg(WUC),     defreg(WUS),     defreg(AIT),
+    defreg(IP6AT),   defreg(IP4AT),   defreg(FFLT),    defreg(FFMT),
+    defreg(FFVT),    defreg(WUPM),    defreg(PBM),     defreg(SCC),
+    defreg(ECOL),    defreg(MCC),     defreg(LATECOL), defreg(COLC),
+    defreg(DC),      defreg(TNCRS),   defreg(SEC),     defreg(CEXTERR),
+    defreg(RLEC),    defreg(XONRXC),  defreg(XONTXC),  defreg(XOFFRXC),
+    defreg(XOFFTXC), defreg(RFC),     defreg(RJC),     defreg(RNBC),
+    defreg(TSCTFC),  defreg(MGTPRC),  defreg(MGTPDC),  defreg(MGTPTC),
+    defreg(RUC),     defreg(ROC),     defreg(GORCL),   defreg(GORCH),
+    defreg(GOTCL),   defreg(GOTCH),   defreg(BPRC),    defreg(MPRC),
+    defreg(TSCTC),   defreg(PRC64),   defreg(PRC127),  defreg(PRC255),
+    defreg(PRC511),  defreg(PRC1023), defreg(PRC1522), defreg(PTC64),
+    defreg(PTC127),  defreg(PTC255),  defreg(PTC511),  defreg(PTC1023),
+    defreg(PTC1522), defreg(MPTC),    defreg(BPTC)
 };
 
 static void
@@ -193,8 +215,7 @@ e1000_link_up(E1000State *s)
 static bool
 have_autoneg(E1000State *s)
 {
-    return (s->compat_flags & E1000_FLAG_AUTONEG) &&
-           (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN);
+    return chkflag(AUTONEG) && (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN);
 }
 
 static void
@@ -226,18 +247,18 @@ enum { NPHYWRITEOPS = ARRAY_SIZE(phyreg_writeops) };
 
 enum { PHY_R = 1, PHY_W = 2, PHY_RW = PHY_R | PHY_W };
 static const char phy_regcap[0x20] = {
-    [PHY_STATUS] = PHY_R,	[M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW,
-    [PHY_ID1] = PHY_R,		[M88E1000_PHY_SPEC_CTRL] = PHY_RW,
-    [PHY_CTRL] = PHY_RW,	[PHY_1000T_CTRL] = PHY_RW,
-    [PHY_LP_ABILITY] = PHY_R,	[PHY_1000T_STATUS] = PHY_R,
-    [PHY_AUTONEG_ADV] = PHY_RW,	[M88E1000_RX_ERR_CNTR] = PHY_R,
-    [PHY_ID2] = PHY_R,		[M88E1000_PHY_SPEC_STATUS] = PHY_R,
+    [PHY_STATUS]      = PHY_R,     [M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW,
+    [PHY_ID1]         = PHY_R,     [M88E1000_PHY_SPEC_CTRL]     = PHY_RW,
+    [PHY_CTRL]        = PHY_RW,    [PHY_1000T_CTRL]             = PHY_RW,
+    [PHY_LP_ABILITY]  = PHY_R,     [PHY_1000T_STATUS]           = PHY_R,
+    [PHY_AUTONEG_ADV] = PHY_RW,    [M88E1000_RX_ERR_CNTR]       = PHY_R,
+    [PHY_ID2]         = PHY_R,     [M88E1000_PHY_SPEC_STATUS]   = PHY_R,
     [PHY_AUTONEG_EXP] = PHY_R,
 };
 
 /* PHY_ID2 documented in 8254x_GBe_SDM.pdf, pp. 250 */
 static const uint16_t phy_reg_init[] = {
-    [PHY_CTRL] =   MII_CR_SPEED_SELECT_MSB |
+    [PHY_CTRL]   = MII_CR_SPEED_SELECT_MSB |
                    MII_CR_FULL_DUPLEX |
                    MII_CR_AUTO_NEG_EN,
 
@@ -264,15 +285,15 @@ static const uint16_t phy_reg_init[] = {
 };
 
 static const uint32_t mac_reg_init[] = {
-    [PBA] =     0x00100030,
-    [LEDCTL] =  0x602,
-    [CTRL] =    E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN0 |
+    [PBA]     = 0x00100030,
+    [LEDCTL]  = 0x602,
+    [CTRL]    = E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN0 |
                 E1000_CTRL_SPD_1000 | E1000_CTRL_SLU,
-    [STATUS] =  0x80000000 | E1000_STATUS_GIO_MASTER_ENABLE |
+    [STATUS]  = 0x80000000 | E1000_STATUS_GIO_MASTER_ENABLE |
                 E1000_STATUS_ASDV | E1000_STATUS_MTXCKOK |
                 E1000_STATUS_SPEED_1000 | E1000_STATUS_FD |
                 E1000_STATUS_LU,
-    [MANC] =    E1000_MANC_EN_MNG2HOST | E1000_MANC_RCV_TCO_EN |
+    [MANC]    = E1000_MANC_EN_MNG2HOST | E1000_MANC_RCV_TCO_EN |
                 E1000_MANC_ARP_EN | E1000_MANC_0298_EN |
                 E1000_MANC_RMCP_EN,
 };
@@ -319,7 +340,7 @@ set_interrupt_cause(E1000State *s, int index, uint32_t val)
         if (s->mit_timer_on) {
             return;
         }
-        if (s->compat_flags & E1000_FLAG_MIT) {
+        if (chkflag(MIT)) {
             /* Compute the next mitigation delay according to pending
              * interrupts and the current values of RADV (provided
              * RDTR!=0), TADV and ITR.
@@ -426,6 +447,11 @@ static void e1000_reset(void *opaque)
         e1000_link_down(d);
     }
 
+    /* Throttle interrupts to prevent guest (e.g Win 2012) from
+     * reinjecting interrupts endlessly. TODO: fix non ITR case.
+     */
+    d->mac_reg[ITR] = 250;
+
     /* Some guests expect pre-initialized RAH/RAL (AddrValid flag + MACaddr) */
     d->mac_reg[RA] = 0;
     d->mac_reg[RA + 1] = E1000_RAH_AV;
@@ -510,17 +536,19 @@ set_eecd(E1000State *s, int index, uint32_t val)
 
     s->eecd_state.old_eecd = val & (E1000_EECD_SK | E1000_EECD_CS |
             E1000_EECD_DI|E1000_EECD_FWE_MASK|E1000_EECD_REQ);
-    if (!(E1000_EECD_CS & val))			// CS inactive; nothing to do
-	return;
-    if (E1000_EECD_CS & (val ^ oldval)) {	// CS rise edge; reset state
-	s->eecd_state.val_in = 0;
-	s->eecd_state.bitnum_in = 0;
-	s->eecd_state.bitnum_out = 0;
-	s->eecd_state.reading = 0;
+    if (!(E1000_EECD_CS & val)) {            /* CS inactive; nothing to do */
+        return;
     }
-    if (!(E1000_EECD_SK & (val ^ oldval)))	// no clock edge
+    if (E1000_EECD_CS & (val ^ oldval)) {    /* CS rise edge; reset state */
+        s->eecd_state.val_in = 0;
+        s->eecd_state.bitnum_in = 0;
+        s->eecd_state.bitnum_out = 0;
+        s->eecd_state.reading = 0;
+    }
+    if (!(E1000_EECD_SK & (val ^ oldval))) {    /* no clock edge */
         return;
-    if (!(E1000_EECD_SK & val)) {		// falling edge
+    }
+    if (!(E1000_EECD_SK & val)) {               /* falling edge */
         s->eecd_state.bitnum_out++;
         return;
     }
@@ -565,6 +593,56 @@ putsum(uint8_t *data, uint32_t n, uint32_t sloc, uint32_t css, uint32_t cse)
     }
 }
 
+static inline void
+inc_reg_if_not_full(E1000State *s, int index)
+{
+    if (s->mac_reg[index] != 0xffffffff) {
+        s->mac_reg[index]++;
+    }
+}
+
+static inline void
+inc_tx_bcast_or_mcast_count(E1000State *s, const unsigned char *arr)
+{
+    if (!memcmp(arr, bcast, sizeof bcast)) {
+        inc_reg_if_not_full(s, BPTC);
+    } else if (arr[0] & 1) {
+        inc_reg_if_not_full(s, MPTC);
+    }
+}
+
+static void
+grow_8reg_if_not_full(E1000State *s, int index, int size)
+{
+    uint64_t sum = s->mac_reg[index] | (uint64_t)s->mac_reg[index+1] << 32;
+
+    if (sum + size < sum) {
+        sum = ~0ULL;
+    } else {
+        sum += size;
+    }
+    s->mac_reg[index] = sum;
+    s->mac_reg[index+1] = sum >> 32;
+}
+
+static void
+increase_size_stats(E1000State *s, const int *size_regs, int size)
+{
+    if (size > 1023) {
+        inc_reg_if_not_full(s, size_regs[5]);
+    } else if (size > 511) {
+        inc_reg_if_not_full(s, size_regs[4]);
+    } else if (size > 255) {
+        inc_reg_if_not_full(s, size_regs[3]);
+    } else if (size > 127) {
+        inc_reg_if_not_full(s, size_regs[2]);
+    } else if (size > 64) {
+        inc_reg_if_not_full(s, size_regs[1]);
+    } else if (size == 64) {
+        inc_reg_if_not_full(s, size_regs[0]);
+    }
+}
+
 static inline int
 vlan_enabled(E1000State *s)
 {
@@ -602,40 +680,49 @@ fcs_len(E1000State *s)
 static void
 e1000_send_packet(E1000State *s, const uint8_t *buf, int size)
 {
+    static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511,
+                                    PTC1023, PTC1522 };
+
     NetClientState *nc = qemu_get_queue(s->nic);
     if (s->phy_reg[PHY_CTRL] & MII_CR_LOOPBACK) {
         nc->info->receive(nc, buf, size);
     } else {
         qemu_send_packet(nc, buf, size);
     }
+    inc_tx_bcast_or_mcast_count(s, buf);
+    increase_size_stats(s, PTCregs, size);
 }
 
 static void
 xmit_seg(E1000State *s)
 {
     uint16_t len, *sp;
-    unsigned int frames = s->tx.tso_frames, css, sofar, n;
+    unsigned int frames = s->tx.tso_frames, css, sofar;
     struct e1000_tx *tp = &s->tx;
 
     if (tp->tse && tp->cptse) {
         css = tp->ipcss;
         DBGOUT(TXSUM, "frames %d size %d ipcss %d\n",
                frames, tp->size, css);
-        if (tp->ip) {		// IPv4
+        if (tp->ip) {    /* IPv4 */
             stw_be_p(tp->data+css+2, tp->size - css);
             stw_be_p(tp->data+css+4,
-                          be16_to_cpup((uint16_t *)(tp->data+css+4))+frames);
-        } else			// IPv6
+                     be16_to_cpup((uint16_t *)(tp->data+css+4))+frames);
+        } else {         /* IPv6 */
             stw_be_p(tp->data+css+4, tp->size - css);
+        }
         css = tp->tucss;
         len = tp->size - css;
         DBGOUT(TXSUM, "tcp %d tucss %d len %d\n", tp->tcp, css, len);
         if (tp->tcp) {
             sofar = frames * tp->mss;
             stl_be_p(tp->data+css+4, ldl_be_p(tp->data+css+4)+sofar); /* seq */
-            if (tp->paylen - sofar > tp->mss)
-                tp->data[css + 13] &= ~9;		// PSH, FIN
-        } else	// UDP
+            if (tp->paylen - sofar > tp->mss) {
+                tp->data[css + 13] &= ~9;    /* PSH, FIN */
+            } else if (frames) {
+                inc_reg_if_not_full(s, TSCTC);
+            }
+        } else    /* UDP */
             stw_be_p(tp->data+css+4, len);
         if (tp->sum_needed & E1000_TXD_POPTS_TXSM) {
             unsigned int phsum;
@@ -657,13 +744,15 @@ xmit_seg(E1000State *s)
         memmove(tp->data, tp->data + 4, 8);
         memcpy(tp->data + 8, tp->vlan_header, 4);
         e1000_send_packet(s, tp->vlan, tp->size + 4);
-    } else
+    } else {
         e1000_send_packet(s, tp->data, tp->size);
-    s->mac_reg[TPT]++;
-    s->mac_reg[GPTC]++;
-    n = s->mac_reg[TOTL];
-    if ((s->mac_reg[TOTL] += s->tx.size) < n)
-        s->mac_reg[TOTH]++;
+    }
+
+    inc_reg_if_not_full(s, TPT);
+    grow_8reg_if_not_full(s, TOTL, s->tx.size);
+    s->mac_reg[GPTC] = s->mac_reg[TPT];
+    s->mac_reg[GOTCL] = s->mac_reg[TOTL];
+    s->mac_reg[GOTCH] = s->mac_reg[TOTH];
 }
 
 static void
@@ -679,7 +768,7 @@ process_tx_desc(E1000State *s, struct e1000_tx_desc *dp)
     struct e1000_tx *tp = &s->tx;
 
     s->mit_ide |= (txd_lower & E1000_TXD_CMD_IDE);
-    if (dtype == E1000_TXD_CMD_DEXT) {	// context descriptor
+    if (dtype == E1000_TXD_CMD_DEXT) {    /* context descriptor */
         op = le32_to_cpu(xp->cmd_and_length);
         tp->ipcss = xp->lower_setup.ip_fields.ipcss;
         tp->ipcso = xp->lower_setup.ip_fields.ipcso;
@@ -694,7 +783,7 @@ process_tx_desc(E1000State *s, struct e1000_tx_desc *dp)
         tp->tcp = (op & E1000_TXD_CMD_TCP) ? 1 : 0;
         tp->tse = (op & E1000_TXD_CMD_TSE) ? 1 : 0;
         tp->tso_frames = 0;
-        if (tp->tucso == 0) {	// this is probably wrong
+        if (tp->tucso == 0) {    /* this is probably wrong */
             DBGOUT(TXSUM, "TCP/UDP: cso 0!\n");
             tp->tucso = tp->tucss + (tp->tcp ? 16 : 6);
         }
@@ -718,7 +807,7 @@ process_tx_desc(E1000State *s, struct e1000_tx_desc *dp)
         stw_be_p(tp->vlan_header + 2,
                       le16_to_cpu(dp->upper.fields.special));
     }
-        
+
     addr = le64_to_cpu(dp->buffer_addr);
     if (tp->tse && tp->cptse) {
         msh = tp->hdr_len + tp->mss;
@@ -819,7 +908,8 @@ start_xmit(E1000State *s)
          * bogus values to TDT/TDLEN.
          * there's nothing too intelligent we could do about this.
          */
-        if (s->mac_reg[TDH] == tdh_start) {
+        if (s->mac_reg[TDH] == tdh_start ||
+            tdh_start >= s->mac_reg[TDLEN] / sizeof(desc)) {
             DBGOUT(TXERR, "TDH wraparound @%x, TDT %x, TDLEN %x\n",
                    tdh_start, s->mac_reg[TDT], s->mac_reg[TDLEN]);
             break;
@@ -831,9 +921,9 @@ start_xmit(E1000State *s)
 static int
 receive_filter(E1000State *s, const uint8_t *buf, int size)
 {
-    static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
     static const int mta_shift[] = {4, 3, 2, 0};
     uint32_t f, rctl = s->mac_reg[RCTL], ra[2], *rp;
+    int isbcast = !memcmp(buf, bcast, sizeof bcast), ismcast = (buf[0] & 1);
 
     if (is_vlan_packet(s, buf) && vlan_rx_filter_enabled(s)) {
         uint16_t vid = be16_to_cpup((uint16_t *)(buf + 14));
@@ -843,14 +933,19 @@ receive_filter(E1000State *s, const uint8_t *buf, int size)
             return 0;
     }
 
-    if (rctl & E1000_RCTL_UPE)			// promiscuous
+    if (!isbcast && !ismcast && (rctl & E1000_RCTL_UPE)) { /* promiscuous ucast */
         return 1;
+    }
 
-    if ((buf[0] & 1) && (rctl & E1000_RCTL_MPE))	// promiscuous mcast
+    if (ismcast && (rctl & E1000_RCTL_MPE)) {          /* promiscuous mcast */
+        inc_reg_if_not_full(s, MPRC);
         return 1;
+    }
 
-    if ((rctl & E1000_RCTL_BAM) && !memcmp(buf, bcast, sizeof bcast))
+    if (isbcast && (rctl & E1000_RCTL_BAM)) {          /* broadcast enabled */
+        inc_reg_if_not_full(s, BPRC);
         return 1;
+    }
 
     for (rp = s->mac_reg + RA; rp < s->mac_reg + RA + 32; rp += 2) {
         if (!(rp[1] & E1000_RAH_AV))
@@ -870,8 +965,10 @@ receive_filter(E1000State *s, const uint8_t *buf, int size)
 
     f = mta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3];
     f = (((buf[5] << 8) | buf[4]) >> f) & 0xfff;
-    if (s->mac_reg[MTA + (f >> 5)] & (1 << (f & 0x1f)))
+    if (s->mac_reg[MTA + (f >> 5)] & (1 << (f & 0x1f))) {
+        inc_reg_if_not_full(s, MPRC);
         return 1;
+    }
     DBGOUT(RXFILTER,
            "dropping, inexact filter mismatch: %02x:%02x:%02x:%02x:%02x:%02x MO %d MTA[%d] %x\n",
            buf[0], buf[1], buf[2], buf[3], buf[4], buf[5],
@@ -960,6 +1057,8 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
     size_t desc_offset;
     size_t desc_size;
     size_t total_size;
+    static const int PRCregs[6] = { PRC64, PRC127, PRC255, PRC511,
+                                    PRC1023, PRC1522 };
 
     if (!(s->mac_reg[STATUS] & E1000_STATUS_LU)) {
         return -1;
@@ -973,6 +1072,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
     if (size < sizeof(min_buf)) {
         iov_to_buf(iov, iovcnt, 0, min_buf, size);
         memset(&min_buf[size], 0, sizeof(min_buf) - size);
+        inc_reg_if_not_full(s, RUC);
         min_iov.iov_base = filter_buf = min_buf;
         min_iov.iov_len = size = sizeof(min_buf);
         iovcnt = 1;
@@ -988,6 +1088,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
         (size > MAXIMUM_ETHERNET_VLAN_SIZE
         && !(s->mac_reg[RCTL] & E1000_RCTL_LPE)))
         && !(s->mac_reg[RCTL] & E1000_RCTL_SBP)) {
+        inc_reg_if_not_full(s, ROC);
         return size;
     }
 
@@ -1065,7 +1166,8 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
         if (++s->mac_reg[RDH] * sizeof(desc) >= s->mac_reg[RDLEN])
             s->mac_reg[RDH] = 0;
         /* see comment in start_xmit; same here */
-        if (s->mac_reg[RDH] == rdh_start) {
+        if (s->mac_reg[RDH] == rdh_start ||
+            rdh_start >= s->mac_reg[RDLEN] / sizeof(desc)) {
             DBGOUT(RXERR, "RDH wraparound @%x, RDT %x, RDLEN %x\n",
                    rdh_start, s->mac_reg[RDT], s->mac_reg[RDLEN]);
             set_ics(s, 0, E1000_ICS_RXO);
@@ -1073,16 +1175,17 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
         }
     } while (desc_offset < total_size);
 
-    s->mac_reg[GPRC]++;
-    s->mac_reg[TPR]++;
+    increase_size_stats(s, PRCregs, total_size);
+    inc_reg_if_not_full(s, TPR);
+    s->mac_reg[GPRC] = s->mac_reg[TPR];
     /* TOR - Total Octets Received:
      * This register includes bytes received in a packet from the <Destination
      * Address> field through the <CRC> field, inclusively.
+     * Always include FCS length (4) in size.
      */
-    n = s->mac_reg[TORL] + size + /* Always include FCS length. */ 4;
-    if (n < s->mac_reg[TORL])
-        s->mac_reg[TORH]++;
-    s->mac_reg[TORL] = n;
+    grow_8reg_if_not_full(s, TORL, size+4);
+    s->mac_reg[GORCL] = s->mac_reg[TORL];
+    s->mac_reg[GORCH] = s->mac_reg[TORH];
 
     n = E1000_ICS_RXT0;
     if ((rdt = s->mac_reg[RDT]) < s->mac_reg[RDH])
@@ -1114,6 +1217,30 @@ mac_readreg(E1000State *s, int index)
 }
 
 static uint32_t
+mac_low4_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0xf;
+}
+
+static uint32_t
+mac_low11_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0x7ff;
+}
+
+static uint32_t
+mac_low13_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0x1fff;
+}
+
+static uint32_t
+mac_low16_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0xffff;
+}
+
+static uint32_t
 mac_icr_read(E1000State *s, int index)
 {
     uint32_t ret = s->mac_reg[ICR];
@@ -1206,46 +1333,144 @@ set_ims(E1000State *s, int index, uint32_t val)
     set_ics(s, 0, 0);
 }
 
-#define getreg(x)	[x] = mac_readreg
+#define getreg(x)    [x] = mac_readreg
 static uint32_t (*macreg_readops[])(E1000State *, int) = {
-    getreg(PBA),	getreg(RCTL),	getreg(TDH),	getreg(TXDCTL),
-    getreg(WUFC),	getreg(TDT),	getreg(CTRL),	getreg(LEDCTL),
-    getreg(MANC),	getreg(MDIC),	getreg(SWSM),	getreg(STATUS),
-    getreg(TORL),	getreg(TOTL),	getreg(IMS),	getreg(TCTL),
-    getreg(RDH),	getreg(RDT),	getreg(VET),	getreg(ICS),
-    getreg(TDBAL),	getreg(TDBAH),	getreg(RDBAH),	getreg(RDBAL),
-    getreg(TDLEN),      getreg(RDLEN),  getreg(RDTR),   getreg(RADV),
-    getreg(TADV),       getreg(ITR),
-
-    [TOTH] = mac_read_clr8,	[TORH] = mac_read_clr8,	[GPRC] = mac_read_clr4,
-    [GPTC] = mac_read_clr4,	[TPR] = mac_read_clr4,	[TPT] = mac_read_clr4,
-    [ICR] = mac_icr_read,	[EECD] = get_eecd,	[EERD] = flash_eerd_read,
-    [CRCERRS ... MPC] = &mac_readreg,
-    [RA ... RA+31] = &mac_readreg,
-    [MTA ... MTA+127] = &mac_readreg,
+    getreg(PBA),      getreg(RCTL),     getreg(TDH),      getreg(TXDCTL),
+    getreg(WUFC),     getreg(TDT),      getreg(CTRL),     getreg(LEDCTL),
+    getreg(MANC),     getreg(MDIC),     getreg(SWSM),     getreg(STATUS),
+    getreg(TORL),     getreg(TOTL),     getreg(IMS),      getreg(TCTL),
+    getreg(RDH),      getreg(RDT),      getreg(VET),      getreg(ICS),
+    getreg(TDBAL),    getreg(TDBAH),    getreg(RDBAH),    getreg(RDBAL),
+    getreg(TDLEN),    getreg(RDLEN),    getreg(RDTR),     getreg(RADV),
+    getreg(TADV),     getreg(ITR),      getreg(FCRUC),    getreg(IPAV),
+    getreg(WUC),      getreg(WUS),      getreg(SCC),      getreg(ECOL),
+    getreg(MCC),      getreg(LATECOL),  getreg(COLC),     getreg(DC),
+    getreg(TNCRS),    getreg(SEC),      getreg(CEXTERR),  getreg(RLEC),
+    getreg(XONRXC),   getreg(XONTXC),   getreg(XOFFRXC),  getreg(XOFFTXC),
+    getreg(RFC),      getreg(RJC),      getreg(RNBC),     getreg(TSCTFC),
+    getreg(MGTPRC),   getreg(MGTPDC),   getreg(MGTPTC),   getreg(GORCL),
+    getreg(GOTCL),
+
+    [TOTH]    = mac_read_clr8,      [TORH]    = mac_read_clr8,
+    [GOTCH]   = mac_read_clr8,      [GORCH]   = mac_read_clr8,
+    [PRC64]   = mac_read_clr4,      [PRC127]  = mac_read_clr4,
+    [PRC255]  = mac_read_clr4,      [PRC511]  = mac_read_clr4,
+    [PRC1023] = mac_read_clr4,      [PRC1522] = mac_read_clr4,
+    [PTC64]   = mac_read_clr4,      [PTC127]  = mac_read_clr4,
+    [PTC255]  = mac_read_clr4,      [PTC511]  = mac_read_clr4,
+    [PTC1023] = mac_read_clr4,      [PTC1522] = mac_read_clr4,
+    [GPRC]    = mac_read_clr4,      [GPTC]    = mac_read_clr4,
+    [TPT]     = mac_read_clr4,      [TPR]     = mac_read_clr4,
+    [RUC]     = mac_read_clr4,      [ROC]     = mac_read_clr4,
+    [BPRC]    = mac_read_clr4,      [MPRC]    = mac_read_clr4,
+    [TSCTC]   = mac_read_clr4,      [BPTC]    = mac_read_clr4,
+    [MPTC]    = mac_read_clr4,
+    [ICR]     = mac_icr_read,       [EECD]    = get_eecd,
+    [EERD]    = flash_eerd_read,
+    [RDFH]    = mac_low13_read,     [RDFT]    = mac_low13_read,
+    [RDFHS]   = mac_low13_read,     [RDFTS]   = mac_low13_read,
+    [RDFPC]   = mac_low13_read,
+    [TDFH]    = mac_low11_read,     [TDFT]    = mac_low11_read,
+    [TDFHS]   = mac_low13_read,     [TDFTS]   = mac_low13_read,
+    [TDFPC]   = mac_low13_read,
+    [AIT]     = mac_low16_read,
+
+    [CRCERRS ... MPC]   = &mac_readreg,
+    [IP6AT ... IP6AT+3] = &mac_readreg,    [IP4AT ... IP4AT+6] = &mac_readreg,
+    [FFLT ... FFLT+6]   = &mac_low11_read,
+    [RA ... RA+31]      = &mac_readreg,
+    [WUPM ... WUPM+31]  = &mac_readreg,
+    [MTA ... MTA+127]   = &mac_readreg,
     [VFTA ... VFTA+127] = &mac_readreg,
+    [FFMT ... FFMT+254] = &mac_low4_read,
+    [FFVT ... FFVT+254] = &mac_readreg,
+    [PBM ... PBM+16383] = &mac_readreg,
 };
 enum { NREADOPS = ARRAY_SIZE(macreg_readops) };
 
-#define putreg(x)	[x] = mac_writereg
+#define putreg(x)    [x] = mac_writereg
 static void (*macreg_writeops[])(E1000State *, int, uint32_t) = {
-    putreg(PBA),	putreg(EERD),	putreg(SWSM),	putreg(WUFC),
-    putreg(TDBAL),	putreg(TDBAH),	putreg(TXDCTL),	putreg(RDBAH),
-    putreg(RDBAL),	putreg(LEDCTL), putreg(VET),
-    [TDLEN] = set_dlen,	[RDLEN] = set_dlen,	[TCTL] = set_tctl,
-    [TDT] = set_tctl,	[MDIC] = set_mdic,	[ICS] = set_ics,
-    [TDH] = set_16bit,	[RDH] = set_16bit,	[RDT] = set_rdt,
-    [IMC] = set_imc,	[IMS] = set_ims,	[ICR] = set_icr,
-    [EECD] = set_eecd,	[RCTL] = set_rx_control, [CTRL] = set_ctrl,
-    [RDTR] = set_16bit, [RADV] = set_16bit,     [TADV] = set_16bit,
-    [ITR] = set_16bit,
-    [RA ... RA+31] = &mac_writereg,
-    [MTA ... MTA+127] = &mac_writereg,
+    putreg(PBA),      putreg(EERD),     putreg(SWSM),     putreg(WUFC),
+    putreg(TDBAL),    putreg(TDBAH),    putreg(TXDCTL),   putreg(RDBAH),
+    putreg(RDBAL),    putreg(LEDCTL),   putreg(VET),      putreg(FCRUC),
+    putreg(TDFH),     putreg(TDFT),     putreg(TDFHS),    putreg(TDFTS),
+    putreg(TDFPC),    putreg(RDFH),     putreg(RDFT),     putreg(RDFHS),
+    putreg(RDFTS),    putreg(RDFPC),    putreg(IPAV),     putreg(WUC),
+    putreg(WUS),      putreg(AIT),
+
+    [TDLEN]  = set_dlen,   [RDLEN]  = set_dlen,       [TCTL] = set_tctl,
+    [TDT]    = set_tctl,   [MDIC]   = set_mdic,       [ICS]  = set_ics,
+    [TDH]    = set_16bit,  [RDH]    = set_16bit,      [RDT]  = set_rdt,
+    [IMC]    = set_imc,    [IMS]    = set_ims,        [ICR]  = set_icr,
+    [EECD]   = set_eecd,   [RCTL]   = set_rx_control, [CTRL] = set_ctrl,
+    [RDTR]   = set_16bit,  [RADV]   = set_16bit,      [TADV] = set_16bit,
+    [ITR]    = set_16bit,
+
+    [IP6AT ... IP6AT+3] = &mac_writereg, [IP4AT ... IP4AT+6] = &mac_writereg,
+    [FFLT ... FFLT+6]   = &mac_writereg,
+    [RA ... RA+31]      = &mac_writereg,
+    [WUPM ... WUPM+31]  = &mac_writereg,
+    [MTA ... MTA+127]   = &mac_writereg,
     [VFTA ... VFTA+127] = &mac_writereg,
+    [FFMT ... FFMT+254] = &mac_writereg, [FFVT ... FFVT+254] = &mac_writereg,
+    [PBM ... PBM+16383] = &mac_writereg,
 };
 
 enum { NWRITEOPS = ARRAY_SIZE(macreg_writeops) };
 
+enum { MAC_ACCESS_PARTIAL = 1, MAC_ACCESS_FLAG_NEEDED = 2 };
+
+#define markflag(x)    ((E1000_FLAG_##x << 2) | MAC_ACCESS_FLAG_NEEDED)
+/* In the array below the meaning of the bits is: [f|f|f|f|f|f|n|p]
+ * f - flag bits (up to 6 possible flags)
+ * n - flag needed
+ * p - partially implenented */
+static const uint8_t mac_reg_access[0x8000] = {
+    [RDTR]    = markflag(MIT),    [TADV]    = markflag(MIT),
+    [RADV]    = markflag(MIT),    [ITR]     = markflag(MIT),
+
+    [IPAV]    = markflag(MAC),    [WUC]     = markflag(MAC),
+    [IP6AT]   = markflag(MAC),    [IP4AT]   = markflag(MAC),
+    [FFVT]    = markflag(MAC),    [WUPM]    = markflag(MAC),
+    [ECOL]    = markflag(MAC),    [MCC]     = markflag(MAC),
+    [DC]      = markflag(MAC),    [TNCRS]   = markflag(MAC),
+    [RLEC]    = markflag(MAC),    [XONRXC]  = markflag(MAC),
+    [XOFFTXC] = markflag(MAC),    [RFC]     = markflag(MAC),
+    [TSCTFC]  = markflag(MAC),    [MGTPRC]  = markflag(MAC),
+    [WUS]     = markflag(MAC),    [AIT]     = markflag(MAC),
+    [FFLT]    = markflag(MAC),    [FFMT]    = markflag(MAC),
+    [SCC]     = markflag(MAC),    [FCRUC]   = markflag(MAC),
+    [LATECOL] = markflag(MAC),    [COLC]    = markflag(MAC),
+    [SEC]     = markflag(MAC),    [CEXTERR] = markflag(MAC),
+    [XONTXC]  = markflag(MAC),    [XOFFRXC] = markflag(MAC),
+    [RJC]     = markflag(MAC),    [RNBC]    = markflag(MAC),
+    [MGTPDC]  = markflag(MAC),    [MGTPTC]  = markflag(MAC),
+    [RUC]     = markflag(MAC),    [ROC]     = markflag(MAC),
+    [GORCL]   = markflag(MAC),    [GORCH]   = markflag(MAC),
+    [GOTCL]   = markflag(MAC),    [GOTCH]   = markflag(MAC),
+    [BPRC]    = markflag(MAC),    [MPRC]    = markflag(MAC),
+    [TSCTC]   = markflag(MAC),    [PRC64]   = markflag(MAC),
+    [PRC127]  = markflag(MAC),    [PRC255]  = markflag(MAC),
+    [PRC511]  = markflag(MAC),    [PRC1023] = markflag(MAC),
+    [PRC1522] = markflag(MAC),    [PTC64]   = markflag(MAC),
+    [PTC127]  = markflag(MAC),    [PTC255]  = markflag(MAC),
+    [PTC511]  = markflag(MAC),    [PTC1023] = markflag(MAC),
+    [PTC1522] = markflag(MAC),    [MPTC]    = markflag(MAC),
+    [BPTC]    = markflag(MAC),
+
+    [TDFH]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFT]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFHS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFTS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFPC] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFH]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFT]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFHS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFTS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFPC] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [PBM]   = markflag(MAC) | MAC_ACCESS_PARTIAL,
+};
+
 static void
 e1000_mmio_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned size)
@@ -1254,9 +1479,20 @@ e1000_mmio_write(void *opaque, hwaddr addr, uint64_t val,
     unsigned int index = (addr & 0x1ffff) >> 2;
 
     if (index < NWRITEOPS && macreg_writeops[index]) {
-        macreg_writeops[index](s, index, val);
+        if (!(mac_reg_access[index] & MAC_ACCESS_FLAG_NEEDED)
+            || (s->compat_flags & (mac_reg_access[index] >> 2))) {
+            if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+                DBGOUT(GENERAL, "Writing to register at offset: 0x%08x. "
+                       "It is not fully implemented.\n", index<<2);
+            }
+            macreg_writeops[index](s, index, val);
+        } else {    /* "flag needed" bit is set, but the flag is not active */
+            DBGOUT(MMIO, "MMIO write attempt to disabled reg. addr=0x%08x\n",
+                   index<<2);
+        }
     } else if (index < NREADOPS && macreg_readops[index]) {
-        DBGOUT(MMIO, "e1000_mmio_writel RO %x: 0x%04"PRIx64"\n", index<<2, val);
+        DBGOUT(MMIO, "e1000_mmio_writel RO %x: 0x%04"PRIx64"\n",
+               index<<2, val);
     } else {
         DBGOUT(UNKNOWN, "MMIO unknown write addr=0x%08x,val=0x%08"PRIx64"\n",
                index<<2, val);
@@ -1269,11 +1505,21 @@ e1000_mmio_read(void *opaque, hwaddr addr, unsigned size)
     E1000State *s = opaque;
     unsigned int index = (addr & 0x1ffff) >> 2;
 
-    if (index < NREADOPS && macreg_readops[index])
-    {
-        return macreg_readops[index](s, index);
+    if (index < NREADOPS && macreg_readops[index]) {
+        if (!(mac_reg_access[index] & MAC_ACCESS_FLAG_NEEDED)
+            || (s->compat_flags & (mac_reg_access[index] >> 2))) {
+            if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+                DBGOUT(GENERAL, "Reading register at offset: 0x%08x. "
+                       "It is not fully implemented.\n", index<<2);
+            }
+            return macreg_readops[index](s, index);
+        } else {    /* "flag needed" bit is set, but the flag is not active */
+            DBGOUT(MMIO, "MMIO read attempt of disabled reg. addr=0x%08x\n",
+                   index<<2);
+        }
+    } else {
+        DBGOUT(UNKNOWN, "MMIO unknown read addr=0x%08x\n", index<<2);
     }
-    DBGOUT(UNKNOWN, "MMIO unknown read addr=0x%08x\n", index<<2);
     return 0;
 }
 
@@ -1340,7 +1586,7 @@ static int e1000_post_load(void *opaque, int version_id)
     E1000State *s = opaque;
     NetClientState *nc = qemu_get_queue(s->nic);
 
-    if (!(s->compat_flags & E1000_FLAG_MIT)) {
+    if (!chkflag(MIT)) {
         s->mac_reg[ITR] = s->mac_reg[RDTR] = s->mac_reg[RADV] =
             s->mac_reg[TADV] = 0;
         s->mit_irq_level = false;
@@ -1367,7 +1613,14 @@ static bool e1000_mit_state_needed(void *opaque)
 {
     E1000State *s = opaque;
 
-    return s->compat_flags & E1000_FLAG_MIT;
+    return chkflag(MIT);
+}
+
+static bool e1000_full_mac_needed(void *opaque)
+{
+    E1000State *s = opaque;
+
+    return chkflag(MAC);
 }
 
 static const VMStateDescription vmstate_e1000_mit_state = {
@@ -1385,6 +1638,17 @@ static const VMStateDescription vmstate_e1000_mit_state = {
     }
 };
 
+static const VMStateDescription vmstate_e1000_full_mac_state = {
+    .name = "e1000/full_mac_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = e1000_full_mac_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(mac_reg, E1000State, 0x8000),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_e1000 = {
     .name = "e1000",
     .version_id = 2,
@@ -1464,6 +1728,7 @@ static const VMStateDescription vmstate_e1000 = {
     },
     .subsections = (const VMStateDescription*[]) {
         &vmstate_e1000_mit_state,
+        &vmstate_e1000_full_mac_state,
         NULL
     }
 };
@@ -1596,6 +1861,8 @@ static Property e1000_properties[] = {
                     compat_flags, E1000_FLAG_AUTONEG_BIT, true),
     DEFINE_PROP_BIT("mitigation", E1000State,
                     compat_flags, E1000_FLAG_MIT_BIT, true),
+    DEFINE_PROP_BIT("extra_mac_registers", E1000State,
+                    compat_flags, E1000_FLAG_MAC_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1647,7 +1914,7 @@ static const TypeInfo e1000_base_info = {
 
 static const E1000Info e1000_devices[] = {
     {
-        .name      = "e1000-82540em",
+        .name      = "e1000",
         .device_id = E1000_DEV_ID_82540EM,
         .revision  = 0x03,
         .phy_id2   = E1000_PHY_ID2_8254xx_DEFAULT,
@@ -1666,11 +1933,6 @@ static const E1000Info e1000_devices[] = {
     },
 };
 
-static const TypeInfo e1000_default_info = {
-    .name          = "e1000",
-    .parent        = "e1000-82540em",
-};
-
 static void e1000_register_types(void)
 {
     int i;
@@ -1688,7 +1950,6 @@ static void e1000_register_types(void)
 
         type_register(&type_info);
     }
-    type_register_static(&e1000_default_info);
 }
 
 type_init(e1000_register_types)
diff --git a/hw/net/e1000_regs.h b/hw/net/e1000_regs.h
index 60b96aaf1..1c40244ab 100644
--- a/hw/net/e1000_regs.h
+++ b/hw/net/e1000_regs.h
@@ -158,7 +158,8 @@
 #define E1000_PHY_CTRL     0x00F10  /* PHY Control Register in CSR */
 #define FEXTNVM_SW_CONFIG  0x0001
 #define E1000_PBA      0x01000  /* Packet Buffer Allocation - RW */
-#define E1000_PBS      0x01008  /* Packet Buffer Size */
+#define E1000_PBM      0x10000  /* Packet Buffer Memory - RW */
+#define E1000_PBS      0x01008  /* Packet Buffer Size - RW */
 #define E1000_EEMNGCTL 0x01010  /* MNG EEprom Control */
 #define E1000_FLASH_UPDATES 1000
 #define E1000_EEARBC   0x01024  /* EEPROM Auto Read Bus Control */
@@ -191,6 +192,11 @@
 #define E1000_RAID     0x02C08  /* Receive Ack Interrupt Delay - RW */
 #define E1000_TXDMAC   0x03000  /* TX DMA Control - RW */
 #define E1000_KABGTXD  0x03004  /* AFE Band Gap Transmit Ref Data */
+#define E1000_RDFH     0x02410  /* Receive Data FIFO Head Register - RW */
+#define E1000_RDFT     0x02418  /* Receive Data FIFO Tail Register - RW */
+#define E1000_RDFHS    0x02420  /* Receive Data FIFO Head Saved Register - RW */
+#define E1000_RDFTS    0x02428  /* Receive Data FIFO Tail Saved Register - RW */
+#define E1000_RDFPC    0x02430  /* Receive Data FIFO Packet Count - RW */
 #define E1000_TDFH     0x03410  /* TX Data FIFO Head - RW */
 #define E1000_TDFT     0x03418  /* TX Data FIFO Tail - RW */
 #define E1000_TDFHS    0x03420  /* TX Data FIFO Head Saved - RW */
diff --git a/hw/net/eepro100.c b/hw/net/eepro100.c
index 60333b7fc..685a4781b 100644
--- a/hw/net/eepro100.c
+++ b/hw/net/eepro100.c
@@ -774,6 +774,11 @@ static void tx_command(EEPRO100State *s)
 #if 0
         uint16_t tx_buffer_el = lduw_le_pci_dma(&s->dev, tbd_address + 6);
 #endif
+        if (tx_buffer_size == 0) {
+            /* Prevent an endless loop. */
+            logout("loop in %s:%u\n", __FILE__, __LINE__);
+            break;
+        }
         tbd_address += 8;
         TRACE(RXTX, logout
             ("TBD (simplified mode): buffer address 0x%08x, size 0x%04x\n",
@@ -855,6 +860,10 @@ static void set_multicast_list(EEPRO100State *s)
 
 static void action_command(EEPRO100State *s)
 {
+    /* The loop below won't stop if it gets special handcrafted data.
+       Therefore we limit the number of iterations. */
+    unsigned max_loop_count = 16;
+
     for (;;) {
         bool bit_el;
         bool bit_s;
@@ -870,6 +879,13 @@ static void action_command(EEPRO100State *s)
 #if 0
         bool bit_sf = ((s->tx.command & COMMAND_SF) != 0);
 #endif
+
+        if (max_loop_count-- == 0) {
+            /* Prevent an endless loop. */
+            logout("loop in %s:%u\n", __FILE__, __LINE__);
+            break;
+        }
+
         s->cu_offset = s->tx.link;
         TRACE(OTHER,
               logout("val=(cu start), status=0x%04x, command=0x%04x, link=0x%08x\n",
diff --git a/hw/net/fsl_etsec/etsec.c b/hw/net/fsl_etsec/etsec.c
index 0f5cf4477..04bb41da6 100644
--- a/hw/net/fsl_etsec/etsec.c
+++ b/hw/net/fsl_etsec/etsec.c
@@ -353,7 +353,7 @@ static ssize_t etsec_receive(NetClientState *nc,
     etsec->need_flush = false;
     ret = etsec_rx_ring_write(etsec, buf, size);
     if (ret == 0) {
-        /* The packet will be queued, let's flush it when buffer is avilable
+        /* The packet will be queued, let's flush it when buffer is available
          * again. */
         etsec->need_flush = true;
     }
diff --git a/hw/net/fsl_etsec/rings.c b/hw/net/fsl_etsec/rings.c
index 68e7b6d16..0a5c6cfe3 100644
--- a/hw/net/fsl_etsec/rings.c
+++ b/hw/net/fsl_etsec/rings.c
@@ -464,9 +464,7 @@ static void rx_init_frame(eTSEC *etsec, const uint8_t *buf, size_t size)
         etsec->rx_fcb_size = 0;
     }
 
-    if (etsec->rx_buffer != NULL) {
-        g_free(etsec->rx_buffer);
-    }
+    g_free(etsec->rx_buffer);
 
     /* Do not copy the frame for now */
     etsec->rx_buffer     = (uint8_t *)buf;
diff --git a/hw/net/imx_fec.c b/hw/net/imx_fec.c
new file mode 100644
index 000000000..c50bf7ff3
--- /dev/null
+++ b/hw/net/imx_fec.c
@@ -0,0 +1,709 @@
+/*
+ * i.MX Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ * Based on Coldfire Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2007 CodeSourcery.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/net/imx_fec.h"
+#include "sysemu/dma.h"
+
+/* For crc32 */
+#include <zlib.h>
+
+#ifndef DEBUG_IMX_FEC
+#define DEBUG_IMX_FEC 0
+#endif
+
+#define FEC_PRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_FEC) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_FEC, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+#ifndef DEBUG_IMX_PHY
+#define DEBUG_IMX_PHY 0
+#endif
+
+#define PHY_PRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_PHY) { \
+            fprintf(stderr, "[%s.phy]%s: " fmt , TYPE_IMX_FEC, \
+                                                 __func__, ##args); \
+        } \
+    } while (0)
+
+static const VMStateDescription vmstate_imx_fec = {
+    .name = TYPE_IMX_FEC,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(irq_state, IMXFECState),
+        VMSTATE_UINT32(eir, IMXFECState),
+        VMSTATE_UINT32(eimr, IMXFECState),
+        VMSTATE_UINT32(rx_enabled, IMXFECState),
+        VMSTATE_UINT32(rx_descriptor, IMXFECState),
+        VMSTATE_UINT32(tx_descriptor, IMXFECState),
+        VMSTATE_UINT32(ecr, IMXFECState),
+        VMSTATE_UINT32(mmfr, IMXFECState),
+        VMSTATE_UINT32(mscr, IMXFECState),
+        VMSTATE_UINT32(mibc, IMXFECState),
+        VMSTATE_UINT32(rcr, IMXFECState),
+        VMSTATE_UINT32(tcr, IMXFECState),
+        VMSTATE_UINT32(tfwr, IMXFECState),
+        VMSTATE_UINT32(frsr, IMXFECState),
+        VMSTATE_UINT32(erdsr, IMXFECState),
+        VMSTATE_UINT32(etdsr, IMXFECState),
+        VMSTATE_UINT32(emrbr, IMXFECState),
+        VMSTATE_UINT32(miigsk_cfgr, IMXFECState),
+        VMSTATE_UINT32(miigsk_enr, IMXFECState),
+
+        VMSTATE_UINT32(phy_status, IMXFECState),
+        VMSTATE_UINT32(phy_control, IMXFECState),
+        VMSTATE_UINT32(phy_advertise, IMXFECState),
+        VMSTATE_UINT32(phy_int, IMXFECState),
+        VMSTATE_UINT32(phy_int_mask, IMXFECState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#define PHY_INT_ENERGYON            (1 << 7)
+#define PHY_INT_AUTONEG_COMPLETE    (1 << 6)
+#define PHY_INT_FAULT               (1 << 5)
+#define PHY_INT_DOWN                (1 << 4)
+#define PHY_INT_AUTONEG_LP          (1 << 3)
+#define PHY_INT_PARFAULT            (1 << 2)
+#define PHY_INT_AUTONEG_PAGE        (1 << 1)
+
+static void imx_fec_update(IMXFECState *s);
+
+/*
+ * The MII phy could raise a GPIO to the processor which in turn
+ * could be handled as an interrpt by the OS.
+ * For now we don't handle any GPIO/interrupt line, so the OS will
+ * have to poll for the PHY status.
+ */
+static void phy_update_irq(IMXFECState *s)
+{
+    imx_fec_update(s);
+}
+
+static void phy_update_link(IMXFECState *s)
+{
+    /* Autonegotiation status mirrors link status.  */
+    if (qemu_get_queue(s->nic)->link_down) {
+        PHY_PRINTF("link is down\n");
+        s->phy_status &= ~0x0024;
+        s->phy_int |= PHY_INT_DOWN;
+    } else {
+        PHY_PRINTF("link is up\n");
+        s->phy_status |= 0x0024;
+        s->phy_int |= PHY_INT_ENERGYON;
+        s->phy_int |= PHY_INT_AUTONEG_COMPLETE;
+    }
+    phy_update_irq(s);
+}
+
+static void imx_fec_set_link(NetClientState *nc)
+{
+    phy_update_link(IMX_FEC(qemu_get_nic_opaque(nc)));
+}
+
+static void phy_reset(IMXFECState *s)
+{
+    s->phy_status = 0x7809;
+    s->phy_control = 0x3000;
+    s->phy_advertise = 0x01e1;
+    s->phy_int_mask = 0;
+    s->phy_int = 0;
+    phy_update_link(s);
+}
+
+static uint32_t do_phy_read(IMXFECState *s, int reg)
+{
+    uint32_t val;
+
+    if (reg > 31) {
+        /* we only advertise one phy */
+        return 0;
+    }
+
+    switch (reg) {
+    case 0:     /* Basic Control */
+        val = s->phy_control;
+        break;
+    case 1:     /* Basic Status */
+        val = s->phy_status;
+        break;
+    case 2:     /* ID1 */
+        val = 0x0007;
+        break;
+    case 3:     /* ID2 */
+        val = 0xc0d1;
+        break;
+    case 4:     /* Auto-neg advertisement */
+        val = s->phy_advertise;
+        break;
+    case 5:     /* Auto-neg Link Partner Ability */
+        val = 0x0f71;
+        break;
+    case 6:     /* Auto-neg Expansion */
+        val = 1;
+        break;
+    case 29:    /* Interrupt source.  */
+        val = s->phy_int;
+        s->phy_int = 0;
+        phy_update_irq(s);
+        break;
+    case 30:    /* Interrupt mask */
+        val = s->phy_int_mask;
+        break;
+    case 17:
+    case 18:
+    case 27:
+    case 31:
+        qemu_log_mask(LOG_UNIMP, "[%s.phy]%s: reg %d not implemented\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        val = 0;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s.phy]%s: Bad address at offset %d\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        val = 0;
+        break;
+    }
+
+    PHY_PRINTF("read 0x%04x @ %d\n", val, reg);
+
+    return val;
+}
+
+static void do_phy_write(IMXFECState *s, int reg, uint32_t val)
+{
+    PHY_PRINTF("write 0x%04x @ %d\n", val, reg);
+
+    if (reg > 31) {
+        /* we only advertise one phy */
+        return;
+    }
+
+    switch (reg) {
+    case 0:     /* Basic Control */
+        if (val & 0x8000) {
+            phy_reset(s);
+        } else {
+            s->phy_control = val & 0x7980;
+            /* Complete autonegotiation immediately.  */
+            if (val & 0x1000) {
+                s->phy_status |= 0x0020;
+            }
+        }
+        break;
+    case 4:     /* Auto-neg advertisement */
+        s->phy_advertise = (val & 0x2d7f) | 0x80;
+        break;
+    case 30:    /* Interrupt mask */
+        s->phy_int_mask = val & 0xff;
+        phy_update_irq(s);
+        break;
+    case 17:
+    case 18:
+    case 27:
+    case 31:
+        qemu_log_mask(LOG_UNIMP, "[%s.phy)%s: reg %d not implemented\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s.phy]%s: Bad address at offset %d\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        break;
+    }
+}
+
+static void imx_fec_read_bd(IMXFECBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_read(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_fec_write_bd(IMXFECBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_write(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_fec_update(IMXFECState *s)
+{
+    uint32_t active;
+    uint32_t changed;
+
+    active = s->eir & s->eimr;
+    changed = active ^ s->irq_state;
+    if (changed) {
+        qemu_set_irq(s->irq, active);
+    }
+    s->irq_state = active;
+}
+
+static void imx_fec_do_tx(IMXFECState *s)
+{
+    int frame_size = 0;
+    uint8_t frame[FEC_MAX_FRAME_SIZE];
+    uint8_t *ptr = frame;
+    uint32_t addr = s->tx_descriptor;
+
+    while (1) {
+        IMXFECBufDesc bd;
+        int len;
+
+        imx_fec_read_bd(&bd, addr);
+        FEC_PRINTF("tx_bd %x flags %04x len %d data %08x\n",
+                   addr, bd.flags, bd.length, bd.data);
+        if ((bd.flags & FEC_BD_R) == 0) {
+            /* Run out of descriptors to transmit.  */
+            break;
+        }
+        len = bd.length;
+        if (frame_size + len > FEC_MAX_FRAME_SIZE) {
+            len = FEC_MAX_FRAME_SIZE - frame_size;
+            s->eir |= FEC_INT_BABT;
+        }
+        dma_memory_read(&address_space_memory, bd.data, ptr, len);
+        ptr += len;
+        frame_size += len;
+        if (bd.flags & FEC_BD_L) {
+            /* Last buffer in frame.  */
+            qemu_send_packet(qemu_get_queue(s->nic), frame, len);
+            ptr = frame;
+            frame_size = 0;
+            s->eir |= FEC_INT_TXF;
+        }
+        s->eir |= FEC_INT_TXB;
+        bd.flags &= ~FEC_BD_R;
+        /* Write back the modified descriptor.  */
+        imx_fec_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & FEC_BD_W) != 0) {
+            addr = s->etdsr;
+        } else {
+            addr += 8;
+        }
+    }
+
+    s->tx_descriptor = addr;
+
+    imx_fec_update(s);
+}
+
+static void imx_fec_enable_rx(IMXFECState *s)
+{
+    IMXFECBufDesc bd;
+    uint32_t tmp;
+
+    imx_fec_read_bd(&bd, s->rx_descriptor);
+
+    tmp = ((bd.flags & FEC_BD_E) != 0);
+
+    if (!tmp) {
+        FEC_PRINTF("RX buffer full\n");
+    } else if (!s->rx_enabled) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+
+    s->rx_enabled = tmp;
+}
+
+static void imx_fec_reset(DeviceState *d)
+{
+    IMXFECState *s = IMX_FEC(d);
+
+    /* Reset the FEC */
+    s->eir = 0;
+    s->eimr = 0;
+    s->rx_enabled = 0;
+    s->ecr = 0;
+    s->mscr = 0;
+    s->mibc = 0xc0000000;
+    s->rcr = 0x05ee0001;
+    s->tcr = 0;
+    s->tfwr = 0;
+    s->frsr = 0x500;
+    s->miigsk_cfgr = 0;
+    s->miigsk_enr = 0x6;
+
+    /* We also reset the PHY */
+    phy_reset(s);
+}
+
+static uint64_t imx_fec_read(void *opaque, hwaddr addr, unsigned size)
+{
+    IMXFECState *s = IMX_FEC(opaque);
+
+    FEC_PRINTF("reading from @ 0x%" HWADDR_PRIx "\n", addr);
+
+    switch (addr & 0x3ff) {
+    case 0x004:
+        return s->eir;
+    case 0x008:
+        return s->eimr;
+    case 0x010:
+        return s->rx_enabled ? (1 << 24) : 0;   /* RDAR */
+    case 0x014:
+        return 0;   /* TDAR */
+    case 0x024:
+        return s->ecr;
+    case 0x040:
+        return s->mmfr;
+    case 0x044:
+        return s->mscr;
+    case 0x064:
+        return s->mibc; /* MIBC */
+    case 0x084:
+        return s->rcr;
+    case 0x0c4:
+        return s->tcr;
+    case 0x0e4:     /* PALR */
+        return (s->conf.macaddr.a[0] << 24)
+               | (s->conf.macaddr.a[1] << 16)
+               | (s->conf.macaddr.a[2] << 8)
+               | s->conf.macaddr.a[3];
+        break;
+    case 0x0e8:     /* PAUR */
+        return (s->conf.macaddr.a[4] << 24)
+               | (s->conf.macaddr.a[5] << 16)
+               | 0x8808;
+    case 0x0ec:
+        return 0x10000; /* OPD */
+    case 0x118:
+        return 0;
+    case 0x11c:
+        return 0;
+    case 0x120:
+        return 0;
+    case 0x124:
+        return 0;
+    case 0x144:
+        return s->tfwr;
+    case 0x14c:
+        return 0x600;
+    case 0x150:
+        return s->frsr;
+    case 0x180:
+        return s->erdsr;
+    case 0x184:
+        return s->etdsr;
+    case 0x188:
+        return s->emrbr;
+    case 0x300:
+        return s->miigsk_cfgr;
+    case 0x308:
+        return s->miigsk_enr;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_FEC, __func__, addr);
+        return 0;
+    }
+}
+
+static void imx_fec_write(void *opaque, hwaddr addr,
+                          uint64_t value, unsigned size)
+{
+    IMXFECState *s = IMX_FEC(opaque);
+
+    FEC_PRINTF("writing 0x%08x @ 0x%" HWADDR_PRIx "\n", (int)value, addr);
+
+    switch (addr & 0x3ff) {
+    case 0x004: /* EIR */
+        s->eir &= ~value;
+        break;
+    case 0x008: /* EIMR */
+        s->eimr = value;
+        break;
+    case 0x010: /* RDAR */
+        if ((s->ecr & FEC_EN) && !s->rx_enabled) {
+            imx_fec_enable_rx(s);
+        }
+        break;
+    case 0x014: /* TDAR */
+        if (s->ecr & FEC_EN) {
+            imx_fec_do_tx(s);
+        }
+        break;
+    case 0x024: /* ECR */
+        s->ecr = value;
+        if (value & FEC_RESET) {
+            imx_fec_reset(DEVICE(s));
+        }
+        if ((s->ecr & FEC_EN) == 0) {
+            s->rx_enabled = 0;
+        }
+        break;
+    case 0x040: /* MMFR */
+        /* store the value */
+        s->mmfr = value;
+        if (extract32(value, 28, 1)) {
+            do_phy_write(s, extract32(value, 18, 9), extract32(value, 0, 16));
+        } else {
+            s->mmfr = do_phy_read(s, extract32(value, 18, 9));
+        }
+        /* raise the interrupt as the PHY operation is done */
+        s->eir |= FEC_INT_MII;
+        break;
+    case 0x044: /* MSCR */
+        s->mscr = value & 0xfe;
+        break;
+    case 0x064: /* MIBC */
+        /* TODO: Implement MIB.  */
+        s->mibc = (value & 0x80000000) ? 0xc0000000 : 0;
+        break;
+    case 0x084: /* RCR */
+        s->rcr = value & 0x07ff003f;
+        /* TODO: Implement LOOP mode.  */
+        break;
+    case 0x0c4: /* TCR */
+        /* We transmit immediately, so raise GRA immediately.  */
+        s->tcr = value;
+        if (value & 1) {
+            s->eir |= FEC_INT_GRA;
+        }
+        break;
+    case 0x0e4: /* PALR */
+        s->conf.macaddr.a[0] = value >> 24;
+        s->conf.macaddr.a[1] = value >> 16;
+        s->conf.macaddr.a[2] = value >> 8;
+        s->conf.macaddr.a[3] = value;
+        break;
+    case 0x0e8: /* PAUR */
+        s->conf.macaddr.a[4] = value >> 24;
+        s->conf.macaddr.a[5] = value >> 16;
+        break;
+    case 0x0ec: /* OPDR */
+        break;
+    case 0x118: /* IAUR */
+    case 0x11c: /* IALR */
+    case 0x120: /* GAUR */
+    case 0x124: /* GALR */
+        /* TODO: implement MAC hash filtering.  */
+        break;
+    case 0x144: /* TFWR */
+        s->tfwr = value & 3;
+        break;
+    case 0x14c: /* FRBR */
+        /* FRBR writes ignored.  */
+        break;
+    case 0x150: /* FRSR */
+        s->frsr = (value & 0x3fc) | 0x400;
+        break;
+    case 0x180: /* ERDSR */
+        s->erdsr = value & ~3;
+        s->rx_descriptor = s->erdsr;
+        break;
+    case 0x184: /* ETDSR */
+        s->etdsr = value & ~3;
+        s->tx_descriptor = s->etdsr;
+        break;
+    case 0x188: /* EMRBR */
+        s->emrbr = value & 0x7f0;
+        break;
+    case 0x300: /* MIIGSK_CFGR */
+        s->miigsk_cfgr = value & 0x53;
+        break;
+    case 0x308: /* MIIGSK_ENR */
+        s->miigsk_enr = (value & 0x2) ? 0x6 : 0;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_FEC, __func__, addr);
+        break;
+    }
+
+    imx_fec_update(s);
+}
+
+static int imx_fec_can_receive(NetClientState *nc)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+
+    return s->rx_enabled;
+}
+
+static ssize_t imx_fec_receive(NetClientState *nc, const uint8_t *buf,
+                               size_t len)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+    IMXFECBufDesc bd;
+    uint32_t flags = 0;
+    uint32_t addr;
+    uint32_t crc;
+    uint32_t buf_addr;
+    uint8_t *crc_ptr;
+    unsigned int buf_len;
+    size_t size = len;
+
+    FEC_PRINTF("len %d\n", (int)size);
+
+    if (!s->rx_enabled) {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Unexpected packet\n",
+                      TYPE_IMX_FEC, __func__);
+        return 0;
+    }
+
+    /* 4 bytes for the CRC.  */
+    size += 4;
+    crc = cpu_to_be32(crc32(~0, buf, size));
+    crc_ptr = (uint8_t *) &crc;
+
+    /* Huge frames are truncted.  */
+    if (size > FEC_MAX_FRAME_SIZE) {
+        size = FEC_MAX_FRAME_SIZE;
+        flags |= FEC_BD_TR | FEC_BD_LG;
+    }
+
+    /* Frames larger than the user limit just set error flags.  */
+    if (size > (s->rcr >> 16)) {
+        flags |= FEC_BD_LG;
+    }
+
+    addr = s->rx_descriptor;
+    while (size > 0) {
+        imx_fec_read_bd(&bd, addr);
+        if ((bd.flags & FEC_BD_E) == 0) {
+            /* No descriptors available.  Bail out.  */
+            /*
+             * FIXME: This is wrong. We should probably either
+             * save the remainder for when more RX buffers are
+             * available, or flag an error.
+             */
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Lost end of frame\n",
+                          TYPE_IMX_FEC, __func__);
+            break;
+        }
+        buf_len = (size <= s->emrbr) ? size : s->emrbr;
+        bd.length = buf_len;
+        size -= buf_len;
+
+        FEC_PRINTF("rx_bd 0x%x length %d\n", addr, bd.length);
+
+        /* The last 4 bytes are the CRC.  */
+        if (size < 4) {
+            buf_len += size - 4;
+        }
+        buf_addr = bd.data;
+        dma_memory_write(&address_space_memory, buf_addr, buf, buf_len);
+        buf += buf_len;
+        if (size < 4) {
+            dma_memory_write(&address_space_memory, buf_addr + buf_len,
+                             crc_ptr, 4 - size);
+            crc_ptr += 4 - size;
+        }
+        bd.flags &= ~FEC_BD_E;
+        if (size == 0) {
+            /* Last buffer in frame.  */
+            bd.flags |= flags | FEC_BD_L;
+            FEC_PRINTF("rx frame flags %04x\n", bd.flags);
+            s->eir |= FEC_INT_RXF;
+        } else {
+            s->eir |= FEC_INT_RXB;
+        }
+        imx_fec_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & FEC_BD_W) != 0) {
+            addr = s->erdsr;
+        } else {
+            addr += 8;
+        }
+    }
+    s->rx_descriptor = addr;
+    imx_fec_enable_rx(s);
+    imx_fec_update(s);
+    return len;
+}
+
+static const MemoryRegionOps imx_fec_ops = {
+    .read = imx_fec_read,
+    .write = imx_fec_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void imx_fec_cleanup(NetClientState *nc)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+
+    s->nic = NULL;
+}
+
+static NetClientInfo net_imx_fec_info = {
+    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .size = sizeof(NICState),
+    .can_receive = imx_fec_can_receive,
+    .receive = imx_fec_receive,
+    .cleanup = imx_fec_cleanup,
+    .link_status_changed = imx_fec_set_link,
+};
+
+
+static void imx_fec_realize(DeviceState *dev, Error **errp)
+{
+    IMXFECState *s = IMX_FEC(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx_fec_ops, s,
+                          TYPE_IMX_FEC, 0x400);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+    qemu_macaddr_default_if_unset(&s->conf.macaddr);
+
+    s->conf.peers.ncs[0] = nd_table[0].netdev;
+
+    s->nic = qemu_new_nic(&net_imx_fec_info, &s->conf,
+                          object_get_typename(OBJECT(dev)), DEVICE(dev)->id,
+                          s);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
+}
+
+static Property imx_fec_properties[] = {
+    DEFINE_NIC_PROPERTIES(IMXFECState, conf),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void imx_fec_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_imx_fec;
+    dc->reset = imx_fec_reset;
+    dc->props = imx_fec_properties;
+    dc->realize = imx_fec_realize;
+}
+
+static const TypeInfo imx_fec_info = {
+    .name = TYPE_IMX_FEC,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXFECState),
+    .class_init = imx_fec_class_init,
+};
+
+static void imx_fec_register_types(void)
+{
+    type_register_static(&imx_fec_info);
+}
+
+type_init(imx_fec_register_types)
diff --git a/hw/net/lan9118.c b/hw/net/lan9118.c
index 4f0e840f0..1734b5276 100644
--- a/hw/net/lan9118.c
+++ b/hw/net/lan9118.c
@@ -56,6 +56,8 @@ do { fprintf(stderr, "lan9118: error: " fmt , ## __VA_ARGS__);} while (0)
 #define CSR_E2P_CMD     0xb0
 #define CSR_E2P_DATA    0xb4
 
+#define E2P_CMD_MAC_ADDR_LOADED 0x100
+
 /* IRQ_CFG */
 #define IRQ_INT         0x00001000
 #define IRQ_EN          0x00000100
@@ -352,14 +354,14 @@ static void lan9118_reload_eeprom(lan9118_state *s)
 {
     int i;
     if (s->eeprom[0] != 0xa5) {
-        s->e2p_cmd &= ~0x10;
+        s->e2p_cmd &= ~E2P_CMD_MAC_ADDR_LOADED;
         DPRINTF("MACADDR load failed\n");
         return;
     }
     for (i = 0; i < 6; i++) {
         s->conf.macaddr.a[i] = s->eeprom[i + 1];
     }
-    s->e2p_cmd |= 0x10;
+    s->e2p_cmd |= E2P_CMD_MAC_ADDR_LOADED;
     DPRINTF("MACADDR loaded from eeprom\n");
     lan9118_mac_changed(s);
 }
@@ -902,7 +904,8 @@ static void do_mac_write(lan9118_state *s, int reg, uint32_t val)
          */
         break;
     default:
-        hw_error("lan9118: Unimplemented MAC register write: %d = 0x%x\n",
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "lan9118: Unimplemented MAC register write: %d = 0x%x\n",
                  s->mac_cmd & 0xf, val);
     }
 }
@@ -930,14 +933,16 @@ static uint32_t do_mac_read(lan9118_state *s, int reg)
     case MAC_FLOW:
         return s->mac_flow;
     default:
-        hw_error("lan9118: Unimplemented MAC register read: %d\n",
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "lan9118: Unimplemented MAC register read: %d\n",
                  s->mac_cmd & 0xf);
+        return 0;
     }
 }
 
 static void lan9118_eeprom_cmd(lan9118_state *s, int cmd, int addr)
 {
-    s->e2p_cmd = (s->e2p_cmd & 0x10) | (cmd << 28) | addr;
+    s->e2p_cmd = (s->e2p_cmd & E2P_CMD_MAC_ADDR_LOADED) | (cmd << 28) | addr;
     switch (cmd) {
     case 0:
         s->e2p_data = s->eeprom[addr];
@@ -1128,7 +1133,8 @@ static void lan9118_writel(void *opaque, hwaddr offset,
         break;
 
     default:
-        hw_error("lan9118_write: Bad reg 0x%x = %x\n", (int)offset, (int)val);
+        qemu_log_mask(LOG_GUEST_ERROR, "lan9118_write: Bad reg 0x%x = %x\n",
+                      (int)offset, (int)val);
         break;
     }
     lan9118_update(s);
@@ -1246,7 +1252,7 @@ static uint64_t lan9118_readl(void *opaque, hwaddr offset,
     case CSR_E2P_DATA:
         return s->e2p_data;
     }
-    hw_error("lan9118_read: Bad reg 0x%x\n", (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR, "lan9118_read: Bad reg 0x%x\n", (int)offset);
     return 0;
 }
 
diff --git a/hw/net/milkymist-minimac2.c b/hw/net/milkymist-minimac2.c
index 5d1cf0851..6302b8bfa 100644
--- a/hw/net/milkymist-minimac2.c
+++ b/hw/net/milkymist-minimac2.c
@@ -463,7 +463,7 @@ static int milkymist_minimac2_init(SysBusDevice *sbd)
 
     /* register buffers memory */
     memory_region_init_ram(&s->buffers, OBJECT(dev), "milkymist-minimac2.buffers",
-                           buffers_size, &error_abort);
+                           buffers_size, &error_fatal);
     vmstate_register_ram_global(&s->buffers);
     s->rx0_buf = memory_region_get_ram_ptr(&s->buffers);
     s->rx1_buf = s->rx0_buf + MINIMAC2_BUFFER_SIZE;
diff --git a/hw/net/ne2000-isa.c b/hw/net/ne2000-isa.c
index 17e7199f7..18b064463 100644
--- a/hw/net/ne2000-isa.c
+++ b/hw/net/ne2000-isa.c
@@ -44,7 +44,6 @@ typedef struct ISANE2000State {
 static NetClientInfo net_ne2000_isa_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
-    .can_receive = ne2000_can_receive,
     .receive = ne2000_receive,
 };
 
diff --git a/hw/net/ne2000.c b/hw/net/ne2000.c
index 2bdb4c927..995115e09 100644
--- a/hw/net/ne2000.c
+++ b/hw/net/ne2000.c
@@ -154,6 +154,10 @@ static int ne2000_buffer_full(NE2000State *s)
 {
     int avail, index, boundary;
 
+    if (s->stop <= s->start) {
+        return 1;
+    }
+
     index = s->curpag << 8;
     boundary = s->boundary << 8;
     if (index < boundary)
@@ -165,15 +169,6 @@ static int ne2000_buffer_full(NE2000State *s)
     return 0;
 }
 
-int ne2000_can_receive(NetClientState *nc)
-{
-    NE2000State *s = qemu_get_nic_opaque(nc);
-
-    if (s->cmd & E8390_STOP)
-        return 1;
-    return !ne2000_buffer_full(s);
-}
-
 #define MIN_BUF_SIZE 60
 
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
@@ -476,8 +471,9 @@ static inline void ne2000_mem_writel(NE2000State *s, uint32_t addr,
                                      uint32_t val)
 {
     addr &= ~1; /* XXX: check exact behaviour if not even */
-    if (addr < 32 ||
-        (addr >= NE2000_PMEM_START && addr < NE2000_MEM_SIZE)) {
+    if (addr < 32
+        || (addr >= NE2000_PMEM_START
+            && addr + sizeof(uint32_t) <= NE2000_MEM_SIZE)) {
         stl_le_p(s->mem + addr, val);
     }
 }
@@ -506,8 +502,9 @@ static inline uint32_t ne2000_mem_readw(NE2000State *s, uint32_t addr)
 static inline uint32_t ne2000_mem_readl(NE2000State *s, uint32_t addr)
 {
     addr &= ~1; /* XXX: check exact behaviour if not even */
-    if (addr < 32 ||
-        (addr >= NE2000_PMEM_START && addr < NE2000_MEM_SIZE)) {
+    if (addr < 32
+        || (addr >= NE2000_PMEM_START
+            && addr + sizeof(uint32_t) <= NE2000_MEM_SIZE)) {
         return ldl_le_p(s->mem + addr);
     } else {
         return 0xffffffff;
@@ -716,7 +713,6 @@ void ne2000_setup_io(NE2000State *s, DeviceState *dev, unsigned size)
 static NetClientInfo net_ne2000_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
-    .can_receive = ne2000_can_receive,
     .receive = ne2000_receive,
 };
 
diff --git a/hw/net/ne2000.h b/hw/net/ne2000.h
index e500306aa..d022b28fc 100644
--- a/hw/net/ne2000.h
+++ b/hw/net/ne2000.h
@@ -34,7 +34,6 @@ typedef struct NE2000State {
 void ne2000_setup_io(NE2000State *s, DeviceState *dev, unsigned size);
 extern const VMStateDescription vmstate_ne2000;
 void ne2000_reset(NE2000State *s);
-int ne2000_can_receive(NetClientState *nc);
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_);
 
 #endif
diff --git a/hw/net/pcnet.c b/hw/net/pcnet.c
index 34373767d..1f4a3dbe4 100644
--- a/hw/net/pcnet.c
+++ b/hw/net/pcnet.c
@@ -670,8 +670,7 @@ static inline hwaddr pcnet_rdra_addr(PCNetState *s, int idx)
 static inline int64_t pcnet_get_next_poll_time(PCNetState *s, int64_t current_time)
 {
     int64_t next_time = current_time +
-        muldiv64(65536 - (CSR_SPND(s) ? 0 : CSR_POLL(s)),
-                 get_ticks_per_sec(), 33000000L);
+                        (65536 - (CSR_SPND(s) ? 0 : CSR_POLL(s))) * 30;
     if (next_time <= current_time)
         next_time = current_time + 1;
     return next_time;
@@ -1065,6 +1064,12 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
             int pktcount = 0;
 
             if (!s->looptest) {
+                if (size > 4092) {
+#ifdef PCNET_DEBUG_RMD
+                    fprintf(stderr, "pcnet: truncates rx packet.\n");
+#endif
+                    size = 4092;
+                }
                 memcpy(src, buf, size);
                 /* no need to compute the CRC */
                 src[size] = 0;
@@ -1085,7 +1090,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
                 uint32_t fcs = ~0;
                 uint8_t *p = src;
 
-                while (p != &src[size-4])
+                while (p != &src[size])
                     CRC(fcs, *p++);
                 crc_err = (*(uint32_t *)p != htonl(fcs));
             }
@@ -1234,8 +1239,10 @@ static void pcnet_transmit(PCNetState *s)
         bcnt = 4096 - GET_FIELD(tmd.length, TMDL, BCNT);
 
         /* if multi-tmd packet outsizes s->buffer then skip it silently.
-           Note: this is not what real hw does */
-        if (s->xmit_pos + bcnt > sizeof(s->buffer)) {
+         * Note: this is not what real hw does.
+         * Last four bytes of s->buffer are used to store CRC FCS code.
+         */
+        if (s->xmit_pos + bcnt > sizeof(s->buffer) - 4) {
             s->xmit_pos = -1;
             goto txdone;
         }
diff --git a/hw/net/rocker/rocker.c b/hw/net/rocker/rocker.c
index 47d080fd3..2e77e5086 100644
--- a/hw/net/rocker/rocker.c
+++ b/hw/net/rocker/rocker.c
@@ -101,8 +101,7 @@ RockerSwitch *qmp_query_rocker(const char *name, Error **errp)
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
@@ -122,8 +121,7 @@ RockerPortList *qmp_query_rocker_ports(const char *name, Error **errp)
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
@@ -234,6 +232,9 @@ static int tx_consume(Rocker *r, DescInfo *info)
         frag_addr = rocker_tlv_get_le64(tlvs[ROCKER_TLV_TX_FRAG_ATTR_ADDR]);
         frag_len = rocker_tlv_get_le16(tlvs[ROCKER_TLV_TX_FRAG_ATTR_LEN]);
 
+        if (iovcnt >= ROCKER_TX_FRAGS_MAX) {
+            goto err_too_many_frags;
+        }
         iov[iovcnt].iov_len = frag_len;
         iov[iovcnt].iov_base = g_malloc(frag_len);
         if (!iov[iovcnt].iov_base) {
@@ -246,10 +247,7 @@ static int tx_consume(Rocker *r, DescInfo *info)
             err = -ROCKER_ENXIO;
             goto err_bad_io;
         }
-
-        if (++iovcnt > ROCKER_TX_FRAGS_MAX) {
-            goto err_too_many_frags;
-        }
+        iovcnt++;
     }
 
     if (iovcnt) {
@@ -265,9 +263,7 @@ err_bad_io:
 err_no_mem:
 err_bad_attr:
     for (i = 0; i < ROCKER_TX_FRAGS_MAX; i++) {
-        if (iov[i].iov_base) {
-            g_free(iov[i].iov_base);
-        }
+        g_free(iov[i].iov_base);
     }
 
     return err;
@@ -1364,7 +1360,7 @@ static int pci_rocker_init(PCIDevice *dev)
         r->fp_ports = ROCKER_FP_PORTS_MAX;
     }
 
-    r->rings = g_malloc(sizeof(DescRing *) * rocker_pci_ring_count(r));
+    r->rings = g_new(DescRing *, rocker_pci_ring_count(r));
     if (!r->rings) {
         goto err_rings_alloc;
     }
diff --git a/hw/net/rocker/rocker_desc.c b/hw/net/rocker/rocker_desc.c
index 9d896fe47..5e697b19e 100644
--- a/hw/net/rocker/rocker_desc.c
+++ b/hw/net/rocker/rocker_desc.c
@@ -136,15 +136,13 @@ bool desc_ring_set_size(DescRing *ring, uint32_t size)
     }
 
     for (i = 0; i < ring->size; i++) {
-        if (ring->info[i].buf) {
-            g_free(ring->info[i].buf);
-        }
+        g_free(ring->info[i].buf);
     }
 
     ring->size = size;
     ring->head = ring->tail = 0;
 
-    ring->info = g_realloc(ring->info, size * sizeof(DescInfo));
+    ring->info = g_renew(DescInfo, ring->info, size);
     if (!ring->info) {
         return false;
     }
@@ -347,7 +345,7 @@ DescRing *desc_ring_alloc(Rocker *r, int index)
 {
     DescRing *ring;
 
-    ring = g_malloc0(sizeof(DescRing));
+    ring = g_new0(DescRing, 1);
     if (!ring) {
         return NULL;
     }
@@ -360,9 +358,7 @@ DescRing *desc_ring_alloc(Rocker *r, int index)
 
 void desc_ring_free(DescRing *ring)
 {
-    if (ring->info) {
-        g_free(ring->info);
-    }
+    g_free(ring->info);
     g_free(ring);
 }
 
diff --git a/hw/net/rocker/rocker_fp.c b/hw/net/rocker/rocker_fp.c
index c693ae508..5906396b9 100644
--- a/hw/net/rocker/rocker_fp.c
+++ b/hw/net/rocker/rocker_fp.c
@@ -218,7 +218,7 @@ FpPort *fp_port_alloc(Rocker *r, char *sw_name,
                       MACAddr *start_mac, unsigned int index,
                       NICPeers *peers)
 {
-    FpPort *port = g_malloc0(sizeof(FpPort));
+    FpPort *port = g_new0(FpPort, 1);
 
     if (!port) {
         return NULL;
diff --git a/hw/net/rocker/rocker_of_dpa.c b/hw/net/rocker/rocker_of_dpa.c
index 874fb01d6..3cf1d61fe 100644
--- a/hw/net/rocker/rocker_of_dpa.c
+++ b/hw/net/rocker/rocker_of_dpa.c
@@ -367,7 +367,7 @@ static OfDpaFlow *of_dpa_flow_alloc(uint64_t cookie)
     OfDpaFlow *flow;
     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) / 1000;
 
-    flow = g_malloc0(sizeof(OfDpaFlow));
+    flow = g_new0(OfDpaFlow, 1);
     if (!flow) {
         return NULL;
     }
@@ -811,7 +811,7 @@ static int of_dpa_group_get_stats(OfDpa *of_dpa, uint32_t id)
 
 static OfDpaGroup *of_dpa_group_alloc(uint32_t id)
 {
-    OfDpaGroup *group = g_malloc0(sizeof(OfDpaGroup));
+    OfDpaGroup *group = g_new0(OfDpaGroup, 1);
 
     if (!group) {
         return NULL;
@@ -2039,15 +2039,14 @@ static int of_dpa_cmd_add_l2_flood(OfDpa *of_dpa, OfDpaGroup *group,
     group->l2_flood.group_count =
         rocker_tlv_get_le16(group_tlvs[ROCKER_TLV_OF_DPA_GROUP_COUNT]);
 
-    tlvs = g_malloc0((group->l2_flood.group_count + 1) *
-                     sizeof(RockerTlv *));
+    tlvs = g_new0(RockerTlv *, group->l2_flood.group_count + 1);
     if (!tlvs) {
         return -ROCKER_ENOMEM;
     }
 
     g_free(group->l2_flood.group_ids);
     group->l2_flood.group_ids =
-        g_malloc0(group->l2_flood.group_count * sizeof(uint32_t));
+        g_new0(uint32_t, group->l2_flood.group_count);
     if (!group->l2_flood.group_ids) {
         err = -ROCKER_ENOMEM;
         goto err_out;
@@ -2463,15 +2462,13 @@ RockerOfDpaFlowList *qmp_query_rocker_of_dpa_flows(const char *name,
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
     w = rocker_get_world(r, ROCKER_WORLD_TYPE_OF_DPA);
     if (!w) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s doesn't have OF-DPA world", name);
+        error_setg(errp, "rocker %s doesn't have OF-DPA world", name);
         return NULL;
     }
 
@@ -2598,15 +2595,13 @@ RockerOfDpaGroupList *qmp_query_rocker_of_dpa_groups(const char *name,
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
     w = rocker_get_world(r, ROCKER_WORLD_TYPE_OF_DPA);
     if (!w) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s doesn't have OF-DPA world", name);
+        error_setg(errp, "rocker %s doesn't have OF-DPA world", name);
         return NULL;
     }
 
diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c
index cb51613eb..68e43f3d4 100644
--- a/hw/net/rtl8139.c
+++ b/hw/net/rtl8139.c
@@ -43,7 +43,7 @@
  *                                  Added rx/tx buffer reset when enabling rx/tx operation
  *
  *  2010-Feb-04  Frediano Ziglio:   Rewrote timer support using QEMU timer only
- *                                  when strictly needed (required for for
+ *                                  when strictly needed (required for
  *                                  Darwin)
  *  2011-Mar-22  Benjamin Poirier:  Implemented VLAN offloading
  */
@@ -56,6 +56,7 @@
 #include "sysemu/dma.h"
 #include "qemu/timer.h"
 #include "net/net.h"
+#include "net/eth.h"
 #include "hw/loader.h"
 #include "sysemu/sysemu.h"
 #include "qemu/iov.h"
@@ -63,7 +64,7 @@
 /* debug RTL8139 card */
 //#define DEBUG_RTL8139 1
 
-#define PCI_FREQUENCY 33000000L
+#define PCI_PERIOD 30    /* 30 ns period = 33.333333 Mhz frequency */
 
 #define SET_MASKED(input, mask, curr) \
     ( ( (input) & ~(mask) ) | ( (curr) & (mask) ) )
@@ -72,11 +73,8 @@
 #define MOD2(input, size) \
     ( ( input ) & ( size - 1 )  )
 
-#define ETHER_ADDR_LEN 6
 #define ETHER_TYPE_LEN 2
-#define ETH_HLEN (ETHER_ADDR_LEN * 2 + ETHER_TYPE_LEN)
-#define ETH_P_IP    0x0800      /* Internet Protocol packet */
-#define ETH_P_8021Q 0x8100      /* 802.1Q VLAN Extended Header  */
+#define ETH_HLEN (ETH_ALEN * 2 + ETHER_TYPE_LEN)
 #define ETH_MTU     1500
 
 #define VLAN_TCI_LEN 2
@@ -1016,8 +1014,8 @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
 
         /* write VLAN info to descriptor variables. */
         if (s->CpCmd & CPlusRxVLAN && be16_to_cpup((uint16_t *)
-                &buf[ETHER_ADDR_LEN * 2]) == ETH_P_8021Q) {
-            dot1q_buf = &buf[ETHER_ADDR_LEN * 2];
+                &buf[ETH_ALEN * 2]) == ETH_P_VLAN) {
+            dot1q_buf = &buf[ETH_ALEN * 2];
             size -= VLAN_HLEN;
             /* if too small buffer, use the tailroom added duing expansion */
             if (size < MIN_BUF_SIZE) {
@@ -1058,10 +1056,10 @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
 
         /* receive/copy to target memory */
         if (dot1q_buf) {
-            pci_dma_write(d, rx_addr, buf, 2 * ETHER_ADDR_LEN);
-            pci_dma_write(d, rx_addr + 2 * ETHER_ADDR_LEN,
-                          buf + 2 * ETHER_ADDR_LEN + VLAN_HLEN,
-                          size - 2 * ETHER_ADDR_LEN);
+            pci_dma_write(d, rx_addr, buf, 2 * ETH_ALEN);
+            pci_dma_write(d, rx_addr + 2 * ETH_ALEN,
+                          buf + 2 * ETH_ALEN + VLAN_HLEN,
+                          size - 2 * ETH_ALEN);
         } else {
             pci_dma_write(d, rx_addr, buf, size);
         }
@@ -1785,12 +1783,12 @@ static void rtl8139_transfer_frame(RTL8139State *s, uint8_t *buf, int size,
         return;
     }
 
-    if (dot1q_buf && size >= ETHER_ADDR_LEN * 2) {
+    if (dot1q_buf && size >= ETH_ALEN * 2) {
         iov = (struct iovec[3]) {
-            { .iov_base = buf, .iov_len = ETHER_ADDR_LEN * 2 },
+            { .iov_base = buf, .iov_len = ETH_ALEN * 2 },
             { .iov_base = (void *) dot1q_buf, .iov_len = VLAN_HLEN },
-            { .iov_base = buf + ETHER_ADDR_LEN * 2,
-                .iov_len = size - ETHER_ADDR_LEN * 2 },
+            { .iov_base = buf + ETH_ALEN * 2,
+                .iov_len = size - ETH_ALEN * 2 },
         };
 
         memcpy(vlan_iov, iov, sizeof(vlan_iov));
@@ -1870,64 +1868,12 @@ static int rtl8139_transmit_one(RTL8139State *s, int descriptor)
 }
 
 /* structures and macros for task offloading */
-typedef struct ip_header
-{
-    uint8_t  ip_ver_len;    /* version and header length */
-    uint8_t  ip_tos;        /* type of service */
-    uint16_t ip_len;        /* total length */
-    uint16_t ip_id;         /* identification */
-    uint16_t ip_off;        /* fragment offset field */
-    uint8_t  ip_ttl;        /* time to live */
-    uint8_t  ip_p;          /* protocol */
-    uint16_t ip_sum;        /* checksum */
-    uint32_t ip_src,ip_dst; /* source and dest address */
-} ip_header;
-
-#define IP_HEADER_VERSION_4 4
-#define IP_HEADER_VERSION(ip) ((ip->ip_ver_len >> 4)&0xf)
-#define IP_HEADER_LENGTH(ip) (((ip->ip_ver_len)&0xf) << 2)
-
-typedef struct tcp_header
-{
-    uint16_t th_sport;		/* source port */
-    uint16_t th_dport;		/* destination port */
-    uint32_t th_seq;			/* sequence number */
-    uint32_t th_ack;			/* acknowledgement number */
-    uint16_t th_offset_flags; /* data offset, reserved 6 bits, TCP protocol flags */
-    uint16_t th_win;			/* window */
-    uint16_t th_sum;			/* checksum */
-    uint16_t th_urp;			/* urgent pointer */
-} tcp_header;
-
-typedef struct udp_header
-{
-    uint16_t uh_sport; /* source port */
-    uint16_t uh_dport; /* destination port */
-    uint16_t uh_ulen;  /* udp length */
-    uint16_t uh_sum;   /* udp checksum */
-} udp_header;
-
-typedef struct ip_pseudo_header
-{
-    uint32_t ip_src;
-    uint32_t ip_dst;
-    uint8_t  zeros;
-    uint8_t  ip_proto;
-    uint16_t ip_payload;
-} ip_pseudo_header;
-
-#define IP_PROTO_TCP 6
-#define IP_PROTO_UDP 17
-
 #define TCP_HEADER_DATA_OFFSET(tcp) (((be16_to_cpu(tcp->th_offset_flags) >> 12)&0xf) << 2)
 #define TCP_FLAGS_ONLY(flags) ((flags)&0x3f)
 #define TCP_HEADER_FLAGS(tcp) TCP_FLAGS_ONLY(be16_to_cpu(tcp->th_offset_flags))
 
 #define TCP_HEADER_CLEAR_FLAGS(tcp, off) ((tcp)->th_offset_flags &= cpu_to_be16(~TCP_FLAGS_ONLY(off)))
 
-#define TCP_FLAG_FIN  0x01
-#define TCP_FLAG_PUSH 0x08
-
 /* produces ones' complement sum of data */
 static uint16_t ones_complement_sum(uint8_t *data, size_t len)
 {
@@ -2136,7 +2082,7 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                 bswap16(txdw1 & CP_TX_VLAN_TAG_MASK));
 
             dot1q_buffer = (uint16_t *) dot1q_buffer_space;
-            dot1q_buffer[0] = cpu_to_be16(ETH_P_8021Q);
+            dot1q_buffer[0] = cpu_to_be16(ETH_P_VLAN);
             /* BE + le_to_cpu() + ~cpu_to_le()~ = BE */
             dot1q_buffer[1] = cpu_to_le16(txdw1 & CP_TX_VLAN_TAG_MASK);
         } else {
@@ -2153,12 +2099,12 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
             DPRINTF("+++ C+ mode offloaded task checksum\n");
 
             /* Large enough for Ethernet and IP headers? */
-            if (saved_size < ETH_HLEN + sizeof(ip_header)) {
+            if (saved_size < ETH_HLEN + sizeof(struct ip_header)) {
                 goto skip_offload;
             }
 
             /* ip packet header */
-            ip_header *ip = NULL;
+            struct ip_header *ip = NULL;
             int hlen = 0;
             uint8_t  ip_protocol = 0;
             uint16_t ip_data_len = 0;
@@ -2174,11 +2120,15 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
 
             DPRINTF("+++ C+ mode has IP packet\n");
 
-            /* not aligned */
+            /* Note on memory alignment: eth_payload_data is 16-bit aligned
+             * since saved_buffer is allocated with g_malloc() and ETH_HLEN is
+             * even.  32-bit accesses must use ldl/stl wrappers to avoid
+             * unaligned accesses.
+             */
             eth_payload_data = saved_buffer + ETH_HLEN;
             eth_payload_len  = saved_size   - ETH_HLEN;
 
-            ip = (ip_header*)eth_payload_data;
+            ip = (struct ip_header*)eth_payload_data;
 
             if (IP_HEADER_VERSION(ip) != IP_HEADER_VERSION_4) {
                 DPRINTF("+++ C+ mode packet has bad IP version %d "
@@ -2187,8 +2137,8 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                 goto skip_offload;
             }
 
-            hlen = IP_HEADER_LENGTH(ip);
-            if (hlen < sizeof(ip_header) || hlen > eth_payload_len) {
+            hlen = IP_HDR_GET_LEN(ip);
+            if (hlen < sizeof(struct ip_header) || hlen > eth_payload_len) {
                 goto skip_offload;
             }
 
@@ -2271,7 +2221,7 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                     }
 
                     DPRINTF("+++ C+ mode TSO TCP seqno %08x\n",
-                        be32_to_cpu(p_tcp_hdr->th_seq));
+                            ldl_be_p(&p_tcp_hdr->th_seq));
 
                     /* add 4 TCP pseudoheader fields */
                     /* copy IP source and destination fields */
@@ -2289,7 +2239,7 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                     /* keep PUSH and FIN flags only for the last frame */
                     if (!is_last_frame)
                     {
-                        TCP_HEADER_CLEAR_FLAGS(p_tcp_hdr, TCP_FLAG_PUSH|TCP_FLAG_FIN);
+                        TCP_HEADER_CLEAR_FLAGS(p_tcp_hdr, TH_PUSH | TH_FIN);
                     }
 
                     /* recalculate TCP checksum */
@@ -2327,7 +2277,8 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                         0, (uint8_t *) dot1q_buffer);
 
                     /* add transferred count to TCP sequence number */
-                    p_tcp_hdr->th_seq = cpu_to_be32(chunk_size + be32_to_cpu(p_tcp_hdr->th_seq));
+                    stl_be_p(&p_tcp_hdr->th_seq,
+                             chunk_size + ldl_be_p(&p_tcp_hdr->th_seq));
                     ++send_count;
                 }
 
@@ -2883,8 +2834,7 @@ static void rtl8139_io_writew(void *opaque, uint8_t addr, uint32_t val)
 
 static void rtl8139_set_next_tctr_time(RTL8139State *s)
 {
-    const uint64_t ns_per_period =
-        muldiv64(0x100000000LL, get_ticks_per_sec(), PCI_FREQUENCY);
+    const uint64_t ns_per_period = (uint64_t)PCI_PERIOD << 32;
 
     DPRINTF("entered rtl8139_set_next_tctr_time\n");
 
@@ -2902,7 +2852,7 @@ static void rtl8139_set_next_tctr_time(RTL8139State *s)
     if (!s->TimerInt) {
         timer_del(s->timer);
     } else {
-        uint64_t delta = muldiv64(s->TimerInt, get_ticks_per_sec(), PCI_FREQUENCY);
+        uint64_t delta = (uint64_t)s->TimerInt * PCI_PERIOD;
         if (s->TCTR_base + delta <= qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)) {
             delta += ns_per_period;
         }
@@ -3176,8 +3126,8 @@ static uint32_t rtl8139_io_readl(void *opaque, uint8_t addr)
             break;
 
         case Timer:
-            ret = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->TCTR_base,
-                           PCI_FREQUENCY, get_ticks_per_sec());
+            ret = (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->TCTR_base) /
+                  PCI_PERIOD;
             DPRINTF("TCTR Timer read val=0x%08x\n", ret);
             break;
 
@@ -3271,8 +3221,7 @@ static void rtl8139_pre_save(void *opaque)
     int64_t current_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     /* for migration to older versions */
-    s->TCTR = muldiv64(current_time - s->TCTR_base, PCI_FREQUENCY,
-                       get_ticks_per_sec());
+    s->TCTR = (current_time - s->TCTR_base) / PCI_PERIOD;
     s->rtl8139_mmio_io_addr_dummy = 0;
 }
 
@@ -3440,10 +3389,8 @@ static void pci_rtl8139_uninit(PCIDevice *dev)
 {
     RTL8139State *s = RTL8139(dev);
 
-    if (s->cplus_txbuffer) {
-        g_free(s->cplus_txbuffer);
-        s->cplus_txbuffer = NULL;
-    }
+    g_free(s->cplus_txbuffer);
+    s->cplus_txbuffer = NULL;
     timer_del(s->timer);
     timer_free(s->timer);
     qemu_del_nic(s->nic);
diff --git a/hw/net/smc91c111.c b/hw/net/smc91c111.c
index 74e06e6c7..c19cdd14d 100644
--- a/hw/net/smc91c111.c
+++ b/hw/net/smc91c111.c
@@ -124,6 +124,25 @@ static void smc91c111_update(smc91c111_state *s)
     qemu_set_irq(s->irq, level);
 }
 
+static int smc91c111_can_receive(smc91c111_state *s)
+{
+    if ((s->rcr & RCR_RXEN) == 0 || (s->rcr & RCR_SOFT_RST)) {
+        return 1;
+    }
+    if (s->allocated == (1 << NUM_PACKETS) - 1 ||
+        s->rx_fifo_len == NUM_PACKETS) {
+        return 0;
+    }
+    return 1;
+}
+
+static inline void smc91c111_flush_queued_packets(smc91c111_state *s)
+{
+    if (smc91c111_can_receive(s)) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+}
+
 /* Try to allocate a packet.  Returns 0x80 on failure.  */
 static int smc91c111_allocate_packet(smc91c111_state *s)
 {
@@ -164,6 +183,7 @@ static void smc91c111_pop_rx_fifo(smc91c111_state *s)
     } else {
         s->int_level &= ~INT_RCV;
     }
+    smc91c111_flush_queued_packets(s);
     smc91c111_update(s);
 }
 
@@ -185,7 +205,7 @@ static void smc91c111_release_packet(smc91c111_state *s, int packet)
     s->allocated &= ~(1 << packet);
     if (s->tx_alloc == 0x80)
         smc91c111_tx_alloc(s);
-    qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    smc91c111_flush_queued_packets(s);
 }
 
 /* Flush the TX FIFO.  */
@@ -311,6 +331,7 @@ static void smc91c111_writeb(void *opaque, hwaddr offset,
             if (s->rcr & RCR_SOFT_RST) {
                 smc91c111_reset(DEVICE(s));
             }
+            smc91c111_flush_queued_packets(s);
             return;
         case 10: case 11: /* RPCR */
             /* Ignored */
@@ -636,15 +657,11 @@ static uint32_t smc91c111_readl(void *opaque, hwaddr offset)
     return val;
 }
 
-static int smc91c111_can_receive(NetClientState *nc)
+static int smc91c111_can_receive_nc(NetClientState *nc)
 {
     smc91c111_state *s = qemu_get_nic_opaque(nc);
 
-    if ((s->rcr & RCR_RXEN) == 0 || (s->rcr & RCR_SOFT_RST))
-        return 1;
-    if (s->allocated == (1 << NUM_PACKETS) - 1)
-        return 0;
-    return 1;
+    return smc91c111_can_receive(s);
 }
 
 static ssize_t smc91c111_receive(NetClientState *nc, const uint8_t *buf, size_t size)
@@ -739,7 +756,7 @@ static const MemoryRegionOps smc91c111_mem_ops = {
 static NetClientInfo net_smc91c111_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
-    .can_receive = smc91c111_can_receive,
+    .can_receive = smc91c111_can_receive_nc,
     .receive = smc91c111_receive,
 };
 
diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
index 1d76b94c8..10e233a7f 100644
--- a/hw/net/vhost_net.c
+++ b/hw/net/vhost_net.c
@@ -77,13 +77,9 @@ static const int user_feature_bits[] = {
     VIRTIO_NET_F_HOST_ECN,
     VIRTIO_NET_F_HOST_UFO,
     VIRTIO_NET_F_MRG_RXBUF,
-    VIRTIO_NET_F_STATUS,
-    VIRTIO_NET_F_CTRL_VQ,
-    VIRTIO_NET_F_CTRL_RX,
-    VIRTIO_NET_F_CTRL_VLAN,
-    VIRTIO_NET_F_CTRL_RX_EXTRA,
-    VIRTIO_NET_F_CTRL_MAC_ADDR,
-    VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
+
+    /* This bit implies RARP isn't sent by QEMU out of band */
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
 
     VIRTIO_NET_F_MQ,
 
@@ -122,6 +118,11 @@ void vhost_net_ack_features(struct vhost_net *net, uint64_t features)
     vhost_ack_features(&net->dev, vhost_net_get_feature_bits(net), features);
 }
 
+uint64_t vhost_net_get_max_queues(VHostNetState *net)
+{
+    return net->dev.max_queues;
+}
+
 static int vhost_net_get_fd(NetClientState *backend)
 {
     switch (backend->info->type) {
@@ -143,6 +144,11 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options)
         fprintf(stderr, "vhost-net requires net backend to be setup\n");
         goto fail;
     }
+    net->nc = options->net_backend;
+
+    net->dev.max_queues = 1;
+    net->dev.nvqs = 2;
+    net->dev.vqs = net->vqs;
 
     if (backend_kernel) {
         r = vhost_net_get_fd(options->net_backend);
@@ -152,14 +158,15 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options)
         net->dev.backend_features = qemu_has_vnet_hdr(options->net_backend)
             ? 0 : (1ULL << VHOST_NET_F_VIRTIO_NET_HDR);
         net->backend = r;
+        net->dev.protocol_features = 0;
     } else {
         net->dev.backend_features = 0;
+        net->dev.protocol_features = 0;
         net->backend = -1;
-    }
-    net->nc = options->net_backend;
 
-    net->dev.nvqs = 2;
-    net->dev.vqs = net->vqs;
+        /* vhost-user needs vq_index to initiate a specific queue pair */
+        net->dev.vq_index = net->nc->queue_index * net->dev.nvqs;
+    }
 
     r = vhost_dev_init(&net->dev, options->opaque,
                        options->backend_type);
@@ -241,8 +248,7 @@ static int vhost_net_start_one(struct vhost_net *net,
         file.fd = net->backend;
         for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
             const VhostOps *vhost_ops = net->dev.vhost_ops;
-            r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                      &file);
+            r = vhost_ops->vhost_net_set_backend(&net->dev, &file);
             if (r < 0) {
                 r = -errno;
                 goto fail;
@@ -255,8 +261,7 @@ fail:
     if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
         while (file.index-- > 0) {
             const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                          &file);
+            int r = vhost_ops->vhost_net_set_backend(&net->dev, &file);
             assert(r >= 0);
         }
     }
@@ -278,15 +283,7 @@ static void vhost_net_stop_one(struct vhost_net *net,
     if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
         for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
             const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                          &file);
-            assert(r >= 0);
-        }
-    } else if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER) {
-        for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
-            const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_RESET_OWNER,
-                                          NULL);
+            int r = vhost_ops->vhost_net_set_backend(&net->dev, &file);
             assert(r >= 0);
         }
     }
@@ -303,21 +300,19 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
     VirtioBusState *vbus = VIRTIO_BUS(qbus);
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
-    int r, e, i;
+    int r, e, i, j;
 
     if (!k->set_guest_notifiers) {
         error_report("binding does not support guest notifiers");
-        r = -ENOSYS;
-        goto err;
-    }
-
-    r = vhost_net_set_vnet_endian(dev, ncs[0].peer, true);
-    if (r < 0) {
-        goto err;
+        return -ENOSYS;
     }
 
-    for (i = 0; i < total_queues; i++) {
-        vhost_net_set_vq_index(get_vhost_net(ncs[i].peer), i * 2);
+    for (j = 0; j < total_queues; j++) {
+        r = vhost_net_set_vnet_endian(dev, ncs[j].peer, true);
+        if (r < 0) {
+            goto err_endian;
+        }
+        vhost_net_set_vq_index(get_vhost_net(ncs[j].peer), j * 2);
     }
 
     r = k->set_guest_notifiers(qbus->parent, total_queues * 2, true);
@@ -346,8 +341,9 @@ err_start:
         fflush(stderr);
     }
 err_endian:
-    vhost_net_set_vnet_endian(dev, ncs[0].peer, false);
-err:
+    while (--j >= 0) {
+        vhost_net_set_vnet_endian(dev, ncs[j].peer, false);
+    }
     return r;
 }
 
@@ -379,6 +375,18 @@ void vhost_net_cleanup(struct vhost_net *net)
     g_free(net);
 }
 
+int vhost_net_notify_migration_done(struct vhost_net *net, char* mac_addr)
+{
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+    int r = -1;
+
+    if (vhost_ops->vhost_migration_done) {
+        r = vhost_ops->vhost_migration_done(&net->dev, mac_addr);
+    }
+
+    return r;
+}
+
 bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
 {
     return vhost_virtqueue_pending(&net->dev, idx);
@@ -411,7 +419,25 @@ VHostNetState *get_vhost_net(NetClientState *nc)
 
     return vhost_net;
 }
+
+int vhost_set_vring_enable(NetClientState *nc, int enable)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+
+    if (vhost_ops->vhost_set_vring_enable) {
+        return vhost_ops->vhost_set_vring_enable(&net->dev, enable);
+    }
+
+    return 0;
+}
+
 #else
+uint64_t vhost_net_get_max_queues(VHostNetState *net)
+{
+    return 1;
+}
+
 struct vhost_net *vhost_net_init(VhostNetOptions *options)
 {
     error_report("vhost-net support is not compiled in");
@@ -452,8 +478,18 @@ void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
 {
 }
 
+int vhost_net_notify_migration_done(struct vhost_net *net, char* mac_addr)
+{
+    return -1;
+}
+
 VHostNetState *get_vhost_net(NetClientState *nc)
 {
     return 0;
 }
+
+int vhost_set_vring_enable(NetClientState *nc, int enable)
+{
+    return 0;
+}
 #endif
diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index 4aa93fb93..a877614e3 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -406,6 +406,10 @@ static int peer_attach(VirtIONet *n, int index)
         return 0;
     }
 
+    if (nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER) {
+        vhost_set_vring_enable(nc->peer, 1);
+    }
+
     if (nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
         return 0;
     }
@@ -421,6 +425,10 @@ static int peer_detach(VirtIONet *n, int index)
         return 0;
     }
 
+    if (nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER) {
+        vhost_set_vring_enable(nc->peer, 0);
+    }
+
     if (nc->peer->info->type !=  NET_CLIENT_OPTIONS_KIND_TAP) {
         return 0;
     }
@@ -1118,7 +1126,7 @@ static void virtio_net_tx_complete(NetClientState *nc, ssize_t len)
     virtqueue_push(q->tx_vq, &q->async_tx.elem, 0);
     virtio_notify(vdev, q->tx_vq);
 
-    q->async_tx.elem.out_num = q->async_tx.len = 0;
+    q->async_tx.elem.out_num = 0;
 
     virtio_queue_set_notification(q->tx_vq, 1);
     virtio_net_flush_tx(q);
@@ -1142,7 +1150,7 @@ static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
     }
 
     while (virtqueue_pop(q->tx_vq, &elem)) {
-        ssize_t ret, len;
+        ssize_t ret;
         unsigned int out_num = elem.out_num;
         struct iovec *out_sg = &elem.out_sg[0];
         struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1];
@@ -1190,18 +1198,14 @@ static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
             out_sg = sg;
         }
 
-        len = n->guest_hdr_len;
-
         ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index),
                                       out_sg, out_num, virtio_net_tx_complete);
         if (ret == 0) {
             virtio_queue_set_notification(q->tx_vq, 0);
             q->async_tx.elem = elem;
-            q->async_tx.len  = len;
             return -EBUSY;
         }
 
-        len += ret;
 drop:
         virtqueue_push(q->tx_vq, &elem, 0);
         virtio_notify(vdev, q->tx_vq);
diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c
index 2504425cf..2b4aad718 100644
--- a/hw/net/vmxnet3.c
+++ b/hw/net/vmxnet3.c
@@ -729,9 +729,7 @@ static void vmxnet3_process_tx_queue(VMXNET3State *s, int qidx)
         }
 
         if (txd.eop) {
-            if (!s->skip_current_tx_pkt) {
-                vmxnet_tx_pkt_parse(s->tx_pkt);
-
+            if (!s->skip_current_tx_pkt && vmxnet_tx_pkt_parse(s->tx_pkt)) {
                 if (s->needs_vlan) {
                     vmxnet_tx_pkt_setup_vlan_header(s->tx_pkt, s->tci);
                 }
@@ -927,9 +925,9 @@ static void vmxnet3_rx_need_csum_calculate(struct VmxnetRxPkt *pkt,
 
     /* Validate packet len: csum_start + scum_offset + length of csum field */
     if (pkt_len < (vhdr->csum_start + vhdr->csum_offset + 2)) {
-        VMW_PKPRN("packet len:%d < csum_start(%d) + csum_offset(%d) + 2, "
+        VMW_PKPRN("packet len:%lu < csum_start(%d) + csum_offset(%d) + 2, "
                   "cannot calculate checksum",
-                  len, vhdr->csum_start, vhdr->csum_offset);
+                  pkt_len, vhdr->csum_start, vhdr->csum_offset);
         return;
     }
 
@@ -1165,9 +1163,13 @@ vmxnet3_io_bar0_write(void *opaque, hwaddr addr,
 static uint64_t
 vmxnet3_io_bar0_read(void *opaque, hwaddr addr, unsigned size)
 {
+    VMXNET3State *s = opaque;
+
     if (VMW_IS_MULTIREG_ADDR(addr, VMXNET3_REG_IMR,
                         VMXNET3_MAX_INTRS, VMXNET3_REG_ALIGN)) {
-        g_assert_not_reached();
+        int l = VMW_MULTIREG_IDX_BY_ADDR(addr, VMXNET3_REG_IMR,
+                                         VMXNET3_REG_ALIGN);
+        return s->interrupt_states[l].is_masked;
     }
 
     VMW_CBPRN("BAR0 unknown read [%" PRIx64 "], size %d", addr, size);
@@ -1192,8 +1194,13 @@ static void vmxnet3_reset_mac(VMXNET3State *s)
 
 static void vmxnet3_deactivate_device(VMXNET3State *s)
 {
-    VMW_CBPRN("Deactivating vmxnet3...");
-    s->device_active = false;
+    if (s->device_active) {
+        VMW_CBPRN("Deactivating vmxnet3...");
+        vmxnet_tx_pkt_reset(s->tx_pkt);
+        vmxnet_tx_pkt_uninit(s->tx_pkt);
+        vmxnet_rx_pkt_uninit(s->rx_pkt);
+        s->device_active = false;
+    }
 }
 
 static void vmxnet3_reset(VMXNET3State *s)
@@ -1202,7 +1209,6 @@ static void vmxnet3_reset(VMXNET3State *s)
 
     vmxnet3_deactivate_device(s);
     vmxnet3_reset_interrupt_states(s);
-    vmxnet_tx_pkt_reset(s->tx_pkt);
     s->drv_shmem = 0;
     s->tx_sop = true;
     s->skip_current_tx_pkt = false;
@@ -1287,6 +1293,10 @@ static uint32_t vmxnet3_get_interrupt_config(VMXNET3State *s)
 static void vmxnet3_fill_stats(VMXNET3State *s)
 {
     int i;
+
+    if (!s->device_active)
+        return;
+
     for (i = 0; i < s->txq_num; i++) {
         cpu_physical_memory_write(s->txq_descr[i].tx_stats_pa,
                                   &s->txq_descr[i].txq_stats,
@@ -1425,6 +1435,12 @@ static void vmxnet3_activate_device(VMXNET3State *s)
         return;
     }
 
+    /* Verify if device is active */
+    if (s->device_active) {
+        VMW_CFPRN("Vmxnet3 device is active");
+        return;
+    }
+
     vmxnet3_adjust_by_guest_type(s);
     vmxnet3_update_features(s);
     vmxnet3_update_pm_state(s);
@@ -1621,7 +1637,7 @@ static void vmxnet3_handle_command(VMXNET3State *s, uint64_t cmd)
         break;
 
     case VMXNET3_CMD_QUIESCE_DEV:
-        VMW_CBPRN("Set: VMXNET3_CMD_QUIESCE_DEV - pause the device");
+        VMW_CBPRN("Set: VMXNET3_CMD_QUIESCE_DEV - deactivate the device");
         vmxnet3_deactivate_device(s);
         break;
 
@@ -1629,6 +1645,11 @@ static void vmxnet3_handle_command(VMXNET3State *s, uint64_t cmd)
         VMW_CBPRN("Set: VMXNET3_CMD_GET_CONF_INTR - interrupt configuration");
         break;
 
+    case VMXNET3_CMD_GET_ADAPTIVE_RING_INFO:
+        VMW_CBPRN("Set: VMXNET3_CMD_GET_ADAPTIVE_RING_INFO - "
+                  "adaptive ring info flags");
+        break;
+
     default:
         VMW_CBPRN("Received unknown command: %" PRIx64, cmd);
         break;
@@ -1668,6 +1689,10 @@ static uint64_t vmxnet3_get_command_status(VMXNET3State *s)
         ret = vmxnet3_get_interrupt_config(s);
         break;
 
+    case VMXNET3_CMD_GET_ADAPTIVE_RING_INFO:
+        ret = VMXNET3_DISABLE_ADAPTIVE_RING;
+        break;
+
     default:
         VMW_WRPRN("Received request for unknown command: %x", s->last_command);
         ret = -1;
@@ -1726,7 +1751,7 @@ vmxnet3_io_bar1_write(void *opaque,
          * shared address only after we get the high part
          */
         if (val == 0) {
-            s->device_active = false;
+            vmxnet3_deactivate_device(s);
         }
         s->temp_shared_guest_driver_memory = val;
         s->drv_shmem = 0;
@@ -2000,16 +2025,13 @@ static bool vmxnet3_peer_has_vnet_hdr(VMXNET3State *s)
         return true;
     }
 
-    VMW_WRPRN("Peer has no virtio extension. Task offloads will be emulated.");
     return false;
 }
 
 static void vmxnet3_net_uninit(VMXNET3State *s)
 {
     g_free(s->mcast_list);
-    vmxnet_tx_pkt_reset(s->tx_pkt);
-    vmxnet_tx_pkt_uninit(s->tx_pkt);
-    vmxnet_rx_pkt_uninit(s->rx_pkt);
+    vmxnet3_deactivate_device(s);
     qemu_del_nic(s->nic);
 }
 
diff --git a/hw/net/vmxnet3.h b/hw/net/vmxnet3.h
index f987d7126..f7006afe9 100644
--- a/hw/net/vmxnet3.h
+++ b/hw/net/vmxnet3.h
@@ -198,9 +198,13 @@ enum {
     VMXNET3_CMD_GET_DID_LO,                               /* 0xF00D0005 */
     VMXNET3_CMD_GET_DID_HI,                               /* 0xF00D0006 */
     VMXNET3_CMD_GET_DEV_EXTRA_INFO,                       /* 0xF00D0007 */
-    VMXNET3_CMD_GET_CONF_INTR                             /* 0xF00D0008 */
+    VMXNET3_CMD_GET_CONF_INTR,                            /* 0xF00D0008 */
+    VMXNET3_CMD_GET_ADAPTIVE_RING_INFO                    /* 0xF00D0009 */
 };
 
+/* Adaptive Ring Info Flags */
+#define VMXNET3_DISABLE_ADAPTIVE_RING 1
+
 /*
  *    Little Endian layout of bitfields -
  *    Byte 0 :    7.....len.....0
diff --git a/hw/net/vmxnet_tx_pkt.c b/hw/net/vmxnet_tx_pkt.c
index f7344c4cb..eb88ddf25 100644
--- a/hw/net/vmxnet_tx_pkt.c
+++ b/hw/net/vmxnet_tx_pkt.c
@@ -142,11 +142,24 @@ static bool vmxnet_tx_pkt_parse_headers(struct VmxnetTxPkt *pkt)
 
     bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
                             ETH_MAX_L2_HDR_LEN);
-    if (bytes_read < ETH_MAX_L2_HDR_LEN) {
+    if (bytes_read < sizeof(struct eth_header)) {
+        l2_hdr->iov_len = 0;
+        return false;
+    }
+
+    l2_hdr->iov_len = sizeof(struct eth_header);
+    switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) {
+    case ETH_P_VLAN:
+        l2_hdr->iov_len += sizeof(struct vlan_header);
+        break;
+    case ETH_P_DVLAN:
+        l2_hdr->iov_len += 2 * sizeof(struct vlan_header);
+        break;
+    }
+
+    if (bytes_read < l2_hdr->iov_len) {
         l2_hdr->iov_len = 0;
         return false;
-    } else {
-        l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
     }
 
     l3_proto = eth_get_l3_proto(l2_hdr->iov_base, l2_hdr->iov_len);
diff --git a/hw/net/xen_nic.c b/hw/net/xen_nic.c
index d7cbfc103..0da16b44f 100644
--- a/hw/net/xen_nic.c
+++ b/hw/net/xen_nic.c
@@ -24,7 +24,6 @@
 #include <stdarg.h>
 #include <string.h>
 #include <unistd.h>
-#include <signal.h>
 #include <inttypes.h>
 #include <fcntl.h>
 #include <errno.h>
diff --git a/hw/nvram/fw_cfg.c b/hw/nvram/fw_cfg.c
index 88481b78c..5a03b5d35 100644
--- a/hw/nvram/fw_cfg.c
+++ b/hw/nvram/fw_cfg.c
@@ -23,6 +23,7 @@
  */
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 #include "hw/isa/isa.h"
 #include "hw/nvram/fw_cfg.h"
 #include "hw/sysbus.h"
@@ -30,7 +31,7 @@
 #include "qemu/error-report.h"
 #include "qemu/config-file.h"
 
-#define FW_CFG_SIZE 2
+#define FW_CFG_CTL_SIZE 2
 #define FW_CFG_NAME "fw_cfg"
 #define FW_CFG_PATH "/machine/" FW_CFG_NAME
 
@@ -42,6 +43,18 @@
 #define FW_CFG_IO(obj)  OBJECT_CHECK(FWCfgIoState,  (obj), TYPE_FW_CFG_IO)
 #define FW_CFG_MEM(obj) OBJECT_CHECK(FWCfgMemState, (obj), TYPE_FW_CFG_MEM)
 
+/* FW_CFG_VERSION bits */
+#define FW_CFG_VERSION      0x01
+#define FW_CFG_VERSION_DMA  0x02
+
+/* FW_CFG_DMA_CONTROL bits */
+#define FW_CFG_DMA_CTL_ERROR   0x01
+#define FW_CFG_DMA_CTL_READ    0x02
+#define FW_CFG_DMA_CTL_SKIP    0x04
+#define FW_CFG_DMA_CTL_SELECT  0x08
+
+#define FW_CFG_DMA_SIGNATURE 0x51454d5520434647ULL /* "QEMU CFG" */
+
 typedef struct FWCfgEntry {
     uint32_t len;
     uint8_t *data;
@@ -59,6 +72,11 @@ struct FWCfgState {
     uint16_t cur_entry;
     uint32_t cur_offset;
     Notifier machine_ready;
+
+    bool dma_enabled;
+    dma_addr_t dma_addr;
+    AddressSpace *dma_as;
+    MemoryRegion dma_iomem;
 };
 
 struct FWCfgIoState {
@@ -67,7 +85,7 @@ struct FWCfgIoState {
     /*< public >*/
 
     MemoryRegion comb_iomem;
-    uint32_t iobase;
+    uint32_t iobase, dma_iobase;
 };
 
 struct FWCfgMemState {
@@ -187,9 +205,7 @@ static void fw_cfg_bootsplash(FWCfgState *s)
             g_free(filename);
             return;
         }
-        if (boot_splash_filedata != NULL) {
-            g_free(boot_splash_filedata);
-        }
+        g_free(boot_splash_filedata);
         boot_splash_filedata = (uint8_t *)file_data;
         boot_splash_filedata_size = file_size;
 
@@ -254,7 +270,8 @@ static int fw_cfg_select(FWCfgState *s, uint16_t key)
 static uint8_t fw_cfg_read(FWCfgState *s)
 {
     int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
-    FWCfgEntry *e = &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
+    FWCfgEntry *e = (s->cur_entry == FW_CFG_INVALID) ? NULL :
+                    &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
     uint8_t ret;
 
     if (s->cur_entry == FW_CFG_INVALID || !e->data || s->cur_offset >= e->len)
@@ -294,6 +311,130 @@ static void fw_cfg_data_mem_write(void *opaque, hwaddr addr,
     } while (i);
 }
 
+static void fw_cfg_dma_transfer(FWCfgState *s)
+{
+    dma_addr_t len;
+    FWCfgDmaAccess dma;
+    int arch;
+    FWCfgEntry *e;
+    int read;
+    dma_addr_t dma_addr;
+
+    /* Reset the address before the next access */
+    dma_addr = s->dma_addr;
+    s->dma_addr = 0;
+
+    if (dma_memory_read(s->dma_as, dma_addr, &dma, sizeof(dma))) {
+        stl_be_dma(s->dma_as, dma_addr + offsetof(FWCfgDmaAccess, control),
+                   FW_CFG_DMA_CTL_ERROR);
+        return;
+    }
+
+    dma.address = be64_to_cpu(dma.address);
+    dma.length = be32_to_cpu(dma.length);
+    dma.control = be32_to_cpu(dma.control);
+
+    if (dma.control & FW_CFG_DMA_CTL_SELECT) {
+        fw_cfg_select(s, dma.control >> 16);
+    }
+
+    arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
+    e = (s->cur_entry == FW_CFG_INVALID) ? NULL :
+        &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
+
+    if (dma.control & FW_CFG_DMA_CTL_READ) {
+        read = 1;
+    } else if (dma.control & FW_CFG_DMA_CTL_SKIP) {
+        read = 0;
+    } else {
+        dma.length = 0;
+    }
+
+    dma.control = 0;
+
+    while (dma.length > 0 && !(dma.control & FW_CFG_DMA_CTL_ERROR)) {
+        if (s->cur_entry == FW_CFG_INVALID || !e->data ||
+                                s->cur_offset >= e->len) {
+            len = dma.length;
+
+            /* If the access is not a read access, it will be a skip access,
+             * tested before.
+             */
+            if (read) {
+                if (dma_memory_set(s->dma_as, dma.address, 0, len)) {
+                    dma.control |= FW_CFG_DMA_CTL_ERROR;
+                }
+            }
+
+        } else {
+            if (dma.length <= (e->len - s->cur_offset)) {
+                len = dma.length;
+            } else {
+                len = (e->len - s->cur_offset);
+            }
+
+            if (e->read_callback) {
+                e->read_callback(e->callback_opaque, s->cur_offset);
+            }
+
+            /* If the access is not a read access, it will be a skip access,
+             * tested before.
+             */
+            if (read) {
+                if (dma_memory_write(s->dma_as, dma.address,
+                                    &e->data[s->cur_offset], len)) {
+                    dma.control |= FW_CFG_DMA_CTL_ERROR;
+                }
+            }
+
+            s->cur_offset += len;
+        }
+
+        dma.address += len;
+        dma.length  -= len;
+
+    }
+
+    stl_be_dma(s->dma_as, dma_addr + offsetof(FWCfgDmaAccess, control),
+                dma.control);
+
+    trace_fw_cfg_read(s, 0);
+}
+
+static uint64_t fw_cfg_dma_mem_read(void *opaque, hwaddr addr,
+                                    unsigned size)
+{
+    /* Return a signature value (and handle various read sizes) */
+    return extract64(FW_CFG_DMA_SIGNATURE, (8 - addr - size) * 8, size * 8);
+}
+
+static void fw_cfg_dma_mem_write(void *opaque, hwaddr addr,
+                                 uint64_t value, unsigned size)
+{
+    FWCfgState *s = opaque;
+
+    if (size == 4) {
+        if (addr == 0) {
+            /* FWCfgDmaAccess high address */
+            s->dma_addr = value << 32;
+        } else if (addr == 4) {
+            /* FWCfgDmaAccess low address */
+            s->dma_addr |= value;
+            fw_cfg_dma_transfer(s);
+        }
+    } else if (size == 8 && addr == 0) {
+        s->dma_addr = value;
+        fw_cfg_dma_transfer(s);
+    }
+}
+
+static bool fw_cfg_dma_mem_valid(void *opaque, hwaddr addr,
+                                  unsigned size, bool is_write)
+{
+    return !is_write || ((size == 4 && (addr == 0 || addr == 4)) ||
+                         (size == 8 && addr == 0));
+}
+
 static bool fw_cfg_data_mem_valid(void *opaque, hwaddr addr,
                                   unsigned size, bool is_write)
 {
@@ -361,6 +502,15 @@ static const MemoryRegionOps fw_cfg_comb_mem_ops = {
     .valid.accepts = fw_cfg_comb_valid,
 };
 
+static const MemoryRegionOps fw_cfg_dma_mem_ops = {
+    .read = fw_cfg_dma_mem_read,
+    .write = fw_cfg_dma_mem_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .valid.accepts = fw_cfg_dma_mem_valid,
+    .valid.max_access_size = 8,
+    .impl.max_access_size = 8,
+};
+
 static void fw_cfg_reset(DeviceState *d)
 {
     FWCfgState *s = FW_CFG(d);
@@ -401,6 +551,22 @@ static bool is_version_1(void *opaque, int version_id)
     return version_id == 1;
 }
 
+static bool fw_cfg_dma_enabled(void *opaque)
+{
+    FWCfgState *s = opaque;
+
+    return s->dma_enabled;
+}
+
+static const VMStateDescription vmstate_fw_cfg_dma = {
+    .name = "fw_cfg/dma",
+    .needed = fw_cfg_dma_enabled,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(dma_addr, FWCfgState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static const VMStateDescription vmstate_fw_cfg = {
     .name = "fw_cfg",
     .version_id = 2,
@@ -410,6 +576,10 @@ static const VMStateDescription vmstate_fw_cfg = {
         VMSTATE_UINT16_HACK(cur_offset, FWCfgState, is_version_1),
         VMSTATE_UINT32_V(cur_offset, FWCfgState, 2),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_fw_cfg_dma,
+        NULL,
     }
 };
 
@@ -595,7 +765,6 @@ static void fw_cfg_init1(DeviceState *dev)
     qdev_init_nofail(dev);
 
     fw_cfg_add_bytes(s, FW_CFG_SIGNATURE, (char *)"QEMU", 4);
-    fw_cfg_add_i32(s, FW_CFG_ID, 1);
     fw_cfg_add_bytes(s, FW_CFG_UUID, qemu_uuid, 16);
     fw_cfg_add_i16(s, FW_CFG_NOGRAPHIC, (uint16_t)(display_type == DT_NOGRAPHIC));
     fw_cfg_add_i16(s, FW_CFG_NB_CPUS, (uint16_t)smp_cpus);
@@ -607,25 +776,57 @@ static void fw_cfg_init1(DeviceState *dev)
     qemu_add_machine_init_done_notifier(&s->machine_ready);
 }
 
-FWCfgState *fw_cfg_init_io(uint32_t iobase)
+FWCfgState *fw_cfg_init_io_dma(uint32_t iobase, uint32_t dma_iobase,
+                                AddressSpace *dma_as)
 {
     DeviceState *dev;
+    FWCfgState *s;
+    uint32_t version = FW_CFG_VERSION;
+    bool dma_requested = dma_iobase && dma_as;
 
     dev = qdev_create(NULL, TYPE_FW_CFG_IO);
     qdev_prop_set_uint32(dev, "iobase", iobase);
+    qdev_prop_set_uint32(dev, "dma_iobase", dma_iobase);
+    if (!dma_requested) {
+        qdev_prop_set_bit(dev, "dma_enabled", false);
+    }
+
     fw_cfg_init1(dev);
+    s = FW_CFG(dev);
+
+    if (s->dma_enabled) {
+        /* 64 bits for the address field */
+        s->dma_as = dma_as;
+        s->dma_addr = 0;
+
+        version |= FW_CFG_VERSION_DMA;
+    }
 
-    return FW_CFG(dev);
+    fw_cfg_add_i32(s, FW_CFG_ID, version);
+
+    return s;
+}
+
+FWCfgState *fw_cfg_init_io(uint32_t iobase)
+{
+    return fw_cfg_init_io_dma(iobase, 0, NULL);
 }
 
-FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr, hwaddr data_addr,
-                                 uint32_t data_width)
+FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr,
+                                 hwaddr data_addr, uint32_t data_width,
+                                 hwaddr dma_addr, AddressSpace *dma_as)
 {
     DeviceState *dev;
     SysBusDevice *sbd;
+    FWCfgState *s;
+    uint32_t version = FW_CFG_VERSION;
+    bool dma_requested = dma_addr && dma_as;
 
     dev = qdev_create(NULL, TYPE_FW_CFG_MEM);
     qdev_prop_set_uint32(dev, "data_width", data_width);
+    if (!dma_requested) {
+        qdev_prop_set_bit(dev, "dma_enabled", false);
+    }
 
     fw_cfg_init1(dev);
 
@@ -633,13 +834,25 @@ FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr, hwaddr data_addr,
     sysbus_mmio_map(sbd, 0, ctl_addr);
     sysbus_mmio_map(sbd, 1, data_addr);
 
-    return FW_CFG(dev);
+    s = FW_CFG(dev);
+
+    if (s->dma_enabled) {
+        s->dma_as = dma_as;
+        s->dma_addr = 0;
+        sysbus_mmio_map(sbd, 2, dma_addr);
+        version |= FW_CFG_VERSION_DMA;
+    }
+
+    fw_cfg_add_i32(s, FW_CFG_ID, version);
+
+    return s;
 }
 
 FWCfgState *fw_cfg_init_mem(hwaddr ctl_addr, hwaddr data_addr)
 {
     return fw_cfg_init_mem_wide(ctl_addr, data_addr,
-                                fw_cfg_data_mem_ops.valid.max_access_size);
+                                fw_cfg_data_mem_ops.valid.max_access_size,
+                                0, NULL);
 }
 
 
@@ -666,6 +879,9 @@ static const TypeInfo fw_cfg_info = {
 
 static Property fw_cfg_io_properties[] = {
     DEFINE_PROP_UINT32("iobase", FWCfgIoState, iobase, -1),
+    DEFINE_PROP_UINT32("dma_iobase", FWCfgIoState, dma_iobase, -1),
+    DEFINE_PROP_BOOL("dma_enabled", FWCfgIoState, parent_obj.dma_enabled,
+                     true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -675,8 +891,15 @@ static void fw_cfg_io_realize(DeviceState *dev, Error **errp)
     SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
 
     memory_region_init_io(&s->comb_iomem, OBJECT(s), &fw_cfg_comb_mem_ops,
-                          FW_CFG(s), "fwcfg", FW_CFG_SIZE);
+                          FW_CFG(s), "fwcfg", FW_CFG_CTL_SIZE);
     sysbus_add_io(sbd, s->iobase, &s->comb_iomem);
+
+    if (FW_CFG(s)->dma_enabled) {
+        memory_region_init_io(&FW_CFG(s)->dma_iomem, OBJECT(s),
+                              &fw_cfg_dma_mem_ops, FW_CFG(s), "fwcfg.dma",
+                              sizeof(dma_addr_t));
+        sysbus_add_io(sbd, s->dma_iobase, &FW_CFG(s)->dma_iomem);
+    }
 }
 
 static void fw_cfg_io_class_init(ObjectClass *klass, void *data)
@@ -697,6 +920,8 @@ static const TypeInfo fw_cfg_io_info = {
 
 static Property fw_cfg_mem_properties[] = {
     DEFINE_PROP_UINT32("data_width", FWCfgMemState, data_width, -1),
+    DEFINE_PROP_BOOL("dma_enabled", FWCfgMemState, parent_obj.dma_enabled,
+                     true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -707,7 +932,7 @@ static void fw_cfg_mem_realize(DeviceState *dev, Error **errp)
     const MemoryRegionOps *data_ops = &fw_cfg_data_mem_ops;
 
     memory_region_init_io(&s->ctl_iomem, OBJECT(s), &fw_cfg_ctl_mem_ops,
-                          FW_CFG(s), "fwcfg.ctl", FW_CFG_SIZE);
+                          FW_CFG(s), "fwcfg.ctl", FW_CFG_CTL_SIZE);
     sysbus_init_mmio(sbd, &s->ctl_iomem);
 
     if (s->data_width > data_ops->valid.max_access_size) {
@@ -725,6 +950,13 @@ static void fw_cfg_mem_realize(DeviceState *dev, Error **errp)
     memory_region_init_io(&s->data_iomem, OBJECT(s), data_ops, FW_CFG(s),
                           "fwcfg.data", data_ops->valid.max_access_size);
     sysbus_init_mmio(sbd, &s->data_iomem);
+
+    if (FW_CFG(s)->dma_enabled) {
+        memory_region_init_io(&FW_CFG(s)->dma_iomem, OBJECT(s),
+                              &fw_cfg_dma_mem_ops, FW_CFG(s), "fwcfg.dma",
+                              sizeof(dma_addr_t));
+        sysbus_init_mmio(sbd, &FW_CFG(s)->dma_iomem);
+    }
 }
 
 static void fw_cfg_mem_class_init(ObjectClass *klass, void *data)
diff --git a/hw/nvram/mac_nvram.c b/hw/nvram/mac_nvram.c
index d35f8a312..9f165664c 100644
--- a/hw/nvram/mac_nvram.c
+++ b/hw/nvram/mac_nvram.c
@@ -123,6 +123,7 @@ static void macio_nvram_class_init(ObjectClass *oc, void *data)
     dc->reset = macio_nvram_reset;
     dc->vmsd = &vmstate_macio_nvram;
     dc->props = macio_nvram_properties;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo macio_nvram_type_info = {
diff --git a/hw/openrisc/cputimer.c b/hw/openrisc/cputimer.c
index 9c5494510..560cb914c 100644
--- a/hw/openrisc/cputimer.c
+++ b/hw/openrisc/cputimer.c
@@ -22,7 +22,7 @@
 #include "hw/hw.h"
 #include "qemu/timer.h"
 
-#define TIMER_FREQ    (20 * 1000 * 1000)    /* 20MHz */
+#define TIMER_PERIOD 50 /* 50 ns period for 20 MHz timer */
 
 /* The time when TTCR changes */
 static uint64_t last_clk;
@@ -36,8 +36,7 @@ void cpu_openrisc_count_update(OpenRISCCPU *cpu)
         return;
     }
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    cpu->env.ttcr += (uint32_t)muldiv64(now - last_clk, TIMER_FREQ,
-                                        get_ticks_per_sec());
+    cpu->env.ttcr += (uint32_t)((now - last_clk) / TIMER_PERIOD);
     last_clk = now;
 }
 
@@ -59,7 +58,7 @@ void cpu_openrisc_timer_update(OpenRISCCPU *cpu)
     } else {
         wait = (cpu->env.ttmr & TTMR_TP) - (cpu->env.ttcr & TTMR_TP);
     }
-    next = now + muldiv64(wait, get_ticks_per_sec(), TIMER_FREQ);
+    next = now + (uint64_t)wait * TIMER_PERIOD;
     timer_mod(cpu->env.timer, next);
 }
 
diff --git a/hw/openrisc/openrisc_sim.c b/hw/openrisc/openrisc_sim.c
index 1da0657dd..be6c9b562 100644
--- a/hw/openrisc/openrisc_sim.c
+++ b/hw/openrisc/openrisc_sim.c
@@ -68,7 +68,7 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size,
 
     if (kernel_filename && !qtest_enabled()) {
         kernel_size = load_elf(kernel_filename, NULL, NULL,
-                               &elf_entry, NULL, NULL, 1, ELF_MACHINE, 1);
+                               &elf_entry, NULL, NULL, 1, EM_OPENRISC, 1);
         entry = elf_entry;
         if (kernel_size < 0) {
             kernel_size = load_uimage(kernel_filename,
@@ -114,7 +114,7 @@ static void openrisc_sim_init(MachineState *machine)
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
@@ -132,17 +132,12 @@ static void openrisc_sim_init(MachineState *machine)
     cpu_openrisc_load_kernel(ram_size, kernel_filename, cpu);
 }
 
-static QEMUMachine openrisc_sim_machine = {
-    .name = "or32-sim",
-    .desc = "or32 simulation",
-    .init = openrisc_sim_init,
-    .max_cpus = 1,
-    .is_default = 1,
-};
-
-static void openrisc_sim_machine_init(void)
+static void openrisc_sim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&openrisc_sim_machine);
+    mc->desc = "or32 simulation";
+    mc->init = openrisc_sim_init;
+    mc->max_cpus = 1;
+    mc->is_default = 1;
 }
 
-machine_init(openrisc_sim_machine_init);
+DEFINE_MACHINE("or32-sim", openrisc_sim_machine_init)
diff --git a/hw/pci-host/bonito.c b/hw/pci-host/bonito.c
index 3a731fe18..4139a2c46 100644
--- a/hw/pci-host/bonito.c
+++ b/hw/pci-host/bonito.c
@@ -355,6 +355,10 @@ static uint64_t bonito_ldma_readl(void *opaque, hwaddr addr,
     uint32_t val;
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->bonldma)) {
+        return 0;
+    }
+
     val = ((uint32_t *)(&s->bonldma))[addr/sizeof(uint32_t)];
 
     return val;
@@ -365,6 +369,10 @@ static void bonito_ldma_writel(void *opaque, hwaddr addr,
 {
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->bonldma)) {
+        return;
+    }
+
     ((uint32_t *)(&s->bonldma))[addr/sizeof(uint32_t)] = val & 0xffffffff;
 }
 
@@ -384,6 +392,10 @@ static uint64_t bonito_cop_readl(void *opaque, hwaddr addr,
     uint32_t val;
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->boncop)) {
+        return 0;
+    }
+
     val = ((uint32_t *)(&s->boncop))[addr/sizeof(uint32_t)];
 
     return val;
@@ -394,6 +406,10 @@ static void bonito_cop_writel(void *opaque, hwaddr addr,
 {
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->boncop)) {
+        return;
+    }
+
     ((uint32_t *)(&s->boncop))[addr/sizeof(uint32_t)] = val & 0xffffffff;
 }
 
diff --git a/hw/pci-host/grackle.c b/hw/pci-host/grackle.c
index bfe707a1a..ea31b72e7 100644
--- a/hw/pci-host/grackle.c
+++ b/hw/pci-host/grackle.c
@@ -146,8 +146,10 @@ static const TypeInfo grackle_pci_info = {
 static void pci_grackle_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     k->init = pci_grackle_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo grackle_pci_host_info = {
diff --git a/hw/pci-host/piix.c b/hw/pci-host/piix.c
index ad55f9966..715208b22 100644
--- a/hw/pci-host/piix.c
+++ b/hw/pci-host/piix.c
@@ -34,13 +34,13 @@
 #include "sysemu/sysemu.h"
 #include "hw/i386/ioapic.h"
 #include "qapi/visitor.h"
+#include "qemu/error-report.h"
 
 /*
  * I440FX chipset data sheet.
  * http://download.intel.com/design/chipsets/datashts/29054901.pdf
  */
 
-#define TYPE_I440FX_PCI_HOST_BRIDGE "i440FX-pcihost"
 #define I440FX_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(I440FXState, (obj), TYPE_I440FX_PCI_HOST_BRIDGE)
 
@@ -95,7 +95,6 @@ typedef struct PIIX3State {
 #define PIIX3_PCI_DEVICE(obj) \
     OBJECT_CHECK(PIIX3State, (obj), TYPE_PIIX3_PCI_DEVICE)
 
-#define TYPE_I440FX_PCI_DEVICE "i440FX"
 #define I440FX_PCI_DEVICE(obj) \
     OBJECT_CHECK(PCII440FXState, (obj), TYPE_I440FX_PCI_DEVICE)
 
@@ -117,6 +116,11 @@ struct PCII440FXState {
 #define I440FX_PAM_SIZE 7
 #define I440FX_SMRAM    0x72
 
+/* Older coreboot versions (4.0 and older) read a config register that doesn't
+ * exist in real hardware, to get the RAM size from QEMU.
+ */
+#define I440FX_COREBOOT_RAM_SIZE 0x57
+
 static void piix3_set_irq(void *opaque, int pirq, int level);
 static PCIINTxRoute piix3_route_intx_pin_to_irq(void *opaque, int pci_intx);
 static void piix3_write_config_xen(PCIDevice *dev,
@@ -298,9 +302,14 @@ static void i440fx_pcihost_realize(DeviceState *dev, Error **errp)
 static void i440fx_realize(PCIDevice *dev, Error **errp)
 {
     dev->config[I440FX_SMRAM] = 0x02;
+
+    if (object_property_get_bool(qdev_get_machine(), "iommu", NULL)) {
+        error_report("warning: i440fx doesn't support emulated iommu");
+    }
 }
 
-PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
+PCIBus *i440fx_init(const char *host_type, const char *pci_type,
+                    PCII440FXState **pi440fx_state,
                     int *piix3_devfn,
                     ISABus **isa_bus, qemu_irq *pic,
                     MemoryRegion *address_space_mem,
@@ -320,7 +329,7 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
     unsigned i;
     I440FXState *i440fx;
 
-    dev = qdev_create(NULL, TYPE_I440FX_PCI_HOST_BRIDGE);
+    dev = qdev_create(NULL, host_type);
     s = PCI_HOST_BRIDGE(dev);
     b = pci_bus_new(dev, NULL, pci_address_space,
                     address_space_io, 0, TYPE_PCI_BUS);
@@ -328,7 +337,7 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
     object_property_add_child(qdev_get_machine(), "i440fx", OBJECT(dev), NULL);
     qdev_init_nofail(dev);
 
-    d = pci_create_simple(b, 0, TYPE_I440FX_PCI_DEVICE);
+    d = pci_create_simple(b, 0, pci_type);
     *pi440fx_state = I440FX_PCI_DEVICE(d);
     f = *pi440fx_state;
     f->system_memory = address_space_mem;
@@ -394,7 +403,7 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
     if (ram_size > 255) {
         ram_size = 255;
     }
-    d->config[0x57] = ram_size;
+    d->config[I440FX_COREBOOT_RAM_SIZE] = ram_size;
 
     i440fx_update_memory_mappings(f);
 
@@ -735,6 +744,93 @@ static const TypeInfo i440fx_info = {
     .class_init    = i440fx_class_init,
 };
 
+/* IGD Passthrough Host Bridge. */
+typedef struct {
+    uint8_t offset;
+    uint8_t len;
+} IGDHostInfo;
+
+/* Here we just expose minimal host bridge offset subset. */
+static const IGDHostInfo igd_host_bridge_infos[] = {
+    {0x08, 2},  /* revision id */
+    {0x2c, 2},  /* sybsystem vendor id */
+    {0x2e, 2},  /* sybsystem id */
+    {0x50, 2},  /* SNB: processor graphics control register */
+    {0x52, 2},  /* processor graphics control register */
+    {0xa4, 4},  /* SNB: graphics base of stolen memory */
+    {0xa8, 4},  /* SNB: base of GTT stolen memory */
+};
+
+static int host_pci_config_read(int pos, int len, uint32_t val)
+{
+    char path[PATH_MAX];
+    int config_fd;
+    ssize_t size = sizeof(path);
+    /* Access real host bridge. */
+    int rc = snprintf(path, size, "/sys/bus/pci/devices/%04x:%02x:%02x.%d/%s",
+                      0, 0, 0, 0, "config");
+    int ret = 0;
+
+    if (rc >= size || rc < 0) {
+        return -ENODEV;
+    }
+
+    config_fd = open(path, O_RDWR);
+    if (config_fd < 0) {
+        return -ENODEV;
+    }
+
+    if (lseek(config_fd, pos, SEEK_SET) != pos) {
+        ret = -errno;
+        goto out;
+    }
+    do {
+        rc = read(config_fd, (uint8_t *)&val, len);
+    } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
+    if (rc != len) {
+        ret = -errno;
+    }
+out:
+    close(config_fd);
+    return ret;
+}
+
+static int igd_pt_i440fx_initfn(struct PCIDevice *pci_dev)
+{
+    uint32_t val = 0;
+    int rc, i, num;
+    int pos, len;
+
+    num = ARRAY_SIZE(igd_host_bridge_infos);
+    for (i = 0; i < num; i++) {
+        pos = igd_host_bridge_infos[i].offset;
+        len = igd_host_bridge_infos[i].len;
+        rc = host_pci_config_read(pos, len, val);
+        if (rc) {
+            return -ENODEV;
+        }
+        pci_default_write_config(pci_dev, pos, val, len);
+    }
+
+    return 0;
+}
+
+static void igd_passthrough_i440fx_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = igd_pt_i440fx_initfn;
+    dc->desc = "IGD Passthrough Host bridge";
+}
+
+static const TypeInfo igd_passthrough_i440fx_info = {
+    .name          = TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE,
+    .parent        = TYPE_I440FX_PCI_DEVICE,
+    .instance_size = sizeof(PCII440FXState),
+    .class_init    = igd_passthrough_i440fx_class_init,
+};
+
 static const char *i440fx_pcihost_root_bus_path(PCIHostState *host_bridge,
                                                 PCIBus *rootbus)
 {
@@ -776,6 +872,7 @@ static const TypeInfo i440fx_pcihost_info = {
 static void i440fx_register_types(void)
 {
     type_register_static(&i440fx_info);
+    type_register_static(&igd_passthrough_i440fx_info);
     type_register_static(&piix3_pci_type_info);
     type_register_static(&piix3_info);
     type_register_static(&piix3_xen_info);
diff --git a/hw/pci-host/ppce500.c b/hw/pci-host/ppce500.c
index 613ba73c6..50add34e5 100644
--- a/hw/pci-host/ppce500.c
+++ b/hw/pci-host/ppce500.c
@@ -140,7 +140,7 @@ static uint64_t pci_reg_read4(void *opaque, hwaddr addr,
     case PPCE500_PCI_OW3:
     case PPCE500_PCI_OW4:
         idx = (addr >> 5) & 0x7;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_POTAR:
             value = pci->pob[idx].potar;
             break;
@@ -162,7 +162,7 @@ static uint64_t pci_reg_read4(void *opaque, hwaddr addr,
     case PPCE500_PCI_IW2:
     case PPCE500_PCI_IW1:
         idx = ((addr >> 5) & 0x3) - 1;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_PITAR:
             value = pci->pib[idx].pitar;
             break;
diff --git a/hw/pci-host/prep.c b/hw/pci-host/prep.c
index c63f45d21..da88cb335 100644
--- a/hw/pci-host/prep.c
+++ b/hw/pci-host/prep.c
@@ -302,7 +302,7 @@ static void raven_realize(PCIDevice *d, Error **errp)
     d->config[0x34] = 0x00; // capabilities_pointer
 
     memory_region_init_ram(&s->bios, OBJECT(s), "bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     memory_region_set_readonly(&s->bios, true);
     memory_region_add_subregion(get_system_memory(), (uint32_t)(-BIOS_SIZE),
                                 &s->bios);
@@ -328,9 +328,7 @@ static void raven_realize(PCIDevice *d, Error **errp)
         if (bios_size < 0 || bios_size > BIOS_SIZE) {
             hw_error("qemu: could not load bios image '%s'\n", s->bios_name);
         }
-        if (filename) {
-            g_free(filename);
-        }
+        g_free(filename);
     }
 }
 
diff --git a/hw/pci-host/q35.c b/hw/pci-host/q35.c
index bd7409456..1fb470758 100644
--- a/hw/pci-host/q35.c
+++ b/hw/pci-host/q35.c
@@ -426,31 +426,12 @@ static void mch_reset(DeviceState *qdev)
 static AddressSpace *q35_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
 {
     IntelIOMMUState *s = opaque;
-    VTDAddressSpace **pvtd_as;
-    int bus_num = pci_bus_num(bus);
+    VTDAddressSpace *vtd_as;
 
-    assert(0 <= bus_num && bus_num <= VTD_PCI_BUS_MAX);
     assert(0 <= devfn && devfn <= VTD_PCI_DEVFN_MAX);
 
-    pvtd_as = s->address_spaces[bus_num];
-    if (!pvtd_as) {
-        /* No corresponding free() */
-        pvtd_as = g_malloc0(sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX);
-        s->address_spaces[bus_num] = pvtd_as;
-    }
-    if (!pvtd_as[devfn]) {
-        pvtd_as[devfn] = g_malloc0(sizeof(VTDAddressSpace));
-
-        pvtd_as[devfn]->bus_num = (uint8_t)bus_num;
-        pvtd_as[devfn]->devfn = (uint8_t)devfn;
-        pvtd_as[devfn]->iommu_state = s;
-        pvtd_as[devfn]->context_cache_entry.context_cache_gen = 0;
-        memory_region_init_iommu(&pvtd_as[devfn]->iommu, OBJECT(s),
-                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);
-        address_space_init(&pvtd_as[devfn]->as,
-                           &pvtd_as[devfn]->iommu, "intel_iommu");
-    }
-    return &pvtd_as[devfn]->as;
+    vtd_as = vtd_find_add_as(s, bus, devfn);
+    return &vtd_as->as;
 }
 
 static void mch_init_dmar(MCHPCIState *mch)
@@ -525,7 +506,7 @@ static void mch_realize(PCIDevice *d, Error **errp)
                  PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE, PAM_EXPAN_SIZE);
     }
     /* Intel IOMMU (VT-d) */
-    if (machine_iommu(current_machine)) {
+    if (object_property_get_bool(qdev_get_machine(), "iommu", NULL)) {
         mch_init_dmar(mch);
     }
 }
diff --git a/hw/pci-host/uninorth.c b/hw/pci-host/uninorth.c
index f0144eb7b..215b64ffe 100644
--- a/hw/pci-host/uninorth.c
+++ b/hw/pci-host/uninorth.c
@@ -446,8 +446,10 @@ static const TypeInfo unin_internal_pci_host_info = {
 static void pci_unin_main_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_main_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_main_info = {
@@ -460,8 +462,10 @@ static const TypeInfo pci_unin_main_info = {
 static void pci_u3_agp_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_u3_agp_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_u3_agp_info = {
@@ -474,8 +478,10 @@ static const TypeInfo pci_u3_agp_info = {
 static void pci_unin_agp_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_agp_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_agp_info = {
@@ -488,8 +494,10 @@ static const TypeInfo pci_unin_agp_info = {
 static void pci_unin_internal_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_internal_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_internal_info = {
diff --git a/hw/pci/msi.c b/hw/pci/msi.c
index f9c048442..c1dd5318c 100644
--- a/hw/pci/msi.c
+++ b/hw/pci/msi.c
@@ -294,7 +294,7 @@ void msi_send_message(PCIDevice *dev, MSIMessage msg)
 {
     MemTxAttrs attrs = {};
 
-    attrs.stream_id = (pci_bus_num(dev->bus) << 8) | dev->devfn;
+    attrs.requester_id = pci_requester_id(dev);
     address_space_stl_le(&dev->bus_master_as, msg.address, msg.data,
                          attrs, NULL);
 }
diff --git a/hw/pci/msix.c b/hw/pci/msix.c
index 7716bf364..64c93d83d 100644
--- a/hw/pci/msix.c
+++ b/hw/pci/msix.c
@@ -200,8 +200,14 @@ static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
     return pci_get_long(dev->msix_pba + addr);
 }
 
+static void msix_pba_mmio_write(void *opaque, hwaddr addr,
+                                uint64_t val, unsigned size)
+{
+}
+
 static const MemoryRegionOps msix_pba_mmio_ops = {
     .read = msix_pba_mmio_read,
+    .write = msix_pba_mmio_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
         .min_access_size = 4,
@@ -314,9 +320,7 @@ int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
         bar_size = bar_pba_offset + bar_pba_size;
     }
 
-    if (bar_size & (bar_size - 1)) {
-        bar_size = 1 << qemu_fls(bar_size);
-    }
+    bar_size = pow2ceil(bar_size);
 
     name = g_strdup_printf("%s-msix", dev->name);
     memory_region_init(&dev->msix_exclusive_bar, OBJECT(dev), name, bar_size);
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index a017614d4..168b9cc56 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -38,6 +38,7 @@
 #include "hw/pci/msix.h"
 #include "exec/address-spaces.h"
 #include "hw/hotplug.h"
+#include "hw/boards.h"
 
 //#define DEBUG_PCI
 #ifdef DEBUG_PCI
@@ -846,6 +847,9 @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, PCIBus *bus,
     PCIConfigWriteFunc *config_write = pc->config_write;
     Error *local_err = NULL;
     AddressSpace *dma_as;
+    DeviceState *dev = DEVICE(pci_dev);
+
+    pci_dev->bus = bus;
 
     if (devfn < 0) {
         for(devfn = bus->devfn_min ; devfn < ARRAY_SIZE(bus->devices);
@@ -863,9 +867,17 @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, PCIBus *bus,
                    PCI_SLOT(devfn), PCI_FUNC(devfn), name,
                    bus->devices[devfn]->name);
         return NULL;
+    } else if (dev->hotplugged &&
+               pci_get_function_0(pci_dev)) {
+        error_setg(errp, "PCI: slot %d function 0 already ocuppied by %s,"
+                   " new func %s cannot be exposed to guest.",
+                   PCI_SLOT(devfn),
+                   bus->devices[PCI_DEVFN(PCI_SLOT(devfn), 0)]->name,
+                   name);
+
+       return NULL;
     }
 
-    pci_dev->bus = bus;
     pci_dev->devfn = devfn;
     dma_as = pci_device_iommu_address_space(pci_dev);
 
@@ -1065,6 +1077,10 @@ static pcibus_t pci_bar_address(PCIDevice *d,
     pcibus_t new_addr, last_addr;
     int bar = pci_bar(d, reg);
     uint16_t cmd = pci_get_word(d->config + PCI_COMMAND);
+    Object *machine = qdev_get_machine();
+    ObjectClass *oc = object_get_class(machine);
+    MachineClass *mc = MACHINE_CLASS(oc);
+    bool allow_0_address = mc->pci_allow_0_address;
 
     if (type & PCI_BASE_ADDRESS_SPACE_IO) {
         if (!(cmd & PCI_COMMAND_IO)) {
@@ -1075,7 +1091,8 @@ static pcibus_t pci_bar_address(PCIDevice *d,
         /* Check if 32 bit BAR wraps around explicitly.
          * TODO: make priorities correct and remove this work around.
          */
-        if (last_addr <= new_addr || new_addr == 0 || last_addr >= UINT32_MAX) {
+        if (last_addr <= new_addr || last_addr >= UINT32_MAX ||
+            (!allow_0_address && new_addr == 0)) {
             return PCI_BAR_UNMAPPED;
         }
         return new_addr;
@@ -1099,8 +1116,8 @@ static pcibus_t pci_bar_address(PCIDevice *d,
     /* XXX: as we cannot support really dynamic
        mappings, we handle specific values as invalid
        mappings. */
-    if (last_addr <= new_addr || new_addr == 0 ||
-        last_addr == PCI_BAR_UNMAPPED) {
+    if (last_addr <= new_addr || last_addr == PCI_BAR_UNMAPPED ||
+        (!allow_0_address && new_addr == 0)) {
         return PCI_BAR_UNMAPPED;
     }
 
@@ -1148,16 +1165,16 @@ static void pci_update_mappings(PCIDevice *d)
         /* now do the real mapping */
         if (r->addr != PCI_BAR_UNMAPPED) {
             trace_pci_update_mappings_del(d, pci_bus_num(d->bus),
-                                          PCI_FUNC(d->devfn),
                                           PCI_SLOT(d->devfn),
+                                          PCI_FUNC(d->devfn),
                                           i, r->addr, r->size);
             memory_region_del_subregion(r->address_space, r->memory);
         }
         r->addr = new_addr;
         if (r->addr != PCI_BAR_UNMAPPED) {
             trace_pci_update_mappings_add(d, pci_bus_num(d->bus),
-                                          PCI_FUNC(d->devfn),
                                           PCI_SLOT(d->devfn),
+                                          PCI_FUNC(d->devfn),
                                           i, r->addr, r->size);
             memory_region_add_subregion_overlap(r->address_space,
                                                 r->addr, r->memory, 1);
@@ -2065,9 +2082,7 @@ static void pci_add_option_rom(PCIDevice *pdev, bool is_default_rom,
         g_free(path);
         return;
     }
-    if (size & (size - 1)) {
-        size = 1 << qemu_fls(size);
-    }
+    size = pow2ceil(size);
 
     vmsd = qdev_get_vmsd(DEVICE(pdev));
 
@@ -2077,7 +2092,7 @@ static void pci_add_option_rom(PCIDevice *pdev, bool is_default_rom,
         snprintf(name, sizeof(name), "%s.rom", object_get_typename(OBJECT(pdev)));
     }
     pdev->has_rom = true;
-    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size, &error_abort);
+    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size, &error_fatal);
     vmstate_register_ram(&pdev->rom, &pdev->qdev);
     ptr = memory_region_get_ram_ptr(&pdev->rom);
     load_image(path, ptr);
@@ -2379,17 +2394,14 @@ static void pci_device_class_init(ObjectClass *klass, void *data)
 AddressSpace *pci_device_iommu_address_space(PCIDevice *dev)
 {
     PCIBus *bus = PCI_BUS(dev->bus);
+    PCIBus *iommu_bus = bus;
 
-    if (bus->iommu_fn) {
-        return bus->iommu_fn(bus, bus->iommu_opaque, dev->devfn);
+    while(iommu_bus && !iommu_bus->iommu_fn && iommu_bus->parent_dev) {
+        iommu_bus = PCI_BUS(iommu_bus->parent_dev->bus);
     }
-
-    if (bus->parent_dev) {
-        /** We are ignoring the bus master DMA bit of the bridge
-         *  as it would complicate things such as VFIO for no good reason */
-        return pci_device_iommu_address_space(bus->parent_dev);
+    if (iommu_bus && iommu_bus->iommu_fn) {
+        return iommu_bus->iommu_fn(bus, iommu_bus->iommu_opaque, dev->devfn);
     }
-
     return &address_space_memory;
 }
 
@@ -2453,6 +2465,33 @@ void pci_bus_get_w64_range(PCIBus *bus, Range *range)
     pci_for_each_device_under_bus(bus, pci_dev_get_w64, range);
 }
 
+static bool pcie_has_upstream_port(PCIDevice *dev)
+{
+    PCIDevice *parent_dev = pci_bridge_get_device(dev->bus);
+
+    /* Device associated with an upstream port.
+     * As there are several types of these, it's easier to check the
+     * parent device: upstream ports are always connected to
+     * root or downstream ports.
+     */
+    return parent_dev &&
+        pci_is_express(parent_dev) &&
+        parent_dev->exp.exp_cap &&
+        (pcie_cap_get_type(parent_dev) == PCI_EXP_TYPE_ROOT_PORT ||
+         pcie_cap_get_type(parent_dev) == PCI_EXP_TYPE_DOWNSTREAM);
+}
+
+PCIDevice *pci_get_function_0(PCIDevice *pci_dev)
+{
+    if(pcie_has_upstream_port(pci_dev)) {
+        /* With an upstream PCIe port, we only support 1 device at slot 0 */
+        return pci_dev->bus->devices[0];
+    } else {
+        /* Other bus types might support multiple devices at slots 0-31 */
+        return pci_dev->bus->devices[PCI_DEVFN(PCI_SLOT(pci_dev->devfn), 0)];
+    }
+}
+
 static const TypeInfo pci_device_type_info = {
     .name = TYPE_PCI_DEVICE,
     .parent = TYPE_DEVICE,
diff --git a/hw/pci/pci_host.c b/hw/pci/pci_host.c
index 3e26f9256..49f59a5db 100644
--- a/hw/pci/pci_host.c
+++ b/hw/pci/pci_host.c
@@ -20,6 +20,7 @@
 
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
+#include "hw/pci/pci_bus.h"
 #include "trace.h"
 
 /* debug PCI */
@@ -52,6 +53,13 @@ void pci_host_config_write_common(PCIDevice *pci_dev, uint32_t addr,
                                   uint32_t limit, uint32_t val, uint32_t len)
 {
     assert(len <= 4);
+    /* non-zero functions are only exposed when function 0 is present,
+     * allowing direct removal of unexposed functions.
+     */
+    if (pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) {
+        return;
+    }
+
     trace_pci_cfg_write(pci_dev->name, PCI_SLOT(pci_dev->devfn),
                         PCI_FUNC(pci_dev->devfn), addr, val);
     pci_dev->config_write(pci_dev, addr, val, MIN(len, limit - addr));
@@ -63,6 +71,13 @@ uint32_t pci_host_config_read_common(PCIDevice *pci_dev, uint32_t addr,
     uint32_t ret;
 
     assert(len <= 4);
+    /* non-zero functions are only exposed when function 0 is present,
+     * allowing direct removal of unexposed functions.
+     */
+    if (pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) {
+        return ~0x0;
+    }
+
     ret = pci_dev->config_read(pci_dev, addr, MIN(len, limit - addr));
     trace_pci_cfg_read(pci_dev->name, PCI_SLOT(pci_dev->devfn),
                        PCI_FUNC(pci_dev->devfn), addr, ret);
diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
index 6e28985bd..0eab29d53 100644
--- a/hw/pci/pcie.c
+++ b/hw/pci/pcie.c
@@ -249,25 +249,43 @@ void pcie_cap_slot_hotplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
         return;
     }
 
-    /* TODO: multifunction hot-plug.
-     * Right now, only a device of function = 0 is allowed to be
-     * hot plugged/unplugged.
+    /* To enable multifunction hot-plug, we just ensure the function
+     * 0 added last. When function 0 is added, we set the sltsta and
+     * inform OS via event notification.
      */
-    assert(PCI_FUNC(pci_dev->devfn) == 0);
+    if (pci_get_function_0(pci_dev)) {
+        pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
+                                   PCI_EXP_SLTSTA_PDS);
+        pcie_cap_slot_event(PCI_DEVICE(hotplug_dev),
+                            PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP);
+    }
+}
 
-    pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
-                               PCI_EXP_SLTSTA_PDS);
-    pcie_cap_slot_event(PCI_DEVICE(hotplug_dev),
-                        PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP);
+static void pcie_unplug_device(PCIBus *bus, PCIDevice *dev, void *opaque)
+{
+    object_unparent(OBJECT(dev));
 }
 
 void pcie_cap_slot_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
                                          DeviceState *dev, Error **errp)
 {
     uint8_t *exp_cap;
+    PCIDevice *pci_dev = PCI_DEVICE(dev);
+    PCIBus *bus = pci_dev->bus;
 
     pcie_cap_slot_hotplug_common(PCI_DEVICE(hotplug_dev), dev, &exp_cap, errp);
 
+    /* In case user cancel the operation of multi-function hot-add,
+     * remove the function that is unexposed to guest individually,
+     * without interaction with guest.
+     */
+    if (pci_dev->devfn &&
+        !bus->devices[0]) {
+        pcie_unplug_device(bus, pci_dev, NULL);
+
+        return;
+    }
+
     pcie_cap_slot_push_attention_button(PCI_DEVICE(hotplug_dev));
 }
 
@@ -378,11 +396,6 @@ void pcie_cap_slot_reset(PCIDevice *dev)
     hotplug_event_update_event_status(dev);
 }
 
-static void pcie_unplug_device(PCIBus *bus, PCIDevice *dev, void *opaque)
-{
-    object_unparent(OBJECT(dev));
-}
-
 void pcie_cap_slot_write_config(PCIDevice *dev,
                                 uint32_t addr, uint32_t val, int len)
 {
@@ -413,13 +426,13 @@ void pcie_cap_slot_write_config(PCIDevice *dev,
      */
     if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
         ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
-            PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
-            pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
-                                pcie_unplug_device, NULL);
+        PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
+        pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
+                            pcie_unplug_device, NULL);
 
-            pci_word_test_and_clear_mask(exp_cap + PCI_EXP_SLTSTA,
-                                         PCI_EXP_SLTSTA_PDS);
-            pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
+        pci_word_test_and_clear_mask(exp_cap + PCI_EXP_SLTSTA,
+                                     PCI_EXP_SLTSTA_PDS);
+        pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
                                        PCI_EXP_SLTSTA_PDC);
     }
 
diff --git a/hw/pci/pcie_aer.c b/hw/pci/pcie_aer.c
index f1847ac21..98d2c183b 100644
--- a/hw/pci/pcie_aer.c
+++ b/hw/pci/pcie_aer.c
@@ -827,10 +827,6 @@ typedef struct PCIEAERErrorName {
  */
 static const struct PCIEAERErrorName pcie_aer_error_list[] = {
     {
-        .name = "TRAIN",
-        .val = PCI_ERR_UNC_TRAIN,
-        .correctable = false,
-    }, {
         .name = "DLP",
         .val = PCI_ERR_UNC_DLP,
         .correctable = false,
@@ -983,7 +979,7 @@ static int do_pcie_aer_inject_error(Monitor *mon,
         }
     }
     err.status = error_status;
-    err.source_id = (pci_bus_num(dev->bus) << 8) | dev->devfn;
+    err.source_id = pci_requester_id(dev);
 
     err.flags = 0;
     if (correctable) {
diff --git a/hw/pci/shpc.c b/hw/pci/shpc.c
index bfb4d31b6..d34fdf3ec 100644
--- a/hw/pci/shpc.c
+++ b/hw/pci/shpc.c
@@ -1,5 +1,4 @@
 #include "qemu-common.h"
-#include <strings.h>
 #include <stdint.h>
 #include "qemu/range.h"
 #include "qemu/error-report.h"
diff --git a/hw/ppc/Makefile.objs b/hw/ppc/Makefile.objs
index c8ab06e7f..c1ffc7771 100644
--- a/hw/ppc/Makefile.objs
+++ b/hw/ppc/Makefile.objs
@@ -3,7 +3,7 @@ obj-y += ppc.o ppc_booke.o
 # IBM pSeries (sPAPR)
 obj-$(CONFIG_PSERIES) += spapr.o spapr_vio.o spapr_events.o
 obj-$(CONFIG_PSERIES) += spapr_hcall.o spapr_iommu.o spapr_rtas.o
-obj-$(CONFIG_PSERIES) += spapr_pci.o spapr_rtc.o spapr_drc.o
+obj-$(CONFIG_PSERIES) += spapr_pci.o spapr_rtc.o spapr_drc.o spapr_rng.o
 ifeq ($(CONFIG_PCI)$(CONFIG_PSERIES)$(CONFIG_LINUX), yyy)
 obj-y += spapr_pci_vfio.o
 endif
diff --git a/hw/ppc/e500.c b/hw/ppc/e500.c
index d300846c3..b3418dbae 100644
--- a/hw/ppc/e500.c
+++ b/hw/ppc/e500.c
@@ -1017,7 +1017,7 @@ void ppce500_init(MachineState *machine, PPCE500Params *params)
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
 
     bios_size = load_elf(filename, NULL, NULL, &bios_entry, &loadaddr, NULL,
-                         1, ELF_MACHINE, 0);
+                         1, PPC_ELF_MACHINE, 0);
     if (bios_size < 0) {
         /*
          * Hrm. No ELF image? Try a uImage, maybe someone is giving us an
@@ -1048,10 +1048,6 @@ void ppce500_init(MachineState *machine, PPCE500Params *params)
     boot_info->entry = bios_entry;
     boot_info->dt_base = dt_base;
     boot_info->dt_size = dt_size;
-
-    if (kvm_enabled()) {
-        kvmppc_init();
-    }
 }
 
 static int e500_ccsr_initfn(SysBusDevice *dev)
diff --git a/hw/ppc/e500plat.c b/hw/ppc/e500plat.c
index 14b14eaa7..384b6e84d 100644
--- a/hw/ppc/e500plat.c
+++ b/hw/ppc/e500plat.c
@@ -57,17 +57,12 @@ static void e500plat_init(MachineState *machine)
     ppce500_init(machine, &params);
 }
 
-static QEMUMachine e500plat_machine = {
-    .name = "ppce500",
-    .desc = "generic paravirt e500 platform",
-    .init = e500plat_init,
-    .max_cpus = 32,
-    .has_dynamic_sysbus = true,
-};
-
-static void e500plat_machine_init(void)
+static void e500plat_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&e500plat_machine);
+    mc->desc = "generic paravirt e500 platform";
+    mc->init = e500plat_init;
+    mc->max_cpus = 32;
+    mc->has_dynamic_sysbus = true;
 }
 
-machine_init(e500plat_machine_init);
+DEFINE_MACHINE("ppce500", e500plat_machine_init)
diff --git a/hw/ppc/mac.h b/hw/ppc/mac.h
index 8bdba30c1..e375ed2b2 100644
--- a/hw/ppc/mac.h
+++ b/hw/ppc/mac.h
@@ -103,6 +103,9 @@ typedef struct CUDAState {
     uint8_t last_b;
     uint8_t last_acr;
 
+    /* MacOS 9 is racy and requires a delay upon setting the SR_INT bit */
+    QEMUTimer *sr_delay_timer;
+
     int data_in_size;
     int data_in_index;
     int data_out_index;
diff --git a/hw/ppc/mac_newworld.c b/hw/ppc/mac_newworld.c
index 77d5c819e..1b9a573ca 100644
--- a/hw/ppc/mac_newworld.c
+++ b/hw/ppc/mac_newworld.c
@@ -207,7 +207,7 @@ static void ppc_core99_init(MachineState *machine)
 
     /* allocate and load BIOS */
     memory_region_init_ram(bios, NULL, "ppc_core99.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -219,7 +219,7 @@ static void ppc_core99_init(MachineState *machine)
     /* Load OpenBIOS (ELF) */
     if (filename) {
         bios_size = load_elf(filename, NULL, NULL, NULL,
-                             NULL, NULL, 1, ELF_MACHINE, 0);
+                             NULL, NULL, 1, PPC_ELF_MACHINE, 0);
 
         g_free(filename);
     } else {
@@ -242,7 +242,7 @@ static void ppc_core99_init(MachineState *machine)
         kernel_base = KERNEL_LOAD_ADDR;
 
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, kernel_base,
                                     ram_size - kernel_base, bswap_needed,
@@ -371,12 +371,13 @@ static void ppc_core99_init(MachineState *machine)
         /* 970 gets a U3 bus */
         pci_bus = pci_pmac_u3_init(pic, get_system_memory(), get_system_io());
         machine_arch = ARCH_MAC99_U3;
-        machine->usb |= defaults_enabled() && !machine->usb_disabled;
     } else {
         pci_bus = pci_pmac_init(pic, get_system_memory(), get_system_io());
         machine_arch = ARCH_MAC99;
     }
 
+    machine->usb |= defaults_enabled() && !machine->usb_disabled;
+
     /* Timebase Frequency */
     if (kvm_enabled()) {
         tbfreq = kvmppc_get_tbfreq();
@@ -508,7 +509,6 @@ static void core99_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = "mac99";
     mc->desc = "Mac99 based PowerMAC";
     mc->init = ppc_core99_init;
     mc->max_cpus = MAX_CPUS;
@@ -517,7 +517,7 @@ static void core99_machine_class_init(ObjectClass *oc, void *data)
 }
 
 static const TypeInfo core99_machine_info = {
-    .name          = "mac99-machine",
+    .name          = MACHINE_TYPE_NAME("mac99"),
     .parent        = TYPE_MACHINE,
     .class_init    = core99_machine_class_init,
 };
diff --git a/hw/ppc/mac_oldworld.c b/hw/ppc/mac_oldworld.c
index 06fdbaf58..21eaf0e66 100644
--- a/hw/ppc/mac_oldworld.c
+++ b/hw/ppc/mac_oldworld.c
@@ -135,7 +135,7 @@ static void ppc_heathrow_init(MachineState *machine)
 
     /* allocate and load BIOS */
     memory_region_init_ram(bios, NULL, "ppc_heathrow.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -147,7 +147,7 @@ static void ppc_heathrow_init(MachineState *machine)
     /* Load OpenBIOS (ELF) */
     if (filename) {
         bios_size = load_elf(filename, 0, NULL, NULL, NULL, NULL,
-                             1, ELF_MACHINE, 0);
+                             1, PPC_ELF_MACHINE, 0);
         g_free(filename);
     } else {
         bios_size = -1;
@@ -168,7 +168,7 @@ static void ppc_heathrow_init(MachineState *machine)
 #endif
         kernel_base = KERNEL_LOAD_ADDR;
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, kernel_base,
                                     ram_size - kernel_base, bswap_needed,
@@ -357,21 +357,17 @@ static int heathrow_kvm_type(const char *arg)
     return 2;
 }
 
-static QEMUMachine heathrow_machine = {
-    .name = "g3beige",
-    .desc = "Heathrow based PowerMAC",
-    .init = ppc_heathrow_init,
-    .max_cpus = MAX_CPUS,
+static void heathrow_machine_init(MachineClass *mc)
+{
+    mc->desc = "Heathrow based PowerMAC";
+    mc->init = ppc_heathrow_init;
+    mc->max_cpus = MAX_CPUS;
 #ifndef TARGET_PPC64
-    .is_default = 1,
+    mc->is_default = 1;
 #endif
-    .default_boot_order = "cd", /* TOFIX "cad" when Mac floppy is implemented */
-    .kvm_type = heathrow_kvm_type,
-};
-
-static void heathrow_machine_init(void)
-{
-    qemu_register_machine(&heathrow_machine);
+    /* TOFIX "cad" when Mac floppy is implemented */
+    mc->default_boot_order = "cd";
+    mc->kvm_type = heathrow_kvm_type;
 }
 
-machine_init(heathrow_machine_init);
+DEFINE_MACHINE("g3beige", heathrow_machine_init)
diff --git a/hw/ppc/mpc8544ds.c b/hw/ppc/mpc8544ds.c
index 3a3b141e4..0afbd34d6 100644
--- a/hw/ppc/mpc8544ds.c
+++ b/hw/ppc/mpc8544ds.c
@@ -50,16 +50,11 @@ static void mpc8544ds_init(MachineState *machine)
 }
 
 
-static QEMUMachine ppce500_machine = {
-    .name = "mpc8544ds",
-    .desc = "mpc8544ds",
-    .init = mpc8544ds_init,
-    .max_cpus = 15,
-};
-
-static void ppce500_machine_init(void)
+static void ppce500_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&ppce500_machine);
+    mc->desc = "mpc8544ds";
+    mc->init = mpc8544ds_init;
+    mc->max_cpus = 15;
 }
 
-machine_init(ppce500_machine_init);
+DEFINE_MACHINE("mpc8544ds", ppce500_machine_init)
diff --git a/hw/ppc/ppc.c b/hw/ppc/ppc.c
index b77e30357..2c604ef49 100644
--- a/hw/ppc/ppc.c
+++ b/hw/ppc/ppc.c
@@ -834,7 +834,7 @@ static void cpu_ppc_set_tb_clk (void *opaque, uint32_t freq)
 static void timebase_pre_save(void *opaque)
 {
     PPCTimebase *tb = opaque;
-    uint64_t ticks = cpu_get_real_ticks();
+    uint64_t ticks = cpu_get_host_ticks();
     PowerPCCPU *first_ppc_cpu = POWERPC_CPU(first_cpu);
 
     if (!first_ppc_cpu->env.tb_env) {
@@ -878,7 +878,7 @@ static int timebase_post_load(void *opaque, int version_id)
                                      NANOSECONDS_PER_SECOND);
     guest_tb = tb_remote->guest_timebase + MIN(0, migration_duration_tb);
 
-    tb_off_adj = guest_tb - cpu_get_real_ticks();
+    tb_off_adj = guest_tb - cpu_get_host_ticks();
 
     tb_off = first_ppc_cpu->env.tb_env->tb_offset;
     trace_ppc_tb_adjust(tb_off, tb_off_adj, tb_off_adj - tb_off,
diff --git a/hw/ppc/ppc405_boards.c b/hw/ppc/ppc405_boards.c
index ec6c4cbaf..31bc18642 100644
--- a/hw/ppc/ppc405_boards.c
+++ b/hw/ppc/ppc405_boards.c
@@ -215,7 +215,8 @@ static void ref405ep_init(MachineState *machine)
                         33333333, &pic, kernel_filename == NULL ? 0 : 1);
     /* allocate SRAM */
     sram_size = 512 * 1024;
-    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size, &error_abort);
+    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size,
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0xFFF00000, sram);
     /* allocate and load BIOS */
@@ -250,7 +251,7 @@ static void ref405ep_init(MachineState *machine)
 #endif
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "ef405ep.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
 
         if (bios_name == NULL)
@@ -368,10 +369,18 @@ static void ref405ep_init(MachineState *machine)
 #endif
 }
 
-static QEMUMachine ref405ep_machine = {
-    .name = "ref405ep",
-    .desc = "ref405ep",
-    .init = ref405ep_init,
+static void ref405ep_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ref405ep";
+    mc->init = ref405ep_init;
+}
+
+static const TypeInfo ref405ep_type = {
+    .name = MACHINE_TYPE_NAME("ref405ep"),
+    .parent = TYPE_MACHINE,
+    .class_init = ref405ep_class_init,
 };
 
 /*****************************************************************************/
@@ -399,7 +408,7 @@ struct taihu_cpld_t {
     uint8_t reg1;
 };
 
-static uint32_t taihu_cpld_readb (void *opaque, hwaddr addr)
+static uint64_t taihu_cpld_read(void *opaque, hwaddr addr, unsigned size)
 {
     taihu_cpld_t *cpld;
     uint32_t ret;
@@ -420,8 +429,8 @@ static uint32_t taihu_cpld_readb (void *opaque, hwaddr addr)
     return ret;
 }
 
-static void taihu_cpld_writeb (void *opaque,
-                               hwaddr addr, uint32_t value)
+static void taihu_cpld_write(void *opaque, hwaddr addr,
+                             uint64_t value, unsigned size)
 {
     taihu_cpld_t *cpld;
 
@@ -438,48 +447,12 @@ static void taihu_cpld_writeb (void *opaque,
     }
 }
 
-static uint32_t taihu_cpld_readw (void *opaque, hwaddr addr)
-{
-    uint32_t ret;
-
-    ret = taihu_cpld_readb(opaque, addr) << 8;
-    ret |= taihu_cpld_readb(opaque, addr + 1);
-
-    return ret;
-}
-
-static void taihu_cpld_writew (void *opaque,
-                               hwaddr addr, uint32_t value)
-{
-    taihu_cpld_writeb(opaque, addr, (value >> 8) & 0xFF);
-    taihu_cpld_writeb(opaque, addr + 1, value & 0xFF);
-}
-
-static uint32_t taihu_cpld_readl (void *opaque, hwaddr addr)
-{
-    uint32_t ret;
-
-    ret = taihu_cpld_readb(opaque, addr) << 24;
-    ret |= taihu_cpld_readb(opaque, addr + 1) << 16;
-    ret |= taihu_cpld_readb(opaque, addr + 2) << 8;
-    ret |= taihu_cpld_readb(opaque, addr + 3);
-
-    return ret;
-}
-
-static void taihu_cpld_writel (void *opaque,
-                               hwaddr addr, uint32_t value)
-{
-    taihu_cpld_writel(opaque, addr, (value >> 24) & 0xFF);
-    taihu_cpld_writel(opaque, addr + 1, (value >> 16) & 0xFF);
-    taihu_cpld_writel(opaque, addr + 2, (value >> 8) & 0xFF);
-    taihu_cpld_writeb(opaque, addr + 3, value & 0xFF);
-}
-
 static const MemoryRegionOps taihu_cpld_ops = {
-    .old_mmio = {
-        .read = { taihu_cpld_readb, taihu_cpld_readw, taihu_cpld_readl, },
-        .write = { taihu_cpld_writeb, taihu_cpld_writew, taihu_cpld_writel, },
+    .read = taihu_cpld_read,
+    .write = taihu_cpld_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
     },
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
@@ -579,7 +552,7 @@ static void taihu_405ep_init(MachineState *machine)
             bios_name = BIOS_FILENAME;
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "taihu_405ep.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
         filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (filename) {
@@ -664,16 +637,24 @@ static void taihu_405ep_init(MachineState *machine)
 #endif
 }
 
-static QEMUMachine taihu_machine = {
-    .name = "taihu",
-    .desc = "taihu",
-    .init = taihu_405ep_init,
+static void taihu_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "taihu";
+    mc->init = taihu_405ep_init;
+}
+
+static const TypeInfo taihu_type = {
+    .name = MACHINE_TYPE_NAME("taihu"),
+    .parent = TYPE_MACHINE,
+    .class_init = taihu_class_init,
 };
 
 static void ppc405_machine_init(void)
 {
-    qemu_register_machine(&ref405ep_machine);
-    qemu_register_machine(&taihu_machine);
+    type_register_static(&ref405ep_type);
+    type_register_static(&taihu_type);
 }
 
-machine_init(ppc405_machine_init);
+machine_init(ppc405_machine_init)
diff --git a/hw/ppc/ppc405_uc.c b/hw/ppc/ppc405_uc.c
index c77434ae0..10f5dda1b 100644
--- a/hw/ppc/ppc405_uc.c
+++ b/hw/ppc/ppc405_uc.c
@@ -975,7 +975,7 @@ static void ppc405_ocm_init(CPUPPCState *env)
     ocm = g_malloc0(sizeof(ppc405_ocm_t));
     /* XXX: Size is 4096 or 0x04000000 */
     memory_region_init_ram(&ocm->isarc_ram, NULL, "ppc405.ocm", 4096,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&ocm->isarc_ram);
     memory_region_init_alias(&ocm->dsarc_ram, NULL, "ppc405.dsarc", &ocm->isarc_ram,
                              0, 4096);
diff --git a/hw/ppc/ppc440_bamboo.c b/hw/ppc/ppc440_bamboo.c
index 032fa803d..b66c11338 100644
--- a/hw/ppc/ppc440_bamboo.c
+++ b/hw/ppc/ppc440_bamboo.c
@@ -256,7 +256,7 @@ static void bamboo_init(MachineState *machine)
                               NULL, NULL);
         if (success < 0) {
             success = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               &elf_lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               &elf_lowaddr, NULL, 1, PPC_ELF_MACHINE, 0);
             entry = elf_entry;
             loadaddr = elf_lowaddr;
         }
@@ -288,20 +288,12 @@ static void bamboo_init(MachineState *machine)
             exit(1);
         }
     }
-
-    if (kvm_enabled())
-        kvmppc_init();
 }
 
-static QEMUMachine bamboo_machine = {
-    .name = "bamboo",
-    .desc = "bamboo",
-    .init = bamboo_init,
-};
-
-static void bamboo_machine_init(void)
+static void bamboo_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&bamboo_machine);
+    mc->desc = "bamboo";
+    mc->init = bamboo_init;
 }
 
-machine_init(bamboo_machine_init);
+DEFINE_MACHINE("bamboo", bamboo_machine_init)
diff --git a/hw/ppc/prep.c b/hw/ppc/prep.c
index 45b5f62d6..5ad28f75c 100644
--- a/hw/ppc/prep.c
+++ b/hw/ppc/prep.c
@@ -42,11 +42,9 @@
 #include "sysemu/arch_init.h"
 #include "sysemu/qtest.h"
 #include "exec/address-spaces.h"
+#include "trace.h"
 #include "elf.h"
 
-//#define HARD_DEBUG_PPC_IO
-//#define DEBUG_PPC_IO
-
 /* SMP is not enabled, for now */
 #define MAX_CPUS 1
 
@@ -57,26 +55,6 @@
 #define KERNEL_LOAD_ADDR 0x01000000
 #define INITRD_LOAD_ADDR 0x01800000
 
-#if defined (HARD_DEBUG_PPC_IO) && !defined (DEBUG_PPC_IO)
-#define DEBUG_PPC_IO
-#endif
-
-#if defined (HARD_DEBUG_PPC_IO)
-#define PPC_IO_DPRINTF(fmt, ...)                         \
-do {                                                     \
-    if (qemu_loglevel_mask(CPU_LOG_IOPORT)) {            \
-        qemu_log("%s: " fmt, __func__ , ## __VA_ARGS__); \
-    } else {                                             \
-        printf("%s : " fmt, __func__ , ## __VA_ARGS__);  \
-    }                                                    \
-} while (0)
-#elif defined (DEBUG_PPC_IO)
-#define PPC_IO_DPRINTF(fmt, ...) \
-qemu_log_mask(CPU_LOG_IOPORT, fmt, ## __VA_ARGS__)
-#else
-#define PPC_IO_DPRINTF(fmt, ...) do { } while (0)
-#endif
-
 /* Constants for devices init */
 static const int ide_iobase[2] = { 0x1f0, 0x170 };
 static const int ide_iobase2[2] = { 0x3f6, 0x376 };
@@ -199,8 +177,7 @@ static void PREP_io_800_writeb (void *opaque, uint32_t addr, uint32_t val)
 {
     sysctrl_t *sysctrl = opaque;
 
-    PPC_IO_DPRINTF("0x%08" PRIx32 " => 0x%02" PRIx32 "\n",
-                   addr - PPC_IO_BASE, val);
+    trace_prep_io_800_writeb(addr - PPC_IO_BASE, val);
     switch (addr) {
     case 0x0092:
         /* Special port 92 */
@@ -327,8 +304,7 @@ static uint32_t PREP_io_800_readb (void *opaque, uint32_t addr)
         printf("ERROR: unaffected IO port: %04" PRIx32 " read\n", addr);
         break;
     }
-    PPC_IO_DPRINTF("0x%08" PRIx32 " <= 0x%02" PRIx32 "\n",
-                   addr - PPC_IO_BASE, retval);
+    trace_prep_io_800_readb(addr - PPC_IO_BASE, retval);
 
     return retval;
 }
@@ -336,15 +312,6 @@ static uint32_t PREP_io_800_readb (void *opaque, uint32_t addr)
 
 #define NVRAM_SIZE        0x2000
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static void ppc_prep_reset(void *opaque)
 {
     PowerPCCPU *cpu = opaque;
@@ -610,7 +577,7 @@ static void ppc_prep_init(MachineState *machine)
         bios_name = BIOS_FILENAME;
     }
     qdev_prop_set_string(dev, "bios-name", bios_name);
-    qdev_prop_set_uint32(dev, "elf-machine", ELF_MACHINE);
+    qdev_prop_set_uint32(dev, "elf-machine", PPC_ELF_MACHINE);
     pcihost = PCI_HOST_BRIDGE(dev);
     object_property_add_child(qdev_get_machine(), "raven", OBJECT(dev), NULL);
     qdev_init_nofail(dev);
@@ -626,8 +593,6 @@ static void ppc_prep_init(MachineState *machine)
     cpu = POWERPC_CPU(first_cpu);
     qdev_connect_gpio_out(&pci->qdev, 0,
                           cpu->env.irq_inputs[PPC6xx_INPUT_INT]);
-    qdev_connect_gpio_out(&pci->qdev, 1,
-                          qemu_allocate_irq(cpu_request_exit, NULL, 0));
     sysbus_connect_irq(&pcihost->busdev, 0, qdev_get_gpio_in(&pci->qdev, 9));
     sysbus_connect_irq(&pcihost->busdev, 1, qdev_get_gpio_in(&pci->qdev, 11));
     sysbus_connect_irq(&pcihost->busdev, 2, qdev_get_gpio_in(&pci->qdev, 9));
@@ -698,17 +663,12 @@ static void ppc_prep_init(MachineState *machine)
                          graphic_width, graphic_height, graphic_depth);
 }
 
-static QEMUMachine prep_machine = {
-    .name = "prep",
-    .desc = "PowerPC PREP platform",
-    .init = ppc_prep_init,
-    .max_cpus = MAX_CPUS,
-    .default_boot_order = "cad",
-};
-
-static void prep_machine_init(void)
+static void prep_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&prep_machine);
+    mc->desc = "PowerPC PREP platform";
+    mc->init = ppc_prep_init;
+    mc->max_cpus = MAX_CPUS;
+    mc->default_boot_order = "cad";
 }
 
-machine_init(prep_machine_init);
+DEFINE_MACHINE("prep", prep_machine_init)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index a6f19473c..ff1537a47 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -30,9 +30,11 @@
 #include "hw/fw-path-provider.h"
 #include "elf.h"
 #include "net/net.h"
+#include "sysemu/device_tree.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/cpus.h"
 #include "sysemu/kvm.h"
+#include "sysemu/device_tree.h"
 #include "kvm_ppc.h"
 #include "migration/migration.h"
 #include "mmu-hash64.h"
@@ -60,6 +62,7 @@
 #include "hw/nmi.h"
 
 #include "hw/compat.h"
+#include "qemu-common.h"
 
 #include <libfdt.h>
 
@@ -73,7 +76,7 @@
  *
  * We load our kernel at 4M, leaving space for SLOF initial image
  */
-#define FDT_MAX_SIZE            0x40000
+#define FDT_MAX_SIZE            0x100000
 #define RTAS_MAX_SIZE           0x10000
 #define RTAS_MAX_ADDR           0x80000000 /* RTAS must stay below that */
 #define FW_MAX_SIZE             0x400000
@@ -85,8 +88,6 @@
 
 #define TIMEBASE_FREQ           512000000ULL
 
-#define MAX_CPUS                255
-
 #define PHANDLE_XICP            0x00001111
 
 #define HTAB_SIZE(spapr)        (1ULL << ((spapr)->htab_shift))
@@ -124,6 +125,7 @@ static XICSState *xics_system_init(MachineState *machine,
             error_report("kernel_irqchip requested but unavailable: %s",
                          error_get_pretty(err));
         }
+        error_free(err);
     }
 
     if (!icp) {
@@ -375,6 +377,11 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
     _FDT((fdt_property_string(fdt, "vm,uuid", buf)));
     g_free(buf);
 
+    if (qemu_get_vm_name()) {
+        _FDT((fdt_property_string(fdt, "ibm,partition-name",
+                                  qemu_get_vm_name())));
+    }
+
     _FDT((fdt_property_cell(fdt, "#address-cells", 0x2)));
     _FDT((fdt_property_cell(fdt, "#size-cells", 0x2)));
 
@@ -427,6 +434,10 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
     _FDT((fdt_property_cell(fdt, "rtas-event-scan-rate",
                             RTAS_EVENT_SCAN_RATE)));
 
+    if (msi_supported) {
+        _FDT((fdt_property(fdt, "ibm,change-msix-capable", NULL, 0)));
+    }
+
     /*
      * According to PAPR, rtas ibm,os-term does not guarantee a return
      * back to the guest cpu.
@@ -495,44 +506,7 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
     return fdt;
 }
 
-int spapr_h_cas_compose_response(sPAPRMachineState *spapr,
-                                 target_ulong addr, target_ulong size)
-{
-    void *fdt, *fdt_skel;
-    sPAPRDeviceTreeUpdateHeader hdr = { .version_id = 1 };
-
-    size -= sizeof(hdr);
-
-    /* Create sceleton */
-    fdt_skel = g_malloc0(size);
-    _FDT((fdt_create(fdt_skel, size)));
-    _FDT((fdt_begin_node(fdt_skel, "")));
-    _FDT((fdt_end_node(fdt_skel)));
-    _FDT((fdt_finish(fdt_skel)));
-    fdt = g_malloc0(size);
-    _FDT((fdt_open_into(fdt_skel, fdt, size)));
-    g_free(fdt_skel);
-
-    /* Fix skeleton up */
-    _FDT((spapr_fixup_cpu_dt(fdt, spapr)));
-
-    /* Pack resulting tree */
-    _FDT((fdt_pack(fdt)));
-
-    if (fdt_totalsize(fdt) + sizeof(hdr) > size) {
-        trace_spapr_cas_failed(size);
-        return -1;
-    }
-
-    cpu_physical_memory_write(addr, &hdr, sizeof(hdr));
-    cpu_physical_memory_write(addr + sizeof(hdr), fdt, fdt_totalsize(fdt));
-    trace_spapr_cas_continue(fdt_totalsize(fdt) + sizeof(hdr));
-    g_free(fdt);
-
-    return 0;
-}
-
-static void spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
+static int spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
                                        hwaddr size)
 {
     uint32_t associativity[] = {
@@ -555,6 +529,7 @@ static void spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
                       sizeof(mem_reg_property))));
     _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
                       sizeof(associativity))));
+    return off;
 }
 
 static int spapr_populate_memory(sPAPRMachineState *spapr, void *fdt)
@@ -620,11 +595,27 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
     uint32_t cpufreq = kvm_enabled() ? kvmppc_get_clockfreq() : 1000000000;
     uint32_t page_sizes_prop[64];
     size_t page_sizes_prop_size;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("smp-opts"), NULL);
-    unsigned sockets = opts ? qemu_opt_get_number(opts, "sockets", 0) : 0;
-    uint32_t cpus_per_socket = sockets ? (smp_cpus / sockets) : 1;
+    uint32_t vcpus_per_socket = smp_threads * smp_cores;
     uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
 
+    /* Note: we keep CI large pages off for now because a 64K capable guest
+     * provisioned with large pages might otherwise try to map a qemu
+     * framebuffer (or other kind of memory mapped PCI BAR) using 64K pages
+     * even if that qemu runs on a 4k host.
+     *
+     * We can later add this bit back when we are confident this is not
+     * an issue (!HV KVM or 64K host)
+     */
+    uint8_t pa_features_206[] = { 6, 0,
+        0xf6, 0x1f, 0xc7, 0x00, 0x80, 0xc0 };
+    uint8_t pa_features_207[] = { 24, 0,
+        0xf6, 0x1f, 0xc7, 0xc0, 0x80, 0xf0,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x80, 0x00,
+        0x80, 0x00, 0x80, 0x00, 0x80, 0x00 };
+    uint8_t *pa_features;
+    size_t pa_size;
+
     _FDT((fdt_setprop_cell(fdt, offset, "reg", index)));
     _FDT((fdt_setprop_string(fdt, offset, "device_type", "cpu")));
 
@@ -653,6 +644,7 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
 
     _FDT((fdt_setprop_cell(fdt, offset, "timebase-frequency", tbfreq)));
     _FDT((fdt_setprop_cell(fdt, offset, "clock-frequency", cpufreq)));
+    _FDT((fdt_setprop_cell(fdt, offset, "slb-size", env->slb_nr)));
     _FDT((fdt_setprop_cell(fdt, offset, "ibm,slb-size", env->slb_nr)));
     _FDT((fdt_setprop_string(fdt, offset, "status", "okay")));
     _FDT((fdt_setprop(fdt, offset, "64-bit", NULL, 0)));
@@ -690,8 +682,21 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
                           page_sizes_prop, page_sizes_prop_size)));
     }
 
+    /* Do the ibm,pa-features property, adjust it for ci-large-pages */
+    if (env->mmu_model == POWERPC_MMU_2_06) {
+        pa_features = pa_features_206;
+        pa_size = sizeof(pa_features_206);
+    } else /* env->mmu_model == POWERPC_MMU_2_07 */ {
+        pa_features = pa_features_207;
+        pa_size = sizeof(pa_features_207);
+    }
+    if (env->ci_large_pages) {
+        pa_features[3] |= 0x20;
+    }
+    _FDT((fdt_setprop(fdt, offset, "ibm,pa-features", pa_features, pa_size)));
+
     _FDT((fdt_setprop_cell(fdt, offset, "ibm,chip-id",
-                           cs->cpu_index / cpus_per_socket)));
+                           cs->cpu_index / vcpus_per_socket)));
 
     _FDT((fdt_setprop(fdt, offset, "ibm,pft-size",
                       pft_size_prop, sizeof(pft_size_prop))));
@@ -738,12 +743,155 @@ static void spapr_populate_cpus_dt_node(void *fdt, sPAPRMachineState *spapr)
 
 }
 
+/*
+ * Adds ibm,dynamic-reconfiguration-memory node.
+ * Refer to docs/specs/ppc-spapr-hotplug.txt for the documentation
+ * of this device tree node.
+ */
+static int spapr_populate_drconf_memory(sPAPRMachineState *spapr, void *fdt)
+{
+    MachineState *machine = MACHINE(spapr);
+    int ret, i, offset;
+    uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
+    uint32_t prop_lmb_size[] = {0, cpu_to_be32(lmb_size)};
+    uint32_t nr_lmbs = (machine->maxram_size - machine->ram_size)/lmb_size;
+    uint32_t *int_buf, *cur_index, buf_len;
+    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
+
+    /*
+     * Allocate enough buffer size to fit in ibm,dynamic-memory
+     * or ibm,associativity-lookup-arrays
+     */
+    buf_len = MAX(nr_lmbs * SPAPR_DR_LMB_LIST_ENTRY_SIZE + 1, nr_nodes * 4 + 2)
+              * sizeof(uint32_t);
+    cur_index = int_buf = g_malloc0(buf_len);
+
+    offset = fdt_add_subnode(fdt, 0, "ibm,dynamic-reconfiguration-memory");
+
+    ret = fdt_setprop(fdt, offset, "ibm,lmb-size", prop_lmb_size,
+                    sizeof(prop_lmb_size));
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = fdt_setprop_cell(fdt, offset, "ibm,memory-flags-mask", 0xff);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = fdt_setprop_cell(fdt, offset, "ibm,memory-preservation-time", 0x0);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* ibm,dynamic-memory */
+    int_buf[0] = cpu_to_be32(nr_lmbs);
+    cur_index++;
+    for (i = 0; i < nr_lmbs; i++) {
+        sPAPRDRConnector *drc;
+        sPAPRDRConnectorClass *drck;
+        uint64_t addr = i * lmb_size + spapr->hotplug_memory.base;;
+        uint32_t *dynamic_memory = cur_index;
+
+        drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_LMB,
+                                       addr/lmb_size);
+        g_assert(drc);
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+        dynamic_memory[0] = cpu_to_be32(addr >> 32);
+        dynamic_memory[1] = cpu_to_be32(addr & 0xffffffff);
+        dynamic_memory[2] = cpu_to_be32(drck->get_index(drc));
+        dynamic_memory[3] = cpu_to_be32(0); /* reserved */
+        dynamic_memory[4] = cpu_to_be32(numa_get_node(addr, NULL));
+        if (addr < machine->ram_size ||
+                    memory_region_present(get_system_memory(), addr)) {
+            dynamic_memory[5] = cpu_to_be32(SPAPR_LMB_FLAGS_ASSIGNED);
+        } else {
+            dynamic_memory[5] = cpu_to_be32(0);
+        }
+
+        cur_index += SPAPR_DR_LMB_LIST_ENTRY_SIZE;
+    }
+    ret = fdt_setprop(fdt, offset, "ibm,dynamic-memory", int_buf, buf_len);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* ibm,associativity-lookup-arrays */
+    cur_index = int_buf;
+    int_buf[0] = cpu_to_be32(nr_nodes);
+    int_buf[1] = cpu_to_be32(4); /* Number of entries per associativity list */
+    cur_index += 2;
+    for (i = 0; i < nr_nodes; i++) {
+        uint32_t associativity[] = {
+            cpu_to_be32(0x0),
+            cpu_to_be32(0x0),
+            cpu_to_be32(0x0),
+            cpu_to_be32(i)
+        };
+        memcpy(cur_index, associativity, sizeof(associativity));
+        cur_index += 4;
+    }
+    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
+            (cur_index - int_buf) * sizeof(uint32_t));
+out:
+    g_free(int_buf);
+    return ret;
+}
+
+int spapr_h_cas_compose_response(sPAPRMachineState *spapr,
+                                 target_ulong addr, target_ulong size,
+                                 bool cpu_update, bool memory_update)
+{
+    void *fdt, *fdt_skel;
+    sPAPRDeviceTreeUpdateHeader hdr = { .version_id = 1 };
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
+    size -= sizeof(hdr);
+
+    /* Create sceleton */
+    fdt_skel = g_malloc0(size);
+    _FDT((fdt_create(fdt_skel, size)));
+    _FDT((fdt_begin_node(fdt_skel, "")));
+    _FDT((fdt_end_node(fdt_skel)));
+    _FDT((fdt_finish(fdt_skel)));
+    fdt = g_malloc0(size);
+    _FDT((fdt_open_into(fdt_skel, fdt, size)));
+    g_free(fdt_skel);
+
+    /* Fixup cpu nodes */
+    if (cpu_update) {
+        _FDT((spapr_fixup_cpu_dt(fdt, spapr)));
+    }
+
+    /* Generate memory nodes or ibm,dynamic-reconfiguration-memory node */
+    if (memory_update && smc->dr_lmb_enabled) {
+        _FDT((spapr_populate_drconf_memory(spapr, fdt)));
+    }
+
+    /* Pack resulting tree */
+    _FDT((fdt_pack(fdt)));
+
+    if (fdt_totalsize(fdt) + sizeof(hdr) > size) {
+        trace_spapr_cas_failed(size);
+        return -1;
+    }
+
+    cpu_physical_memory_write(addr, &hdr, sizeof(hdr));
+    cpu_physical_memory_write(addr + sizeof(hdr), fdt, fdt_totalsize(fdt));
+    trace_spapr_cas_continue(fdt_totalsize(fdt) + sizeof(hdr));
+    g_free(fdt);
+
+    return 0;
+}
+
 static void spapr_finalize_fdt(sPAPRMachineState *spapr,
                                hwaddr fdt_addr,
                                hwaddr rtas_addr,
                                hwaddr rtas_size)
 {
     MachineState *machine = MACHINE(qdev_get_machine());
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
     const char *boot_device = machine->boot_order;
     int ret, i;
     size_t cb = 0;
@@ -768,6 +916,14 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr,
         exit(1);
     }
 
+    if (object_resolve_path_type("", TYPE_SPAPR_RNG, NULL)) {
+        ret = spapr_rng_populate_dt(fdt);
+        if (ret < 0) {
+            fprintf(stderr, "could not set up rng device in the fdt\n");
+            exit(1);
+        }
+    }
+
     QLIST_FOREACH(phb, &spapr->phbs, list) {
         ret = spapr_populate_pci_dt(phb, PHANDLE_XICP, fdt);
     }
@@ -814,6 +970,10 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr,
         spapr_populate_chosen_stdout(fdt, spapr->vio_bus);
     }
 
+    if (smc->dr_lmb_enabled) {
+        _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB));
+    }
+
     _FDT((fdt_pack(fdt)));
 
     if (fdt_totalsize(fdt) > FDT_MAX_SIZE) {
@@ -822,6 +982,7 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr,
         exit(1);
     }
 
+    qemu_fdt_dumpdtb(fdt, fdt_totalsize(fdt));
     cpu_physical_memory_write(fdt_addr, fdt, fdt_totalsize(fdt));
 
     g_free(bootlist);
@@ -851,7 +1012,7 @@ static void emulate_spapr_hypercall(PowerPCCPU *cpu)
 #define CLEAN_HPTE(_hpte)  ((*(uint64_t *)(_hpte)) &= tswap64(~HPTE64_V_HPTE_DIRTY))
 #define DIRTY_HPTE(_hpte)  ((*(uint64_t *)(_hpte)) |= tswap64(HPTE64_V_HPTE_DIRTY))
 
-static void spapr_reset_htab(sPAPRMachineState *spapr)
+static void spapr_alloc_htab(sPAPRMachineState *spapr)
 {
     long shift;
     int index;
@@ -861,23 +1022,62 @@ static void spapr_reset_htab(sPAPRMachineState *spapr)
      * RAM */
 
     shift = kvmppc_reset_htab(spapr->htab_shift);
+    if (shift < 0) {
+        /*
+         * For HV KVM, host kernel will return -ENOMEM when requested
+         * HTAB size can't be allocated.
+         */
+        error_setg(&error_abort, "Failed to allocate HTAB of requested size, try with smaller maxmem");
+    } else if (shift > 0) {
+        /*
+         * Kernel handles htab, we don't need to allocate one
+         *
+         * Older kernels can fall back to lower HTAB shift values,
+         * but we don't allow booting of such guests.
+         */
+        if (shift != spapr->htab_shift) {
+            error_setg(&error_abort, "Failed to allocate HTAB of requested size, try with smaller maxmem");
+        }
 
-    if (shift > 0) {
-        /* Kernel handles htab, we don't need to allocate one */
         spapr->htab_shift = shift;
         kvmppc_kern_htab = true;
+    } else {
+        /* Allocate htab */
+        spapr->htab = qemu_memalign(HTAB_SIZE(spapr), HTAB_SIZE(spapr));
+
+        /* And clear it */
+        memset(spapr->htab, 0, HTAB_SIZE(spapr));
+
+        for (index = 0; index < HTAB_SIZE(spapr) / HASH_PTE_SIZE_64; index++) {
+            DIRTY_HPTE(HPTE(spapr->htab, index));
+        }
+    }
+}
+
+/*
+ * Clear HTAB entries during reset.
+ *
+ * If host kernel has allocated HTAB, KVM_PPC_ALLOCATE_HTAB ioctl is
+ * used to clear HTAB. Otherwise QEMU-allocated HTAB is cleared manually.
+ */
+static void spapr_reset_htab(sPAPRMachineState *spapr)
+{
+    long shift;
+    int index;
+
+    shift = kvmppc_reset_htab(spapr->htab_shift);
+    if (shift < 0) {
+        error_setg(&error_abort, "Failed to reset HTAB");
+    } else if (shift > 0) {
+        if (shift != spapr->htab_shift) {
+            error_setg(&error_abort, "Requested HTAB allocation failed during reset");
+        }
 
         /* Tell readers to update their file descriptor */
         if (spapr->htab_fd >= 0) {
             spapr->htab_fd_stale = true;
         }
     } else {
-        if (!spapr->htab) {
-            /* Allocate an htab if we don't yet have one */
-            spapr->htab = qemu_memalign(HTAB_SIZE(spapr), HTAB_SIZE(spapr));
-        }
-
-        /* And clear it */
         memset(spapr->htab, 0, HTAB_SIZE(spapr));
 
         for (index = 0; index < HTAB_SIZE(spapr) / HASH_PTE_SIZE_64; index++) {
@@ -1041,6 +1241,7 @@ static int spapr_vga_init(PCIBus *pci_bus)
     case VGA_DEVICE:
         return true;
     case VGA_STD:
+    case VGA_VIRTIO:
         return pci_vga_init(pci_bus) != NULL;
     default:
         fprintf(stderr, "This vga model is not supported,"
@@ -1329,6 +1530,8 @@ static int htab_load(QEMUFile *f, void *opaque, int version_id)
     if (section_hdr) {
         /* First section, just the hash shift */
         if (spapr->htab_shift != section_hdr) {
+            error_report("htab_shift mismatch: source %d target %d",
+                         section_hdr, spapr->htab_shift);
             return -EINVAL;
         }
         return 0;
@@ -1398,7 +1601,7 @@ static int htab_load(QEMUFile *f, void *opaque, int version_id)
 static SaveVMHandlers savevm_htab_handlers = {
     .save_live_setup = htab_save_setup,
     .save_live_iterate = htab_save_iterate,
-    .save_live_complete = htab_save_complete,
+    .save_live_complete_precopy = htab_save_complete,
     .load_state = htab_load,
 };
 
@@ -1437,10 +1640,77 @@ static void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu)
     qemu_register_reset(spapr_cpu_reset, cpu);
 }
 
+/*
+ * Reset routine for LMB DR devices.
+ *
+ * Unlike PCI DR devices, LMB DR devices explicitly register this reset
+ * routine. Reset for PCI DR devices will be handled by PHB reset routine
+ * when it walks all its children devices. LMB devices reset occurs
+ * as part of spapr_ppc_reset().
+ */
+static void spapr_drc_reset(void *opaque)
+{
+    sPAPRDRConnector *drc = opaque;
+    DeviceState *d = DEVICE(drc);
+
+    if (d) {
+        device_reset(d);
+    }
+}
+
+static void spapr_create_lmb_dr_connectors(sPAPRMachineState *spapr)
+{
+    MachineState *machine = MACHINE(spapr);
+    uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
+    uint32_t nr_lmbs = (machine->maxram_size - machine->ram_size)/lmb_size;
+    int i;
+
+    for (i = 0; i < nr_lmbs; i++) {
+        sPAPRDRConnector *drc;
+        uint64_t addr;
+
+        addr = i * lmb_size + spapr->hotplug_memory.base;
+        drc = spapr_dr_connector_new(OBJECT(spapr), SPAPR_DR_CONNECTOR_TYPE_LMB,
+                                     addr/lmb_size);
+        qemu_register_reset(spapr_drc_reset, drc);
+    }
+}
+
+/*
+ * If RAM size, maxmem size and individual node mem sizes aren't aligned
+ * to SPAPR_MEMORY_BLOCK_SIZE(256MB), then refuse to start the guest
+ * since we can't support such unaligned sizes with DRCONF_MEMORY.
+ */
+static void spapr_validate_node_memory(MachineState *machine)
+{
+    int i;
+
+    if (machine->maxram_size % SPAPR_MEMORY_BLOCK_SIZE ||
+        machine->ram_size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_report("Can't support memory configuration where RAM size "
+                     "0x" RAM_ADDR_FMT " or maxmem size "
+                     "0x" RAM_ADDR_FMT " isn't aligned to %llu MB",
+                     machine->ram_size, machine->maxram_size,
+                     SPAPR_MEMORY_BLOCK_SIZE/M_BYTE);
+        exit(EXIT_FAILURE);
+    }
+
+    for (i = 0; i < nb_numa_nodes; i++) {
+        if (numa_info[i].node_mem % SPAPR_MEMORY_BLOCK_SIZE) {
+            error_report("Can't support memory configuration where memory size"
+                         " %" PRIx64 " of node %d isn't aligned to %llu MB",
+                         numa_info[i].node_mem, i,
+                         SPAPR_MEMORY_BLOCK_SIZE/M_BYTE);
+            exit(EXIT_FAILURE);
+        }
+    }
+}
+
 /* pSeries LPAR / sPAPR hardware init */
 static void ppc_spapr_init(MachineState *machine)
 {
     sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -1507,11 +1777,12 @@ static void ppc_spapr_init(MachineState *machine)
      * more than needed for the Linux guests we support. */
     spapr->htab_shift = 18; /* Minimum architected size */
     while (spapr->htab_shift <= 46) {
-        if ((1ULL << (spapr->htab_shift + 7)) >= machine->ram_size) {
+        if ((1ULL << (spapr->htab_shift + 7)) >= machine->maxram_size) {
             break;
         }
         spapr->htab_shift++;
     }
+    spapr_alloc_htab(spapr);
 
     /* Set up Interrupt Controller before we create the VCPUs */
     spapr->icp = xics_system_init(machine,
@@ -1519,6 +1790,10 @@ static void ppc_spapr_init(MachineState *machine)
                                                smp_threads),
                                   XICS_IRQS);
 
+    if (smc->dr_lmb_enabled) {
+        spapr_validate_node_memory(machine);
+    }
+
     /* init CPUs */
     if (machine->cpu_model == NULL) {
         machine->cpu_model = kvm_enabled() ? "host" : "POWER7";
@@ -1535,6 +1810,7 @@ static void ppc_spapr_init(MachineState *machine)
     if (kvm_enabled()) {
         /* Enable H_LOGICAL_CI_* so SLOF can talk to in-kernel devices */
         kvmppc_enable_logical_ci_hcalls();
+        kvmppc_enable_set_mode_hcall();
     }
 
     /* allocate RAM */
@@ -1550,6 +1826,28 @@ static void ppc_spapr_init(MachineState *machine)
         memory_region_add_subregion(sysmem, 0, rma_region);
     }
 
+    /* initialize hotplug memory address space */
+    if (machine->ram_size < machine->maxram_size) {
+        ram_addr_t hotplug_mem_size = machine->maxram_size - machine->ram_size;
+
+        if (machine->ram_slots > SPAPR_MAX_RAM_SLOTS) {
+            error_report("Specified number of memory slots %"PRIu64" exceeds max supported %d\n",
+                         machine->ram_slots, SPAPR_MAX_RAM_SLOTS);
+            exit(EXIT_FAILURE);
+        }
+
+        spapr->hotplug_memory.base = ROUND_UP(machine->ram_size,
+                                              SPAPR_HOTPLUG_MEM_ALIGN);
+        memory_region_init(&spapr->hotplug_memory.mr, OBJECT(spapr),
+                           "hotplug-memory", hotplug_mem_size);
+        memory_region_add_subregion(sysmem, spapr->hotplug_memory.base,
+                                    &spapr->hotplug_memory.mr);
+    }
+
+    if (smc->dr_lmb_enabled) {
+        spapr_create_lmb_dr_connectors(spapr);
+    }
+
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "spapr-rtas.bin");
     if (!filename) {
         error_report("Could not find LPAR rtas '%s'", "spapr-rtas.bin");
@@ -1636,11 +1934,11 @@ static void ppc_spapr_init(MachineState *machine)
         uint64_t lowaddr = 0;
 
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE, 0);
         if (kernel_size == ELF_LOAD_WRONG_ENDIAN) {
             kernel_size = load_elf(kernel_filename,
                                    translate_kernel_address, NULL,
-                                   NULL, &lowaddr, NULL, 0, ELF_MACHINE, 0);
+                                   NULL, &lowaddr, NULL, 0, PPC_ELF_MACHINE, 0);
             kernel_le = kernel_size > 0;
         }
         if (kernel_size < 0) {
@@ -1820,22 +2118,166 @@ static void spapr_nmi(NMIState *n, int cpu_index, Error **errp)
     }
 }
 
+static void spapr_add_lmbs(DeviceState *dev, uint64_t addr, uint64_t size,
+                           uint32_t node, Error **errp)
+{
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    uint32_t nr_lmbs = size/SPAPR_MEMORY_BLOCK_SIZE;
+    int i, fdt_offset, fdt_size;
+    void *fdt;
+
+    /*
+     * Check for DRC connectors and send hotplug notification to the
+     * guest only in case of hotplugged memory. This allows cold plugged
+     * memory to be specified at boot time.
+     */
+    if (!dev->hotplugged) {
+        return;
+    }
+
+    for (i = 0; i < nr_lmbs; i++) {
+        drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_LMB,
+                addr/SPAPR_MEMORY_BLOCK_SIZE);
+        g_assert(drc);
+
+        fdt = create_device_tree(&fdt_size);
+        fdt_offset = spapr_populate_memory_node(fdt, node, addr,
+                                                SPAPR_MEMORY_BLOCK_SIZE);
+
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+        drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, errp);
+        addr += SPAPR_MEMORY_BLOCK_SIZE;
+    }
+    spapr_hotplug_req_add_by_count(SPAPR_DR_CONNECTOR_TYPE_LMB, nr_lmbs);
+}
+
+static void spapr_memory_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                              uint32_t node, Error **errp)
+{
+    Error *local_err = NULL;
+    sPAPRMachineState *ms = SPAPR_MACHINE(hotplug_dev);
+    PCDIMMDevice *dimm = PC_DIMM(dev);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
+    MemoryRegion *mr = ddc->get_memory_region(dimm);
+    uint64_t align = memory_region_get_alignment(mr);
+    uint64_t size = memory_region_size(mr);
+    uint64_t addr;
+
+    if (size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_setg(&local_err, "Hotplugged memory size must be a multiple of "
+                      "%lld MB", SPAPR_MEMORY_BLOCK_SIZE/M_BYTE);
+        goto out;
+    }
+
+    pc_dimm_memory_plug(dev, &ms->hotplug_memory, mr, align, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    addr = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP, &local_err);
+    if (local_err) {
+        pc_dimm_memory_unplug(dev, &ms->hotplug_memory, mr);
+        goto out;
+    }
+
+    spapr_add_lmbs(dev, addr, size, node, &error_abort);
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        int node;
+
+        if (!smc->dr_lmb_enabled) {
+            error_setg(errp, "Memory hotplug not supported for this machine");
+            return;
+        }
+        node = object_property_get_int(OBJECT(dev), PC_DIMM_NODE_PROP, errp);
+        if (*errp) {
+            return;
+        }
+
+        /*
+         * Currently PowerPC kernel doesn't allow hot-adding memory to
+         * memory-less node, but instead will silently add the memory
+         * to the first node that has some memory. This causes two
+         * unexpected behaviours for the user.
+         *
+         * - Memory gets hotplugged to a different node than what the user
+         *   specified.
+         * - Since pc-dimm subsystem in QEMU still thinks that memory belongs
+         *   to memory-less node, a reboot will set things accordingly
+         *   and the previously hotplugged memory now ends in the right node.
+         *   This appears as if some memory moved from one node to another.
+         *
+         * So until kernel starts supporting memory hotplug to memory-less
+         * nodes, just prevent such attempts upfront in QEMU.
+         */
+        if (nb_numa_nodes && !numa_info[node].node_mem) {
+            error_setg(errp, "Can't hotplug memory to memory-less node %d",
+                       node);
+            return;
+        }
+
+        spapr_memory_plug(hotplug_dev, dev, node, errp);
+    }
+}
+
+static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        error_setg(errp, "Memory hot unplug not supported by sPAPR");
+    }
+}
+
+static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
+                                             DeviceState *dev)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        return HOTPLUG_HANDLER(machine);
+    }
+    return NULL;
+}
+
+static unsigned spapr_cpu_index_to_socket_id(unsigned cpu_index)
+{
+    /* Allocate to NUMA nodes on a "socket" basis (not that concept of
+     * socket means much for the paravirtualized PAPR platform) */
+    return cpu_index / smp_threads / smp_cores;
+}
+
 static void spapr_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(oc);
     FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(oc);
     NMIClass *nc = NMI_CLASS(oc);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
 
     mc->init = ppc_spapr_init;
     mc->reset = ppc_spapr_reset;
     mc->block_default_type = IF_SCSI;
-    mc->max_cpus = MAX_CPUS;
+    mc->max_cpus = MAX_CPUMASK_BITS;
     mc->no_parallel = 1;
     mc->default_boot_order = "";
     mc->default_ram_size = 512 * M_BYTE;
     mc->kvm_type = spapr_kvm_type;
     mc->has_dynamic_sysbus = true;
+    mc->pci_allow_0_address = true;
+    mc->get_hotplug_handler = spapr_get_hotpug_handler;
+    hc->plug = spapr_machine_device_plug;
+    hc->unplug = spapr_machine_device_unplug;
+    mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id;
 
+    smc->dr_lmb_enabled = false;
     fwc->get_dev_path = spapr_get_fw_dev_path;
     nc->nmi_monitor_handler = spapr_nmi;
 }
@@ -1851,11 +2293,16 @@ static const TypeInfo spapr_machine_info = {
     .interfaces = (InterfaceInfo[]) {
         { TYPE_FW_PATH_PROVIDER },
         { TYPE_NMI },
+        { TYPE_HOTPLUG_HANDLER },
         { }
     },
 };
 
+#define SPAPR_COMPAT_2_4 \
+        HW_COMPAT_2_4
+
 #define SPAPR_COMPAT_2_3 \
+        SPAPR_COMPAT_2_4 \
         HW_COMPAT_2_3 \
         {\
             .driver   = "spapr-pci-host-bridge",\
@@ -1880,6 +2327,7 @@ static void spapr_compat_2_3(Object *obj)
 {
     savevm_skip_section_footers();
     global_state_set_optional();
+    savevm_skip_configuration();
 }
 
 static void spapr_compat_2_2(Object *obj)
@@ -1918,13 +2366,12 @@ static void spapr_machine_2_1_class_init(ObjectClass *oc, void *data)
         { /* end of list */ }
     };
 
-    mc->name = "pseries-2.1";
     mc->desc = "pSeries Logical Partition (PAPR compliant) v2.1";
     mc->compat_props = compat_props;
 }
 
 static const TypeInfo spapr_machine_2_1_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.1",
+    .name          = MACHINE_TYPE_NAME("pseries-2.1"),
     .parent        = TYPE_SPAPR_MACHINE,
     .class_init    = spapr_machine_2_1_class_init,
     .instance_init = spapr_machine_2_1_instance_init,
@@ -1938,13 +2385,12 @@ static void spapr_machine_2_2_class_init(ObjectClass *oc, void *data)
     };
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = "pseries-2.2";
     mc->desc = "pSeries Logical Partition (PAPR compliant) v2.2";
     mc->compat_props = compat_props;
 }
 
 static const TypeInfo spapr_machine_2_2_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.2",
+    .name          = MACHINE_TYPE_NAME("pseries-2.2"),
     .parent        = TYPE_SPAPR_MACHINE,
     .class_init    = spapr_machine_2_2_class_init,
     .instance_init = spapr_machine_2_2_instance_init,
@@ -1958,13 +2404,12 @@ static void spapr_machine_2_3_class_init(ObjectClass *oc, void *data)
     };
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = "pseries-2.3";
     mc->desc = "pSeries Logical Partition (PAPR compliant) v2.3";
     mc->compat_props = compat_props;
 }
 
 static const TypeInfo spapr_machine_2_3_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.3",
+    .name          = MACHINE_TYPE_NAME("pseries-2.3"),
     .parent        = TYPE_SPAPR_MACHINE,
     .class_init    = spapr_machine_2_3_class_init,
     .instance_init = spapr_machine_2_3_instance_init,
@@ -1972,20 +2417,40 @@ static const TypeInfo spapr_machine_2_3_info = {
 
 static void spapr_machine_2_4_class_init(ObjectClass *oc, void *data)
 {
+    static GlobalProperty compat_props[] = {
+        SPAPR_COMPAT_2_4
+        { /* end of list */ }
+    };
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = "pseries-2.4";
     mc->desc = "pSeries Logical Partition (PAPR compliant) v2.4";
-    mc->alias = "pseries";
-    mc->is_default = 1;
+    mc->compat_props = compat_props;
 }
 
 static const TypeInfo spapr_machine_2_4_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.4",
+    .name          = MACHINE_TYPE_NAME("pseries-2.4"),
     .parent        = TYPE_SPAPR_MACHINE,
     .class_init    = spapr_machine_2_4_class_init,
 };
 
+static void spapr_machine_2_5_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(oc);
+
+    mc->name = "pseries-2.5";
+    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.5";
+    mc->alias = "pseries";
+    mc->is_default = 1;
+    smc->dr_lmb_enabled = true;
+}
+
+static const TypeInfo spapr_machine_2_5_info = {
+    .name          = MACHINE_TYPE_NAME("pseries-2.5"),
+    .parent        = TYPE_SPAPR_MACHINE,
+    .class_init    = spapr_machine_2_5_class_init,
+};
+
 static void spapr_machine_register_types(void)
 {
     type_register_static(&spapr_machine_info);
@@ -1993,6 +2458,7 @@ static void spapr_machine_register_types(void)
     type_register_static(&spapr_machine_2_2_info);
     type_register_static(&spapr_machine_2_3_info);
     type_register_static(&spapr_machine_2_4_info);
+    type_register_static(&spapr_machine_2_5_info);
 }
 
 type_init(spapr_machine_register_types)
diff --git a/hw/ppc/spapr_drc.c b/hw/ppc/spapr_drc.c
index ee874326e..8be62c349 100644
--- a/hw/ppc/spapr_drc.c
+++ b/hw/ppc/spapr_drc.c
@@ -15,6 +15,7 @@
 #include "hw/qdev.h"
 #include "qapi/visitor.h"
 #include "qemu/error-report.h"
+#include "hw/ppc/spapr.h" /* for RTAS return codes */
 
 /* #define DEBUG_SPAPR_DRC */
 
@@ -32,7 +33,7 @@
 
 #define DRC_CONTAINER_PATH "/dr-connector"
 #define DRC_INDEX_TYPE_SHIFT 28
-#define DRC_INDEX_ID_MASK (~(~0 << DRC_INDEX_TYPE_SHIFT))
+#define DRC_INDEX_ID_MASK ((1ULL << DRC_INDEX_TYPE_SHIFT) - 1)
 
 static sPAPRDRConnectorTypeShift get_type_shift(sPAPRDRConnectorType type)
 {
@@ -59,13 +60,23 @@ static uint32_t get_index(sPAPRDRConnector *drc)
             (drc->id & DRC_INDEX_ID_MASK);
 }
 
-static int set_isolation_state(sPAPRDRConnector *drc,
-                               sPAPRDRIsolationState state)
+static uint32_t set_isolation_state(sPAPRDRConnector *drc,
+                                    sPAPRDRIsolationState state)
 {
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
 
     DPRINTFN("drc: %x, set_isolation_state: %x", get_index(drc), state);
 
+    if (state == SPAPR_DR_ISOLATION_STATE_UNISOLATED) {
+        /* cannot unisolate a non-existant resource, and, or resources
+         * which are in an 'UNUSABLE' allocation state. (PAPR 2.7, 13.5.3.5)
+         */
+        if (!drc->dev ||
+            drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
+            return RTAS_OUT_NO_SUCH_INDICATOR;
+        }
+    }
+
     drc->isolation_state = state;
 
     if (drc->isolation_state == SPAPR_DR_ISOLATION_STATE_ISOLATED) {
@@ -89,24 +100,35 @@ static int set_isolation_state(sPAPRDRConnector *drc,
         drc->configured = false;
     }
 
-    return 0;
+    return RTAS_OUT_SUCCESS;
 }
 
-static int set_indicator_state(sPAPRDRConnector *drc,
-                               sPAPRDRIndicatorState state)
+static uint32_t set_indicator_state(sPAPRDRConnector *drc,
+                                    sPAPRDRIndicatorState state)
 {
     DPRINTFN("drc: %x, set_indicator_state: %x", get_index(drc), state);
     drc->indicator_state = state;
-    return 0;
+    return RTAS_OUT_SUCCESS;
 }
 
-static int set_allocation_state(sPAPRDRConnector *drc,
-                                sPAPRDRAllocationState state)
+static uint32_t set_allocation_state(sPAPRDRConnector *drc,
+                                     sPAPRDRAllocationState state)
 {
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
 
     DPRINTFN("drc: %x, set_allocation_state: %x", get_index(drc), state);
 
+    if (state == SPAPR_DR_ALLOCATION_STATE_USABLE) {
+        /* if there's no resource/device associated with the DRC, there's
+         * no way for us to put it in an allocation state consistent with
+         * being 'USABLE'. PAPR 2.7, 13.5.3.4 documents that this should
+         * result in an RTAS return code of -3 / "no such indicator"
+         */
+        if (!drc->dev) {
+            return RTAS_OUT_NO_SUCH_INDICATOR;
+        }
+    }
+
     if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI) {
         drc->allocation_state = state;
         if (drc->awaiting_release &&
@@ -116,7 +138,7 @@ static int set_allocation_state(sPAPRDRConnector *drc,
                          drc->detach_cb_opaque, NULL);
         }
     }
-    return 0;
+    return RTAS_OUT_SUCCESS;
 }
 
 static uint32_t get_type(sPAPRDRConnector *drc)
@@ -157,10 +179,8 @@ static void set_configured(sPAPRDRConnector *drc)
  * based on the current allocation/indicator/power states
  * for the DR connector.
  */
-static sPAPRDREntitySense entity_sense(sPAPRDRConnector *drc)
+static uint32_t entity_sense(sPAPRDRConnector *drc, sPAPRDREntitySense *state)
 {
-    sPAPRDREntitySense state;
-
     if (drc->dev) {
         if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI &&
             drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
@@ -169,7 +189,7 @@ static sPAPRDREntitySense entity_sense(sPAPRDRConnector *drc)
              * Otherwise, report the state as USABLE/PRESENT,
              * as we would for PCI.
              */
-            state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
+            *state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
         } else {
             /* this assumes all PCI devices are assigned to
              * a 'live insertion' power domain, where QEMU
@@ -177,21 +197,21 @@ static sPAPRDREntitySense entity_sense(sPAPRDRConnector *drc)
              * to the guest. present, non-PCI resources are
              * unaffected by power state.
              */
-            state = SPAPR_DR_ENTITY_SENSE_PRESENT;
+            *state = SPAPR_DR_ENTITY_SENSE_PRESENT;
         }
     } else {
         if (drc->type == SPAPR_DR_CONNECTOR_TYPE_PCI) {
             /* PCI devices, and only PCI devices, use EMPTY
              * in cases where we'd otherwise use UNUSABLE
              */
-            state = SPAPR_DR_ENTITY_SENSE_EMPTY;
+            *state = SPAPR_DR_ENTITY_SENSE_EMPTY;
         } else {
-            state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
+            *state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
         }
     }
 
     DPRINTFN("drc: %x, entity_sense: %x", get_index(drc), state);
-    return state;
+    return RTAS_OUT_SUCCESS;
 }
 
 static void prop_get_index(Object *obj, Visitor *v, void *opaque,
@@ -224,7 +244,9 @@ static void prop_get_entity_sense(Object *obj, Visitor *v, void *opaque,
 {
     sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
-    uint32_t value = (uint32_t)drck->entity_sense(drc);
+    uint32_t value;
+
+    drck->entity_sense(drc, &value);
     visit_type_uint32(v, &value, name, errp);
 }
 
@@ -232,10 +254,16 @@ static void prop_get_fdt(Object *obj, Visitor *v, void *opaque,
                         const char *name, Error **errp)
 {
     sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+    Error *err = NULL;
     int fdt_offset_next, fdt_offset, fdt_depth;
     void *fdt;
 
     if (!drc->fdt) {
+        visit_start_struct(v, NULL, NULL, name, 0, &err);
+        if (!err) {
+            visit_end_struct(v, &err);
+        }
+        error_propagate(errp, err);
         return;
     }
 
@@ -254,24 +282,43 @@ static void prop_get_fdt(Object *obj, Visitor *v, void *opaque,
         case FDT_BEGIN_NODE:
             fdt_depth++;
             name = fdt_get_name(fdt, fdt_offset, &name_len);
-            visit_start_struct(v, NULL, NULL, name, 0, NULL);
+            visit_start_struct(v, NULL, NULL, name, 0, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
             break;
         case FDT_END_NODE:
             /* shouldn't ever see an FDT_END_NODE before FDT_BEGIN_NODE */
             g_assert(fdt_depth > 0);
-            visit_end_struct(v, NULL);
+            visit_end_struct(v, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
             fdt_depth--;
             break;
         case FDT_PROP: {
             int i;
             prop = fdt_get_property_by_offset(fdt, fdt_offset, &prop_len);
             name = fdt_string(fdt, fdt32_to_cpu(prop->nameoff));
-            visit_start_list(v, name, NULL);
+            visit_start_list(v, name, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
             for (i = 0; i < prop_len; i++) {
-                visit_type_uint8(v, (uint8_t *)&prop->data[i], NULL, NULL);
-
+                visit_type_uint8(v, (uint8_t *)&prop->data[i], NULL, &err);
+                if (err) {
+                    error_propagate(errp, err);
+                    return;
+                }
+            }
+            visit_end_list(v, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
             }
-            visit_end_list(v, NULL);
             break;
         }
         default:
@@ -310,7 +357,7 @@ static void attach(sPAPRDRConnector *drc, DeviceState *d, void *fdt,
     drc->dev = d;
     drc->fdt = fdt;
     drc->fdt_start_offset = fdt_start_offset;
-    drc->configured = false;
+    drc->configured = coldplug;
 
     object_property_add_link(OBJECT(drc), "device",
                              object_get_typename(OBJECT(drc->dev)),
@@ -451,14 +498,17 @@ sPAPRDRConnector *spapr_dr_connector_new(Object *owner,
 {
     sPAPRDRConnector *drc =
         SPAPR_DR_CONNECTOR(object_new(TYPE_SPAPR_DR_CONNECTOR));
+    char *prop_name;
 
     g_assert(type);
 
     drc->type = type;
     drc->id = id;
     drc->owner = owner;
-    object_property_add_child(owner, "dr-connector[*]", OBJECT(drc), NULL);
+    prop_name = g_strdup_printf("dr-connector[%"PRIu32"]", get_index(drc));
+    object_property_add_child(owner, prop_name, OBJECT(drc), NULL);
     object_property_set_bool(OBJECT(drc), true, "realized", NULL);
+    g_free(prop_name);
 
     /* human-readable name for a DRC to encode into the DT
      * description. this is mainly only used within a guest in place
@@ -549,6 +599,10 @@ static void spapr_dr_connector_class_init(ObjectClass *k, void *data)
     drck->attach = attach;
     drck->detach = detach;
     drck->release_pending = release_pending;
+    /*
+     * Reason: it crashes FIXME find and document the real reason
+     */
+    dk->cannot_instantiate_with_device_add_yet = true;
 }
 
 static const TypeInfo spapr_dr_connector_info = {
@@ -632,6 +686,7 @@ int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner,
 {
     Object *root_container;
     ObjectProperty *prop;
+    ObjectPropertyIterator *iter;
     uint32_t drc_count = 0;
     GArray *drc_indexes, *drc_power_domains;
     GString *drc_names, *drc_types;
@@ -655,7 +710,8 @@ int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner,
      */
     root_container = container_get(object_get_root(), DRC_CONTAINER_PATH);
 
-    QTAILQ_FOREACH(prop, &root_container->properties, node) {
+    iter = object_property_iter_init(root_container);
+    while ((prop = object_property_iter_next(iter))) {
         Object *obj;
         sPAPRDRConnector *drc;
         sPAPRDRConnectorClass *drck;
@@ -696,6 +752,7 @@ int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner,
                                     spapr_drc_get_type_str(drc->type));
         drc_types = g_string_insert_len(drc_types, -1, "\0", 1);
     }
+    object_property_iter_free(iter);
 
     /* now write the drc count into the space we reserved at the
      * beginning of the arrays previously
diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c
index f626eb7b3..744ea62ac 100644
--- a/hw/ppc/spapr_events.c
+++ b/hw/ppc/spapr_events.c
@@ -386,7 +386,9 @@ static void spapr_powerdown_req(Notifier *n, void *opaque)
     qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->check_exception_irq));
 }
 
-static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
+static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action,
+                                    sPAPRDRConnectorType drc_type,
+                                    uint32_t drc)
 {
     sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
     struct hp_log_full *new_hp;
@@ -395,8 +397,6 @@ static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
     struct rtas_event_log_v6_maina *maina;
     struct rtas_event_log_v6_mainb *mainb;
     struct rtas_event_log_v6_hp *hp;
-    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
-    sPAPRDRConnectorType drc_type = drck->get_type(drc);
 
     new_hp = g_malloc0(sizeof(struct hp_log_full));
     hdr = &new_hp->hdr;
@@ -427,14 +427,15 @@ static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
     hp->hdr.section_length = cpu_to_be16(sizeof(*hp));
     hp->hdr.section_version = 1; /* includes extended modifier */
     hp->hotplug_action = hp_action;
-
+    hp->hotplug_identifier = hp_id;
 
     switch (drc_type) {
     case SPAPR_DR_CONNECTOR_TYPE_PCI:
-        hp->drc.index = cpu_to_be32(drck->get_index(drc));
-        hp->hotplug_identifier = RTAS_LOG_V6_HP_ID_DRC_INDEX;
         hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_PCI;
         break;
+    case SPAPR_DR_CONNECTOR_TYPE_LMB:
+        hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_MEMORY;
+        break;
     default:
         /* we shouldn't be signaling hotplug events for resources
          * that don't support them
@@ -443,19 +444,49 @@ static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
         return;
     }
 
+    if (hp_id == RTAS_LOG_V6_HP_ID_DRC_COUNT) {
+        hp->drc.count = cpu_to_be32(drc);
+    } else if (hp_id == RTAS_LOG_V6_HP_ID_DRC_INDEX) {
+        hp->drc.index = cpu_to_be32(drc);
+    }
+
     rtas_event_log_queue(RTAS_LOG_TYPE_HOTPLUG, new_hp, true);
 
     qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->check_exception_irq));
 }
 
-void spapr_hotplug_req_add_event(sPAPRDRConnector *drc)
+void spapr_hotplug_req_add_by_index(sPAPRDRConnector *drc)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDRConnectorType drc_type = drck->get_type(drc);
+    uint32 index = drck->get_index(drc);
+
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_INDEX,
+                            RTAS_LOG_V6_HP_ACTION_ADD, drc_type, index);
+}
+
+void spapr_hotplug_req_remove_by_index(sPAPRDRConnector *drc)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDRConnectorType drc_type = drck->get_type(drc);
+    uint32 index = drck->get_index(drc);
+
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_INDEX,
+                            RTAS_LOG_V6_HP_ACTION_REMOVE, drc_type, index);
+}
+
+void spapr_hotplug_req_add_by_count(sPAPRDRConnectorType drc_type,
+                                       uint32_t count)
 {
-    spapr_hotplug_req_event(drc, RTAS_LOG_V6_HP_ACTION_ADD);
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_COUNT,
+                            RTAS_LOG_V6_HP_ACTION_ADD, drc_type, count);
 }
 
-void spapr_hotplug_req_remove_event(sPAPRDRConnector *drc)
+void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type,
+                                          uint32_t count)
 {
-    spapr_hotplug_req_event(drc, RTAS_LOG_V6_HP_ACTION_REMOVE);
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_COUNT,
+                            RTAS_LOG_V6_HP_ACTION_REMOVE, drc_type, count);
 }
 
 static void check_exception(PowerPCCPU *cpu, sPAPRMachineState *spapr,
diff --git a/hw/ppc/spapr_hcall.c b/hw/ppc/spapr_hcall.c
index 652ddf6e3..cebceea69 100644
--- a/hw/ppc/spapr_hcall.c
+++ b/hw/ppc/spapr_hcall.c
@@ -808,6 +808,32 @@ static target_ulong h_set_mode(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     return ret;
 }
 
+/*
+ * Return the offset to the requested option vector @vector in the
+ * option vector table @table.
+ */
+static target_ulong cas_get_option_vector(int vector, target_ulong table)
+{
+    int i;
+    char nr_vectors, nr_entries;
+
+    if (!table) {
+        return 0;
+    }
+
+    nr_vectors = (ldl_phys(&address_space_memory, table) >> 24) + 1;
+    if (!vector || vector > nr_vectors) {
+        return 0;
+    }
+    table++; /* skip nr option vectors */
+
+    for (i = 0; i < vector - 1; i++) {
+        nr_entries = ldl_phys(&address_space_memory, table) >> 24;
+        table += nr_entries + 2;
+    }
+    return table;
+}
+
 typedef struct {
     PowerPCCPU *cpu;
     uint32_t cpu_version;
@@ -828,19 +854,22 @@ static void do_set_compat(void *arg)
     ((cpuver) == CPU_POWERPC_LOGICAL_2_06_PLUS) ? 2061 : \
     ((cpuver) == CPU_POWERPC_LOGICAL_2_07) ? 2070 : 0)
 
+#define OV5_DRCONF_MEMORY 0x20
+
 static target_ulong h_client_architecture_support(PowerPCCPU *cpu_,
                                                   sPAPRMachineState *spapr,
                                                   target_ulong opcode,
                                                   target_ulong *args)
 {
-    target_ulong list = args[0];
+    target_ulong list = args[0], ov_table;
     PowerPCCPUClass *pcc_ = POWERPC_CPU_GET_CLASS(cpu_);
     CPUState *cs;
-    bool cpu_match = false;
+    bool cpu_match = false, cpu_update = true, memory_update = false;
     unsigned old_cpu_version = cpu_->cpu_version;
     unsigned compat_lvl = 0, cpu_version = 0;
     unsigned max_lvl = get_compat_level(cpu_->max_compat);
     int counter;
+    char ov5_byte2;
 
     /* Parse PVR list */
     for (counter = 0; counter < 512; ++counter) {
@@ -890,8 +919,6 @@ static target_ulong h_client_architecture_support(PowerPCCPU *cpu_,
         }
     }
 
-    /* For the future use: here @list points to the first capability */
-
     /* Parsing finished */
     trace_spapr_cas_pvr(cpu_->cpu_version, cpu_match,
                         cpu_version, pcc_->pcr_mask);
@@ -915,14 +942,26 @@ static target_ulong h_client_architecture_support(PowerPCCPU *cpu_,
     }
 
     if (!cpu_version) {
-        return H_SUCCESS;
+        cpu_update = false;
     }
 
+    /* For the future use: here @ov_table points to the first option vector */
+    ov_table = list;
+
+    list = cas_get_option_vector(5, ov_table);
     if (!list) {
         return H_SUCCESS;
     }
 
-    if (spapr_h_cas_compose_response(spapr, args[1], args[2])) {
+    /* @list now points to OV 5 */
+    list += 2;
+    ov5_byte2 = rtas_ld(list, 0) >> 24;
+    if (ov5_byte2 & OV5_DRCONF_MEMORY) {
+        memory_update = true;
+    }
+
+    if (spapr_h_cas_compose_response(spapr, args[1], args[2],
+                                     cpu_update, memory_update)) {
         qemu_system_reset_request();
     }
 
@@ -971,7 +1010,8 @@ target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
         }
     }
 
-    hcall_dprintf("Unimplemented hcall 0x" TARGET_FMT_lx "\n", opcode);
+    qemu_log_mask(LOG_UNIMP, "Unimplemented SPAPR hcall 0x" TARGET_FMT_lx "\n",
+                  opcode);
     return H_FUNCTION;
 }
 
diff --git a/hw/ppc/spapr_iommu.c b/hw/ppc/spapr_iommu.c
index f61504e0c..ed28565d8 100644
--- a/hw/ppc/spapr_iommu.c
+++ b/hw/ppc/spapr_iommu.c
@@ -146,7 +146,7 @@ static int spapr_tce_table_realize(DeviceState *dev)
         tcet->table = kvmppc_create_spapr_tce(tcet->liobn,
                                               window_size,
                                               &tcet->fd,
-                                              tcet->vfio_accel);
+                                              tcet->need_vfio);
     }
 
     if (!tcet->table) {
@@ -168,11 +168,43 @@ static int spapr_tce_table_realize(DeviceState *dev)
     return 0;
 }
 
+void spapr_tce_set_need_vfio(sPAPRTCETable *tcet, bool need_vfio)
+{
+    size_t table_size = tcet->nb_table * sizeof(uint64_t);
+    void *newtable;
+
+    if (need_vfio == tcet->need_vfio) {
+        /* Nothing to do */
+        return;
+    }
+
+    if (!need_vfio) {
+        /* FIXME: We don't support transition back to KVM accelerated
+         * TCEs yet */
+        return;
+    }
+
+    tcet->need_vfio = true;
+
+    if (tcet->fd < 0) {
+        /* Table is already in userspace, nothing to be do */
+        return;
+    }
+
+    newtable = g_malloc(table_size);
+    memcpy(newtable, tcet->table, table_size);
+
+    kvmppc_remove_spapr_tce(tcet->table, tcet->fd, tcet->nb_table);
+
+    tcet->fd = -1;
+    tcet->table = newtable;
+}
+
 sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
                                    uint64_t bus_offset,
                                    uint32_t page_shift,
                                    uint32_t nb_table,
-                                   bool vfio_accel)
+                                   bool need_vfio)
 {
     sPAPRTCETable *tcet;
     char tmp[64];
@@ -192,7 +224,7 @@ sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
     tcet->bus_offset = bus_offset;
     tcet->page_shift = page_shift;
     tcet->nb_table = nb_table;
-    tcet->vfio_accel = vfio_accel;
+    tcet->need_vfio = need_vfio;
 
     snprintf(tmp, sizeof(tmp), "tce-table-%x", liobn);
     object_property_add_child(OBJECT(owner), tmp, OBJECT(tcet), NULL);
diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c
index 119fa5e46..55fa8db9e 100644
--- a/hw/ppc/spapr_pci.c
+++ b/hw/ppc/spapr_pci.c
@@ -140,7 +140,7 @@ static void rtas_ibm_read_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         return;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     size = rtas_ld(args, 3);
     addr = rtas_ld(args, 0);
 
@@ -206,7 +206,7 @@ static void rtas_ibm_write_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         return;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     val = rtas_ld(args, 4);
     size = rtas_ld(args, 3);
     addr = rtas_ld(args, 0);
@@ -269,7 +269,7 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                 target_ulong rets)
 {
     uint32_t config_addr = rtas_ld(args, 0);
-    uint64_t buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    uint64_t buid = rtas_ldq(args, 1);
     unsigned int func = rtas_ld(args, 3);
     unsigned int req_num = rtas_ld(args, 4); /* 0 == remove all */
     unsigned int seq_num = rtas_ld(args, 5);
@@ -375,7 +375,9 @@ out:
     rtas_st(rets, 0, RTAS_OUT_SUCCESS);
     rtas_st(rets, 1, req_num);
     rtas_st(rets, 2, ++seq_num);
-    rtas_st(rets, 3, ret_intr_type);
+    if (nret > 3) {
+        rtas_st(rets, 3, ret_intr_type);
+    }
 
     trace_spapr_pci_rtas_ibm_change_msi(config_addr, func, req_num, irq);
 }
@@ -389,7 +391,7 @@ static void rtas_ibm_query_interrupt_source_number(PowerPCCPU *cpu,
                                                    target_ulong rets)
 {
     uint32_t config_addr = rtas_ld(args, 0);
-    uint64_t buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    uint64_t buid = rtas_ldq(args, 1);
     unsigned int intr_src_num = -1, ioa_intr_num = rtas_ld(args, 3);
     sPAPRPHBState *phb = NULL;
     PCIDevice *pdev = NULL;
@@ -429,7 +431,6 @@ static void rtas_ibm_set_eeh_option(PowerPCCPU *cpu,
 {
     sPAPRPHBState *sphb;
     sPAPRPHBClass *spc;
-    PCIDevice *pdev;
     uint32_t addr, option;
     uint64_t buid;
     int ret;
@@ -438,7 +439,7 @@ static void rtas_ibm_set_eeh_option(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     addr = rtas_ld(args, 0);
     option = rtas_ld(args, 3);
 
@@ -447,12 +448,6 @@ static void rtas_ibm_set_eeh_option(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    pdev = pci_find_device(PCI_HOST_BRIDGE(sphb)->bus,
-                           (addr >> 16) & 0xFF, (addr >> 8) & 0xFF);
-    if (!pdev || !object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
-        goto param_error_exit;
-    }
-
     spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
     if (!spc->eeh_set_option) {
         goto param_error_exit;
@@ -482,7 +477,7 @@ static void rtas_ibm_get_config_addr_info2(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
@@ -537,7 +532,7 @@ static void rtas_ibm_read_slot_reset_state2(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
@@ -582,7 +577,7 @@ static void rtas_ibm_set_slot_reset(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     option = rtas_ld(args, 3);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
@@ -617,7 +612,7 @@ static void rtas_ibm_configure_pe(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
@@ -652,7 +647,7 @@ static void rtas_ibm_slot_error_detail(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
@@ -1088,6 +1083,12 @@ static void spapr_phb_add_pci_device(sPAPRDRConnector *drc,
     void *fdt = NULL;
     int fdt_start_offset = 0, fdt_size;
 
+    if (object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+        sPAPRTCETable *tcet = spapr_tce_find_by_liobn(phb->dma_liobn);
+
+        spapr_tce_set_need_vfio(tcet, true);
+    }
+
     if (dev->hotplugged) {
         fdt = create_device_tree(&fdt_size);
         fdt_start_offset = spapr_create_pci_child_dt(phb, pdev, fdt, 0);
@@ -1185,7 +1186,7 @@ static void spapr_phb_hot_plug_child(HotplugHandler *plug_handler,
         return;
     }
     if (plugged_dev->hotplugged) {
-        spapr_hotplug_req_add_event(drc);
+        spapr_hotplug_req_add_by_index(drc);
     }
 }
 
@@ -1213,7 +1214,7 @@ static void spapr_phb_hot_unplug_child(HotplugHandler *plug_handler,
             error_propagate(errp, local_err);
             return;
         }
-        spapr_hotplug_req_remove_event(drc);
+        spapr_hotplug_req_remove_by_index(drc);
     }
 }
 
@@ -1392,7 +1393,7 @@ static void spapr_phb_finish_realize(sPAPRPHBState *sphb, Error **errp)
     sPAPRTCETable *tcet;
     uint32_t nb_table;
 
-    nb_table = SPAPR_PCI_DMA32_SIZE >> SPAPR_TCE_PAGE_SHIFT;
+    nb_table = sphb->dma_win_size >> SPAPR_TCE_PAGE_SHIFT;
     tcet = spapr_tce_new_table(DEVICE(sphb), sphb->dma_liobn,
                                0, SPAPR_TCE_PAGE_SHIFT, nb_table, false);
     if (!tcet) {
@@ -1402,7 +1403,7 @@ static void spapr_phb_finish_realize(sPAPRPHBState *sphb, Error **errp)
     }
 
     /* Register default 32bit DMA window */
-    memory_region_add_subregion(&sphb->iommu_root, 0,
+    memory_region_add_subregion(&sphb->iommu_root, sphb->dma_win_addr,
                                 spapr_tce_get_iommu(tcet));
 }
 
@@ -1435,6 +1436,9 @@ static Property spapr_phb_properties[] = {
                        SPAPR_PCI_IO_WIN_SIZE),
     DEFINE_PROP_BOOL("dynamic-reconfiguration", sPAPRPHBState, dr_enabled,
                      true),
+    /* Default DMA window is 0..1GB */
+    DEFINE_PROP_UINT64("dma_win_addr", sPAPRPHBState, dma_win_addr, 0),
+    DEFINE_PROP_UINT64("dma_win_size", sPAPRPHBState, dma_win_size, 0x40000000),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1468,10 +1472,8 @@ static void spapr_pci_pre_save(void *opaque)
     gpointer key, value;
     int i;
 
-    if (sphb->msi_devs) {
-        g_free(sphb->msi_devs);
-        sphb->msi_devs = NULL;
-    }
+    g_free(sphb->msi_devs);
+    sphb->msi_devs = NULL;
     sphb->msi_devs_num = g_hash_table_size(sphb->msi);
     if (!sphb->msi_devs_num) {
         return;
@@ -1498,10 +1500,8 @@ static int spapr_pci_post_load(void *opaque, int version_id)
                          sizeof(sphb->msi_devs[i].value));
         g_hash_table_insert(sphb->msi, key, value);
     }
-    if (sphb->msi_devs) {
-        g_free(sphb->msi_devs);
-        sphb->msi_devs = NULL;
-    }
+    g_free(sphb->msi_devs);
+    sphb->msi_devs = NULL;
     sphb->msi_devs_num = 0;
 
     return 0;
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index cca45ed31..a61b41813 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -117,7 +117,7 @@ static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
         phb = PCI_HOST_BRIDGE(sphb);
         pdev = pci_find_device(phb->bus,
                                (addr >> 16) & 0xFF, (addr >> 8) & 0xFF);
-        if (!pdev) {
+        if (!pdev || !object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
             return RTAS_OUT_PARAM_ERROR;
         }
 
diff --git a/hw/ppc/spapr_rng.c b/hw/ppc/spapr_rng.c
new file mode 100644
index 000000000..ed43d5e04
--- /dev/null
+++ b/hw/ppc/spapr_rng.c
@@ -0,0 +1,186 @@
+/*
+ * QEMU sPAPR random number generator "device" for H_RANDOM hypercall
+ *
+ * Copyright 2015 Thomas Huth, Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/device_tree.h"
+#include "sysemu/rng.h"
+#include "hw/ppc/spapr.h"
+#include "kvm_ppc.h"
+
+#define SPAPR_RNG(obj) \
+    OBJECT_CHECK(sPAPRRngState, (obj), TYPE_SPAPR_RNG)
+
+struct sPAPRRngState {
+    /*< private >*/
+    DeviceState ds;
+    RngBackend *backend;
+    bool use_kvm;
+};
+typedef struct sPAPRRngState sPAPRRngState;
+
+struct HRandomData {
+    QemuSemaphore sem;
+    union {
+        uint64_t v64;
+        uint8_t v8[8];
+    } val;
+    int received;
+};
+typedef struct HRandomData HRandomData;
+
+/* Callback function for the RngBackend */
+static void random_recv(void *dest, const void *src, size_t size)
+{
+    HRandomData *hrdp = dest;
+
+    if (src && size > 0) {
+        assert(size + hrdp->received <= sizeof(hrdp->val.v8));
+        memcpy(&hrdp->val.v8[hrdp->received], src, size);
+        hrdp->received += size;
+    }
+
+    qemu_sem_post(&hrdp->sem);
+}
+
+/* Handler for the H_RANDOM hypercall */
+static target_ulong h_random(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                             target_ulong opcode, target_ulong *args)
+{
+    sPAPRRngState *rngstate;
+    HRandomData hrdata;
+
+    rngstate = SPAPR_RNG(object_resolve_path_type("", TYPE_SPAPR_RNG, NULL));
+
+    if (!rngstate || !rngstate->backend) {
+        return H_HARDWARE;
+    }
+
+    qemu_sem_init(&hrdata.sem, 0);
+    hrdata.val.v64 = 0;
+    hrdata.received = 0;
+
+    qemu_mutex_unlock_iothread();
+    while (hrdata.received < 8) {
+        rng_backend_request_entropy(rngstate->backend, 8 - hrdata.received,
+                                    random_recv, &hrdata);
+        qemu_sem_wait(&hrdata.sem);
+    }
+    qemu_mutex_lock_iothread();
+
+    qemu_sem_destroy(&hrdata.sem);
+    args[0] = hrdata.val.v64;
+
+    return H_SUCCESS;
+}
+
+static void spapr_rng_instance_init(Object *obj)
+{
+    sPAPRRngState *rngstate = SPAPR_RNG(obj);
+
+    if (object_resolve_path_type("", TYPE_SPAPR_RNG, NULL) != NULL) {
+        error_report("spapr-rng can not be instantiated twice!");
+        return;
+    }
+
+    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
+                             (Object **)&rngstate->backend,
+                             object_property_allow_set_link,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
+    object_property_set_description(obj, "rng",
+                                    "ID of the random number generator backend",
+                                    NULL);
+}
+
+static void spapr_rng_realize(DeviceState *dev, Error **errp)
+{
+
+    sPAPRRngState *rngstate = SPAPR_RNG(dev);
+
+    if (rngstate->use_kvm) {
+        if (kvmppc_enable_hwrng() == 0) {
+            return;
+        }
+        /*
+         * If user specified both, use-kvm and a backend, we fall back to
+         * the backend now. If not, provide an appropriate error message.
+         */
+        if (!rngstate->backend) {
+            error_setg(errp, "Could not initialize in-kernel H_RANDOM call!");
+            return;
+        }
+    }
+
+    if (rngstate->backend) {
+        spapr_register_hypercall(H_RANDOM, h_random);
+    } else {
+        error_setg(errp, "spapr-rng needs an RNG backend!");
+    }
+}
+
+int spapr_rng_populate_dt(void *fdt)
+{
+    int node;
+    int ret;
+
+    node = qemu_fdt_add_subnode(fdt, "/ibm,platform-facilities");
+    if (node <= 0) {
+        return -1;
+    }
+    ret = fdt_setprop_string(fdt, node, "device_type",
+                             "ibm,platform-facilities");
+    ret |= fdt_setprop_cell(fdt, node, "#address-cells", 0x1);
+    ret |= fdt_setprop_cell(fdt, node, "#size-cells", 0x0);
+
+    node = fdt_add_subnode(fdt, node, "ibm,random-v1");
+    if (node <= 0) {
+        return -1;
+    }
+    ret |= fdt_setprop_string(fdt, node, "compatible", "ibm,random");
+
+    return ret ? -1 : 0;
+}
+
+static Property spapr_rng_properties[] = {
+    DEFINE_PROP_BOOL("use-kvm", sPAPRRngState, use_kvm, false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void spapr_rng_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = spapr_rng_realize;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->props = spapr_rng_properties;
+}
+
+static const TypeInfo spapr_rng_info = {
+    .name          = TYPE_SPAPR_RNG,
+    .parent        = TYPE_DEVICE,
+    .instance_size = sizeof(sPAPRRngState),
+    .instance_init = spapr_rng_instance_init,
+    .class_init    = spapr_rng_class_init,
+};
+
+static void spapr_rng_register_type(void)
+{
+    type_register_static(&spapr_rng_info);
+}
+type_init(spapr_rng_register_type)
diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c
index 2986f94f0..34b12a3b9 100644
--- a/hw/ppc/spapr_rtas.c
+++ b/hw/ppc/spapr_rtas.c
@@ -34,6 +34,7 @@
 #include "hw/ppc/spapr.h"
 #include "hw/ppc/spapr_vio.h"
 #include "qapi-event.h"
+#include "hw/boards.h"
 
 #include <libfdt.h>
 #include "hw/ppc/spapr_drc.h"
@@ -214,7 +215,7 @@ static void rtas_stop_self(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     CPUPPCState *env = &cpu->env;
 
     cs->halted = 1;
-    cpu_exit(cs);
+    qemu_cpu_kick(cs);
     /*
      * While stopping a CPU, the guest calls H_CPPR which
      * effectively disables interrupts on XICS level.
@@ -240,8 +241,14 @@ static void rtas_ibm_get_system_parameter(PowerPCCPU *cpu,
 
     switch (parameter) {
     case RTAS_SYSPARM_SPLPAR_CHARACTERISTICS: {
-        char *param_val = g_strdup_printf("MaxEntCap=%d,MaxPlatProcs=%d",
-                                          max_cpus, smp_cpus);
+        char *param_val = g_strdup_printf("MaxEntCap=%d,"
+                                          "DesMem=%llu,"
+                                          "DesProcs=%d,"
+                                          "MaxPlatProcs=%d",
+                                          max_cpus,
+                                          current_machine->ram_size / M_BYTE,
+                                          smp_cpus,
+                                          max_cpus);
         rtas_st_buffer(buffer, length, (uint8_t *)param_val, strlen(param_val));
         g_free(param_val);
         break;
@@ -365,12 +372,13 @@ static void rtas_set_indicator(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     uint32_t sensor_type;
     uint32_t sensor_index;
     uint32_t sensor_state;
+    uint32_t ret = RTAS_OUT_SUCCESS;
     sPAPRDRConnector *drc;
     sPAPRDRConnectorClass *drck;
 
     if (nargs != 3 || nret != 1) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
 
     sensor_type = rtas_ld(args, 0);
@@ -386,8 +394,8 @@ static void rtas_set_indicator(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     if (!drc) {
         DPRINTF("rtas_set_indicator: invalid sensor/DRC index: %xh\n",
                 sensor_index);
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
     drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
 
@@ -406,19 +414,20 @@ static void rtas_set_indicator(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                 spapr_ccs_remove(spapr, ccs);
             }
         }
-        drck->set_isolation_state(drc, sensor_state);
+        ret = drck->set_isolation_state(drc, sensor_state);
         break;
     case RTAS_SENSOR_TYPE_DR:
-        drck->set_indicator_state(drc, sensor_state);
+        ret = drck->set_indicator_state(drc, sensor_state);
         break;
     case RTAS_SENSOR_TYPE_ALLOCATION_STATE:
-        drck->set_allocation_state(drc, sensor_state);
+        ret = drck->set_allocation_state(drc, sensor_state);
         break;
     default:
         goto out_unimplemented;
     }
 
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+out:
+    rtas_st(rets, 0, ret);
     return;
 
 out_unimplemented:
@@ -435,13 +444,14 @@ static void rtas_get_sensor_state(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 {
     uint32_t sensor_type;
     uint32_t sensor_index;
+    uint32_t sensor_state = 0;
     sPAPRDRConnector *drc;
     sPAPRDRConnectorClass *drck;
-    uint32_t entity_sense;
+    uint32_t ret = RTAS_OUT_SUCCESS;
 
     if (nargs != 2 || nret != 2) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
 
     sensor_type = rtas_ld(args, 0);
@@ -451,22 +461,23 @@ static void rtas_get_sensor_state(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         /* currently only DR-related sensors are implemented */
         DPRINTF("rtas_get_sensor_state: sensor/indicator not implemented: %d\n",
                 sensor_type);
-        rtas_st(rets, 0, RTAS_OUT_NOT_SUPPORTED);
-        return;
+        ret = RTAS_OUT_NOT_SUPPORTED;
+        goto out;
     }
 
     drc = spapr_dr_connector_by_index(sensor_index);
     if (!drc) {
         DPRINTF("rtas_get_sensor_state: invalid sensor/DRC index: %xh\n",
                 sensor_index);
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
     drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
-    entity_sense = drck->entity_sense(drc);
+    ret = drck->entity_sense(drc, &sensor_state);
 
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-    rtas_st(rets, 1, entity_sense);
+out:
+    rtas_st(rets, 0, ret);
+    rtas_st(rets, 1, sensor_state);
 }
 
 /* configure-connector work area offsets, int32_t units for field
@@ -515,6 +526,12 @@ static void rtas_ibm_configure_connector(PowerPCCPU *cpu,
 
     drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
     fdt = drck->get_fdt(drc, NULL);
+    if (!fdt) {
+        DPRINTF("rtas_ibm_configure_connector: Missing FDT for DRC index: %xh\n",
+                drc_index);
+        rc = SPAPR_DR_CC_RESPONSE_NOT_CONFIGURABLE;
+        goto out;
+    }
 
     ccs = spapr_ccs_find(spapr, drc_index);
     if (!ccs) {
diff --git a/hw/ppc/virtex_ml507.c b/hw/ppc/virtex_ml507.c
index de86f7c64..c2b5e441a 100644
--- a/hw/ppc/virtex_ml507.c
+++ b/hw/ppc/virtex_ml507.c
@@ -257,7 +257,7 @@ static void virtex_init(MachineState *machine)
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
-                               &entry, &low, &high, 1, ELF_MACHINE, 0);
+                               &entry, &low, &high, 1, PPC_ELF_MACHINE, 0);
         boot_info.bootstrap_pc = entry & 0x00ffffff;
 
         if (kernel_size < 0) {
@@ -297,15 +297,10 @@ static void virtex_init(MachineState *machine)
     env->load_info = &boot_info;
 }
 
-static QEMUMachine virtex_machine = {
-    .name = "virtex-ml507",
-    .desc = "Xilinx Virtex ML507 reference design",
-    .init = virtex_init,
-};
-
-static void virtex_machine_init(void)
+static void virtex_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&virtex_machine);
+    mc->desc = "Xilinx Virtex ML507 reference design";
+    mc->init = virtex_init;
 }
 
-machine_init(virtex_machine_init);
+DEFINE_MACHINE("virtex-ml507", virtex_machine_init)
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 27cd75a93..527d75400 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -9,3 +9,5 @@ obj-y += css.o
 obj-y += s390-virtio-ccw.o
 obj-y += virtio-ccw.o
 obj-y += s390-pci-bus.o s390-pci-inst.o
+obj-y += s390-skeys.o
+obj-$(CONFIG_KVM) += s390-skeys-kvm.o
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index 97d93d56d..c6ca8bec4 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -261,6 +261,9 @@ static CCW1 copy_ccw_from_guest(hwaddr addr, bool fmt1)
         ret.flags = tmp0.flags;
         ret.count = be16_to_cpu(tmp0.count);
         ret.cda = be16_to_cpu(tmp0.cda1) | (tmp0.cda0 << 16);
+        if ((ret.cmd_code & 0x0f) == CCW_CMD_TIC) {
+            ret.cmd_code &= 0x0f;
+        }
     }
     return ret;
 }
@@ -287,6 +290,10 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
         ((ccw.cmd_code & 0xf0) != 0)) {
         return -EINVAL;
     }
+    if (!sch->ccw_fmt_1 && (ccw.count == 0) &&
+        (ccw.cmd_code != CCW_CMD_TIC)) {
+        return -EINVAL;
+    }
 
     if (ccw.flags & CCW_FLAG_SUSPEND) {
         return -EINPROGRESS;
@@ -694,7 +701,7 @@ int css_do_csch(SubchDev *sch)
 
     /* Trigger the clear function. */
     s->ctrl &= ~(SCSW_CTRL_MASK_FCTL | SCSW_CTRL_MASK_ACTL);
-    s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_FCTL_CLEAR_FUNC;
+    s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_ACTL_CLEAR_PEND;
 
     do_subchannel_work(sch, NULL);
     ret = 0;
@@ -885,8 +892,14 @@ int css_do_tsch_get_irb(SubchDev *sch, IRB *target_irb, int *irb_len)
         /* If a unit check is pending, copy sense data. */
         if ((s->dstat & SCSW_DSTAT_UNIT_CHECK) &&
             (p->chars & PMCW_CHARS_MASK_CSENSE)) {
+            int i;
+
             irb.scsw.flags |= SCSW_FLAGS_MASK_ESWF | SCSW_FLAGS_MASK_ECTL;
+            /* Attention: sense_data is already BE! */
             memcpy(irb.ecw, sch->sense_data, sizeof(sch->sense_data));
+            for (i = 0; i < ARRAY_SIZE(irb.ecw); i++) {
+                irb.ecw[i] = be32_to_cpu(irb.ecw[i]);
+            }
             irb.esw[1] = 0x01000000 | (sizeof(sch->sense_data) << 8);
         }
     }
diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c
index 0c700effb..907b48560 100644
--- a/hw/s390x/event-facility.c
+++ b/hw/s390x/event-facility.c
@@ -31,8 +31,6 @@ struct SCLPEventFacility {
     unsigned int receive_mask;
 };
 
-static SCLPEvent cpu_hotplug;
-
 /* return true if any child has event pending set */
 static bool event_pending(SCLPEventFacility *ef)
 {
@@ -240,12 +238,13 @@ static void read_event_data(SCLPEventFacility *ef, SCCB *sccb)
         sclp_active_selection_mask = sclp_cp_receive_mask;
         break;
     case SCLP_SELECTIVE_READ:
-        if (!(sclp_cp_receive_mask & be32_to_cpu(red->mask))) {
+        sclp_active_selection_mask = be32_to_cpu(red->mask);
+        if (!sclp_cp_receive_mask ||
+            (sclp_active_selection_mask & ~sclp_cp_receive_mask)) {
             sccb->h.response_code =
                     cpu_to_be16(SCLP_RC_INVALID_SELECTION_MASK);
             goto out;
         }
-        sclp_active_selection_mask = be32_to_cpu(red->mask);
         break;
     default:
         sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_FUNCTION);
@@ -286,8 +285,26 @@ out:
 
 #define TYPE_SCLP_EVENTS_BUS "s390-sclp-events-bus"
 
+static void sclp_events_bus_realize(BusState *bus, Error **errp)
+{
+    BusChild *kid;
+
+    /* TODO: recursive realization has to be done in common code */
+    QTAILQ_FOREACH(kid, &bus->children, sibling) {
+        DeviceState *dev = kid->child;
+
+        object_property_set_bool(OBJECT(dev), true, "realized", errp);
+        if (*errp) {
+            return;
+        }
+    }
+}
+
 static void sclp_events_bus_class_init(ObjectClass *klass, void *data)
 {
+    BusClass *bc = BUS_CLASS(klass);
+
+    bc->realize = sclp_events_bus_realize;
 }
 
 static const TypeInfo sclp_events_bus_info = {
@@ -324,26 +341,26 @@ static const VMStateDescription vmstate_event_facility = {
      }
 };
 
-static int init_event_facility(SCLPEventFacility *event_facility)
+static void init_event_facility(Object *obj)
 {
-    DeviceState *sdev = DEVICE(event_facility);
-    DeviceState *quiesce;
+    SCLPEventFacility *event_facility = EVENT_FACILITY(obj);
+    DeviceState *sdev = DEVICE(obj);
+    Object *new;
 
     /* Spawn a new bus for SCLP events */
     qbus_create_inplace(&event_facility->sbus, sizeof(event_facility->sbus),
                         TYPE_SCLP_EVENTS_BUS, sdev, NULL);
 
-    quiesce = qdev_create(&event_facility->sbus.qbus, "sclpquiesce");
-    if (!quiesce) {
-        return -1;
-    }
-    qdev_init_nofail(quiesce);
-
-    object_initialize(&cpu_hotplug, sizeof(cpu_hotplug), TYPE_SCLP_CPU_HOTPLUG);
-    qdev_set_parent_bus(DEVICE(&cpu_hotplug), BUS(&event_facility->sbus));
-    object_property_set_bool(OBJECT(&cpu_hotplug), true, "realized", NULL);
+    new = object_new(TYPE_SCLP_QUIESCE);
+    object_property_add_child(obj, TYPE_SCLP_QUIESCE, new, NULL);
+    object_unref(new);
+    qdev_set_parent_bus(DEVICE(new), &event_facility->sbus.qbus);
 
-    return 0;
+    new = object_new(TYPE_SCLP_CPU_HOTPLUG);
+    object_property_add_child(obj, TYPE_SCLP_CPU_HOTPLUG, new, NULL);
+    object_unref(new);
+    qdev_set_parent_bus(DEVICE(new), &event_facility->sbus.qbus);
+    /* the facility will automatically realize the devices via the bus */
 }
 
 static void reset_event_facility(DeviceState *dev)
@@ -362,7 +379,6 @@ static void init_event_facility_class(ObjectClass *klass, void *data)
     dc->reset = reset_event_facility;
     dc->vmsd = &vmstate_event_facility;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
-    k->init = init_event_facility;
     k->command_handler = command_handler;
     k->event_pending = event_pending;
 }
@@ -370,6 +386,7 @@ static void init_event_facility_class(ObjectClass *klass, void *data)
 static const TypeInfo sclp_event_facility_info = {
     .name          = TYPE_SCLP_EVENT_FACILITY,
     .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_init = init_event_facility,
     .instance_size = sizeof(SCLPEventFacility),
     .class_init    = init_event_facility_class,
     .class_size    = sizeof(SCLPEventFacilityClass),
diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
index 2e0a8b6e0..b91fcc6e7 100644
--- a/hw/s390x/ipl.c
+++ b/hw/s390x/ipl.c
@@ -15,7 +15,6 @@
 #include "cpu.h"
 #include "elf.h"
 #include "hw/loader.h"
-#include "hw/sysbus.h"
 #include "hw/s390x/virtio-ccw.h"
 #include "hw/s390x/css.h"
 #include "ipl.h"
@@ -29,44 +28,6 @@
 #define ZIPL_IMAGE_START                0x009000UL
 #define IPL_PSW_MASK                    (PSW_MASK_32 | PSW_MASK_64)
 
-#define TYPE_S390_IPL "s390-ipl"
-#define S390_IPL(obj) \
-    OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
-#if 0
-#define S390_IPL_CLASS(klass) \
-    OBJECT_CLASS_CHECK(S390IPLState, (klass), TYPE_S390_IPL)
-#define S390_IPL_GET_CLASS(obj) \
-    OBJECT_GET_CLASS(S390IPLState, (obj), TYPE_S390_IPL)
-#endif
-
-typedef struct S390IPLClass {
-    /*< private >*/
-    SysBusDeviceClass parent_class;
-    /*< public >*/
-
-    void (*parent_reset) (SysBusDevice *dev);
-} S390IPLClass;
-
-typedef struct S390IPLState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    uint64_t start_addr;
-    uint64_t bios_start_addr;
-    bool enforce_bios;
-    IplParameterBlock iplb;
-    bool iplb_valid;
-    bool reipl_requested;
-
-    /*< public >*/
-    char *kernel;
-    char *initrd;
-    char *cmdline;
-    char *firmware;
-    uint8_t cssid;
-    uint8_t ssid;
-    uint16_t devno;
-} S390IPLState;
-
 static const VMStateDescription vmstate_iplb = {
     .name = "ipl/iplb",
     .version_id = 0,
@@ -95,6 +56,11 @@ static const VMStateDescription vmstate_ipl = {
      }
 };
 
+static S390IPLState *get_ipl_device(void)
+{
+    return S390_IPL(object_resolve_path_type("", TYPE_S390_IPL, NULL));
+}
+
 static uint64_t bios_translate_addr(void *opaque, uint64_t srcaddr)
 {
     uint64_t dstaddr = *(uint64_t *) opaque;
@@ -105,11 +71,12 @@ static uint64_t bios_translate_addr(void *opaque, uint64_t srcaddr)
     return srcaddr + dstaddr;
 }
 
-static int s390_ipl_init(SysBusDevice *dev)
+static void s390_ipl_realize(DeviceState *dev, Error **errp)
 {
     S390IPLState *ipl = S390_IPL(dev);
     uint64_t pentry = KERN_IMAGE_START;
     int kernel_size;
+    Error *l_err = NULL;
 
     int bios_size;
     char *bios_filename;
@@ -127,12 +94,13 @@ static int s390_ipl_init(SysBusDevice *dev)
 
         bios_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (bios_filename == NULL) {
-            hw_error("could not find stage1 bootloader\n");
+            error_setg(&l_err, "could not find stage1 bootloader\n");
+            goto error;
         }
 
         bios_size = load_elf(bios_filename, bios_translate_addr, &fwbase,
                              &ipl->bios_start_addr, NULL, NULL, 1,
-                             ELF_MACHINE, 0);
+                             EM_S390, 0);
         if (bios_size > 0) {
             /* Adjust ELF start address to final location */
             ipl->bios_start_addr += fwbase;
@@ -145,7 +113,8 @@ static int s390_ipl_init(SysBusDevice *dev)
         g_free(bios_filename);
 
         if (bios_size == -1) {
-            hw_error("could not load bootloader '%s'\n", bios_name);
+            error_setg(&l_err, "could not load bootloader '%s'\n", bios_name);
+            goto error;
         }
 
         /* default boot target is the bios */
@@ -154,13 +123,13 @@ static int s390_ipl_init(SysBusDevice *dev)
 
     if (ipl->kernel) {
         kernel_size = load_elf(ipl->kernel, NULL, NULL, &pentry, NULL,
-                               NULL, 1, ELF_MACHINE, 0);
+                               NULL, 1, EM_S390, 0);
         if (kernel_size < 0) {
             kernel_size = load_image_targphys(ipl->kernel, 0, ram_size);
         }
         if (kernel_size < 0) {
-            fprintf(stderr, "could not load kernel '%s'\n", ipl->kernel);
-            return -1;
+            error_setg(&l_err, "could not load kernel '%s'\n", ipl->kernel);
+            goto error;
         }
         /*
          * Is it a Linux kernel (starting at 0x10000)? If yes, we fill in the
@@ -187,9 +156,8 @@ static int s390_ipl_init(SysBusDevice *dev)
             initrd_size = load_image_targphys(ipl->initrd, initrd_offset,
                                               ram_size - initrd_offset);
             if (initrd_size == -1) {
-                fprintf(stderr, "qemu: could not load initrd '%s'\n",
-                        ipl->initrd);
-                exit(1);
+                error_setg(&l_err, "could not load initrd '%s'\n", ipl->initrd);
+                goto error;
             }
 
             /*
@@ -200,7 +168,9 @@ static int s390_ipl_init(SysBusDevice *dev)
             stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
         }
     }
-    return 0;
+    qemu_register_reset(qdev_reset_all_fn, dev);
+error:
+    error_propagate(errp, l_err);
 }
 
 static Property s390_ipl_properties[] = {
@@ -218,7 +188,7 @@ static Property s390_ipl_properties[] = {
  * - -1 if no valid boot device was found
  * - ccw id of the boot device otherwise
  */
-static uint64_t s390_update_iplstate(CPUS390XState *env, S390IPLState *ipl)
+static uint64_t s390_update_iplstate(S390IPLState *ipl)
 {
     DeviceState *dev_st;
 
@@ -251,25 +221,19 @@ out:
     return (uint32_t) (ipl->cssid << 24 | ipl->ssid << 16 | ipl->devno);
 }
 
-int s390_ipl_update_diag308(IplParameterBlock *iplb)
+void s390_ipl_update_diag308(IplParameterBlock *iplb)
 {
-    S390IPLState *ipl;
+    S390IPLState *ipl = get_ipl_device();
 
-    ipl = S390_IPL(object_resolve_path(TYPE_S390_IPL, NULL));
-    if (ipl) {
-        ipl->iplb = *iplb;
-        ipl->iplb_valid = true;
-        return 0;
-    }
-    return -1;
+    ipl->iplb = *iplb;
+    ipl->iplb_valid = true;
 }
 
 IplParameterBlock *s390_ipl_get_iplb(void)
 {
-    S390IPLState *ipl;
+    S390IPLState *ipl = get_ipl_device();
 
-    ipl = S390_IPL(object_resolve_path(TYPE_S390_IPL, NULL));
-    if (!ipl || !ipl->iplb_valid) {
+    if (!ipl->iplb_valid) {
         return NULL;
     }
     return &ipl->iplb;
@@ -277,41 +241,40 @@ IplParameterBlock *s390_ipl_get_iplb(void)
 
 void s390_reipl_request(void)
 {
-    S390IPLState *ipl;
+    S390IPLState *ipl = get_ipl_device();
 
-    ipl = S390_IPL(object_resolve_path(TYPE_S390_IPL, NULL));
     ipl->reipl_requested = true;
     qemu_system_reset_request();
 }
 
+void s390_ipl_prepare_cpu(S390CPU *cpu)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    cpu->env.psw.addr = ipl->start_addr;
+    cpu->env.psw.mask = IPL_PSW_MASK;
+
+    if (!ipl->kernel || ipl->iplb_valid) {
+        cpu->env.psw.addr = ipl->bios_start_addr;
+        cpu->env.regs[7] = s390_update_iplstate(ipl);
+    }
+}
+
 static void s390_ipl_reset(DeviceState *dev)
 {
     S390IPLState *ipl = S390_IPL(dev);
-    S390CPU *cpu = S390_CPU(qemu_get_cpu(0));
-    CPUS390XState *env = &cpu->env;
-
-    env->psw.addr = ipl->start_addr;
-    env->psw.mask = IPL_PSW_MASK;
 
     if (!ipl->reipl_requested) {
         ipl->iplb_valid = false;
     }
     ipl->reipl_requested = false;
-
-    if (!ipl->kernel || ipl->iplb_valid) {
-        env->psw.addr = ipl->bios_start_addr;
-        env->regs[7] = s390_update_iplstate(env, ipl);
-    }
-
-    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
 }
 
 static void s390_ipl_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = s390_ipl_init;
+    dc->realize = s390_ipl_realize;
     dc->props = s390_ipl_properties;
     dc->reset = s390_ipl_reset;
     dc->vmsd = &vmstate_ipl;
@@ -320,8 +283,8 @@ static void s390_ipl_class_init(ObjectClass *klass, void *data)
 
 static const TypeInfo s390_ipl_info = {
     .class_init = s390_ipl_class_init,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .name  = "s390-ipl",
+    .parent = TYPE_DEVICE,
+    .name  = TYPE_S390_IPL,
     .instance_size  = sizeof(S390IPLState),
 };
 
diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
index 70497bc65..6b48ed7b9 100644
--- a/hw/s390x/ipl.h
+++ b/hw/s390x/ipl.h
@@ -12,14 +12,42 @@
 #ifndef HW_S390_IPL_H
 #define HW_S390_IPL_H
 
+#include "hw/qdev.h"
+#include "cpu.h"
+
 typedef struct IplParameterBlock {
       uint8_t  reserved1[110];
       uint16_t devno;
       uint8_t  reserved2[88];
 } IplParameterBlock;
 
-int s390_ipl_update_diag308(IplParameterBlock *iplb);
+void s390_ipl_update_diag308(IplParameterBlock *iplb);
+void s390_ipl_prepare_cpu(S390CPU *cpu);
 IplParameterBlock *s390_ipl_get_iplb(void);
 void s390_reipl_request(void);
 
+#define TYPE_S390_IPL "s390-ipl"
+#define S390_IPL(obj) OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
+
+struct S390IPLState {
+    /*< private >*/
+    DeviceState parent_obj;
+    uint64_t start_addr;
+    uint64_t bios_start_addr;
+    bool enforce_bios;
+    IplParameterBlock iplb;
+    bool iplb_valid;
+    bool reipl_requested;
+
+    /*< public >*/
+    char *kernel;
+    char *initrd;
+    char *cmdline;
+    char *firmware;
+    uint8_t cssid;
+    uint8_t ssid;
+    uint16_t devno;
+};
+typedef struct S390IPLState S390IPLState;
+
 #endif
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 560b66a50..98c726cfc 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -308,9 +308,8 @@ static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
 {
     uint64_t pte;
     uint32_t flags;
-    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, mr);
-    S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)
-                                           ->qbus.parent);
+    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, iommu_mr);
+    S390pciState *s;
     IOMMUTLBEntry ret = {
         .target_as = &address_space_memory,
         .iova = 0,
@@ -319,8 +318,13 @@ static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
         .perm = IOMMU_NONE,
     };
 
+    if (!pbdev->configured || !pbdev->pdev) {
+        return ret;
+    }
+
     DPRINTF("iommu trans addr 0x%" PRIx64 "\n", addr);
 
+    s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)->qbus.parent);
     /* s390 does not have an APIC mapped to main storage so we use
      * a separate AddressSpace only for msix notifications
      */
@@ -450,14 +454,32 @@ static const MemoryRegionOps s390_msi_ctrl_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+void s390_pcihost_iommu_configure(S390PCIBusDevice *pbdev, bool enable)
+{
+    pbdev->configured = false;
+
+    if (enable) {
+        uint64_t size = pbdev->pal - pbdev->pba + 1;
+        memory_region_init_iommu(&pbdev->iommu_mr, OBJECT(&pbdev->mr),
+                                 &s390_iommu_ops, "iommu-s390", size);
+        memory_region_add_subregion(&pbdev->mr, pbdev->pba, &pbdev->iommu_mr);
+    } else {
+        memory_region_del_subregion(&pbdev->mr, &pbdev->iommu_mr);
+    }
+
+    pbdev->configured = true;
+}
+
 static void s390_pcihost_init_as(S390pciState *s)
 {
     int i;
+    S390PCIBusDevice *pbdev;
 
     for (i = 0; i < PCI_SLOT_MAX; i++) {
-        memory_region_init_iommu(&s->pbdev[i].mr, OBJECT(s),
-                                 &s390_iommu_ops, "iommu-s390", UINT64_MAX);
-        address_space_init(&s->pbdev[i].as, &s->pbdev[i].mr, "iommu-pci");
+        pbdev = &s->pbdev[i];
+        memory_region_init(&pbdev->mr, OBJECT(s),
+                           "iommu-root-s390", UINT64_MAX);
+        address_space_init(&pbdev->as, &pbdev->mr, "iommu-pci");
     }
 
     memory_region_init_io(&s->msix_notify_mr, OBJECT(s),
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 464a92eed..80345dacb 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -231,6 +231,7 @@ typedef struct S390PCIBusDevice {
     AdapterRoutes routes;
     AddressSpace as;
     MemoryRegion mr;
+    MemoryRegion iommu_mr;
 } S390PCIBusDevice;
 
 typedef struct S390pciState {
@@ -244,6 +245,7 @@ typedef struct S390pciState {
 int chsc_sei_nt2_get_event(void *res);
 int chsc_sei_nt2_have_event(void);
 void s390_pci_sclp_configure(int configure, SCCB *sccb);
+void s390_pcihost_iommu_configure(S390PCIBusDevice *pbdev, bool enable);
 S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh);
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid);
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index f9151a9af..8c1dc82b1 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -528,7 +528,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
         goto out;
     }
 
-    mr = pci_device_iommu_address_space(pbdev->pdev)->root;
+    mr = &pbdev->iommu_mr;
     while (start < end) {
         entry = mr->iommu_ops->translate(mr, start, 0);
 
@@ -689,6 +689,9 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
     pbdev->pba = pba;
     pbdev->pal = pal;
     pbdev->g_iota = g_iota;
+
+    s390_pcihost_iommu_configure(pbdev, true);
+
     return 0;
 }
 
@@ -697,6 +700,8 @@ static void dereg_ioat(S390PCIBusDevice *pbdev)
     pbdev->pba = 0;
     pbdev->pal = 0;
     pbdev->g_iota = 0;
+
+    s390_pcihost_iommu_configure(pbdev, false);
 }
 
 int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
diff --git a/hw/s390x/s390-skeys-kvm.c b/hw/s390x/s390-skeys-kvm.c
new file mode 100644
index 000000000..682949afb
--- /dev/null
+++ b/hw/s390x/s390-skeys-kvm.c
@@ -0,0 +1,75 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "hw/s390x/storage-keys.h"
+#include "sysemu/kvm.h"
+#include "qemu/error-report.h"
+
+static int kvm_s390_skeys_enabled(S390SKeysState *ss)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint8_t single_key;
+    int r;
+
+    r = skeyclass->get_skeys(ss, 0, 1, &single_key);
+    if (r != 0 && r != KVM_S390_GET_SKEYS_NONE) {
+        error_report("S390_GET_KEYS error %d\n", r);
+    }
+    return (r == 0);
+}
+
+static int kvm_s390_skeys_get(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    struct kvm_s390_skeys args = {
+        .start_gfn = start_gfn,
+        .count = count,
+        .skeydata_addr = (__u64)keys
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_GET_SKEYS, &args);
+}
+
+static int kvm_s390_skeys_set(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    struct kvm_s390_skeys args = {
+        .start_gfn = start_gfn,
+        .count = count,
+        .skeydata_addr = (__u64)keys
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_SET_SKEYS, &args);
+}
+
+static void kvm_s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_CLASS(oc);
+
+    skeyclass->skeys_enabled = kvm_s390_skeys_enabled;
+    skeyclass->get_skeys = kvm_s390_skeys_get;
+    skeyclass->set_skeys = kvm_s390_skeys_set;
+}
+
+static const TypeInfo kvm_s390_skeys_info = {
+    .name          = TYPE_KVM_S390_SKEYS,
+    .parent        = TYPE_S390_SKEYS,
+    .instance_size = sizeof(S390SKeysState),
+    .class_init    = kvm_s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+};
+
+static void kvm_s390_skeys_register_types(void)
+{
+    type_register_static(&kvm_s390_skeys_info);
+}
+
+type_init(kvm_s390_skeys_register_types)
diff --git a/hw/s390x/s390-skeys.c b/hw/s390x/s390-skeys.c
new file mode 100644
index 000000000..539ef6d3a
--- /dev/null
+++ b/hw/s390x/s390-skeys.c
@@ -0,0 +1,415 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "hw/boards.h"
+#include "qmp-commands.h"
+#include "migration/qemu-file.h"
+#include "hw/s390x/storage-keys.h"
+#include "qemu/error-report.h"
+
+#define S390_SKEYS_BUFFER_SIZE 131072  /* Room for 128k storage keys */
+#define S390_SKEYS_SAVE_FLAG_EOS 0x01
+#define S390_SKEYS_SAVE_FLAG_SKEYS 0x02
+#define S390_SKEYS_SAVE_FLAG_ERROR 0x04
+
+S390SKeysState *s390_get_skeys_device(void)
+{
+    S390SKeysState *ss;
+
+    ss = S390_SKEYS(object_resolve_path_type("", TYPE_S390_SKEYS, NULL));
+    assert(ss);
+    return ss;
+}
+
+void s390_skeys_init(void)
+{
+    Object *obj;
+
+    if (kvm_enabled()) {
+        obj = object_new(TYPE_KVM_S390_SKEYS);
+    } else {
+        obj = object_new(TYPE_QEMU_S390_SKEYS);
+    }
+    object_property_add_child(qdev_get_machine(), TYPE_S390_SKEYS,
+                              obj, NULL);
+    object_unref(obj);
+
+    qdev_init_nofail(DEVICE(obj));
+}
+
+static void write_keys(QEMUFile *f, uint8_t *keys, uint64_t startgfn,
+                       uint64_t count, Error **errp)
+{
+    uint64_t curpage = startgfn;
+    uint64_t maxpage = curpage + count - 1;
+    const char *fmt = "page=%03" PRIx64 ": key(%d) => ACC=%X, FP=%d, REF=%d,"
+                      " ch=%d, reserved=%d\n";
+    char buf[128];
+    int len;
+
+    for (; curpage <= maxpage; curpage++) {
+        uint8_t acc = (*keys & 0xF0) >> 4;
+        int fp =  (*keys & 0x08);
+        int ref = (*keys & 0x04);
+        int ch = (*keys & 0x02);
+        int res = (*keys & 0x01);
+
+        len = snprintf(buf, sizeof(buf), fmt, curpage,
+                       *keys, acc, fp, ref, ch, res);
+        assert(len < sizeof(buf));
+        qemu_put_buffer(f, (uint8_t *)buf, len);
+        keys++;
+    }
+}
+
+void hmp_info_skeys(Monitor *mon, const QDict *qdict)
+{
+    S390SKeysState *ss = s390_get_skeys_device();
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint8_t key;
+    int r;
+
+    /* Quick check to see if guest is using storage keys*/
+    if (!skeyclass->skeys_enabled(ss)) {
+        monitor_printf(mon, "Error: This guest is not using storage keys\n");
+        return;
+    }
+
+    r = skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
+    if (r < 0) {
+        monitor_printf(mon, "Error: %s\n", strerror(-r));
+        return;
+    }
+
+    monitor_printf(mon, "  key: 0x%X\n", key);
+}
+
+void hmp_dump_skeys(Monitor *mon, const QDict *qdict)
+{
+    const char *filename = qdict_get_str(qdict, "filename");
+    Error *err = NULL;
+
+    qmp_dump_skeys(filename, &err);
+    if (err) {
+        monitor_printf(mon, "%s\n", error_get_pretty(err));
+        error_free(err);
+    }
+}
+
+void qmp_dump_skeys(const char *filename, Error **errp)
+{
+    S390SKeysState *ss = s390_get_skeys_device();
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    const uint64_t total_count = ram_size / TARGET_PAGE_SIZE;
+    uint64_t handled_count = 0, cur_count;
+    Error *lerr = NULL;
+    vaddr cur_gfn = 0;
+    uint8_t *buf;
+    int ret;
+    QEMUFile *f;
+
+    /* Quick check to see if guest is using storage keys*/
+    if (!skeyclass->skeys_enabled(ss)) {
+        error_setg(errp, "This guest is not using storage keys - "
+                         "nothing to dump");
+        return;
+    }
+
+    f = qemu_fopen(filename, "wb");
+    if (!f) {
+        error_setg_file_open(errp, errno, filename);
+        return;
+    }
+
+    buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+    if (!buf) {
+        error_setg(errp, "Could not allocate memory");
+        goto out;
+    }
+
+    /* we'll only dump initial memory for now */
+    while (handled_count < total_count) {
+        /* Calculate how many keys to ask for & handle overflow case */
+        cur_count = MIN(total_count - handled_count, S390_SKEYS_BUFFER_SIZE);
+
+        ret = skeyclass->get_skeys(ss, cur_gfn, cur_count, buf);
+        if (ret < 0) {
+            error_setg(errp, "get_keys error %d", ret);
+            goto out_free;
+        }
+
+        /* write keys to stream */
+        write_keys(f, buf, cur_gfn, cur_count, &lerr);
+        if (lerr) {
+            goto out_free;
+        }
+
+        cur_gfn += cur_count;
+        handled_count += cur_count;
+    }
+
+out_free:
+    error_propagate(errp, lerr);
+    g_free(buf);
+out:
+    qemu_fclose(f);
+}
+
+static void qemu_s390_skeys_init(Object *obj)
+{
+    QEMUS390SKeysState *skeys = QEMU_S390_SKEYS(obj);
+    MachineState *machine = MACHINE(qdev_get_machine());
+
+    skeys->key_count = machine->maxram_size / TARGET_PAGE_SIZE;
+    skeys->keydata = g_malloc0(skeys->key_count);
+}
+
+static int qemu_s390_skeys_enabled(S390SKeysState *ss)
+{
+    return 1;
+}
+
+/*
+ * TODO: for memory hotplug support qemu_s390_skeys_set and qemu_s390_skeys_get
+ * will have to make sure that the given gfn belongs to a memory region and not
+ * a memory hole.
+ */
+static int qemu_s390_skeys_set(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    QEMUS390SKeysState *skeydev = QEMU_S390_SKEYS(ss);
+    int i;
+
+    /* Check for uint64 overflow and access beyond end of key data */
+    if (start_gfn + count > skeydev->key_count || start_gfn + count < count) {
+        error_report("Error: Setting storage keys for page beyond the end "
+                "of memory: gfn=%" PRIx64 " count=%" PRId64 "\n", start_gfn,
+                count);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < count; i++) {
+        skeydev->keydata[start_gfn + i] = keys[i];
+    }
+    return 0;
+}
+
+static int qemu_s390_skeys_get(S390SKeysState *ss, uint64_t start_gfn,
+                               uint64_t count, uint8_t *keys)
+{
+    QEMUS390SKeysState *skeydev = QEMU_S390_SKEYS(ss);
+    int i;
+
+    /* Check for uint64 overflow and access beyond end of key data */
+    if (start_gfn + count > skeydev->key_count || start_gfn + count < count) {
+        error_report("Error: Getting storage keys for page beyond the end "
+                "of memory: gfn=%" PRIx64 " count=%" PRId64 "\n", start_gfn,
+                count);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < count; i++) {
+        keys[i] = skeydev->keydata[start_gfn + i];
+    }
+    return 0;
+}
+
+static void qemu_s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_CLASS(oc);
+
+    skeyclass->skeys_enabled = qemu_s390_skeys_enabled;
+    skeyclass->get_skeys = qemu_s390_skeys_get;
+    skeyclass->set_skeys = qemu_s390_skeys_set;
+}
+
+static const TypeInfo qemu_s390_skeys_info = {
+    .name          = TYPE_QEMU_S390_SKEYS,
+    .parent        = TYPE_S390_SKEYS,
+    .instance_init = qemu_s390_skeys_init,
+    .instance_size = sizeof(QEMUS390SKeysState),
+    .class_init    = qemu_s390_skeys_class_init,
+    .instance_size = sizeof(S390SKeysClass),
+};
+
+static void s390_storage_keys_save(QEMUFile *f, void *opaque)
+{
+    S390SKeysState *ss = S390_SKEYS(opaque);
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint64_t pages_left = ram_size / TARGET_PAGE_SIZE;
+    uint64_t read_count, eos = S390_SKEYS_SAVE_FLAG_EOS;
+    vaddr cur_gfn = 0;
+    int error = 0;
+    uint8_t *buf;
+
+    if (!skeyclass->skeys_enabled(ss)) {
+        goto end_stream;
+    }
+
+    buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+    if (!buf) {
+        error_report("storage key save could not allocate memory\n");
+        goto end_stream;
+    }
+
+    /* We only support initial memory. Standby memory is not handled yet. */
+    qemu_put_be64(f, (cur_gfn * TARGET_PAGE_SIZE) | S390_SKEYS_SAVE_FLAG_SKEYS);
+    qemu_put_be64(f, pages_left);
+
+    while (pages_left) {
+        read_count = MIN(pages_left, S390_SKEYS_BUFFER_SIZE);
+
+        if (!error) {
+            error = skeyclass->get_skeys(ss, cur_gfn, read_count, buf);
+            if (error) {
+                /*
+                 * If error: we want to fill the stream with valid data instead
+                 * of stopping early so we pad the stream with 0x00 values and
+                 * use S390_SKEYS_SAVE_FLAG_ERROR to indicate failure to the
+                 * reading side.
+                 */
+                error_report("S390_GET_KEYS error %d\n", error);
+                memset(buf, 0, S390_SKEYS_BUFFER_SIZE);
+                eos = S390_SKEYS_SAVE_FLAG_ERROR;
+            }
+        }
+
+        qemu_put_buffer(f, buf, read_count);
+        cur_gfn += read_count;
+        pages_left -= read_count;
+    }
+
+    g_free(buf);
+end_stream:
+    qemu_put_be64(f, eos);
+}
+
+static int s390_storage_keys_load(QEMUFile *f, void *opaque, int version_id)
+{
+    S390SKeysState *ss = S390_SKEYS(opaque);
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    int ret = 0;
+
+    while (!ret) {
+        ram_addr_t addr;
+        int flags;
+
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        switch (flags) {
+        case S390_SKEYS_SAVE_FLAG_SKEYS: {
+            const uint64_t total_count = qemu_get_be64(f);
+            uint64_t handled_count = 0, cur_count;
+            uint64_t cur_gfn = addr / TARGET_PAGE_SIZE;
+            uint8_t *buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+
+            if (!buf) {
+                error_report("storage key load could not allocate memory\n");
+                ret = -ENOMEM;
+                break;
+            }
+
+            while (handled_count < total_count) {
+                cur_count = MIN(total_count - handled_count,
+                                S390_SKEYS_BUFFER_SIZE);
+                qemu_get_buffer(f, buf, cur_count);
+
+                ret = skeyclass->set_skeys(ss, cur_gfn, cur_count, buf);
+                if (ret < 0) {
+                    error_report("S390_SET_KEYS error %d\n", ret);
+                    break;
+                }
+                handled_count += cur_count;
+                cur_gfn += cur_count;
+            }
+            g_free(buf);
+            break;
+        }
+        case S390_SKEYS_SAVE_FLAG_ERROR: {
+            error_report("Storage key data is incomplete");
+            ret = -EINVAL;
+            break;
+        }
+        case S390_SKEYS_SAVE_FLAG_EOS:
+            /* normal exit */
+            return 0;
+        default:
+            error_report("Unexpected storage key flag data: %#x", flags);
+            ret = -EINVAL;
+        }
+    }
+
+    return ret;
+}
+
+static inline bool s390_skeys_get_migration_enabled(Object *obj, Error **errp)
+{
+    S390SKeysState *ss = S390_SKEYS(obj);
+
+    return ss->migration_enabled;
+}
+
+static inline void s390_skeys_set_migration_enabled(Object *obj, bool value,
+                                            Error **errp)
+{
+    S390SKeysState *ss = S390_SKEYS(obj);
+
+    /* Prevent double registration of savevm handler */
+    if (ss->migration_enabled == value) {
+        return;
+    }
+
+    ss->migration_enabled = value;
+
+    if (ss->migration_enabled) {
+        register_savevm(NULL, TYPE_S390_SKEYS, 0, 1, s390_storage_keys_save,
+                        s390_storage_keys_load, ss);
+    } else {
+        unregister_savevm(DEVICE(ss), TYPE_S390_SKEYS, ss);
+    }
+}
+
+static void s390_skeys_instance_init(Object *obj)
+{
+    object_property_add_bool(obj, "migration-enabled",
+                             s390_skeys_get_migration_enabled,
+                             s390_skeys_set_migration_enabled, NULL);
+    object_property_set_bool(obj, true, "migration-enabled", NULL);
+}
+
+static void s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+}
+
+static const TypeInfo s390_skeys_info = {
+    .name          = TYPE_S390_SKEYS,
+    .parent        = TYPE_DEVICE,
+    .instance_init = s390_skeys_instance_init,
+    .instance_size = sizeof(S390SKeysState),
+    .class_init    = s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+    .abstract = true,
+};
+
+static void qemu_s390_skeys_register_types(void)
+{
+    type_register_static(&s390_skeys_info);
+    type_register_static(&qemu_s390_skeys_info);
+}
+
+type_init(qemu_s390_skeys_register_types)
diff --git a/hw/s390x/s390-virtio-bus.c b/hw/s390x/s390-virtio-bus.c
index 77aec8a5b..98cb1293c 100644
--- a/hw/s390x/s390-virtio-bus.c
+++ b/hw/s390x/s390-virtio-bus.c
@@ -577,17 +577,12 @@ static const TypeInfo s390_virtio_blk = {
     .class_init    = s390_virtio_blk_class_init,
 };
 
-static Property s390_virtio_serial_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void s390_virtio_serial_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
 
     k->realize = s390_virtio_serial_realize;
-    dc->props = s390_virtio_serial_properties;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 4c51d1a5b..5a52ff26e 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -19,6 +19,8 @@
 #include "virtio-ccw.h"
 #include "qemu/config-file.h"
 #include "s390-pci-bus.h"
+#include "hw/s390x/storage-keys.h"
+#include "hw/compat.h"
 
 #define TYPE_S390_CCW_MACHINE               "s390-ccw-machine"
 
@@ -34,26 +36,23 @@ typedef struct S390CcwMachineState {
     bool dea_key_wrap;
 } S390CcwMachineState;
 
-void io_subsystem_reset(void)
+static const char *const reset_dev_types[] = {
+    "virtual-css-bridge",
+    "s390-sclp-event-facility",
+    "s390-flic",
+    "diag288",
+};
+
+void subsystem_reset(void)
 {
-    DeviceState *css, *sclp, *flic, *diag288;
+    DeviceState *dev;
+    int i;
 
-    css = DEVICE(object_resolve_path_type("", "virtual-css-bridge", NULL));
-    if (css) {
-        qdev_reset_all(css);
-    }
-    sclp = DEVICE(object_resolve_path_type("",
-                  "s390-sclp-event-facility", NULL));
-    if (sclp) {
-        qdev_reset_all(sclp);
-    }
-    flic = DEVICE(object_resolve_path_type("", "s390-flic", NULL));
-    if (flic) {
-        qdev_reset_all(flic);
-    }
-    diag288 = DEVICE(object_resolve_path_type("", "diag288", NULL));
-    if (diag288) {
-        qdev_reset_all(diag288);
+    for (i = 0; i < ARRAY_SIZE(reset_dev_types); i++) {
+        dev = DEVICE(object_resolve_path_type("", reset_dev_types[i], NULL));
+        if (dev) {
+            qdev_reset_all(dev);
+        }
     }
 }
 
@@ -99,58 +98,30 @@ static void virtio_ccw_register_hcalls(void)
                                    virtio_ccw_hcall_early_printk);
 }
 
-static void ccw_init(MachineState *machine)
+void s390_memory_init(ram_addr_t mem_size)
 {
-    ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
-    uint8_t *storage_keys;
+
+    /* allocate RAM for core */
+    memory_region_allocate_system_memory(ram, NULL, "s390.ram", mem_size);
+    memory_region_add_subregion(sysmem, 0, ram);
+
+    /* Initialize storage key device */
+    s390_skeys_init();
+}
+
+static void ccw_init(MachineState *machine)
+{
     int ret;
     VirtualCssBus *css_bus;
     DeviceState *dev;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
-    ram_addr_t pad_size = 0;
-    ram_addr_t maxmem = qemu_opt_get_size(opts, "maxmem", my_ram_size);
-    ram_addr_t standby_mem_size = maxmem - my_ram_size;
-    uint64_t kvm_limit;
-
-    /* The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
-     * The variable 'mhd->increment_size' is an exponent of 2 that can be
-     * used to calculate the size (in bytes) of an increment. */
-    mhd->increment_size = 20;
-    while ((my_ram_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
-        mhd->increment_size++;
-    }
-    while ((standby_mem_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
-        mhd->increment_size++;
-    }
 
-    /* The core and standby memory areas need to be aligned with
-     * the increment size.  In effect, this can cause the
-     * user-specified memory size to be rounded down to align
-     * with the nearest increment boundary. */
-    standby_mem_size = standby_mem_size >> mhd->increment_size
-                                        << mhd->increment_size;
-    my_ram_size = my_ram_size >> mhd->increment_size
-                              << mhd->increment_size;
-
-    /* let's propagate the changed ram size into the global variable. */
-    ram_size = my_ram_size;
-    machine->maxram_size = my_ram_size + standby_mem_size;
-
-    ret = s390_set_memory_limit(machine->maxram_size, &kvm_limit);
-    if (ret == -E2BIG) {
-        hw_error("qemu: host supports a maximum of %" PRIu64 " GB",
-                 kvm_limit >> 30);
-    } else if (ret) {
-        hw_error("qemu: setting the guest size failed");
-    }
+    s390_sclp_init();
+    s390_memory_init(machine->ram_size);
 
     /* get a BUS */
     css_bus = virtual_css_bus_init();
-    s390_sclp_init();
     s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
                       machine->initrd_filename, "s390-ccw.img", true);
     s390_flic_init();
@@ -163,27 +134,8 @@ static void ccw_init(MachineState *machine)
     /* register hypercalls */
     virtio_ccw_register_hcalls();
 
-    /* allocate RAM for core */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(sysmem, 0, ram);
-
-    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
-       calculate the pad size necessary to force this boundary. */
-    if (standby_mem_size) {
-        if (my_ram_size % MEM_SECTION_SIZE) {
-            pad_size = MEM_SECTION_SIZE - my_ram_size % MEM_SECTION_SIZE;
-        }
-        my_ram_size += standby_mem_size + pad_size;
-        mhd->pad_size = pad_size;
-        mhd->standby_mem_size = standby_mem_size;
-    }
-
-    /* allocate storage keys */
-    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
-
     /* init CPUs */
-    s390_init_cpus(machine->cpu_model, storage_keys);
+    s390_init_cpus(machine->cpu_model);
 
     if (kvm_enabled()) {
         kvm_s390_enable_css_support(s390_cpu_addr2state(0));
@@ -209,6 +161,7 @@ static void ccw_machine_class_init(ObjectClass *oc, void *data)
     NMIClass *nc = NMI_CLASS(oc);
 
     mc->init = ccw_init;
+    mc->reset = s390_machine_reset;
     mc->block_default_type = IF_VIRTIO;
     mc->no_cdrom = 1;
     mc->no_floppy = 1;
@@ -282,26 +235,84 @@ static const TypeInfo ccw_machine_info = {
     },
 };
 
+#define CCW_COMPAT_2_4 \
+        HW_COMPAT_2_4 \
+        {\
+            .driver   = TYPE_S390_SKEYS,\
+            .property = "migration-enabled",\
+            .value    = "off",\
+        },{\
+            .driver   = "virtio-blk-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-balloon-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-serial-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-9p-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-rng-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-net-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-scsi-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "vhost-scsi-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },
+
 static void ccw_machine_2_4_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
+    static GlobalProperty compat_props[] = {
+        CCW_COMPAT_2_4
+        { /* end of list */ }
+    };
 
-    mc->name = "s390-ccw-virtio-2.4";
-    mc->alias = "s390-ccw-virtio";
     mc->desc = "VirtIO-ccw based S390 machine v2.4";
-    mc->is_default = 1;
+    mc->compat_props = compat_props;
 }
 
 static const TypeInfo ccw_machine_2_4_info = {
-    .name          = TYPE_S390_CCW_MACHINE "2.4",
+    .name          = MACHINE_TYPE_NAME("s390-ccw-virtio-2.4"),
     .parent        = TYPE_S390_CCW_MACHINE,
     .class_init    = ccw_machine_2_4_class_init,
 };
 
+static void ccw_machine_2_5_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->alias = "s390-ccw-virtio";
+    mc->desc = "VirtIO-ccw based S390 machine v2.5";
+    mc->is_default = 1;
+}
+
+static const TypeInfo ccw_machine_2_5_info = {
+    .name          = MACHINE_TYPE_NAME("s390-ccw-virtio-2.5"),
+    .parent        = TYPE_S390_CCW_MACHINE,
+    .class_init    = ccw_machine_2_5_class_init,
+};
+
 static void ccw_machine_register_types(void)
 {
     type_register_static(&ccw_machine_info);
     type_register_static(&ccw_machine_2_4_info);
+    type_register_static(&ccw_machine_2_5_info);
 }
 
 type_init(ccw_machine_register_types)
diff --git a/hw/s390x/s390-virtio.c b/hw/s390x/s390-virtio.c
index 1284e77b2..ae55760d6 100644
--- a/hw/s390x/s390-virtio.c
+++ b/hw/s390x/s390-virtio.c
@@ -23,6 +23,7 @@
 
 #include "hw/hw.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
@@ -30,14 +31,16 @@
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "hw/virtio/virtio.h"
-#include "hw/sysbus.h"
 #include "sysemu/kvm.h"
 #include "exec/address-spaces.h"
+#include "sysemu/qtest.h"
 
 #include "hw/s390x/s390-virtio-bus.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/s390-virtio.h"
+#include "hw/s390x/storage-keys.h"
+#include "hw/s390x/ipl.h"
 #include "cpu.h"
 
 //#define DEBUG_S390
@@ -52,7 +55,8 @@
 
 #define MAX_BLK_DEVS                    10
 #define ZIPL_FILENAME                   "s390-zipl.rom"
-#define TYPE_S390_MACHINE               "s390-machine"
+#define S390_MACHINE                    "s390-virtio"
+#define TYPE_S390_MACHINE               MACHINE_TYPE_NAME(S390_MACHINE)
 
 #define S390_TOD_CLOCK_VALUE_MISSING    0x00
 #define S390_TOD_CLOCK_VALUE_PRESENT    0x01
@@ -147,9 +151,9 @@ void s390_init_ipl_dev(const char *kernel_filename,
                        const char *firmware,
                        bool enforce_bios)
 {
-    DeviceState *dev;
+    Object *new = object_new(TYPE_S390_IPL);
+    DeviceState *dev = DEVICE(new);
 
-    dev  = qdev_create(NULL, "s390-ipl");
     if (kernel_filename) {
         qdev_prop_set_string(dev, "kernel", kernel_filename);
     }
@@ -159,12 +163,13 @@ void s390_init_ipl_dev(const char *kernel_filename,
     qdev_prop_set_string(dev, "cmdline", kernel_cmdline);
     qdev_prop_set_string(dev, "firmware", firmware);
     qdev_prop_set_bit(dev, "enforce_bios", enforce_bios);
-    object_property_add_child(qdev_get_machine(), "s390-ipl",
-                              OBJECT(dev), NULL);
+    object_property_add_child(qdev_get_machine(), TYPE_S390_IPL,
+                              new, NULL);
+    object_unref(new);
     qdev_init_nofail(dev);
 }
 
-void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys)
+void s390_init_cpus(const char *cpu_model)
 {
     int i;
 
@@ -184,7 +189,6 @@ void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys)
         ipi_states[i] = cpu;
         cs->halted = 1;
         cs->exception_index = EXCP_HLT;
-        cpu->env.storage_keys = storage_keys;
     }
 }
 
@@ -260,31 +264,26 @@ int gtod_load(QEMUFile *f, void *opaque, int version_id)
 /* PC hardware initialisation */
 static void s390_init(MachineState *machine)
 {
-    ram_addr_t my_ram_size = machine->ram_size;
-    MemoryRegion *sysmem = get_system_memory();
-    MemoryRegion *ram = g_new(MemoryRegion, 1);
-    int increment_size = 20;
-    uint8_t *storage_keys;
+    ram_addr_t my_ram_size;
     void *virtio_region;
     hwaddr virtio_region_len;
     hwaddr virtio_region_start;
 
-    /*
-     * The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or
-     * fewer.
-     */
-    while ((my_ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
-        increment_size++;
+    if (!qtest_enabled()) {
+        error_printf("WARNING\n"
+                     "The s390-virtio machine (non-ccw) is deprecated.\n"
+                     "It will be removed in 2.6. Please use s390-ccw-virtio\n");
     }
-    my_ram_size = my_ram_size >> increment_size << increment_size;
 
-    /* let's propagate the changed ram size into the global variable. */
-    ram_size = my_ram_size;
+    if (machine->ram_slots) {
+        error_report("Memory hotplug not supported by the selected machine.");
+        exit(EXIT_FAILURE);
+    }
+    s390_sclp_init();
+    my_ram_size = machine->ram_size;
 
     /* get a BUS */
     s390_bus = s390_virtio_bus_init(&my_ram_size);
-    s390_sclp_init();
     s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
                       machine->initrd_filename, ZIPL_FILENAME, false);
     s390_flic_init();
@@ -293,9 +292,7 @@ static void s390_init(MachineState *machine)
     s390_virtio_register_hcalls();
 
     /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(sysmem, 0, ram);
+    s390_memory_init(my_ram_size);
 
     /* clear virtio region */
     virtio_region_len = my_ram_size - ram_size;
@@ -306,11 +303,8 @@ static void s390_init(MachineState *machine)
     cpu_physical_memory_unmap(virtio_region, virtio_region_len, 1,
                               virtio_region_len);
 
-    /* allocate storage keys */
-    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
-
     /* init CPUs */
-    s390_init_cpus(machine->cpu_model, storage_keys);
+    s390_init_cpus(machine->cpu_model);
 
     /* Create VirtIO network adapters */
     s390_create_virtio_net((BusState *)s390_bus, "virtio-net-s390");
@@ -328,15 +322,28 @@ void s390_nmi(NMIState *n, int cpu_index, Error **errp)
     }
 }
 
+void s390_machine_reset(void)
+{
+    S390CPU *ipl_cpu = S390_CPU(qemu_get_cpu(0));
+
+    qemu_devices_reset();
+    s390_cmma_reset();
+    s390_crypto_reset();
+
+    /* all cpus are stopped - configure and start the ipl cpu only */
+    s390_ipl_prepare_cpu(ipl_cpu);
+    s390_cpu_set_state(CPU_STATE_OPERATING, ipl_cpu);
+}
+
 static void s390_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
     NMIClass *nc = NMI_CLASS(oc);
 
-    mc->name = "s390-virtio";
     mc->alias = "s390";
-    mc->desc = "VirtIO based S390 machine";
+    mc->desc = "VirtIO based S390 machine (deprecated)";
     mc->init = s390_init;
+    mc->reset = s390_machine_reset;
     mc->block_default_type = IF_VIRTIO;
     mc->max_cpus = 255;
     mc->no_serial = 1;
diff --git a/hw/s390x/s390-virtio.h b/hw/s390x/s390-virtio.h
index c84785395..eebce8e5e 100644
--- a/hw/s390x/s390-virtio.h
+++ b/hw/s390x/s390-virtio.h
@@ -19,7 +19,7 @@
 typedef int (*s390_virtio_fn)(const uint64_t *args);
 void s390_register_virtio_hypercall(uint64_t code, s390_virtio_fn fn);
 
-void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys);
+void s390_init_cpus(const char *cpu_model);
 void s390_init_ipl_dev(const char *kernel_filename,
                        const char *kernel_cmdline,
                        const char *initrd_filename,
@@ -27,4 +27,6 @@ void s390_init_ipl_dev(const char *kernel_filename,
                        bool enforce_bios);
 void s390_create_virtio_net(BusState *bus, const char *name);
 void s390_nmi(NMIState *n, int cpu_index, Error **errp);
+void s390_machine_reset(void);
+void s390_memory_init(ram_addr_t mem_size);
 #endif
diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index b3a6c5e5a..a061b49f1 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -17,37 +17,27 @@
 #include "exec/memory.h"
 #include "sysemu/sysemu.h"
 #include "exec/address-spaces.h"
-#include "qemu/config-file.h"
+#include "hw/boards.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
 #include "hw/s390x/s390-pci-bus.h"
 
-static inline SCLPEventFacility *get_event_facility(void)
+static inline SCLPDevice *get_sclp_device(void)
 {
-    ObjectProperty *op = object_property_find(qdev_get_machine(),
-                                              TYPE_SCLP_EVENT_FACILITY,
-                                              NULL);
-    assert(op);
-    return op->opaque;
+    return SCLP(object_resolve_path_type("", TYPE_SCLP, NULL));
 }
 
 /* Provide information about the configuration, CPUs and storage */
-static void read_SCP_info(SCCB *sccb)
+static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadInfo *read_info = (ReadInfo *) sccb;
+    MachineState *machine = MACHINE(qdev_get_machine());
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
     CPUState *cpu;
     int cpu_count = 0;
     int i = 0;
-    int increment_size = 20;
     int rnsize, rnmax;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
-    int slots = qemu_opt_get_number(opts, "slots", 0);
-    int max_avail_slots = s390_get_memslot_count(kvm_state);
-
-    if (slots > max_avail_slots) {
-        slots = max_avail_slots;
-    }
+    int slots = MIN(machine->ram_slots, s390_get_memslot_count(kvm_state));
 
     CPU_FOREACH(cpu) {
         cpu_count++;
@@ -66,23 +56,8 @@ static void read_SCP_info(SCCB *sccb)
     read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO |
                                         SCLP_HAS_PCI_RECONFIG);
 
-    /*
-     * The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
-     */
-    while ((ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
-        increment_size++;
-    }
-    rnmax = ram_size >> increment_size;
-
     /* Memory Hotplug is only supported for the ccw machine type */
     if (mhd) {
-        while ((mhd->standby_mem_size >> increment_size) >
-               MAX_STORAGE_INCREMENTS) {
-            increment_size++;
-        }
-        assert(increment_size == mhd->increment_size);
-
         mhd->standby_subregion_size = MEM_SECTION_SIZE;
         /* Deduct the memory slot already used for core */
         if (slots > 0) {
@@ -108,13 +83,11 @@ static void read_SCP_info(SCCB *sccb)
         }
         mhd->padded_ram_size = ram_size + mhd->pad_size;
         mhd->rzm = 1 << mhd->increment_size;
-        rnmax = ((ram_size + mhd->standby_mem_size + mhd->pad_size)
-             >> mhd->increment_size);
 
         read_info->facilities |= cpu_to_be64(SCLP_FC_ASSIGN_ATTACH_READ_STOR);
     }
 
-    rnsize = 1 << (increment_size - 20);
+    rnsize = 1 << (sclp->increment_size - 20);
     if (rnsize <= 128) {
         read_info->rnsize = rnsize;
     } else {
@@ -122,6 +95,7 @@ static void read_SCP_info(SCCB *sccb)
         read_info->rnsize2 = cpu_to_be32(rnsize);
     }
 
+    rnmax = machine->maxram_size >> sclp->increment_size;
     if (rnmax < 0x10000) {
         read_info->rnmax = cpu_to_be16(rnmax);
     } else {
@@ -132,14 +106,17 @@ static void read_SCP_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void read_storage_element0_info(SCCB *sccb)
+static void read_storage_element0_info(SCLPDevice *sclp, SCCB *sccb)
 {
     int i, assigned;
     int subincrement_id = SCLP_STARTING_SUBINCREMENT_ID;
     ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if ((ram_size >> mhd->increment_size) >= 0x10000) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
@@ -158,12 +135,15 @@ static void read_storage_element0_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void read_storage_element1_info(SCCB *sccb)
+static void read_storage_element1_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if ((mhd->standby_mem_size >> mhd->increment_size) >= 0x10000) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
@@ -179,13 +159,17 @@ static void read_storage_element1_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_STANDBY_READ_COMPLETION);
 }
 
-static void attach_storage_element(SCCB *sccb, uint16_t element)
+static void attach_storage_element(SCLPDevice *sclp, SCCB *sccb,
+                                   uint16_t element)
 {
     int i, assigned, subincrement_id;
     AttachStorageElement *attach_info = (AttachStorageElement *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if (element != 1) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
@@ -203,20 +187,26 @@ static void attach_storage_element(SCCB *sccb, uint16_t element)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
 }
 
-static void assign_storage(SCCB *sccb)
+static void assign_storage(SCLPDevice *sclp, SCCB *sccb)
 {
     MemoryRegion *mr = NULL;
     uint64_t this_subregion_size;
     AssignStorage *assign_info = (AssignStorage *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
-    assert(mhd);
-    ram_addr_t assign_addr = (assign_info->rn - 1) * mhd->rzm;
+    ram_addr_t assign_addr;
     MemoryRegion *sysmem = get_system_memory();
 
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+    assign_addr = (assign_info->rn - 1) * mhd->rzm;
+
     if ((assign_addr % MEM_SECTION_SIZE == 0) &&
         (assign_addr >= mhd->padded_ram_size)) {
         /* Re-use existing memory region if found */
         mr = memory_region_find(sysmem, assign_addr, 1).mr;
+        memory_region_unref(mr);
         if (!mr) {
 
             MemoryRegion *standby_ram = g_new(MemoryRegion, 1);
@@ -241,7 +231,13 @@ static void assign_storage(SCCB *sccb)
                 this_subregion_size = mhd->standby_subregion_size;
             }
 
-            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size, &error_abort);
+            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size,
+                                   &error_fatal);
+            /* This is a hack to make memory hotunplug work again. Once we have
+             * subdevices, we have to unparent them when unassigning memory,
+             * instead of doing it via the ref count of the MemoryRegion. */
+            object_ref(OBJECT(standby_ram));
+            object_unparent(OBJECT(standby_ram));
             vmstate_register_ram_global(standby_ram);
             memory_region_add_subregion(sysmem, offset, standby_ram);
         }
@@ -252,15 +248,20 @@ static void assign_storage(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
 }
 
-static void unassign_storage(SCCB *sccb)
+static void unassign_storage(SCLPDevice *sclp, SCCB *sccb)
 {
     MemoryRegion *mr = NULL;
     AssignStorage *assign_info = (AssignStorage *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
-    assert(mhd);
-    ram_addr_t unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+    ram_addr_t unassign_addr;
     MemoryRegion *sysmem = get_system_memory();
 
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+    unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+
     /* if the addr is a multiple of 256 MB */
     if ((unassign_addr % MEM_SECTION_SIZE == 0) &&
         (unassign_addr >= mhd->padded_ram_size)) {
@@ -269,6 +270,7 @@ static void unassign_storage(SCCB *sccb)
 
         /* find the specified memory region and destroy it */
         mr = memory_region_find(sysmem, unassign_addr, 1).mr;
+        memory_region_unref(mr);
         if (mr) {
             int i;
             int is_removable = 1;
@@ -287,8 +289,7 @@ static void unassign_storage(SCCB *sccb)
             }
             if (is_removable) {
                 memory_region_del_subregion(sysmem, mr);
-                object_unparent(OBJECT(mr));
-                g_free(mr);
+                object_unref(OBJECT(mr));
             }
         }
     }
@@ -296,7 +297,7 @@ static void unassign_storage(SCCB *sccb)
 }
 
 /* Provide information about the CPU */
-static void sclp_read_cpu_info(SCCB *sccb)
+static void sclp_read_cpu_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadCpuInfo *cpu_info = (ReadCpuInfo *) sccb;
     CPUState *cpu;
@@ -323,34 +324,35 @@ static void sclp_read_cpu_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void sclp_execute(SCCB *sccb, uint32_t code)
+static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
 {
-    SCLPEventFacility *ef = get_event_facility();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+    SCLPEventFacility *ef = sclp->event_facility;
     SCLPEventFacilityClass *efc = EVENT_FACILITY_GET_CLASS(ef);
 
     switch (code & SCLP_CMD_CODE_MASK) {
     case SCLP_CMDW_READ_SCP_INFO:
     case SCLP_CMDW_READ_SCP_INFO_FORCED:
-        read_SCP_info(sccb);
+        sclp_c->read_SCP_info(sclp, sccb);
         break;
     case SCLP_CMDW_READ_CPU_INFO:
-        sclp_read_cpu_info(sccb);
+        sclp_c->read_cpu_info(sclp, sccb);
         break;
     case SCLP_READ_STORAGE_ELEMENT_INFO:
         if (code & 0xff00) {
-            read_storage_element1_info(sccb);
+            sclp_c->read_storage_element1_info(sclp, sccb);
         } else {
-            read_storage_element0_info(sccb);
+            sclp_c->read_storage_element0_info(sclp, sccb);
         }
         break;
     case SCLP_ATTACH_STORAGE_ELEMENT:
-        attach_storage_element(sccb, (code & 0xff00) >> 8);
+        sclp_c->attach_storage_element(sclp, sccb, (code & 0xff00) >> 8);
         break;
     case SCLP_ASSIGN_STORAGE:
-        assign_storage(sccb);
+        sclp_c->assign_storage(sclp, sccb);
         break;
     case SCLP_UNASSIGN_STORAGE:
-        unassign_storage(sccb);
+        sclp_c->unassign_storage(sclp, sccb);
         break;
     case SCLP_CMDW_CONFIGURE_PCI:
         s390_pci_sclp_configure(1, sccb);
@@ -366,6 +368,8 @@ static void sclp_execute(SCCB *sccb, uint32_t code)
 
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
 {
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
     int r = 0;
     SCCB work_sccb;
 
@@ -400,20 +404,20 @@ int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
         goto out;
     }
 
-    sclp_execute((SCCB *)&work_sccb, code);
+    sclp_c->execute(sclp, (SCCB *)&work_sccb, code);
 
     cpu_physical_memory_write(sccb, &work_sccb,
                               be16_to_cpu(work_sccb.h.length));
 
-    sclp_service_interrupt(sccb);
+    sclp_c->service_interrupt(sclp, sccb);
 
 out:
     return r;
 }
 
-void sclp_service_interrupt(uint32_t sccb)
+static void service_interrupt(SCLPDevice *sclp, uint32_t sccb)
 {
-    SCLPEventFacility *ef = get_event_facility();
+    SCLPEventFacility *ef = sclp->event_facility;
     SCLPEventFacilityClass *efc = EVENT_FACILITY_GET_CLASS(ef);
 
     uint32_t param = sccb & ~3;
@@ -428,17 +432,148 @@ void sclp_service_interrupt(uint32_t sccb)
     s390_sclp_extint(param);
 }
 
+void sclp_service_interrupt(uint32_t sccb)
+{
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+
+    sclp_c->service_interrupt(sclp, sccb);
+}
+
 /* qemu object creation and initialization functions */
 
 void s390_sclp_init(void)
 {
-    DeviceState *dev  = qdev_create(NULL, TYPE_SCLP_EVENT_FACILITY);
+    Object *new = object_new(TYPE_SCLP);
 
-    object_property_add_child(qdev_get_machine(), TYPE_SCLP_EVENT_FACILITY,
-                              OBJECT(dev), NULL);
-    qdev_init_nofail(dev);
+    object_property_add_child(qdev_get_machine(), TYPE_SCLP, new,
+                              NULL);
+    object_unref(OBJECT(new));
+    qdev_init_nofail(DEVICE(new));
+}
+
+static void sclp_realize(DeviceState *dev, Error **errp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    SCLPDevice *sclp = SCLP(dev);
+    Error *l_err = NULL;
+    uint64_t hw_limit;
+    int ret;
+
+    object_property_set_bool(OBJECT(sclp->event_facility), true, "realized",
+                             &l_err);
+    if (l_err) {
+        goto error;
+    }
+
+    ret = s390_set_memory_limit(machine->maxram_size, &hw_limit);
+    if (ret == -E2BIG) {
+        error_setg(&l_err, "qemu: host supports a maximum of %" PRIu64 " GB",
+                   hw_limit >> 30);
+        goto error;
+    } else if (ret) {
+        error_setg(&l_err, "qemu: setting the guest size failed");
+        goto error;
+    }
+    return;
+error:
+    assert(l_err);
+    error_propagate(errp, l_err);
+}
+
+static void sclp_memory_init(SCLPDevice *sclp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    ram_addr_t initial_mem = machine->ram_size;
+    ram_addr_t max_mem = machine->maxram_size;
+    ram_addr_t standby_mem = max_mem - initial_mem;
+    ram_addr_t pad_mem = 0;
+    int increment_size = 20;
+
+    /* The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
+     * The variable 'increment_size' is an exponent of 2 that can be
+     * used to calculate the size (in bytes) of an increment. */
+    while ((initial_mem >> increment_size) > MAX_STORAGE_INCREMENTS) {
+        increment_size++;
+    }
+    if (machine->ram_slots) {
+        while ((standby_mem >> increment_size) > MAX_STORAGE_INCREMENTS) {
+            increment_size++;
+        }
+    }
+    sclp->increment_size = increment_size;
+
+    /* The core and standby memory areas need to be aligned with
+     * the increment size.  In effect, this can cause the
+     * user-specified memory size to be rounded down to align
+     * with the nearest increment boundary. */
+    initial_mem = initial_mem >> increment_size << increment_size;
+    standby_mem = standby_mem >> increment_size << increment_size;
+
+    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
+       calculate the pad size necessary to force this boundary. */
+    if (machine->ram_slots && standby_mem) {
+        sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
+
+        if (initial_mem % MEM_SECTION_SIZE) {
+            pad_mem = MEM_SECTION_SIZE - initial_mem % MEM_SECTION_SIZE;
+        }
+        mhd->increment_size = increment_size;
+        mhd->pad_size = pad_mem;
+        mhd->standby_mem_size = standby_mem;
+    }
+    machine->ram_size = initial_mem;
+    machine->maxram_size = initial_mem + pad_mem + standby_mem;
+    /* let's propagate the changed ram size into the global variable. */
+    ram_size = initial_mem;
+}
+
+static void sclp_init(Object *obj)
+{
+    SCLPDevice *sclp = SCLP(obj);
+    Object *new;
+
+    new = object_new(TYPE_SCLP_EVENT_FACILITY);
+    object_property_add_child(obj, TYPE_SCLP_EVENT_FACILITY, new, NULL);
+    /* qdev_device_add searches the sysbus for TYPE_SCLP_EVENTS_BUS */
+    qdev_set_parent_bus(DEVICE(new), sysbus_get_default());
+    object_unref(new);
+    sclp->event_facility = EVENT_FACILITY(new);
+
+    sclp_memory_init(sclp);
 }
 
+static void sclp_class_init(ObjectClass *oc, void *data)
+{
+    SCLPDeviceClass *sc = SCLP_CLASS(oc);
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->desc = "SCLP (Service-Call Logical Processor)";
+    dc->realize = sclp_realize;
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+
+    sc->read_SCP_info = read_SCP_info;
+    sc->read_storage_element0_info = read_storage_element0_info;
+    sc->read_storage_element1_info = read_storage_element1_info;
+    sc->attach_storage_element = attach_storage_element;
+    sc->assign_storage = assign_storage;
+    sc->unassign_storage = unassign_storage;
+    sc->read_cpu_info = sclp_read_cpu_info;
+    sc->execute = sclp_execute;
+    sc->service_interrupt = service_interrupt;
+}
+
+static TypeInfo sclp_info = {
+    .name = TYPE_SCLP,
+    .parent = TYPE_DEVICE,
+    .instance_init = sclp_init,
+    .instance_size = sizeof(SCLPDevice),
+    .class_init = sclp_class_init,
+    .class_size = sizeof(SCLPDeviceClass),
+};
+
 sclpMemoryHotplugDev *init_sclp_memory_hotplug_dev(void)
 {
     DeviceState *dev;
@@ -475,5 +610,6 @@ static TypeInfo sclp_memory_hotplug_dev_info = {
 static void register_types(void)
 {
     type_register_static(&sclp_memory_hotplug_dev_info);
+    type_register_static(&sclp_info);
 }
 type_init(register_types);
diff --git a/hw/s390x/sclpcpu.c b/hw/s390x/sclpcpu.c
index 2fe8b5aa4..322eb31d9 100644
--- a/hw/s390x/sclpcpu.c
+++ b/hw/s390x/sclpcpu.c
@@ -25,13 +25,16 @@ typedef struct ConfigMgtData {
     uint8_t event_qualifier;
 } QEMU_PACKED ConfigMgtData;
 
-static qemu_irq *irq_cpu_hotplug; /* Only used in this file */
-
 #define EVENT_QUAL_CPU_CHANGE  1
 
 void raise_irq_cpu_hotplug(void)
 {
-    qemu_irq_raise(*irq_cpu_hotplug);
+    Object *obj = object_resolve_path_type("", TYPE_SCLP_CPU_HOTPLUG, NULL);
+
+    SCLP_EVENT(obj)->event_pending = true;
+
+    /* Trigger SCLP read operation */
+    sclp_service_interrupt(0);
 }
 
 static unsigned int send_mask(void)
@@ -70,36 +73,19 @@ static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
     return 1;
 }
 
-static void trigger_signal(void *opaque, int n, int level)
-{
-    SCLPEvent *event = opaque;
-    event->event_pending = true;
-
-    /* Trigger SCLP read operation */
-    sclp_service_interrupt(0);
-}
-
-static int irq_cpu_hotplug_init(SCLPEvent *event)
-{
-    irq_cpu_hotplug = qemu_allocate_irqs(trigger_signal, event, 1);
-    return 0;
-}
-
 static void cpu_class_init(ObjectClass *oc, void *data)
 {
     SCLPEventClass *k = SCLP_EVENT_CLASS(oc);
     DeviceClass *dc = DEVICE_CLASS(oc);
 
-    k->init = irq_cpu_hotplug_init;
     k->get_send_mask = send_mask;
     k->get_receive_mask = receive_mask;
     k->read_event_data = read_event_data;
-    k->write_event_data = NULL;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo sclp_cpu_info = {
-    .name          = "sclp-cpu-hotplug",
+    .name          = TYPE_SCLP_CPU_HOTPLUG,
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPEvent),
     .class_init    = cpu_class_init,
diff --git a/hw/s390x/sclpquiesce.c b/hw/s390x/sclpquiesce.c
index ffa555313..15b06e108 100644
--- a/hw/s390x/sclpquiesce.c
+++ b/hw/s390x/sclpquiesce.c
@@ -66,7 +66,7 @@ static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
 }
 
 static const VMStateDescription vmstate_sclpquiesce = {
-    .name = "sclpquiesce",
+    .name = TYPE_SCLP_QUIESCE,
     .version_id = 0,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
@@ -127,7 +127,7 @@ static void quiesce_class_init(ObjectClass *klass, void *data)
 }
 
 static const TypeInfo sclp_quiesce_info = {
-    .name          = "sclpquiesce",
+    .name          = TYPE_SCLP_QUIESCE,
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPEvent),
     .class_init    = quiesce_class_init,
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index d36373e88..63da30386 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -307,11 +307,18 @@ static int virtio_ccw_set_vqs(SubchDev *sch, VqInfoBlock *info,
     if (!desc) {
         virtio_queue_set_vector(vdev, index, VIRTIO_NO_VECTOR);
     } else {
-        /* Fail if we don't have a big enough queue. */
-        /* TODO: Add interface to handle vring.num changing */
-        if (virtio_queue_get_num(vdev, index) > num) {
+        if (info) {
+            /* virtio-1 allows changing the ring size. */
+            if (virtio_queue_get_num(vdev, index) < num) {
+                /* Fail if we exceed the maximum number. */
+                return -EINVAL;
+            }
+            virtio_queue_set_num(vdev, index, num);
+        } else if (virtio_queue_get_num(vdev, index) > num) {
+            /* Fail if we don't have a big enough queue. */
             return -EINVAL;
         }
+        /* We ignore possible increased num for legacy for compatibility. */
         virtio_queue_set_vector(vdev, index, index);
     }
     /* tell notify handler in case of config change */
@@ -460,16 +467,19 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
                                                 MEMTXATTRS_UNSPECIFIED,
                                                 NULL);
             if (features.index == 0) {
-                features.features = (uint32_t)vdev->host_features;
-            } else if (features.index == 1) {
-                features.features = (uint32_t)(vdev->host_features >> 32);
+                if (dev->revision >= 1) {
+                    /* Don't offer legacy features for modern devices. */
+                    features.features = (uint32_t)
+                        (vdev->host_features & ~VIRTIO_LEGACY_FEATURES);
+                } else {
+                    features.features = (uint32_t)vdev->host_features;
+                }
+            } else if ((features.index == 1) && (dev->revision >= 1)) {
                 /*
-                 * Don't offer version 1 to the guest if it did not
-                 * negotiate at least revision 1.
+                 * Only offer feature bits beyond 31 if the guest has
+                 * negotiated at least revision 1.
                  */
-                if (dev->revision <= 0) {
-                    features.features &= ~(1 << (VIRTIO_F_VERSION_1 - 32));
-                }
+                features.features = (uint32_t)(vdev->host_features >> 32);
             } else {
                 /* Return zeroes if the guest supports more feature bits. */
                 features.features = 0;
@@ -508,14 +518,12 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
                 virtio_set_features(vdev,
                                     (vdev->guest_features & 0xffffffff00000000ULL) |
                                     features.features);
-            } else if (features.index == 1) {
+            } else if ((features.index == 1) && (dev->revision >= 1)) {
                 /*
-                 * The guest should not set version 1 if it didn't
-                 * negotiate a revision >= 1.
+                 * If the guest did not negotiate at least revision 1,
+                 * we did not offer it any feature bits beyond 31. Such a
+                 * guest passing us any bit here is therefore buggy.
                  */
-                if (dev->revision <= 0) {
-                    features.features &= ~(1 << (VIRTIO_F_VERSION_1 - 32));
-                }
                 virtio_set_features(vdev,
                                     (vdev->guest_features & 0x00000000ffffffffULL) |
                                     ((uint64_t)features.features << 32));
@@ -766,7 +774,7 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
          * need to fetch it here. Nothing to do for now, though.
          */
         if (dev->revision >= 0 ||
-            revinfo.revision > virtio_ccw_rev_max(vdev)) {
+            revinfo.revision > virtio_ccw_rev_max(dev)) {
             ret = -ENOSYS;
             break;
         }
@@ -1539,10 +1547,25 @@ static void virtio_ccw_device_plugged(DeviceState *d, Error **errp)
 
     sch->id.cu_model = virtio_bus_get_vdev_id(&dev->bus);
 
+    if (dev->max_rev >= 1) {
+        virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
+    }
+
     css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid,
                           d->hotplugged, 1);
 }
 
+static void virtio_ccw_post_plugged(DeviceState *d, Error **errp)
+{
+   VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+   VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+
+   if (!virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        /* A backend didn't support modern virtio. */
+       dev->max_rev = 0;
+   }
+}
+
 static void virtio_ccw_device_unplugged(DeviceState *d)
 {
     VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
@@ -1555,6 +1578,8 @@ static Property virtio_ccw_net_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1582,6 +1607,8 @@ static Property virtio_ccw_blk_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1609,6 +1636,8 @@ static Property virtio_ccw_serial_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1636,6 +1665,8 @@ static Property virtio_ccw_balloon_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1663,6 +1694,8 @@ static Property virtio_ccw_scsi_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1689,6 +1722,8 @@ static const TypeInfo virtio_ccw_scsi = {
 #ifdef CONFIG_VHOST_SCSI
 static Property vhost_ccw_scsi_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1727,6 +1762,8 @@ static Property virtio_ccw_rng_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1865,6 +1902,7 @@ static void virtio_ccw_bus_class_init(ObjectClass *klass, void *data)
     k->save_config = virtio_ccw_save_config;
     k->load_config = virtio_ccw_load_config;
     k->device_plugged = virtio_ccw_device_plugged;
+    k->post_plugged = virtio_ccw_post_plugged;
     k->device_unplugged = virtio_ccw_device_unplugged;
 }
 
@@ -1880,6 +1918,8 @@ static Property virtio_ccw_9p_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
             VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/s390x/virtio-ccw.h b/hw/s390x/virtio-ccw.h
index 692ddd731..7ab8367ba 100644
--- a/hw/s390x/virtio-ccw.h
+++ b/hw/s390x/virtio-ccw.h
@@ -88,6 +88,7 @@ struct VirtioCcwDevice {
     SubchDev *sch;
     char *bus_id;
     int revision;
+    uint32_t max_rev;
     VirtioBusState bus;
     bool ioeventfd_started;
     bool ioeventfd_disabled;
@@ -102,9 +103,10 @@ struct VirtioCcwDevice {
 };
 
 /* The maximum virtio revision we support. */
-static inline int virtio_ccw_rev_max(VirtIODevice *vdev)
+#define VIRTIO_CCW_MAX_REV 1
+static inline int virtio_ccw_rev_max(VirtioCcwDevice *dev)
 {
-    return 0;
+    return dev->max_rev;
 }
 
 /* virtual css bus type */
diff --git a/hw/scsi/megasas.c b/hw/scsi/megasas.c
index a04369c5a..576f56cbf 100644
--- a/hw/scsi/megasas.c
+++ b/hw/scsi/megasas.c
@@ -431,7 +431,7 @@ static uint64_t megasas_fw_time(void)
 static uint64_t megasas_get_sata_addr(uint16_t id)
 {
     uint64_t addr = (0x1221ULL << 48);
-    return addr & (id << 24);
+    return addr | ((uint64_t)id << 24);
 }
 
 /*
@@ -718,7 +718,7 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
     BusChild *kid;
     int num_pd_disks = 0;
 
-    memset(&info, 0x0, cmd->iov_size);
+    memset(&info, 0x0, dcmd_size);
     if (cmd->iov_size < dcmd_size) {
         trace_megasas_dcmd_invalid_xfer_len(cmd->index, cmd->iov_size,
                                             dcmd_size);
@@ -757,7 +757,7 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
 
     memcpy(info.product_name, base_class->product_name, 24);
     snprintf(info.serial_number, 32, "%s", s->hba_serial);
-    snprintf(info.package_version, 0x60, "%s-QEMU", QEMU_VERSION);
+    snprintf(info.package_version, 0x60, "%s-QEMU", qemu_hw_version());
     memcpy(info.image_component[0].name, "APP", 3);
     snprintf(info.image_component[0].version, 10, "%s-QEMU",
              base_class->product_version);
diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index f0ae4625f..fd1171e48 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -136,7 +136,8 @@ static void scsi_dma_restart_cb(void *opaque, int running, RunState state)
         return;
     }
     if (!s->bh) {
-        s->bh = qemu_bh_new(scsi_dma_restart_bh, s);
+        AioContext *ctx = blk_get_aio_context(s->conf.blk);
+        s->bh = aio_bh_new(ctx, scsi_dma_restart_bh, s);
         qemu_bh_schedule(s->bh);
     }
 }
@@ -452,7 +453,7 @@ static bool scsi_target_emulate_inquiry(SCSITargetReq *r)
         r->buf[7] = 0x10 | (r->req.bus->info->tcq ? 0x02 : 0); /* Sync, TCQ.  */
         memcpy(&r->buf[8], "QEMU    ", 8);
         memcpy(&r->buf[16], "QEMU TARGET     ", 16);
-        pstrcpy((char *) &r->buf[32], 4, qemu_get_version());
+        pstrcpy((char *) &r->buf[32], 4, qemu_hw_version());
     }
     return true;
 }
@@ -557,7 +558,7 @@ SCSIRequest *scsi_req_alloc(const SCSIReqOps *reqops, SCSIDevice *d,
     const int memset_off = offsetof(SCSIRequest, sense)
                            + sizeof(req->sense);
 
-    req = g_slice_alloc(reqops->size);
+    req = g_malloc(reqops->size);
     memset((uint8_t *)req + memset_off, 0, reqops->size - memset_off);
     req->refcount = 1;
     req->bus = bus;
@@ -1621,7 +1622,7 @@ void scsi_req_unref(SCSIRequest *req)
         }
         object_unref(OBJECT(req->dev));
         object_unref(OBJECT(qbus->parent));
-        g_slice_free1(req->ops->size, req);
+        g_free(req);
     }
 }
 
diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
index 0e0bc6440..4797d8368 100644
--- a/hw/scsi/scsi-disk.c
+++ b/hw/scsi/scsi-disk.c
@@ -90,7 +90,7 @@ struct SCSIDiskState
     bool tray_locked;
 };
 
-static int scsi_handle_rw_error(SCSIDiskReq *r, int error);
+static int scsi_handle_rw_error(SCSIDiskReq *r, int error, bool acct_failed);
 
 static void scsi_free_request(SCSIRequest *req)
 {
@@ -169,18 +169,18 @@ static void scsi_aio_complete(void *opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, true)) {
             goto done;
         }
     }
 
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     scsi_req_complete(&r->req, GOOD);
 
 done:
@@ -217,6 +217,8 @@ static void scsi_write_do_fua(SCSIDiskReq *r)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
+    assert(r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
@@ -235,22 +237,17 @@ done:
     scsi_req_unref(&r->req);
 }
 
-static void scsi_dma_complete_noio(void *opaque, int ret)
+static void scsi_dma_complete_noio(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+    assert(r->req.aiocb == NULL);
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -271,9 +268,17 @@ done:
 static void scsi_dma_complete(void *opaque, int ret)
 {
     SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
     assert(r->req.aiocb != NULL);
-    scsi_dma_complete_noio(opaque, ret);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_dma_complete_noio(r, ret);
 }
 
 static void scsi_read_complete(void * opaque, int ret)
@@ -284,18 +289,18 @@ static void scsi_read_complete(void * opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, true)) {
             goto done;
         }
     }
 
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     DPRINTF("Data ready tag=0x%x len=%zd\n", r->req.tag, r->qiov.size);
 
     n = r->qiov.size / 512;
@@ -308,23 +313,20 @@ done:
 }
 
 /* Actually issue a read to the block device.  */
-static void scsi_do_read(void *opaque, int ret)
+static void scsi_do_read(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = opaque;
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint32_t n;
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
+    assert (r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -349,6 +351,22 @@ done:
     scsi_req_unref(&r->req);
 }
 
+static void scsi_do_read_cb(void *opaque, int ret)
+{
+    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+
+    assert (r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_do_read(opaque, ret);
+}
+
 /* Read more data from scsi device into buffer.  */
 static void scsi_read_data(SCSIRequest *req)
 {
@@ -384,7 +402,7 @@ static void scsi_read_data(SCSIRequest *req)
     if (first && scsi_is_cmd_fua(&r->req.cmd)) {
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
                          BLOCK_ACCT_FLUSH);
-        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_do_read, r);
+        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_do_read_cb, r);
     } else {
         scsi_do_read(r, 0);
     }
@@ -397,14 +415,17 @@ static void scsi_read_data(SCSIRequest *req)
  * scsi_handle_rw_error always manages its reference counts, independent
  * of the return value.
  */
-static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
+static int scsi_handle_rw_error(SCSIDiskReq *r, int error, bool acct_failed)
 {
-    bool is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
+    bool is_read = (r->req.cmd.mode == SCSI_XFER_FROM_DEV);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     BlockErrorAction action = blk_get_error_action(s->qdev.conf.blk,
                                                    is_read, error);
 
     if (action == BLOCK_ERROR_ACTION_REPORT) {
+        if (acct_failed) {
+            block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+        }
         switch (error) {
         case ENOMEDIUM:
             scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
@@ -430,23 +451,19 @@ static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
     return action != BLOCK_ERROR_ACTION_IGNORE;
 }
 
-static void scsi_write_complete(void * opaque, int ret)
+static void scsi_write_complete_noio(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint32_t n;
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
+    assert (r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -467,6 +484,22 @@ done:
     scsi_req_unref(&r->req);
 }
 
+static void scsi_write_complete(void * opaque, int ret)
+{
+    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+
+    assert (r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_write_complete_noio(r, ret);
+}
+
 static void scsi_write_data(SCSIRequest *req)
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
@@ -480,18 +513,18 @@ static void scsi_write_data(SCSIRequest *req)
     scsi_req_ref(&r->req);
     if (r->req.cmd.mode != SCSI_XFER_TO_DEV) {
         DPRINTF("Data transfer direction invalid\n");
-        scsi_write_complete(r, -EINVAL);
+        scsi_write_complete_noio(r, -EINVAL);
         return;
     }
 
     if (!r->req.sg && !r->qiov.size) {
         /* Called for the first time.  Ask the driver to send us more data.  */
         r->started = true;
-        scsi_write_complete(r, 0);
+        scsi_write_complete_noio(r, 0);
         return;
     }
     if (s->tray_open) {
-        scsi_write_complete(r, -ENOMEDIUM);
+        scsi_write_complete_noio(r, -ENOMEDIUM);
         return;
     }
 
@@ -500,7 +533,7 @@ static void scsi_write_data(SCSIRequest *req)
         if (r->req.sg) {
             scsi_dma_complete_noio(r, 0);
         } else {
-            scsi_write_complete(r, 0);
+            scsi_write_complete_noio(r, 0);
         }
         return;
     }
@@ -1557,22 +1590,24 @@ typedef struct UnmapCBData {
     int count;
 } UnmapCBData;
 
-static void scsi_unmap_complete(void *opaque, int ret)
+static void scsi_unmap_complete(void *opaque, int ret);
+
+static void scsi_unmap_complete_noio(UnmapCBData *data, int ret)
 {
-    UnmapCBData *data = opaque;
     SCSIDiskReq *r = data->r;
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint64_t sector_num;
     uint32_t nb_sectors;
 
-    r->req.aiocb = NULL;
+    assert(r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -1601,6 +1636,17 @@ done:
     g_free(data);
 }
 
+static void scsi_unmap_complete(void *opaque, int ret)
+{
+    UnmapCBData *data = opaque;
+    SCSIDiskReq *r = data->r;
+
+    assert(r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    scsi_unmap_complete_noio(data, ret);
+}
+
 static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
@@ -1638,7 +1684,7 @@ static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
 
     /* The matching unref is in scsi_unmap_complete, before data is freed.  */
     scsi_req_ref(&r->req);
-    scsi_unmap_complete(data, 0);
+    scsi_unmap_complete_noio(data, 0);
     return;
 
 invalid_param_len:
@@ -1665,18 +1711,19 @@ static void scsi_write_same_complete(void *opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, true)) {
             goto done;
         }
     }
 
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+
     data->nb_sectors -= data->iov.iov_len / 512;
     data->sector += data->iov.iov_len / 512;
     data->iov.iov_len = MIN(data->nb_sectors * 512, data->iov.iov_len);
@@ -2284,7 +2331,7 @@ static void scsi_realize(SCSIDevice *dev, Error **errp)
     }
 
     if (!s->version) {
-        s->version = g_strdup(qemu_get_version());
+        s->version = g_strdup(qemu_hw_version());
     }
     if (!s->vendor) {
         s->vendor = g_strdup("QEMU");
diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c
index e53470f85..a4626f72c 100644
--- a/hw/scsi/scsi-generic.c
+++ b/hw/scsi/scsi-generic.c
@@ -88,12 +88,12 @@ static void scsi_free_request(SCSIRequest *req)
 }
 
 /* Helper function for command completion.  */
-static void scsi_command_complete(void *opaque, int ret)
+static void scsi_command_complete_noio(SCSIGenericReq *r, int ret)
 {
     int status;
-    SCSIGenericReq *r = (SCSIGenericReq *)opaque;
 
-    r->req.aiocb = NULL;
+    assert(r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
@@ -142,6 +142,15 @@ done:
     scsi_req_unref(&r->req);
 }
 
+static void scsi_command_complete(void *opaque, int ret)
+{
+    SCSIGenericReq *r = (SCSIGenericReq *)opaque;
+
+    assert(r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+    scsi_command_complete_noio(r, ret);
+}
+
 static int execute_command(BlockBackend *blk,
                            SCSIGenericReq *r, int direction,
                            BlockCompletionFunc *complete)
@@ -172,33 +181,51 @@ static void scsi_read_complete(void * opaque, int ret)
     SCSIDevice *s = r->req.dev;
     int len;
 
+    assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
+
     if (ret || r->req.io_canceled) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
         return;
     }
+
     len = r->io_header.dxfer_len - r->io_header.resid;
     DPRINTF("Data ready tag=0x%x len=%d\n", r->req.tag, len);
 
     r->len = -1;
     if (len == 0) {
-        scsi_command_complete(r, 0);
-    } else {
-        /* Snoop READ CAPACITY output to set the blocksize.  */
-        if (r->req.cmd.buf[0] == READ_CAPACITY_10 &&
-            (ldl_be_p(&r->buf[0]) != 0xffffffffU || s->max_lba == 0)) {
-            s->blocksize = ldl_be_p(&r->buf[4]);
-            s->max_lba = ldl_be_p(&r->buf[0]) & 0xffffffffULL;
-        } else if (r->req.cmd.buf[0] == SERVICE_ACTION_IN_16 &&
-                   (r->req.cmd.buf[1] & 31) == SAI_READ_CAPACITY_16) {
-            s->blocksize = ldl_be_p(&r->buf[8]);
-            s->max_lba = ldq_be_p(&r->buf[0]);
-        }
-        blk_set_guest_block_size(s->conf.blk, s->blocksize);
+        scsi_command_complete_noio(r, 0);
+        return;
+    }
 
-        scsi_req_data(&r->req, len);
-        scsi_req_unref(&r->req);
+    /* Snoop READ CAPACITY output to set the blocksize.  */
+    if (r->req.cmd.buf[0] == READ_CAPACITY_10 &&
+        (ldl_be_p(&r->buf[0]) != 0xffffffffU || s->max_lba == 0)) {
+        s->blocksize = ldl_be_p(&r->buf[4]);
+        s->max_lba = ldl_be_p(&r->buf[0]) & 0xffffffffULL;
+    } else if (r->req.cmd.buf[0] == SERVICE_ACTION_IN_16 &&
+               (r->req.cmd.buf[1] & 31) == SAI_READ_CAPACITY_16) {
+        s->blocksize = ldl_be_p(&r->buf[8]);
+        s->max_lba = ldq_be_p(&r->buf[0]);
     }
+    blk_set_guest_block_size(s->conf.blk, s->blocksize);
+
+    /* Patch MODE SENSE device specific parameters if the BDS is opened
+     * readonly.
+     */
+    if ((s->type == TYPE_DISK || s->type == TYPE_TAPE) &&
+        blk_is_read_only(s->conf.blk) &&
+        (r->req.cmd.buf[0] == MODE_SENSE ||
+         r->req.cmd.buf[0] == MODE_SENSE_10) &&
+        (r->req.cmd.buf[1] & 0x8) == 0) {
+        if (r->req.cmd.buf[0] == MODE_SENSE) {
+            r->buf[2] |= 0x80;
+        } else  {
+            r->buf[3] |= 0x80;
+        }
+    }
+    scsi_req_data(&r->req, len);
+    scsi_req_unref(&r->req);
 }
 
 /* Read more data from scsi device into buffer.  */
@@ -213,14 +240,14 @@ static void scsi_read_data(SCSIRequest *req)
     /* The request is used as the AIO opaque value, so add a ref.  */
     scsi_req_ref(&r->req);
     if (r->len == -1) {
-        scsi_command_complete(r, 0);
+        scsi_command_complete_noio(r, 0);
         return;
     }
 
     ret = execute_command(s->conf.blk, r, SG_DXFER_FROM_DEV,
                           scsi_read_complete);
     if (ret < 0) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
     }
 }
 
@@ -230,9 +257,12 @@ static void scsi_write_complete(void * opaque, int ret)
     SCSIDevice *s = r->req.dev;
 
     DPRINTF("scsi_write_complete() ret = %d\n", ret);
+
+    assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
+
     if (ret || r->req.io_canceled) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
         return;
     }
 
@@ -242,7 +272,7 @@ static void scsi_write_complete(void * opaque, int ret)
         DPRINTF("block size %d\n", s->blocksize);
     }
 
-    scsi_command_complete(r, ret);
+    scsi_command_complete_noio(r, ret);
 }
 
 /* Write data to a scsi device.  Returns nonzero on failure.
@@ -264,7 +294,7 @@ static void scsi_write_data(SCSIRequest *req)
     scsi_req_ref(&r->req);
     ret = execute_command(s->conf.blk, r, SG_DXFER_TO_DEV, scsi_write_complete);
     if (ret < 0) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
     }
 }
 
@@ -306,7 +336,7 @@ static int32_t scsi_send_command(SCSIRequest *req, uint8_t *cmd)
         ret = execute_command(s->conf.blk, r, SG_DXFER_NONE,
                               scsi_command_complete);
         if (ret < 0) {
-            scsi_command_complete(r, ret);
+            scsi_command_complete_noio(r, ret);
             return 0;
         }
         return 0;
diff --git a/hw/scsi/spapr_vscsi.c b/hw/scsi/spapr_vscsi.c
index 891424fae..f4f5140a4 100644
--- a/hw/scsi/spapr_vscsi.c
+++ b/hw/scsi/spapr_vscsi.c
@@ -750,7 +750,6 @@ static void vscsi_report_luns(VSCSIState *s, vscsi_req *req)
     len = n+8;
 
     resp_data = g_malloc0(len);
-    memset(resp_data, 0, len);
     stl_be_p(resp_data, n);
     i = found_lun0 ? 8 : 16;
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index 62d91003b..00cdac62f 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -26,6 +26,7 @@
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
 #include "hw/fw-path-provider.h"
+#include "linux/vhost.h"
 
 /* Features supported by host kernel. */
 static const int kernel_feature_bits[] = {
@@ -45,7 +46,7 @@ static int vhost_scsi_set_endpoint(VHostSCSI *s)
 
     memset(&backend, 0, sizeof(backend));
     pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->conf.wwpn);
-    ret = vhost_ops->vhost_call(&s->dev, VHOST_SCSI_SET_ENDPOINT, &backend);
+    ret = vhost_ops->vhost_scsi_set_endpoint(&s->dev, &backend);
     if (ret < 0) {
         return -errno;
     }
@@ -60,7 +61,7 @@ static void vhost_scsi_clear_endpoint(VHostSCSI *s)
 
     memset(&backend, 0, sizeof(backend));
     pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->conf.wwpn);
-    vhost_ops->vhost_call(&s->dev, VHOST_SCSI_CLEAR_ENDPOINT, &backend);
+    vhost_ops->vhost_scsi_clear_endpoint(&s->dev, &backend);
 }
 
 static int vhost_scsi_start(VHostSCSI *s)
@@ -76,8 +77,7 @@ static int vhost_scsi_start(VHostSCSI *s)
         return -ENOSYS;
     }
 
-    ret = vhost_ops->vhost_call(&s->dev,
-                                VHOST_SCSI_GET_ABI_VERSION, &abi_version);
+    ret = vhost_ops->vhost_scsi_get_abi_version(&s->dev, &abi_version);
     if (ret < 0) {
         return -errno;
     }
@@ -118,7 +118,7 @@ static int vhost_scsi_start(VHostSCSI *s)
      * enabling/disabling irqfd.
      */
     for (i = 0; i < s->dev.nvqs; i++) {
-        vhost_virtqueue_mask(&s->dev, vdev, i, false);
+        vhost_virtqueue_mask(&s->dev, vdev, s->dev.vq_index + i, false);
     }
 
     return ret;
@@ -277,6 +277,7 @@ static void vhost_scsi_unrealize(DeviceState *dev, Error **errp)
     /* This will stop vhost backend. */
     vhost_scsi_set_status(vdev, 0);
 
+    vhost_dev_cleanup(&s->dev);
     g_free(s->dev.vqs);
 
     virtio_scsi_common_unrealize(dev, errp);
diff --git a/hw/scsi/virtio-scsi-dataplane.c b/hw/scsi/virtio-scsi-dataplane.c
index 5575648a9..0d8d71e24 100644
--- a/hw/scsi/virtio-scsi-dataplane.c
+++ b/hw/scsi/virtio-scsi-dataplane.c
@@ -57,10 +57,10 @@ static VirtIOSCSIVring *virtio_scsi_vring_init(VirtIOSCSI *s,
         return NULL;
     }
 
-    r = g_slice_new(VirtIOSCSIVring);
+    r = g_new(VirtIOSCSIVring, 1);
     r->host_notifier = *virtio_queue_get_host_notifier(vq);
     r->guest_notifier = *virtio_queue_get_guest_notifier(vq);
-    aio_set_event_notifier(s->ctx, &r->host_notifier, handler);
+    aio_set_event_notifier(s->ctx, &r->host_notifier, true, handler);
 
     r->parent = s;
 
@@ -71,9 +71,9 @@ static VirtIOSCSIVring *virtio_scsi_vring_init(VirtIOSCSI *s,
     return r;
 
 fail_vring:
-    aio_set_event_notifier(s->ctx, &r->host_notifier, NULL);
+    aio_set_event_notifier(s->ctx, &r->host_notifier, true, NULL);
     k->set_host_notifier(qbus->parent, n, false);
-    g_slice_free(VirtIOSCSIVring, r);
+    g_free(r);
     return NULL;
 }
 
@@ -162,14 +162,17 @@ static void virtio_scsi_clear_aio(VirtIOSCSI *s)
     int i;
 
     if (s->ctrl_vring) {
-        aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
+        aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier,
+                               true, NULL);
     }
     if (s->event_vring) {
-        aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
+        aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier,
+                               true, NULL);
     }
     if (s->cmd_vrings) {
         for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
-            aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
+            aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier,
+                                   true, NULL);
         }
     }
 }
@@ -182,18 +185,18 @@ static void virtio_scsi_vring_teardown(VirtIOSCSI *s)
 
     if (s->ctrl_vring) {
         vring_teardown(&s->ctrl_vring->vring, vdev, 0);
-        g_slice_free(VirtIOSCSIVring, s->ctrl_vring);
+        g_free(s->ctrl_vring);
         s->ctrl_vring = NULL;
     }
     if (s->event_vring) {
         vring_teardown(&s->event_vring->vring, vdev, 1);
-        g_slice_free(VirtIOSCSIVring, s->event_vring);
+        g_free(s->event_vring);
         s->event_vring = NULL;
     }
     if (s->cmd_vrings) {
         for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
             vring_teardown(&s->cmd_vrings[i]->vring, vdev, 2 + i);
-            g_slice_free(VirtIOSCSIVring, s->cmd_vrings[i]);
+            g_free(s->cmd_vrings[i]);
             s->cmd_vrings[i] = NULL;
         }
         free(s->cmd_vrings);
@@ -290,10 +293,13 @@ void virtio_scsi_dataplane_stop(VirtIOSCSI *s)
 
     aio_context_acquire(s->ctx);
 
-    aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
-    aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
+    aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier,
+                           true, NULL);
+    aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier,
+                           true, NULL);
     for (i = 0; i < vs->conf.num_queues; i++) {
-        aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
+        aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier,
+                               true, NULL);
     }
 
     blk_drain_all(); /* ensure there are no in-flight requests */
diff --git a/hw/scsi/virtio-scsi.c b/hw/scsi/virtio-scsi.c
index 93c33e11f..3a4f520fb 100644
--- a/hw/scsi/virtio-scsi.c
+++ b/hw/scsi/virtio-scsi.c
@@ -47,7 +47,7 @@ VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq)
     const size_t zero_skip = offsetof(VirtIOSCSIReq, elem)
                              + sizeof(VirtQueueElement);
 
-    req = g_slice_alloc(sizeof(*req) + vs->cdb_size);
+    req = g_malloc(sizeof(*req) + vs->cdb_size);
     req->vq = vq;
     req->dev = s;
     qemu_sglist_init(&req->qsgl, DEVICE(s), 8, &address_space_memory);
@@ -58,11 +58,9 @@ VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq)
 
 void virtio_scsi_free_req(VirtIOSCSIReq *req)
 {
-    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)req->dev;
-
     qemu_iovec_destroy(&req->resp_iov);
     qemu_sglist_destroy(&req->qsgl);
-    g_slice_free1(sizeof(*req) + vs->cdb_size, req);
+    g_free(req);
 }
 
 static void virtio_scsi_complete_req(VirtIOSCSIReq *req)
@@ -207,15 +205,8 @@ static void *virtio_scsi_load_request(QEMUFile *f, SCSIRequest *sreq)
     assert(n < vs->conf.num_queues);
     req = virtio_scsi_init_req(s, vs->cmd_vqs[n]);
     qemu_get_buffer(f, (unsigned char *)&req->elem, sizeof(req->elem));
-    /* TODO: add a way for SCSIBusInfo's load_request to fail,
-     * and fail migration instead of asserting here.
-     * When we do, we might be able to re-enable NDEBUG below.
-     */
-#ifdef NDEBUG
-#error building with NDEBUG is not supported
-#endif
-    assert(req->elem.in_num <= ARRAY_SIZE(req->elem.in_sg));
-    assert(req->elem.out_num <= ARRAY_SIZE(req->elem.out_sg));
+
+    virtqueue_map(&req->elem);
 
     if (virtio_scsi_parse_req(req, sizeof(VirtIOSCSICmdReq) + vs->cdb_size,
                               sizeof(VirtIOSCSICmdResp) + vs->sense_size) < 0) {
@@ -245,7 +236,7 @@ static void virtio_scsi_cancel_notify(Notifier *notifier, void *data)
     if (--n->tmf_req->remaining == 0) {
         virtio_scsi_complete_req(n->tmf_req);
     }
-    g_slice_free(VirtIOSCSICancelNotifier, n);
+    g_free(n);
 }
 
 /* Return 0 if the request is ready to be completed and return to guest;
@@ -259,7 +250,7 @@ static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
     int target;
     int ret = 0;
 
-    if (s->dataplane_started) {
+    if (s->dataplane_started && d) {
         assert(blk_get_aio_context(d->conf.blk) == s->ctx);
     }
     /* Here VIRTIO_SCSI_S_OK means "FUNCTION COMPLETE".  */
@@ -296,7 +287,7 @@ static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
                 VirtIOSCSICancelNotifier *notifier;
 
                 req->remaining = 1;
-                notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                notifier = g_new(VirtIOSCSICancelNotifier, 1);
                 notifier->tmf_req = req;
                 notifier->notifier.notify = virtio_scsi_cancel_notify;
                 scsi_req_cancel_async(r, &notifier->notifier);
@@ -345,7 +336,7 @@ static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
                     VirtIOSCSICancelNotifier *notifier;
 
                     req->remaining++;
-                    notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                    notifier = g_new(VirtIOSCSICancelNotifier, 1);
                     notifier->notifier.notify = virtio_scsi_cancel_notify;
                     notifier->tmf_req = req;
                     scsi_req_cancel_async(r, &notifier->notifier);
diff --git a/hw/sd/milkymist-memcard.c b/hw/sd/milkymist-memcard.c
index 2209ef1d5..b430d5668 100644
--- a/hw/sd/milkymist-memcard.c
+++ b/hw/sd/milkymist-memcard.c
@@ -28,7 +28,7 @@
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 enum {
     ENABLE_CMD_TX   = (1<<0),
diff --git a/hw/sd/omap_mmc.c b/hw/sd/omap_mmc.c
index d072deca1..5bc47193f 100644
--- a/hw/sd/omap_mmc.c
+++ b/hw/sd/omap_mmc.c
@@ -18,7 +18,7 @@
  */
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 struct omap_mmc_s {
     qemu_irq irq;
@@ -578,8 +578,7 @@ struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 BlockBackend *blk,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk)
 {
-    struct omap_mmc_s *s = (struct omap_mmc_s *)
-            g_malloc0(sizeof(struct omap_mmc_s));
+    struct omap_mmc_s *s = g_new0(struct omap_mmc_s, 1);
 
     s->irq = irq;
     s->dma = dma;
@@ -605,8 +604,7 @@ struct omap_mmc_s *omap2_mmc_init(struct omap_target_agent_s *ta,
                 BlockBackend *blk, qemu_irq irq, qemu_irq dma[],
                 omap_clk fclk, omap_clk iclk)
 {
-    struct omap_mmc_s *s = (struct omap_mmc_s *)
-            g_malloc0(sizeof(struct omap_mmc_s));
+    struct omap_mmc_s *s = g_new0(struct omap_mmc_s, 1);
 
     s->irq = irq;
     s->dma = dma;
diff --git a/hw/sd/pl181.c b/hw/sd/pl181.c
index 11fcd479d..326c53ad9 100644
--- a/hw/sd/pl181.c
+++ b/hw/sd/pl181.c
@@ -10,7 +10,7 @@
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/sysbus.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 //#define DEBUG_PL181 1
 
@@ -46,7 +46,7 @@ typedef struct PL181State {
     int32_t fifo_pos;
     int32_t fifo_len;
     /* The linux 2.6.21 driver is buggy, and misbehaves if new data arrives
-       while it is reading the FIFO.  We hack around this be defering
+       while it is reading the FIFO.  We hack around this by deferring
        subsequent transfers until after the driver polls the status word.
        http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=4446/1
      */
diff --git a/hw/sd/pxa2xx_mmci.c b/hw/sd/pxa2xx_mmci.c
index d1fe6d58e..b21708007 100644
--- a/hw/sd/pxa2xx_mmci.c
+++ b/hw/sd/pxa2xx_mmci.c
@@ -12,7 +12,7 @@
 
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 #include "hw/qdev.h"
 
 struct PXA2xxMMCIState {
diff --git a/hw/sd/sd.c b/hw/sd/sd.c
index a1ff465a6..1a9935cf9 100644
--- a/hw/sd/sd.c
+++ b/hw/sd/sd.c
@@ -31,7 +31,7 @@
 
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 #include "qemu/bitmap.h"
 
 //#define DEBUG_SD 1
@@ -412,8 +412,7 @@ static void sd_reset(SDState *sd)
     sd_set_cardstatus(sd);
     sd_set_sdstatus(sd);
 
-    if (sd->wp_groups)
-        g_free(sd->wp_groups);
+    g_free(sd->wp_groups);
     sd->wp_switch = sd->blk ? blk_is_read_only(sd->blk) : false;
     sd->wpgrps_size = sect;
     sd->wp_groups = bitmap_new(sd->wpgrps_size);
@@ -493,7 +492,10 @@ SDState *sd_init(BlockBackend *blk, bool is_spi)
     sd->blk = blk;
     sd_reset(sd);
     if (sd->blk) {
-        blk_attach_dev_nofail(sd->blk, sd);
+        /* Attach dev if not already attached.  (This call ignores an
+         * error return code if sd->blk is already attached.) */
+        /* FIXME ignoring blk_attach_dev() failure is dangerously brittle */
+        blk_attach_dev(sd->blk, sd);
         blk_set_dev_ops(sd->blk, &sd_block_ops, sd);
     }
     vmstate_register(NULL, -1, &sd_vmstate, sd);
diff --git a/hw/sd/sdhci.h b/hw/sd/sdhci-internal.h
index 3352d23d6..c712daf4e 100644
--- a/hw/sd/sdhci.h
+++ b/hw/sd/sdhci-internal.h
@@ -21,14 +21,10 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef SDHCI_INTERNAL_H
+#define SDHCI_INTERNAL_H
 
-#ifndef SDHCI_H
-#define SDHCI_H
-
-#include "qemu-common.h"
-#include "hw/pci/pci.h"
-#include "hw/sysbus.h"
-#include "hw/sd.h"
+#include "hw/sd/sdhci.h"
 
 /* R/W SDMA System Address register 0x0 */
 #define SDHC_SYSAD                     0x00
@@ -231,65 +227,6 @@ enum {
     sdhc_gap_write  = 2   /* SDHC stopped at block gap during write operation */
 };
 
-/* SD/MMC host controller state */
-typedef struct SDHCIState {
-    union {
-        PCIDevice pcidev;
-        SysBusDevice busdev;
-    };
-    SDState *card;
-    MemoryRegion iomem;
-
-    QEMUTimer *insert_timer;       /* timer for 'changing' sd card. */
-    QEMUTimer *transfer_timer;
-    qemu_irq eject_cb;
-    qemu_irq ro_cb;
-    qemu_irq irq;
-
-    uint32_t sdmasysad;    /* SDMA System Address register */
-    uint16_t blksize;      /* Host DMA Buff Boundary and Transfer BlkSize Reg */
-    uint16_t blkcnt;       /* Blocks count for current transfer */
-    uint32_t argument;     /* Command Argument Register */
-    uint16_t trnmod;       /* Transfer Mode Setting Register */
-    uint16_t cmdreg;       /* Command Register */
-    uint32_t rspreg[4];    /* Response Registers 0-3 */
-    uint32_t prnsts;       /* Present State Register */
-    uint8_t  hostctl;      /* Host Control Register */
-    uint8_t  pwrcon;       /* Power control Register */
-    uint8_t  blkgap;       /* Block Gap Control Register */
-    uint8_t  wakcon;       /* WakeUp Control Register */
-    uint16_t clkcon;       /* Clock control Register */
-    uint8_t  timeoutcon;   /* Timeout Control Register */
-    uint8_t  admaerr;      /* ADMA Error Status Register */
-    uint16_t norintsts;    /* Normal Interrupt Status Register */
-    uint16_t errintsts;    /* Error Interrupt Status Register */
-    uint16_t norintstsen;  /* Normal Interrupt Status Enable Register */
-    uint16_t errintstsen;  /* Error Interrupt Status Enable Register */
-    uint16_t norintsigen;  /* Normal Interrupt Signal Enable Register */
-    uint16_t errintsigen;  /* Error Interrupt Signal Enable Register */
-    uint16_t acmd12errsts; /* Auto CMD12 error status register */
-    uint64_t admasysaddr;  /* ADMA System Address Register */
-
-    uint32_t capareg;      /* Capabilities Register */
-    uint32_t maxcurr;      /* Maximum Current Capabilities Register */
-    uint8_t  *fifo_buffer; /* SD host i/o FIFO buffer */
-    uint32_t buf_maxsz;
-    uint16_t data_count;   /* current element in FIFO buffer */
-    uint8_t  stopped_state;/* Current SDHC state */
-    /* Buffer Data Port Register - virtual access point to R and W buffers */
-    /* Software Reset Register - always reads as 0 */
-    /* Force Event Auto CMD12 Error Interrupt Reg - write only */
-    /* Force Event Error Interrupt Register- write only */
-    /* RO Host Controller Version Register always reads as 0x2401 */
-} SDHCIState;
-
 extern const VMStateDescription sdhci_vmstate;
 
-#define TYPE_PCI_SDHCI "sdhci-pci"
-#define PCI_SDHCI(obj) OBJECT_CHECK(SDHCIState, (obj), TYPE_PCI_SDHCI)
-
-#define TYPE_SYSBUS_SDHCI "generic-sdhci"
-#define SYSBUS_SDHCI(obj)                               \
-     OBJECT_CHECK(SDHCIState, (obj), TYPE_SYSBUS_SDHCI)
-
-#endif /* SDHCI_H */
+#endif
diff --git a/hw/sd/sdhci.c b/hw/sd/sdhci.c
index e63367ba5..861276065 100644
--- a/hw/sd/sdhci.c
+++ b/hw/sd/sdhci.c
@@ -22,38 +22,38 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <inttypes.h>
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/dma.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
-
-#include "sdhci.h"
+#include "sdhci-internal.h"
 
 /* host controller debug messages */
 #ifndef SDHC_DEBUG
 #define SDHC_DEBUG                        0
 #endif
 
-#if SDHC_DEBUG == 0
-    #define DPRINT_L1(fmt, args...)       do { } while (0)
-    #define DPRINT_L2(fmt, args...)       do { } while (0)
-    #define ERRPRINT(fmt, args...)        do { } while (0)
-#elif SDHC_DEBUG == 1
-    #define DPRINT_L1(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define DPRINT_L2(fmt, args...)       do { } while (0)
-    #define ERRPRINT(fmt, args...)        \
-        do {fprintf(stderr, "QEMU SDHC ERROR: "fmt, ## args); } while (0)
-#else
-    #define DPRINT_L1(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define DPRINT_L2(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define ERRPRINT(fmt, args...)        \
-        do {fprintf(stderr, "QEMU SDHC ERROR: "fmt, ## args); } while (0)
-#endif
+#define DPRINT_L1(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG) { \
+            fprintf(stderr, "QEMU SDHC: " fmt, ## args); \
+        } \
+    } while (0)
+#define DPRINT_L2(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG > 1) { \
+            fprintf(stderr, "QEMU SDHC: " fmt, ## args); \
+        } \
+    } while (0)
+#define ERRPRINT(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG) { \
+            fprintf(stderr, "QEMU SDHC ERROR: " fmt, ## args); \
+        } \
+    } while (0)
 
 /* Default SD/MMC host controller features information, which will be
  * presented in CAPABILITIES register of generic SD host controller at reset.
@@ -719,7 +719,8 @@ static void sdhci_do_adma(SDHCIState *s)
             break;
         case SDHC_ADMA_ATTR_ACT_LINK:   /* link to next descriptor table */
             s->admasysaddr = dscr.addr;
-            DPRINT_L1("ADMA link: admasysaddr=0x%lx\n", s->admasysaddr);
+            DPRINT_L1("ADMA link: admasysaddr=0x%" PRIx64 "\n",
+                      s->admasysaddr);
             break;
         default:
             s->admasysaddr += dscr.incr;
@@ -727,7 +728,8 @@ static void sdhci_do_adma(SDHCIState *s)
         }
 
         if (dscr.attr & SDHC_ADMA_ATTR_INT) {
-            DPRINT_L1("ADMA interrupt: admasysaddr=0x%lx\n", s->admasysaddr);
+            DPRINT_L1("ADMA interrupt: admasysaddr=0x%" PRIx64 "\n",
+                      s->admasysaddr);
             if (s->norintstsen & SDHC_NISEN_DMA) {
                 s->norintsts |= SDHC_NIS_DMA;
             }
@@ -1006,6 +1008,16 @@ sdhci_write(void *opaque, hwaddr offset, uint64_t val, unsigned size)
             MASKED_WRITE(s->blksize, mask, value);
             MASKED_WRITE(s->blkcnt, mask >> 16, value >> 16);
         }
+
+        /* Limit block size to the maximum buffer size */
+        if (extract32(s->blksize, 0, 12) > s->buf_maxsz) {
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: Size 0x%x is larger than " \
+                          "the maximum buffer 0x%x", __func__, s->blksize,
+                          s->buf_maxsz);
+
+            s->blksize = deposit32(s->blksize, 0, 12, s->buf_maxsz);
+        }
+
         break;
     case SDHC_ARGUMENT:
         MASKED_WRITE(s->argument, mask, value);
@@ -1142,13 +1154,9 @@ static inline unsigned int sdhci_get_fifolen(SDHCIState *s)
     }
 }
 
-static void sdhci_initfn(SDHCIState *s)
+static void sdhci_initfn(SDHCIState *s, BlockBackend *blk)
 {
-    DriveInfo *di;
-
-    /* FIXME use a qdev drive property instead of drive_get_next() */
-    di = drive_get_next(IF_SD);
-    s->card = sd_init(di ? blk_by_legacy_dinfo(di) : NULL, false);
+    s->card = sd_init(blk, false);
     if (s->card == NULL) {
         exit(1);
     }
@@ -1169,10 +1177,8 @@ static void sdhci_uninitfn(SDHCIState *s)
     qemu_free_irq(s->eject_cb);
     qemu_free_irq(s->ro_cb);
 
-    if (s->fifo_buffer) {
-        g_free(s->fifo_buffer);
-        s->fifo_buffer = NULL;
-    }
+    g_free(s->fifo_buffer);
+    s->fifo_buffer = NULL;
 }
 
 const VMStateDescription sdhci_vmstate = {
@@ -1214,7 +1220,13 @@ const VMStateDescription sdhci_vmstate = {
 
 /* Capabilities registers provide information on supported features of this
  * specific host controller implementation */
-static Property sdhci_properties[] = {
+static Property sdhci_pci_properties[] = {
+    /*
+     * We currently fuse controller and card into a single device
+     * model, but we intend to separate them.  For that purpose, the
+     * properties that belong to the card are marked as experimental.
+     */
+    DEFINE_PROP_DRIVE("x-drive", SDHCIState, blk),
     DEFINE_PROP_UINT32("capareg", SDHCIState, capareg,
             SDHC_CAPAB_REG_DEFAULT),
     DEFINE_PROP_UINT32("maxcurr", SDHCIState, maxcurr, 0),
@@ -1226,7 +1238,7 @@ static void sdhci_pci_realize(PCIDevice *dev, Error **errp)
     SDHCIState *s = PCI_SDHCI(dev);
     dev->config[PCI_CLASS_PROG] = 0x01; /* Standard Host supported DMA */
     dev->config[PCI_INTERRUPT_PIN] = 0x01; /* interrupt pin A */
-    sdhci_initfn(s);
+    sdhci_initfn(s, s->blk);
     s->buf_maxsz = sdhci_get_fifolen(s);
     s->fifo_buffer = g_malloc0(s->buf_maxsz);
     s->irq = pci_allocate_irq(dev);
@@ -1253,9 +1265,7 @@ static void sdhci_pci_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_SYSTEM_SDHCI;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     dc->vmsd = &sdhci_vmstate;
-    dc->props = sdhci_properties;
-    /* Reason: realize() method uses drive_get_next() */
-    dc->cannot_instantiate_with_device_add_yet = true;
+    dc->props = sdhci_pci_properties;
 }
 
 static const TypeInfo sdhci_pci_info = {
@@ -1265,10 +1275,21 @@ static const TypeInfo sdhci_pci_info = {
     .class_init = sdhci_pci_class_init,
 };
 
+static Property sdhci_sysbus_properties[] = {
+    DEFINE_PROP_UINT32("capareg", SDHCIState, capareg,
+            SDHC_CAPAB_REG_DEFAULT),
+    DEFINE_PROP_UINT32("maxcurr", SDHCIState, maxcurr, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void sdhci_sysbus_init(Object *obj)
 {
     SDHCIState *s = SYSBUS_SDHCI(obj);
-    sdhci_initfn(s);
+    DriveInfo *di;
+
+    /* FIXME use a qdev drive property instead of drive_get_next() */
+    di = drive_get_next(IF_SD);
+    sdhci_initfn(s, di ? blk_by_legacy_dinfo(di) : NULL);
 }
 
 static void sdhci_sysbus_finalize(Object *obj)
@@ -1295,7 +1316,7 @@ static void sdhci_sysbus_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     dc->vmsd = &sdhci_vmstate;
-    dc->props = sdhci_properties;
+    dc->props = sdhci_sysbus_properties;
     dc->realize = sdhci_sysbus_realize;
     /* Reason: instance_init() method uses drive_get_next() */
     dc->cannot_instantiate_with_device_add_yet = true;
diff --git a/hw/sd/ssi-sd.c b/hw/sd/ssi-sd.c
index e4b2d4f83..c49ff62f5 100644
--- a/hw/sd/ssi-sd.c
+++ b/hw/sd/ssi-sd.c
@@ -13,7 +13,7 @@
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/ssi.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 //#define DEBUG_SSI_SD 1
 
diff --git a/hw/sh4/r2d.c b/hw/sh4/r2d.c
index 5e22ed79b..c1ff9a0cf 100644
--- a/hw/sh4/r2d.c
+++ b/hw/sh4/r2d.c
@@ -255,7 +255,7 @@ static void r2d_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 
     /* Allocate memory space */
-    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE, &error_abort);
+    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(sdram);
     memory_region_add_subregion(address_space_mem, SDRAM_BASE, sdram);
     /* Register peripherals */
@@ -338,9 +338,9 @@ static void r2d_init(MachineState *machine)
         }
 
         /* initialization which should be done by firmware */
-        boot_params.loader_type = 1;
-        boot_params.initrd_start = INITRD_LOAD_OFFSET;
-        boot_params.initrd_size = initrd_size;
+        boot_params.loader_type = tswap32(1);
+        boot_params.initrd_start = tswap32(INITRD_LOAD_OFFSET);
+        boot_params.initrd_size = tswap32(initrd_size);
     }
 
     if (kernel_cmdline) {
@@ -354,15 +354,10 @@ static void r2d_init(MachineState *machine)
                        SDRAM_BASE + BOOT_PARAMS_OFFSET);
 }
 
-static QEMUMachine r2d_machine = {
-    .name = "r2d",
-    .desc = "r2d-plus board",
-    .init = r2d_init,
-};
-
-static void r2d_machine_init(void)
+static void r2d_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&r2d_machine);
+    mc->desc = "r2d-plus board";
+    mc->init = r2d_init;
 }
 
-machine_init(r2d_machine_init);
+DEFINE_MACHINE("r2d", r2d_machine_init)
diff --git a/hw/sh4/shix.c b/hw/sh4/shix.c
index f93f98e56..d508be958 100644
--- a/hw/sh4/shix.c
+++ b/hw/sh4/shix.c
@@ -59,16 +59,16 @@ static void shix_init(MachineState *machine)
     }
 
     /* Allocate memory space */
-    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000, &error_abort);
+    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(sysmem, 0x00000000, rom);
     memory_region_init_ram(&sdram[0], NULL, "shix.sdram1", 0x01000000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&sdram[0]);
     memory_region_add_subregion(sysmem, 0x08000000, &sdram[0]);
     memory_region_init_ram(&sdram[1], NULL, "shix.sdram2", 0x01000000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&sdram[1]);
     memory_region_add_subregion(sysmem, 0x0c000000, &sdram[1]);
 
@@ -87,16 +87,11 @@ static void shix_init(MachineState *machine)
     tc58128_init(s, "shix_linux_nand.bin", NULL);
 }
 
-static QEMUMachine shix_machine = {
-    .name = "shix",
-    .desc = "shix card",
-    .init = shix_init,
-    .is_default = 1,
-};
-
-static void shix_machine_init(void)
+static void shix_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&shix_machine);
+    mc->desc = "shix card";
+    mc->init = shix_init;
+    mc->is_default = 1;
 }
 
-machine_init(shix_machine_init);
+DEFINE_MACHINE("shix", shix_machine_init)
diff --git a/hw/smbios/Makefile.objs b/hw/smbios/Makefile.objs
new file mode 100644
index 000000000..f69a92f96
--- /dev/null
+++ b/hw/smbios/Makefile.objs
@@ -0,0 +1 @@
+common-obj-$(CONFIG_SMBIOS) += smbios.o
diff --git a/hw/i386/smbios.c b/hw/smbios/smbios.c
index 1341e0234..b81a1d349 100644
--- a/hw/i386/smbios.c
+++ b/hw/smbios/smbios.c
@@ -19,10 +19,9 @@
 #include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/cpus.h"
-#include "hw/i386/pc.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/loader.h"
-
+#include "exec/cpu-common.h"
 
 /* legacy structures and constants for <= 2.0 machines */
 struct smbios_header {
@@ -56,7 +55,9 @@ static uint8_t *smbios_tables;
 static size_t smbios_tables_len;
 static unsigned smbios_table_max;
 static unsigned smbios_table_cnt;
-static struct smbios_entry_point ep;
+static SmbiosEntryPointType smbios_ep_type = SMBIOS_ENTRY_POINT_21;
+
+static SmbiosEntryPoint ep;
 
 static int smbios_type4_count = 0;
 static bool smbios_immutable;
@@ -772,11 +773,12 @@ void smbios_set_cpuid(uint32_t version, uint32_t features)
 
 void smbios_set_defaults(const char *manufacturer, const char *product,
                          const char *version, bool legacy_mode,
-                         bool uuid_encoded)
+                         bool uuid_encoded, SmbiosEntryPointType ep_type)
 {
     smbios_have_defaults = true;
     smbios_legacy = legacy_mode;
     smbios_uuid_encoded = uuid_encoded;
+    smbios_ep_type = ep_type;
 
     /* drop unwanted version of command-line file blob(s) */
     if (smbios_legacy) {
@@ -809,32 +811,61 @@ void smbios_set_defaults(const char *manufacturer, const char *product,
 
 static void smbios_entry_point_setup(void)
 {
-    memcpy(ep.anchor_string, "_SM_", 4);
-    memcpy(ep.intermediate_anchor_string, "_DMI_", 5);
-    ep.length = sizeof(struct smbios_entry_point);
-    ep.entry_point_revision = 0; /* formatted_area reserved, per spec v2.1+ */
-    memset(ep.formatted_area, 0, 5);
-
-    /* compliant with smbios spec v2.8 */
-    ep.smbios_major_version = 2;
-    ep.smbios_minor_version = 8;
-    ep.smbios_bcd_revision = 0x28;
-
-    /* set during table construction, but BIOS may override: */
-    ep.structure_table_length = cpu_to_le16(smbios_tables_len);
-    ep.max_structure_size = cpu_to_le16(smbios_table_max);
-    ep.number_of_structures = cpu_to_le16(smbios_table_cnt);
-
-    /* BIOS must recalculate: */
-    ep.checksum = 0;
-    ep.intermediate_checksum = 0;
-    ep.structure_table_address = cpu_to_le32(0);
+    switch (smbios_ep_type) {
+    case SMBIOS_ENTRY_POINT_21:
+        memcpy(ep.ep21.anchor_string, "_SM_", 4);
+        memcpy(ep.ep21.intermediate_anchor_string, "_DMI_", 5);
+        ep.ep21.length = sizeof(struct smbios_21_entry_point);
+        ep.ep21.entry_point_revision = 0; /* formatted_area reserved */
+        memset(ep.ep21.formatted_area, 0, 5);
+
+        /* compliant with smbios spec v2.8 */
+        ep.ep21.smbios_major_version = 2;
+        ep.ep21.smbios_minor_version = 8;
+        ep.ep21.smbios_bcd_revision = 0x28;
+
+        /* set during table construction, but BIOS may override: */
+        ep.ep21.structure_table_length = cpu_to_le16(smbios_tables_len);
+        ep.ep21.max_structure_size = cpu_to_le16(smbios_table_max);
+        ep.ep21.number_of_structures = cpu_to_le16(smbios_table_cnt);
+
+        /* BIOS must recalculate */
+        ep.ep21.checksum = 0;
+        ep.ep21.intermediate_checksum = 0;
+        ep.ep21.structure_table_address = cpu_to_le32(0);
+
+        break;
+    case SMBIOS_ENTRY_POINT_30:
+        memcpy(ep.ep30.anchor_string, "_SM3_", 5);
+        ep.ep30.length = sizeof(struct smbios_30_entry_point);
+        ep.ep30.entry_point_revision = 1;
+        ep.ep30.reserved = 0;
+
+        /* compliant with smbios spec 3.0 */
+        ep.ep30.smbios_major_version = 3;
+        ep.ep30.smbios_minor_version = 0;
+        ep.ep30.smbios_doc_rev = 0;
+
+        /* set during table construct, but BIOS might override */
+        ep.ep30.structure_table_max_size = cpu_to_le32(smbios_tables_len);
+
+        /* BIOS must recalculate */
+        ep.ep30.checksum = 0;
+        ep.ep30.structure_table_address = cpu_to_le64(0);
+
+        break;
+    default:
+        abort();
+        break;
+    }
 }
 
-void smbios_get_tables(uint8_t **tables, size_t *tables_len,
+void smbios_get_tables(const struct smbios_phys_mem_area *mem_array,
+                       const unsigned int mem_array_size,
+                       uint8_t **tables, size_t *tables_len,
                        uint8_t **anchor, size_t *anchor_len)
 {
-    unsigned i, dimm_cnt, instance;
+    unsigned i, dimm_cnt;
 
     if (smbios_legacy) {
         *tables = *anchor = NULL;
@@ -867,11 +898,9 @@ void smbios_get_tables(uint8_t **tables, size_t *tables_len,
             smbios_build_type_17_table(i, GET_DIMM_SZ);
         }
 
-        for (i = 0, instance = 0; i < e820_get_num_entries(); i++) {
-            uint64_t address, length;
-            if (e820_get_entry(i, E820_RAM, &address, &length)) {
-                smbios_build_type_19_table(instance++, address, length);
-            }
+        for (i = 0; i < mem_array_size; i++) {
+            smbios_build_type_19_table(i, mem_array[i].address,
+                                       mem_array[i].length);
         }
 
         smbios_build_type_32_table();
@@ -886,7 +915,15 @@ void smbios_get_tables(uint8_t **tables, size_t *tables_len,
     *tables = smbios_tables;
     *tables_len = smbios_tables_len;
     *anchor = (uint8_t *)&ep;
-    *anchor_len = sizeof(struct smbios_entry_point);
+
+    /* calculate length based on anchor string */
+    if (!strncmp((char *)&ep, "_SM_", 4)) {
+        *anchor_len = sizeof(struct smbios_21_entry_point);
+    } else if (!strncmp((char *)&ep, "_SM3_", 5)) {
+        *anchor_len = sizeof(struct smbios_30_entry_point);
+    } else {
+        abort();
+    }
 }
 
 static void save_opt(const char **dest, QemuOpts *opts, const char *name)
diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index 7f5dcd6d8..22d1e7e31 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -156,7 +156,7 @@ static void leon3_generic_hw_init(MachineState *machine)
 
     /* Allocate BIOS */
     prom_size = 8 * 1024 * 1024; /* 8Mb */
-    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size, &error_abort);
+    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size, &error_fatal);
     vmstate_register_ram_global(prom);
     memory_region_set_readonly(prom, true);
     memory_region_add_subregion(address_space_mem, 0x00000000, prom);
@@ -193,7 +193,7 @@ static void leon3_generic_hw_init(MachineState *machine)
         uint64_t entry;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1 /* big endian */, ELF_MACHINE, 0);
+                               1 /* big endian */, EM_SPARC, 0);
         if (kernel_size < 0) {
             fprintf(stderr, "qemu: could not load kernel '%s'\n",
                     kernel_filename);
@@ -216,15 +216,10 @@ static void leon3_generic_hw_init(MachineState *machine)
     }
 }
 
-static QEMUMachine leon3_generic_machine = {
-    .name     = "leon3_generic",
-    .desc     = "Leon-3 generic",
-    .init     = leon3_generic_hw_init,
-};
-
-static void leon3_machine_init(void)
+static void leon3_generic_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&leon3_generic_machine);
+    mc->desc = "Leon-3 generic";
+    mc->init = leon3_generic_hw_init;
 }
 
-machine_init(leon3_machine_init);
+DEFINE_MACHINE("leon3_generic", leon3_generic_machine_init)
diff --git a/hw/sparc/sun4m.c b/hw/sparc/sun4m.c
index 68ac4d8bb..230dac955 100644
--- a/hw/sparc/sun4m.c
+++ b/hw/sparc/sun4m.c
@@ -109,9 +109,9 @@ int DMA_write_memory (int nchan, void *buf, int pos, int size)
 }
 void DMA_hold_DREQ (int nchan) {}
 void DMA_release_DREQ (int nchan) {}
-void DMA_schedule(int nchan) {}
+void DMA_schedule(void) {}
 
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
+void DMA_init(int high_page_enable)
 {
 }
 
@@ -300,7 +300,7 @@ static unsigned long sun4m_load_kernel(const char *kernel_filename,
         bswap_needed = 0;
 #endif
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, NULL, NULL, 1, ELF_MACHINE, 0);
+                               NULL, NULL, NULL, 1, EM_SPARC, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, KERNEL_LOAD_ADDR,
                                     RAM_size - KERNEL_LOAD_ADDR, bswap_needed,
@@ -641,7 +641,7 @@ static int idreg_init1(SysBusDevice *dev)
     IDRegState *s = MACIO_ID_REGISTER(dev);
 
     memory_region_init_ram(&s->mem, OBJECT(s),
-                           "sun4m.idreg", sizeof(idreg_data), &error_abort);
+                           "sun4m.idreg", sizeof(idreg_data), &error_fatal);
     vmstate_register_ram_global(&s->mem);
     memory_region_set_readonly(&s->mem, true);
     sysbus_init_mmio(dev, &s->mem);
@@ -688,7 +688,7 @@ static int afx_init1(SysBusDevice *dev)
 {
     AFXState *s = TCX_AFX(dev);
 
-    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4, &error_abort);
+    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4, &error_fatal);
     vmstate_register_ram_global(&s->mem);
     sysbus_init_mmio(dev, &s->mem);
     return 0;
@@ -744,7 +744,7 @@ static void prom_init(hwaddr addr, const char *bios_name)
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
     if (filename) {
         ret = load_elf(filename, translate_prom_address, &addr, NULL,
-                       NULL, NULL, 1, ELF_MACHINE, 0);
+                       NULL, NULL, 1, EM_SPARC, 0);
         if (ret < 0 || ret > PROM_SIZE_MAX) {
             ret = load_image_targphys(filename, addr, PROM_SIZE_MAX);
         }
@@ -763,7 +763,7 @@ static int prom_init1(SysBusDevice *dev)
     PROMState *s = OPENPROM(dev);
 
     memory_region_init_ram(&s->prom, OBJECT(s), "sun4m.prom", PROM_SIZE_MAX,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -1420,80 +1420,152 @@ static void sbook_init(MachineState *machine)
     sun4m_hw_init(&sun4m_hwdefs[8], machine);
 }
 
-static QEMUMachine ss5_machine = {
-    .name = "SS-5",
-    .desc = "Sun4m platform, SPARCstation 5",
-    .init = ss5_init,
-    .block_default_type = IF_SCSI,
-    .is_default = 1,
-    .default_boot_order = "c",
+static void ss5_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 5";
+    mc->init = ss5_init;
+    mc->block_default_type = IF_SCSI;
+    mc->is_default = 1;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss5_type = {
+    .name = MACHINE_TYPE_NAME("SS-5"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss5_class_init,
 };
 
-static QEMUMachine ss10_machine = {
-    .name = "SS-10",
-    .desc = "Sun4m platform, SPARCstation 10",
-    .init = ss10_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss10_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 10";
+    mc->init = ss10_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss10_type = {
+    .name = MACHINE_TYPE_NAME("SS-10"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss10_class_init,
 };
 
-static QEMUMachine ss600mp_machine = {
-    .name = "SS-600MP",
-    .desc = "Sun4m platform, SPARCserver 600MP",
-    .init = ss600mp_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss600mp_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCserver 600MP";
+    mc->init = ss600mp_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss600mp_type = {
+    .name = MACHINE_TYPE_NAME("SS-600MP"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss600mp_class_init,
 };
 
-static QEMUMachine ss20_machine = {
-    .name = "SS-20",
-    .desc = "Sun4m platform, SPARCstation 20",
-    .init = ss20_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss20_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 20";
+    mc->init = ss20_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss20_type = {
+    .name = MACHINE_TYPE_NAME("SS-20"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss20_class_init,
 };
 
-static QEMUMachine voyager_machine = {
-    .name = "Voyager",
-    .desc = "Sun4m platform, SPARCstation Voyager",
-    .init = vger_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void voyager_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation Voyager";
+    mc->init = vger_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo voyager_type = {
+    .name = MACHINE_TYPE_NAME("Voyager"),
+    .parent = TYPE_MACHINE,
+    .class_init = voyager_class_init,
 };
 
-static QEMUMachine ss_lx_machine = {
-    .name = "LX",
-    .desc = "Sun4m platform, SPARCstation LX",
-    .init = ss_lx_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void ss_lx_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation LX";
+    mc->init = ss_lx_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss_lx_type = {
+    .name = MACHINE_TYPE_NAME("LX"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss_lx_class_init,
 };
 
-static QEMUMachine ss4_machine = {
-    .name = "SS-4",
-    .desc = "Sun4m platform, SPARCstation 4",
-    .init = ss4_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void ss4_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 4";
+    mc->init = ss4_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss4_type = {
+    .name = MACHINE_TYPE_NAME("SS-4"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss4_class_init,
 };
 
-static QEMUMachine scls_machine = {
-    .name = "SPARCClassic",
-    .desc = "Sun4m platform, SPARCClassic",
-    .init = scls_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void scls_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCClassic";
+    mc->init = scls_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo scls_type = {
+    .name = MACHINE_TYPE_NAME("SPARCClassic"),
+    .parent = TYPE_MACHINE,
+    .class_init = scls_class_init,
 };
 
-static QEMUMachine sbook_machine = {
-    .name = "SPARCbook",
-    .desc = "Sun4m platform, SPARCbook",
-    .init = sbook_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void sbook_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCbook";
+    mc->init = sbook_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sbook_type = {
+    .name = MACHINE_TYPE_NAME("SPARCbook"),
+    .parent = TYPE_MACHINE,
+    .class_init = sbook_class_init,
 };
 
 static void sun4m_register_types(void)
@@ -1506,16 +1578,16 @@ static void sun4m_register_types(void)
 
 static void sun4m_machine_init(void)
 {
-    qemu_register_machine(&ss5_machine);
-    qemu_register_machine(&ss10_machine);
-    qemu_register_machine(&ss600mp_machine);
-    qemu_register_machine(&ss20_machine);
-    qemu_register_machine(&voyager_machine);
-    qemu_register_machine(&ss_lx_machine);
-    qemu_register_machine(&ss4_machine);
-    qemu_register_machine(&scls_machine);
-    qemu_register_machine(&sbook_machine);
+    type_register_static(&ss5_type);
+    type_register_static(&ss10_type);
+    type_register_static(&ss600mp_type);
+    type_register_static(&ss20_type);
+    type_register_static(&voyager_type);
+    type_register_static(&ss_lx_type);
+    type_register_static(&ss4_type);
+    type_register_static(&scls_type);
+    type_register_static(&sbook_type);
 }
 
 type_init(sun4m_register_types)
-machine_init(sun4m_machine_init);
+machine_init(sun4m_machine_init)
diff --git a/hw/sparc64/sun4u.c b/hw/sparc64/sun4u.c
index 30cfa0e0a..d6b929cf8 100644
--- a/hw/sparc64/sun4u.c
+++ b/hw/sparc64/sun4u.c
@@ -112,9 +112,9 @@ int DMA_write_memory (int nchan, void *buf, int pos, int size)
 }
 void DMA_hold_DREQ (int nchan) {}
 void DMA_release_DREQ (int nchan) {}
-void DMA_schedule(int nchan) {}
+void DMA_schedule(void) {}
 
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
+void DMA_init(int high_page_enable)
 {
 }
 
@@ -208,7 +208,7 @@ static uint64_t sun4u_load_kernel(const char *kernel_filename,
         bswap_needed = 0;
 #endif
         kernel_size = load_elf(kernel_filename, NULL, NULL, kernel_entry,
-                               kernel_addr, &kernel_top, 1, ELF_MACHINE, 0);
+                               kernel_addr, &kernel_top, 1, EM_SPARCV9, 0);
         if (kernel_size < 0) {
             *kernel_addr = KERNEL_LOAD_ADDR;
             *kernel_entry = KERNEL_LOAD_ADDR;
@@ -671,7 +671,7 @@ static void prom_init(hwaddr addr, const char *bios_name)
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
     if (filename) {
         ret = load_elf(filename, translate_prom_address, &addr,
-                       NULL, NULL, NULL, 1, ELF_MACHINE, 0);
+                       NULL, NULL, NULL, 1, EM_SPARCV9, 0);
         if (ret < 0 || ret > PROM_SIZE_MAX) {
             ret = load_image_targphys(filename, addr, PROM_SIZE_MAX);
         }
@@ -690,7 +690,7 @@ static int prom_init1(SysBusDevice *dev)
     PROMState *s = OPENPROM(dev);
 
     memory_region_init_ram(&s->prom, OBJECT(s), "sun4u.prom", PROM_SIZE_MAX,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -734,7 +734,7 @@ static int ram_init1(SysBusDevice *dev)
     RamDevice *d = SUN4U_RAM(dev);
 
     memory_region_init_ram(&d->ram, OBJECT(d), "sun4u.ram", d->size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&d->ram);
     sysbus_init_mmio(dev, &d->ram);
     return 0;
@@ -965,29 +965,53 @@ static void niagara_init(MachineState *machine)
     sun4uv_init(get_system_memory(), machine, &hwdefs[2]);
 }
 
-static QEMUMachine sun4u_machine = {
-    .name = "sun4u",
-    .desc = "Sun4u platform",
-    .init = sun4u_init,
-    .max_cpus = 1, // XXX for now
-    .is_default = 1,
-    .default_boot_order = "c",
+static void sun4u_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4u platform";
+    mc->init = sun4u_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->is_default = 1;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sun4u_type = {
+    .name = MACHINE_TYPE_NAME("sun4u"),
+    .parent = TYPE_MACHINE,
+    .class_init = sun4u_class_init,
 };
 
-static QEMUMachine sun4v_machine = {
-    .name = "sun4v",
-    .desc = "Sun4v platform",
-    .init = sun4v_init,
-    .max_cpus = 1, // XXX for now
-    .default_boot_order = "c",
+static void sun4v_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4v platform";
+    mc->init = sun4v_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sun4v_type = {
+    .name = MACHINE_TYPE_NAME("sun4v"),
+    .parent = TYPE_MACHINE,
+    .class_init = sun4v_class_init,
 };
 
-static QEMUMachine niagara_machine = {
-    .name = "Niagara",
-    .desc = "Sun4v platform, Niagara",
-    .init = niagara_init,
-    .max_cpus = 1, // XXX for now
-    .default_boot_order = "c",
+static void niagara_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4v platform, Niagara";
+    mc->init = niagara_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo niagara_type = {
+    .name = MACHINE_TYPE_NAME("Niagara"),
+    .parent = TYPE_MACHINE,
+    .class_init = niagara_class_init,
 };
 
 static void sun4u_register_types(void)
@@ -999,10 +1023,10 @@ static void sun4u_register_types(void)
 
 static void sun4u_machine_init(void)
 {
-    qemu_register_machine(&sun4u_machine);
-    qemu_register_machine(&sun4v_machine);
-    qemu_register_machine(&niagara_machine);
+    type_register_static(&sun4u_type);
+    type_register_static(&sun4v_type);
+    type_register_static(&niagara_type);
 }
 
 type_init(sun4u_register_types)
-machine_init(sun4u_machine_init);
+machine_init(sun4u_machine_init)
diff --git a/hw/ssi/omap_spi.c b/hw/ssi/omap_spi.c
index 119e325a6..27263299b 100644
--- a/hw/ssi/omap_spi.c
+++ b/hw/ssi/omap_spi.c
@@ -342,8 +342,7 @@ static const MemoryRegionOps omap_mcspi_ops = {
 struct omap_mcspi_s *omap_mcspi_init(struct omap_target_agent_s *ta, int chnum,
                 qemu_irq irq, qemu_irq *drq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_mcspi_s *s = (struct omap_mcspi_s *)
-            g_malloc0(sizeof(struct omap_mcspi_s));
+    struct omap_mcspi_s *s = g_new0(struct omap_mcspi_s, 1);
     struct omap_mcspi_ch_s *ch = s->ch;
 
     s->irq = irq;
diff --git a/hw/timer/hpet.c b/hw/timer/hpet.c
index 2bb62211c..7f0391c78 100644
--- a/hw/timer/hpet.c
+++ b/hw/timer/hpet.c
@@ -116,22 +116,22 @@ static uint32_t timer_enabled(HPETTimer *t)
 
 static uint32_t hpet_time_after(uint64_t a, uint64_t b)
 {
-    return ((int32_t)(b) - (int32_t)(a) < 0);
+    return ((int32_t)(b - a) < 0);
 }
 
 static uint32_t hpet_time_after64(uint64_t a, uint64_t b)
 {
-    return ((int64_t)(b) - (int64_t)(a) < 0);
+    return ((int64_t)(b - a) < 0);
 }
 
 static uint64_t ticks_to_ns(uint64_t value)
 {
-    return (muldiv64(value, HPET_CLK_PERIOD, FS_PER_NS));
+    return value * HPET_CLK_PERIOD;
 }
 
 static uint64_t ns_to_ticks(uint64_t value)
 {
-    return (muldiv64(value, FS_PER_NS, HPET_CLK_PERIOD));
+    return value / HPET_CLK_PERIOD;
 }
 
 static uint64_t hpet_fixup_reg(uint64_t new, uint64_t old, uint64_t mask)
@@ -758,7 +758,7 @@ static void hpet_realize(DeviceState *dev, Error **errp)
     /* 64-bit main counter; LegacyReplacementRoute. */
     s->capability = 0x8086a001ULL;
     s->capability |= (s->num_timers - 1) << HPET_ID_NUM_TIM_SHIFT;
-    s->capability |= ((HPET_CLK_PERIOD) << 32);
+    s->capability |= ((uint64_t)(HPET_CLK_PERIOD * FS_PER_NS) << 32);
 
     qdev_init_gpio_in(dev, hpet_handle_legacy_irq, 2);
     qdev_init_gpio_out(dev, &s->pit_enabled, 1);
diff --git a/hw/timer/imx_epit.c b/hw/timer/imx_epit.c
index ffefc22f4..967be4ad2 100644
--- a/hw/timer/imx_epit.c
+++ b/hw/timer/imx_epit.c
@@ -5,25 +5,28 @@
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
  * Updated by Peter Chubb
- * Updated by Jean-Christophe Dubois
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
 
-#include "hw/hw.h"
-#include "qemu/bitops.h"
-#include "qemu/timer.h"
-#include "hw/ptimer.h"
-#include "hw/sysbus.h"
-#include "hw/arm/imx.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/misc/imx_ccm.h"
 #include "qemu/main-loop.h"
 
-#define TYPE_IMX_EPIT "imx.epit"
+#ifndef DEBUG_IMX_EPIT
+#define DEBUG_IMX_EPIT 0
+#endif
 
-#define DEBUG_TIMER 0
-#if DEBUG_TIMER
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_EPIT) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_EPIT, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static char const *imx_epit_reg_name(uint32_t reg)
 {
@@ -43,48 +46,6 @@ static char const *imx_epit_reg_name(uint32_t reg)
     }
 }
 
-#  define DPRINTF(fmt, args...) \
-    do { fprintf(stderr, "%s: " fmt , __func__, ##args); } while (0)
-#else
-#  define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-          do { fprintf(stderr, "%s: " fmt, __func__, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_EPIT(obj) \
-        OBJECT_CHECK(IMXEPITState, (obj), TYPE_IMX_EPIT)
-
-/*
- * EPIT: Enhanced periodic interrupt timer
- */
-
-#define CR_EN       (1 << 0)
-#define CR_ENMOD    (1 << 1)
-#define CR_OCIEN    (1 << 2)
-#define CR_RLD      (1 << 3)
-#define CR_PRESCALE_SHIFT (4)
-#define CR_PRESCALE_MASK  (0xfff)
-#define CR_SWR      (1 << 16)
-#define CR_IOVW     (1 << 17)
-#define CR_DBGEN    (1 << 18)
-#define CR_WAITEN   (1 << 19)
-#define CR_DOZEN    (1 << 20)
-#define CR_STOPEN   (1 << 21)
-#define CR_CLKSRC_SHIFT (24)
-#define CR_CLKSRC_MASK  (0x3 << CR_CLKSRC_SHIFT)
-
-#define EPIT_TIMER_MAX  0XFFFFFFFFUL
-
 /*
  * Exact clock frequencies vary from board to board.
  * These are typical.
@@ -96,23 +57,6 @@ static const IMXClk imx_epit_clocks[] =  {
     CLK_32k,  /* 11 ipg_clk_32k -- ~32kHz */
 };
 
-typedef struct {
-    SysBusDevice busdev;
-    ptimer_state *timer_reload;
-    ptimer_state *timer_cmp;
-    MemoryRegion iomem;
-    DeviceState *ccm;
-
-    uint32_t cr;
-    uint32_t sr;
-    uint32_t lr;
-    uint32_t cmp;
-    uint32_t cnt;
-
-    uint32_t freq;
-    qemu_irq irq;
-} IMXEPITState;
-
 /*
  * Update interrupt status
  */
@@ -174,18 +118,17 @@ static void imx_epit_reset(DeviceState *dev)
 
 static uint32_t imx_epit_update_count(IMXEPITState *s)
 {
-     s->cnt = ptimer_get_count(s->timer_reload);
+    s->cnt = ptimer_get_count(s->timer_reload);
 
-     return s->cnt;
+    return s->cnt;
 }
 
 static uint64_t imx_epit_read(void *opaque, hwaddr offset, unsigned size)
 {
     IMXEPITState *s = IMX_EPIT(opaque);
     uint32_t reg_value = 0;
-    uint32_t reg = offset >> 2;
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* Control Register */
         reg_value = s->cr;
         break;
@@ -208,11 +151,12 @@ static uint64_t imx_epit_read(void *opaque, hwaddr offset, unsigned size)
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_EPIT, __func__, offset);
         break;
     }
 
-    DPRINTF("(%s) = 0x%08x\n", imx_epit_reg_name(reg), reg_value);
+    DPRINTF("(%s) = 0x%08x\n", imx_epit_reg_name(offset >> 2), reg_value);
 
     return reg_value;
 }
@@ -237,12 +181,12 @@ static void imx_epit_write(void *opaque, hwaddr offset, uint64_t value,
                            unsigned size)
 {
     IMXEPITState *s = IMX_EPIT(opaque);
-    uint32_t reg = offset >> 2;
     uint64_t oldcr;
 
-    DPRINTF("(%s, value = 0x%08x)\n", imx_epit_reg_name(reg), (uint32_t)value);
+    DPRINTF("(%s, value = 0x%08x)\n", imx_epit_reg_name(offset >> 2),
+            (uint32_t)value);
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* CR */
 
         oldcr = s->cr;
@@ -318,7 +262,8 @@ static void imx_epit_write(void *opaque, hwaddr offset, uint64_t value,
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_EPIT, __func__, offset);
 
         break;
     }
@@ -333,24 +278,14 @@ static void imx_epit_cmp(void *opaque)
     imx_epit_update_int(s);
 }
 
-void imx_timerp_create(const hwaddr addr, qemu_irq irq, DeviceState *ccm)
-{
-    IMXEPITState *pp;
-    DeviceState *dev;
-
-    dev = sysbus_create_simple(TYPE_IMX_EPIT, addr, irq);
-    pp = IMX_EPIT(dev);
-    pp->ccm = ccm;
-}
-
 static const MemoryRegionOps imx_epit_ops = {
-  .read = imx_epit_read,
-  .write = imx_epit_write,
-  .endianness = DEVICE_NATIVE_ENDIAN,
+    .read = imx_epit_read,
+    .write = imx_epit_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
 static const VMStateDescription vmstate_imx_timer_epit = {
-    .name = "imx.epit",
+    .name = TYPE_IMX_EPIT,
     .version_id = 2,
     .minimum_version_id = 2,
     .fields = (VMStateField[]) {
diff --git a/hw/timer/imx_gpt.c b/hw/timer/imx_gpt.c
index 3b3101084..7257f4201 100644
--- a/hw/timer/imx_gpt.c
+++ b/hw/timer/imx_gpt.c
@@ -5,28 +5,28 @@
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
  * Updated by Peter Chubb
- * Updated by Jean-Christophe Dubois
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
 
-#include "hw/hw.h"
-#include "qemu/bitops.h"
-#include "qemu/timer.h"
-#include "hw/ptimer.h"
-#include "hw/sysbus.h"
-#include "hw/arm/imx.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/misc/imx_ccm.h"
 #include "qemu/main-loop.h"
 
-#define TYPE_IMX_GPT "imx.gpt"
+#ifndef DEBUG_IMX_GPT
+#define DEBUG_IMX_GPT 0
+#endif
 
-/*
- * Define to 1 for debug messages
- */
-#define DEBUG_TIMER 0
-#if DEBUG_TIMER
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_GPT) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_GPT, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static char const *imx_gpt_reg_name(uint32_t reg)
 {
@@ -56,94 +56,8 @@ static char const *imx_gpt_reg_name(uint32_t reg)
     }
 }
 
-#  define DPRINTF(fmt, args...) \
-          do { printf("%s: " fmt , __func__, ##args); } while (0)
-#else
-#  define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-          do { fprintf(stderr, "%s: " fmt, __func__, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_GPT(obj) \
-        OBJECT_CHECK(IMXGPTState, (obj), TYPE_IMX_GPT)
-/*
- * GPT : General purpose timer
- *
- * This timer counts up continuously while it is enabled, resetting itself
- * to 0 when it reaches GPT_TIMER_MAX (in freerun mode) or when it
- * reaches the value of one of the ocrX (in periodic mode).
- */
-
-#define GPT_TIMER_MAX  0XFFFFFFFFUL
-
-/* Control register.  Not all of these bits have any effect (yet) */
-#define GPT_CR_EN     (1 << 0)  /* GPT Enable */
-#define GPT_CR_ENMOD  (1 << 1)  /* GPT Enable Mode */
-#define GPT_CR_DBGEN  (1 << 2)  /* GPT Debug mode enable */
-#define GPT_CR_WAITEN (1 << 3)  /* GPT Wait Mode Enable  */
-#define GPT_CR_DOZEN  (1 << 4)  /* GPT Doze mode enable */
-#define GPT_CR_STOPEN (1 << 5)  /* GPT Stop Mode Enable */
-#define GPT_CR_CLKSRC_SHIFT (6)
-#define GPT_CR_CLKSRC_MASK  (0x7)
-
-#define GPT_CR_FRR    (1 << 9)  /* Freerun or Restart */
-#define GPT_CR_SWR    (1 << 15) /* Software Reset */
-#define GPT_CR_IM1    (3 << 16) /* Input capture channel 1 mode (2 bits) */
-#define GPT_CR_IM2    (3 << 18) /* Input capture channel 2 mode (2 bits) */
-#define GPT_CR_OM1    (7 << 20) /* Output Compare Channel 1 Mode (3 bits) */
-#define GPT_CR_OM2    (7 << 23) /* Output Compare Channel 2 Mode (3 bits) */
-#define GPT_CR_OM3    (7 << 26) /* Output Compare Channel 3 Mode (3 bits) */
-#define GPT_CR_FO1    (1 << 29) /* Force Output Compare Channel 1 */
-#define GPT_CR_FO2    (1 << 30) /* Force Output Compare Channel 2 */
-#define GPT_CR_FO3    (1 << 31) /* Force Output Compare Channel 3 */
-
-#define GPT_SR_OF1  (1 << 0)
-#define GPT_SR_OF2  (1 << 1)
-#define GPT_SR_OF3  (1 << 2)
-#define GPT_SR_ROV  (1 << 5)
-
-#define GPT_IR_OF1IE  (1 << 0)
-#define GPT_IR_OF2IE  (1 << 1)
-#define GPT_IR_OF3IE  (1 << 2)
-#define GPT_IR_ROVIE  (1 << 5)
-
-typedef struct {
-    SysBusDevice busdev;
-    ptimer_state *timer;
-    MemoryRegion iomem;
-    DeviceState *ccm;
-
-    uint32_t cr;
-    uint32_t pr;
-    uint32_t sr;
-    uint32_t ir;
-    uint32_t ocr1;
-    uint32_t ocr2;
-    uint32_t ocr3;
-    uint32_t icr1;
-    uint32_t icr2;
-    uint32_t cnt;
-
-    uint32_t next_timeout;
-    uint32_t next_int;
-
-    uint32_t freq;
-
-    qemu_irq irq;
-} IMXGPTState;
-
 static const VMStateDescription vmstate_imx_timer_gpt = {
-    .name = "imx.gpt",
+    .name = TYPE_IMX_GPT,
     .version_id = 3,
     .minimum_version_id = 3,
     .fields = (VMStateField[]) {
@@ -180,7 +94,7 @@ static void imx_gpt_set_freq(IMXGPTState *s)
 {
     uint32_t clksrc = extract32(s->cr, GPT_CR_CLKSRC_SHIFT, 3);
     uint32_t freq = imx_clock_frequency(s->ccm, imx_gpt_clocks[clksrc])
-                                                / (1 + s->pr);
+                    / (1 + s->pr);
     s->freq = freq;
 
     DPRINTF("Setting clksrc %d to frequency %d\n", clksrc, freq);
@@ -207,7 +121,7 @@ static uint32_t imx_gpt_update_count(IMXGPTState *s)
 }
 
 static inline uint32_t imx_gpt_find_limit(uint32_t count, uint32_t reg,
-                                             uint32_t timeout)
+                                          uint32_t timeout)
 {
     if ((count < reg) && (timeout > reg)) {
         timeout = reg;
@@ -298,9 +212,8 @@ static uint64_t imx_gpt_read(void *opaque, hwaddr offset, unsigned size)
 {
     IMXGPTState *s = IMX_GPT(opaque);
     uint32_t reg_value = 0;
-    uint32_t reg = offset >> 2;
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* Control Register */
         reg_value = s->cr;
         break;
@@ -330,12 +243,14 @@ static uint64_t imx_gpt_read(void *opaque, hwaddr offset, unsigned size)
         break;
 
     case 7: /* input Capture Register 1 */
-        qemu_log_mask(LOG_UNIMP, "icr1 feature is not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: icr1 feature is not implemented\n",
+                      TYPE_IMX_GPT, __func__);
         reg_value = s->icr1;
         break;
 
     case 8: /* input Capture Register 2 */
-        qemu_log_mask(LOG_UNIMP, "icr2 feature is not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: icr2 feature is not implemented\n",
+                      TYPE_IMX_GPT, __func__);
         reg_value = s->icr2;
         break;
 
@@ -345,11 +260,12 @@ static uint64_t imx_gpt_read(void *opaque, hwaddr offset, unsigned size)
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPT, __func__, offset);
         break;
     }
 
-    DPRINTF("(%s) = 0x%08x\n", imx_gpt_reg_name(reg), reg_value);
+    DPRINTF("(%s) = 0x%08x\n", imx_gpt_reg_name(offset >> 2), reg_value);
 
     return reg_value;
 }
@@ -396,12 +312,11 @@ static void imx_gpt_write(void *opaque, hwaddr offset, uint64_t value,
 {
     IMXGPTState *s = IMX_GPT(opaque);
     uint32_t oldreg;
-    uint32_t reg = offset >> 2;
 
-    DPRINTF("(%s, value = 0x%08x)\n", imx_gpt_reg_name(reg),
+    DPRINTF("(%s, value = 0x%08x)\n", imx_gpt_reg_name(offset >> 2),
             (uint32_t)value);
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0:
         oldreg = s->cr;
         s->cr = value & ~0x7c14;
@@ -477,7 +392,8 @@ static void imx_gpt_write(void *opaque, hwaddr offset, uint64_t value,
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPT, __func__, offset);
         break;
     }
 }
@@ -522,16 +438,6 @@ static void imx_gpt_realize(DeviceState *dev, Error **errp)
     s->timer = ptimer_init(bh);
 }
 
-void imx_timerg_create(const hwaddr addr, qemu_irq irq, DeviceState *ccm)
-{
-    IMXGPTState *pp;
-    DeviceState *dev;
-
-    dev = sysbus_create_simple(TYPE_IMX_GPT, addr, irq);
-    pp = IMX_GPT(dev);
-    pp->ccm = ccm;
-}
-
 static void imx_gpt_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
diff --git a/hw/timer/m48t59.c b/hw/timer/m48t59.c
index 8ab683dda..b3df8f9ec 100644
--- a/hw/timer/m48t59.c
+++ b/hw/timer/m48t59.c
@@ -590,10 +590,8 @@ static void nvram_writel (void *opaque, hwaddr addr, uint32_t value)
 static uint32_t nvram_readb (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
-    uint32_t retval;
 
-    retval = m48t59_read(NVRAM, addr);
-    return retval;
+    return m48t59_read(NVRAM, addr);
 }
 
 static uint32_t nvram_readw (void *opaque, hwaddr addr)
diff --git a/hw/timer/omap_gptimer.c b/hw/timer/omap_gptimer.c
index b8c8c0137..dcf706c46 100644
--- a/hw/timer/omap_gptimer.c
+++ b/hw/timer/omap_gptimer.c
@@ -468,8 +468,7 @@ static const MemoryRegionOps omap_gp_timer_ops = {
 struct omap_gp_timer_s *omap_gp_timer_init(struct omap_target_agent_s *ta,
                 qemu_irq irq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_gp_timer_s *s = (struct omap_gp_timer_s *)
-            g_malloc0(sizeof(struct omap_gp_timer_s));
+    struct omap_gp_timer_s *s = g_new0(struct omap_gp_timer_s, 1);
 
     s->ta = ta;
     s->irq = irq;
diff --git a/hw/tpm/tpm_passthrough.c b/hw/tpm/tpm_passthrough.c
index 79a8f98af..be160c19b 100644
--- a/hw/tpm/tpm_passthrough.c
+++ b/hw/tpm/tpm_passthrough.c
@@ -22,8 +22,6 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
-#include <dirent.h>
-
 #include "qemu-common.h"
 #include "qapi/error.h"
 #include "qemu/error-report.h"
diff --git a/hw/tpm/tpm_tis.c b/hw/tpm/tpm_tis.c
index 0806b5f82..ff073d501 100644
--- a/hw/tpm/tpm_tis.c
+++ b/hw/tpm/tpm_tis.c
@@ -141,7 +141,7 @@
 
 #define TPM_TIS_IFACE_ID_SUPPORTED_FLAGS1_3 \
     (TPM_TIS_IFACE_ID_INTERFACE_TIS1_3 | \
-     (~0 << 4)/* all of it is don't care */)
+     (~0u << 4)/* all of it is don't care */)
 
 /* if backend was a TPM 2.0: */
 #define TPM_TIS_IFACE_ID_SUPPORTED_FLAGS2_0 \
diff --git a/hw/tricore/tricore_testboard.c b/hw/tricore/tricore_testboard.c
index a059a20a3..4ff5e7b68 100644
--- a/hw/tricore/tricore_testboard.c
+++ b/hw/tricore/tricore_testboard.c
@@ -44,7 +44,7 @@ static void tricore_load_kernel(CPUTriCoreState *env)
     kernel_size = load_elf(tricoretb_binfo.kernel_filename, NULL,
                            NULL, (uint64_t *)&entry, NULL,
                            NULL, 0,
-                           ELF_MACHINE, 1);
+                           EM_TRICORE, 1);
     if (kernel_size <= 0) {
         error_report("qemu: no kernel file '%s'",
                 tricoretb_binfo.kernel_filename);
@@ -76,17 +76,23 @@ static void tricore_testboard_init(MachineState *machine, int board_id)
         exit(1);
     }
     env = &cpu->env;
-    memory_region_init_ram(ext_cram, NULL, "powerlink_ext_c.ram", 2*1024*1024, &error_abort);
+    memory_region_init_ram(ext_cram, NULL, "powerlink_ext_c.ram", 2*1024*1024,
+                           &error_fatal);
     vmstate_register_ram_global(ext_cram);
-    memory_region_init_ram(ext_dram, NULL, "powerlink_ext_d.ram", 4*1024*1024, &error_abort);
+    memory_region_init_ram(ext_dram, NULL, "powerlink_ext_d.ram", 4*1024*1024,
+                           &error_fatal);
     vmstate_register_ram_global(ext_dram);
-    memory_region_init_ram(int_cram, NULL, "powerlink_int_c.ram", 48*1024, &error_abort);
+    memory_region_init_ram(int_cram, NULL, "powerlink_int_c.ram", 48*1024,
+                           &error_fatal);
     vmstate_register_ram_global(int_cram);
-    memory_region_init_ram(int_dram, NULL, "powerlink_int_d.ram", 48*1024, &error_abort);
+    memory_region_init_ram(int_dram, NULL, "powerlink_int_d.ram", 48*1024,
+                           &error_fatal);
     vmstate_register_ram_global(int_dram);
-    memory_region_init_ram(pcp_data, NULL, "powerlink_pcp_data.ram", 16*1024, &error_abort);
+    memory_region_init_ram(pcp_data, NULL, "powerlink_pcp_data.ram", 16*1024,
+                           &error_fatal);
     vmstate_register_ram_global(pcp_data);
-    memory_region_init_ram(pcp_text, NULL, "powerlink_pcp_text.ram", 32*1024, &error_abort);
+    memory_region_init_ram(pcp_text, NULL, "powerlink_pcp_text.ram", 32*1024,
+                           &error_fatal);
     vmstate_register_ram_global(pcp_text);
 
     memory_region_add_subregion(sysmem, 0x80000000, ext_cram);
@@ -109,16 +115,11 @@ static void tricoreboard_init(MachineState *machine)
     tricore_testboard_init(machine, 0x183);
 }
 
-static QEMUMachine ttb_machine = {
-    .name = "tricore_testboard",
-    .desc = "a minimal TriCore board",
-    .init = tricoreboard_init,
-    .is_default = 0,
-};
-
-static void tricore_testboard_machine_init(void)
+static void ttb_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&ttb_machine);
+    mc->desc = "a minimal TriCore board";
+    mc->init = tricoreboard_init;
+    mc->is_default = 0;
 }
 
-machine_init(tricore_testboard_machine_init);
+DEFINE_MACHINE("tricore_testboard", ttb_machine_init)
diff --git a/hw/unicore32/puv3.c b/hw/unicore32/puv3.c
index 703e29d6d..91117b2b9 100644
--- a/hw/unicore32/puv3.c
+++ b/hw/unicore32/puv3.c
@@ -75,7 +75,7 @@ static void puv3_board_init(CPUUniCore32State *env, ram_addr_t ram_size)
 
     /* SDRAM at address zero.  */
     memory_region_init_ram(ram_memory, NULL, "puv3.ram", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_memory);
     memory_region_add_subregion(get_system_memory(), 0, ram_memory);
 }
@@ -130,16 +130,11 @@ static void puv3_init(MachineState *machine)
     puv3_load_kernel(kernel_filename);
 }
 
-static QEMUMachine puv3_machine = {
-    .name = "puv3",
-    .desc = "PKUnity Version-3 based on UniCore32",
-    .init = puv3_init,
-    .is_default = 1,
-};
-
-static void puv3_machine_init(void)
+static void puv3_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&puv3_machine);
+    mc->desc = "PKUnity Version-3 based on UniCore32";
+    mc->init = puv3_init;
+    mc->is_default = 1;
 }
 
-machine_init(puv3_machine_init)
+DEFINE_MACHINE("puv3", puv3_machine_init)
diff --git a/hw/usb/Makefile.objs b/hw/usb/Makefile.objs
index 7443e386b..8f00fbd8f 100644
--- a/hw/usb/Makefile.objs
+++ b/hw/usb/Makefile.objs
@@ -23,9 +23,8 @@ common-obj-$(CONFIG_USB_BLUETOOTH)    += dev-bluetooth.o
 
 ifeq ($(CONFIG_USB_SMARTCARD),y)
 common-obj-y                          += dev-smartcard-reader.o
-common-obj-y                          += ccid-card-passthru.o
-common-obj-$(CONFIG_SMARTCARD_NSS)    += ccid-card-emulated.o
-ccid-card-emulated.o-cflags := -I$(SRC_PATH)/libcacard
+common-obj-$(CONFIG_SMARTCARD)        += ccid-card-passthru.o
+common-obj-$(CONFIG_SMARTCARD)        += ccid-card-emulated.o
 endif
 
 ifeq ($(CONFIG_POSIX),y)
diff --git a/hw/usb/bus.c b/hw/usb/bus.c
index 5f39e1e3a..ee6b43abc 100644
--- a/hw/usb/bus.c
+++ b/hw/usb/bus.c
@@ -655,9 +655,12 @@ void hmp_info_usb(Monitor *mon, const QDict *qdict)
             dev = port->dev;
             if (!dev)
                 continue;
-            monitor_printf(mon, "  Device %d.%d, Port %s, Speed %s Mb/s, Product %s\n",
-                           bus->busnr, dev->addr, port->path, usb_speed(dev->speed),
-                           dev->product_desc);
+            monitor_printf(mon, "  Device %d.%d, Port %s, Speed %s Mb/s, "
+                           "Product %s%s%s\n",
+                           bus->busnr, dev->addr, port->path,
+                           usb_speed(dev->speed), dev->product_desc,
+                           dev->qdev.id ? ", ID: " : "",
+                           dev->qdev.id ?: "");
         }
     }
 }
diff --git a/hw/usb/ccid-card-emulated.c b/hw/usb/ccid-card-emulated.c
index 72329ed7d..869a63c5b 100644
--- a/hw/usb/ccid-card-emulated.c
+++ b/hw/usb/ccid-card-emulated.c
@@ -166,7 +166,7 @@ static void emulated_push_event(EmulatedState *card, EmulEvent *event)
 
 static void emulated_push_type(EmulatedState *card, uint32_t type)
 {
-    EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent));
+    EmulEvent *event = g_new(EmulEvent, 1);
 
     assert(event);
     event->p.gen.type = type;
@@ -175,7 +175,7 @@ static void emulated_push_type(EmulatedState *card, uint32_t type)
 
 static void emulated_push_error(EmulatedState *card, uint64_t code)
 {
-    EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent));
+    EmulEvent *event = g_new(EmulEvent, 1);
 
     assert(event);
     event->p.error.type = EMUL_ERROR;
diff --git a/hw/usb/ccid-card-passthru.c b/hw/usb/ccid-card-passthru.c
index 85a4fc3e5..9f49c0537 100644
--- a/hw/usb/ccid-card-passthru.c
+++ b/hw/usb/ccid-card-passthru.c
@@ -12,7 +12,7 @@
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "ccid.h"
-#include "libcacard/vscard_common.h"
+#include "cacard/vscard_common.h"
 
 #define DPRINTF(card, lvl, fmt, ...)                    \
 do {                                                    \
diff --git a/hw/usb/core.c b/hw/usb/core.c
index d0025db60..7f4637029 100644
--- a/hw/usb/core.c
+++ b/hw/usb/core.c
@@ -128,9 +128,16 @@ static void do_token_setup(USBDevice *s, USBPacket *p)
     }
 
     usb_packet_copy(p, s->setup_buf, p->iov.size);
+    s->setup_index = 0;
     p->actual_length = 0;
     s->setup_len   = (s->setup_buf[7] << 8) | s->setup_buf[6];
-    s->setup_index = 0;
+    if (s->setup_len > sizeof(s->data_buf)) {
+        fprintf(stderr,
+                "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
+                s->setup_len, sizeof(s->data_buf));
+        p->status = USB_RET_STALL;
+        return;
+    }
 
     request = (s->setup_buf[0] << 8) | s->setup_buf[1];
     value   = (s->setup_buf[3] << 8) | s->setup_buf[2];
@@ -151,13 +158,6 @@ static void do_token_setup(USBDevice *s, USBPacket *p)
         }
         s->setup_state = SETUP_STATE_DATA;
     } else {
-        if (s->setup_len > sizeof(s->data_buf)) {
-            fprintf(stderr,
-                "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
-                s->setup_len, sizeof(s->data_buf));
-            p->status = USB_RET_STALL;
-            return;
-        }
         if (s->setup_len == 0)
             s->setup_state = SETUP_STATE_ACK;
         else
@@ -176,7 +176,7 @@ static void do_token_in(USBDevice *s, USBPacket *p)
     request = (s->setup_buf[0] << 8) | s->setup_buf[1];
     value   = (s->setup_buf[3] << 8) | s->setup_buf[2];
     index   = (s->setup_buf[5] << 8) | s->setup_buf[4];
- 
+
     switch(s->setup_state) {
     case SETUP_STATE_ACK:
         if (!(s->setup_buf[0] & USB_DIR_IN)) {
diff --git a/hw/usb/dev-audio.c b/hw/usb/dev-audio.c
index f092bb849..02fb1103c 100644
--- a/hw/usb/dev-audio.c
+++ b/hw/usb/dev-audio.c
@@ -664,7 +664,7 @@ static const VMStateDescription vmstate_usb_audio = {
 static Property usb_audio_properties[] = {
     DEFINE_PROP_UINT32("debug", USBAudioState, debug, 0),
     DEFINE_PROP_UINT32("buffer", USBAudioState, buffer,
-                       8 * USBAUDIO_PACKET_SIZE),
+                       32 * USBAUDIO_PACKET_SIZE),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/usb/dev-mtp.c b/hw/usb/dev-mtp.c
index 809b1cb11..a2762679e 100644
--- a/hw/usb/dev-mtp.c
+++ b/hw/usb/dev-mtp.c
@@ -359,8 +359,7 @@ static void usb_mtp_object_readdir(MTPState *s, MTPObject *o)
     }
     while ((entry = readdir(dir)) != NULL) {
         if ((o->nchildren % 32) == 0) {
-            o->children = g_realloc(o->children,
-                                    (o->nchildren + 32) * sizeof(MTPObject *));
+            o->children = g_renew(MTPObject *, o->children, o->nchildren + 32);
         }
         o->children[o->nchildren] =
             usb_mtp_object_alloc(s, s->next_handle++, o, entry->d_name);
diff --git a/hw/usb/dev-network.c b/hw/usb/dev-network.c
index 7800ceea5..180adce61 100644
--- a/hw/usb/dev-network.c
+++ b/hw/usb/dev-network.c
@@ -653,7 +653,8 @@ typedef struct USBNetState {
 
 static int is_rndis(USBNetState *s)
 {
-    return s->dev.config->bConfigurationValue == DEV_RNDIS_CONFIG_VALUE;
+    return s->dev.config ?
+            s->dev.config->bConfigurationValue == DEV_RNDIS_CONFIG_VALUE : 0;
 }
 
 static int ndis_query(USBNetState *s, uint32_t oid,
@@ -914,8 +915,9 @@ static int rndis_query_response(USBNetState *s,
 
     bufoffs = le32_to_cpu(buf->InformationBufferOffset) + 8;
     buflen = le32_to_cpu(buf->InformationBufferLength);
-    if (bufoffs + buflen > length)
+    if (buflen > length || bufoffs >= length || bufoffs + buflen > length) {
         return USB_RET_STALL;
+    }
 
     infobuflen = ndis_query(s, le32_to_cpu(buf->OID),
                             bufoffs + (uint8_t *) buf, buflen, infobuf,
@@ -960,8 +962,9 @@ static int rndis_set_response(USBNetState *s,
 
     bufoffs = le32_to_cpu(buf->InformationBufferOffset) + 8;
     buflen = le32_to_cpu(buf->InformationBufferLength);
-    if (bufoffs + buflen > length)
+    if (buflen > length || bufoffs >= length || bufoffs + buflen > length) {
         return USB_RET_STALL;
+    }
 
     ret = ndis_set(s, le32_to_cpu(buf->OID),
                     bufoffs + (uint8_t *) buf, buflen);
@@ -1211,8 +1214,9 @@ static void usb_net_handle_dataout(USBNetState *s, USBPacket *p)
     if (le32_to_cpu(msg->MessageType) == RNDIS_PACKET_MSG) {
         uint32_t offs = 8 + le32_to_cpu(msg->DataOffset);
         uint32_t size = le32_to_cpu(msg->DataLength);
-        if (offs + size <= len)
+        if (offs < len && size < len && offs + size <= len) {
             qemu_send_packet(qemu_get_queue(s->nic), s->out_buf + offs, size);
+        }
     }
     s->out_ptr -= len;
     memmove(s->out_buf, &s->out_buf[len], s->out_ptr);
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index 9a4e7dc0c..597d8fd18 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -613,20 +613,22 @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
         return;
     }
 
-    bdrv_add_key(blk_bs(blk), NULL, &err);
-    if (err) {
-        if (monitor_cur_is_qmp()) {
-            error_propagate(errp, err);
-            return;
-        }
-        error_free(err);
-        err = NULL;
-        if (cur_mon) {
-            monitor_read_bdrv_key_start(cur_mon, blk_bs(blk),
-                                        usb_msd_password_cb, s);
-            s->dev.auto_attach = 0;
-        } else {
-            autostart = 0;
+    if (blk_bs(blk)) {
+        bdrv_add_key(blk_bs(blk), NULL, &err);
+        if (err) {
+            if (monitor_cur_is_qmp()) {
+                error_propagate(errp, err);
+                return;
+            }
+            error_free(err);
+            err = NULL;
+            if (cur_mon) {
+                monitor_read_bdrv_key_start(cur_mon, blk_bs(blk),
+                                            usb_msd_password_cb, s);
+                s->dev.auto_attach = 0;
+            } else {
+                autostart = 0;
+            }
         }
     }
 
diff --git a/hw/usb/hcd-ehci-pci.c b/hw/usb/hcd-ehci-pci.c
index 7afa5f9d6..16fb845d0 100644
--- a/hw/usb/hcd-ehci-pci.c
+++ b/hw/usb/hcd-ehci-pci.c
@@ -95,10 +95,8 @@ static void usb_ehci_pci_exit(PCIDevice *dev)
 
     usb_ehci_unrealize(s, DEVICE(dev), NULL);
 
-    if (s->irq) {
-        g_free(s->irq);
-        s->irq = NULL;
-    }
+    g_free(s->irq);
+    s->irq = NULL;
 }
 
 static void usb_ehci_pci_reset(DeviceState *dev)
diff --git a/hw/usb/hcd-ehci.c b/hw/usb/hcd-ehci.c
index 64a54c6e8..4120bb7c3 100644
--- a/hw/usb/hcd-ehci.c
+++ b/hw/usb/hcd-ehci.c
@@ -726,7 +726,7 @@ static void ehci_detach(USBPort *port)
     ehci_queues_rip_device(s, port->dev, 0);
     ehci_queues_rip_device(s, port->dev, 1);
 
-    *portsc &= ~(PORTSC_CONNECT|PORTSC_PED);
+    *portsc &= ~(PORTSC_CONNECT|PORTSC_PED|PORTSC_SUSPEND);
     *portsc |= PORTSC_CSC;
 
     ehci_raise_irq(s, USBSTS_PCD);
@@ -865,6 +865,7 @@ void ehci_reset(void *opaque)
     s->usbsts = USBSTS_HALT;
     s->usbsts_pending = 0;
     s->usbsts_frindex = 0;
+    ehci_update_irq(s);
 
     s->astate = EST_INACTIVE;
     s->pstate = EST_INACTIVE;
@@ -1404,21 +1405,23 @@ static int ehci_process_itd(EHCIState *ehci,
         if (itd->transact[i] & ITD_XACT_ACTIVE) {
             pg   = get_field(itd->transact[i], ITD_XACT_PGSEL);
             off  = itd->transact[i] & ITD_XACT_OFFSET_MASK;
-            ptr1 = (itd->bufptr[pg] & ITD_BUFPTR_MASK);
-            ptr2 = (itd->bufptr[pg+1] & ITD_BUFPTR_MASK);
             len  = get_field(itd->transact[i], ITD_XACT_LENGTH);
 
             if (len > max * mult) {
                 len = max * mult;
             }
-
-            if (len > BUFF_SIZE) {
+            if (len > BUFF_SIZE || pg > 6) {
                 return -1;
             }
 
+            ptr1 = (itd->bufptr[pg] & ITD_BUFPTR_MASK);
             qemu_sglist_init(&ehci->isgl, ehci->device, 2, ehci->as);
             if (off + len > 4096) {
                 /* transfer crosses page border */
+                if (pg == 6) {
+                    return -1;  /* avoid page pg + 1 */
+                }
+                ptr2 = (itd->bufptr[pg + 1] & ITD_BUFPTR_MASK);
                 uint32_t len2 = off + len - 4096;
                 uint32_t len1 = len - len2;
                 qemu_sglist_add(&ehci->isgl, ptr1 + off, len1);
@@ -2000,6 +2003,7 @@ static int ehci_state_writeback(EHCIQueue *q)
 static void ehci_advance_state(EHCIState *ehci, int async)
 {
     EHCIQueue *q = NULL;
+    int itd_count = 0;
     int again;
 
     do {
@@ -2024,10 +2028,12 @@ static void ehci_advance_state(EHCIState *ehci, int async)
 
         case EST_FETCHITD:
             again = ehci_state_fetchitd(ehci, async);
+            itd_count++;
             break;
 
         case EST_FETCHSITD:
             again = ehci_state_fetchsitd(ehci, async);
+            itd_count++;
             break;
 
         case EST_ADVANCEQUEUE:
@@ -2076,7 +2082,8 @@ static void ehci_advance_state(EHCIState *ehci, int async)
             break;
         }
 
-        if (again < 0) {
+        if (again < 0 || itd_count > 16) {
+            /* TODO: notify guest (raise HSE irq?) */
             fprintf(stderr, "processing error - resetting ehci HC\n");
             ehci_reset(ehci);
             again = 0;
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index c673bed4c..268ab3646 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -1453,9 +1453,7 @@ static int xhci_ep_nuke_one_xfer(XHCITransfer *t, TRBCCode report)
         t->running_retry = 0;
         killed = 1;
     }
-    if (t->trbs) {
-        g_free(t->trbs);
-    }
+    g_free(t->trbs);
 
     t->trbs = NULL;
     t->trb_count = t->trb_alloced = 0;
@@ -2190,7 +2188,7 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
             xfer->trbs = NULL;
         }
         if (!xfer->trbs) {
-            xfer->trbs = g_malloc(sizeof(XHCITRB) * length);
+            xfer->trbs = g_new(XHCITRB, length);
             xfer->trb_alloced = length;
         }
         xfer->trb_count = length;
diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c
index 11429f5e7..3f8e540bd 100644
--- a/hw/usb/host-libusb.c
+++ b/hw/usb/host-libusb.c
@@ -451,6 +451,7 @@ static void usb_host_req_complete_iso(struct libusb_transfer *transfer)
     }
     if (xfer->ring->ep->pid == USB_TOKEN_IN) {
         QTAILQ_INSERT_TAIL(&xfer->ring->copy, xfer, next);
+        usb_wakeup(xfer->ring->ep, 0);
     } else {
         QTAILQ_INSERT_TAIL(&xfer->ring->unused, xfer, next);
     }
@@ -1239,7 +1240,7 @@ static void usb_host_handle_control(USBDevice *udev, USBPacket *p,
 
     /* Fix up USB-3 ep0 maxpacket size to allow superspeed connected devices
      * to work redirected to a not superspeed capable hcd */
-    if (udev->speed == USB_SPEED_SUPER &&
+    if ((udev->speedmask & USB_SPEED_MASK_SUPER) &&
         !(udev->port->speedmask & USB_SPEED_MASK_SUPER) &&
         request == 0x8006 && value == 0x100 && index == 0) {
         r->usb3ep0quirk = true;
@@ -1429,7 +1430,7 @@ static void usb_host_free_streams(USBDevice *udev, USBEndpoint **eps,
  * still present in the first place.  Attemping to contine where we
  * left off is impossible.
  *
- * What we are going to to to here is emulate a surprise removal of
+ * What we are going to do here is emulate a surprise removal of
  * the usb device passed through, then kick host scan so the device
  * will get re-attached (and re-initialized by the guest) in case it
  * is still present.
diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index 25df25fd0..30ff74273 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -33,9 +33,7 @@
 #include "qemu/iov.h"
 #include "sysemu/char.h"
 
-#include <dirent.h>
 #include <sys/ioctl.h>
-#include <signal.h>
 #include <usbredirparser.h>
 #include <usbredirfilter.h>
 
@@ -324,7 +322,7 @@ static void packet_id_queue_add(struct PacketIdQueue *q, uint64_t id)
 
     DPRINTF("adding packet id %"PRIu64" to %s queue\n", id, q->name);
 
-    e = g_malloc0(sizeof(struct PacketIdQueueEntry));
+    e = g_new0(struct PacketIdQueueEntry, 1);
     e->id = id;
     QTAILQ_INSERT_TAIL(&q->head, e, next);
     q->size++;
@@ -470,7 +468,7 @@ static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
         dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
     }
 
-    bufp = g_malloc(sizeof(struct buf_packet));
+    bufp = g_new(struct buf_packet, 1);
     bufp->data   = data;
     bufp->len    = len;
     bufp->offset = 0;
@@ -2236,7 +2234,7 @@ static int usbredir_get_bufpq(QEMUFile *f, void *priv, size_t unused)
 
     endp->bufpq_size = qemu_get_be32(f);
     for (i = 0; i < endp->bufpq_size; i++) {
-        bufp = g_malloc(sizeof(struct buf_packet));
+        bufp = g_new(struct buf_packet, 1);
         bufp->len = qemu_get_be32(f);
         bufp->status = qemu_get_be32(f);
         bufp->offset = 0;
diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
index d540c9d14..d3248633c 100644
--- a/hw/vfio/Makefile.objs
+++ b/hw/vfio/Makefile.objs
@@ -1,6 +1,6 @@
 ifeq ($(CONFIG_LINUX), y)
 obj-$(CONFIG_SOFTMMU) += common.o
-obj-$(CONFIG_PCI) += pci.o
+obj-$(CONFIG_PCI) += pci.o pci-quirks.o
 obj-$(CONFIG_SOFTMMU) += platform.o
 obj-$(CONFIG_SOFTMMU) += calxeda-xgmac.o
 endif
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 85ee9b005..6797208cc 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -226,7 +226,7 @@ static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
     /*
      * Try the mapping, if it fails with EBUSY, unmap the region and try
      * again.  This shouldn't be necessary, but we sometimes see it in
-     * the the VGA ROM space.
+     * the VGA ROM space.
      */
     if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
         (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
@@ -312,11 +312,15 @@ out:
     rcu_read_unlock();
 }
 
+static hwaddr vfio_container_granularity(VFIOContainer *container)
+{
+    return (hwaddr)1 << ctz64(container->iova_pgsizes);
+}
+
 static void vfio_listener_region_add(MemoryListener *listener,
                                      MemoryRegionSection *section)
 {
-    VFIOContainer *container = container_of(listener, VFIOContainer,
-                                            iommu_data.type1.listener);
+    VFIOContainer *container = container_of(listener, VFIOContainer, listener);
     hwaddr iova, end;
     Int128 llend;
     void *vaddr;
@@ -344,14 +348,22 @@ static void vfio_listener_region_add(MemoryListener *listener,
     if (int128_ge(int128_make64(iova), llend)) {
         return;
     }
+    end = int128_get64(llend);
+
+    if ((iova < container->min_iova) || ((end - 1) > container->max_iova)) {
+        error_report("vfio: IOMMU container %p can't map guest IOVA region"
+                     " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx,
+                     container, iova, end - 1);
+        ret = -EFAULT;
+        goto fail;
+    }
 
     memory_region_ref(section->mr);
 
     if (memory_region_is_iommu(section->mr)) {
         VFIOGuestIOMMU *giommu;
 
-        trace_vfio_listener_region_add_iommu(iova,
-                    int128_get64(int128_sub(llend, int128_one())));
+        trace_vfio_listener_region_add_iommu(iova, end - 1);
         /*
          * FIXME: We should do some checking to see if the
          * capabilities of the host VFIO IOMMU are adequate to model
@@ -362,33 +374,22 @@ static void vfio_listener_region_add(MemoryListener *listener,
          * would be the right place to wire that up (tell the KVM
          * device emulation the VFIO iommu handles to use).
          */
-        /*
-         * This assumes that the guest IOMMU is empty of
-         * mappings at this point.
-         *
-         * One way of doing this is:
-         * 1. Avoid sharing IOMMUs between emulated devices or different
-         * IOMMU groups.
-         * 2. Implement VFIO_IOMMU_ENABLE in the host kernel to fail if
-         * there are some mappings in IOMMU.
-         *
-         * VFIO on SPAPR does that. Other IOMMU models may do that different,
-         * they must make sure there are no existing mappings or
-         * loop through existing mappings to map them into VFIO.
-         */
         giommu = g_malloc0(sizeof(*giommu));
         giommu->iommu = section->mr;
         giommu->container = container;
         giommu->n.notify = vfio_iommu_map_notify;
         QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
+
         memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
+        memory_region_iommu_replay(giommu->iommu, &giommu->n,
+                                   vfio_container_granularity(container),
+                                   false);
 
         return;
     }
 
     /* Here we assume that memory_region_is_ram(section->mr)==true */
 
-    end = int128_get64(llend);
     vaddr = memory_region_get_ram_ptr(section->mr) +
             section->offset_within_region +
             (iova - section->offset_within_address_space);
@@ -400,27 +401,30 @@ static void vfio_listener_region_add(MemoryListener *listener,
         error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
                      "0x%"HWADDR_PRIx", %p) = %d (%m)",
                      container, iova, end - iova, vaddr, ret);
+        goto fail;
+    }
 
-        /*
-         * On the initfn path, store the first error in the container so we
-         * can gracefully fail.  Runtime, there's not much we can do other
-         * than throw a hardware error.
-         */
-        if (!container->iommu_data.type1.initialized) {
-            if (!container->iommu_data.type1.error) {
-                container->iommu_data.type1.error = ret;
-            }
-        } else {
-            hw_error("vfio: DMA mapping failed, unable to continue");
+    return;
+
+fail:
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.
+     */
+    if (!container->initialized) {
+        if (!container->error) {
+            container->error = ret;
         }
+    } else {
+        hw_error("vfio: DMA mapping failed, unable to continue");
     }
 }
 
 static void vfio_listener_region_del(MemoryListener *listener,
                                      MemoryRegionSection *section)
 {
-    VFIOContainer *container = container_of(listener, VFIOContainer,
-                                            iommu_data.type1.listener);
+    VFIOContainer *container = container_of(listener, VFIOContainer, listener);
     hwaddr iova, end;
     int ret;
 
@@ -485,7 +489,7 @@ static const MemoryListener vfio_memory_listener = {
 
 static void vfio_listener_release(VFIOContainer *container)
 {
-    memory_listener_unregister(&container->iommu_data.type1.listener);
+    memory_listener_unregister(&container->listener);
 }
 
 int vfio_mmap_region(Object *obj, VFIORegion *region,
@@ -496,7 +500,7 @@ int vfio_mmap_region(Object *obj, VFIORegion *region,
     int ret = 0;
     VFIODevice *vbasedev = region->vbasedev;
 
-    if (vbasedev->allow_mmap && size && region->flags &
+    if (!vbasedev->no_mmap && size && region->flags &
         VFIO_REGION_INFO_FLAG_MMAP) {
         int prot = 0;
 
@@ -668,6 +672,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
     if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ||
         ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)) {
         bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU);
+        struct vfio_iommu_type1_info info;
 
         ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
         if (ret) {
@@ -684,21 +689,27 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
             goto free_container_exit;
         }
 
-        container->iommu_data.type1.listener = vfio_memory_listener;
-        container->iommu_data.release = vfio_listener_release;
-
-        memory_listener_register(&container->iommu_data.type1.listener,
-                                 container->space->as);
-
-        if (container->iommu_data.type1.error) {
-            ret = container->iommu_data.type1.error;
-            error_report("vfio: memory listener initialization failed for container");
-            goto listener_release_exit;
+        /*
+         * FIXME: This assumes that a Type1 IOMMU can map any 64-bit
+         * IOVA whatsoever.  That's not actually true, but the current
+         * kernel interface doesn't tell us what it can map, and the
+         * existing Type1 IOMMUs generally support any IOVA we're
+         * going to actually try in practice.
+         */
+        container->min_iova = 0;
+        container->max_iova = (hwaddr)-1;
+
+        /* Assume just 4K IOVA page size */
+        container->iova_pgsizes = 0x1000;
+        info.argsz = sizeof(info);
+        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info);
+        /* Ignore errors */
+        if ((ret == 0) && (info.flags & VFIO_IOMMU_INFO_PGSIZES)) {
+            container->iova_pgsizes = info.iova_pgsizes;
         }
-
-        container->iommu_data.type1.initialized = true;
-
     } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU)) {
+        struct vfio_iommu_spapr_tce_info info;
+
         ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
         if (ret) {
             error_report("vfio: failed to set group container: %m");
@@ -724,18 +735,41 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
             goto free_container_exit;
         }
 
-        container->iommu_data.type1.listener = vfio_memory_listener;
-        container->iommu_data.release = vfio_listener_release;
-
-        memory_listener_register(&container->iommu_data.type1.listener,
-                                 container->space->as);
+        /*
+         * This only considers the host IOMMU's 32-bit window.  At
+         * some point we need to add support for the optional 64-bit
+         * window and dynamic windows
+         */
+        info.argsz = sizeof(info);
+        ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
+        if (ret) {
+            error_report("vfio: VFIO_IOMMU_SPAPR_TCE_GET_INFO failed: %m");
+            ret = -errno;
+            goto free_container_exit;
+        }
+        container->min_iova = info.dma32_window_start;
+        container->max_iova = container->min_iova + info.dma32_window_size - 1;
 
+        /* Assume just 4K IOVA pages for now */
+        container->iova_pgsizes = 0x1000;
     } else {
         error_report("vfio: No available IOMMU models");
         ret = -EINVAL;
         goto free_container_exit;
     }
 
+    container->listener = vfio_memory_listener;
+
+    memory_listener_register(&container->listener, container->space->as);
+
+    if (container->error) {
+        ret = container->error;
+        error_report("vfio: memory listener initialization failed for container");
+        goto listener_release_exit;
+    }
+
+    container->initialized = true;
+
     QLIST_INIT(&container->group_list);
     QLIST_INSERT_HEAD(&space->containers, container, next);
 
@@ -774,9 +808,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
         VFIOAddressSpace *space = container->space;
         VFIOGuestIOMMU *giommu, *tmp;
 
-        if (container->iommu_data.release) {
-            container->iommu_data.release(container);
-        }
+        vfio_listener_release(container);
         QLIST_REMOVE(container, next);
 
         QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) {
diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c
new file mode 100644
index 000000000..30c68a1e2
--- /dev/null
+++ b/hw/vfio/pci-quirks.c
@@ -0,0 +1,1204 @@
+/*
+ * device quirks for PCI devices
+ *
+ * Copyright Red Hat, Inc. 2012-2015
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "pci.h"
+#include "trace.h"
+#include "qemu/range.h"
+
+/* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */
+static bool vfio_pci_is(VFIOPCIDevice *vdev, uint32_t vendor, uint32_t device)
+{
+    return (vendor == PCI_ANY_ID || vendor == vdev->vendor_id) &&
+           (device == PCI_ANY_ID || device == vdev->device_id);
+}
+
+static bool vfio_is_vga(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint16_t class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
+
+    return class == PCI_CLASS_DISPLAY_VGA;
+}
+
+/*
+ * List of device ids/vendor ids for which to disable
+ * option rom loading. This avoids the guest hangs during rom
+ * execution as noticed with the BCM 57810 card for lack of a
+ * more better way to handle such issues.
+ * The  user can still override by specifying a romfile or
+ * rombar=1.
+ * Please see https://bugs.launchpad.net/qemu/+bug/1284874
+ * for an analysis of the 57810 card hang. When adding
+ * a new vendor id/device id combination below, please also add
+ * your card/environment details and information that could
+ * help in debugging to the bug tracking this issue
+ */
+static const struct {
+    uint32_t vendor;
+    uint32_t device;
+} romblacklist[] = {
+    { 0x14e4, 0x168e }, /* Broadcom BCM 57810 */
+};
+
+bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0 ; i < ARRAY_SIZE(romblacklist); i++) {
+        if (vfio_pci_is(vdev, romblacklist[i].vendor, romblacklist[i].device)) {
+            trace_vfio_quirk_rom_blacklisted(vdev->vbasedev.name,
+                                             romblacklist[i].vendor,
+                                             romblacklist[i].device);
+            return true;
+        }
+    }
+    return false;
+}
+
+/*
+ * Device specific region quirks (mostly backdoors to PCI config space)
+ */
+
+/*
+ * The generic window quirks operate on an address and data register,
+ * vfio_generic_window_address_quirk handles the address register and
+ * vfio_generic_window_data_quirk handles the data register.  These ops
+ * pass reads and writes through to hardware until a value matching the
+ * stored address match/mask is written.  When this occurs, the data
+ * register access emulated PCI config space for the device rather than
+ * passing through accesses.  This enables devices where PCI config space
+ * is accessible behind a window register to maintain the virtualization
+ * provided through vfio.
+ */
+typedef struct VFIOConfigWindowMatch {
+    uint32_t match;
+    uint32_t mask;
+} VFIOConfigWindowMatch;
+
+typedef struct VFIOConfigWindowQuirk {
+    struct VFIOPCIDevice *vdev;
+
+    uint32_t address_val;
+
+    uint32_t address_offset;
+    uint32_t data_offset;
+
+    bool window_enabled;
+    uint8_t bar;
+
+    MemoryRegion *addr_mem;
+    MemoryRegion *data_mem;
+
+    uint32_t nr_matches;
+    VFIOConfigWindowMatch matches[];
+} VFIOConfigWindowQuirk;
+
+static uint64_t vfio_generic_window_quirk_address_read(void *opaque,
+                                                       hwaddr addr,
+                                                       unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+
+    return vfio_region_read(&vdev->bars[window->bar].region,
+                            addr + window->address_offset, size);
+}
+
+static void vfio_generic_window_quirk_address_write(void *opaque, hwaddr addr,
+                                                    uint64_t data,
+                                                    unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+    int i;
+
+    window->window_enabled = false;
+
+    vfio_region_write(&vdev->bars[window->bar].region,
+                      addr + window->address_offset, data, size);
+
+    for (i = 0; i < window->nr_matches; i++) {
+        if ((data & ~window->matches[i].mask) == window->matches[i].match) {
+            window->window_enabled = true;
+            window->address_val = data & window->matches[i].mask;
+            trace_vfio_quirk_generic_window_address_write(vdev->vbasedev.name,
+                                    memory_region_name(window->addr_mem), data);
+            break;
+        }
+    }
+}
+
+static const MemoryRegionOps vfio_generic_window_address_quirk = {
+    .read = vfio_generic_window_quirk_address_read,
+    .write = vfio_generic_window_quirk_address_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_generic_window_quirk_data_read(void *opaque,
+                                                    hwaddr addr, unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+    uint64_t data;
+
+    /* Always read data reg, discard if window enabled */
+    data = vfio_region_read(&vdev->bars[window->bar].region,
+                            addr + window->data_offset, size);
+
+    if (window->window_enabled) {
+        data = vfio_pci_read_config(&vdev->pdev, window->address_val, size);
+        trace_vfio_quirk_generic_window_data_read(vdev->vbasedev.name,
+                                    memory_region_name(window->data_mem), data);
+    }
+
+    return data;
+}
+
+static void vfio_generic_window_quirk_data_write(void *opaque, hwaddr addr,
+                                                 uint64_t data, unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+
+    if (window->window_enabled) {
+        vfio_pci_write_config(&vdev->pdev, window->address_val, data, size);
+        trace_vfio_quirk_generic_window_data_write(vdev->vbasedev.name,
+                                    memory_region_name(window->data_mem), data);
+        return;
+    }
+
+    vfio_region_write(&vdev->bars[window->bar].region,
+                      addr + window->data_offset, data, size);
+}
+
+static const MemoryRegionOps vfio_generic_window_data_quirk = {
+    .read = vfio_generic_window_quirk_data_read,
+    .write = vfio_generic_window_quirk_data_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/*
+ * The generic mirror quirk handles devices which expose PCI config space
+ * through a region within a BAR.  When enabled, reads and writes are
+ * redirected through to emulated PCI config space.  XXX if PCI config space
+ * used memory regions, this could just be an alias.
+ */
+typedef struct VFIOConfigMirrorQuirk {
+    struct VFIOPCIDevice *vdev;
+    uint32_t offset;
+    uint8_t bar;
+    MemoryRegion *mem;
+} VFIOConfigMirrorQuirk;
+
+static uint64_t vfio_generic_quirk_mirror_read(void *opaque,
+                                               hwaddr addr, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+    uint64_t data;
+
+    /* Read and discard in case the hardware cares */
+    (void)vfio_region_read(&vdev->bars[mirror->bar].region,
+                           addr + mirror->offset, size);
+
+    data = vfio_pci_read_config(&vdev->pdev, addr, size);
+    trace_vfio_quirk_generic_mirror_read(vdev->vbasedev.name,
+                                         memory_region_name(mirror->mem),
+                                         addr, data);
+    return data;
+}
+
+static void vfio_generic_quirk_mirror_write(void *opaque, hwaddr addr,
+                                            uint64_t data, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+
+    vfio_pci_write_config(&vdev->pdev, addr, data, size);
+    trace_vfio_quirk_generic_mirror_write(vdev->vbasedev.name,
+                                          memory_region_name(mirror->mem),
+                                          addr, data);
+}
+
+static const MemoryRegionOps vfio_generic_mirror_quirk = {
+    .read = vfio_generic_quirk_mirror_read,
+    .write = vfio_generic_quirk_mirror_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/* Is range1 fully contained within range2?  */
+static bool vfio_range_contained(uint64_t first1, uint64_t len1,
+                                 uint64_t first2, uint64_t len2) {
+    return (first1 >= first2 && first1 + len1 <= first2 + len2);
+}
+
+#define PCI_VENDOR_ID_ATI               0x1002
+
+/*
+ * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
+ * through VGA register 0x3c3.  On newer cards, the I/O port BAR is always
+ * BAR4 (older cards like the X550 used BAR1, but we don't care to support
+ * those).  Note that on bare metal, a read of 0x3c3 doesn't always return the
+ * I/O port BAR address.  Originally this was coded to return the virtual BAR
+ * address only if the physical register read returns the actual BAR address,
+ * but users have reported greater success if we return the virtual address
+ * unconditionally.
+ */
+static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
+                                        hwaddr addr, unsigned size)
+{
+    VFIOPCIDevice *vdev = opaque;
+    uint64_t data = vfio_pci_read_config(&vdev->pdev,
+                                         PCI_BASE_ADDRESS_4 + 1, size);
+
+    trace_vfio_quirk_ati_3c3_read(vdev->vbasedev.name, data);
+
+    return data;
+}
+
+static const MemoryRegionOps vfio_ati_3c3_quirk = {
+    .read = vfio_ati_3c3_quirk_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+
+    /*
+     * As long as the BAR is >= 256 bytes it will be aligned such that the
+     * lower byte is always zero.  Filter out anything else, if it exists.
+     */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->bars[4].ioport || vdev->bars[4].region.size < 256) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+
+    memory_region_init_io(quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, vdev,
+                          "vfio-ati-3c3-quirk", 1);
+    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
+                                3 /* offset 3 bytes from 0x3c0 */, quirk->mem);
+
+    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
+                      quirk, next);
+
+    trace_vfio_quirk_ati_3c3_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Newer ATI/AMD devices, including HD5450 and HD7850, have a mirror to PCI
+ * config space through MMIO BAR2 at offset 0x4000.  Nothing seems to access
+ * the MMIO space directly, but a window to this space is provided through
+ * I/O port BAR4.  Offset 0x0 is the address register and offset 0x4 is the
+ * data register.  When the address is programmed to a range of 0x4000-0x4fff
+ * PCI configuration space is available.  Experimentation seems to indicate
+ * that read-only may be provided by hardware.
+ */
+static void vfio_probe_ati_bar4_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigWindowQuirk *window;
+
+    /* This windows doesn't seem to be used except by legacy VGA code */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->has_vga || nr != 4) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    window = quirk->data = g_malloc0(sizeof(*window) +
+                                     sizeof(VFIOConfigWindowMatch));
+    window->vdev = vdev;
+    window->address_offset = 0;
+    window->data_offset = 4;
+    window->nr_matches = 1;
+    window->matches[0].match = 0x4000;
+    window->matches[0].mask = PCIE_CONFIG_SPACE_SIZE - 1;
+    window->bar = nr;
+    window->addr_mem = &quirk->mem[0];
+    window->data_mem = &quirk->mem[1];
+
+    memory_region_init_io(window->addr_mem, OBJECT(vdev),
+                          &vfio_generic_window_address_quirk, window,
+                          "vfio-ati-bar4-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        window->address_offset,
+                                        window->addr_mem, 1);
+
+    memory_region_init_io(window->data_mem, OBJECT(vdev),
+                          &vfio_generic_window_data_quirk, window,
+                          "vfio-ati-bar4-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        window->data_offset,
+                                        window->data_mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_ati_bar4_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Trap the BAR2 MMIO mirror to config space as well.
+ */
+static void vfio_probe_ati_bar2_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigMirrorQuirk *mirror;
+
+    /* Only enable on newer devices where BAR2 is 64bit */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->has_vga || nr != 2 || !vdev->bars[2].mem64) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    mirror = quirk->data = g_malloc0(sizeof(*mirror));
+    mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+    mirror->vdev = vdev;
+    mirror->offset = 0x4000;
+    mirror->bar = nr;
+
+    memory_region_init_io(mirror->mem, OBJECT(vdev),
+                          &vfio_generic_mirror_quirk, mirror,
+                          "vfio-ati-bar2-4000-quirk", PCI_CONFIG_SPACE_SIZE);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        mirror->offset, mirror->mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_ati_bar2_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Older ATI/AMD cards like the X550 have a similar window to that above.
+ * I/O port BAR1 provides a window to a mirror of PCI config space located
+ * in BAR2 at offset 0xf00.  We don't care to support such older cards, but
+ * note it for future reference.
+ */
+
+#define PCI_VENDOR_ID_NVIDIA                    0x10de
+
+/*
+ * Nvidia has several different methods to get to config space, the
+ * nouveu project has several of these documented here:
+ * https://github.com/pathscale/envytools/tree/master/hwdocs
+ *
+ * The first quirk is actually not documented in envytools and is found
+ * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]).  This is an
+ * NV46 chipset.  The backdoor uses the legacy VGA I/O ports to access
+ * the mirror of PCI config space found at BAR0 offset 0x1800.  The access
+ * sequence first writes 0x338 to I/O port 0x3d4.  The target offset is
+ * then written to 0x3d0.  Finally 0x538 is written for a read and 0x738
+ * is written for a write to 0x3d4.  The BAR0 offset is then accessible
+ * through 0x3d0.  This quirk doesn't seem to be necessary on newer cards
+ * that use the I/O port BAR5 window but it doesn't hurt to leave it.
+ */
+typedef enum {NONE = 0, SELECT, WINDOW, READ, WRITE} VFIONvidia3d0State;
+static const char *nv3d0_states[] = { "NONE", "SELECT",
+                                      "WINDOW", "READ", "WRITE" };
+
+typedef struct VFIONvidia3d0Quirk {
+    VFIOPCIDevice *vdev;
+    VFIONvidia3d0State state;
+    uint32_t offset;
+} VFIONvidia3d0Quirk;
+
+static uint64_t vfio_nvidia_3d4_quirk_read(void *opaque,
+                                           hwaddr addr, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+
+    quirk->state = NONE;
+
+    return vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
+                         addr + 0x14, size);
+}
+
+static void vfio_nvidia_3d4_quirk_write(void *opaque, hwaddr addr,
+                                        uint64_t data, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+
+    quirk->state = NONE;
+
+    switch (data) {
+    case 0x338:
+        if (old_state == NONE) {
+            quirk->state = SELECT;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    case 0x538:
+        if (old_state == WINDOW) {
+            quirk->state = READ;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    case 0x738:
+        if (old_state == WINDOW) {
+            quirk->state = WRITE;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    }
+
+    vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
+                   addr + 0x14, data, size);
+}
+
+static const MemoryRegionOps vfio_nvidia_3d4_quirk = {
+    .read = vfio_nvidia_3d4_quirk_read,
+    .write = vfio_nvidia_3d4_quirk_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
+                                           hwaddr addr, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+    uint64_t data = vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
+                                  addr + 0x10, size);
+
+    quirk->state = NONE;
+
+    if (old_state == READ &&
+        (quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) {
+        uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1);
+
+        data = vfio_pci_read_config(&vdev->pdev, offset, size);
+        trace_vfio_quirk_nvidia_3d0_read(vdev->vbasedev.name,
+                                         offset, size, data);
+    }
+
+    return data;
+}
+
+static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
+                                        uint64_t data, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+
+    quirk->state = NONE;
+
+    if (old_state == SELECT) {
+        quirk->offset = (uint32_t)data;
+        quirk->state = WINDOW;
+        trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                          nv3d0_states[quirk->state]);
+    } else if (old_state == WRITE) {
+        if ((quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) {
+            uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1);
+
+            vfio_pci_write_config(&vdev->pdev, offset, data, size);
+            trace_vfio_quirk_nvidia_3d0_write(vdev->vbasedev.name,
+                                              offset, data, size);
+            return;
+        }
+    }
+
+    vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
+                   addr + 0x10, data, size);
+}
+
+static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
+    .read = vfio_nvidia_3d0_quirk_read,
+    .write = vfio_nvidia_3d0_quirk_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+    VFIONvidia3d0Quirk *data;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vdev->bars[1].region.size) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->data = data = g_malloc0(sizeof(*data));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    data->vdev = vdev;
+
+    memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_nvidia_3d4_quirk,
+                          data, "vfio-nvidia-3d4-quirk", 2);
+    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
+                                0x14 /* 0x3c0 + 0x14 */, &quirk->mem[0]);
+
+    memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_nvidia_3d0_quirk,
+                          data, "vfio-nvidia-3d0-quirk", 2);
+    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
+                                0x10 /* 0x3c0 + 0x10 */, &quirk->mem[1]);
+
+    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
+                      quirk, next);
+
+    trace_vfio_quirk_nvidia_3d0_probe(vdev->vbasedev.name);
+}
+
+/*
+ * The second quirk is documented in envytools.  The I/O port BAR5 is just
+ * a set of address/data ports to the MMIO BARs.  The BAR we care about is
+ * again BAR0.  This backdoor is apparently a bit newer than the one above
+ * so we need to not only trap 256 bytes @0x1800, but all of PCI config
+ * space, including extended space is available at the 4k @0x88000.
+ */
+typedef struct VFIONvidiaBAR5Quirk {
+    uint32_t master;
+    uint32_t enable;
+    MemoryRegion *addr_mem;
+    MemoryRegion *data_mem;
+    bool enabled;
+    VFIOConfigWindowQuirk window; /* last for match data */
+} VFIONvidiaBAR5Quirk;
+
+static void vfio_nvidia_bar5_enable(VFIONvidiaBAR5Quirk *bar5)
+{
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    if (((bar5->master & bar5->enable) & 0x1) == bar5->enabled) {
+        return;
+    }
+
+    bar5->enabled = !bar5->enabled;
+    trace_vfio_quirk_nvidia_bar5_state(vdev->vbasedev.name,
+                                       bar5->enabled ?  "Enable" : "Disable");
+    memory_region_set_enabled(bar5->addr_mem, bar5->enabled);
+    memory_region_set_enabled(bar5->data_mem, bar5->enabled);
+}
+
+static uint64_t vfio_nvidia_bar5_quirk_master_read(void *opaque,
+                                                   hwaddr addr, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    return vfio_region_read(&vdev->bars[5].region, addr, size);
+}
+
+static void vfio_nvidia_bar5_quirk_master_write(void *opaque, hwaddr addr,
+                                                uint64_t data, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    vfio_region_write(&vdev->bars[5].region, addr, data, size);
+
+    bar5->master = data;
+    vfio_nvidia_bar5_enable(bar5);
+}
+
+static const MemoryRegionOps vfio_nvidia_bar5_quirk_master = {
+    .read = vfio_nvidia_bar5_quirk_master_read,
+    .write = vfio_nvidia_bar5_quirk_master_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_nvidia_bar5_quirk_enable_read(void *opaque,
+                                                   hwaddr addr, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    return vfio_region_read(&vdev->bars[5].region, addr + 4, size);
+}
+
+static void vfio_nvidia_bar5_quirk_enable_write(void *opaque, hwaddr addr,
+                                                uint64_t data, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    vfio_region_write(&vdev->bars[5].region, addr + 4, data, size);
+
+    bar5->enable = data;
+    vfio_nvidia_bar5_enable(bar5);
+}
+
+static const MemoryRegionOps vfio_nvidia_bar5_quirk_enable = {
+    .read = vfio_nvidia_bar5_quirk_enable_read,
+    .write = vfio_nvidia_bar5_quirk_enable_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_nvidia_bar5_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIONvidiaBAR5Quirk *bar5;
+    VFIOConfigWindowQuirk *window;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vdev->has_vga || nr != 5) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 4);
+    quirk->nr_mem = 4;
+    bar5 = quirk->data = g_malloc0(sizeof(*bar5) +
+                                   (sizeof(VFIOConfigWindowMatch) * 2));
+    window = &bar5->window;
+
+    window->vdev = vdev;
+    window->address_offset = 0x8;
+    window->data_offset = 0xc;
+    window->nr_matches = 2;
+    window->matches[0].match = 0x1800;
+    window->matches[0].mask = PCI_CONFIG_SPACE_SIZE - 1;
+    window->matches[1].match = 0x88000;
+    window->matches[1].mask = PCIE_CONFIG_SPACE_SIZE - 1;
+    window->bar = nr;
+    window->addr_mem = bar5->addr_mem = &quirk->mem[0];
+    window->data_mem = bar5->data_mem = &quirk->mem[1];
+
+    memory_region_init_io(window->addr_mem, OBJECT(vdev),
+                          &vfio_generic_window_address_quirk, window,
+                          "vfio-nvidia-bar5-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        window->address_offset,
+                                        window->addr_mem, 1);
+    memory_region_set_enabled(window->addr_mem, false);
+
+    memory_region_init_io(window->data_mem, OBJECT(vdev),
+                          &vfio_generic_window_data_quirk, window,
+                          "vfio-nvidia-bar5-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        window->data_offset,
+                                        window->data_mem, 1);
+    memory_region_set_enabled(window->data_mem, false);
+
+    memory_region_init_io(&quirk->mem[2], OBJECT(vdev),
+                          &vfio_nvidia_bar5_quirk_master, bar5,
+                          "vfio-nvidia-bar5-master-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        0, &quirk->mem[2], 1);
+
+    memory_region_init_io(&quirk->mem[3], OBJECT(vdev),
+                          &vfio_nvidia_bar5_quirk_enable, bar5,
+                          "vfio-nvidia-bar5-enable-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        4, &quirk->mem[3], 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_nvidia_bar5_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Finally, BAR0 itself.  We want to redirect any accesses to either
+ * 0x1800 or 0x88000 through the PCI config space access functions.
+ */
+static void vfio_nvidia_quirk_mirror_write(void *opaque, hwaddr addr,
+                                           uint64_t data, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+    PCIDevice *pdev = &vdev->pdev;
+
+    vfio_generic_quirk_mirror_write(opaque, addr, data, size);
+
+    /*
+     * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
+     * MSI capability ID register.  Both the ID and next register are
+     * read-only, so we allow writes covering either of those to real hw.
+     */
+    if ((pdev->cap_present & QEMU_PCI_CAP_MSI) &&
+        vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) {
+        vfio_region_write(&vdev->bars[mirror->bar].region,
+                          addr + mirror->offset, data, size);
+        trace_vfio_quirk_nvidia_bar0_msi_ack(vdev->vbasedev.name);
+    }
+}
+
+static const MemoryRegionOps vfio_nvidia_mirror_quirk = {
+    .read = vfio_generic_quirk_mirror_read,
+    .write = vfio_nvidia_quirk_mirror_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_nvidia_bar0_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigMirrorQuirk *mirror;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vfio_is_vga(vdev) || nr != 0) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    mirror = quirk->data = g_malloc0(sizeof(*mirror));
+    mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+    mirror->vdev = vdev;
+    mirror->offset = 0x88000;
+    mirror->bar = nr;
+
+    memory_region_init_io(mirror->mem, OBJECT(vdev),
+                          &vfio_nvidia_mirror_quirk, mirror,
+                          "vfio-nvidia-bar0-88000-mirror-quirk",
+                          PCIE_CONFIG_SPACE_SIZE);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        mirror->offset, mirror->mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    /* The 0x1800 offset mirror only seems to get used by legacy VGA */
+    if (vdev->has_vga) {
+        quirk = g_malloc0(sizeof(*quirk));
+        mirror = quirk->data = g_malloc0(sizeof(*mirror));
+        mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+        quirk->nr_mem = 1;
+        mirror->vdev = vdev;
+        mirror->offset = 0x1800;
+        mirror->bar = nr;
+
+        memory_region_init_io(mirror->mem, OBJECT(vdev),
+                              &vfio_nvidia_mirror_quirk, mirror,
+                              "vfio-nvidia-bar0-1800-mirror-quirk",
+                              PCI_CONFIG_SPACE_SIZE);
+        memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                            mirror->offset, mirror->mem, 1);
+
+        QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+    }
+
+    trace_vfio_quirk_nvidia_bar0_probe(vdev->vbasedev.name);
+}
+
+/*
+ * TODO - Some Nvidia devices provide config access to their companion HDA
+ * device and even to their parent bridge via these config space mirrors.
+ * Add quirks for those regions.
+ */
+
+#define PCI_VENDOR_ID_REALTEK 0x10ec
+
+/*
+ * RTL8168 devices have a backdoor that can access the MSI-X table.  At BAR2
+ * offset 0x70 there is a dword data register, offset 0x74 is a dword address
+ * register.  According to the Linux r8169 driver, the MSI-X table is addressed
+ * when the "type" portion of the address register is set to 0x1.  This appears
+ * to be bits 16:30.  Bit 31 is both a write indicator and some sort of
+ * "address latched" indicator.  Bits 12:15 are a mask field, which we can
+ * ignore because the MSI-X table should always be accessed as a dword (full
+ * mask).  Bits 0:11 is offset within the type.
+ *
+ * Example trace:
+ *
+ * Read from MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
+ *
+ * Write 0xfee00000 to MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
+ */
+typedef struct VFIOrtl8168Quirk {
+    VFIOPCIDevice *vdev;
+    uint32_t addr;
+    uint32_t data;
+    bool enabled;
+} VFIOrtl8168Quirk;
+
+static uint64_t vfio_rtl8168_quirk_address_read(void *opaque,
+                                                hwaddr addr, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+    uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size);
+
+    if (rtl->enabled) {
+        data = rtl->addr ^ 0x80000000U; /* latch/complete */
+        trace_vfio_quirk_rtl8168_fake_latch(vdev->vbasedev.name, data);
+    }
+
+    return data;
+}
+
+static void vfio_rtl8168_quirk_address_write(void *opaque, hwaddr addr,
+                                             uint64_t data, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+
+    rtl->enabled = false;
+
+    if ((data & 0x7fff0000) == 0x10000) { /* MSI-X table */
+        rtl->enabled = true;
+        rtl->addr = (uint32_t)data;
+
+        if (data & 0x80000000U) { /* Do write */
+            if (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) {
+                hwaddr offset = data & 0xfff;
+                uint64_t val = rtl->data;
+
+                trace_vfio_quirk_rtl8168_msix_write(vdev->vbasedev.name,
+                                                    (uint16_t)offset, val);
+
+                /* Write to the proper guest MSI-X table instead */
+                memory_region_dispatch_write(&vdev->pdev.msix_table_mmio,
+                                             offset, val, size,
+                                             MEMTXATTRS_UNSPECIFIED);
+            }
+            return; /* Do not write guest MSI-X data to hardware */
+        }
+    }
+
+    vfio_region_write(&vdev->bars[2].region, addr + 0x74, data, size);
+}
+
+static const MemoryRegionOps vfio_rtl_address_quirk = {
+    .read = vfio_rtl8168_quirk_address_read,
+    .write = vfio_rtl8168_quirk_address_write,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_rtl8168_quirk_data_read(void *opaque,
+                                             hwaddr addr, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+    uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size);
+
+    if (rtl->enabled && (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) {
+        hwaddr offset = rtl->addr & 0xfff;
+        memory_region_dispatch_read(&vdev->pdev.msix_table_mmio, offset,
+                                    &data, size, MEMTXATTRS_UNSPECIFIED);
+        trace_vfio_quirk_rtl8168_msix_read(vdev->vbasedev.name, offset, data);
+    }
+
+    return data;
+}
+
+static void vfio_rtl8168_quirk_data_write(void *opaque, hwaddr addr,
+                                          uint64_t data, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+
+    rtl->data = (uint32_t)data;
+
+    vfio_region_write(&vdev->bars[2].region, addr + 0x70, data, size);
+}
+
+static const MemoryRegionOps vfio_rtl_data_quirk = {
+    .read = vfio_rtl8168_quirk_data_read,
+    .write = vfio_rtl8168_quirk_data_write,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_rtl8168_bar2_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOrtl8168Quirk *rtl;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_REALTEK, 0x8168) || nr != 2) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    quirk->data = rtl = g_malloc0(sizeof(*rtl));
+    rtl->vdev = vdev;
+
+    memory_region_init_io(&quirk->mem[0], OBJECT(vdev),
+                          &vfio_rtl_address_quirk, rtl,
+                          "vfio-rtl8168-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        0x74, &quirk->mem[0], 1);
+
+    memory_region_init_io(&quirk->mem[1], OBJECT(vdev),
+                          &vfio_rtl_data_quirk, rtl,
+                          "vfio-rtl8168-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
+                                        0x70, &quirk->mem[1], 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_rtl8168_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Common quirk probe entry points.
+ */
+void vfio_vga_quirk_setup(VFIOPCIDevice *vdev)
+{
+    vfio_vga_probe_ati_3c3_quirk(vdev);
+    vfio_vga_probe_nvidia_3d0_quirk(vdev);
+}
+
+void vfio_vga_quirk_teardown(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+    int i, j;
+
+    for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
+        QLIST_FOREACH(quirk, &vdev->vga.region[i].quirks, next) {
+            for (j = 0; j < quirk->nr_mem; j++) {
+                memory_region_del_subregion(&vdev->vga.region[i].mem,
+                                            &quirk->mem[j]);
+            }
+        }
+    }
+}
+
+void vfio_vga_quirk_free(VFIOPCIDevice *vdev)
+{
+    int i, j;
+
+    for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
+        while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) {
+            VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks);
+            QLIST_REMOVE(quirk, next);
+            for (j = 0; j < quirk->nr_mem; j++) {
+                object_unparent(OBJECT(&quirk->mem[j]));
+            }
+            g_free(quirk->mem);
+            g_free(quirk->data);
+            g_free(quirk);
+        }
+    }
+}
+
+void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr)
+{
+    vfio_probe_ati_bar4_quirk(vdev, nr);
+    vfio_probe_ati_bar2_quirk(vdev, nr);
+    vfio_probe_nvidia_bar5_quirk(vdev, nr);
+    vfio_probe_nvidia_bar0_quirk(vdev, nr);
+    vfio_probe_rtl8168_bar2_quirk(vdev, nr);
+}
+
+void vfio_bar_quirk_teardown(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    VFIOQuirk *quirk;
+    int i;
+
+    QLIST_FOREACH(quirk, &bar->quirks, next) {
+        for (i = 0; i < quirk->nr_mem; i++) {
+            memory_region_del_subregion(&bar->region.mem, &quirk->mem[i]);
+        }
+    }
+}
+
+void vfio_bar_quirk_free(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    int i;
+
+    while (!QLIST_EMPTY(&bar->quirks)) {
+        VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
+        QLIST_REMOVE(quirk, next);
+        for (i = 0; i < quirk->nr_mem; i++) {
+            object_unparent(OBJECT(&quirk->mem[i]));
+        }
+        g_free(quirk->mem);
+        g_free(quirk->data);
+        g_free(quirk);
+    }
+}
+
+/*
+ * Reset quirks
+ */
+
+/*
+ * AMD Radeon PCI config reset, based on Linux:
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running()
+ *   drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc()
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock()
+ * IDs: include/drm/drm_pciids.h
+ * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0
+ *
+ * Bonaire and Hawaii GPUs do not respond to a bus reset.  This is a bug in the
+ * hardware that should be fixed on future ASICs.  The symptom of this is that
+ * once the accerlated driver loads, Windows guests will bsod on subsequent
+ * attmpts to load the driver, such as after VM reset or shutdown/restart.  To
+ * work around this, we do an AMD specific PCI config reset, followed by an SMC
+ * reset.  The PCI config reset only works if SMC firmware is running, so we
+ * have a dependency on the state of the device as to whether this reset will
+ * be effective.  There are still cases where we won't be able to kick the
+ * device into working, but this greatly improves the usability overall.  The
+ * config reset magic is relatively common on AMD GPUs, but the setup and SMC
+ * poking is largely ASIC specific.
+ */
+static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev)
+{
+    uint32_t clk, pc_c;
+
+    /*
+     * Registers 200h and 204h are index and data registers for accessing
+     * indirect configuration registers within the device.
+     */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
+    clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4);
+    pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+
+    return (!(clk & 1) && (0x20100 <= pc_c));
+}
+
+/*
+ * The scope of a config reset is controlled by a mode bit in the misc register
+ * and a fuse, exposed as a bit in another register.  The fuse is the default
+ * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula
+ * scope = !(misc ^ fuse), where the resulting scope is defined the same as
+ * the fuse.  A truth table therefore tells us that if misc == fuse, we need
+ * to flip the value of the bit in the misc register.
+ */
+static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev)
+{
+    uint32_t misc, fuse;
+    bool a, b;
+
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4);
+    fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    b = fuse & 64;
+
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4);
+    misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    a = misc & 2;
+
+    if (a == b) {
+        vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4);
+        vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */
+    }
+}
+
+static int vfio_radeon_reset(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    int i, ret = 0;
+    uint32_t data;
+
+    /* Defer to a kernel implemented reset */
+    if (vdev->vbasedev.reset_works) {
+        trace_vfio_quirk_ati_bonaire_reset_skipped(vdev->vbasedev.name);
+        return -ENODEV;
+    }
+
+    /* Enable only memory BAR access */
+    vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2);
+
+    /* Reset only works if SMC firmware is loaded and running */
+    if (!vfio_radeon_smc_is_running(vdev)) {
+        ret = -EINVAL;
+        trace_vfio_quirk_ati_bonaire_reset_no_smc(vdev->vbasedev.name);
+        goto out;
+    }
+
+    /* Make sure only the GFX function is reset */
+    vfio_radeon_set_gfx_only_reset(vdev);
+
+    /* AMD PCI config reset */
+    vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4);
+    usleep(100);
+
+    /* Read back the memory size to make sure we're out of reset */
+    for (i = 0; i < 100000; i++) {
+        if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) {
+            goto reset_smc;
+        }
+        usleep(1);
+    }
+
+    trace_vfio_quirk_ati_bonaire_reset_timeout(vdev->vbasedev.name);
+
+reset_smc:
+    /* Reset SMC */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4);
+    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    data |= 1;
+    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
+
+    /* Disable SMC clock */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
+    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    data |= 1;
+    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
+
+    trace_vfio_quirk_ati_bonaire_reset_done(vdev->vbasedev.name);
+
+out:
+    /* Restore PCI command register */
+    vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2);
+
+    return ret;
+}
+
+void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev)
+{
+    switch (vdev->vendor_id) {
+    case 0x1002:
+        switch (vdev->device_id) {
+        /* Bonaire */
+        case 0x6649: /* Bonaire [FirePro W5100] */
+        case 0x6650:
+        case 0x6651:
+        case 0x6658: /* Bonaire XTX [Radeon R7 260X] */
+        case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */
+        case 0x665d: /* Bonaire [Radeon R7 200 Series] */
+        /* Hawaii */
+        case 0x67A0: /* Hawaii XT GL [FirePro W9100] */
+        case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */
+        case 0x67A2:
+        case 0x67A8:
+        case 0x67A9:
+        case 0x67AA:
+        case 0x67B0: /* Hawaii XT [Radeon R9 290X] */
+        case 0x67B1: /* Hawaii PRO [Radeon R9 290] */
+        case 0x67B8:
+        case 0x67B9:
+        case 0x67BA:
+        case 0x67BE:
+            vdev->resetfn = vfio_radeon_reset;
+            trace_vfio_quirk_ati_bonaire_reset(vdev->vbasedev.name);
+            break;
+        }
+        break;
+    }
+}
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 4023d8e82..1fb868c24 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -18,7 +18,6 @@
  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
  */
 
-#include <dirent.h>
 #include <linux/vfio.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
@@ -27,178 +26,19 @@
 #include <unistd.h>
 
 #include "config.h"
-#include "exec/address-spaces.h"
-#include "exec/memory.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
-#include "hw/pci/pci.h"
-#include "qemu-common.h"
+#include "hw/pci/pci_bridge.h"
 #include "qemu/error-report.h"
-#include "qemu/event_notifier.h"
-#include "qemu/queue.h"
 #include "qemu/range.h"
 #include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
+#include "pci.h"
 #include "trace.h"
-#include "hw/vfio/vfio.h"
-#include "hw/vfio/vfio-common.h"
-
-struct VFIOPCIDevice;
-
-typedef struct VFIOQuirk {
-    MemoryRegion mem;
-    struct VFIOPCIDevice *vdev;
-    QLIST_ENTRY(VFIOQuirk) next;
-    struct {
-        uint32_t base_offset:TARGET_PAGE_BITS;
-        uint32_t address_offset:TARGET_PAGE_BITS;
-        uint32_t address_size:3;
-        uint32_t bar:3;
-
-        uint32_t address_match;
-        uint32_t address_mask;
-
-        uint32_t address_val:TARGET_PAGE_BITS;
-        uint32_t data_offset:TARGET_PAGE_BITS;
-        uint32_t data_size:3;
-
-        uint8_t flags;
-        uint8_t read_flags;
-        uint8_t write_flags;
-    } data;
-} VFIOQuirk;
-
-typedef struct VFIOBAR {
-    VFIORegion region;
-    bool ioport;
-    bool mem64;
-    QLIST_HEAD(, VFIOQuirk) quirks;
-} VFIOBAR;
-
-typedef struct VFIOVGARegion {
-    MemoryRegion mem;
-    off_t offset;
-    int nr;
-    QLIST_HEAD(, VFIOQuirk) quirks;
-} VFIOVGARegion;
-
-typedef struct VFIOVGA {
-    off_t fd_offset;
-    int fd;
-    VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
-} VFIOVGA;
-
-typedef struct VFIOINTx {
-    bool pending; /* interrupt pending */
-    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
-    uint8_t pin; /* which pin to pull for qemu_set_irq */
-    EventNotifier interrupt; /* eventfd triggered on interrupt */
-    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
-    PCIINTxRoute route; /* routing info for QEMU bypass */
-    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
-    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
-} VFIOINTx;
-
-typedef struct VFIOMSIVector {
-    /*
-     * Two interrupt paths are configured per vector.  The first, is only used
-     * for interrupts injected via QEMU.  This is typically the non-accel path,
-     * but may also be used when we want QEMU to handle masking and pending
-     * bits.  The KVM path bypasses QEMU and is therefore higher performance,
-     * but requires masking at the device.  virq is used to track the MSI route
-     * through KVM, thus kvm_interrupt is only available when virq is set to a
-     * valid (>= 0) value.
-     */
-    EventNotifier interrupt;
-    EventNotifier kvm_interrupt;
-    struct VFIOPCIDevice *vdev; /* back pointer to device */
-    int virq;
-    bool use;
-} VFIOMSIVector;
-
-enum {
-    VFIO_INT_NONE = 0,
-    VFIO_INT_INTx = 1,
-    VFIO_INT_MSI  = 2,
-    VFIO_INT_MSIX = 3,
-};
-
-/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
-typedef struct VFIOMSIXInfo {
-    uint8_t table_bar;
-    uint8_t pba_bar;
-    uint16_t entries;
-    uint32_t table_offset;
-    uint32_t pba_offset;
-    MemoryRegion mmap_mem;
-    void *mmap;
-} VFIOMSIXInfo;
-
-typedef struct VFIOPCIDevice {
-    PCIDevice pdev;
-    VFIODevice vbasedev;
-    VFIOINTx intx;
-    unsigned int config_size;
-    uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
-    off_t config_offset; /* Offset of config space region within device fd */
-    unsigned int rom_size;
-    off_t rom_offset; /* Offset of ROM region within device fd */
-    void *rom;
-    int msi_cap_size;
-    VFIOMSIVector *msi_vectors;
-    VFIOMSIXInfo *msix;
-    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
-    int interrupt; /* Current interrupt type */
-    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
-    VFIOVGA vga; /* 0xa0000, 0x3b0, 0x3c0 */
-    PCIHostDeviceAddress host;
-    EventNotifier err_notifier;
-    EventNotifier req_notifier;
-    int (*resetfn)(struct VFIOPCIDevice *);
-    uint32_t features;
-#define VFIO_FEATURE_ENABLE_VGA_BIT 0
-#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
-#define VFIO_FEATURE_ENABLE_REQ_BIT 1
-#define VFIO_FEATURE_ENABLE_REQ (1 << VFIO_FEATURE_ENABLE_REQ_BIT)
-    int32_t bootindex;
-    uint8_t pm_cap;
-    bool has_vga;
-    bool pci_aer;
-    bool req_enabled;
-    bool has_flr;
-    bool has_pm_reset;
-    bool rom_read_failed;
-} VFIOPCIDevice;
-
-typedef struct VFIORomBlacklistEntry {
-    uint16_t vendor_id;
-    uint16_t device_id;
-} VFIORomBlacklistEntry;
-
-/*
- * List of device ids/vendor ids for which to disable
- * option rom loading. This avoids the guest hangs during rom
- * execution as noticed with the BCM 57810 card for lack of a
- * more better way to handle such issues.
- * The  user can still override by specifying a romfile or
- * rombar=1.
- * Please see https://bugs.launchpad.net/qemu/+bug/1284874
- * for an analysis of the 57810 card hang. When adding
- * a new vendor id/device id combination below, please also add
- * your card/environment details and information that could
- * help in debugging to the bug tracking this issue
- */
-static const VFIORomBlacklistEntry romblacklist[] = {
-    /* Broadcom BCM 57810 */
-    { 0x14e4, 0x168e }
-};
 
 #define MSIX_CAP_LENGTH 12
 
 static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
-static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
-static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
-                                  uint32_t val, int len);
 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
 
 /*
@@ -248,7 +88,7 @@ static void vfio_intx_interrupt(void *opaque)
     }
 }
 
-static void vfio_eoi(VFIODevice *vbasedev)
+static void vfio_intx_eoi(VFIODevice *vbasedev)
 {
     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
 
@@ -256,14 +96,14 @@ static void vfio_eoi(VFIODevice *vbasedev)
         return;
     }
 
-    trace_vfio_eoi(vbasedev->name);
+    trace_vfio_intx_eoi(vbasedev->name);
 
     vdev->intx.pending = false;
     pci_irq_deassert(&vdev->pdev);
     vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
 }
 
-static void vfio_enable_intx_kvm(VFIOPCIDevice *vdev)
+static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev)
 {
 #ifdef CONFIG_KVM
     struct kvm_irqfd irqfd = {
@@ -275,7 +115,7 @@ static void vfio_enable_intx_kvm(VFIOPCIDevice *vdev)
     int ret, argsz;
     int32_t *pfd;
 
-    if (!VFIO_ALLOW_KVM_INTX || !kvm_irqfds_enabled() ||
+    if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
         vdev->intx.route.mode != PCI_INTX_ENABLED ||
         !kvm_resamplefds_enabled()) {
         return;
@@ -325,7 +165,7 @@ static void vfio_enable_intx_kvm(VFIOPCIDevice *vdev)
 
     vdev->intx.kvm_accel = true;
 
-    trace_vfio_enable_intx_kvm(vdev->vbasedev.name);
+    trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
 
     return;
 
@@ -340,7 +180,7 @@ fail:
 #endif
 }
 
-static void vfio_disable_intx_kvm(VFIOPCIDevice *vdev)
+static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
 {
 #ifdef CONFIG_KVM
     struct kvm_irqfd irqfd = {
@@ -377,11 +217,11 @@ static void vfio_disable_intx_kvm(VFIOPCIDevice *vdev)
     /* If we've missed an event, let it re-fire through QEMU */
     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
 
-    trace_vfio_disable_intx_kvm(vdev->vbasedev.name);
+    trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
 #endif
 }
 
-static void vfio_update_irq(PCIDevice *pdev)
+static void vfio_intx_update(PCIDevice *pdev)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
     PCIINTxRoute route;
@@ -396,10 +236,10 @@ static void vfio_update_irq(PCIDevice *pdev)
         return; /* Nothing changed */
     }
 
-    trace_vfio_update_irq(vdev->vbasedev.name,
-                          vdev->intx.route.irq, route.irq);
+    trace_vfio_intx_update(vdev->vbasedev.name,
+                           vdev->intx.route.irq, route.irq);
 
-    vfio_disable_intx_kvm(vdev);
+    vfio_intx_disable_kvm(vdev);
 
     vdev->intx.route = route;
 
@@ -407,13 +247,13 @@ static void vfio_update_irq(PCIDevice *pdev)
         return;
     }
 
-    vfio_enable_intx_kvm(vdev);
+    vfio_intx_enable_kvm(vdev);
 
     /* Re-enable the interrupt in cased we missed an EOI */
-    vfio_eoi(&vdev->vbasedev);
+    vfio_intx_eoi(&vdev->vbasedev);
 }
 
-static int vfio_enable_intx(VFIOPCIDevice *vdev)
+static int vfio_intx_enable(VFIOPCIDevice *vdev)
 {
     uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
     int ret, argsz;
@@ -468,21 +308,21 @@ static int vfio_enable_intx(VFIOPCIDevice *vdev)
         return -errno;
     }
 
-    vfio_enable_intx_kvm(vdev);
+    vfio_intx_enable_kvm(vdev);
 
     vdev->interrupt = VFIO_INT_INTx;
 
-    trace_vfio_enable_intx(vdev->vbasedev.name);
+    trace_vfio_intx_enable(vdev->vbasedev.name);
 
     return 0;
 }
 
-static void vfio_disable_intx(VFIOPCIDevice *vdev)
+static void vfio_intx_disable(VFIOPCIDevice *vdev)
 {
     int fd;
 
     timer_del(vdev->intx.mmap_timer);
-    vfio_disable_intx_kvm(vdev);
+    vfio_intx_disable_kvm(vdev);
     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
     vdev->intx.pending = false;
     pci_irq_deassert(&vdev->pdev);
@@ -494,7 +334,7 @@ static void vfio_disable_intx(VFIOPCIDevice *vdev)
 
     vdev->interrupt = VFIO_INT_NONE;
 
-    trace_vfio_disable_intx(vdev->vbasedev.name);
+    trace_vfio_intx_disable(vdev->vbasedev.name);
 }
 
 /*
@@ -504,33 +344,28 @@ static void vfio_msi_interrupt(void *opaque)
 {
     VFIOMSIVector *vector = opaque;
     VFIOPCIDevice *vdev = vector->vdev;
+    MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
+    void (*notify)(PCIDevice *dev, unsigned vector);
+    MSIMessage msg;
     int nr = vector - vdev->msi_vectors;
 
     if (!event_notifier_test_and_clear(&vector->interrupt)) {
         return;
     }
 
-#ifdef DEBUG_VFIO
-    MSIMessage msg;
-
     if (vdev->interrupt == VFIO_INT_MSIX) {
-        msg = msix_get_message(&vdev->pdev, nr);
+        get_msg = msix_get_message;
+        notify = msix_notify;
     } else if (vdev->interrupt == VFIO_INT_MSI) {
-        msg = msi_get_message(&vdev->pdev, nr);
+        get_msg = msi_get_message;
+        notify = msi_notify;
     } else {
         abort();
     }
 
+    msg = get_msg(&vdev->pdev, nr);
     trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
-#endif
-
-    if (vdev->interrupt == VFIO_INT_MSIX) {
-        msix_notify(&vdev->pdev, nr);
-    } else if (vdev->interrupt == VFIO_INT_MSI) {
-        msi_notify(&vdev->pdev, nr);
-    } else {
-        error_report("vfio: MSI interrupt receieved, but not enabled?");
-    }
+    notify(&vdev->pdev, nr);
 }
 
 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
@@ -577,13 +412,12 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
     return ret;
 }
 
-static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg,
-                                  bool msix)
+static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
+                                  MSIMessage *msg, bool msix)
 {
     int virq;
 
-    if ((msix && !VFIO_ALLOW_KVM_MSIX) ||
-        (!msix && !VFIO_ALLOW_KVM_MSI) || !msg) {
+    if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi) || !msg) {
         return;
     }
 
@@ -591,7 +425,7 @@ static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg,
         return;
     }
 
-    virq = kvm_irqchip_add_msi_route(kvm_state, *msg);
+    virq = kvm_irqchip_add_msi_route(kvm_state, *msg, &vdev->pdev);
     if (virq < 0) {
         event_notifier_cleanup(&vector->kvm_interrupt);
         return;
@@ -616,9 +450,10 @@ static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
     event_notifier_cleanup(&vector->kvm_interrupt);
 }
 
-static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg)
+static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
+                                     PCIDevice *pdev)
 {
-    kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg);
+    kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
 }
 
 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
@@ -653,10 +488,10 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         if (!msg) {
             vfio_remove_kvm_msi_virq(vector);
         } else {
-            vfio_update_kvm_msi_virq(vector, *msg);
+            vfio_update_kvm_msi_virq(vector, *msg, pdev);
         }
     } else {
-        vfio_add_kvm_msi_virq(vector, msg, true);
+        vfio_add_kvm_msi_virq(vdev, vector, msg, true);
     }
 
     /*
@@ -748,11 +583,11 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
     }
 }
 
-static void vfio_enable_msix(VFIOPCIDevice *vdev)
+static void vfio_msix_enable(VFIOPCIDevice *vdev)
 {
     vfio_disable_interrupts(vdev);
 
-    vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector));
+    vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
 
     vdev->interrupt = VFIO_INT_MSIX;
 
@@ -777,10 +612,10 @@ static void vfio_enable_msix(VFIOPCIDevice *vdev)
         error_report("vfio: msix_set_vector_notifiers failed");
     }
 
-    trace_vfio_enable_msix(vdev->vbasedev.name);
+    trace_vfio_msix_enable(vdev->vbasedev.name);
 }
 
-static void vfio_enable_msi(VFIOPCIDevice *vdev)
+static void vfio_msi_enable(VFIOPCIDevice *vdev)
 {
     int ret, i;
 
@@ -788,7 +623,7 @@ static void vfio_enable_msi(VFIOPCIDevice *vdev)
 
     vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
 retry:
-    vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
+    vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
 
     for (i = 0; i < vdev->nr_vectors; i++) {
         VFIOMSIVector *vector = &vdev->msi_vectors[i];
@@ -809,7 +644,7 @@ retry:
          * Attempt to enable route through KVM irqchip,
          * default to userspace handling if unavailable.
          */
-        vfio_add_kvm_msi_virq(vector, &msg, false);
+        vfio_add_kvm_msi_virq(vdev, vector, &msg, false);
     }
 
     /* Set interrupt type prior to possible interrupts */
@@ -853,10 +688,10 @@ retry:
         return;
     }
 
-    trace_vfio_enable_msi(vdev->vbasedev.name, vdev->nr_vectors);
+    trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
 }
 
-static void vfio_disable_msi_common(VFIOPCIDevice *vdev)
+static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
 {
     int i;
 
@@ -877,10 +712,10 @@ static void vfio_disable_msi_common(VFIOPCIDevice *vdev)
     vdev->nr_vectors = 0;
     vdev->interrupt = VFIO_INT_NONE;
 
-    vfio_enable_intx(vdev);
+    vfio_intx_enable(vdev);
 }
 
-static void vfio_disable_msix(VFIOPCIDevice *vdev)
+static void vfio_msix_disable(VFIOPCIDevice *vdev)
 {
     int i;
 
@@ -901,17 +736,17 @@ static void vfio_disable_msix(VFIOPCIDevice *vdev)
         vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
     }
 
-    vfio_disable_msi_common(vdev);
+    vfio_msi_disable_common(vdev);
 
-    trace_vfio_disable_msix(vdev->vbasedev.name);
+    trace_vfio_msix_disable(vdev->vbasedev.name);
 }
 
-static void vfio_disable_msi(VFIOPCIDevice *vdev)
+static void vfio_msi_disable(VFIOPCIDevice *vdev)
 {
     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
-    vfio_disable_msi_common(vdev);
+    vfio_msi_disable_common(vdev);
 
-    trace_vfio_disable_msi(vdev->vbasedev.name);
+    trace_vfio_msi_disable(vdev->vbasedev.name);
 }
 
 static void vfio_update_msi(VFIOPCIDevice *vdev)
@@ -927,7 +762,7 @@ static void vfio_update_msi(VFIOPCIDevice *vdev)
         }
 
         msg = msi_get_message(&vdev->pdev, i);
-        vfio_update_kvm_msi_virq(vector, msg);
+        vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
     }
 }
 
@@ -1034,26 +869,6 @@ static const MemoryRegionOps vfio_rom_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    uint16_t vendor_id, device_id;
-    int count = 0;
-
-    vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
-
-    while (count < ARRAY_SIZE(romblacklist)) {
-        if (romblacklist[count].vendor_id == vendor_id &&
-            romblacklist[count].device_id == device_id) {
-                return true;
-        }
-        count++;
-    }
-
-    return false;
-}
-
 static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
 {
     uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
@@ -1131,7 +946,7 @@ static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
     vdev->rom_read_failed = false;
 }
 
-static void vfio_vga_write(void *opaque, hwaddr addr,
+void vfio_vga_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size)
 {
     VFIOVGARegion *region = opaque;
@@ -1167,7 +982,7 @@ static void vfio_vga_write(void *opaque, hwaddr addr,
     trace_vfio_vga_write(region->offset + addr, data, size);
 }
 
-static uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
+uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
 {
     VFIOVGARegion *region = opaque;
     VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
@@ -1213,858 +1028,9 @@ static const MemoryRegionOps vfio_vga_ops = {
 };
 
 /*
- * Device specific quirks
- */
-
-/* Is range1 fully contained within range2?  */
-static bool vfio_range_contained(uint64_t first1, uint64_t len1,
-                                 uint64_t first2, uint64_t len2) {
-    return (first1 >= first2 && first1 + len1 <= first2 + len2);
-}
-
-static bool vfio_flags_enabled(uint8_t flags, uint8_t mask)
-{
-    return (mask && (flags & mask) == mask);
-}
-
-static uint64_t vfio_generic_window_quirk_read(void *opaque,
-                                               hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    uint64_t data;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
-        ranges_overlap(addr, size,
-                       quirk->data.data_offset, quirk->data.data_size)) {
-        hwaddr offset = addr - quirk->data.data_offset;
-
-        if (!vfio_range_contained(addr, size, quirk->data.data_offset,
-                                  quirk->data.data_size)) {
-            hw_error("%s: window data read not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        data = vfio_pci_read_config(&vdev->pdev,
-                                    quirk->data.address_val + offset, size);
-
-        trace_vfio_generic_window_quirk_read(memory_region_name(&quirk->mem),
-                                             vdev->vbasedev.name,
-                                             quirk->data.bar,
-                                             addr, size, data);
-    } else {
-        data = vfio_region_read(&vdev->bars[quirk->data.bar].region,
-                                addr + quirk->data.base_offset, size);
-    }
-
-    return data;
-}
-
-static void vfio_generic_window_quirk_write(void *opaque, hwaddr addr,
-                                            uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-
-    if (ranges_overlap(addr, size,
-                       quirk->data.address_offset, quirk->data.address_size)) {
-
-        if (addr != quirk->data.address_offset) {
-            hw_error("%s: offset write into address window: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        if ((data & ~quirk->data.address_mask) == quirk->data.address_match) {
-            quirk->data.flags |= quirk->data.write_flags |
-                                 quirk->data.read_flags;
-            quirk->data.address_val = data & quirk->data.address_mask;
-        } else {
-            quirk->data.flags &= ~(quirk->data.write_flags |
-                                   quirk->data.read_flags);
-        }
-    }
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
-        ranges_overlap(addr, size,
-                       quirk->data.data_offset, quirk->data.data_size)) {
-        hwaddr offset = addr - quirk->data.data_offset;
-
-        if (!vfio_range_contained(addr, size, quirk->data.data_offset,
-                                  quirk->data.data_size)) {
-            hw_error("%s: window data write not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        vfio_pci_write_config(&vdev->pdev,
-                              quirk->data.address_val + offset, data, size);
-        trace_vfio_generic_window_quirk_write(memory_region_name(&quirk->mem),
-                                              vdev->vbasedev.name,
-                                              quirk->data.bar,
-                                              addr, data, size);
-        return;
-    }
-
-    vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                   addr + quirk->data.base_offset, data, size);
-}
-
-static const MemoryRegionOps vfio_generic_window_quirk = {
-    .read = vfio_generic_window_quirk_read,
-    .write = vfio_generic_window_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static uint64_t vfio_generic_quirk_read(void *opaque,
-                                        hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-    hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
-    uint64_t data;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
-        ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
-        if (!vfio_range_contained(addr, size, offset,
-                                  quirk->data.address_mask + 1)) {
-            hw_error("%s: read not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        data = vfio_pci_read_config(&vdev->pdev, addr - offset, size);
-
-        trace_vfio_generic_quirk_read(memory_region_name(&quirk->mem),
-                                      vdev->vbasedev.name, quirk->data.bar,
-                                      addr + base, size, data);
-    } else {
-        data = vfio_region_read(&vdev->bars[quirk->data.bar].region,
-                                addr + base, size);
-    }
-
-    return data;
-}
-
-static void vfio_generic_quirk_write(void *opaque, hwaddr addr,
-                                     uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-    hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
-        ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
-        if (!vfio_range_contained(addr, size, offset,
-                                  quirk->data.address_mask + 1)) {
-            hw_error("%s: write not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        vfio_pci_write_config(&vdev->pdev, addr - offset, data, size);
-
-        trace_vfio_generic_quirk_write(memory_region_name(&quirk->mem),
-                                       vdev->vbasedev.name, quirk->data.bar,
-                                       addr + base, data, size);
-    } else {
-        vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                          addr + base, data, size);
-    }
-}
-
-static const MemoryRegionOps vfio_generic_quirk = {
-    .read = vfio_generic_quirk_read,
-    .write = vfio_generic_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-#define PCI_VENDOR_ID_ATI               0x1002
-
-/*
- * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
- * through VGA register 0x3c3.  On newer cards, the I/O port BAR is always
- * BAR4 (older cards like the X550 used BAR1, but we don't care to support
- * those).  Note that on bare metal, a read of 0x3c3 doesn't always return the
- * I/O port BAR address.  Originally this was coded to return the virtual BAR
- * address only if the physical register read returns the actual BAR address,
- * but users have reported greater success if we return the virtual address
- * unconditionally.
- */
-static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
-                                        hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    uint64_t data = vfio_pci_read_config(&vdev->pdev,
-                                         PCI_BASE_ADDRESS_0 + (4 * 4) + 1,
-                                         size);
-    trace_vfio_ati_3c3_quirk_read(data);
-
-    return data;
-}
-
-static const MemoryRegionOps vfio_ati_3c3_quirk = {
-    .read = vfio_ati_3c3_quirk_read,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    /*
-     * As long as the BAR is >= 256 bytes it will be aligned such that the
-     * lower byte is always zero.  Filter out anything else, if it exists.
-     */
-    if (!vdev->bars[4].ioport || vdev->bars[4].region.size < 256) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, quirk,
-                          "vfio-ati-3c3-quirk", 1);
-    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
-                                3 /* offset 3 bytes from 0x3c0 */, &quirk->mem);
-
-    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
-                      quirk, next);
-
-    trace_vfio_vga_probe_ati_3c3_quirk(vdev->vbasedev.name);
-}
-
-/*
- * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
- * config space through MMIO BAR2 at offset 0x4000.  Nothing seems to access
- * the MMIO space directly, but a window to this space is provided through
- * I/O port BAR4.  Offset 0x0 is the address register and offset 0x4 is the
- * data register.  When the address is programmed to a range of 0x4000-0x4fff
- * PCI configuration space is available.  Experimentation seems to indicate
- * that only read-only access is provided, but we drop writes when the window
- * is enabled to config space nonetheless.
- */
-static void vfio_probe_ati_bar4_window_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 4 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.address_size = 4;
-    quirk->data.data_offset = 4;
-    quirk->data.data_size = 4;
-    quirk->data.address_match = 0x4000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-    quirk->data.read_flags = quirk->data.write_flags = 1;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev),
-                          &vfio_generic_window_quirk, quirk,
-                          "vfio-ati-bar4-window-quirk", 8);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.base_offset, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_ati_bar4_window_quirk(vdev->vbasedev.name);
-}
-
-#define PCI_VENDOR_ID_REALTEK 0x10ec
-
-/*
- * RTL8168 devices have a backdoor that can access the MSI-X table.  At BAR2
- * offset 0x70 there is a dword data register, offset 0x74 is a dword address
- * register.  According to the Linux r8169 driver, the MSI-X table is addressed
- * when the "type" portion of the address register is set to 0x1.  This appears
- * to be bits 16:30.  Bit 31 is both a write indicator and some sort of
- * "address latched" indicator.  Bits 12:15 are a mask field, which we can
- * ignore because the MSI-X table should always be accessed as a dword (full
- * mask).  Bits 0:11 is offset within the type.
- *
- * Example trace:
- *
- * Read from MSI-X table offset 0
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
- *
- * Write 0xfee00000 to MSI-X table offset 0
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
- */
-
-static uint64_t vfio_rtl8168_window_quirk_read(void *opaque,
-                                               hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-
-    switch (addr) {
-    case 4: /* address */
-        if (quirk->data.flags) {
-            trace_vfio_rtl8168_window_quirk_read_fake(
-                    memory_region_name(&quirk->mem),
-                    vdev->vbasedev.name);
-
-            return quirk->data.address_match ^ 0x80000000U;
-        }
-        break;
-    case 0: /* data */
-        if (quirk->data.flags) {
-            uint64_t val;
-
-            trace_vfio_rtl8168_window_quirk_read_table(
-                    memory_region_name(&quirk->mem),
-                    vdev->vbasedev.name);
-
-            if (!(vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) {
-                return 0;
-            }
-
-            memory_region_dispatch_read(&vdev->pdev.msix_table_mmio,
-                                        (hwaddr)(quirk->data.address_match
-                                                 & 0xfff),
-                                        &val,
-                                        size,
-                                        MEMTXATTRS_UNSPECIFIED);
-            return val;
-        }
-    }
-
-    trace_vfio_rtl8168_window_quirk_read_direct(memory_region_name(&quirk->mem),
-                                                vdev->vbasedev.name);
-
-    return vfio_region_read(&vdev->bars[quirk->data.bar].region,
-                            addr + 0x70, size);
-}
-
-static void vfio_rtl8168_window_quirk_write(void *opaque, hwaddr addr,
-                                            uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-
-    switch (addr) {
-    case 4: /* address */
-        if ((data & 0x7fff0000) == 0x10000) {
-            if (data & 0x80000000U &&
-                vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) {
-
-                trace_vfio_rtl8168_window_quirk_write_table(
-                        memory_region_name(&quirk->mem),
-                        vdev->vbasedev.name);
-
-                memory_region_dispatch_write(&vdev->pdev.msix_table_mmio,
-                                             (hwaddr)(data & 0xfff),
-                                             (uint64_t)quirk->data.address_mask,
-                                             size, MEMTXATTRS_UNSPECIFIED);
-            }
-
-            quirk->data.flags = 1;
-            quirk->data.address_match = data;
-
-            return;
-        }
-        quirk->data.flags = 0;
-        break;
-    case 0: /* data */
-        quirk->data.address_mask = data;
-        break;
-    }
-
-    trace_vfio_rtl8168_window_quirk_write_direct(
-            memory_region_name(&quirk->mem),
-            vdev->vbasedev.name);
-
-    vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                      addr + 0x70, data, size);
-}
-
-static const MemoryRegionOps vfio_rtl8168_window_quirk = {
-    .read = vfio_rtl8168_window_quirk_read,
-    .write = vfio_rtl8168_window_quirk_write,
-    .valid = {
-        .min_access_size = 4,
-        .max_access_size = 4,
-        .unaligned = false,
-    },
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_probe_rtl8168_bar2_window_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_REALTEK ||
-        pci_get_word(pdev->config + PCI_DEVICE_ID) != 0x8168 || nr != 2) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_rtl8168_window_quirk,
-                          quirk, "vfio-rtl8168-window-quirk", 8);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                                        0x70, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_rtl8168_bar2_window_quirk(vdev->vbasedev.name);
-}
-/*
- * Trap the BAR2 MMIO window to config space as well.
- */
-static void vfio_probe_ati_bar2_4000_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    /* Only enable on newer devices where BAR2 is 64bit */
-    if (!vdev->has_vga || nr != 2 || !vdev->bars[2].mem64 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x4000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
-                          "vfio-ati-bar2-4000-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_ati_bar2_4000_quirk(vdev->vbasedev.name);
-}
-
-/*
- * Older ATI/AMD cards like the X550 have a similar window to that above.
- * I/O port BAR1 provides a window to a mirror of PCI config space located
- * in BAR2 at offset 0xf00.  We don't care to support such older cards, but
- * note it for future reference.
- */
-
-#define PCI_VENDOR_ID_NVIDIA                    0x10de
-
-/*
- * Nvidia has several different methods to get to config space, the
- * nouveu project has several of these documented here:
- * https://github.com/pathscale/envytools/tree/master/hwdocs
- *
- * The first quirk is actually not documented in envytools and is found
- * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]).  This is an
- * NV46 chipset.  The backdoor uses the legacy VGA I/O ports to access
- * the mirror of PCI config space found at BAR0 offset 0x1800.  The access
- * sequence first writes 0x338 to I/O port 0x3d4.  The target offset is
- * then written to 0x3d0.  Finally 0x538 is written for a read and 0x738
- * is written for a write to 0x3d4.  The BAR0 offset is then accessible
- * through 0x3d0.  This quirk doesn't seem to be necessary on newer cards
- * that use the I/O port BAR5 window but it doesn't hurt to leave it.
- */
-enum {
-    NV_3D0_NONE = 0,
-    NV_3D0_SELECT,
-    NV_3D0_WINDOW,
-    NV_3D0_READ,
-    NV_3D0_WRITE,
-};
-
-static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
-                                           hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-    uint64_t data = vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
-                                  addr + quirk->data.base_offset, size);
-
-    if (quirk->data.flags == NV_3D0_READ && addr == quirk->data.data_offset) {
-        data = vfio_pci_read_config(pdev, quirk->data.address_val, size);
-        trace_vfio_nvidia_3d0_quirk_read(size, data);
-    }
-
-    quirk->data.flags = NV_3D0_NONE;
-
-    return data;
-}
-
-static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
-                                        uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-
-    switch (quirk->data.flags) {
-    case NV_3D0_NONE:
-        if (addr == quirk->data.address_offset && data == 0x338) {
-            quirk->data.flags = NV_3D0_SELECT;
-        }
-        break;
-    case NV_3D0_SELECT:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.data_offset &&
-            (data & ~quirk->data.address_mask) == quirk->data.address_match) {
-            quirk->data.flags = NV_3D0_WINDOW;
-            quirk->data.address_val = data & quirk->data.address_mask;
-        }
-        break;
-    case NV_3D0_WINDOW:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.address_offset) {
-            if (data == 0x538) {
-                quirk->data.flags = NV_3D0_READ;
-            } else if (data == 0x738) {
-                quirk->data.flags = NV_3D0_WRITE;
-            }
-        }
-        break;
-    case NV_3D0_WRITE:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.data_offset) {
-            vfio_pci_write_config(pdev, quirk->data.address_val, data, size);
-            trace_vfio_nvidia_3d0_quirk_write(data, size);
-            return;
-        }
-        break;
-    }
-
-    vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
-                   addr + quirk->data.base_offset, data, size);
-}
-
-static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
-    .read = vfio_nvidia_3d0_quirk_read,
-    .write = vfio_nvidia_3d0_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA ||
-        !vdev->bars[1].region.size) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.base_offset = 0x10;
-    quirk->data.address_offset = 4;
-    quirk->data.address_size = 2;
-    quirk->data.address_match = 0x1800;
-    quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
-    quirk->data.data_offset = 0;
-    quirk->data.data_size = 4;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_3d0_quirk,
-                          quirk, "vfio-nvidia-3d0-quirk", 6);
-    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
-                                quirk->data.base_offset, &quirk->mem);
-
-    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
-                      quirk, next);
-
-    trace_vfio_vga_probe_nvidia_3d0_quirk(vdev->vbasedev.name);
-}
-
-/*
- * The second quirk is documented in envytools.  The I/O port BAR5 is just
- * a set of address/data ports to the MMIO BARs.  The BAR we care about is
- * again BAR0.  This backdoor is apparently a bit newer than the one above
- * so we need to not only trap 256 bytes @0x1800, but all of PCI config
- * space, including extended space is available at the 4k @0x88000.
- */
-enum {
-    NV_BAR5_ADDRESS = 0x1,
-    NV_BAR5_ENABLE = 0x2,
-    NV_BAR5_MASTER = 0x4,
-    NV_BAR5_VALID = 0x7,
-};
-
-static void vfio_nvidia_bar5_window_quirk_write(void *opaque, hwaddr addr,
-                                                uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-
-    switch (addr) {
-    case 0x0:
-        if (data & 0x1) {
-            quirk->data.flags |= NV_BAR5_MASTER;
-        } else {
-            quirk->data.flags &= ~NV_BAR5_MASTER;
-        }
-        break;
-    case 0x4:
-        if (data & 0x1) {
-            quirk->data.flags |= NV_BAR5_ENABLE;
-        } else {
-            quirk->data.flags &= ~NV_BAR5_ENABLE;
-        }
-        break;
-    case 0x8:
-        if (quirk->data.flags & NV_BAR5_MASTER) {
-            if ((data & ~0xfff) == 0x88000) {
-                quirk->data.flags |= NV_BAR5_ADDRESS;
-                quirk->data.address_val = data & 0xfff;
-            } else if ((data & ~0xff) == 0x1800) {
-                quirk->data.flags |= NV_BAR5_ADDRESS;
-                quirk->data.address_val = data & 0xff;
-            } else {
-                quirk->data.flags &= ~NV_BAR5_ADDRESS;
-            }
-        }
-        break;
-    }
-
-    vfio_generic_window_quirk_write(opaque, addr, data, size);
-}
-
-static const MemoryRegionOps vfio_nvidia_bar5_window_quirk = {
-    .read = vfio_generic_window_quirk_read,
-    .write = vfio_nvidia_bar5_window_quirk_write,
-    .valid.min_access_size = 4,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_probe_nvidia_bar5_window_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 5 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.read_flags = quirk->data.write_flags = NV_BAR5_VALID;
-    quirk->data.address_offset = 0x8;
-    quirk->data.address_size = 0; /* actually 4, but avoids generic code */
-    quirk->data.data_offset = 0xc;
-    quirk->data.data_size = 4;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev),
-                          &vfio_nvidia_bar5_window_quirk, quirk,
-                          "vfio-nvidia-bar5-window-quirk", 16);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                                        0, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_nvidia_bar5_window_quirk(vdev->vbasedev.name);
-}
-
-static void vfio_nvidia_88000_quirk_write(void *opaque, hwaddr addr,
-                                          uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-
-    vfio_generic_quirk_write(opaque, addr, data, size);
-
-    /*
-     * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
-     * MSI capability ID register.  Both the ID and next register are
-     * read-only, so we allow writes covering either of those to real hw.
-     * NB - only fixed for the 0x88000 MMIO window.
-     */
-    if ((pdev->cap_present & QEMU_PCI_CAP_MSI) &&
-        vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) {
-        vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                          addr + base, data, size);
-    }
-}
-
-static const MemoryRegionOps vfio_nvidia_88000_quirk = {
-    .read = vfio_generic_quirk_read,
-    .write = vfio_nvidia_88000_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-/*
- * Finally, BAR0 itself.  We want to redirect any accesses to either
- * 0x1800 or 0x88000 through the PCI config space access functions.
- *
- * NB - quirk at a page granularity or else they don't seem to work when
- *      BARs are mmap'd
- *
- * Here's offset 0x88000...
- */
-static void vfio_probe_nvidia_bar0_88000_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-    uint16_t vendor, class;
-
-    vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
-
-    if (nr != 0 || vendor != PCI_VENDOR_ID_NVIDIA ||
-        class != PCI_CLASS_DISPLAY_VGA) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x88000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_88000_quirk,
-                          quirk, "vfio-nvidia-bar0-88000-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_nvidia_bar0_88000_quirk(vdev->vbasedev.name);
-}
-
-/*
- * And here's the same for BAR0 offset 0x1800...
- */
-static void vfio_probe_nvidia_bar0_1800_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 0 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
-        return;
-    }
-
-    /* Log the chipset ID */
-    trace_vfio_probe_nvidia_bar0_1800_quirk_id(
-            (unsigned int)(vfio_region_read(&vdev->bars[0].region, 0, 4) >> 20)
-            & 0xff);
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x1800;
-    quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
-                          "vfio-nvidia-bar0-1800-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_nvidia_bar0_1800_quirk(vdev->vbasedev.name);
-}
-
-/*
- * TODO - Some Nvidia devices provide config access to their companion HDA
- * device and even to their parent bridge via these config space mirrors.
- * Add quirks for those regions.
- */
-
-/*
- * Common quirk probe entry points.
- */
-static void vfio_vga_quirk_setup(VFIOPCIDevice *vdev)
-{
-    vfio_vga_probe_ati_3c3_quirk(vdev);
-    vfio_vga_probe_nvidia_3d0_quirk(vdev);
-}
-
-static void vfio_vga_quirk_teardown(VFIOPCIDevice *vdev)
-{
-    VFIOQuirk *quirk;
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
-        QLIST_FOREACH(quirk, &vdev->vga.region[i].quirks, next) {
-            memory_region_del_subregion(&vdev->vga.region[i].mem, &quirk->mem);
-        }
-    }
-}
-
-static void vfio_vga_quirk_free(VFIOPCIDevice *vdev)
-{
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
-        while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) {
-            VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks);
-            object_unparent(OBJECT(&quirk->mem));
-            QLIST_REMOVE(quirk, next);
-            g_free(quirk);
-        }
-    }
-}
-
-static void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr)
-{
-    vfio_probe_ati_bar4_window_quirk(vdev, nr);
-    vfio_probe_ati_bar2_4000_quirk(vdev, nr);
-    vfio_probe_nvidia_bar5_window_quirk(vdev, nr);
-    vfio_probe_nvidia_bar0_88000_quirk(vdev, nr);
-    vfio_probe_nvidia_bar0_1800_quirk(vdev, nr);
-    vfio_probe_rtl8168_bar2_window_quirk(vdev, nr);
-}
-
-static void vfio_bar_quirk_teardown(VFIOPCIDevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-    VFIOQuirk *quirk;
-
-    QLIST_FOREACH(quirk, &bar->quirks, next) {
-        memory_region_del_subregion(&bar->region.mem, &quirk->mem);
-    }
-}
-
-static void vfio_bar_quirk_free(VFIOPCIDevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-
-    while (!QLIST_EMPTY(&bar->quirks)) {
-        VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
-        object_unparent(OBJECT(&quirk->mem));
-        QLIST_REMOVE(quirk, next);
-        g_free(quirk);
-    }
-}
-
-/*
  * PCI config space
  */
-static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
+uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
     uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
@@ -2097,8 +1063,8 @@ static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
     return val;
 }
 
-static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
-                                  uint32_t val, int len)
+void vfio_pci_write_config(PCIDevice *pdev,
+                           uint32_t addr, uint32_t val, int len)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
     uint32_t val_le = cpu_to_le32(val);
@@ -2124,11 +1090,11 @@ static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
 
         if (!was_enabled) {
             if (is_enabled) {
-                vfio_enable_msi(vdev);
+                vfio_msi_enable(vdev);
             }
         } else {
             if (!is_enabled) {
-                vfio_disable_msi(vdev);
+                vfio_msi_disable(vdev);
             } else {
                 vfio_update_msi(vdev);
             }
@@ -2142,9 +1108,9 @@ static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
         is_enabled = msix_enabled(pdev);
 
         if (!was_enabled && is_enabled) {
-            vfio_enable_msix(vdev);
+            vfio_msix_enable(vdev);
         } else if (was_enabled && !is_enabled) {
-            vfio_disable_msix(vdev);
+            vfio_msix_disable(vdev);
         }
     } else {
         /* Write everything to QEMU to keep emulated bits correct */
@@ -2163,17 +1129,17 @@ static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
      * disable MSI/X and then cleanup by disabling INTx.
      */
     if (vdev->interrupt == VFIO_INT_MSIX) {
-        vfio_disable_msix(vdev);
+        vfio_msix_disable(vdev);
     } else if (vdev->interrupt == VFIO_INT_MSI) {
-        vfio_disable_msi(vdev);
+        vfio_msi_disable(vdev);
     }
 
     if (vdev->interrupt == VFIO_INT_INTx) {
-        vfio_disable_intx(vdev);
+        vfio_intx_disable(vdev);
     }
 }
 
-static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos)
+static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos)
 {
     uint16_t ctrl;
     bool msi_64bit, msi_maskbit;
@@ -2189,7 +1155,7 @@ static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos)
     msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
     entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
 
-    trace_vfio_setup_msi(vdev->vbasedev.name, pos);
+    trace_vfio_msi_setup(vdev->vbasedev.name, pos);
 
     ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
     if (ret < 0) {
@@ -2212,12 +1178,13 @@ static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos)
  * need to first look for where the MSI-X table lives.  So we
  * unfortunately split MSI-X setup across two functions.
  */
-static int vfio_early_setup_msix(VFIOPCIDevice *vdev)
+static int vfio_msix_early_setup(VFIOPCIDevice *vdev)
 {
     uint8_t pos;
     uint16_t ctrl;
     uint32_t table, pba;
     int fd = vdev->vbasedev.fd;
+    VFIOMSIXInfo *msix;
 
     pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
     if (!pos) {
@@ -2243,49 +1210,44 @@ static int vfio_early_setup_msix(VFIOPCIDevice *vdev)
     table = le32_to_cpu(table);
     pba = le32_to_cpu(pba);
 
-    vdev->msix = g_malloc0(sizeof(*(vdev->msix)));
-    vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
+    msix = g_malloc0(sizeof(*msix));
+    msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
+    msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
+    msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
+    msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
+    msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
 
     /*
      * Test the size of the pba_offset variable and catch if it extends outside
      * of the specified BAR. If it is the case, we need to apply a hardware
      * specific quirk if the device is known or we have a broken configuration.
      */
-    if (vdev->msix->pba_offset >=
-        vdev->bars[vdev->msix->pba_bar].region.size) {
-
-        PCIDevice *pdev = &vdev->pdev;
-        uint16_t vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
-        uint16_t device = pci_get_word(pdev->config + PCI_DEVICE_ID);
-
+    if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
         /*
          * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
          * adapters. The T5 hardware returns an incorrect value of 0x8000 for
          * the VF PBA offset while the BAR itself is only 8k. The correct value
          * is 0x1000, so we hard code that here.
          */
-        if (vendor == PCI_VENDOR_ID_CHELSIO && (device & 0xff00) == 0x5800) {
-            vdev->msix->pba_offset = 0x1000;
+        if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
+            (vdev->device_id & 0xff00) == 0x5800) {
+            msix->pba_offset = 0x1000;
         } else {
             error_report("vfio: Hardware reports invalid configuration, "
                          "MSIX PBA outside of specified BAR");
+            g_free(msix);
             return -EINVAL;
         }
     }
 
-    trace_vfio_early_setup_msix(vdev->vbasedev.name, pos,
-                                vdev->msix->table_bar,
-                                vdev->msix->table_offset,
-                                vdev->msix->entries);
+    trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
+                                msix->table_offset, msix->entries);
+    vdev->msix = msix;
 
     return 0;
 }
 
-static int vfio_setup_msix(VFIOPCIDevice *vdev, int pos)
+static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos)
 {
     int ret;
 
@@ -2563,13 +1525,38 @@ static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size)
     }
 
     if (!pci_bus_is_express(vdev->pdev.bus)) {
+        PCIBus *bus = vdev->pdev.bus;
+        PCIDevice *bridge;
+
         /*
-         * Use express capability as-is on PCI bus.  It doesn't make much
-         * sense to even expose, but some drivers (ex. tg3) depend on it
-         * and guests don't seem to be particular about it.  We'll need
-         * to revist this or force express devices to express buses if we
-         * ever expose an IOMMU to the guest.
+         * Traditionally PCI device assignment exposes the PCIe capability
+         * as-is on non-express buses.  The reason being that some drivers
+         * simply assume that it's there, for example tg3.  However when
+         * we're running on a native PCIe machine type, like Q35, we need
+         * to hide the PCIe capability.  The reason for this is twofold;
+         * first Windows guests get a Code 10 error when the PCIe capability
+         * is exposed in this configuration.  Therefore express devices won't
+         * work at all unless they're attached to express buses in the VM.
+         * Second, a native PCIe machine introduces the possibility of fine
+         * granularity IOMMUs supporting both translation and isolation.
+         * Guest code to discover the IOMMU visibility of a device, such as
+         * IOMMU grouping code on Linux, is very aware of device types and
+         * valid transitions between bus types.  An express device on a non-
+         * express bus is not a valid combination on bare metal systems.
+         *
+         * Drivers that require a PCIe capability to make the device
+         * functional are simply going to need to have their devices placed
+         * on a PCIe bus in the VM.
          */
+        while (!pci_bus_is_root(bus)) {
+            bridge = pci_bridge_get_device(bus);
+            bus = bridge->bus;
+        }
+
+        if (pci_bus_is_express(bus)) {
+            return 0;
+        }
+
     } else if (pci_bus_is_root(vdev->pdev.bus)) {
         /*
          * On a Root Complex bus Endpoints become Root Complex Integrated
@@ -2708,14 +1695,14 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
 
     switch (cap_id) {
     case PCI_CAP_ID_MSI:
-        ret = vfio_setup_msi(vdev, pos);
+        ret = vfio_msi_setup(vdev, pos);
         break;
     case PCI_CAP_ID_EXP:
         vfio_check_pcie_flr(vdev, pos);
         ret = vfio_setup_pcie_cap(vdev, pos, size);
         break;
     case PCI_CAP_ID_MSIX:
-        ret = vfio_setup_msix(vdev, pos);
+        ret = vfio_msix_setup(vdev, pos);
         break;
     case PCI_CAP_ID_PM:
         vfio_check_pm_reset(vdev, pos);
@@ -2793,7 +1780,7 @@ static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
 
 static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
 {
-    vfio_enable_intx(vdev);
+    vfio_intx_enable(vdev);
 }
 
 static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
@@ -3017,7 +2004,7 @@ static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
 static VFIODeviceOps vfio_pci_ops = {
     .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
     .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
-    .vfio_eoi = vfio_eoi,
+    .vfio_eoi = vfio_intx_eoi,
 };
 
 static int vfio_populate_device(VFIOPCIDevice *vdev)
@@ -3352,162 +2339,6 @@ static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
     vdev->req_enabled = false;
 }
 
-/*
- * AMD Radeon PCI config reset, based on Linux:
- *   drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running()
- *   drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset
- *   drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc()
- *   drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock()
- * IDs: include/drm/drm_pciids.h
- * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0
- *
- * Bonaire and Hawaii GPUs do not respond to a bus reset.  This is a bug in the
- * hardware that should be fixed on future ASICs.  The symptom of this is that
- * once the accerlated driver loads, Windows guests will bsod on subsequent
- * attmpts to load the driver, such as after VM reset or shutdown/restart.  To
- * work around this, we do an AMD specific PCI config reset, followed by an SMC
- * reset.  The PCI config reset only works if SMC firmware is running, so we
- * have a dependency on the state of the device as to whether this reset will
- * be effective.  There are still cases where we won't be able to kick the
- * device into working, but this greatly improves the usability overall.  The
- * config reset magic is relatively common on AMD GPUs, but the setup and SMC
- * poking is largely ASIC specific.
- */
-static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev)
-{
-    uint32_t clk, pc_c;
-
-    /*
-     * Registers 200h and 204h are index and data registers for acessing
-     * indirect configuration registers within the device.
-     */
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
-    clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4);
-    pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-
-    return (!(clk & 1) && (0x20100 <= pc_c));
-}
-
-/*
- * The scope of a config reset is controlled by a mode bit in the misc register
- * and a fuse, exposed as a bit in another register.  The fuse is the default
- * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula
- * scope = !(misc ^ fuse), where the resulting scope is defined the same as
- * the fuse.  A truth table therefore tells us that if misc == fuse, we need
- * to flip the value of the bit in the misc register.
- */
-static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev)
-{
-    uint32_t misc, fuse;
-    bool a, b;
-
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4);
-    fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    b = fuse & 64;
-
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4);
-    misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    a = misc & 2;
-
-    if (a == b) {
-        vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4);
-        vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */
-    }
-}
-
-static int vfio_radeon_reset(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    int i, ret = 0;
-    uint32_t data;
-
-    /* Defer to a kernel implemented reset */
-    if (vdev->vbasedev.reset_works) {
-        return -ENODEV;
-    }
-
-    /* Enable only memory BAR access */
-    vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2);
-
-    /* Reset only works if SMC firmware is loaded and running */
-    if (!vfio_radeon_smc_is_running(vdev)) {
-        ret = -EINVAL;
-        goto out;
-    }
-
-    /* Make sure only the GFX function is reset */
-    vfio_radeon_set_gfx_only_reset(vdev);
-
-    /* AMD PCI config reset */
-    vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4);
-    usleep(100);
-
-    /* Read back the memory size to make sure we're out of reset */
-    for (i = 0; i < 100000; i++) {
-        if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) {
-            break;
-        }
-        usleep(1);
-    }
-
-    /* Reset SMC */
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4);
-    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    data |= 1;
-    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
-
-    /* Disable SMC clock */
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
-    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    data |= 1;
-    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
-
-out:
-    /* Restore PCI command register */
-    vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2);
-
-    return ret;
-}
-
-static void vfio_setup_resetfn(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    uint16_t vendor, device;
-
-    vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    device = pci_get_word(pdev->config + PCI_DEVICE_ID);
-
-    switch (vendor) {
-    case 0x1002:
-        switch (device) {
-        /* Bonaire */
-        case 0x6649: /* Bonaire [FirePro W5100] */
-        case 0x6650:
-        case 0x6651:
-        case 0x6658: /* Bonaire XTX [Radeon R7 260X] */
-        case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */
-        case 0x665d: /* Bonaire [Radeon R7 200 Series] */
-        /* Hawaii */
-        case 0x67A0: /* Hawaii XT GL [FirePro W9100] */
-        case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */
-        case 0x67A2:
-        case 0x67A8:
-        case 0x67A9:
-        case 0x67AA:
-        case 0x67B0: /* Hawaii XT [Radeon R9 290X] */
-        case 0x67B1: /* Hawaii PRO [Radeon R9 290] */
-        case 0x67B8:
-        case 0x67B9:
-        case 0x67BA:
-        case 0x67BE:
-            vdev->resetfn = vfio_radeon_reset;
-            break;
-        }
-        break;
-    }
-}
-
 static int vfio_initfn(PCIDevice *pdev)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
@@ -3600,6 +2431,54 @@ static int vfio_initfn(PCIDevice *pdev)
     /* QEMU can choose to expose the ROM or not */
     memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
 
+    /*
+     * The PCI spec reserves vendor ID 0xffff as an invalid value.  The
+     * device ID is managed by the vendor and need only be a 16-bit value.
+     * Allow any 16-bit value for subsystem so they can be hidden or changed.
+     */
+    if (vdev->vendor_id != PCI_ANY_ID) {
+        if (vdev->vendor_id >= 0xffff) {
+            error_report("vfio: Invalid PCI vendor ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
+        trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
+    } else {
+        vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
+    }
+
+    if (vdev->device_id != PCI_ANY_ID) {
+        if (vdev->device_id > 0xffff) {
+            error_report("vfio: Invalid PCI device ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
+        trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
+    } else {
+        vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
+    }
+
+    if (vdev->sub_vendor_id != PCI_ANY_ID) {
+        if (vdev->sub_vendor_id > 0xffff) {
+            error_report("vfio: Invalid PCI subsystem vendor ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
+                               vdev->sub_vendor_id, ~0);
+        trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
+                                              vdev->sub_vendor_id);
+    }
+
+    if (vdev->sub_device_id != PCI_ANY_ID) {
+        if (vdev->sub_device_id > 0xffff) {
+            error_report("vfio: Invalid PCI subsystem device ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
+        trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
+                                              vdev->sub_device_id);
+    }
+
     /* QEMU can change multi-function devices to single function, or reverse */
     vdev->emulated_config_bits[PCI_HEADER_TYPE] =
                                               PCI_HEADER_TYPE_MULTI_FUNCTION;
@@ -3621,7 +2500,7 @@ static int vfio_initfn(PCIDevice *pdev)
 
     vfio_pci_size_rom(vdev);
 
-    ret = vfio_early_setup_msix(vdev);
+    ret = vfio_msix_early_setup(vdev);
     if (ret) {
         return ret;
     }
@@ -3647,8 +2526,8 @@ static int vfio_initfn(PCIDevice *pdev)
     if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
         vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
                                                   vfio_intx_mmap_enable, vdev);
-        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
-        ret = vfio_enable_intx(vdev);
+        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_intx_update);
+        ret = vfio_intx_enable(vdev);
         if (ret) {
             goto out_teardown;
         }
@@ -3656,7 +2535,7 @@ static int vfio_initfn(PCIDevice *pdev)
 
     vfio_register_err_notifier(vdev);
     vfio_register_req_notifier(vdev);
-    vfio_setup_resetfn(vdev);
+    vfio_setup_resetfn_quirk(vdev);
 
     return 0;
 
@@ -3749,7 +2628,16 @@ static Property vfio_pci_dev_properties[] = {
                     VFIO_FEATURE_ENABLE_VGA_BIT, false),
     DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
                     VFIO_FEATURE_ENABLE_REQ_BIT, true),
-    DEFINE_PROP_BOOL("x-mmap", VFIOPCIDevice, vbasedev.allow_mmap, true),
+    DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
+    DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
+    DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
+    DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
+    DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
+                       sub_vendor_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
+                       sub_device_id, PCI_ANY_ID),
     /*
      * TODO - support passed fds... is this necessary?
      * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
new file mode 100644
index 000000000..f004d52b6
--- /dev/null
+++ b/hw/vfio/pci.h
@@ -0,0 +1,159 @@
+/*
+ * vfio based device assignment support - PCI devices
+ *
+ * Copyright Red Hat, Inc. 2012-2015
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+#ifndef HW_VFIO_VFIO_PCI_H
+#define HW_VFIO_VFIO_PCI_H
+
+#include "qemu-common.h"
+#include "exec/memory.h"
+#include "hw/pci/pci.h"
+#include "hw/vfio/vfio-common.h"
+#include "qemu/event_notifier.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+
+#define PCI_ANY_ID (~0)
+
+struct VFIOPCIDevice;
+
+typedef struct VFIOQuirk {
+    QLIST_ENTRY(VFIOQuirk) next;
+    void *data;
+    int nr_mem;
+    MemoryRegion *mem;
+} VFIOQuirk;
+
+typedef struct VFIOBAR {
+    VFIORegion region;
+    bool ioport;
+    bool mem64;
+    QLIST_HEAD(, VFIOQuirk) quirks;
+} VFIOBAR;
+
+typedef struct VFIOVGARegion {
+    MemoryRegion mem;
+    off_t offset;
+    int nr;
+    QLIST_HEAD(, VFIOQuirk) quirks;
+} VFIOVGARegion;
+
+typedef struct VFIOVGA {
+    off_t fd_offset;
+    int fd;
+    VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
+} VFIOVGA;
+
+typedef struct VFIOINTx {
+    bool pending; /* interrupt pending */
+    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
+    uint8_t pin; /* which pin to pull for qemu_set_irq */
+    EventNotifier interrupt; /* eventfd triggered on interrupt */
+    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+    PCIINTxRoute route; /* routing info for QEMU bypass */
+    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
+    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
+} VFIOINTx;
+
+typedef struct VFIOMSIVector {
+    /*
+     * Two interrupt paths are configured per vector.  The first, is only used
+     * for interrupts injected via QEMU.  This is typically the non-accel path,
+     * but may also be used when we want QEMU to handle masking and pending
+     * bits.  The KVM path bypasses QEMU and is therefore higher performance,
+     * but requires masking at the device.  virq is used to track the MSI route
+     * through KVM, thus kvm_interrupt is only available when virq is set to a
+     * valid (>= 0) value.
+     */
+    EventNotifier interrupt;
+    EventNotifier kvm_interrupt;
+    struct VFIOPCIDevice *vdev; /* back pointer to device */
+    int virq;
+    bool use;
+} VFIOMSIVector;
+
+enum {
+    VFIO_INT_NONE = 0,
+    VFIO_INT_INTx = 1,
+    VFIO_INT_MSI  = 2,
+    VFIO_INT_MSIX = 3,
+};
+
+/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
+typedef struct VFIOMSIXInfo {
+    uint8_t table_bar;
+    uint8_t pba_bar;
+    uint16_t entries;
+    uint32_t table_offset;
+    uint32_t pba_offset;
+    MemoryRegion mmap_mem;
+    void *mmap;
+} VFIOMSIXInfo;
+
+typedef struct VFIOPCIDevice {
+    PCIDevice pdev;
+    VFIODevice vbasedev;
+    VFIOINTx intx;
+    unsigned int config_size;
+    uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
+    off_t config_offset; /* Offset of config space region within device fd */
+    unsigned int rom_size;
+    off_t rom_offset; /* Offset of ROM region within device fd */
+    void *rom;
+    int msi_cap_size;
+    VFIOMSIVector *msi_vectors;
+    VFIOMSIXInfo *msix;
+    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
+    int interrupt; /* Current interrupt type */
+    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
+    VFIOVGA vga; /* 0xa0000, 0x3b0, 0x3c0 */
+    PCIHostDeviceAddress host;
+    EventNotifier err_notifier;
+    EventNotifier req_notifier;
+    int (*resetfn)(struct VFIOPCIDevice *);
+    uint32_t vendor_id;
+    uint32_t device_id;
+    uint32_t sub_vendor_id;
+    uint32_t sub_device_id;
+    uint32_t features;
+#define VFIO_FEATURE_ENABLE_VGA_BIT 0
+#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
+#define VFIO_FEATURE_ENABLE_REQ_BIT 1
+#define VFIO_FEATURE_ENABLE_REQ (1 << VFIO_FEATURE_ENABLE_REQ_BIT)
+    int32_t bootindex;
+    uint8_t pm_cap;
+    bool has_vga;
+    bool pci_aer;
+    bool req_enabled;
+    bool has_flr;
+    bool has_pm_reset;
+    bool rom_read_failed;
+    bool no_kvm_intx;
+    bool no_kvm_msi;
+    bool no_kvm_msix;
+} VFIOPCIDevice;
+
+uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
+void vfio_pci_write_config(PCIDevice *pdev,
+                           uint32_t addr, uint32_t val, int len);
+
+uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size);
+void vfio_vga_write(void *opaque, hwaddr addr, uint64_t data, unsigned size);
+
+bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_setup(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_teardown(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_free(VFIOPCIDevice *vdev);
+void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr);
+void vfio_bar_quirk_teardown(VFIOPCIDevice *vdev, int nr);
+void vfio_bar_quirk_free(VFIOPCIDevice *vdev, int nr);
+void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev);
+
+#endif /* HW_VFIO_VFIO_PCI_H */
diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c
index 60365d127..289b498ca 100644
--- a/hw/vfio/platform.c
+++ b/hw/vfio/platform.c
@@ -32,6 +32,11 @@
  * Functions used whatever the injection method
  */
 
+static inline bool vfio_irq_is_automasked(VFIOINTp *intp)
+{
+    return intp->flags & VFIO_IRQ_INFO_AUTOMASKED;
+}
+
 /**
  * vfio_init_intp - allocate, initialize the IRQ struct pointer
  * and add it into the list of IRQs
@@ -57,18 +62,25 @@ static VFIOINTp *vfio_init_intp(VFIODevice *vbasedev,
     sysbus_init_irq(sbdev, &intp->qemuirq);
 
     /* Get an eventfd for trigger */
-    ret = event_notifier_init(&intp->interrupt, 0);
+    intp->interrupt = g_malloc0(sizeof(EventNotifier));
+    ret = event_notifier_init(intp->interrupt, 0);
     if (ret) {
+        g_free(intp->interrupt);
         g_free(intp);
         error_report("vfio: Error: trigger event_notifier_init failed ");
         return NULL;
     }
-    /* Get an eventfd for resample/unmask */
-    ret = event_notifier_init(&intp->unmask, 0);
-    if (ret) {
-        g_free(intp);
-        error_report("vfio: Error: resamplefd event_notifier_init failed");
-        return NULL;
+    if (vfio_irq_is_automasked(intp)) {
+        /* Get an eventfd for resample/unmask */
+        intp->unmask = g_malloc0(sizeof(EventNotifier));
+        ret = event_notifier_init(intp->unmask, 0);
+        if (ret) {
+            g_free(intp->interrupt);
+            g_free(intp->unmask);
+            g_free(intp);
+            error_report("vfio: Error: resamplefd event_notifier_init failed");
+            return NULL;
+        }
     }
 
     QLIST_INSERT_HEAD(&vdev->intp_list, intp, next);
@@ -100,7 +112,7 @@ static int vfio_set_trigger_eventfd(VFIOINTp *intp,
     irq_set->start = 0;
     irq_set->count = 1;
     pfd = (int32_t *)&irq_set->data;
-    *pfd = event_notifier_get_fd(&intp->interrupt);
+    *pfd = event_notifier_get_fd(intp->interrupt);
     qemu_set_fd_handler(*pfd, (IOHandler *)handler, NULL, intp);
     ret = ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     g_free(irq_set);
@@ -182,7 +194,7 @@ static void vfio_intp_mmap_enable(void *opaque)
 static void vfio_intp_inject_pending_lockheld(VFIOINTp *intp)
 {
     trace_vfio_platform_intp_inject_pending_lockheld(intp->pin,
-                              event_notifier_get_fd(&intp->interrupt));
+                              event_notifier_get_fd(intp->interrupt));
 
     intp->state = VFIO_IRQ_ACTIVE;
 
@@ -224,18 +236,18 @@ static void vfio_intp_interrupt(VFIOINTp *intp)
         trace_vfio_intp_interrupt_set_pending(intp->pin);
         QSIMPLEQ_INSERT_TAIL(&vdev->pending_intp_queue,
                              intp, pqnext);
-        ret = event_notifier_test_and_clear(&intp->interrupt);
+        ret = event_notifier_test_and_clear(intp->interrupt);
         qemu_mutex_unlock(&vdev->intp_mutex);
         return;
     }
 
     trace_vfio_platform_intp_interrupt(intp->pin,
-                              event_notifier_get_fd(&intp->interrupt));
+                              event_notifier_get_fd(intp->interrupt));
 
-    ret = event_notifier_test_and_clear(&intp->interrupt);
+    ret = event_notifier_test_and_clear(intp->interrupt);
     if (!ret) {
-        error_report("Error when clearing fd=%d (ret = %d)\n",
-                     event_notifier_get_fd(&intp->interrupt), ret);
+        error_report("Error when clearing fd=%d (ret = %d)",
+                     event_notifier_get_fd(intp->interrupt), ret);
     }
 
     intp->state = VFIO_IRQ_ACTIVE;
@@ -283,13 +295,13 @@ static void vfio_platform_eoi(VFIODevice *vbasedev)
     QLIST_FOREACH(intp, &vdev->intp_list, next) {
         if (intp->state == VFIO_IRQ_ACTIVE) {
             trace_vfio_platform_eoi(intp->pin,
-                                event_notifier_get_fd(&intp->interrupt));
+                                event_notifier_get_fd(intp->interrupt));
             intp->state = VFIO_IRQ_INACTIVE;
 
             /* deassert the virtual IRQ */
             qemu_set_irq(intp->qemuirq, 0);
 
-            if (intp->flags & VFIO_IRQ_INFO_AUTOMASKED) {
+            if (vfio_irq_is_automasked(intp)) {
                 /* unmasks the physical level-sensitive IRQ */
                 vfio_unmask_single_irqindex(vbasedev, intp->pin);
             }
@@ -310,18 +322,29 @@ static void vfio_platform_eoi(VFIODevice *vbasedev)
 /**
  * vfio_start_eventfd_injection - starts the virtual IRQ injection using
  * user-side handled eventfds
- * @intp: the IRQ struct pointer
+ * @sbdev: the sysbus device handle
+ * @irq: the qemu irq handle
  */
 
-static int vfio_start_eventfd_injection(VFIOINTp *intp)
+static void vfio_start_eventfd_injection(SysBusDevice *sbdev, qemu_irq irq)
 {
     int ret;
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
+    VFIOINTp *intp;
+
+    QLIST_FOREACH(intp, &vdev->intp_list, next) {
+        if (intp->qemuirq == irq) {
+            break;
+        }
+    }
+    assert(intp);
 
     ret = vfio_set_trigger_eventfd(intp, vfio_intp_interrupt);
     if (ret) {
-        error_report("vfio: Error: Failed to pass IRQ fd to the driver: %m");
+        error_report("vfio: failed to start eventfd signaling for IRQ %d: %m",
+                     intp->pin);
+        abort();
     }
-    return ret;
 }
 
 /*
@@ -349,7 +372,7 @@ static int vfio_set_resample_eventfd(VFIOINTp *intp)
     irq_set->start = 0;
     irq_set->count = 1;
     pfd = (int32_t *)&irq_set->data;
-    *pfd = event_notifier_get_fd(&intp->unmask);
+    *pfd = event_notifier_get_fd(intp->unmask);
     qemu_set_fd_handler(*pfd, NULL, NULL, NULL);
     ret = ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     g_free(irq_set);
@@ -359,6 +382,15 @@ static int vfio_set_resample_eventfd(VFIOINTp *intp)
     return ret;
 }
 
+/**
+ * vfio_start_irqfd_injection - starts the virtual IRQ injection using
+ * irqfd
+ *
+ * @sbdev: the sysbus device handle
+ * @irq: the qemu irq handle
+ *
+ * In case the irqfd setup fails, we fallback to userspace handled eventfd
+ */
 static void vfio_start_irqfd_injection(SysBusDevice *sbdev, qemu_irq irq)
 {
     VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
@@ -366,7 +398,7 @@ static void vfio_start_irqfd_injection(SysBusDevice *sbdev, qemu_irq irq)
 
     if (!kvm_irqfds_enabled() || !kvm_resamplefds_enabled() ||
         !vdev->irqfd_allowed) {
-        return;
+        goto fail_irqfd;
     }
 
     QLIST_FOREACH(intp, &vdev->intp_list, next) {
@@ -376,39 +408,36 @@ static void vfio_start_irqfd_injection(SysBusDevice *sbdev, qemu_irq irq)
     }
     assert(intp);
 
-    /* Get to a known interrupt state */
-    qemu_set_fd_handler(event_notifier_get_fd(&intp->interrupt),
-                        NULL, NULL, vdev);
-
-    vfio_mask_single_irqindex(&vdev->vbasedev, intp->pin);
-    qemu_set_irq(intp->qemuirq, 0);
-
-    if (kvm_irqchip_add_irqfd_notifier(kvm_state, &intp->interrupt,
-                                   &intp->unmask, irq) < 0) {
+    if (kvm_irqchip_add_irqfd_notifier(kvm_state, intp->interrupt,
+                                   intp->unmask, irq) < 0) {
         goto fail_irqfd;
     }
 
     if (vfio_set_trigger_eventfd(intp, NULL) < 0) {
         goto fail_vfio;
     }
-    if (vfio_set_resample_eventfd(intp) < 0) {
-        goto fail_vfio;
+    if (vfio_irq_is_automasked(intp)) {
+        if (vfio_set_resample_eventfd(intp) < 0) {
+            goto fail_vfio;
+        }
+        trace_vfio_platform_start_level_irqfd_injection(intp->pin,
+                                    event_notifier_get_fd(intp->interrupt),
+                                    event_notifier_get_fd(intp->unmask));
+    } else {
+        trace_vfio_platform_start_edge_irqfd_injection(intp->pin,
+                                    event_notifier_get_fd(intp->interrupt));
     }
 
-    /* Let's resume injection with irqfd setup */
-    vfio_unmask_single_irqindex(&vdev->vbasedev, intp->pin);
-
     intp->kvm_accel = true;
 
-    trace_vfio_platform_start_irqfd_injection(intp->pin,
-                                     event_notifier_get_fd(&intp->interrupt),
-                                     event_notifier_get_fd(&intp->unmask));
     return;
 fail_vfio:
-    kvm_irqchip_remove_irqfd_notifier(kvm_state, &intp->interrupt, irq);
+    kvm_irqchip_remove_irqfd_notifier(kvm_state, intp->interrupt, irq);
+    error_report("vfio: failed to start eventfd signaling for IRQ %d: %m",
+                 intp->pin);
+    abort();
 fail_irqfd:
-    vfio_start_eventfd_injection(intp);
-    vfio_unmask_single_irqindex(&vdev->vbasedev, intp->pin);
+    vfio_start_eventfd_injection(sbdev, irq);
     return;
 }
 
@@ -449,7 +478,7 @@ static int vfio_populate_device(VFIODevice *vbasedev)
         struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
         VFIORegion *ptr;
 
-        vdev->regions[i] = g_malloc0(sizeof(VFIORegion));
+        vdev->regions[i] = g_new0(VFIORegion, 1);
         ptr = vdev->regions[i];
         reg_info.index = i;
         ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
@@ -646,7 +675,6 @@ static void vfio_platform_realize(DeviceState *dev, Error **errp)
     VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(dev);
     SysBusDevice *sbdev = SYS_BUS_DEVICE(dev);
     VFIODevice *vbasedev = &vdev->vbasedev;
-    VFIOINTp *intp;
     int i, ret;
 
     vbasedev->type = VFIO_DEVICE_TYPE_PLATFORM;
@@ -665,10 +693,6 @@ static void vfio_platform_realize(DeviceState *dev, Error **errp)
         vfio_map_region(vdev, i);
         sysbus_init_mmio(sbdev, &vdev->regions[i]->mem);
     }
-
-    QLIST_FOREACH(intp, &vdev->intp_list, next) {
-        vfio_start_eventfd_injection(intp);
-    }
 }
 
 static const VMStateDescription vfio_platform_vmstate = {
@@ -678,7 +702,7 @@ static const VMStateDescription vfio_platform_vmstate = {
 
 static Property vfio_platform_dev_properties[] = {
     DEFINE_PROP_STRING("host", VFIOPlatformDevice, vbasedev.name),
-    DEFINE_PROP_BOOL("x-mmap", VFIOPlatformDevice, vbasedev.allow_mmap, true),
+    DEFINE_PROP_BOOL("x-no-mmap", VFIOPlatformDevice, vbasedev.no_mmap, false),
     DEFINE_PROP_UINT32("mmap-timeout-ms", VFIOPlatformDevice,
                        mmap_timeout, 1100),
     DEFINE_PROP_BOOL("x-irqfd", VFIOPlatformDevice, irqfd_allowed, true),
diff --git a/hw/virtio/dataplane/vring.c b/hw/virtio/dataplane/vring.c
index 68f199443..23f667ef4 100644
--- a/hw/virtio/dataplane/vring.c
+++ b/hw/virtio/dataplane/vring.c
@@ -25,15 +25,30 @@
 
 /* vring_map can be coupled with vring_unmap or (if you still have the
  * value returned in *mr) memory_region_unref.
+ * Returns NULL on failure.
+ * Callers that can handle a partial mapping must supply mapped_len pointer to
+ * get the actual length mapped.
+ * Passing mapped_len == NULL requires either a full mapping or a failure.
  */
-static void *vring_map(MemoryRegion **mr, hwaddr phys, hwaddr len,
+static void *vring_map(MemoryRegion **mr, hwaddr phys,
+                       hwaddr len, hwaddr *mapped_len,
                        bool is_write)
 {
     MemoryRegionSection section = memory_region_find(get_system_memory(), phys, len);
+    uint64_t size;
 
-    if (!section.mr || int128_get64(section.size) < len) {
+    if (!section.mr) {
         goto out;
     }
+
+    size = int128_get64(section.size);
+    assert(size);
+
+    /* Passing mapped_len == NULL requires either a full mapping or a failure. */
+    if (!mapped_len && size < len) {
+        goto out;
+    }
+
     if (is_write && section.readonly) {
         goto out;
     }
@@ -46,6 +61,10 @@ static void *vring_map(MemoryRegion **mr, hwaddr phys, hwaddr len,
         goto out;
     }
 
+    if (mapped_len) {
+        *mapped_len = MIN(size, len);
+    }
+
     *mr = section.mr;
     return memory_region_get_ram_ptr(section.mr) + section.offset_within_region;
 
@@ -78,7 +97,7 @@ bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
     addr = virtio_queue_get_desc_addr(vdev, n);
     size = virtio_queue_get_desc_size(vdev, n);
     /* Map the descriptor area as read only */
-    ptr = vring_map(&vring->mr_desc, addr, size, false);
+    ptr = vring_map(&vring->mr_desc, addr, size, NULL, false);
     if (!ptr) {
         error_report("Failed to map 0x%" HWADDR_PRIx " byte for vring desc "
                      "at 0x%" HWADDR_PRIx,
@@ -92,7 +111,7 @@ bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
     /* Add the size of the used_event_idx */
     size += sizeof(uint16_t);
     /* Map the driver area as read only */
-    ptr = vring_map(&vring->mr_avail, addr, size, false);
+    ptr = vring_map(&vring->mr_avail, addr, size, NULL, false);
     if (!ptr) {
         error_report("Failed to map 0x%" HWADDR_PRIx " byte for vring avail "
                      "at 0x%" HWADDR_PRIx,
@@ -106,7 +125,7 @@ bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
     /* Add the size of the avail_event_idx */
     size += sizeof(uint16_t);
     /* Map the device area as read-write */
-    ptr = vring_map(&vring->mr_used, addr, size, true);
+    ptr = vring_map(&vring->mr_used, addr, size, NULL, true);
     if (!ptr) {
         error_report("Failed to map 0x%" HWADDR_PRIx " byte for vring used "
                      "at 0x%" HWADDR_PRIx,
@@ -206,6 +225,7 @@ static int get_desc(Vring *vring, VirtQueueElement *elem,
     struct iovec *iov;
     hwaddr *addr;
     MemoryRegion *mr;
+    hwaddr len;
 
     if (desc->flags & VRING_DESC_F_WRITE) {
         num = &elem->in_num;
@@ -224,26 +244,30 @@ static int get_desc(Vring *vring, VirtQueueElement *elem,
         }
     }
 
-    /* Stop for now if there are not enough iovecs available. */
-    if (*num >= VIRTQUEUE_MAX_SIZE) {
-        error_report("Invalid SG num: %u", *num);
-        return -EFAULT;
-    }
+    while (desc->len) {
+        /* Stop for now if there are not enough iovecs available. */
+        if (*num >= VIRTQUEUE_MAX_SIZE) {
+            error_report("Invalid SG num: %u", *num);
+            return -EFAULT;
+        }
 
-    /* TODO handle non-contiguous memory across region boundaries */
-    iov->iov_base = vring_map(&mr, desc->addr, desc->len,
-                              desc->flags & VRING_DESC_F_WRITE);
-    if (!iov->iov_base) {
-        error_report("Failed to map descriptor addr %#" PRIx64 " len %u",
-                     (uint64_t)desc->addr, desc->len);
-        return -EFAULT;
+        iov->iov_base = vring_map(&mr, desc->addr, desc->len, &len,
+                                  desc->flags & VRING_DESC_F_WRITE);
+        if (!iov->iov_base) {
+            error_report("Failed to map descriptor addr %#" PRIx64 " len %u",
+                         (uint64_t)desc->addr, desc->len);
+            return -EFAULT;
+        }
+
+        /* The MemoryRegion is looked up again and unref'ed later, leave the
+         * ref in place.  */
+        (iov++)->iov_len = len;
+        *addr++ = desc->addr;
+        desc->len -= len;
+        desc->addr += len;
+        *num += 1;
     }
 
-    /* The MemoryRegion is looked up again and unref'ed later, leave the
-     * ref in place.  */
-    iov->iov_len = desc->len;
-    *addr = desc->addr;
-    *num += 1;
     return 0;
 }
 
@@ -257,6 +281,21 @@ static void copy_in_vring_desc(VirtIODevice *vdev,
     host->next = virtio_lduw_p(vdev, &guest->next);
 }
 
+static bool read_vring_desc(VirtIODevice *vdev,
+                            hwaddr guest,
+                            struct vring_desc *host)
+{
+    if (address_space_read(&address_space_memory, guest, MEMTXATTRS_UNSPECIFIED,
+                           (uint8_t *)host, sizeof *host)) {
+        return false;
+    }
+    host->addr = virtio_tswap64(vdev, host->addr);
+    host->len = virtio_tswap32(vdev, host->len);
+    host->flags = virtio_tswap16(vdev, host->flags);
+    host->next = virtio_tswap16(vdev, host->next);
+    return true;
+}
+
 /* This is stolen from linux/drivers/vhost/vhost.c. */
 static int get_indirect(VirtIODevice *vdev, Vring *vring,
                         VirtQueueElement *elem, struct vring_desc *indirect)
@@ -284,23 +323,16 @@ static int get_indirect(VirtIODevice *vdev, Vring *vring,
     }
 
     do {
-        struct vring_desc *desc_ptr;
-        MemoryRegion *mr;
-
         /* Translate indirect descriptor */
-        desc_ptr = vring_map(&mr,
-                             indirect->addr + found * sizeof(desc),
-                             sizeof(desc), false);
-        if (!desc_ptr) {
-            error_report("Failed to map indirect descriptor "
+        if (!read_vring_desc(vdev, indirect->addr + found * sizeof(desc),
+                             &desc)) {
+            error_report("Failed to read indirect descriptor "
                          "addr %#" PRIx64 " len %zu",
                          (uint64_t)indirect->addr + found * sizeof(desc),
                          sizeof(desc));
             vring->broken = true;
             return -EFAULT;
         }
-        copy_in_vring_desc(vdev, desc_ptr, &desc);
-        memory_region_unref(mr);
 
         /* Ensure descriptor has been loaded before accessing fields */
         barrier(); /* read_barrier_depends(); */
diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c
index 4d68a2765..b734a601a 100644
--- a/hw/virtio/vhost-backend.c
+++ b/hw/virtio/vhost-backend.c
@@ -11,6 +11,7 @@
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/vhost-backend.h"
 #include "qemu/error-report.h"
+#include "linux/vhost.h"
 
 #include <sys/ioctl.h>
 
@@ -42,11 +43,152 @@ static int vhost_kernel_cleanup(struct vhost_dev *dev)
     return close(fd);
 }
 
+static int vhost_kernel_memslots_limit(struct vhost_dev *dev)
+{
+    int limit = 64;
+    char *s;
+
+    if (g_file_get_contents("/sys/module/vhost/parameters/max_mem_regions",
+                            &s, NULL, NULL)) {
+        uint64_t val = g_ascii_strtoull(s, NULL, 10);
+        if (!((val == G_MAXUINT64 || !val) && errno)) {
+            return val;
+        }
+        error_report("ignoring invalid max_mem_regions value in vhost module:"
+                     " %s", s);
+    }
+    return limit;
+}
+
+static int vhost_kernel_net_set_backend(struct vhost_dev *dev,
+                                        struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_NET_SET_BACKEND, file);
+}
+
+static int vhost_kernel_scsi_set_endpoint(struct vhost_dev *dev,
+                                          struct vhost_scsi_target *target)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_SET_ENDPOINT, target);
+}
+
+static int vhost_kernel_scsi_clear_endpoint(struct vhost_dev *dev,
+                                            struct vhost_scsi_target *target)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_CLEAR_ENDPOINT, target);
+}
+
+static int vhost_kernel_scsi_get_abi_version(struct vhost_dev *dev, int *version)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_GET_ABI_VERSION, version);
+}
+
+static int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                     struct vhost_log *log)
+{
+    return vhost_kernel_call(dev, VHOST_SET_LOG_BASE, &base);
+}
+
+static int vhost_kernel_set_mem_table(struct vhost_dev *dev,
+                                      struct vhost_memory *mem)
+{
+    return vhost_kernel_call(dev, VHOST_SET_MEM_TABLE, mem);
+}
+
+static int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
+                                       struct vhost_vring_addr *addr)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_ADDR, addr);
+}
+
+static int vhost_kernel_set_vring_endian(struct vhost_dev *dev,
+                                         struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_ENDIAN, ring);
+}
+
+static int vhost_kernel_set_vring_num(struct vhost_dev *dev,
+                                      struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_NUM, ring);
+}
+
+static int vhost_kernel_set_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_BASE, ring);
+}
+
+static int vhost_kernel_get_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_GET_VRING_BASE, ring);
+}
+
+static int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_KICK, file);
+}
+
+static int vhost_kernel_set_vring_call(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_CALL, file);
+}
+
+static int vhost_kernel_set_features(struct vhost_dev *dev,
+                                     uint64_t features)
+{
+    return vhost_kernel_call(dev, VHOST_SET_FEATURES, &features);
+}
+
+static int vhost_kernel_get_features(struct vhost_dev *dev,
+                                     uint64_t *features)
+{
+    return vhost_kernel_call(dev, VHOST_GET_FEATURES, features);
+}
+
+static int vhost_kernel_set_owner(struct vhost_dev *dev)
+{
+    return vhost_kernel_call(dev, VHOST_SET_OWNER, NULL);
+}
+
+static int vhost_kernel_reset_device(struct vhost_dev *dev)
+{
+    return vhost_kernel_call(dev, VHOST_RESET_OWNER, NULL);
+}
+
+static int vhost_kernel_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
 static const VhostOps kernel_ops = {
         .backend_type = VHOST_BACKEND_TYPE_KERNEL,
-        .vhost_call = vhost_kernel_call,
         .vhost_backend_init = vhost_kernel_init,
-        .vhost_backend_cleanup = vhost_kernel_cleanup
+        .vhost_backend_cleanup = vhost_kernel_cleanup,
+        .vhost_backend_memslots_limit = vhost_kernel_memslots_limit,
+        .vhost_net_set_backend = vhost_kernel_net_set_backend,
+        .vhost_scsi_set_endpoint = vhost_kernel_scsi_set_endpoint,
+        .vhost_scsi_clear_endpoint = vhost_kernel_scsi_clear_endpoint,
+        .vhost_scsi_get_abi_version = vhost_kernel_scsi_get_abi_version,
+        .vhost_set_log_base = vhost_kernel_set_log_base,
+        .vhost_set_mem_table = vhost_kernel_set_mem_table,
+        .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
+        .vhost_set_vring_endian = vhost_kernel_set_vring_endian,
+        .vhost_set_vring_num = vhost_kernel_set_vring_num,
+        .vhost_set_vring_base = vhost_kernel_set_vring_base,
+        .vhost_get_vring_base = vhost_kernel_get_vring_base,
+        .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
+        .vhost_set_vring_call = vhost_kernel_set_vring_call,
+        .vhost_set_features = vhost_kernel_set_features,
+        .vhost_get_features = vhost_kernel_get_features,
+        .vhost_set_owner = vhost_kernel_set_owner,
+        .vhost_reset_device = vhost_kernel_reset_device,
+        .vhost_get_vq_index = vhost_kernel_get_vq_index,
 };
 
 int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index e7ab8293d..577c95e35 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -10,11 +10,13 @@
 
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
 #include "sysemu/char.h"
 #include "sysemu/kvm.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "exec/ram_addr.h"
+#include "migration/migration.h"
 
 #include <fcntl.h>
 #include <unistd.h>
@@ -24,6 +26,17 @@
 #include <linux/vhost.h>
 
 #define VHOST_MEMORY_MAX_NREGIONS    8
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+
+enum VhostUserProtocolFeature {
+    VHOST_USER_PROTOCOL_F_MQ = 0,
+    VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
+    VHOST_USER_PROTOCOL_F_RARP = 2,
+
+    VHOST_USER_PROTOCOL_F_MAX
+};
+
+#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
 
 typedef enum VhostUserRequest {
     VHOST_USER_NONE = 0,
@@ -41,6 +54,11 @@ typedef enum VhostUserRequest {
     VHOST_USER_SET_VRING_KICK = 12,
     VHOST_USER_SET_VRING_CALL = 13,
     VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_GET_QUEUE_NUM = 17,
+    VHOST_USER_SET_VRING_ENABLE = 18,
+    VHOST_USER_SEND_RARP = 19,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -57,6 +75,11 @@ typedef struct VhostUserMemory {
     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 } VhostUserMemory;
 
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
 typedef struct VhostUserMsg {
     VhostUserRequest request;
 
@@ -71,7 +94,8 @@ typedef struct VhostUserMsg {
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
-    };
+        VhostUserLog log;
+    } payload;
 } QEMU_PACKED VhostUserMsg;
 
 static VhostUserMsg m __attribute__ ((unused));
@@ -89,37 +113,6 @@ static bool ioeventfd_enabled(void)
     return kvm_enabled() && kvm_eventfds_enabled();
 }
 
-static unsigned long int ioctl_to_vhost_user_request[VHOST_USER_MAX] = {
-    -1,                     /* VHOST_USER_NONE */
-    VHOST_GET_FEATURES,     /* VHOST_USER_GET_FEATURES */
-    VHOST_SET_FEATURES,     /* VHOST_USER_SET_FEATURES */
-    VHOST_SET_OWNER,        /* VHOST_USER_SET_OWNER */
-    VHOST_RESET_OWNER,      /* VHOST_USER_RESET_OWNER */
-    VHOST_SET_MEM_TABLE,    /* VHOST_USER_SET_MEM_TABLE */
-    VHOST_SET_LOG_BASE,     /* VHOST_USER_SET_LOG_BASE */
-    VHOST_SET_LOG_FD,       /* VHOST_USER_SET_LOG_FD */
-    VHOST_SET_VRING_NUM,    /* VHOST_USER_SET_VRING_NUM */
-    VHOST_SET_VRING_ADDR,   /* VHOST_USER_SET_VRING_ADDR */
-    VHOST_SET_VRING_BASE,   /* VHOST_USER_SET_VRING_BASE */
-    VHOST_GET_VRING_BASE,   /* VHOST_USER_GET_VRING_BASE */
-    VHOST_SET_VRING_KICK,   /* VHOST_USER_SET_VRING_KICK */
-    VHOST_SET_VRING_CALL,   /* VHOST_USER_SET_VRING_CALL */
-    VHOST_SET_VRING_ERR     /* VHOST_USER_SET_VRING_ERR */
-};
-
-static VhostUserRequest vhost_user_request_translate(unsigned long int request)
-{
-    VhostUserRequest idx;
-
-    for (idx = 0; idx < VHOST_USER_MAX; idx++) {
-        if (ioctl_to_vhost_user_request[idx] == request) {
-            break;
-        }
-    }
-
-    return (idx == VHOST_USER_MAX) ? VHOST_USER_NONE : idx;
-}
-
 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 {
     CharDriverState *chr = dev->opaque;
@@ -128,8 +121,8 @@ static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 
     r = qemu_chr_fe_read_all(chr, p, size);
     if (r != size) {
-        error_report("Failed to read msg header. Read %d instead of %d.", r,
-                size);
+        error_report("Failed to read msg header. Read %d instead of %d."
+                     " Original request %d.", r, size, msg->request);
         goto fail;
     }
 
@@ -166,12 +159,35 @@ fail:
     return -1;
 }
 
+static bool vhost_user_one_time_request(VhostUserRequest request)
+{
+    switch (request) {
+    case VHOST_USER_SET_OWNER:
+    case VHOST_USER_RESET_OWNER:
+    case VHOST_USER_SET_MEM_TABLE:
+    case VHOST_USER_GET_QUEUE_NUM:
+        return true;
+    default:
+        return false;
+    }
+}
+
+/* most non-init callers ignore the error */
 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
                             int *fds, int fd_num)
 {
     CharDriverState *chr = dev->opaque;
     int size = VHOST_USER_HDR_SIZE + msg->size;
 
+    /*
+     * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
+     * we just need send it once in the first time. For later such
+     * request, we just ignore it.
+     */
+    if (vhost_user_one_time_request(msg->request) && dev->vq_index != 0) {
+        return 0;
+    }
+
     if (fd_num) {
         qemu_chr_fe_set_msgfds(chr, fds, fd_num);
     }
@@ -180,157 +196,364 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
             0 : -1;
 }
 
-static int vhost_user_call(struct vhost_dev *dev, unsigned long int request,
-        void *arg)
+static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                   struct vhost_log *log)
 {
-    VhostUserMsg msg;
-    VhostUserRequest msg_request;
-    struct vhost_vring_file *file = 0;
-    int need_reply = 0;
     int fds[VHOST_MEMORY_MAX_NREGIONS];
-    int i, fd;
     size_t fd_num = 0;
+    bool shmfd = virtio_has_feature(dev->protocol_features,
+                                    VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_LOG_BASE,
+        .flags = VHOST_USER_VERSION,
+        .payload.log.mmap_size = log->size * sizeof(*(log->log)),
+        .payload.log.mmap_offset = 0,
+        .size = sizeof(msg.payload.log),
+    };
 
-    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+    if (shmfd && log->fd != -1) {
+        fds[fd_num++] = log->fd;
+    }
 
-    msg_request = vhost_user_request_translate(request);
-    msg.request = msg_request;
-    msg.flags = VHOST_USER_VERSION;
-    msg.size = 0;
+    vhost_user_write(dev, &msg, fds, fd_num);
 
-    switch (request) {
-    case VHOST_GET_FEATURES:
-        need_reply = 1;
-        break;
-
-    case VHOST_SET_FEATURES:
-    case VHOST_SET_LOG_BASE:
-        msg.u64 = *((__u64 *) arg);
-        msg.size = sizeof(m.u64);
-        break;
-
-    case VHOST_SET_OWNER:
-    case VHOST_RESET_OWNER:
-        break;
-
-    case VHOST_SET_MEM_TABLE:
-        for (i = 0; i < dev->mem->nregions; ++i) {
-            struct vhost_memory_region *reg = dev->mem->regions + i;
-            ram_addr_t ram_addr;
-
-            assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
-            qemu_ram_addr_from_host((void *)(uintptr_t)reg->userspace_addr, &ram_addr);
-            fd = qemu_get_ram_fd(ram_addr);
-            if (fd > 0) {
-                msg.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
-                msg.memory.regions[fd_num].memory_size  = reg->memory_size;
-                msg.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
-                msg.memory.regions[fd_num].mmap_offset = reg->userspace_addr -
-                    (uintptr_t) qemu_get_ram_block_host_ptr(ram_addr);
-                assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
-                fds[fd_num++] = fd;
-            }
+    if (shmfd) {
+        msg.size = 0;
+        if (vhost_user_read(dev, &msg) < 0) {
+            return 0;
         }
 
-        msg.memory.nregions = fd_num;
-
-        if (!fd_num) {
-            error_report("Failed initializing vhost-user memory map, "
-                    "consider using -object memory-backend-file share=on");
+        if (msg.request != VHOST_USER_SET_LOG_BASE) {
+            error_report("Received unexpected msg type. "
+                         "Expected %d received %d",
+                         VHOST_USER_SET_LOG_BASE, msg.request);
             return -1;
         }
+    }
 
-        msg.size = sizeof(m.memory.nregions);
-        msg.size += sizeof(m.memory.padding);
-        msg.size += fd_num * sizeof(VhostUserMemoryRegion);
-
-        break;
-
-    case VHOST_SET_LOG_FD:
-        fds[fd_num++] = *((int *) arg);
-        break;
-
-    case VHOST_SET_VRING_NUM:
-    case VHOST_SET_VRING_BASE:
-        memcpy(&msg.state, arg, sizeof(struct vhost_vring_state));
-        msg.size = sizeof(m.state);
-        break;
-
-    case VHOST_GET_VRING_BASE:
-        memcpy(&msg.state, arg, sizeof(struct vhost_vring_state));
-        msg.size = sizeof(m.state);
-        need_reply = 1;
-        break;
-
-    case VHOST_SET_VRING_ADDR:
-        memcpy(&msg.addr, arg, sizeof(struct vhost_vring_addr));
-        msg.size = sizeof(m.addr);
-        break;
-
-    case VHOST_SET_VRING_KICK:
-    case VHOST_SET_VRING_CALL:
-    case VHOST_SET_VRING_ERR:
-        file = arg;
-        msg.u64 = file->index & VHOST_USER_VRING_IDX_MASK;
-        msg.size = sizeof(m.u64);
-        if (ioeventfd_enabled() && file->fd > 0) {
-            fds[fd_num++] = file->fd;
-        } else {
-            msg.u64 |= VHOST_USER_VRING_NOFD_MASK;
+    return 0;
+}
+
+static int vhost_user_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    int i, fd;
+    size_t fd_num = 0;
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_MEM_TABLE,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    for (i = 0; i < dev->mem->nregions; ++i) {
+        struct vhost_memory_region *reg = dev->mem->regions + i;
+        ram_addr_t ram_addr;
+
+        assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
+        qemu_ram_addr_from_host((void *)(uintptr_t)reg->userspace_addr,
+                                &ram_addr);
+        fd = qemu_get_ram_fd(ram_addr);
+        if (fd > 0) {
+            msg.payload.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
+            msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
+            msg.payload.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
+            msg.payload.memory.regions[fd_num].mmap_offset = reg->userspace_addr -
+                (uintptr_t) qemu_get_ram_block_host_ptr(ram_addr);
+            assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
+            fds[fd_num++] = fd;
         }
-        break;
-    default:
-        error_report("vhost-user trying to send unhandled ioctl");
+    }
+
+    msg.payload.memory.nregions = fd_num;
+
+    if (!fd_num) {
+        error_report("Failed initializing vhost-user memory map, "
+                     "consider using -object memory-backend-file share=on");
         return -1;
-        break;
     }
 
-    if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
+    msg.size = sizeof(msg.payload.memory.nregions);
+    msg.size += sizeof(msg.payload.memory.padding);
+    msg.size += fd_num * sizeof(VhostUserMemoryRegion);
+
+    vhost_user_write(dev, &msg, fds, fd_num);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_addr(struct vhost_dev *dev,
+                                     struct vhost_vring_addr *addr)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_VRING_ADDR,
+        .flags = VHOST_USER_VERSION,
+        .payload.addr = *addr,
+        .size = sizeof(msg.payload.addr),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_endian(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    error_report("vhost-user trying to send unhandled ioctl");
+    return -1;
+}
+
+static int vhost_set_vring(struct vhost_dev *dev,
+                           unsigned long int request,
+                           struct vhost_vring_state *ring)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.state = *ring,
+        .size = sizeof(msg.payload.state),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_num(struct vhost_dev *dev,
+                                    struct vhost_vring_state *ring)
+{
+    return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
+}
+
+static int vhost_user_set_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
+}
+
+static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
+{
+    int i;
+
+    if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
+        return -1;
+    }
+
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num   = enable,
+        };
+
+        vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
+    }
+
+    return 0;
+}
+
+static int vhost_user_get_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_GET_VRING_BASE,
+        .flags = VHOST_USER_VERSION,
+        .payload.state = *ring,
+        .size = sizeof(msg.payload.state),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    if (vhost_user_read(dev, &msg) < 0) {
         return 0;
     }
 
-    if (need_reply) {
-        if (vhost_user_read(dev, &msg) < 0) {
-            return 0;
-        }
+    if (msg.request != VHOST_USER_GET_VRING_BASE) {
+        error_report("Received unexpected msg type. Expected %d received %d",
+                     VHOST_USER_GET_VRING_BASE, msg.request);
+        return -1;
+    }
 
-        if (msg_request != msg.request) {
-            error_report("Received unexpected msg type."
-                    " Expected %d received %d", msg_request, msg.request);
-            return -1;
-        }
+    if (msg.size != sizeof(msg.payload.state)) {
+        error_report("Received bad msg size.");
+        return -1;
+    }
 
-        switch (msg_request) {
-        case VHOST_USER_GET_FEATURES:
-            if (msg.size != sizeof(m.u64)) {
-                error_report("Received bad msg size.");
-                return -1;
-            }
-            *((__u64 *) arg) = msg.u64;
-            break;
-        case VHOST_USER_GET_VRING_BASE:
-            if (msg.size != sizeof(m.state)) {
-                error_report("Received bad msg size.");
-                return -1;
-            }
-            memcpy(arg, &msg.state, sizeof(struct vhost_vring_state));
-            break;
-        default:
-            error_report("Received unexpected msg type.");
-            return -1;
-            break;
-        }
+    *ring = msg.payload.state;
+
+    return 0;
+}
+
+static int vhost_set_vring_file(struct vhost_dev *dev,
+                                VhostUserRequest request,
+                                struct vhost_vring_file *file)
+{
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    size_t fd_num = 0;
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
+        .size = sizeof(msg.payload.u64),
+    };
+
+    if (ioeventfd_enabled() && file->fd > 0) {
+        fds[fd_num++] = file->fd;
+    } else {
+        msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
     }
 
+    vhost_user_write(dev, &msg, fds, fd_num);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_kick(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
+}
+
+static int vhost_user_set_vring_call(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
+}
+
+static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.u64 = u64,
+        .size = sizeof(msg.payload.u64),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+    return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
+}
+
+static int vhost_user_set_protocol_features(struct vhost_dev *dev,
+                                            uint64_t features)
+{
+    return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
+}
+
+static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
+        return 0;
+    }
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    if (vhost_user_read(dev, &msg) < 0) {
+        return 0;
+    }
+
+    if (msg.request != request) {
+        error_report("Received unexpected msg type. Expected %d received %d",
+                     request, msg.request);
+        return -1;
+    }
+
+    if (msg.size != sizeof(msg.payload.u64)) {
+        error_report("Received bad msg size.");
+        return -1;
+    }
+
+    *u64 = msg.payload.u64;
+
+    return 0;
+}
+
+static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
+{
+    return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
+}
+
+static int vhost_user_set_owner(struct vhost_dev *dev)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_OWNER,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_reset_device(struct vhost_dev *dev)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_RESET_OWNER,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
     return 0;
 }
 
 static int vhost_user_init(struct vhost_dev *dev, void *opaque)
 {
+    uint64_t features;
+    int err;
+
     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 
     dev->opaque = opaque;
 
+    err = vhost_user_get_features(dev, &features);
+    if (err < 0) {
+        return err;
+    }
+
+    if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
+        dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
+
+        err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
+                                 &features);
+        if (err < 0) {
+            return err;
+        }
+
+        dev->protocol_features = features & VHOST_USER_PROTOCOL_FEATURE_MASK;
+        err = vhost_user_set_protocol_features(dev, dev->protocol_features);
+        if (err < 0) {
+            return err;
+        }
+
+        /* query the max queues we support if backend supports Multiple Queue */
+        if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
+            err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
+                                     &dev->max_queues);
+            if (err < 0) {
+                return err;
+            }
+        }
+    }
+
+    if (dev->migration_blocker == NULL &&
+        !virtio_has_feature(dev->protocol_features,
+                            VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
+        error_setg(&dev->migration_blocker,
+                   "Migration disabled: vhost-user backend lacks "
+                   "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
+    }
+
     return 0;
 }
 
@@ -343,9 +566,92 @@ static int vhost_user_cleanup(struct vhost_dev *dev)
     return 0;
 }
 
+static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx;
+}
+
+static int vhost_user_memslots_limit(struct vhost_dev *dev)
+{
+    return VHOST_MEMORY_MAX_NREGIONS;
+}
+
+static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
+{
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+
+    return virtio_has_feature(dev->protocol_features,
+                              VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+}
+
+static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
+{
+    VhostUserMsg msg = { 0 };
+    int err;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+
+    /* If guest supports GUEST_ANNOUNCE do nothing */
+    if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
+        return 0;
+    }
+
+    /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
+    if (virtio_has_feature(dev->protocol_features,
+                           VHOST_USER_PROTOCOL_F_RARP)) {
+        msg.request = VHOST_USER_SEND_RARP;
+        msg.flags = VHOST_USER_VERSION;
+        memcpy((char *)&msg.payload.u64, mac_addr, 6);
+        msg.size = sizeof(msg.payload.u64);
+
+        err = vhost_user_write(dev, &msg, NULL, 0);
+        return err;
+    }
+    return -1;
+}
+
+static bool vhost_user_can_merge(struct vhost_dev *dev,
+                                 uint64_t start1, uint64_t size1,
+                                 uint64_t start2, uint64_t size2)
+{
+    ram_addr_t ram_addr;
+    int mfd, rfd;
+    MemoryRegion *mr;
+
+    mr = qemu_ram_addr_from_host((void *)(uintptr_t)start1, &ram_addr);
+    assert(mr);
+    mfd = qemu_get_ram_fd(ram_addr);
+
+    mr = qemu_ram_addr_from_host((void *)(uintptr_t)start2, &ram_addr);
+    assert(mr);
+    rfd = qemu_get_ram_fd(ram_addr);
+
+    return mfd == rfd;
+}
+
 const VhostOps user_ops = {
         .backend_type = VHOST_BACKEND_TYPE_USER,
-        .vhost_call = vhost_user_call,
         .vhost_backend_init = vhost_user_init,
-        .vhost_backend_cleanup = vhost_user_cleanup
-        };
+        .vhost_backend_cleanup = vhost_user_cleanup,
+        .vhost_backend_memslots_limit = vhost_user_memslots_limit,
+        .vhost_set_log_base = vhost_user_set_log_base,
+        .vhost_set_mem_table = vhost_user_set_mem_table,
+        .vhost_set_vring_addr = vhost_user_set_vring_addr,
+        .vhost_set_vring_endian = vhost_user_set_vring_endian,
+        .vhost_set_vring_num = vhost_user_set_vring_num,
+        .vhost_set_vring_base = vhost_user_set_vring_base,
+        .vhost_get_vring_base = vhost_user_get_vring_base,
+        .vhost_set_vring_kick = vhost_user_set_vring_kick,
+        .vhost_set_vring_call = vhost_user_set_vring_call,
+        .vhost_set_features = vhost_user_set_features,
+        .vhost_get_features = vhost_user_get_features,
+        .vhost_set_owner = vhost_user_set_owner,
+        .vhost_reset_device = vhost_user_reset_device,
+        .vhost_get_vq_index = vhost_user_get_vq_index,
+        .vhost_set_vring_enable = vhost_user_set_vring_enable,
+        .vhost_requires_shm_log = vhost_user_requires_shm_log,
+        .vhost_migration_done = vhost_user_migration_done,
+        .vhost_backend_can_merge = vhost_user_can_merge,
+};
diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index a08c36bb4..90c60a7e9 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -18,6 +18,7 @@
 #include "qemu/atomic.h"
 #include "qemu/range.h"
 #include "qemu/error-report.h"
+#include "qemu/memfd.h"
 #include <linux/vhost.h>
 #include "exec/address-spaces.h"
 #include "hw/virtio/virtio-bus.h"
@@ -25,6 +26,23 @@
 #include "migration/migration.h"
 
 static struct vhost_log *vhost_log;
+static struct vhost_log *vhost_log_shm;
+
+static unsigned int used_memslots;
+static QLIST_HEAD(, vhost_dev) vhost_devices =
+    QLIST_HEAD_INITIALIZER(vhost_devices);
+
+bool vhost_has_free_slot(void)
+{
+    unsigned int slots_limit = ~0U;
+    struct vhost_dev *hdev;
+
+    QLIST_FOREACH(hdev, &vhost_devices, entry) {
+        unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev);
+        slots_limit = MIN(slots_limit, r);
+    }
+    return slots_limit > used_memslots;
+}
 
 static void vhost_dev_sync_region(struct vhost_dev *dev,
                                   MemoryRegionSection *section,
@@ -241,6 +259,13 @@ static void vhost_dev_assign_memory(struct vhost_dev *dev,
             continue;
         }
 
+        if (dev->vhost_ops->vhost_backend_can_merge &&
+            !dev->vhost_ops->vhost_backend_can_merge(dev, uaddr, size,
+                                                     reg->userspace_addr,
+                                                     reg->memory_size)) {
+            continue;
+        }
+
         if (merged) {
             --to;
             assert(to >= 0);
@@ -286,25 +311,46 @@ static uint64_t vhost_get_log_size(struct vhost_dev *dev)
     }
     return log_size;
 }
-static struct vhost_log *vhost_log_alloc(uint64_t size)
+
+static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
 {
-    struct vhost_log *log = g_malloc0(sizeof *log + size * sizeof(*(log->log)));
+    struct vhost_log *log;
+    uint64_t logsize = size * sizeof(*(log->log));
+    int fd = -1;
+
+    log = g_new0(struct vhost_log, 1);
+    if (share) {
+        log->log = qemu_memfd_alloc("vhost-log", logsize,
+                                    F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
+                                    &fd);
+        memset(log->log, 0, logsize);
+    } else {
+        log->log = g_malloc0(logsize);
+    }
 
     log->size = size;
     log->refcnt = 1;
+    log->fd = fd;
 
     return log;
 }
 
-static struct vhost_log *vhost_log_get(uint64_t size)
+static struct vhost_log *vhost_log_get(uint64_t size, bool share)
 {
-    if (!vhost_log || vhost_log->size != size) {
-        vhost_log = vhost_log_alloc(size);
+    struct vhost_log *log = share ? vhost_log_shm : vhost_log;
+
+    if (!log || log->size != size) {
+        log = vhost_log_alloc(size, share);
+        if (share) {
+            vhost_log_shm = log;
+        } else {
+            vhost_log = log;
+        }
     } else {
-        ++vhost_log->refcnt;
+        ++log->refcnt;
     }
 
-    return vhost_log;
+    return log;
 }
 
 static void vhost_log_put(struct vhost_dev *dev, bool sync)
@@ -321,20 +367,35 @@ static void vhost_log_put(struct vhost_dev *dev, bool sync)
         if (dev->log_size && sync) {
             vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
         }
+
         if (vhost_log == log) {
+            g_free(log->log);
             vhost_log = NULL;
+        } else if (vhost_log_shm == log) {
+            qemu_memfd_free(log->log, log->size * sizeof(*(log->log)),
+                            log->fd);
+            vhost_log_shm = NULL;
         }
+
         g_free(log);
     }
 }
 
-static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
+static bool vhost_dev_log_is_shared(struct vhost_dev *dev)
+{
+    return dev->vhost_ops->vhost_requires_shm_log &&
+           dev->vhost_ops->vhost_requires_shm_log(dev);
+}
+
+static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size)
 {
-    struct vhost_log *log = vhost_log_get(size);
+    struct vhost_log *log = vhost_log_get(size, vhost_dev_log_is_shared(dev));
     uint64_t log_base = (uintptr_t)log->log;
     int r;
 
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_LOG_BASE, &log_base);
+    /* inform backend of log switching, this must be done before
+       releasing the current log, to ensure no logging is lost */
+    r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
     assert(r >= 0);
     vhost_log_put(dev, true);
     dev->log = log;
@@ -457,6 +518,7 @@ static void vhost_set_memory(MemoryListener *listener,
     dev->mem_changed_start_addr = MIN(dev->mem_changed_start_addr, start_addr);
     dev->mem_changed_end_addr = MAX(dev->mem_changed_end_addr, start_addr + size - 1);
     dev->memory_changed = true;
+    used_memslots = dev->mem->nregions;
 }
 
 static bool vhost_section(MemoryRegionSection *section)
@@ -500,7 +562,7 @@ static void vhost_commit(MemoryListener *listener)
     }
 
     if (!dev->log_enabled) {
-        r = dev->vhost_ops->vhost_call(dev, VHOST_SET_MEM_TABLE, dev->mem);
+        r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
         assert(r >= 0);
         dev->memory_changed = false;
         return;
@@ -513,7 +575,7 @@ static void vhost_commit(MemoryListener *listener)
     if (dev->log_size < log_size) {
         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
     }
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_MEM_TABLE, dev->mem);
+    r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
     assert(r >= 0);
     /* To log less, can only decrease log size after table update. */
     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
@@ -581,7 +643,7 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
         .log_guest_addr = vq->used_phys,
         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
     };
-    int r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_ADDR, &addr);
+    int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
     if (r < 0) {
         return -errno;
     }
@@ -595,19 +657,20 @@ static int vhost_dev_set_features(struct vhost_dev *dev, bool enable_log)
     if (enable_log) {
         features |= 0x1ULL << VHOST_F_LOG_ALL;
     }
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_FEATURES, &features);
+    r = dev->vhost_ops->vhost_set_features(dev, features);
     return r < 0 ? -errno : 0;
 }
 
 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 {
-    int r, t, i;
+    int r, t, i, idx;
     r = vhost_dev_set_features(dev, enable_log);
     if (r < 0) {
         goto err_features;
     }
     for (i = 0; i < dev->nvqs; ++i) {
-        r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
+        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
+        r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
                                      enable_log);
         if (r < 0) {
             goto err_vq;
@@ -616,7 +679,8 @@ static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
     return 0;
 err_vq:
     for (; i >= 0; --i) {
-        t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
+        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
+        t = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
                                      dev->log_enabled);
         assert(t >= 0);
     }
@@ -700,7 +764,7 @@ static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
         .num = is_big_endian
     };
 
-    if (!dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_ENDIAN, &s)) {
+    if (!dev->vhost_ops->vhost_set_vring_endian(dev, &s)) {
         return 0;
     }
 
@@ -719,7 +783,7 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
 {
     hwaddr s, l, a;
     int r;
-    int vhost_vq_index = idx - dev->vq_index;
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
     struct vhost_vring_file file = {
         .index = vhost_vq_index
     };
@@ -728,16 +792,15 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
     };
     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 
-    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
 
     vq->num = state.num = virtio_queue_get_num(vdev, idx);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_NUM, &state);
+    r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
     if (r) {
         return -errno;
     }
 
     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_BASE, &state);
+    r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
     if (r) {
         return -errno;
     }
@@ -789,7 +852,7 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
     }
 
     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_KICK, &file);
+    r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
     if (r) {
         r = -errno;
         goto fail_kick;
@@ -822,13 +885,13 @@ static void vhost_virtqueue_stop(struct vhost_dev *dev,
                                     struct vhost_virtqueue *vq,
                                     unsigned idx)
 {
-    int vhost_vq_index = idx - dev->vq_index;
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
     struct vhost_vring_state state = {
         .index = vhost_vq_index,
     };
     int r;
-    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_GET_VRING_BASE, &state);
+
+    r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
     if (r < 0) {
         fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
         fflush(stderr);
@@ -875,8 +938,9 @@ static void vhost_eventfd_del(MemoryListener *listener,
 static int vhost_virtqueue_init(struct vhost_dev *dev,
                                 struct vhost_virtqueue *vq, int n)
 {
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
     struct vhost_vring_file file = {
-        .index = n,
+        .index = vhost_vq_index,
     };
     int r = event_notifier_init(&vq->masked_notifier, 0);
     if (r < 0) {
@@ -884,7 +948,7 @@ static int vhost_virtqueue_init(struct vhost_dev *dev,
     }
 
     file.fd = event_notifier_get_fd(&vq->masked_notifier);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_CALL, &file);
+    r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
     if (r) {
         r = -errno;
         goto fail_call;
@@ -906,6 +970,8 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
     uint64_t features;
     int i, r;
 
+    hdev->migration_blocker = NULL;
+
     if (vhost_set_backend_type(hdev, backend_type) < 0) {
         close((uintptr_t)opaque);
         return -1;
@@ -916,18 +982,26 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
         return -errno;
     }
 
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_OWNER, NULL);
+    if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) {
+        fprintf(stderr, "vhost backend memory slots limit is less"
+                " than current number of present memory slots\n");
+        close((uintptr_t)opaque);
+        return -1;
+    }
+    QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
+
+    r = hdev->vhost_ops->vhost_set_owner(hdev);
     if (r < 0) {
         goto fail;
     }
 
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_GET_FEATURES, &features);
+    r = hdev->vhost_ops->vhost_get_features(hdev, &features);
     if (r < 0) {
         goto fail;
     }
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = vhost_virtqueue_init(hdev, hdev->vqs + i, i);
+        r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i);
         if (r < 0) {
             goto fail_vq;
         }
@@ -949,12 +1023,21 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
         .eventfd_del = vhost_eventfd_del,
         .priority = 10
     };
-    hdev->migration_blocker = NULL;
-    if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
-        error_setg(&hdev->migration_blocker,
-                   "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
+
+    if (hdev->migration_blocker == NULL) {
+        if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
+            error_setg(&hdev->migration_blocker,
+                       "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
+        } else if (!qemu_memfd_check()) {
+            error_setg(&hdev->migration_blocker,
+                       "Migration disabled: failed to allocate shared memory");
+        }
+    }
+
+    if (hdev->migration_blocker != NULL) {
         migrate_add_blocker(hdev->migration_blocker);
     }
+
     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
     hdev->n_mem_sections = 0;
     hdev->mem_sections = NULL;
@@ -972,6 +1055,7 @@ fail_vq:
 fail:
     r = -errno;
     hdev->vhost_ops->vhost_backend_cleanup(hdev);
+    QLIST_REMOVE(hdev, entry);
     return r;
 }
 
@@ -989,6 +1073,7 @@ void vhost_dev_cleanup(struct vhost_dev *hdev)
     g_free(hdev->mem);
     g_free(hdev->mem_sections);
     hdev->vhost_ops->vhost_backend_cleanup(hdev);
+    QLIST_REMOVE(hdev, entry);
 }
 
 /* Stop processing guest IO notifications in qemu.
@@ -1066,18 +1151,16 @@ void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
 {
     struct VirtQueue *vvq = virtio_get_queue(vdev, n);
     int r, index = n - hdev->vq_index;
+    struct vhost_vring_file file;
 
-    assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
-
-    struct vhost_vring_file file = {
-        .index = index
-    };
     if (mask) {
         file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
     } else {
         file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
     }
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_VRING_CALL, &file);
+
+    file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
+    r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
     assert(r >= 0);
 }
 
@@ -1119,7 +1202,7 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
     if (r < 0) {
         goto fail_features;
     }
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_MEM_TABLE, hdev->mem);
+    r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
     if (r < 0) {
         r = -errno;
         goto fail_mem;
@@ -1138,10 +1221,12 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         uint64_t log_base;
 
         hdev->log_size = vhost_get_log_size(hdev);
-        hdev->log = vhost_log_get(hdev->log_size);
+        hdev->log = vhost_log_get(hdev->log_size,
+                                  vhost_dev_log_is_shared(hdev));
         log_base = (uintptr_t)hdev->log->log;
-        r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_LOG_BASE,
-                                        hdev->log_size ? &log_base : NULL);
+        r = hdev->vhost_ops->vhost_set_log_base(hdev,
+                                                hdev->log_size ? log_base : 0,
+                                                hdev->log);
         if (r < 0) {
             r = -errno;
             goto fail_log;
diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c
index c419b1714..9671635e6 100644
--- a/hw/virtio/virtio-balloon.c
+++ b/hw/virtio/virtio-balloon.c
@@ -37,9 +37,11 @@
 static void balloon_page(void *addr, int deflate)
 {
 #if defined(__linux__)
-    if (!kvm_enabled() || kvm_has_sync_mmu())
+    if (!qemu_balloon_is_inhibited() && (!kvm_enabled() ||
+                                         kvm_has_sync_mmu())) {
         qemu_madvise(addr, TARGET_PAGE_SIZE,
                 deflate ? QEMU_MADV_WILLNEED : QEMU_MADV_DONTNEED);
+    }
 #endif
 }
 
diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c
index febda76b9..81c7cdd50 100644
--- a/hw/virtio/virtio-bus.c
+++ b/hw/virtio/virtio-bus.c
@@ -56,6 +56,9 @@ void virtio_bus_device_plugged(VirtIODevice *vdev, Error **errp)
     assert(vdc->get_features != NULL);
     vdev->host_features = vdc->get_features(vdev, vdev->host_features,
                                             errp);
+    if (klass->post_plugged != NULL) {
+        klass->post_plugged(qbus->parent, errp);
+    }
 }
 
 /* Reset the virtio_bus */
diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index c024161f5..94667e625 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -86,6 +86,129 @@ static void virtio_pci_save_config(DeviceState *d, QEMUFile *f)
         qemu_put_be16(f, vdev->config_vector);
 }
 
+static void virtio_pci_load_modern_queue_state(VirtIOPCIQueue *vq,
+                                               QEMUFile *f)
+{
+    vq->num = qemu_get_be16(f);
+    vq->enabled = qemu_get_be16(f);
+    vq->desc[0] = qemu_get_be32(f);
+    vq->desc[1] = qemu_get_be32(f);
+    vq->avail[0] = qemu_get_be32(f);
+    vq->avail[1] = qemu_get_be32(f);
+    vq->used[0] = qemu_get_be32(f);
+    vq->used[1] = qemu_get_be32(f);
+}
+
+static bool virtio_pci_has_extra_state(DeviceState *d)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA;
+}
+
+static int get_virtio_pci_modern_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIOPCIProxy *proxy = pv;
+    int i;
+
+    proxy->dfselect = qemu_get_be32(f);
+    proxy->gfselect = qemu_get_be32(f);
+    proxy->guest_features[0] = qemu_get_be32(f);
+    proxy->guest_features[1] = qemu_get_be32(f);
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        virtio_pci_load_modern_queue_state(&proxy->vqs[i], f);
+    }
+
+    return 0;
+}
+
+static void virtio_pci_save_modern_queue_state(VirtIOPCIQueue *vq,
+                                               QEMUFile *f)
+{
+    qemu_put_be16(f, vq->num);
+    qemu_put_be16(f, vq->enabled);
+    qemu_put_be32(f, vq->desc[0]);
+    qemu_put_be32(f, vq->desc[1]);
+    qemu_put_be32(f, vq->avail[0]);
+    qemu_put_be32(f, vq->avail[1]);
+    qemu_put_be32(f, vq->used[0]);
+    qemu_put_be32(f, vq->used[1]);
+}
+
+static void put_virtio_pci_modern_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIOPCIProxy *proxy = pv;
+    int i;
+
+    qemu_put_be32(f, proxy->dfselect);
+    qemu_put_be32(f, proxy->gfselect);
+    qemu_put_be32(f, proxy->guest_features[0]);
+    qemu_put_be32(f, proxy->guest_features[1]);
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        virtio_pci_save_modern_queue_state(&proxy->vqs[i], f);
+    }
+}
+
+static const VMStateInfo vmstate_info_virtio_pci_modern_state = {
+    .name = "virtqueue_state",
+    .get = get_virtio_pci_modern_state,
+    .put = put_virtio_pci_modern_state,
+};
+
+static bool virtio_pci_modern_state_needed(void *opaque)
+{
+    VirtIOPCIProxy *proxy = opaque;
+
+    return !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+}
+
+static const VMStateDescription vmstate_virtio_pci_modern_state = {
+    .name = "virtio_pci/modern_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_pci_modern_state_needed,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "modern_state",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &vmstate_info_virtio_pci_modern_state,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_virtio_pci = {
+    .name = "virtio_pci",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_virtio_pci_modern_state,
+        NULL
+    }
+};
+
+static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL);
+}
+
+static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1);
+}
+
 static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
 {
     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
@@ -133,6 +256,7 @@ static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)
     if (vector != VIRTIO_NO_VECTOR) {
         return msix_vector_use(&proxy->pci_dev, vector);
     }
+
     return 0;
 }
 
@@ -146,7 +270,10 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
     bool legacy = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_LEGACY);
     bool modern = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+    bool fast_mmio = kvm_ioeventfd_any_length_enabled();
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
     MemoryRegion *modern_mr = &proxy->notify.mr;
+    MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr;
     MemoryRegion *legacy_mr = &proxy->bar;
     hwaddr modern_addr = QEMU_VIRTIO_PCI_QUEUE_MEM_MULT *
                          virtio_get_queue_index(vq);
@@ -162,8 +289,17 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
         }
         virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
         if (modern) {
-            memory_region_add_eventfd(modern_mr, modern_addr, 2,
-                                      true, n, notifier);
+            if (fast_mmio) {
+                memory_region_add_eventfd(modern_mr, modern_addr, 0,
+                                          false, n, notifier);
+            } else {
+                memory_region_add_eventfd(modern_mr, modern_addr, 2,
+                                          false, n, notifier);
+            }
+            if (modern_pio) {
+                memory_region_add_eventfd(modern_notify_mr, 0, 2,
+                                              true, n, notifier);
+            }
         }
         if (legacy) {
             memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
@@ -171,8 +307,17 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
         }
     } else {
         if (modern) {
-            memory_region_del_eventfd(modern_mr, modern_addr, 2,
-                                      true, n, notifier);
+            if (fast_mmio) {
+                memory_region_del_eventfd(modern_mr, modern_addr, 0,
+                                          false, n, notifier);
+            } else {
+                memory_region_del_eventfd(modern_mr, modern_addr, 2,
+                                          false, n, notifier);
+            }
+            if (modern_pio) {
+                memory_region_del_eventfd(modern_notify_mr, 0, 2,
+                                          true, n, notifier);
+            }
         }
         if (legacy) {
             memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
@@ -590,7 +735,7 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
     int ret;
 
     if (irqfd->users == 0) {
-        ret = kvm_irqchip_add_msi_route(kvm_state, msg);
+        ret = kvm_irqchip_add_msi_route(kvm_state, msg, &proxy->pci_dev);
         if (ret < 0) {
             return ret;
         }
@@ -726,7 +871,8 @@ static int virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,
     if (proxy->vector_irqfd) {
         irqfd = &proxy->vector_irqfd[vector];
         if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
-            ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg);
+            ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg,
+                                               &proxy->pci_dev);
             if (ret < 0) {
                 return ret;
             }
@@ -1238,6 +1384,7 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr,
                        proxy->vqs[vdev->queue_sel].avail[0],
                        ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 |
                        proxy->vqs[vdev->queue_sel].used[0]);
+        proxy->vqs[vdev->queue_sel].enabled = 1;
         break;
     case VIRTIO_PCI_COMMON_Q_DESCLO:
         proxy->vqs[vdev->queue_sel].desc[0] = val;
@@ -1280,6 +1427,17 @@ static void virtio_pci_notify_write(void *opaque, hwaddr addr,
     }
 }
 
+static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr,
+                                        uint64_t val, unsigned size)
+{
+    VirtIODevice *vdev = opaque;
+    unsigned queue = val;
+
+    if (queue < VIRTIO_QUEUE_MAX) {
+        virtio_queue_notify(vdev, queue);
+    }
+}
+
 static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
@@ -1373,6 +1531,16 @@ static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy)
         },
         .endianness = DEVICE_LITTLE_ENDIAN,
     };
+    static const MemoryRegionOps notify_pio_ops = {
+        .read = virtio_pci_notify_read,
+        .write = virtio_pci_notify_write_pio,
+        .impl = {
+            .min_access_size = 1,
+            .max_access_size = 4,
+        },
+        .endianness = DEVICE_LITTLE_ENDIAN,
+    };
+
 
     memory_region_init_io(&proxy->common.mr, OBJECT(proxy),
                           &common_ops,
@@ -1397,30 +1565,60 @@ static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy)
                           virtio_bus_get_device(&proxy->bus),
                           "virtio-pci-notify",
                           proxy->notify.size);
+
+    memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy),
+                          &notify_pio_ops,
+                          virtio_bus_get_device(&proxy->bus),
+                          "virtio-pci-notify-pio",
+                          proxy->notify.size);
 }
 
 static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy,
                                          VirtIOPCIRegion *region,
-                                         struct virtio_pci_cap *cap)
+                                         struct virtio_pci_cap *cap,
+                                         MemoryRegion *mr,
+                                         uint8_t bar)
 {
-    memory_region_add_subregion(&proxy->modern_bar,
-                                region->offset,
-                                &region->mr);
+    memory_region_add_subregion(mr, region->offset, &region->mr);
 
     cap->cfg_type = region->type;
-    cap->bar = proxy->modern_mem_bar;
+    cap->bar = bar;
     cap->offset = cpu_to_le32(region->offset);
     cap->length = cpu_to_le32(region->size);
     virtio_pci_add_mem_cap(proxy, cap);
+
+}
+
+static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy,
+                                             VirtIOPCIRegion *region,
+                                             struct virtio_pci_cap *cap)
+{
+    virtio_pci_modern_region_map(proxy, region, cap,
+                                 &proxy->modern_bar, proxy->modern_mem_bar);
+}
+
+static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy,
+                                            VirtIOPCIRegion *region,
+                                            struct virtio_pci_cap *cap)
+{
+    virtio_pci_modern_region_map(proxy, region, cap,
+                                 &proxy->io_bar, proxy->modern_io_bar);
 }
 
-static void virtio_pci_modern_region_unmap(VirtIOPCIProxy *proxy,
-                                           VirtIOPCIRegion *region)
+static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy,
+                                               VirtIOPCIRegion *region)
 {
     memory_region_del_subregion(&proxy->modern_bar,
                                 &region->mr);
 }
 
+static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy,
+                                              VirtIOPCIRegion *region)
+{
+    memory_region_del_subregion(&proxy->io_bar,
+                                &region->mr);
+}
+
 /* This is called by virtio-bus just after the device is plugged. */
 static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
 {
@@ -1428,6 +1626,7 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
     VirtioBusState *bus = &proxy->bus;
     bool legacy = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_LEGACY);
     bool modern = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
     uint8_t *config;
     uint32_t size;
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
@@ -1466,16 +1665,31 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
             .cap.cap_len = sizeof cfg,
             .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG,
         };
-        struct virtio_pci_cfg_cap *cfg_mask;
+        struct virtio_pci_notify_cap notify_pio = {
+            .cap.cap_len = sizeof notify,
+            .notify_off_multiplier = cpu_to_le32(0x0),
+        };
 
-        /* TODO: add io access for speed */
+        struct virtio_pci_cfg_cap *cfg_mask;
 
         virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
         virtio_pci_modern_regions_init(proxy);
-        virtio_pci_modern_region_map(proxy, &proxy->common, &cap);
-        virtio_pci_modern_region_map(proxy, &proxy->isr, &cap);
-        virtio_pci_modern_region_map(proxy, &proxy->device, &cap);
-        virtio_pci_modern_region_map(proxy, &proxy->notify, &notify.cap);
+
+        virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->notify, &notify.cap);
+
+        if (modern_pio) {
+            memory_region_init(&proxy->io_bar, OBJECT(proxy),
+                               "virtio-pci-io", 0x4);
+
+            pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar,
+                             PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar);
+
+            virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio,
+                                            &notify_pio.cap);
+        }
 
         pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar,
                          PCI_BASE_ADDRESS_SPACE_MEMORY |
@@ -1491,12 +1705,17 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
         pci_set_long(cfg_mask->pci_cfg_data, ~0x0);
     }
 
-    if (proxy->nvectors &&
-        msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
-                                proxy->msix_bar)) {
-        error_report("unable to init msix vectors to %" PRIu32,
-                     proxy->nvectors);
-        proxy->nvectors = 0;
+    if (proxy->nvectors) {
+        int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
+                                          proxy->msix_bar);
+        if (err) {
+            /* Notice when a system that supports MSIx can't initialize it.  */
+            if (err != -ENOTSUP) {
+                error_report("unable to init msix vectors to %" PRIu32,
+                             proxy->nvectors);
+            }
+            proxy->nvectors = 0;
+        }
     }
 
     proxy->pci_dev.config_write = virtio_write_config;
@@ -1505,9 +1724,7 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
     if (legacy) {
         size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
             + virtio_bus_get_vdev_config_len(bus);
-        if (size & (size - 1)) {
-            size = 1 << qemu_fls(size);
-        }
+        size = pow2ceil(size);
 
         memory_region_init_io(&proxy->bar, OBJECT(proxy),
                               &virtio_pci_config_ops,
@@ -1528,14 +1745,18 @@ static void virtio_pci_device_unplugged(DeviceState *d)
 {
     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
     bool modern = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 
     virtio_pci_stop_ioeventfd(proxy);
 
     if (modern) {
-        virtio_pci_modern_region_unmap(proxy, &proxy->common);
-        virtio_pci_modern_region_unmap(proxy, &proxy->isr);
-        virtio_pci_modern_region_unmap(proxy, &proxy->device);
-        virtio_pci_modern_region_unmap(proxy, &proxy->notify);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->common);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->device);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify);
+        if (modern_pio) {
+            virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio);
+        }
     }
 }
 
@@ -1555,6 +1776,7 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
      */
     proxy->legacy_io_bar  = 0;
     proxy->msix_bar       = 1;
+    proxy->modern_io_bar  = 2;
     proxy->modern_mem_bar = 4;
 
     proxy->common.offset = 0x0;
@@ -1574,6 +1796,10 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
         QEMU_VIRTIO_PCI_QUEUE_MEM_MULT * VIRTIO_QUEUE_MAX;
     proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
 
+    proxy->notify_pio.offset = 0x0;
+    proxy->notify_pio.size = 0x4;
+    proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
+
     /* subclasses can enforce modern, so do this unconditionally */
     memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci",
                        2 * QEMU_VIRTIO_PCI_QUEUE_MEM_MULT *
@@ -1588,6 +1814,29 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
 
     address_space_init(&proxy->modern_as, &proxy->modern_cfg, "virtio-pci-cfg-as");
 
+    if (pci_is_express(pci_dev) && pci_bus_is_express(pci_dev->bus) &&
+        !pci_bus_is_root(pci_dev->bus)) {
+        int pos;
+
+        pos = pcie_endpoint_cap_init(pci_dev, 0);
+        assert(pos > 0);
+
+        pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0, PCI_PM_SIZEOF);
+        assert(pos > 0);
+
+        /*
+         * Indicates that this function complies with revision 1.2 of the
+         * PCI Power Management Interface Specification.
+         */
+        pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
+    } else {
+        /*
+         * make future invocations of pci_is_express() return false
+         * and pci_config_size() return PCI_CONFIG_SPACE_SIZE.
+         */
+        pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
+    }
+
     virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
     if (k->realize) {
         k->realize(proxy, errp);
@@ -1606,9 +1855,15 @@ static void virtio_pci_reset(DeviceState *qdev)
 {
     VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
     VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
+    int i;
+
     virtio_pci_stop_ioeventfd(proxy);
     virtio_bus_reset(bus);
     msix_unuse_all_vectors(&proxy->pci_dev);
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        proxy->vqs[i].enabled = 0;
+    }
 }
 
 static Property virtio_pci_properties[] = {
@@ -1618,13 +1873,34 @@ static Property virtio_pci_properties[] = {
                     VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT, false),
     DEFINE_PROP_BIT("disable-modern", VirtIOPCIProxy, flags,
                     VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT, true),
+    DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true),
+    DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false),
+    DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp)
+{
+    VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev);
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
+    PCIDevice *pci_dev = &proxy->pci_dev;
+
+    if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) &&
+        !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN)) {
+        pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
+    }
+
+    vpciklass->parent_dc_realize(qdev, errp);
+}
+
 static void virtio_pci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass);
 
     dc->props = virtio_pci_properties;
     k->realize = virtio_pci_realize;
@@ -1632,6 +1908,8 @@ static void virtio_pci_class_init(ObjectClass *klass, void *data)
     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     k->revision = VIRTIO_PCI_ABI_VERSION;
     k->class_id = PCI_CLASS_OTHERS;
+    vpciklass->parent_dc_realize = dc->realize;
+    dc->realize = virtio_pci_dc_realize;
     dc->reset = virtio_pci_reset;
 }
 
@@ -2009,10 +2287,6 @@ static const TypeInfo virtio_net_pci_info = {
 
 /* virtio-rng-pci */
 
-static Property virtio_rng_pci_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void virtio_rng_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VirtIORngPCI *vrng = VIRTIO_RNG_PCI(vpci_dev);
@@ -2039,7 +2313,6 @@ static void virtio_rng_pci_class_init(ObjectClass *klass, void *data)
 
     k->realize = virtio_rng_pci_realize;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
-    dc->props = virtio_rng_pci_properties;
 
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_RNG;
@@ -2136,14 +2409,6 @@ static void virtio_tablet_initfn(Object *obj)
                                 TYPE_VIRTIO_TABLET);
 }
 
-static void virtio_host_initfn(Object *obj)
-{
-    VirtIOInputHostPCI *dev = VIRTIO_INPUT_HOST_PCI(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_INPUT_HOST);
-}
-
 static const TypeInfo virtio_input_pci_info = {
     .name          = TYPE_VIRTIO_INPUT_PCI,
     .parent        = TYPE_VIRTIO_PCI,
@@ -2182,12 +2447,22 @@ static const TypeInfo virtio_tablet_pci_info = {
     .instance_init = virtio_tablet_initfn,
 };
 
+#ifdef CONFIG_LINUX
+static void virtio_host_initfn(Object *obj)
+{
+    VirtIOInputHostPCI *dev = VIRTIO_INPUT_HOST_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_INPUT_HOST);
+}
+
 static const TypeInfo virtio_host_pci_info = {
     .name          = TYPE_VIRTIO_INPUT_HOST_PCI,
     .parent        = TYPE_VIRTIO_INPUT_PCI,
     .instance_size = sizeof(VirtIOInputHostPCI),
     .instance_init = virtio_host_initfn,
 };
+#endif
 
 /* virtio-pci-bus */
 
@@ -2211,6 +2486,9 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
     k->load_config = virtio_pci_load_config;
     k->save_queue = virtio_pci_save_queue;
     k->load_queue = virtio_pci_load_queue;
+    k->save_extra_state = virtio_pci_save_extra_state;
+    k->load_extra_state = virtio_pci_load_extra_state;
+    k->has_extra_state = virtio_pci_has_extra_state;
     k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
     k->set_host_notifier = virtio_pci_set_host_notifier;
     k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
@@ -2235,7 +2513,9 @@ static void virtio_pci_register_types(void)
     type_register_static(&virtio_keyboard_pci_info);
     type_register_static(&virtio_mouse_pci_info);
     type_register_static(&virtio_tablet_pci_info);
+#ifdef CONFIG_LINUX
     type_register_static(&virtio_host_pci_info);
+#endif
     type_register_static(&virtio_pci_bus_info);
     type_register_static(&virtio_pci_info);
 #ifdef CONFIG_VIRTFS
diff --git a/hw/virtio/virtio-pci.h b/hw/virtio/virtio-pci.h
index b6c442f52..c8f9cfdfc 100644
--- a/hw/virtio/virtio-pci.h
+++ b/hw/virtio/virtio-pci.h
@@ -59,21 +59,35 @@ typedef struct VirtioBusClass VirtioPCIBusClass;
 #define VIRTIO_PCI_BUS_CLASS(klass) \
         OBJECT_CLASS_CHECK(VirtioPCIBusClass, klass, TYPE_VIRTIO_PCI_BUS)
 
+enum {
+    VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT,
+    VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT,
+    VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT,
+    VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT,
+    VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT,
+    VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT,
+    VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT,
+};
+
 /* Need to activate work-arounds for buggy guests at vmstate load. */
-#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT  0
 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION \
     (1 << VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT)
 
 /* Performance improves when virtqueue kick processing is decoupled from the
  * vcpu thread using ioeventfd for some devices. */
-#define VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT 1
 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD   (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT)
 
 /* virtio version flags */
-#define VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT 2
-#define VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT 3
 #define VIRTIO_PCI_FLAG_DISABLE_LEGACY (1 << VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT)
 #define VIRTIO_PCI_FLAG_DISABLE_MODERN (1 << VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT)
+#define VIRTIO_PCI_FLAG_DISABLE_PCIE (1 << VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT)
+
+/* migrate extra state */
+#define VIRTIO_PCI_FLAG_MIGRATE_EXTRA (1 << VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT)
+
+/* have pio notification for modern device ? */
+#define VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY \
+    (1 << VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT)
 
 typedef struct {
     MSIMessage msg;
@@ -94,6 +108,7 @@ typedef struct {
 
 typedef struct VirtioPCIClass {
     PCIDeviceClass parent_class;
+    DeviceRealize parent_dc_realize;
     void (*realize)(VirtIOPCIProxy *vpci_dev, Error **errp);
 } VirtioPCIClass;
 
@@ -104,6 +119,14 @@ typedef struct VirtIOPCIRegion {
     uint32_t type;
 } VirtIOPCIRegion;
 
+typedef struct VirtIOPCIQueue {
+  uint16_t num;
+  bool enabled;
+  uint32_t desc[2];
+  uint32_t avail[2];
+  uint32_t used[2];
+} VirtIOPCIQueue;
+
 struct VirtIOPCIProxy {
     PCIDevice pci_dev;
     MemoryRegion bar;
@@ -111,11 +134,14 @@ struct VirtIOPCIProxy {
     VirtIOPCIRegion isr;
     VirtIOPCIRegion device;
     VirtIOPCIRegion notify;
+    VirtIOPCIRegion notify_pio;
     MemoryRegion modern_bar;
+    MemoryRegion io_bar;
     MemoryRegion modern_cfg;
     AddressSpace modern_as;
     uint32_t legacy_io_bar;
     uint32_t msix_bar;
+    uint32_t modern_io_bar;
     uint32_t modern_mem_bar;
     int config_cap;
     uint32_t flags;
@@ -124,13 +150,7 @@ struct VirtIOPCIProxy {
     uint32_t dfselect;
     uint32_t gfselect;
     uint32_t guest_features[2];
-    struct {
-        uint16_t num;
-        bool enabled;
-        uint32_t desc[2];
-        uint32_t avail[2];
-        uint32_t used[2];
-    } vqs[VIRTIO_QUEUE_MAX];
+    VirtIOPCIQueue vqs[VIRTIO_QUEUE_MAX];
 
     bool ioeventfd_disabled;
     bool ioeventfd_started;
@@ -267,6 +287,8 @@ struct VirtIOInputHIDPCI {
     VirtIOInputHID vdev;
 };
 
+#ifdef CONFIG_LINUX
+
 #define TYPE_VIRTIO_INPUT_HOST_PCI "virtio-input-host-pci"
 #define VIRTIO_INPUT_HOST_PCI(obj) \
         OBJECT_CHECK(VirtIOInputHostPCI, (obj), TYPE_VIRTIO_INPUT_HOST_PCI)
@@ -276,6 +298,8 @@ struct VirtIOInputHostPCI {
     VirtIOInputHost vdev;
 };
 
+#endif
+
 /*
  * virtio-gpu-pci: This extends VirtioPCIProxy.
  */
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index d24f77551..1edef5945 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -60,6 +60,7 @@ typedef struct VRingUsed
 typedef struct VRing
 {
     unsigned int num;
+    unsigned int num_default;
     unsigned int align;
     hwaddr desc;
     hwaddr avail;
@@ -447,28 +448,59 @@ int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
     return in_bytes <= in_total && out_bytes <= out_total;
 }
 
-void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
-    size_t num_sg, int is_write)
+static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
+                                unsigned int *num_sg, unsigned int max_size,
+                                int is_write)
 {
     unsigned int i;
     hwaddr len;
 
-    if (num_sg > VIRTQUEUE_MAX_SIZE) {
-        error_report("virtio: map attempt out of bounds: %zd > %d",
-                     num_sg, VIRTQUEUE_MAX_SIZE);
-        exit(1);
-    }
+    /* Note: this function MUST validate input, some callers
+     * are passing in num_sg values received over the network.
+     */
+    /* TODO: teach all callers that this can fail, and return failure instead
+     * of asserting here.
+     * When we do, we might be able to re-enable NDEBUG below.
+     */
+#ifdef NDEBUG
+#error building with NDEBUG is not supported
+#endif
+    assert(*num_sg <= max_size);
 
-    for (i = 0; i < num_sg; i++) {
+    for (i = 0; i < *num_sg; i++) {
         len = sg[i].iov_len;
         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
-        if (sg[i].iov_base == NULL || len != sg[i].iov_len) {
+        if (!sg[i].iov_base) {
             error_report("virtio: error trying to map MMIO memory");
             exit(1);
         }
+        if (len == sg[i].iov_len) {
+            continue;
+        }
+        if (*num_sg >= max_size) {
+            error_report("virtio: memory split makes iovec too large");
+            exit(1);
+        }
+        memmove(sg + i + 1, sg + i, sizeof(*sg) * (*num_sg - i));
+        memmove(addr + i + 1, addr + i, sizeof(*addr) * (*num_sg - i));
+        assert(len < sg[i + 1].iov_len);
+        sg[i].iov_len = len;
+        addr[i + 1] += len;
+        sg[i + 1].iov_len -= len;
+        ++*num_sg;
     }
 }
 
+void virtqueue_map(VirtQueueElement *elem)
+{
+    virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
+                        MIN(ARRAY_SIZE(elem->in_sg), ARRAY_SIZE(elem->in_addr)),
+                        1);
+    virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
+                        MIN(ARRAY_SIZE(elem->out_sg), ARRAY_SIZE(elem->out_addr)),
+                        0);
+}
+
 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
 {
     unsigned int i, head, max;
@@ -530,8 +562,7 @@ int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
     } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
 
     /* Now map what we have collected */
-    virtqueue_map_sg(elem->in_sg, elem->in_addr, elem->in_num, 1);
-    virtqueue_map_sg(elem->out_sg, elem->out_addr, elem->out_num, 0);
+    virtqueue_map(elem);
 
     elem->index = head;
 
@@ -646,6 +677,7 @@ void virtio_reset(void *opaque)
         vdev->vq[i].signalled_used = 0;
         vdev->vq[i].signalled_used_valid = false;
         vdev->vq[i].notification = true;
+        vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
     }
 }
 
@@ -977,6 +1009,7 @@ VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
         abort();
 
     vdev->vq[i].vring.num = queue_size;
+    vdev->vq[i].vring.num_default = queue_size;
     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
     vdev->vq[i].handle_output = handle_output;
 
@@ -990,6 +1023,7 @@ void virtio_del_queue(VirtIODevice *vdev, int n)
     }
 
     vdev->vq[n].vring.num = 0;
+    vdev->vq[n].vring.num_default = 0;
 }
 
 void virtio_irq(VirtQueue *vq)
@@ -1069,6 +1103,29 @@ static bool virtio_virtqueue_needed(void *opaque)
     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
 }
 
+static bool virtio_ringsize_needed(void *opaque)
+{
+    VirtIODevice *vdev = opaque;
+    int i;
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool virtio_extra_state_needed(void *opaque)
+{
+    VirtIODevice *vdev = opaque;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    return k->has_extra_state &&
+        k->has_extra_state(qbus->parent);
+}
+
 static void put_virtqueue_state(QEMUFile *f, void *pv, size_t size)
 {
     VirtIODevice *vdev = pv;
@@ -1117,6 +1174,99 @@ static const VMStateDescription vmstate_virtio_virtqueues = {
     }
 };
 
+static void put_ringsize_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    int i;
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        qemu_put_be32(f, vdev->vq[i].vring.num_default);
+    }
+}
+
+static int get_ringsize_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    int i;
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        vdev->vq[i].vring.num_default = qemu_get_be32(f);
+    }
+    return 0;
+}
+
+static VMStateInfo vmstate_info_ringsize = {
+    .name = "ringsize_state",
+    .get = get_ringsize_state,
+    .put = put_ringsize_state,
+};
+
+static const VMStateDescription vmstate_virtio_ringsize = {
+    .name = "virtio/ringsize",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_ringsize_needed,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "ringsize",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &vmstate_info_ringsize,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static int get_extra_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    if (!k->load_extra_state) {
+        return -1;
+    } else {
+        return k->load_extra_state(qbus->parent, f);
+    }
+}
+
+static void put_extra_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    k->save_extra_state(qbus->parent, f);
+}
+
+static const VMStateInfo vmstate_info_extra_state = {
+    .name = "virtqueue_extra_state",
+    .get = get_extra_state,
+    .put = put_extra_state,
+};
+
+static const VMStateDescription vmstate_virtio_extra_state = {
+    .name = "virtio/extra_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_extra_state_needed,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "extra_state",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &vmstate_info_extra_state,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_virtio_device_endian = {
     .name = "virtio/device_endian",
     .version_id = 1,
@@ -1151,6 +1301,8 @@ static const VMStateDescription vmstate_virtio = {
         &vmstate_virtio_device_endian,
         &vmstate_virtio_64bit_features,
         &vmstate_virtio_virtqueues,
+        &vmstate_virtio_ringsize,
+        &vmstate_virtio_extra_state,
         NULL
     }
 };
@@ -1473,7 +1625,7 @@ hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
 {
     return offsetof(VRingAvail, ring) +
-        sizeof(uint64_t) * vdev->vq[n].vring.num;
+        sizeof(uint16_t) * vdev->vq[n].vring.num;
 }
 
 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
diff --git a/hw/watchdog/wdt_i6300esb.c b/hw/watchdog/wdt_i6300esb.c
index cfa2b1be1..a91c8fdad 100644
--- a/hw/watchdog/wdt_i6300esb.c
+++ b/hw/watchdog/wdt_i6300esb.c
@@ -129,14 +129,9 @@ static void i6300esb_restart_timer(I6300State *d, int stage)
     else
         timeout <<= 5;
 
-    /* Get the timeout in units of ticks_per_sec.
-     *
-     * ticks_per_sec is typically 10^9 == 0x3B9ACA00 (30 bits), with
-     * 20 bits of user supplied preload, and 15 bits of scale, the
-     * multiply here can exceed 64-bits, before we divide by 33MHz, so
-     * we use a higher-precision intermediate result.
-     */
-    timeout = muldiv64(get_ticks_per_sec(), timeout, 33000000);
+    /* Get the timeout in nanoseconds. */
+
+    timeout = timeout * 30; /* on a PCI bus, 1 tick is 30 ns*/
 
     i6300esb_debug("stage %d, timeout %" PRIi64 "\n", d->stage, timeout);
 
diff --git a/hw/xen/Makefile.objs b/hw/xen/Makefile.objs
index a0ca0aa3d..a9ad7e70f 100644
--- a/hw/xen/Makefile.objs
+++ b/hw/xen/Makefile.objs
@@ -3,3 +3,4 @@ common-obj-$(CONFIG_XEN_BACKEND) += xen_backend.o xen_devconfig.o
 
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen-host-pci-device.o
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_msi.o
+obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_msi.o xen_pt_graphics.o
diff --git a/hw/xen/xen-host-pci-device.c b/hw/xen/xen-host-pci-device.c
index 743b37b99..7d8a0237c 100644
--- a/hw/xen/xen-host-pci-device.c
+++ b/hw/xen/xen-host-pci-device.c
@@ -40,7 +40,7 @@ static int xen_host_pci_sysfs_path(const XenHostPCIDevice *d,
                   d->domain, d->bus, d->dev, d->func, name);
 
     if (rc >= size || rc < 0) {
-        /* The ouput is truncated or an other error is encountered */
+        /* The output is truncated, or some other error was encountered */
         return -ENODEV;
     }
     return 0;
@@ -376,6 +376,11 @@ int xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
         goto error;
     }
     d->irq = v;
+    rc = xen_host_pci_get_hex_value(d, "class", &v);
+    if (rc) {
+        goto error;
+    }
+    d->class_code = v;
     d->is_virtfn = xen_host_pci_dev_is_virtfn(d);
 
     return 0;
@@ -387,6 +392,11 @@ error:
     return rc;
 }
 
+bool xen_host_pci_device_closed(XenHostPCIDevice *d)
+{
+    return d->config_fd == -1;
+}
+
 void xen_host_pci_device_put(XenHostPCIDevice *d)
 {
     if (d->config_fd >= 0) {
diff --git a/hw/xen/xen-host-pci-device.h b/hw/xen/xen-host-pci-device.h
index c2486f0c1..3d44e044f 100644
--- a/hw/xen/xen-host-pci-device.h
+++ b/hw/xen/xen-host-pci-device.h
@@ -25,6 +25,7 @@ typedef struct XenHostPCIDevice {
 
     uint16_t vendor_id;
     uint16_t device_id;
+    uint32_t class_code;
     int irq;
 
     XenHostPCIIORegion io_regions[PCI_NUM_REGIONS - 1];
@@ -38,6 +39,7 @@ typedef struct XenHostPCIDevice {
 int xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
                             uint8_t bus, uint8_t dev, uint8_t func);
 void xen_host_pci_device_put(XenHostPCIDevice *pci_dev);
+bool xen_host_pci_device_closed(XenHostPCIDevice *d);
 
 int xen_host_pci_get_byte(XenHostPCIDevice *d, int pos, uint8_t *p);
 int xen_host_pci_get_word(XenHostPCIDevice *d, int pos, uint16_t *p);
diff --git a/hw/xen/xen_pt.c b/hw/xen/xen_pt.c
index ed5fcaec0..aa9628823 100644
--- a/hw/xen/xen_pt.c
+++ b/hw/xen/xen_pt.c
@@ -56,6 +56,7 @@
 
 #include "hw/pci/pci.h"
 #include "hw/xen/xen.h"
+#include "hw/i386/pc.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_pt.h"
 #include "qemu/range.h"
@@ -378,7 +379,7 @@ static void xen_pt_pci_write_config(PCIDevice *d, uint32_t addr,
         }
     }
 
-    /* need to shift back before passing them to xen_host_pci_device */
+    /* need to shift back before passing them to xen_host_pci_set_block. */
     val >>= (addr & 3) << 3;
 
     memory_region_transaction_commit();
@@ -406,7 +407,7 @@ out:
                                     (uint8_t *)&val + index, len);
 
         if (rc < 0) {
-            XEN_PT_ERR(d, "pci_write_block failed. return value: %d.\n", rc);
+            XEN_PT_ERR(d, "xen_host_pci_set_block failed. return value: %d.\n", rc);
         }
     }
 }
@@ -502,6 +503,7 @@ static int xen_pt_register_regions(XenPCIPassthroughState *s, uint16_t *cmd)
                    d->rom.size, d->rom.base_addr);
     }
 
+    xen_pt_register_vga_regions(d);
     return 0;
 }
 
@@ -683,13 +685,86 @@ static const MemoryListener xen_pt_io_listener = {
     .priority = 10,
 };
 
+static void
+xen_igd_passthrough_isa_bridge_create(XenPCIPassthroughState *s,
+                                      XenHostPCIDevice *dev)
+{
+    uint16_t gpu_dev_id;
+    PCIDevice *d = &s->dev;
+
+    gpu_dev_id = dev->device_id;
+    igd_passthrough_isa_bridge_create(d->bus, gpu_dev_id);
+}
+
+/* destroy. */
+static void xen_pt_destroy(PCIDevice *d) {
+
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
+    XenHostPCIDevice *host_dev = &s->real_device;
+    uint8_t machine_irq = s->machine_irq;
+    uint8_t intx;
+    int rc;
+
+    if (machine_irq && !xen_host_pci_device_closed(&s->real_device)) {
+        intx = xen_pt_pci_intx(s);
+        rc = xc_domain_unbind_pt_irq(xen_xc, xen_domid, machine_irq,
+                                     PT_IRQ_TYPE_PCI,
+                                     pci_bus_num(d->bus),
+                                     PCI_SLOT(s->dev.devfn),
+                                     intx,
+                                     0 /* isa_irq */);
+        if (rc < 0) {
+            XEN_PT_ERR(d, "unbinding of interrupt INT%c failed."
+                       " (machine irq: %i, err: %d)"
+                       " But bravely continuing on..\n",
+                       'a' + intx, machine_irq, errno);
+        }
+    }
+
+    /* N.B. xen_pt_config_delete takes care of freeing them. */
+    if (s->msi) {
+        xen_pt_msi_disable(s);
+    }
+    if (s->msix) {
+        xen_pt_msix_disable(s);
+    }
+
+    if (machine_irq) {
+        xen_pt_mapped_machine_irq[machine_irq]--;
+
+        if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
+            rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq);
+
+            if (rc < 0) {
+                XEN_PT_ERR(d, "unmapping of interrupt %i failed. (err: %d)"
+                           " But bravely continuing on..\n",
+                           machine_irq, errno);
+            }
+        }
+        s->machine_irq = 0;
+    }
+
+    /* delete all emulated config registers */
+    xen_pt_config_delete(s);
+
+    xen_pt_unregister_vga_regions(host_dev);
+
+    if (s->listener_set) {
+        memory_listener_unregister(&s->memory_listener);
+        memory_listener_unregister(&s->io_listener);
+        s->listener_set = false;
+    }
+    if (!xen_host_pci_device_closed(&s->real_device)) {
+        xen_host_pci_device_put(&s->real_device);
+    }
+}
 /* init */
 
 static int xen_pt_initfn(PCIDevice *d)
 {
     XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
     int rc = 0;
-    uint8_t machine_irq = 0;
+    uint8_t machine_irq = 0, scratch;
     uint16_t cmd = 0;
     int pirq = XEN_PT_UNASSIGNED_PIRQ;
 
@@ -715,27 +790,48 @@ static int xen_pt_initfn(PCIDevice *d)
     }
 
     /* Initialize virtualized PCI configuration (Extended 256 Bytes) */
-    if (xen_host_pci_get_block(&s->real_device, 0, d->config,
-                               PCI_CONFIG_SPACE_SIZE) == -1) {
-        xen_host_pci_device_put(&s->real_device);
-        return -1;
-    }
+    memset(d->config, 0, PCI_CONFIG_SPACE_SIZE);
 
     s->memory_listener = xen_pt_memory_listener;
     s->io_listener = xen_pt_io_listener;
 
+    /* Setup VGA bios for passthrough GFX */
+    if ((s->real_device.domain == 0) && (s->real_device.bus == 0) &&
+        (s->real_device.dev == 2) && (s->real_device.func == 0)) {
+        if (!is_igd_vga_passthrough(&s->real_device)) {
+            XEN_PT_ERR(d, "Need to enable igd-passthru if you're trying"
+                       " to passthrough IGD GFX.\n");
+            xen_host_pci_device_put(&s->real_device);
+            return -1;
+        }
+
+        if (xen_pt_setup_vga(s, &s->real_device) < 0) {
+            XEN_PT_ERR(d, "Setup VGA BIOS of passthrough GFX failed!\n");
+            xen_host_pci_device_put(&s->real_device);
+            return -1;
+        }
+
+        /* Register ISA bridge for passthrough GFX. */
+        xen_igd_passthrough_isa_bridge_create(s, &s->real_device);
+    }
+
     /* Handle real device's MMIO/PIO BARs */
     xen_pt_register_regions(s, &cmd);
 
     /* reinitialize each config register to be emulated */
-    if (xen_pt_config_init(s)) {
+    rc = xen_pt_config_init(s);
+    if (rc) {
         XEN_PT_ERR(d, "PCI Config space initialisation failed.\n");
-        xen_host_pci_device_put(&s->real_device);
-        return -1;
+        goto err_out;
     }
 
     /* Bind interrupt */
-    if (!s->dev.config[PCI_INTERRUPT_PIN]) {
+    rc = xen_host_pci_get_byte(&s->real_device, PCI_INTERRUPT_PIN, &scratch);
+    if (rc) {
+        XEN_PT_ERR(d, "Failed to read PCI_INTERRUPT_PIN! (rc:%d)\n", rc);
+        goto err_out;
+    }
+    if (!scratch) {
         XEN_PT_LOG(d, "no pin interrupt\n");
         goto out;
     }
@@ -785,69 +881,41 @@ static int xen_pt_initfn(PCIDevice *d)
 
 out:
     if (cmd) {
-        xen_host_pci_set_word(&s->real_device, PCI_COMMAND,
-                              pci_get_word(d->config + PCI_COMMAND) | cmd);
+        uint16_t val;
+
+        rc = xen_host_pci_get_word(&s->real_device, PCI_COMMAND, &val);
+        if (rc) {
+            XEN_PT_ERR(d, "Failed to read PCI_COMMAND! (rc: %d)\n", rc);
+            goto err_out;
+        } else {
+            val |= cmd;
+            rc = xen_host_pci_set_word(&s->real_device, PCI_COMMAND, val);
+            if (rc) {
+                XEN_PT_ERR(d, "Failed to write PCI_COMMAND val=0x%x!(rc: %d)\n",
+                           val, rc);
+                goto err_out;
+            }
+        }
     }
 
     memory_listener_register(&s->memory_listener, &s->dev.bus_master_as);
     memory_listener_register(&s->io_listener, &address_space_io);
+    s->listener_set = true;
     XEN_PT_LOG(d,
                "Real physical device %02x:%02x.%d registered successfully!\n",
                s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function);
 
     return 0;
+
+err_out:
+    xen_pt_destroy(d);
+    assert(rc);
+    return rc;
 }
 
 static void xen_pt_unregister_device(PCIDevice *d)
 {
-    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
-    uint8_t machine_irq = s->machine_irq;
-    uint8_t intx = xen_pt_pci_intx(s);
-    int rc;
-
-    if (machine_irq) {
-        rc = xc_domain_unbind_pt_irq(xen_xc, xen_domid, machine_irq,
-                                     PT_IRQ_TYPE_PCI,
-                                     pci_bus_num(d->bus),
-                                     PCI_SLOT(s->dev.devfn),
-                                     intx,
-                                     0 /* isa_irq */);
-        if (rc < 0) {
-            XEN_PT_ERR(d, "unbinding of interrupt INT%c failed."
-                       " (machine irq: %i, err: %d)"
-                       " But bravely continuing on..\n",
-                       'a' + intx, machine_irq, errno);
-        }
-    }
-
-    if (s->msi) {
-        xen_pt_msi_disable(s);
-    }
-    if (s->msix) {
-        xen_pt_msix_disable(s);
-    }
-
-    if (machine_irq) {
-        xen_pt_mapped_machine_irq[machine_irq]--;
-
-        if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
-            rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq);
-
-            if (rc < 0) {
-                XEN_PT_ERR(d, "unmapping of interrupt %i failed. (err: %d)"
-                           " But bravely continuing on..\n",
-                           machine_irq, errno);
-            }
-        }
-    }
-
-    /* delete all emulated config registers */
-    xen_pt_config_delete(s);
-
-    memory_listener_unregister(&s->memory_listener);
-    memory_listener_unregister(&s->io_listener);
-
-    xen_host_pci_device_put(&s->real_device);
+    xen_pt_destroy(d);
 }
 
 static Property xen_pci_passthrough_properties[] = {
@@ -870,10 +938,18 @@ static void xen_pci_passthrough_class_init(ObjectClass *klass, void *data)
     dc->props = xen_pci_passthrough_properties;
 };
 
+static void xen_pci_passthrough_finalize(Object *obj)
+{
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(obj);
+
+    xen_pt_msix_delete(s);
+}
+
 static const TypeInfo xen_pci_passthrough_info = {
     .name = TYPE_XEN_PT_DEVICE,
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(XenPCIPassthroughState),
+    .instance_finalize = xen_pci_passthrough_finalize,
     .class_init = xen_pci_passthrough_class_init,
 };
 
diff --git a/hw/xen/xen_pt.h b/hw/xen/xen_pt.h
index 393f36ccb..c54528008 100644
--- a/hw/xen/xen_pt.h
+++ b/hw/xen/xen_pt.h
@@ -40,6 +40,9 @@ typedef struct XenPCIPassthroughState XenPCIPassthroughState;
 #define XEN_PT_DEVICE(obj) \
     OBJECT_CHECK(XenPCIPassthroughState, (obj), TYPE_XEN_PT_DEVICE)
 
+uint32_t igd_read_opregion(XenPCIPassthroughState *s);
+void igd_write_opregion(XenPCIPassthroughState *s, uint32_t val);
+
 /* function type for config reg */
 typedef int (*xen_pt_conf_reg_init)
     (XenPCIPassthroughState *, XenPTRegInfo *, uint32_t real_offset,
@@ -66,8 +69,9 @@ typedef int (*xen_pt_conf_byte_read)
 #define XEN_PT_BAR_ALLF 0xFFFFFFFF
 #define XEN_PT_BAR_UNMAPPED (-1)
 
-#define PCI_CAP_MAX 48
+#define XEN_PCI_CAP_MAX 48
 
+#define XEN_PCI_INTEL_OPREGION 0xfc
 
 typedef enum {
     XEN_PT_GRP_TYPE_HARDWIRED = 0,  /* 0 Hardwired reg group */
@@ -134,7 +138,11 @@ struct XenPTRegInfo {
 struct XenPTReg {
     QLIST_ENTRY(XenPTReg) entries;
     XenPTRegInfo *reg;
-    uint32_t data; /* emulated value */
+    union {
+        uint8_t *byte;
+        uint16_t *half_word;
+        uint32_t *word;
+    } ptr; /* pointer to dev.config. */
 };
 
 typedef const struct XenPTRegGroupInfo XenPTRegGroupInfo;
@@ -217,6 +225,7 @@ struct XenPCIPassthroughState {
 
     MemoryListener memory_listener;
     MemoryListener io_listener;
+    bool listener_set;
 };
 
 int xen_pt_config_init(XenPCIPassthroughState *s);
@@ -282,6 +291,7 @@ static inline uint8_t xen_pt_pci_intx(XenPCIPassthroughState *s)
                    " value=%i, acceptable range is 1 - 4\n", r_val);
         r_val = 0;
     } else {
+        /* Note that if s.real_device.config_fd is closed we make 0xff. */
         r_val -= 1;
     }
 
@@ -289,13 +299,13 @@ static inline uint8_t xen_pt_pci_intx(XenPCIPassthroughState *s)
 }
 
 /* MSI/MSI-X */
-int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool en);
 int xen_pt_msi_setup(XenPCIPassthroughState *s);
 int xen_pt_msi_update(XenPCIPassthroughState *d);
 void xen_pt_msi_disable(XenPCIPassthroughState *s);
 
 int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base);
 void xen_pt_msix_delete(XenPCIPassthroughState *s);
+void xen_pt_msix_unmap(XenPCIPassthroughState *s);
 int xen_pt_msix_update(XenPCIPassthroughState *s);
 int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index);
 void xen_pt_msix_disable(XenPCIPassthroughState *s);
@@ -305,5 +315,18 @@ static inline bool xen_pt_has_msix_mapping(XenPCIPassthroughState *s, int bar)
     return s->msix && s->msix->bar_index == bar;
 }
 
-
+extern void *pci_assign_dev_load_option_rom(PCIDevice *dev,
+                                            struct Object *owner, int *size,
+                                            unsigned int domain,
+                                            unsigned int bus, unsigned int slot,
+                                            unsigned int function);
+extern bool has_igd_gfx_passthru;
+static inline bool is_igd_vga_passthrough(XenHostPCIDevice *dev)
+{
+    return (has_igd_gfx_passthru
+            && ((dev->class_code >> 0x8) == PCI_CLASS_DISPLAY_VGA));
+}
+int xen_pt_register_vga_regions(XenHostPCIDevice *dev);
+int xen_pt_unregister_vga_regions(XenHostPCIDevice *dev);
+int xen_pt_setup_vga(XenPCIPassthroughState *s, XenHostPCIDevice *dev);
 #endif /* !XEN_PT_H */
diff --git a/hw/xen/xen_pt_config_init.c b/hw/xen/xen_pt_config_init.c
index dd37be38a..3d8686d55 100644
--- a/hw/xen/xen_pt_config_init.c
+++ b/hw/xen/xen_pt_config_init.c
@@ -128,10 +128,11 @@ static int xen_pt_byte_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint8_t valid_emu_mask = 0;
+    uint8_t *data = cfg_entry->ptr.byte;
 
     /* emulate byte register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -140,10 +141,11 @@ static int xen_pt_word_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t valid_emu_mask = 0;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* emulate word register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -152,10 +154,11 @@ static int xen_pt_long_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t valid_emu_mask = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* emulate long register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -169,10 +172,11 @@ static int xen_pt_byte_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint8_t writable_mask = 0;
     uint8_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint8_t *data = cfg_entry->ptr.byte;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -186,10 +190,11 @@ static int xen_pt_word_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -203,10 +208,11 @@ static int xen_pt_long_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
     uint32_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -255,7 +261,7 @@ static int xen_pt_status_reg_init(XenPCIPassthroughState *s,
         reg_entry = xen_pt_find_reg(reg_grp_entry, PCI_CAPABILITY_LIST);
         if (reg_entry) {
             /* check Capabilities Pointer register */
-            if (reg_entry->data) {
+            if (*reg_entry->ptr.half_word) {
                 reg_field |= PCI_STATUS_CAP_LIST;
             } else {
                 reg_field &= ~PCI_STATUS_CAP_LIST;
@@ -301,10 +307,11 @@ static int xen_pt_cmd_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     if (*val & PCI_COMMAND_INTX_DISABLE) {
@@ -447,7 +454,7 @@ static int xen_pt_bar_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 
     /* emulate BAR */
     valid_emu_mask = bar_emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *cfg_entry->ptr.word, ~valid_emu_mask);
 
     return 0;
 }
@@ -464,6 +471,7 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     uint32_t bar_ro_mask = 0;
     uint32_t r_size = 0;
     int index = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     index = xen_pt_bar_offset_to_index(reg->offset);
     if (index < 0 || index >= PCI_NUM_REGIONS) {
@@ -500,7 +508,7 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 
     /* modify emulate register */
     writable_mask = bar_emu_mask & ~bar_ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* check whether we need to update the virtual region address or not */
     switch (s->bases[index].bar_flag) {
@@ -533,6 +541,7 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
     uint32_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
     pcibus_t r_size = 0;
     uint32_t bar_ro_mask = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     r_size = d->io_regions[PCI_ROM_SLOT].size;
     base = &s->bases[PCI_ROM_SLOT];
@@ -544,7 +553,7 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = ~bar_ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -552,6 +561,22 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
+static int xen_pt_intel_opregion_read(XenPCIPassthroughState *s,
+                                      XenPTReg *cfg_entry,
+                                      uint32_t *value, uint32_t valid_mask)
+{
+    *value = igd_read_opregion(s);
+    return 0;
+}
+
+static int xen_pt_intel_opregion_write(XenPCIPassthroughState *s,
+                                       XenPTReg *cfg_entry, uint32_t *value,
+                                       uint32_t dev_value, uint32_t valid_mask)
+{
+    igd_write_opregion(s, *value);
+    return 0;
+}
+
 /* Header Type0 reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_header0[] = {
     /* Vendor ID reg */
@@ -800,15 +825,21 @@ static XenPTRegInfo xen_pt_emu_reg_vendor[] = {
 static inline uint8_t get_capability_version(XenPCIPassthroughState *s,
                                              uint32_t offset)
 {
-    uint8_t flags = pci_get_byte(s->dev.config + offset + PCI_EXP_FLAGS);
-    return flags & PCI_EXP_FLAGS_VERS;
+    uint8_t flag;
+    if (xen_host_pci_get_byte(&s->real_device, offset + PCI_EXP_FLAGS, &flag)) {
+        return 0;
+    }
+    return flag & PCI_EXP_FLAGS_VERS;
 }
 
 static inline uint8_t get_device_type(XenPCIPassthroughState *s,
                                       uint32_t offset)
 {
-    uint8_t flags = pci_get_byte(s->dev.config + offset + PCI_EXP_FLAGS);
-    return (flags & PCI_EXP_FLAGS_TYPE) >> 4;
+    uint8_t flag;
+    if (xen_host_pci_get_byte(&s->real_device, offset + PCI_EXP_FLAGS, &flag)) {
+        return 0;
+    }
+    return (flag & PCI_EXP_FLAGS_TYPE) >> 4;
 }
 
 /* initialize Link Control register */
@@ -857,8 +888,14 @@ static int xen_pt_linkctrl2_reg_init(XenPCIPassthroughState *s,
         reg_field = XEN_PT_INVALID_REG;
     } else {
         /* set Supported Link Speed */
-        uint8_t lnkcap = pci_get_byte(s->dev.config + real_offset - reg->offset
-                                      + PCI_EXP_LNKCAP);
+        uint8_t lnkcap;
+        int rc;
+        rc = xen_host_pci_get_byte(&s->real_device,
+                                   real_offset - reg->offset + PCI_EXP_LNKCAP,
+                                   &lnkcap);
+        if (rc) {
+            return rc;
+        }
         reg_field |= PCI_EXP_LNKCAP_SLS & lnkcap;
     }
 
@@ -971,10 +1008,11 @@ static int xen_pt_pmcsr_reg_write(XenPCIPassthroughState *s,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~PCI_PM_CTRL_PME_STATUS,
@@ -1039,13 +1077,15 @@ static int xen_pt_msgctrl_reg_init(XenPCIPassthroughState *s,
                                    XenPTRegInfo *reg, uint32_t real_offset,
                                    uint32_t *data)
 {
-    PCIDevice *d = &s->dev;
     XenPTMSI *msi = s->msi;
-    uint16_t reg_field = 0;
+    uint16_t reg_field;
+    int rc;
 
     /* use I/O device register's value as initial value */
-    reg_field = pci_get_word(d->config + real_offset);
-
+    rc = xen_host_pci_get_word(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     if (reg_field & PCI_MSI_FLAGS_ENABLE) {
         XEN_PT_LOG(&s->dev, "MSI already enabled, disabling it first\n");
         xen_host_pci_set_word(&s->real_device, real_offset,
@@ -1067,6 +1107,7 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
     XenPTMSI *msi = s->msi;
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* Currently no support for multi-vector */
     if (*val & PCI_MSI_FLAGS_QSIZE) {
@@ -1075,8 +1116,8 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-    msi->flags |= cfg_entry->data & ~PCI_MSI_FLAGS_ENABLE;
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
+    msi->flags |= *data & ~PCI_MSI_FLAGS_ENABLE;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -1086,7 +1127,7 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
         /* setup MSI pirq for the first time */
         if (!msi->initialized) {
             /* Init physical one */
-            XEN_PT_LOG(&s->dev, "setup MSI\n");
+            XEN_PT_LOG(&s->dev, "setup MSI (register: %x).\n", *val);
             if (xen_pt_msi_setup(s)) {
                 /* We do not broadcast the error to the framework code, so
                  * that MSI errors are contained in MSI emulation code and
@@ -1094,12 +1135,12 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
                  * Guest MSI would be actually not working.
                  */
                 *val &= ~PCI_MSI_FLAGS_ENABLE;
-                XEN_PT_WARN(&s->dev, "Can not map MSI.\n");
+                XEN_PT_WARN(&s->dev, "Can not map MSI (register: %x)!\n", *val);
                 return 0;
             }
             if (xen_pt_msi_update(s)) {
                 *val &= ~PCI_MSI_FLAGS_ENABLE;
-                XEN_PT_WARN(&s->dev, "Can not bind MSI\n");
+                XEN_PT_WARN(&s->dev, "Can not bind MSI (register: %x)!\n", *val);
                 return 0;
             }
             msi->initialized = true;
@@ -1190,18 +1231,19 @@ static int xen_pt_msgaddr32_reg_write(XenPCIPassthroughState *s,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
-    uint32_t old_addr = cfg_entry->data;
+    uint32_t old_addr = *cfg_entry->ptr.word;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-    s->msi->addr_lo = cfg_entry->data;
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
+    s->msi->addr_lo = *data;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_addr) {
+    if (*data != old_addr) {
         if (s->msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1216,7 +1258,8 @@ static int xen_pt_msgaddr64_reg_write(XenPCIPassthroughState *s,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
-    uint32_t old_addr = cfg_entry->data;
+    uint32_t old_addr = *cfg_entry->ptr.word;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* check whether the type is 64 bit or not */
     if (!(s->msi->flags & PCI_MSI_FLAGS_64BIT)) {
@@ -1227,15 +1270,15 @@ static int xen_pt_msgaddr64_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
     /* update the msi_info too */
-    s->msi->addr_hi = cfg_entry->data;
+    s->msi->addr_hi = *data;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_addr) {
+    if (*data != old_addr) {
         if (s->msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1254,8 +1297,9 @@ static int xen_pt_msgdata_reg_write(XenPCIPassthroughState *s,
     XenPTRegInfo *reg = cfg_entry->reg;
     XenPTMSI *msi = s->msi;
     uint16_t writable_mask = 0;
-    uint16_t old_data = cfg_entry->data;
+    uint16_t old_data = *cfg_entry->ptr.half_word;
     uint32_t offset = reg->offset;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* check the offset whether matches the type or not */
     if (!xen_pt_msi_check_type(offset, msi->flags, DATA)) {
@@ -1266,15 +1310,15 @@ static int xen_pt_msgdata_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
     /* update the msi_info too */
-    msi->data = cfg_entry->data;
+    msi->data = *data;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_data) {
+    if (*data != old_data) {
         if (msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1411,14 +1455,16 @@ static int xen_pt_msixctrl_reg_init(XenPCIPassthroughState *s,
                                     XenPTRegInfo *reg, uint32_t real_offset,
                                     uint32_t *data)
 {
-    PCIDevice *d = &s->dev;
-    uint16_t reg_field = 0;
+    uint16_t reg_field;
+    int rc;
 
     /* use I/O device register's value as initial value */
-    reg_field = pci_get_word(d->config + real_offset);
-
+    rc = xen_host_pci_get_word(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     if (reg_field & PCI_MSIX_FLAGS_ENABLE) {
-        XEN_PT_LOG(d, "MSIX already enabled, disabling it first\n");
+        XEN_PT_LOG(&s->dev, "MSIX already enabled, disabling it first\n");
         xen_host_pci_set_word(&s->real_device, real_offset,
                               reg_field & ~PCI_MSIX_FLAGS_ENABLE);
     }
@@ -1436,10 +1482,11 @@ static int xen_pt_msixctrl_reg_write(XenPCIPassthroughState *s,
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
     int debug_msix_enabled_old;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -1492,6 +1539,19 @@ static XenPTRegInfo xen_pt_emu_reg_msix[] = {
     },
 };
 
+static XenPTRegInfo xen_pt_emu_reg_igd_opregion[] = {
+    /* Intel IGFX OpRegion reg */
+    {
+        .offset     = 0x0,
+        .size       = 4,
+        .init_val   = 0,
+        .u.dw.read   = xen_pt_intel_opregion_read,
+        .u.dw.write  = xen_pt_intel_opregion_write,
+    },
+    {
+        .size = 0,
+    },
+};
 
 /****************************
  * Capabilities
@@ -1511,8 +1571,7 @@ static int xen_pt_vendor_size_init(XenPCIPassthroughState *s,
                                    const XenPTRegGroupInfo *grp_reg,
                                    uint32_t base_offset, uint8_t *size)
 {
-    *size = pci_get_byte(s->dev.config + base_offset + 0x02);
-    return 0;
+    return xen_host_pci_get_byte(&s->real_device, base_offset + 0x02, size);
 }
 /* get PCI Express Capability Structure register group size */
 static int xen_pt_pcie_size_init(XenPCIPassthroughState *s,
@@ -1591,12 +1650,15 @@ static int xen_pt_msi_size_init(XenPCIPassthroughState *s,
                                 const XenPTRegGroupInfo *grp_reg,
                                 uint32_t base_offset, uint8_t *size)
 {
-    PCIDevice *d = &s->dev;
     uint16_t msg_ctrl = 0;
     uint8_t msi_size = 0xa;
+    int rc;
 
-    msg_ctrl = pci_get_word(d->config + (base_offset + PCI_MSI_FLAGS));
-
+    rc = xen_host_pci_get_word(&s->real_device, base_offset + PCI_MSI_FLAGS,
+                               &msg_ctrl);
+    if (rc) {
+        return rc;
+    }
     /* check if 64-bit address is capable of per-vector masking */
     if (msg_ctrl & PCI_MSI_FLAGS_64BIT) {
         msi_size += 4;
@@ -1729,6 +1791,14 @@ static const XenPTRegGroupInfo xen_pt_emu_reg_grps[] = {
         .size_init   = xen_pt_msix_size_init,
         .emu_regs = xen_pt_emu_reg_msix,
     },
+    /* Intel IGD Opregion group */
+    {
+        .grp_id      = XEN_PCI_INTEL_OPREGION,
+        .grp_type    = XEN_PT_GRP_TYPE_EMU,
+        .grp_size    = 0x4,
+        .size_init   = xen_pt_reg_grp_size_init,
+        .emu_regs    = xen_pt_emu_reg_igd_opregion,
+    },
     {
         .grp_size = 0,
     },
@@ -1739,11 +1809,14 @@ static int xen_pt_ptr_reg_init(XenPCIPassthroughState *s,
                                XenPTRegInfo *reg, uint32_t real_offset,
                                uint32_t *data)
 {
-    int i;
-    uint8_t *config = s->dev.config;
-    uint32_t reg_field = pci_get_byte(config + real_offset);
+    int i, rc;
+    uint8_t reg_field;
     uint8_t cap_id = 0;
 
+    rc = xen_host_pci_get_byte(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     /* find capability offset */
     while (reg_field) {
         for (i = 0; xen_pt_emu_reg_grps[i].grp_size != 0; i++) {
@@ -1752,7 +1825,13 @@ static int xen_pt_ptr_reg_init(XenPCIPassthroughState *s,
                 continue;
             }
 
-            cap_id = pci_get_byte(config + reg_field + PCI_CAP_LIST_ID);
+            rc = xen_host_pci_get_byte(&s->real_device,
+                                       reg_field + PCI_CAP_LIST_ID, &cap_id);
+            if (rc) {
+                XEN_PT_ERR(&s->dev, "Failed to read capability @0x%x (rc:%d)\n",
+                           reg_field + PCI_CAP_LIST_ID, rc);
+                return rc;
+            }
             if (xen_pt_emu_reg_grps[i].grp_id == cap_id) {
                 if (xen_pt_emu_reg_grps[i].grp_type == XEN_PT_GRP_TYPE_EMU) {
                     goto out;
@@ -1763,7 +1842,11 @@ static int xen_pt_ptr_reg_init(XenPCIPassthroughState *s,
         }
 
         /* next capability */
-        reg_field = pci_get_byte(config + reg_field + PCI_CAP_LIST_NEXT);
+        rc = xen_host_pci_get_byte(&s->real_device,
+                                   reg_field + PCI_CAP_LIST_NEXT, &reg_field);
+        if (rc) {
+            return rc;
+        }
     }
 
 out:
@@ -1779,7 +1862,7 @@ out:
 static uint8_t find_cap_offset(XenPCIPassthroughState *s, uint8_t cap)
 {
     uint8_t id;
-    unsigned max_cap = PCI_CAP_MAX;
+    unsigned max_cap = XEN_PCI_CAP_MAX;
     uint8_t pos = PCI_CAPABILITY_LIST;
     uint8_t status = 0;
 
@@ -1827,6 +1910,10 @@ static int xen_pt_config_reg_init(XenPCIPassthroughState *s,
     reg_entry->reg = reg;
 
     if (reg->init) {
+        uint32_t host_mask, size_mask;
+        unsigned int offset;
+        uint32_t val;
+
         /* initialize emulate register */
         rc = reg->init(s, reg_entry->reg,
                        reg_grp->base_offset + reg->offset, &data);
@@ -1839,8 +1926,67 @@ static int xen_pt_config_reg_init(XenPCIPassthroughState *s,
             g_free(reg_entry);
             return 0;
         }
-        /* set register value */
-        reg_entry->data = data;
+        /* Sync up the data to dev.config */
+        offset = reg_grp->base_offset + reg->offset;
+        size_mask = 0xFFFFFFFF >> ((4 - reg->size) << 3);
+
+        switch (reg->size) {
+        case 1: rc = xen_host_pci_get_byte(&s->real_device, offset, (uint8_t *)&val);
+                break;
+        case 2: rc = xen_host_pci_get_word(&s->real_device, offset, (uint16_t *)&val);
+                break;
+        case 4: rc = xen_host_pci_get_long(&s->real_device, offset, &val);
+                break;
+        default: abort();
+        }
+        if (rc) {
+            /* Serious issues when we cannot read the host values! */
+            g_free(reg_entry);
+            return rc;
+        }
+        /* Set bits in emu_mask are the ones we emulate. The dev.config shall
+         * contain the emulated view of the guest - therefore we flip the mask
+         * to mask out the host values (which dev.config initially has) . */
+        host_mask = size_mask & ~reg->emu_mask;
+
+        if ((data & host_mask) != (val & host_mask)) {
+            uint32_t new_val;
+
+            /* Mask out host (including past size). */
+            new_val = val & host_mask;
+            /* Merge emulated ones (excluding the non-emulated ones). */
+            new_val |= data & host_mask;
+            /* Leave intact host and emulated values past the size - even though
+             * we do not care as we write per reg->size granularity, but for the
+             * logging below lets have the proper value. */
+            new_val |= ((val | data)) & ~size_mask;
+            XEN_PT_LOG(&s->dev,"Offset 0x%04x mismatch! Emulated=0x%04x, host=0x%04x, syncing to 0x%04x.\n",
+                       offset, data, val, new_val);
+            val = new_val;
+        } else
+            val = data;
+
+        if (val & ~size_mask) {
+            XEN_PT_ERR(&s->dev,"Offset 0x%04x:0x%04x expands past register size(%d)!\n",
+                       offset, val, reg->size);
+            g_free(reg_entry);
+            return -ENXIO;
+        }
+        /* This could be just pci_set_long as we don't modify the bits
+         * past reg->size, but in case this routine is run in parallel or the
+         * init value is larger, we do not want to over-write registers. */
+        switch (reg->size) {
+        case 1: pci_set_byte(s->dev.config + offset, (uint8_t)val);
+                break;
+        case 2: pci_set_word(s->dev.config + offset, (uint16_t)val);
+                break;
+        case 4: pci_set_long(s->dev.config + offset, val);
+                break;
+        default: abort();
+        }
+        /* set register value pointer to the data. */
+        reg_entry->ptr.byte = s->dev.config + offset;
+
     }
     /* list add register entry */
     QLIST_INSERT_HEAD(&reg_grp->reg_tbl_list, reg_entry, entries);
@@ -1858,7 +2004,8 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
         uint32_t reg_grp_offset = 0;
         XenPTRegGroup *reg_grp_entry = NULL;
 
-        if (xen_pt_emu_reg_grps[i].grp_id != 0xFF) {
+        if (xen_pt_emu_reg_grps[i].grp_id != 0xFF
+            && xen_pt_emu_reg_grps[i].grp_id != XEN_PCI_INTEL_OPREGION) {
             if (xen_pt_hide_dev_cap(&s->real_device,
                                     xen_pt_emu_reg_grps[i].grp_id)) {
                 continue;
@@ -1871,6 +2018,15 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
             }
         }
 
+        /*
+         * By default we will trap up to 0x40 in the cfg space.
+         * If an intel device is pass through we need to trap 0xfc,
+         * therefore the size should be 0xff.
+         */
+        if (xen_pt_emu_reg_grps[i].grp_id == XEN_PCI_INTEL_OPREGION) {
+            reg_grp_offset = XEN_PCI_INTEL_OPREGION;
+        }
+
         reg_grp_entry = g_new0(XenPTRegGroup, 1);
         QLIST_INIT(&reg_grp_entry->reg_tbl_list);
         QLIST_INSERT_HEAD(&s->reg_grps, reg_grp_entry, entries);
@@ -1883,6 +2039,9 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
                                                   reg_grp_offset,
                                                   &reg_grp_entry->size);
             if (rc < 0) {
+                XEN_PT_LOG(&s->dev, "Failed to initialize %d/%ld, type=0x%x, rc:%d\n",
+                           i, ARRAY_SIZE(xen_pt_emu_reg_grps),
+                           xen_pt_emu_reg_grps[i].grp_type, rc);
                 xen_pt_config_delete(s);
                 return rc;
             }
@@ -1897,6 +2056,10 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
                     /* initialize capability register */
                     rc = xen_pt_config_reg_init(s, reg_grp_entry, regs);
                     if (rc < 0) {
+                        XEN_PT_LOG(&s->dev, "Failed to initialize %d/%ld reg 0x%x in grp_type=0x%x (%d/%ld), rc=%d\n",
+                                   j, ARRAY_SIZE(xen_pt_emu_reg_grps[i].emu_regs),
+                                   regs->offset, xen_pt_emu_reg_grps[i].grp_type,
+                                   i, ARRAY_SIZE(xen_pt_emu_reg_grps), rc);
                         xen_pt_config_delete(s);
                         return rc;
                     }
@@ -1916,11 +2079,9 @@ void xen_pt_config_delete(XenPCIPassthroughState *s)
 
     /* free MSI/MSI-X info table */
     if (s->msix) {
-        xen_pt_msix_delete(s);
-    }
-    if (s->msi) {
-        g_free(s->msi);
+        xen_pt_msix_unmap(s);
     }
+    g_free(s->msi);
 
     /* free all register group entry */
     QLIST_FOREACH_SAFE(reg_group, &s->reg_grps, entries, next_grp) {
diff --git a/hw/xen/xen_pt_graphics.c b/hw/xen/xen_pt_graphics.c
new file mode 100644
index 000000000..df6069bf6
--- /dev/null
+++ b/hw/xen/xen_pt_graphics.c
@@ -0,0 +1,272 @@
+/*
+ * graphics passthrough
+ */
+#include "xen_pt.h"
+#include "xen-host-pci-device.h"
+#include "hw/xen/xen_backend.h"
+
+static unsigned long igd_guest_opregion;
+static unsigned long igd_host_opregion;
+
+#define XEN_PCI_INTEL_OPREGION_MASK 0xfff
+
+typedef struct VGARegion {
+    int type;           /* Memory or port I/O */
+    uint64_t guest_base_addr;
+    uint64_t machine_base_addr;
+    uint64_t size;    /* size of the region */
+    int rc;
+} VGARegion;
+
+#define IORESOURCE_IO           0x00000100
+#define IORESOURCE_MEM          0x00000200
+
+static struct VGARegion vga_args[] = {
+    {
+        .type = IORESOURCE_IO,
+        .guest_base_addr = 0x3B0,
+        .machine_base_addr = 0x3B0,
+        .size = 0xC,
+        .rc = -1,
+    },
+    {
+        .type = IORESOURCE_IO,
+        .guest_base_addr = 0x3C0,
+        .machine_base_addr = 0x3C0,
+        .size = 0x20,
+        .rc = -1,
+    },
+    {
+        .type = IORESOURCE_MEM,
+        .guest_base_addr = 0xa0000 >> XC_PAGE_SHIFT,
+        .machine_base_addr = 0xa0000 >> XC_PAGE_SHIFT,
+        .size = 0x20,
+        .rc = -1,
+    },
+};
+
+/*
+ * register VGA resources for the domain with assigned gfx
+ */
+int xen_pt_register_vga_regions(XenHostPCIDevice *dev)
+{
+    int i = 0;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        return 0;
+    }
+
+    for (i = 0 ; i < ARRAY_SIZE(vga_args); i++) {
+        if (vga_args[i].type == IORESOURCE_IO) {
+            vga_args[i].rc = xc_domain_ioport_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_ADD_MAPPING);
+        } else {
+            vga_args[i].rc = xc_domain_memory_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_ADD_MAPPING);
+        }
+
+        if (vga_args[i].rc) {
+            XEN_PT_ERR(NULL, "VGA %s mapping failed! (rc: %i)\n",
+                    vga_args[i].type == IORESOURCE_IO ? "ioport" : "memory",
+                    vga_args[i].rc);
+            return vga_args[i].rc;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * unregister VGA resources for the domain with assigned gfx
+ */
+int xen_pt_unregister_vga_regions(XenHostPCIDevice *dev)
+{
+    int i = 0;
+    int ret = 0;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        return 0;
+    }
+
+    for (i = 0 ; i < ARRAY_SIZE(vga_args); i++) {
+        if (vga_args[i].type == IORESOURCE_IO) {
+            vga_args[i].rc = xc_domain_ioport_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_REMOVE_MAPPING);
+        } else {
+            vga_args[i].rc = xc_domain_memory_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_REMOVE_MAPPING);
+        }
+
+        if (vga_args[i].rc) {
+            XEN_PT_ERR(NULL, "VGA %s unmapping failed! (rc: %i)\n",
+                    vga_args[i].type == IORESOURCE_IO ? "ioport" : "memory",
+                    vga_args[i].rc);
+            return vga_args[i].rc;
+        }
+    }
+
+    if (igd_guest_opregion) {
+        ret = xc_domain_memory_mapping(xen_xc, xen_domid,
+                (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT),
+                (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                3,
+                DPCI_REMOVE_MAPPING);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void *get_vgabios(XenPCIPassthroughState *s, int *size,
+                       XenHostPCIDevice *dev)
+{
+    return pci_assign_dev_load_option_rom(&s->dev, OBJECT(&s->dev), size,
+                                          dev->domain, dev->bus,
+                                          dev->dev, dev->func);
+}
+
+/* Refer to Seabios. */
+struct rom_header {
+    uint16_t signature;
+    uint8_t size;
+    uint8_t initVector[4];
+    uint8_t reserved[17];
+    uint16_t pcioffset;
+    uint16_t pnpoffset;
+} __attribute__((packed));
+
+struct pci_data {
+    uint32_t signature;
+    uint16_t vendor;
+    uint16_t device;
+    uint16_t vitaldata;
+    uint16_t dlen;
+    uint8_t drevision;
+    uint8_t class_lo;
+    uint16_t class_hi;
+    uint16_t ilen;
+    uint16_t irevision;
+    uint8_t type;
+    uint8_t indicator;
+    uint16_t reserved;
+} __attribute__((packed));
+
+int xen_pt_setup_vga(XenPCIPassthroughState *s, XenHostPCIDevice *dev)
+{
+    unsigned char *bios = NULL;
+    struct rom_header *rom;
+    int bios_size;
+    char *c = NULL;
+    char checksum = 0;
+    uint32_t len = 0;
+    struct pci_data *pd = NULL;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        return -1;
+    }
+
+    bios = get_vgabios(s, &bios_size, dev);
+    if (!bios) {
+        XEN_PT_ERR(&s->dev, "VGA: Can't getting VBIOS!\n");
+        return -1;
+    }
+
+    /* Currently we fixed this address as a primary. */
+    rom = (struct rom_header *)bios;
+    pd = (void *)(bios + (unsigned char)rom->pcioffset);
+
+    /* We may need to fixup Device Identification. */
+    if (pd->device != s->real_device.device_id) {
+        pd->device = s->real_device.device_id;
+
+        len = rom->size * 512;
+        /* Then adjust the bios checksum */
+        for (c = (char *)bios; c < ((char *)bios + len); c++) {
+            checksum += *c;
+        }
+        if (checksum) {
+            bios[len - 1] -= checksum;
+            XEN_PT_LOG(&s->dev, "vga bios checksum is adjusted %x!\n",
+                       checksum);
+        }
+    }
+
+    /* Currently we fixed this address as a primary for legacy BIOS. */
+    cpu_physical_memory_rw(0xc0000, bios, bios_size, 1);
+    return 0;
+}
+
+uint32_t igd_read_opregion(XenPCIPassthroughState *s)
+{
+    uint32_t val = 0;
+
+    if (!igd_guest_opregion) {
+        return val;
+    }
+
+    val = igd_guest_opregion;
+
+    XEN_PT_LOG(&s->dev, "Read opregion val=%x\n", val);
+    return val;
+}
+
+#define XEN_PCI_INTEL_OPREGION_PAGES 0x3
+#define XEN_PCI_INTEL_OPREGION_ENABLE_ACCESSED 0x1
+void igd_write_opregion(XenPCIPassthroughState *s, uint32_t val)
+{
+    int ret;
+
+    if (igd_guest_opregion) {
+        XEN_PT_LOG(&s->dev, "opregion register already been set, ignoring %x\n",
+                   val);
+        return;
+    }
+
+    /* We just work with LE. */
+    xen_host_pci_get_block(&s->real_device, XEN_PCI_INTEL_OPREGION,
+            (uint8_t *)&igd_host_opregion, 4);
+    igd_guest_opregion = (unsigned long)(val & ~XEN_PCI_INTEL_OPREGION_MASK)
+                            | (igd_host_opregion & XEN_PCI_INTEL_OPREGION_MASK);
+
+    ret = xc_domain_iomem_permission(xen_xc, xen_domid,
+            (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+            XEN_PCI_INTEL_OPREGION_PAGES,
+            XEN_PCI_INTEL_OPREGION_ENABLE_ACCESSED);
+
+    if (ret) {
+        XEN_PT_ERR(&s->dev, "[%d]:Can't enable to access IGD host opregion:"
+                    " 0x%lx.\n", ret,
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT)),
+        igd_guest_opregion = 0;
+        return;
+    }
+
+    ret = xc_domain_memory_mapping(xen_xc, xen_domid,
+            (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT),
+            (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+            XEN_PCI_INTEL_OPREGION_PAGES,
+            DPCI_ADD_MAPPING);
+
+    if (ret) {
+        XEN_PT_ERR(&s->dev, "[%d]:Can't map IGD host opregion:0x%lx to"
+                    " guest opregion:0x%lx.\n", ret,
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                    (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT));
+        igd_guest_opregion = 0;
+        return;
+    }
+
+    XEN_PT_LOG(&s->dev, "Map OpRegion: 0x%lx -> 0x%lx\n",
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                    (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT));
+}
diff --git a/hw/xen/xen_pt_msi.c b/hw/xen/xen_pt_msi.c
index 263e0514a..82de2bced 100644
--- a/hw/xen/xen_pt_msi.c
+++ b/hw/xen/xen_pt_msi.c
@@ -75,19 +75,29 @@ static int msi_msix_enable(XenPCIPassthroughState *s,
                            bool enable)
 {
     uint16_t val = 0;
+    int rc;
 
     if (!address) {
         return -1;
     }
 
-    xen_host_pci_get_word(&s->real_device, address, &val);
+    rc = xen_host_pci_get_word(&s->real_device, address, &val);
+    if (rc) {
+        XEN_PT_ERR(&s->dev, "Failed to read MSI/MSI-X register (0x%x), rc:%d\n",
+                   address, rc);
+        return rc;
+    }
     if (enable) {
         val |= flag;
     } else {
         val &= ~flag;
     }
-    xen_host_pci_set_word(&s->real_device, address, val);
-    return 0;
+    rc = xen_host_pci_set_word(&s->real_device, address, val);
+    if (rc) {
+        XEN_PT_ERR(&s->dev, "Failed to write MSI/MSI-X register (0x%x), rc:%d\n",
+                   address, rc);
+    }
+    return rc;
 }
 
 static int msi_msix_setup(XenPCIPassthroughState *s,
@@ -220,7 +230,7 @@ static int msi_msix_disable(XenPCIPassthroughState *s,
  * MSI virtualization functions
  */
 
-int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable)
+static int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable)
 {
     XEN_PT_LOG(&s->dev, "%s MSI.\n", enable ? "enabling" : "disabling");
 
@@ -276,7 +286,7 @@ void xen_pt_msi_disable(XenPCIPassthroughState *s)
         return;
     }
 
-    xen_pt_msi_set_enable(s, false);
+    (void)xen_pt_msi_set_enable(s, false);
 
     msi_msix_disable(s, msi_addr64(msi), msi->data, msi->pirq, false,
                      msi->initialized);
@@ -600,7 +610,7 @@ error_out:
     return rc;
 }
 
-void xen_pt_msix_delete(XenPCIPassthroughState *s)
+void xen_pt_msix_unmap(XenPCIPassthroughState *s)
 {
     XenPTMSIX *msix = s->msix;
 
@@ -617,6 +627,17 @@ void xen_pt_msix_delete(XenPCIPassthroughState *s)
     }
 
     memory_region_del_subregion(&s->bar[msix->bar_index], &msix->mmio);
+}
+
+void xen_pt_msix_delete(XenPCIPassthroughState *s)
+{
+    XenPTMSIX *msix = s->msix;
+
+    if (!msix) {
+        return;
+    }
+
+    object_unparent(OBJECT(&msix->mmio));
 
     g_free(s->msix);
     s->msix = NULL;
diff --git a/hw/xenpv/xen_domainbuild.c b/hw/xenpv/xen_domainbuild.c
index c0ab7537d..ac0e5ac9f 100644
--- a/hw/xenpv/xen_domainbuild.c
+++ b/hw/xenpv/xen_domainbuild.c
@@ -234,7 +234,7 @@ int xen_domain_build_pv(const char *kernel, const char *ramdisk,
     int rc;
 
     memcpy(uuid, qemu_uuid, sizeof(uuid));
-    rc = xc_domain_create(xen_xc, ssidref, uuid, flags, &xen_domid);
+    rc = xen_domain_create(xen_xc, ssidref, uuid, flags, &xen_domid);
     if (rc < 0) {
         fprintf(stderr, "xen: xc_domain_create() failed\n");
         goto err;
diff --git a/hw/xenpv/xen_machine_pv.c b/hw/xenpv/xen_machine_pv.c
index 2e545d241..23d6ef07e 100644
--- a/hw/xenpv/xen_machine_pv.c
+++ b/hw/xenpv/xen_machine_pv.c
@@ -93,17 +93,12 @@ static void xen_init_pv(MachineState *machine)
     xen_init_display(xen_domid);
 }
 
-static QEMUMachine xenpv_machine = {
-    .name = "xenpv",
-    .desc = "Xen Para-virtualized PC",
-    .init = xen_init_pv,
-    .max_cpus = 1,
-    .default_machine_opts = "accel=xen",
-};
-
-static void xenpv_machine_init(void)
+static void xenpv_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xenpv_machine);
+    mc->desc = "Xen Para-virtualized PC";
+    mc->init = xen_init_pv;
+    mc->max_cpus = 1;
+    mc->default_machine_opts = "accel=xen";
 }
 
-machine_init(xenpv_machine_init);
+DEFINE_MACHINE("xenpv", xenpv_machine_init)
diff --git a/hw/xtensa/sim.c b/hw/xtensa/sim.c
index 328d20975..6266b8d44 100644
--- a/hw/xtensa/sim.c
+++ b/hw/xtensa/sim.c
@@ -79,12 +79,12 @@ static void xtensa_sim_init(MachineState *machine)
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
     rom = g_malloc(sizeof(*rom));
-    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000, &error_abort);
+    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0xfe000000, rom);
 
@@ -93,10 +93,10 @@ static void xtensa_sim_init(MachineState *machine)
         uint64_t elf_lowaddr;
 #ifdef TARGET_WORDS_BIGENDIAN
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, 1, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, 1, EM_XTENSA, 0);
 #else
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, 0, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, 0, EM_XTENSA, 0);
 #endif
         if (success > 0) {
             env->pc = elf_entry;
@@ -104,17 +104,12 @@ static void xtensa_sim_init(MachineState *machine)
     }
 }
 
-static QEMUMachine xtensa_sim_machine = {
-    .name = "sim",
-    .desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .is_default = true,
-    .init = xtensa_sim_init,
-    .max_cpus = 4,
-};
-
-static void xtensa_sim_machine_init(void)
+static void xtensa_sim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xtensa_sim_machine);
+    mc->desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->is_default = true;
+    mc->init = xtensa_sim_init;
+    mc->max_cpus = 4;
 }
 
-machine_init(xtensa_sim_machine_init);
+DEFINE_MACHINE("sim", xtensa_sim_machine_init)
diff --git a/hw/xtensa/xtfpga.c b/hw/xtensa/xtfpga.c
index ab4d0e412..c1bc5aef5 100644
--- a/hw/xtensa/xtfpga.c
+++ b/hw/xtensa/xtfpga.c
@@ -143,11 +143,34 @@ static void lx60_net_init(MemoryRegion *address_space,
             sysbus_mmio_get_region(s, 1));
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384, &error_abort);
+    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384,
+                           &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space, buffers, ram);
 }
 
+static pflash_t *xtfpga_flash_init(MemoryRegion *address_space,
+                                   const LxBoardDesc *board,
+                                   DriveInfo *dinfo, int be)
+{
+    SysBusDevice *s;
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+
+    qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
+                        &error_abort);
+    qdev_prop_set_uint32(dev, "num-blocks",
+                         board->flash_size / board->flash_sector_size);
+    qdev_prop_set_uint64(dev, "sector-length", board->flash_sector_size);
+    qdev_prop_set_uint8(dev, "width", 4);
+    qdev_prop_set_bit(dev, "big-endian", be);
+    qdev_prop_set_string(dev, "name", "lx60.io.flash");
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    memory_region_add_subregion(address_space, board->flash_base,
+                                sysbus_mmio_get_region(s, 0));
+    return OBJECT_CHECK(pflash_t, (dev), "cfi.pflash01");
+}
+
 static uint64_t translate_phys_addr(void *opaque, uint64_t addr)
 {
     XtensaCPU *cpu = opaque;
@@ -223,7 +246,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 
     ram = g_malloc(sizeof(*ram));
     memory_region_init_ram(ram, NULL, "lx60.dram", machine->ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(system_memory, 0, ram);
 
@@ -246,16 +269,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     if (dinfo) {
-        flash = pflash_cfi01_register(board->flash_base,
-                NULL, "lx60.io.flash", board->flash_size,
-                blk_by_legacy_dinfo(dinfo),
-                board->flash_sector_size,
-                board->flash_size / board->flash_sector_size,
-                4, 0x0000, 0x0000, 0x0000, 0x0000, be);
-        if (flash == NULL) {
-            error_report("unable to mount pflash");
-            exit(EXIT_FAILURE);
-        }
+        flash = xtfpga_flash_init(system_io, board, dinfo, be);
     }
 
     /* Use presence of kernel file name as 'boot from SRAM' switch. */
@@ -275,7 +289,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 
         rom = g_malloc(sizeof(*rom));
         memory_region_init_ram(rom, NULL, "lx60.sram", board->sram_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(rom);
         memory_region_add_subregion(system_memory, 0xfe000000, rom);
 
@@ -340,7 +354,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
         uint64_t elf_entry;
         uint64_t elf_lowaddr;
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, be, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, be, EM_XTENSA, 0);
         if (success > 0) {
             entry_point = elf_entry;
         } else {
@@ -385,7 +399,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 static void xtensa_lx60_init(MachineState *machine)
 {
     static const LxBoardDesc lx60_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x00400000,
         .flash_sector_size = 0x10000,
         .sram_size = 0x20000,
@@ -396,7 +410,7 @@ static void xtensa_lx60_init(MachineState *machine)
 static void xtensa_lx200_init(MachineState *machine)
 {
     static const LxBoardDesc lx200_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x01000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
@@ -407,7 +421,7 @@ static void xtensa_lx200_init(MachineState *machine)
 static void xtensa_ml605_init(MachineState *machine)
 {
     static const LxBoardDesc ml605_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x01000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
@@ -418,7 +432,7 @@ static void xtensa_ml605_init(MachineState *machine)
 static void xtensa_kc705_init(MachineState *machine)
 {
     static const LxBoardDesc kc705_board = {
-        .flash_base = 0xf0000000,
+        .flash_base = 0x00000000,
         .flash_size = 0x08000000,
         .flash_boot_base = 0x06000000,
         .flash_sector_size = 0x20000,
@@ -427,40 +441,72 @@ static void xtensa_kc705_init(MachineState *machine)
     lx_init(&kc705_board, machine);
 }
 
-static QEMUMachine xtensa_lx60_machine = {
-    .name = "lx60",
-    .desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_lx60_init,
-    .max_cpus = 4,
+static void xtensa_lx60_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_lx60_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_lx60_type = {
+    .name = MACHINE_TYPE_NAME("lx60"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_lx60_class_init,
 };
 
-static QEMUMachine xtensa_lx200_machine = {
-    .name = "lx200",
-    .desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_lx200_init,
-    .max_cpus = 4,
+static void xtensa_lx200_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_lx200_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_lx200_type = {
+    .name = MACHINE_TYPE_NAME("lx200"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_lx200_class_init,
 };
 
-static QEMUMachine xtensa_ml605_machine = {
-    .name = "ml605",
-    .desc = "ml605 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_ml605_init,
-    .max_cpus = 4,
+static void xtensa_ml605_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ml605 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_ml605_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_ml605_type = {
+    .name = MACHINE_TYPE_NAME("ml605"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_ml605_class_init,
 };
 
-static QEMUMachine xtensa_kc705_machine = {
-    .name = "kc705",
-    .desc = "kc705 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_kc705_init,
-    .max_cpus = 4,
+static void xtensa_kc705_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "kc705 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_kc705_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_kc705_type = {
+    .name = MACHINE_TYPE_NAME("kc705"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_kc705_class_init,
 };
 
 static void xtensa_lx_machines_init(void)
 {
-    qemu_register_machine(&xtensa_lx60_machine);
-    qemu_register_machine(&xtensa_lx200_machine);
-    qemu_register_machine(&xtensa_ml605_machine);
-    qemu_register_machine(&xtensa_kc705_machine);
+    type_register_static(&xtensa_lx60_type);
+    type_register_static(&xtensa_lx200_type);
+    type_register_static(&xtensa_ml605_type);
+    type_register_static(&xtensa_kc705_type);
 }
 
-machine_init(xtensa_lx_machines_init);
+machine_init(xtensa_lx_machines_init)
diff --git a/include/block/accounting.h b/include/block/accounting.h
index 4c406cff7..0f46cb4ec 100644
--- a/include/block/accounting.h
+++ b/include/block/accounting.h
@@ -2,6 +2,7 @@
  * QEMU System Emulator block accounting
  *
  * Copyright (c) 2011 Christoph Hellwig
+ * Copyright (c) 2015 Igalia, S.L.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -25,8 +26,12 @@
 #define BLOCK_ACCOUNTING_H
 
 #include <stdint.h>
+#include <stdbool.h>
 
 #include "qemu/typedefs.h"
+#include "qemu/timed-average.h"
+
+typedef struct BlockAcctTimedStats BlockAcctTimedStats;
 
 enum BlockAcctType {
     BLOCK_ACCT_READ,
@@ -35,12 +40,23 @@ enum BlockAcctType {
     BLOCK_MAX_IOTYPE,
 };
 
+struct BlockAcctTimedStats {
+    TimedAverage latency[BLOCK_MAX_IOTYPE];
+    unsigned interval_length; /* in seconds */
+    QSLIST_ENTRY(BlockAcctTimedStats) entries;
+};
+
 typedef struct BlockAcctStats {
     uint64_t nr_bytes[BLOCK_MAX_IOTYPE];
     uint64_t nr_ops[BLOCK_MAX_IOTYPE];
+    uint64_t invalid_ops[BLOCK_MAX_IOTYPE];
+    uint64_t failed_ops[BLOCK_MAX_IOTYPE];
     uint64_t total_time_ns[BLOCK_MAX_IOTYPE];
     uint64_t merged[BLOCK_MAX_IOTYPE];
-    uint64_t wr_highest_sector;
+    int64_t last_access_time_ns;
+    QSLIST_HEAD(, BlockAcctTimedStats) intervals;
+    bool account_invalid;
+    bool account_failed;
 } BlockAcctStats;
 
 typedef struct BlockAcctCookie {
@@ -49,12 +65,21 @@ typedef struct BlockAcctCookie {
     enum BlockAcctType type;
 } BlockAcctCookie;
 
+void block_acct_init(BlockAcctStats *stats, bool account_invalid,
+                     bool account_failed);
+void block_acct_cleanup(BlockAcctStats *stats);
+void block_acct_add_interval(BlockAcctStats *stats, unsigned interval_length);
+BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,
+                                              BlockAcctTimedStats *s);
 void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
                       int64_t bytes, enum BlockAcctType type);
 void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie);
-void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
-                               unsigned int nb_sectors);
+void block_acct_failed(BlockAcctStats *stats, BlockAcctCookie *cookie);
+void block_acct_invalid(BlockAcctStats *stats, enum BlockAcctType type);
 void block_acct_merge_done(BlockAcctStats *stats, enum BlockAcctType type,
                            int num_requests);
+int64_t block_acct_idle_time_ns(BlockAcctStats *stats);
+double block_acct_queue_depth(BlockAcctTimedStats *stats,
+                              enum BlockAcctType type);
 
 #endif
diff --git a/include/block/aio.h b/include/block/aio.h
index 400b1b002..e086e3b4e 100644
--- a/include/block/aio.h
+++ b/include/block/aio.h
@@ -122,6 +122,13 @@ struct AioContext {
 
     /* TimerLists for calling timers - one per clock type */
     QEMUTimerListGroup tlg;
+
+    int external_disable_cnt;
+
+    /* epoll(7) state used when built with CONFIG_EPOLL */
+    int epollfd;
+    bool epoll_enabled;
+    bool epoll_available;
 };
 
 /**
@@ -207,6 +214,11 @@ void aio_notify(AioContext *ctx);
 void aio_notify_accept(AioContext *ctx);
 
 /**
+ * aio_bh_call: Executes callback function of the specified BH.
+ */
+void aio_bh_call(QEMUBH *bh);
+
+/**
  * aio_bh_poll: Poll bottom halves for an AioContext.
  *
  * These are internal functions used by the QEMU main loop.
@@ -299,6 +311,7 @@ bool aio_poll(AioContext *ctx, bool blocking);
  */
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque);
@@ -312,6 +325,7 @@ void aio_set_fd_handler(AioContext *ctx,
  */
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *notifier,
+                            bool is_external,
                             EventNotifierHandler *io_read);
 
 /* Return a GSource that lets the main loop poll the file descriptors attached
@@ -373,4 +387,59 @@ static inline void aio_timer_init(AioContext *ctx,
  */
 int64_t aio_compute_timeout(AioContext *ctx);
 
+/**
+ * aio_disable_external:
+ * @ctx: the aio context
+ *
+ * Disable the further processing of external clients.
+ */
+static inline void aio_disable_external(AioContext *ctx)
+{
+    atomic_inc(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_enable_external:
+ * @ctx: the aio context
+ *
+ * Enable the processing of external clients.
+ */
+static inline void aio_enable_external(AioContext *ctx)
+{
+    assert(ctx->external_disable_cnt > 0);
+    atomic_dec(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_external_disabled:
+ * @ctx: the aio context
+ *
+ * Return true if the external clients are disabled.
+ */
+static inline bool aio_external_disabled(AioContext *ctx)
+{
+    return atomic_read(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_node_check:
+ * @ctx: the aio context
+ * @is_external: Whether or not the checked node is an external event source.
+ *
+ * Check if the node's is_external flag is okay to be polled by the ctx at this
+ * moment. True means green light.
+ */
+static inline bool aio_node_check(AioContext *ctx, bool is_external)
+{
+    return !is_external || !atomic_read(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_context_setup:
+ * @ctx: the aio context
+ *
+ * Initialize the aio context.
+ */
+void aio_context_setup(AioContext *ctx, Error **errp);
+
 #endif
diff --git a/include/block/block.h b/include/block/block.h
index 37916f720..d83d420d1 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -4,7 +4,7 @@
 #include "block/aio.h"
 #include "qemu-common.h"
 #include "qemu/option.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/accounting.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi-types.h"
@@ -14,6 +14,7 @@ typedef struct BlockDriver BlockDriver;
 typedef struct BlockJob BlockJob;
 typedef struct BdrvChild BdrvChild;
 typedef struct BdrvChildRole BdrvChildRole;
+typedef struct BlockJobTxn BlockJobTxn;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
@@ -23,7 +24,7 @@ typedef struct BlockDriverInfo {
     bool is_dirty;
     /*
      * True if unallocated blocks read back as zeroes. This is equivalent
-     * to the the LBPRZ flag in the SCSI logical block provisioning page.
+     * to the LBPRZ flag in the SCSI logical block provisioning page.
      */
     bool unallocated_blocks_are_zero;
     /*
@@ -51,15 +52,16 @@ typedef struct BlockFragInfo {
 } BlockFragInfo;
 
 typedef enum {
-    BDRV_REQ_COPY_ON_READ = 0x1,
-    BDRV_REQ_ZERO_WRITE   = 0x2,
+    BDRV_REQ_COPY_ON_READ       = 0x1,
+    BDRV_REQ_ZERO_WRITE         = 0x2,
     /* The BDRV_REQ_MAY_UNMAP flag is used to indicate that the block driver
      * is allowed to optimize a write zeroes request by unmapping (discarding)
      * blocks if it is guaranteed that the result will read back as
      * zeroes. The flag is only passed to the driver if the block device is
      * opened with BDRV_O_UNMAP.
      */
-    BDRV_REQ_MAY_UNMAP    = 0x4,
+    BDRV_REQ_MAY_UNMAP          = 0x4,
+    BDRV_REQ_NO_SERIALISING     = 0x8,
 } BdrvRequestFlags;
 
 typedef struct BlockSizes {
@@ -147,6 +149,7 @@ typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
 typedef struct BDRVReopenState {
     BlockDriverState *bs;
     int flags;
+    QDict *options;
     void *opaque;
 } BDRVReopenState;
 
@@ -172,11 +175,6 @@ typedef enum BlockOpType {
     BLOCK_OP_TYPE_MAX,
 } BlockOpType;
 
-void bdrv_iostatus_enable(BlockDriverState *bs);
-void bdrv_iostatus_reset(BlockDriverState *bs);
-void bdrv_iostatus_disable(BlockDriverState *bs);
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs);
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error);
 void bdrv_info_print(Monitor *mon, const QObject *data);
 void bdrv_info(Monitor *mon, QObject **ret_data);
 void bdrv_stats_print(Monitor *mon, const QObject *data);
@@ -193,22 +191,20 @@ BlockDriver *bdrv_find_protocol(const char *filename,
                                 bool allow_protocol_prefix,
                                 Error **errp);
 BlockDriver *bdrv_find_format(const char *format_name);
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool readonly);
 int bdrv_create(BlockDriver *drv, const char* filename,
                 QemuOpts *opts, Error **errp);
 int bdrv_create_file(const char *filename, QemuOpts *opts, Error **errp);
 BlockDriverState *bdrv_new_root(void);
 BlockDriverState *bdrv_new(void);
+void bdrv_device_remove(BlockDriverState *bs);
 void bdrv_make_anon(BlockDriverState *bs);
 void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old);
 void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
+void bdrv_replace_in_backing_chain(BlockDriverState *old,
+                                   BlockDriverState *new);
+
 int bdrv_parse_cache_flags(const char *mode, int *flags);
 int bdrv_parse_discard_flags(const char *mode, int *flags);
-int bdrv_open_image(BlockDriverState **pbs, const char *filename,
-                    QDict *options, const char *bdref_key,
-                    BlockDriverState* parent, const BdrvChildRole *child_role,
-                    bool allow_none, Error **errp);
 BdrvChild *bdrv_open_child(const char *filename,
                            QDict *options, const char *bdref_key,
                            BlockDriverState* parent,
@@ -218,10 +214,10 @@ void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd);
 int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp);
 int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp);
 int bdrv_open(BlockDriverState **pbs, const char *filename,
-              const char *reference, QDict *options, int flags,
-              BlockDriver *drv, Error **errp);
+              const char *reference, QDict *options, int flags, Error **errp);
 BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags);
+                                    BlockDriverState *bs,
+                                    QDict *options, int flags);
 int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
 int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp);
 int bdrv_reopen_prepare(BDRVReopenState *reopen_state,
@@ -253,6 +249,8 @@ int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
     int nb_sectors, QEMUIOVector *qiov);
 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
     int nb_sectors, QEMUIOVector *qiov);
 /*
@@ -317,7 +315,8 @@ bool bdrv_recurse_is_first_non_filter(BlockDriverState *bs,
 bool bdrv_is_first_non_filter(BlockDriverState *candidate);
 
 /* check if a named node can be replaced when doing drive-mirror */
-BlockDriverState *check_to_replace_node(const char *node_name, Error **errp);
+BlockDriverState *check_to_replace_node(BlockDriverState *parent_bs,
+                                        const char *node_name, Error **errp);
 
 /* async block I/O */
 typedef void BlockDriverDirtyHandler(BlockDriverState *bs, int64_t sector,
@@ -338,10 +337,18 @@ void bdrv_aio_cancel_async(BlockAIOCB *acb);
 
 typedef struct BlockRequest {
     /* Fields to be filled by multiwrite caller */
-    int64_t sector;
-    int nb_sectors;
-    int flags;
-    QEMUIOVector *qiov;
+    union {
+        struct {
+            int64_t sector;
+            int nb_sectors;
+            int flags;
+            QEMUIOVector *qiov;
+        };
+        struct {
+            int req;
+            void *buf;
+        };
+    };
     BlockCompletionFunc *cb;
     void *opaque;
 
@@ -387,17 +394,11 @@ int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
 int bdrv_is_allocated_above(BlockDriverState *top, BlockDriverState *base,
                             int64_t sector_num, int nb_sectors, int *pnum);
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error);
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read);
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error);
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error);
 int bdrv_is_read_only(BlockDriverState *bs);
 int bdrv_is_sg(BlockDriverState *bs);
 int bdrv_enable_write_cache(BlockDriverState *bs);
 void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce);
-int bdrv_is_inserted(BlockDriverState *bs);
+bool bdrv_is_inserted(BlockDriverState *bs);
 int bdrv_media_changed(BlockDriverState *bs);
 void bdrv_lock_medium(BlockDriverState *bs, bool locked);
 void bdrv_eject(BlockDriverState *bs, bool eject_flag);
@@ -464,7 +465,6 @@ void bdrv_img_create(const char *filename, const char *fmt,
 size_t bdrv_min_mem_align(BlockDriverState *bs);
 /* Returns optimal alignment in bytes for bounce buffer */
 size_t bdrv_opt_mem_align(BlockDriverState *bs);
-void bdrv_set_guest_block_size(BlockDriverState *bs, int align);
 void *qemu_blockalign(BlockDriverState *bs, size_t size);
 void *qemu_blockalign0(BlockDriverState *bs, size_t size);
 void *qemu_try_blockalign(BlockDriverState *bs, size_t size);
@@ -503,7 +503,6 @@ void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
                            int64_t cur_sector, int nr_sectors);
 void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
                              int64_t cur_sector, int nr_sectors);
-void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_dirty_iter_init(BdrvDirtyBitmap *bitmap, struct HBitmapIter *hbi);
 void bdrv_set_dirty_iter(struct HBitmapIter *hbi, int64_t offset);
 int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);
@@ -582,7 +581,13 @@ typedef enum {
     BLKDBG_EVENT_MAX,
 } BlkDebugEvent;
 
-#define BLKDBG_EVENT(bs, evt) bdrv_debug_event(bs, evt)
+#define BLKDBG_EVENT(child, evt) \
+    do { \
+        if (child) { \
+            bdrv_debug_event(child->bs, evt); \
+        } \
+    } while (0)
+
 void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event);
 
 int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
@@ -614,6 +619,23 @@ void bdrv_io_plug(BlockDriverState *bs);
 void bdrv_io_unplug(BlockDriverState *bs);
 void bdrv_flush_io_queue(BlockDriverState *bs);
 
-BlockAcctStats *bdrv_get_stats(BlockDriverState *bs);
+/**
+ * bdrv_drained_begin:
+ *
+ * Begin a quiesced section for exclusive access to the BDS, by disabling
+ * external request sources including NBD server and device model. Note that
+ * this doesn't block timers or coroutines from submitting more requests, which
+ * means block_job_pause is still necessary.
+ *
+ * This function can be recursive.
+ */
+void bdrv_drained_begin(BlockDriverState *bs);
+
+/**
+ * bdrv_drained_end:
+ *
+ * End a quiescent section started by bdrv_drained_begin().
+ */
+void bdrv_drained_end(BlockDriverState *bs);
 
 #endif
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 14ad4c334..7029d41c1 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -26,9 +26,10 @@
 
 #include "block/accounting.h"
 #include "block/block.h"
+#include "block/throttle-groups.h"
 #include "qemu/option.h"
 #include "qemu/queue.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qemu/timer.h"
 #include "qapi-types.h"
 #include "qemu/hbitmap.h"
@@ -59,11 +60,19 @@
 
 #define BLOCK_PROBE_BUF_SIZE        512
 
+enum BdrvTrackedRequestType {
+    BDRV_TRACKED_READ,
+    BDRV_TRACKED_WRITE,
+    BDRV_TRACKED_FLUSH,
+    BDRV_TRACKED_IOCTL,
+    BDRV_TRACKED_DISCARD,
+};
+
 typedef struct BdrvTrackedRequest {
     BlockDriverState *bs;
     int64_t offset;
     unsigned int bytes;
-    bool is_write;
+    enum BdrvTrackedRequestType type;
 
     bool serialising;
     int64_t overlap_offset;
@@ -122,7 +131,6 @@ struct BlockDriver {
     int (*bdrv_write)(BlockDriverState *bs, int64_t sector_num,
                       const uint8_t *buf, int nb_sectors);
     void (*bdrv_close)(BlockDriverState *bs);
-    void (*bdrv_rebind)(BlockDriverState *bs);
     int (*bdrv_create)(const char *filename, QemuOpts *opts, Error **errp);
     int (*bdrv_set_key)(BlockDriverState *bs, const char *key);
     int (*bdrv_make_empty)(BlockDriverState *bs);
@@ -213,13 +221,12 @@ struct BlockDriver {
         const char *backing_file, const char *backing_fmt);
 
     /* removable device specific */
-    int (*bdrv_is_inserted)(BlockDriverState *bs);
+    bool (*bdrv_is_inserted)(BlockDriverState *bs);
     int (*bdrv_media_changed)(BlockDriverState *bs);
     void (*bdrv_eject)(BlockDriverState *bs, bool eject_flag);
     void (*bdrv_lock_medium)(BlockDriverState *bs, bool locked);
 
     /* to control generic scsi devices */
-    int (*bdrv_ioctl)(BlockDriverState *bs, unsigned long int req, void *buf);
     BlockAIOCB *(*bdrv_aio_ioctl)(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque);
@@ -288,6 +295,12 @@ struct BlockDriver {
      */
     int (*bdrv_probe_geometry)(BlockDriverState *bs, HDGeometry *geo);
 
+    /**
+     * Drain and stop any internal sources of requests in the driver, and
+     * remain so until next I/O callback (e.g. bdrv_co_writev) is called.
+     */
+    void (*bdrv_drain)(BlockDriverState *bs);
+
     QLIST_ENTRY(BlockDriver) list;
 };
 
@@ -339,6 +352,7 @@ struct BdrvChild {
     BlockDriverState *bs;
     const BdrvChildRole *role;
     QLIST_ENTRY(BdrvChild) next;
+    QLIST_ENTRY(BdrvChild) next_parent;
 };
 
 /*
@@ -378,9 +392,8 @@ struct BlockDriverState {
     QDict *full_open_options;
     char exact_filename[PATH_MAX];
 
-    BlockDriverState *backing_hd;
-    BdrvChild *backing_child;
-    BlockDriverState *file;
+    BdrvChild *backing;
+    BdrvChild *file;
 
     NotifierList close_notifiers;
 
@@ -390,7 +403,10 @@ struct BlockDriverState {
     /* number of in-flight serialising requests */
     unsigned int serialising_in_flight;
 
-    /* I/O throttling */
+    /* I/O throttling.
+     * throttle_state tells us if this BDS has I/O limits configured.
+     * io_limits_enabled tells us if they are currently being
+     * enforced, but it can be temporarily set to false */
     CoQueue      throttled_reqs[2];
     bool         io_limits_enabled;
     /* The following fields are protected by the ThrottleGroup lock.
@@ -400,8 +416,8 @@ struct BlockDriverState {
     unsigned       pending_reqs[2];
     QLIST_ENTRY(BlockDriverState) round_robin;
 
-    /* I/O stats (display with "info blockstats"). */
-    BlockAcctStats stats;
+    /* Offset after the highest byte written to */
+    uint64_t wr_highest_offset;
 
     /* I/O Limits */
     BlockLimits bl;
@@ -412,18 +428,9 @@ struct BlockDriverState {
     /* Alignment requirement for offset/length of I/O requests */
     unsigned int request_alignment;
 
-    /* the block size for which the guest device expects atomicity */
-    int guest_block_size;
-
     /* do we need to tell the quest if we have a volatile write cache? */
     int enable_write_cache;
 
-    /* NOTE: the following infos are only hints for real hardware
-       drivers. They are not used by the block driver */
-    BlockdevOnError on_read_error, on_write_error;
-    bool iostatus_enabled;
-    BlockDeviceIoStatus iostatus;
-
     /* the following member gives a name to every node on the bs graph. */
     char node_name[32];
     /* element of the list of named nodes building the graph */
@@ -446,6 +453,7 @@ struct BlockDriverState {
      * parent node of this node. */
     BlockDriverState *inherits_from;
     QLIST_HEAD(, BdrvChild) children;
+    QLIST_HEAD(, BdrvChild) parents;
 
     QDict *options;
     BlockdevDetectZeroesOptions detect_zeroes;
@@ -456,8 +464,24 @@ struct BlockDriverState {
     /* threshold limit for writes, in bytes. "High water mark". */
     uint64_t write_threshold_offset;
     NotifierWithReturn write_threshold_notifier;
+
+    int quiesce_counter;
 };
 
+struct BlockBackendRootState {
+    int open_flags;
+    bool read_only;
+    BlockdevDetectZeroesOptions detect_zeroes;
+
+    char *throttle_group;
+    ThrottleState *throttle_state;
+};
+
+static inline BlockDriverState *backing_bs(BlockDriverState *bs)
+{
+    return bs->backing ? bs->backing->bs : NULL;
+}
+
 
 /* Essential block drivers which must always be statically linked into qemu, and
  * which therefore can be accessed without using bdrv_find_format() */
@@ -465,6 +489,8 @@ extern BlockDriver bdrv_file;
 extern BlockDriver bdrv_raw;
 extern BlockDriver bdrv_qcow2;
 
+extern QTAILQ_HEAD(BdrvStates, BlockDriverState) bdrv_states;
+
 /**
  * bdrv_setup_io_funcs:
  *
@@ -496,7 +522,7 @@ void bdrv_add_before_write_notifier(BlockDriverState *bs,
  *
  * May be called from .bdrv_detach_aio_context() to detach children from the
  * current #AioContext.  This is only needed by block drivers that manage their
- * own children.  Both ->file and ->backing_hd are automatically handled and
+ * own children.  Both ->file and ->backing are automatically handled and
  * block drivers should not call this function on them explicitly.
  */
 void bdrv_detach_aio_context(BlockDriverState *bs);
@@ -506,7 +532,7 @@ void bdrv_detach_aio_context(BlockDriverState *bs);
  *
  * May be called from .bdrv_attach_aio_context() to attach children to the new
  * #AioContext.  This is only needed by block drivers that manage their own
- * children.  Both ->file and ->backing_hd are automatically handled and block
+ * children.  Both ->file and ->backing are automatically handled and block
  * drivers should not call this function on them explicitly.
  */
 void bdrv_attach_aio_context(BlockDriverState *bs,
@@ -643,6 +669,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
  * @on_target_error: The action to take upon error writing to the target.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
+ * @txn: Transaction that this job is part of (may be NULL).
  *
  * Start a backup operation on @bs.  Clusters in @bs are written to @target
  * until the job is cancelled or manually completed.
@@ -653,15 +680,22 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockCompletionFunc *cb, void *opaque,
-                  Error **errp);
+                  BlockJobTxn *txn, Error **errp);
+
+void blk_set_bs(BlockBackend *blk, BlockDriverState *bs);
 
 void blk_dev_change_media_cb(BlockBackend *blk, bool load);
 bool blk_dev_has_removable_media(BlockBackend *blk);
+bool blk_dev_has_tray(BlockBackend *blk);
 void blk_dev_eject_request(BlockBackend *blk, bool force);
 bool blk_dev_is_tray_open(BlockBackend *blk);
 bool blk_dev_is_medium_locked(BlockBackend *blk);
 void blk_dev_resize_cb(BlockBackend *blk);
 
 void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+bool bdrv_requests_pending(BlockDriverState *bs);
+
+void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out);
+void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in);
 
 #endif /* BLOCK_INT_H */
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index dd9d5e6aa..d84ccd8d2 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -50,6 +50,26 @@ typedef struct BlockJobDriver {
      * manually.
      */
     void (*complete)(BlockJob *job, Error **errp);
+
+    /**
+     * If the callback is not NULL, it will be invoked when all the jobs
+     * belonging to the same transaction complete; or upon this job's
+     * completion if it is not in a transaction. Skipped if NULL.
+     *
+     * All jobs will complete with a call to either .commit() or .abort() but
+     * never both.
+     */
+    void (*commit)(BlockJob *job);
+
+    /**
+     * If the callback is not NULL, it will be invoked when any job in the
+     * same transaction fails; or upon this job's failure (due to error or
+     * cancellation) if it is not in a transaction. Skipped if NULL.
+     *
+     * All jobs will complete with a call to either .commit() or .abort() but
+     * never both.
+     */
+    void (*abort)(BlockJob *job);
 } BlockJobDriver;
 
 /**
@@ -65,6 +85,14 @@ struct BlockJob {
     BlockDriverState *bs;
 
     /**
+     * The ID of the block job. Currently the BlockBackend name of the BDS
+     * owning the job at the time when the job is started.
+     *
+     * TODO Decouple block job IDs from BlockBackend names
+     */
+    char *id;
+
+    /**
      * The coroutine that executes the job.  If not NULL, it is
      * reentered when busy is false and the job is cancelled.
      */
@@ -122,6 +150,21 @@ struct BlockJob {
 
     /** The opaque value that is passed to the completion function.  */
     void *opaque;
+
+    /** Reference count of the block job */
+    int refcnt;
+
+    /* True if this job has reported completion by calling block_job_completed.
+     */
+    bool completed;
+
+    /* ret code passed to block_job_completed.
+     */
+    int ret;
+
+    /** Non-NULL if this job is part of a transaction */
+    BlockJobTxn *txn;
+    QLIST_ENTRY(BlockJob) txn_list;
 };
 
 /**
@@ -166,12 +209,21 @@ void block_job_sleep_ns(BlockJob *job, QEMUClockType type, int64_t ns);
 void block_job_yield(BlockJob *job);
 
 /**
- * block_job_release:
+ * block_job_ref:
+ * @bs: The block device.
+ *
+ * Grab a reference to the block job. Should be paired with block_job_unref.
+ */
+void block_job_ref(BlockJob *job);
+
+/**
+ * block_job_unref:
  * @bs: The block device.
  *
- * Release job resources when an error occurred or job completed.
+ * Release reference to the block job and release resources if it is the last
+ * reference.
  */
-void block_job_release(BlockDriverState *bs);
+void block_job_unref(BlockJob *job);
 
 /**
  * block_job_completed:
@@ -356,4 +408,39 @@ void block_job_defer_to_main_loop(BlockJob *job,
                                   BlockJobDeferToMainLoopFn *fn,
                                   void *opaque);
 
+/**
+ * block_job_txn_new:
+ *
+ * Allocate and return a new block job transaction.  Jobs can be added to the
+ * transaction using block_job_txn_add_job().
+ *
+ * The transaction is automatically freed when the last job completes or is
+ * cancelled.
+ *
+ * All jobs in the transaction either complete successfully or fail/cancel as a
+ * group.  Jobs wait for each other before completing.  Cancelling one job
+ * cancels all jobs in the transaction.
+ */
+BlockJobTxn *block_job_txn_new(void);
+
+/**
+ * block_job_txn_unref:
+ *
+ * Release a reference that was previously acquired with block_job_txn_add_job
+ * or block_job_txn_new. If it's the last reference to the object, it will be
+ * freed.
+ */
+void block_job_txn_unref(BlockJobTxn *txn);
+
+/**
+ * block_job_txn_add_job:
+ * @txn: The transaction (may be NULL)
+ * @job: Job to add to the transaction
+ *
+ * Add @job to the transaction.  The @job must not already be in a transaction.
+ * The caller must call either block_job_txn_unref() or block_job_completed()
+ * to release the reference that is automatically grabbed here.
+ */
+void block_job_txn_add_job(BlockJobTxn *txn, BlockJob *job);
+
 #endif
diff --git a/include/block/scsi.h b/include/block/scsi.h
index edde960d1..a311341e6 100644
--- a/include/block/scsi.h
+++ b/include/block/scsi.h
@@ -229,7 +229,7 @@ const char *scsi_command_name(uint8_t cmd);
 #define MODE_PAGE_TO_PROTECT                  0x1d
 #define MODE_PAGE_CAPABILITIES                0x2a
 #define MODE_PAGE_ALLS                        0x3f
-/* Not in Mt. Fuji, but in ATAPI 2.6 -- depricated now in favor
+/* Not in Mt. Fuji, but in ATAPI 2.6 -- deprecated now in favor
  * of MODE_PAGE_SENSE_POWER */
 #define MODE_PAGE_CDROM                       0x0d
 
diff --git a/include/block/snapshot.h b/include/block/snapshot.h
index 770d9bbc8..c6910da63 100644
--- a/include/block/snapshot.h
+++ b/include/block/snapshot.h
@@ -63,9 +63,9 @@ int bdrv_snapshot_delete(BlockDriverState *bs,
                          const char *snapshot_id,
                          const char *name,
                          Error **errp);
-void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
-                                        const char *id_or_name,
-                                        Error **errp);
+int bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                       const char *id_or_name,
+                                       Error **errp);
 int bdrv_snapshot_list(BlockDriverState *bs,
                        QEMUSnapshotInfo **psn_info);
 int bdrv_snapshot_load_tmp(BlockDriverState *bs,
@@ -75,4 +75,22 @@ int bdrv_snapshot_load_tmp(BlockDriverState *bs,
 int bdrv_snapshot_load_tmp_by_id_or_name(BlockDriverState *bs,
                                          const char *id_or_name,
                                          Error **errp);
+
+
+/* Group operations. All block drivers are involved.
+ * These functions will properly handle dataplane (take aio_context_acquire
+ * when appropriate for appropriate block drivers */
+
+bool bdrv_all_can_snapshot(BlockDriverState **first_bad_bs);
+int bdrv_all_delete_snapshot(const char *name, BlockDriverState **first_bsd_bs,
+                             Error **err);
+int bdrv_all_goto_snapshot(const char *name, BlockDriverState **first_bsd_bs);
+int bdrv_all_find_snapshot(const char *name, BlockDriverState **first_bad_bs);
+int bdrv_all_create_snapshot(QEMUSnapshotInfo *sn,
+                             BlockDriverState *vm_state_bs,
+                             uint64_t vm_state_size,
+                             BlockDriverState **first_bad_bs);
+
+BlockDriverState *bdrv_all_find_vmstate_bs(void);
+
 #endif
diff --git a/include/block/throttle-groups.h b/include/block/throttle-groups.h
index fab113f6d..aba28f30b 100644
--- a/include/block/throttle-groups.h
+++ b/include/block/throttle-groups.h
@@ -30,6 +30,9 @@
 
 const char *throttle_group_get_name(BlockDriverState *bs);
 
+ThrottleState *throttle_group_incref(const char *name);
+void throttle_group_unref(ThrottleState *ts);
+
 void throttle_group_config(BlockDriverState *bs, ThrottleConfig *cfg);
 void throttle_group_get_config(BlockDriverState *bs, ThrottleConfig *cfg);
 
@@ -40,7 +43,4 @@ void coroutine_fn throttle_group_co_io_limits_intercept(BlockDriverState *bs,
                                                         unsigned int bytes,
                                                         bool is_write);
 
-void throttle_group_lock(BlockDriverState *bs);
-void throttle_group_unlock(BlockDriverState *bs);
-
 #endif
diff --git a/include/crypto/tlscreds.h b/include/crypto/tlscreds.h
new file mode 100644
index 000000000..21761b7ce
--- /dev/null
+++ b/include/crypto/tlscreds.h
@@ -0,0 +1,68 @@
+/*
+ * QEMU crypto TLS credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_H__
+#define QCRYPTO_TLSCRED_H__
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "qom/object.h"
+
+#ifdef CONFIG_GNUTLS
+#include <gnutls/gnutls.h>
+#endif
+
+#define TYPE_QCRYPTO_TLS_CREDS "tls-creds"
+#define QCRYPTO_TLS_CREDS(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCreds, (obj), TYPE_QCRYPTO_TLS_CREDS)
+
+typedef struct QCryptoTLSCreds QCryptoTLSCreds;
+typedef struct QCryptoTLSCredsClass QCryptoTLSCredsClass;
+
+#define QCRYPTO_TLS_CREDS_DH_PARAMS "dh-params.pem"
+
+
+/**
+ * QCryptoTLSCreds:
+ *
+ * The QCryptoTLSCreds object is an abstract base for different
+ * types of TLS handshake credentials. Most commonly the
+ * QCryptoTLSCredsX509 subclass will be used to provide x509
+ * certificate credentials.
+ */
+
+struct QCryptoTLSCreds {
+    Object parent_obj;
+    char *dir;
+    QCryptoTLSCredsEndpoint endpoint;
+#ifdef CONFIG_GNUTLS
+    gnutls_dh_params_t dh_params;
+#endif
+    bool verifyPeer;
+};
+
+
+struct QCryptoTLSCredsClass {
+    ObjectClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCRED_H__ */
+
diff --git a/include/crypto/tlscredsanon.h b/include/crypto/tlscredsanon.h
new file mode 100644
index 000000000..d3976b84b
--- /dev/null
+++ b/include/crypto/tlscredsanon.h
@@ -0,0 +1,112 @@
+/*
+ * QEMU crypto TLS anonymous credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_ANON_H__
+#define QCRYPTO_TLSCRED_ANON_H__
+
+#include "crypto/tlscreds.h"
+
+#define TYPE_QCRYPTO_TLS_CREDS_ANON "tls-creds-anon"
+#define QCRYPTO_TLS_CREDS_ANON(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCredsAnon, (obj), TYPE_QCRYPTO_TLS_CREDS_ANON)
+
+
+typedef struct QCryptoTLSCredsAnon QCryptoTLSCredsAnon;
+typedef struct QCryptoTLSCredsAnonClass QCryptoTLSCredsAnonClass;
+
+/**
+ * QCryptoTLSCredsAnon:
+ *
+ * The QCryptoTLSCredsAnon object provides a representation
+ * of anonymous credentials used perform a TLS handshake.
+ * This is primarily provided for backwards compatibility and
+ * its use is discouraged as it has poor security characteristics
+ * due to lacking MITM attack protection amongst other problems.
+ *
+ * This is a user creatable object, which can be instantiated
+ * via object_new_propv():
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects in code</title>
+ *   <programlisting>
+ *   Object *obj;
+ *   Error *err = NULL;
+ *   obj = object_new_propv(TYPE_QCRYPTO_TLS_CREDS_ANON,
+ *                          "tlscreds0",
+ *                          &err,
+ *                          "endpoint", "server",
+ *                          "dir", "/path/x509/cert/dir",
+ *                          "verify-peer", "yes",
+ *                          NULL);
+ *   </programlisting>
+ * </example>
+ *
+ * Or via QMP:
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects via QMP</title>
+ *   <programlisting>
+ *    {
+ *       "execute": "object-add", "arguments": {
+ *          "id": "tlscreds0",
+ *          "qom-type": "tls-creds-anon",
+ *          "props": {
+ *             "endpoint": "server",
+ *             "dir": "/path/to/x509/cert/dir",
+ *             "verify-peer": false
+ *          }
+ *       }
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ *
+ * Or via the CLI:
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects via CLI</title>
+ *   <programlisting>
+ *  qemu-system-x86_64 -object tls-creds-anon,id=tlscreds0,\
+ *          endpoint=server,verify-peer=off,\
+ *          dir=/path/to/x509/certdir/
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+
+struct QCryptoTLSCredsAnon {
+    QCryptoTLSCreds parent_obj;
+#ifdef CONFIG_GNUTLS
+    union {
+        gnutls_anon_server_credentials_t server;
+        gnutls_anon_client_credentials_t client;
+    } data;
+#endif
+};
+
+
+struct QCryptoTLSCredsAnonClass {
+    QCryptoTLSCredsClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCRED_H__ */
+
diff --git a/include/crypto/tlscredsx509.h b/include/crypto/tlscredsx509.h
new file mode 100644
index 000000000..b9785fddc
--- /dev/null
+++ b/include/crypto/tlscredsx509.h
@@ -0,0 +1,113 @@
+/*
+ * QEMU crypto TLS x509 credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_X509_H__
+#define QCRYPTO_TLSCRED_X509_H__
+
+#include "crypto/tlscreds.h"
+
+#define TYPE_QCRYPTO_TLS_CREDS_X509 "tls-creds-x509"
+#define QCRYPTO_TLS_CREDS_X509(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCredsX509, (obj), TYPE_QCRYPTO_TLS_CREDS_X509)
+
+typedef struct QCryptoTLSCredsX509 QCryptoTLSCredsX509;
+typedef struct QCryptoTLSCredsX509Class QCryptoTLSCredsX509Class;
+
+#define QCRYPTO_TLS_CREDS_X509_CA_CERT "ca-cert.pem"
+#define QCRYPTO_TLS_CREDS_X509_CA_CRL "ca-crl.pem"
+#define QCRYPTO_TLS_CREDS_X509_SERVER_KEY "server-key.pem"
+#define QCRYPTO_TLS_CREDS_X509_SERVER_CERT "server-cert.pem"
+#define QCRYPTO_TLS_CREDS_X509_CLIENT_KEY "client-key.pem"
+#define QCRYPTO_TLS_CREDS_X509_CLIENT_CERT "client-cert.pem"
+
+
+/**
+ * QCryptoTLSCredsX509:
+ *
+ * The QCryptoTLSCredsX509 object provides a representation
+ * of x509 credentials used to perform a TLS handshake.
+ *
+ * This is a user creatable object, which can be instantiated
+ * via object_new_propv():
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects in code</title>
+ *   <programlisting>
+ *   Object *obj;
+ *   Error *err = NULL;
+ *   obj = object_new_propv(TYPE_QCRYPTO_TLS_CREDS_X509,
+ *                          "tlscreds0",
+ *                          &err,
+ *                          "endpoint", "server",
+ *                          "dir", "/path/x509/cert/dir",
+ *                          "verify-peer", "yes",
+ *                          NULL);
+ *   </programlisting>
+ * </example>
+ *
+ * Or via QMP:
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects via QMP</title>
+ *   <programlisting>
+ *    {
+ *       "execute": "object-add", "arguments": {
+ *          "id": "tlscreds0",
+ *          "qom-type": "tls-creds-x509",
+ *          "props": {
+ *             "endpoint": "server",
+ *             "dir": "/path/to/x509/cert/dir",
+ *             "verify-peer": false
+ *          }
+ *       }
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ *
+ * Or via the CLI:
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects via CLI</title>
+ *   <programlisting>
+ *  qemu-system-x86_64 -object tls-creds-x509,id=tlscreds0,\
+ *          endpoint=server,verify-peer=off,\
+ *          dir=/path/to/x509/certdir/
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+struct QCryptoTLSCredsX509 {
+    QCryptoTLSCreds parent_obj;
+#ifdef CONFIG_GNUTLS
+    gnutls_certificate_credentials_t data;
+#endif
+    bool sanityCheck;
+};
+
+
+struct QCryptoTLSCredsX509Class {
+    QCryptoTLSCredsClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCRED_X509_H__ */
+
diff --git a/include/crypto/tlssession.h b/include/crypto/tlssession.h
new file mode 100644
index 000000000..b38fe6954
--- /dev/null
+++ b/include/crypto/tlssession.h
@@ -0,0 +1,322 @@
+/*
+ * QEMU crypto TLS session support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLS_SESSION_H__
+#define QCRYPTO_TLS_SESSION_H__
+
+#include "crypto/tlscreds.h"
+
+/**
+ * QCryptoTLSSession:
+ *
+ * The QCryptoTLSSession object encapsulates the
+ * logic to integrate with a TLS providing library such
+ * as GNUTLS, to setup and run TLS sessions.
+ *
+ * The API is designed such that it has no assumption about
+ * the type of transport it is running over. It may be a
+ * traditional TCP socket, or something else entirely. The
+ * only requirement is a full-duplex stream of some kind.
+ *
+ * <example>
+ *   <title>Using TLS session objects</title>
+ *   <programlisting>
+ * static ssize_t mysock_send(const char *buf, size_t len,
+ *                            void *opaque)
+ * {
+ *    int fd = GPOINTER_TO_INT(opaque);
+ *
+ *    return write(*fd, buf, len);
+ * }
+ *
+ * static ssize_t mysock_recv(const char *buf, size_t len,
+ *                            void *opaque)
+ * {
+ *    int fd = GPOINTER_TO_INT(opaque);
+ *
+ *    return read(*fd, buf, len);
+ * }
+ *
+ * static int mysock_run_tls(int sockfd,
+ *                           QCryptoTLSCreds *creds,
+ *                           Error *erp)
+ * {
+ *    QCryptoTLSSession *sess;
+ *
+ *    sess = qcrypto_tls_session_new(creds,
+ *                                   "vnc.example.com",
+ *                                   NULL,
+ *                                   QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+ *                                   errp);
+ *    if (sess == NULL) {
+ *       return -1;
+ *    }
+ *
+ *    qcrypto_tls_session_set_callbacks(sess,
+ *                                      mysock_send,
+ *                                      mysock_recv
+ *                                      GINT_TO_POINTER(fd));
+ *
+ *    while (1) {
+ *       if (qcrypto_tls_session_handshake(sess, errp) < 0) {
+ *           qcrypto_tls_session_free(sess);
+ *           return -1;
+ *       }
+ *
+ *       switch(qcrypto_tls_session_get_handshake_status(sess)) {
+ *       case QCRYPTO_TLS_HANDSHAKE_COMPLETE:
+ *           if (qcrypto_tls_session_check_credentials(sess, errp) < )) {
+ *               qcrypto_tls_session_free(sess);
+ *               return -1;
+ *           }
+ *           goto done;
+ *       case QCRYPTO_TLS_HANDSHAKE_RECVING:
+ *           ...wait for GIO_IN event on fd...
+ *           break;
+ *       case QCRYPTO_TLS_HANDSHAKE_SENDING:
+ *           ...wait for GIO_OUT event on fd...
+ *           break;
+ *       }
+ *    }
+ *   done:
+ *
+ *    ....send/recv payload data on sess...
+ *
+ *    qcrypto_tls_session_free(sess):
+ * }
+ *   </programlisting>
+ * </example>
+ */
+
+typedef struct QCryptoTLSSession QCryptoTLSSession;
+
+
+/**
+ * qcrypto_tls_session_new:
+ * @creds: pointer to a TLS credentials object
+ * @hostname: optional hostname to validate
+ * @aclname: optional ACL to validate peer credentials against
+ * @endpoint: role of the TLS session, client or server
+ * @errp: pointer to an uninitialized error object
+ *
+ * Create a new TLS session object that will be used to
+ * negotiate a TLS session over an arbitrary data channel.
+ * The session object can operate as either the server or
+ * client, according to the value of the @endpoint argument.
+ *
+ * For clients, the @hostname parameter should hold the full
+ * unmodified hostname as requested by the user. This will
+ * be used to verify the against the hostname reported in
+ * the server's credentials (aka x509 certificate).
+ *
+ * The @aclname parameter (optionally) specifies the name
+ * of an access control list that will be used to validate
+ * the peer's credentials. For x509 credentials, the ACL
+ * will be matched against the CommonName shown in the peer's
+ * certificate. If the session is acting as a server, setting
+ * an ACL will require that the client provide a validate
+ * x509 client certificate.
+ *
+ * After creating the session object, the I/O callbacks
+ * must be set using the qcrypto_tls_session_set_callbacks()
+ * method. A TLS handshake sequence must then be completed
+ * using qcrypto_tls_session_handshake(), before payload
+ * data is permitted to be sent/received.
+ *
+ * The session object must be released by calling
+ * qcrypto_tls_session_free() when no longer required
+ *
+ * Returns: a TLS session object, or NULL on error.
+ */
+QCryptoTLSSession *qcrypto_tls_session_new(QCryptoTLSCreds *creds,
+                                           const char *hostname,
+                                           const char *aclname,
+                                           QCryptoTLSCredsEndpoint endpoint,
+                                           Error **errp);
+
+/**
+ * qcrypto_tls_session_free:
+ * @sess: the TLS session object
+ *
+ * Release all memory associated with the TLS session
+ * object previously allocated by qcrypto_tls_session_new()
+ */
+void qcrypto_tls_session_free(QCryptoTLSSession *sess);
+
+/**
+ * qcrypto_tls_session_check_credentials:
+ * @sess: the TLS session object
+ * @errp: pointer to an uninitialized error object
+ *
+ * Validate the peer's credentials after a successful
+ * TLS handshake. It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns 0 if the credentials validated, -1 on error
+ */
+int qcrypto_tls_session_check_credentials(QCryptoTLSSession *sess,
+                                          Error **errp);
+
+typedef ssize_t (*QCryptoTLSSessionWriteFunc)(const char *buf,
+                                              size_t len,
+                                              void *opaque);
+typedef ssize_t (*QCryptoTLSSessionReadFunc)(char *buf,
+                                             size_t len,
+                                             void *opaque);
+
+/**
+ * qcrypto_tls_session_set_callbacks:
+ * @sess: the TLS session object
+ * @writeFunc: callback for sending data
+ * @readFunc: callback to receiving data
+ * @opaque: data to pass to callbacks
+ *
+ * Sets the callback functions that are to be used for sending
+ * and receiving data on the underlying data channel. Typically
+ * the callbacks to write/read to/from a TCP socket, but there
+ * is no assumption made about the type of channel used.
+ *
+ * The @writeFunc callback will be passed the encrypted
+ * data to send to the remote peer.
+ *
+ * The @readFunc callback will be passed a pointer to fill
+ * with encrypted data received from the remote peer
+ */
+void qcrypto_tls_session_set_callbacks(QCryptoTLSSession *sess,
+                                       QCryptoTLSSessionWriteFunc writeFunc,
+                                       QCryptoTLSSessionReadFunc readFunc,
+                                       void *opaque);
+
+/**
+ * qcrypto_tls_session_write:
+ * @sess: the TLS session object
+ * @buf: the plain text to send
+ * @len: the length of @buf
+ *
+ * Encrypt @len bytes of the data in @buf and send
+ * it to the remote peer using the callback previously
+ * registered with qcrypto_tls_session_set_callbacks()
+ *
+ * It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns: the number of bytes sent, or -1 on error
+ */
+ssize_t qcrypto_tls_session_write(QCryptoTLSSession *sess,
+                                  const char *buf,
+                                  size_t len);
+
+/**
+ * qcrypto_tls_session_read:
+ * @sess: the TLS session object
+ * @buf: to fill with plain text received
+ * @len: the length of @buf
+ *
+ * Receive up to @len bytes of data from the remote peer
+ * using the callback previously registered with
+ * qcrypto_tls_session_set_callbacks(), decrypt it and
+ * store it in @buf.
+ *
+ * It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns: the number of bytes received, or -1 on error
+ */
+ssize_t qcrypto_tls_session_read(QCryptoTLSSession *sess,
+                                 char *buf,
+                                 size_t len);
+
+/**
+ * qcrypto_tls_session_handshake:
+ * @sess: the TLS session object
+ * @errp: pointer to an uninitialized error object
+ *
+ * Start, or continue, a TLS handshake sequence. If
+ * the underlying data channel is non-blocking, then
+ * this method may return control before the handshake
+ * is complete. On non-blocking channels the
+ * qcrypto_tls_session_get_handshake_status() method
+ * should be used to determine whether the handshake
+ * has completed, or is waiting to send or receive
+ * data. In the latter cases, the caller should setup
+ * an event loop watch and call this method again
+ * once the underlying data channel is ready to read
+ * or write again
+ */
+int qcrypto_tls_session_handshake(QCryptoTLSSession *sess,
+                                  Error **errp);
+
+typedef enum {
+    QCRYPTO_TLS_HANDSHAKE_COMPLETE,
+    QCRYPTO_TLS_HANDSHAKE_SENDING,
+    QCRYPTO_TLS_HANDSHAKE_RECVING,
+} QCryptoTLSSessionHandshakeStatus;
+
+/**
+ * qcrypto_tls_session_get_handshake_status:
+ * @sess: the TLS session object
+ *
+ * Check the status of the TLS handshake. This
+ * is used with non-blocking data channels to
+ * determine whether the handshake is waiting
+ * to send or receive further data to/from the
+ * remote peer.
+ *
+ * Once this returns QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ * it is permitted to send/receive payload data on
+ * the channel
+ */
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *sess);
+
+/**
+ * qcrypto_tls_session_get_key_size:
+ * @sess: the TLS session object
+ * @errp: pointer to an uninitialized error object
+ *
+ * Check the size of the data channel encryption key
+ *
+ * Returns: the length in bytes of the encryption key
+ * or -1 on error
+ */
+int qcrypto_tls_session_get_key_size(QCryptoTLSSession *sess,
+                                     Error **errp);
+
+/**
+ * qcrypto_tls_session_get_peer_name:
+ * @sess: the TLS session object
+ *
+ * Get the identified name of the remote peer. If the
+ * TLS session was negotiated using x509 certificate
+ * credentials, this will return the CommonName from
+ * the peer's certificate. If no identified name is
+ * available it will return NULL.
+ *
+ * The returned data must be released with g_free()
+ * when no longer required.
+ *
+ * Returns: the peer's name or NULL.
+ */
+char *qcrypto_tls_session_get_peer_name(QCryptoTLSSession *sess);
+
+#endif /* QCRYPTO_TLS_SESSION_H__ */
diff --git a/include/elf.h b/include/elf.h
index 4afd474d5..66add810d 100644
--- a/include/elf.h
+++ b/include/elf.h
@@ -133,6 +133,11 @@ typedef int64_t  Elf64_Sxword;
 
 #define EM_AARCH64  183
 
+#define EM_TILEGX   191 /* TILE-Gx */
+
+#define EM_MOXIE           223     /* Moxie processor family */
+#define EM_MOXIE_OLD       0xFEED
+
 /* This is the info that is needed to parse the dynamic section of the file */
 #define DT_NULL		0
 #define DT_NEEDED	1
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index ea6a9a667..83b1781af 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -160,18 +160,11 @@ static inline void tswap64s(uint64_t *s)
 /* On some host systems the guest address space is reserved on the host.
  * This allows the guest address space to be offset to a convenient location.
  */
-#if defined(CONFIG_USE_GUEST_BASE)
 extern unsigned long guest_base;
 extern int have_guest_base;
 extern unsigned long reserved_va;
-#define GUEST_BASE guest_base
-#define RESERVED_VA reserved_va
-#else
-#define GUEST_BASE 0ul
-#define RESERVED_VA 0ul
-#endif
 
-#define GUEST_ADDR_MAX (RESERVED_VA ? RESERVED_VA : \
+#define GUEST_ADDR_MAX (reserved_va ? reserved_va : \
                                     (1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
 #endif
 
@@ -181,11 +174,13 @@ extern unsigned long reserved_va;
 #define TARGET_PAGE_MASK ~(TARGET_PAGE_SIZE - 1)
 #define TARGET_PAGE_ALIGN(addr) (((addr) + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK)
 
-/* ??? These should be the larger of uintptr_t and target_ulong.  */
+/* Using intptr_t ensures that qemu_*_page_mask is sign-extended even
+ * when intptr_t is 32-bit and we are aligning a long long.
+ */
 extern uintptr_t qemu_real_host_page_size;
-extern uintptr_t qemu_real_host_page_mask;
+extern intptr_t qemu_real_host_page_mask;
 extern uintptr_t qemu_host_page_size;
-extern uintptr_t qemu_host_page_mask;
+extern intptr_t qemu_host_page_mask;
 
 #define HOST_PAGE_ALIGN(addr) (((addr) + qemu_host_page_size - 1) & qemu_host_page_mask)
 #define REAL_HOST_PAGE_ALIGN(addr) (((addr) + qemu_real_host_page_size - 1) & \
@@ -273,44 +268,6 @@ CPUArchState *cpu_copy(CPUArchState *env);
 
 #if !defined(CONFIG_USER_ONLY)
 
-/* memory API */
-
-typedef struct RAMBlock RAMBlock;
-
-struct RAMBlock {
-    struct rcu_head rcu;
-    struct MemoryRegion *mr;
-    uint8_t *host;
-    ram_addr_t offset;
-    ram_addr_t used_length;
-    ram_addr_t max_length;
-    void (*resized)(const char*, uint64_t length, void *host);
-    uint32_t flags;
-    /* Protected by iothread lock.  */
-    char idstr[256];
-    /* RCU-enabled, writes protected by the ramlist lock */
-    QLIST_ENTRY(RAMBlock) next;
-    int fd;
-};
-
-static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
-{
-    assert(offset < block->used_length);
-    assert(block->host);
-    return (char *)block->host + offset;
-}
-
-typedef struct RAMList {
-    QemuMutex mutex;
-    /* Protected by the iothread lock.  */
-    unsigned long *dirty_memory[DIRTY_MEMORY_NUM];
-    RAMBlock *mru_block;
-    /* RCU-enabled, writes protected by the ramlist lock. */
-    QLIST_HEAD(, RAMBlock) blocks;
-    uint32_t version;
-} RAMList;
-extern RAMList ram_list;
-
 /* Flags stored in the low bits of the TLB virtual address.  These are
    defined so that fast path ram access is all zeros.  */
 /* Zero if TLB entry is valid.  */
@@ -323,9 +280,6 @@ extern RAMList ram_list;
 
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);
 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf);
-ram_addr_t last_ram_offset(void);
-void qemu_mutex_lock_ramlist(void);
-void qemu_mutex_unlock_ramlist(void);
 #endif /* !CONFIG_USER_ONLY */
 
 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
index 9fb1d541d..85aa4033e 100644
--- a/include/exec/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -64,8 +64,12 @@ typedef uint32_t CPUReadMemoryFunc(void *opaque, hwaddr addr);
 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
 /* This should not be used by devices.  */
 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr);
+RAMBlock *qemu_ram_block_by_name(const char *name);
+RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
+                                   ram_addr_t *ram_addr, ram_addr_t *offset);
 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev);
 void qemu_ram_unset_idstr(ram_addr_t addr);
+const char *qemu_ram_get_idstr(RAMBlock *rb);
 
 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
                             int len, int is_write);
diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
index 98b9cff31..5093be26a 100644
--- a/include/exec/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -105,17 +105,18 @@ typedef struct CPUTLBEntry {
        bit 3                      : indicates that the entry is invalid
        bit 2..0                   : zero
     */
-    target_ulong addr_read;
-    target_ulong addr_write;
-    target_ulong addr_code;
-    /* Addend to virtual address to get host address.  IO accesses
-       use the corresponding iotlb value.  */
-    uintptr_t addend;
-    /* padding to get a power of two size */
-    uint8_t dummy[(1 << CPU_TLB_ENTRY_BITS) -
-                  (sizeof(target_ulong) * 3 +
-                   ((-sizeof(target_ulong) * 3) & (sizeof(uintptr_t) - 1)) +
-                   sizeof(uintptr_t))];
+    union {
+        struct {
+            target_ulong addr_read;
+            target_ulong addr_write;
+            target_ulong addr_code;
+            /* Addend to virtual address to get host address.  IO accesses
+               use the corresponding iotlb value.  */
+            uintptr_t addend;
+        };
+        /* padding to get a power of two size */
+        uint8_t dummy[1 << CPU_TLB_ENTRY_BITS];
+    };
 } CPUTLBEntry;
 
 QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS));
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index 1239c60f2..b573df53b 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -49,20 +49,20 @@
 
 #if defined(CONFIG_USER_ONLY)
 /* All direct uses of g2h and h2g need to go away for usermode softmmu.  */
-#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + GUEST_BASE))
+#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + guest_base))
 
 #if HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS
 #define h2g_valid(x) 1
 #else
 #define h2g_valid(x) ({ \
-    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
+    unsigned long __guest = (unsigned long)(x) - guest_base; \
     (__guest < (1ul << TARGET_VIRT_ADDR_SPACE_BITS)) && \
-    (!RESERVED_VA || (__guest < RESERVED_VA)); \
+    (!reserved_va || (__guest < reserved_va)); \
 })
 #endif
 
 #define h2g_nocheck(x) ({ \
-    unsigned long __ret = (unsigned long)(x) - GUEST_BASE; \
+    unsigned long __ret = (unsigned long)(x) - guest_base; \
     (abi_ulong)__ret; \
 })
 
@@ -113,25 +113,6 @@
 /* The memory helpers for tcg-generated code need tcg_target_long etc.  */
 #include "tcg.h"
 
-uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
-void helper_stb_mmu(CPUArchState *env, target_ulong addr,
-                    uint8_t val, int mmu_idx);
-void helper_stw_mmu(CPUArchState *env, target_ulong addr,
-                    uint16_t val, int mmu_idx);
-void helper_stl_mmu(CPUArchState *env, target_ulong addr,
-                    uint32_t val, int mmu_idx);
-void helper_stq_mmu(CPUArchState *env, target_ulong addr,
-                    uint64_t val, int mmu_idx);
-
-uint8_t helper_ldb_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
 #ifdef MMU_MODE0_SUFFIX
 #define CPU_MMU_INDEX 0
 #define MEMSUFFIX MMU_MODE0_SUFFIX
@@ -363,7 +344,7 @@ uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 #endif /* (NB_MMU_MODES > 12) */
 
 /* these access are slower, they must be as rare as possible */
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define CPU_MMU_INDEX (cpu_mmu_index(env, false))
 #define MEMSUFFIX _data
 #define DATA_SIZE 1
 #include "exec/cpu_ldst_template.h"
@@ -379,7 +360,7 @@ uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 #undef CPU_MMU_INDEX
 #undef MEMSUFFIX
 
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define CPU_MMU_INDEX (cpu_mmu_index(env, true))
 #define MEMSUFFIX _code
 #define SOFTMMU_CODE_ACCESS
 
diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h
index 95ab7504e..3091c0003 100644
--- a/include/exec/cpu_ldst_template.h
+++ b/include/exec/cpu_ldst_template.h
@@ -27,20 +27,24 @@
 #define SUFFIX q
 #define USUFFIX q
 #define DATA_TYPE uint64_t
+#define SHIFT 3
 #elif DATA_SIZE == 4
 #define SUFFIX l
 #define USUFFIX l
 #define DATA_TYPE uint32_t
+#define SHIFT 2
 #elif DATA_SIZE == 2
 #define SUFFIX w
 #define USUFFIX uw
 #define DATA_TYPE uint16_t
 #define DATA_STYPE int16_t
+#define SHIFT 1
 #elif DATA_SIZE == 1
 #define SUFFIX b
 #define USUFFIX ub
 #define DATA_TYPE uint8_t
 #define DATA_STYPE int8_t
+#define SHIFT 0
 #else
 #error unsupported data size
 #endif
@@ -54,27 +58,36 @@
 #ifdef SOFTMMU_CODE_ACCESS
 #define ADDR_READ addr_code
 #define MMUSUFFIX _cmmu
+#define URETSUFFIX SUFFIX
+#define SRETSUFFIX SUFFIX
 #else
 #define ADDR_READ addr_read
 #define MMUSUFFIX _mmu
+#define URETSUFFIX USUFFIX
+#define SRETSUFFIX glue(s, SUFFIX)
 #endif
 
 /* generic load/store macros */
 
 static inline RES_TYPE
-glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
 {
     int page_index;
     RES_TYPE res;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(env, addr, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        res = glue(glue(helper_ret_ld, URETSUFFIX), MMUSUFFIX)(env, addr,
+                                                            oi, retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(ld, USUFFIX), _p)((uint8_t *)hostaddr);
@@ -82,27 +95,43 @@ glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
     return res;
 }
 
+static inline RES_TYPE
+glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+    return glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(env, ptr, 0);
+}
+
 #if DATA_SIZE <= 2
 static inline int
-glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
 {
     int res, page_index;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = (DATA_STYPE)glue(glue(helper_ld, SUFFIX),
-                               MMUSUFFIX)(env, addr, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        res = (DATA_STYPE)glue(glue(helper_ret_ld, SRETSUFFIX),
+                               MMUSUFFIX)(env, addr, oi, retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(lds, SUFFIX), _p)((uint8_t *)hostaddr);
     }
     return res;
 }
+
+static inline int
+glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+    return glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(env, ptr, 0);
+}
 #endif
 
 #ifndef SOFTMMU_CODE_ACCESS
@@ -110,25 +139,36 @@ glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 /* generic store macro */
 
 static inline void
-glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
-                                      RES_TYPE v)
+glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                 target_ulong ptr,
+                                                 RES_TYPE v, uintptr_t retaddr)
 {
     int page_index;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        glue(glue(helper_st, SUFFIX), MMUSUFFIX)(env, addr, v, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi,
+                                                     retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         glue(glue(st, SUFFIX), _p)((uint8_t *)hostaddr, v);
     }
 }
 
+static inline void
+glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
+                                      RES_TYPE v)
+{
+    glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(env, ptr, v, 0);
+}
+
 #endif /* !SOFTMMU_CODE_ACCESS */
 
 #undef RES_TYPE
@@ -139,3 +179,6 @@ glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
 #undef DATA_SIZE
 #undef MMUSUFFIX
 #undef ADDR_READ
+#undef URETSUFFIX
+#undef SRETSUFFIX
+#undef SHIFT
diff --git a/include/exec/cpu_ldst_useronly_template.h b/include/exec/cpu_ldst_useronly_template.h
index b3b865fae..040b14743 100644
--- a/include/exec/cpu_ldst_useronly_template.h
+++ b/include/exec/cpu_ldst_useronly_template.h
@@ -56,12 +56,28 @@ glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
     return glue(glue(ld, USUFFIX), _p)(g2h(ptr));
 }
 
+static inline RES_TYPE
+glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
+{
+    return glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(env, ptr);
+}
+
 #if DATA_SIZE <= 2
 static inline int
 glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 {
     return glue(glue(lds, SUFFIX), _p)(g2h(ptr));
 }
+
+static inline int
+glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
+{
+    return glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(env, ptr);
+}
 #endif
 
 #ifndef CODE_ACCESS
@@ -71,6 +87,15 @@ glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
 {
     glue(glue(st, SUFFIX), _p)(g2h(ptr), v);
 }
+
+static inline void
+glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  RES_TYPE v,
+                                                  uintptr_t retaddr)
+{
+    glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(env, ptr, v);
+}
 #endif
 
 #undef RES_TYPE
diff --git a/include/exec/cputlb.h b/include/exec/cputlb.h
index 360815e1b..d454c005b 100644
--- a/include/exec/cputlb.h
+++ b/include/exec/cputlb.h
@@ -25,23 +25,7 @@ void tlb_protect_code(ram_addr_t ram_addr);
 void tlb_unprotect_code(ram_addr_t ram_addr);
 void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
                            uintptr_t length);
-void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length);
-void tlb_set_dirty(CPUArchState *env, target_ulong vaddr);
 extern int tlb_flush_count;
 
-/* exec.c */
-void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr);
-
-MemoryRegionSection *
-address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr, hwaddr *xlat,
-                                  hwaddr *plen);
-hwaddr memory_region_section_get_iotlb(CPUState *cpu,
-                                       MemoryRegionSection *section,
-                                       target_ulong vaddr,
-                                       hwaddr paddr, hwaddr xlat,
-                                       int prot,
-                                       target_ulong *address);
-bool memory_region_is_unassigned(MemoryRegion *mr);
-
 #endif
 #endif
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 60f12bc65..d900b0d07 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -62,26 +62,16 @@ typedef struct TranslationBlock TranslationBlock;
 #define OPC_BUF_SIZE 640
 #define OPC_MAX_SIZE (OPC_BUF_SIZE - MAX_OP_PER_INSTR)
 
-/* Maximum size a TCG op can expand to.  This is complicated because a
-   single op may require several host instructions and register reloads.
-   For now take a wild guess at 192 bytes, which should allow at least
-   a couple of fixup instructions per argument.  */
-#define TCG_MAX_OP_SIZE 192
-
 #define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * MAX_OPC_PARAM)
 
 #include "qemu/log.h"
 
 void gen_intermediate_code(CPUArchState *env, struct TranslationBlock *tb);
-void gen_intermediate_code_pc(CPUArchState *env, struct TranslationBlock *tb);
 void restore_state_to_opc(CPUArchState *env, struct TranslationBlock *tb,
-                          int pc_pos);
+                          target_ulong *data);
 
 void cpu_gen_init(void);
-int cpu_gen_code(CPUArchState *env, struct TranslationBlock *tb,
-                 int *gen_code_size_ptr);
 bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc);
-void page_size_init(void);
 
 void QEMU_NORETURN cpu_resume_from_signal(CPUState *cpu, void *puc);
 void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
@@ -90,14 +80,52 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
                               int cflags);
 void cpu_exec_init(CPUState *cpu, Error **errp);
 void QEMU_NORETURN cpu_loop_exit(CPUState *cpu);
+void QEMU_NORETURN cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc);
 
 #if !defined(CONFIG_USER_ONLY)
-bool qemu_in_vcpu_thread(void);
-void cpu_reload_memory_map(CPUState *cpu);
+void cpu_reloading_memory_map(void);
 void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as);
 /* cputlb.c */
+/**
+ * tlb_flush_page:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of page to be flushed
+ *
+ * Flush one page from the TLB of the specified CPU, for all
+ * MMU indexes.
+ */
 void tlb_flush_page(CPUState *cpu, target_ulong addr);
+/**
+ * tlb_flush:
+ * @cpu: CPU whose TLB should be flushed
+ * @flush_global: ignored
+ *
+ * Flush the entire TLB for the specified CPU.
+ * The flush_global flag is in theory an indicator of whether the whole
+ * TLB should be flushed, or only those entries not marked global.
+ * In practice QEMU does not implement any global/not global flag for
+ * TLB entries, and the argument is ignored.
+ */
 void tlb_flush(CPUState *cpu, int flush_global);
+/**
+ * tlb_flush_page_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of page to be flushed
+ * @...: list of MMU indexes to flush, terminated by a negative value
+ *
+ * Flush one page from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...);
+/**
+ * tlb_flush_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @...: list of MMU indexes to flush, terminated by a negative value
+ *
+ * Flush all entries from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_by_mmuidx(CPUState *cpu, ...);
 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
                   hwaddr paddr, int prot,
                   int mmu_idx, target_ulong size);
@@ -115,6 +143,15 @@ static inline void tlb_flush_page(CPUState *cpu, target_ulong addr)
 static inline void tlb_flush(CPUState *cpu, int flush_global)
 {
 }
+
+static inline void tlb_flush_page_by_mmuidx(CPUState *cpu,
+                                            target_ulong addr, ...)
+{
+}
+
+static inline void tlb_flush_by_mmuidx(CPUState *cpu, ...)
+{
+}
 #endif
 
 #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */
@@ -122,13 +159,14 @@ static inline void tlb_flush(CPUState *cpu, int flush_global)
 #define CODE_GEN_PHYS_HASH_BITS     15
 #define CODE_GEN_PHYS_HASH_SIZE     (1 << CODE_GEN_PHYS_HASH_BITS)
 
-/* estimated block size for TB allocation */
-/* XXX: use a per code average code fragment size and modulate it
-   according to the host CPU */
+/* Estimated block size for TB allocation.  */
+/* ??? The following is based on a 2015 survey of x86_64 host output.
+   Better would seem to be some sort of dynamically sized TB array,
+   adapting to the block sizes actually being produced.  */
 #if defined(CONFIG_SOFTMMU)
-#define CODE_GEN_AVG_BLOCK_SIZE 128
+#define CODE_GEN_AVG_BLOCK_SIZE 400
 #else
-#define CODE_GEN_AVG_BLOCK_SIZE 64
+#define CODE_GEN_AVG_BLOCK_SIZE 150
 #endif
 
 #if defined(__arm__) || defined(_ARCH_PPC) \
@@ -151,10 +189,14 @@ struct TranslationBlock {
 #define CF_LAST_IO     0x8000 /* Last insn may be an IO access.  */
 #define CF_NOCACHE     0x10000 /* To be freed after execution */
 #define CF_USE_ICOUNT  0x20000
+#define CF_IGNORE_ICOUNT 0x40000 /* Do not generate icount code */
 
     void *tc_ptr;    /* pointer to the translated code */
+    uint8_t *tc_search;  /* pointer to search data */
     /* next matching tb for physical address. */
     struct TranslationBlock *phys_hash_next;
+    /* original tb when cflags has CF_NOCACHE */
+    struct TranslationBlock *orig_tb;
     /* first and second physical page containing code. The lower bit
        of the pointer tells the index in page_next[] */
     struct TranslationBlock *page_next[2];
@@ -176,7 +218,7 @@ struct TranslationBlock {
     struct TranslationBlock *jmp_first;
 };
 
-#include "exec/spinlock.h"
+#include "qemu/thread.h"
 
 typedef struct TBContext TBContext;
 
@@ -186,7 +228,7 @@ struct TBContext {
     TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
     int nb_tbs;
     /* any access to the tbs or the page table must use this lock */
-    spinlock_t tb_lock;
+    QemuMutex tb_lock;
 
     /* statistics */
     int tb_flush_count;
@@ -314,8 +356,6 @@ extern uintptr_t tci_tb_ptr;
 
 #if !defined(CONFIG_USER_ONLY)
 
-void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align));
-
 struct MemoryRegion *iotlb_to_region(CPUState *cpu,
                                      hwaddr index);
 
@@ -325,43 +365,44 @@ void tlb_fill(CPUState *cpu, target_ulong addr, int is_write, int mmu_idx,
 #endif
 
 #if defined(CONFIG_USER_ONLY)
+void mmap_lock(void);
+void mmap_unlock(void);
+
 static inline tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 {
     return addr;
 }
 #else
+static inline void mmap_lock(void) {}
+static inline void mmap_unlock(void) {}
+
 /* cputlb.c */
 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr);
+
+void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length);
+void tlb_set_dirty(CPUState *cpu, target_ulong vaddr);
+
+/* exec.c */
+void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr);
+
+MemoryRegionSection *
+address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr, hwaddr *xlat,
+                                  hwaddr *plen);
+hwaddr memory_region_section_get_iotlb(CPUState *cpu,
+                                       MemoryRegionSection *section,
+                                       target_ulong vaddr,
+                                       hwaddr paddr, hwaddr xlat,
+                                       int prot,
+                                       target_ulong *address);
+bool memory_region_is_unassigned(MemoryRegion *mr);
+
 #endif
 
 /* vl.c */
 extern int singlestep;
 
-/* cpu-exec.c */
-extern volatile sig_atomic_t exit_request;
-
-/**
- * cpu_can_do_io:
- * @cpu: The CPU for which to check IO.
- *
- * Deterministic execution requires that IO only be performed on the last
- * instruction of a TB so that interrupts take effect immediately.
- *
- * Returns: %true if memory-mapped IO is safe, %false otherwise.
- */
-static inline bool cpu_can_do_io(CPUState *cpu)
-{
-    if (!use_icount) {
-        return true;
-    }
-    /* If not executing code then assume we are ok.  */
-    if (cpu->current_tb == NULL) {
-        return true;
-    }
-    return cpu->can_do_io != 0;
-}
+/* cpu-exec.c, accessed with atomic_mb_read/atomic_mb_set */
+extern CPUState *tcg_current_cpu;
+extern bool exit_request;
 
-#if !defined(CONFIG_USER_ONLY)
-void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
-#endif
 #endif
diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index 05f57c243..d9e8cf771 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -14,7 +14,34 @@
 typedef void (*gdb_syscall_complete_cb)(CPUState *cpu,
                                         target_ulong ret, target_ulong err);
 
+/**
+ * gdb_do_syscall:
+ * @cb: function to call when the system call has completed
+ * @fmt: gdb syscall format string
+ * ...: list of arguments to interpolate into @fmt
+ *
+ * Send a GDB syscall request. This function will return immediately;
+ * the callback function will be called later when the remote system
+ * call has completed.
+ *
+ * @fmt should be in the 'call-id,parameter,parameter...' format documented
+ * for the F request packet in the GDB remote protocol. A limited set of
+ * printf-style format specifiers is supported:
+ *   %x  - target_ulong argument printed in hex
+ *   %lx - 64-bit argument printed in hex
+ *   %s  - string pointer (target_ulong) and length (int) pair
+ */
 void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...);
+/**
+ * gdb_do_syscallv:
+ * @cb: function to call when the system call has completed
+ * @fmt: gdb syscall format string
+ * @va: arguments to interpolate into @fmt
+ *
+ * As gdb_do_syscall, but taking a va_list rather than a variable
+ * argument list.
+ */
+void gdb_do_syscallv(gdb_syscall_complete_cb cb, const char *fmt, va_list va);
 int use_gdb_syscalls(void);
 void gdb_set_stop_cpu(CPUState *cpu);
 void gdb_exit(CPUArchState *, int);
diff --git a/include/exec/memattrs.h b/include/exec/memattrs.h
index f8537a8d9..e60106184 100644
--- a/include/exec/memattrs.h
+++ b/include/exec/memattrs.h
@@ -35,8 +35,8 @@ typedef struct MemTxAttrs {
     unsigned int secure:1;
     /* Memory access is usermode (unprivileged) */
     unsigned int user:1;
-    /* Stream ID (for MSI for example) */
-    unsigned int stream_id:16;
+    /* Requester ID (for MSI for example) */
+    unsigned int requester_id:16;
 } MemTxAttrs;
 
 /* Bus masters which don't specify any attributes will get this,
diff --git a/include/exec/memory.h b/include/exec/memory.h
index 94d20eae0..0f07159bb 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -324,7 +324,7 @@ void memory_region_unref(MemoryRegion *mr);
  * @owner: the object that tracks the region's reference count
  * @ops: a structure containing read and write callbacks to be used when
  *       I/O is performed on the region.
- * @opaque: passed to to the read and write callbacks of the @ops structure.
+ * @opaque: passed to the read and write callbacks of the @ops structure.
  * @name: used for debugging; not visible to the user or ABI
  * @size: size of the region.
  */
@@ -437,6 +437,9 @@ void memory_region_init_alias(MemoryRegion *mr,
  * memory_region_init_rom_device:  Initialize a ROM memory region.  Writes are
  *                                 handled via callbacks.
  *
+ * If NULL callbacks pointer is given, then I/O space is not supposed to be
+ * handled by QEMU itself. Any access via the memory API will cause an abort().
+ *
  * @mr: the #MemoryRegion to be initialized.
  * @owner: the object that tracks the region's reference count
  * @ops: callbacks for write access handling.
@@ -459,16 +462,21 @@ void memory_region_init_rom_device(MemoryRegion *mr,
  * A reservation region primariy serves debugging purposes.  It claims I/O
  * space that is not supposed to be handled by QEMU itself.  Any access via
  * the memory API will cause an abort().
+ * This function is deprecated. Use memory_region_init_io() with NULL
+ * callbacks instead.
  *
  * @mr: the #MemoryRegion to be initialized
  * @owner: the object that tracks the region's reference count
  * @name: used for debugging; not visible to the user or ABI
  * @size: size of the region.
  */
-void memory_region_init_reservation(MemoryRegion *mr,
-                                    struct Object *owner,
+static inline void memory_region_init_reservation(MemoryRegion *mr,
+                                    Object *owner,
                                     const char *name,
-                                    uint64_t size);
+                                    uint64_t size)
+{
+    memory_region_init_io(mr, owner, NULL, mr, name, size);
+}
 
 /**
  * memory_region_init_iommu: Initialize a memory region that translates
@@ -575,6 +583,19 @@ void memory_region_notify_iommu(MemoryRegion *mr,
 void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n);
 
 /**
+ * memory_region_iommu_replay: replay existing IOMMU translations to
+ * a notifier
+ *
+ * @mr: the memory region to observe
+ * @n: the notifier to which to replay iommu mappings
+ * @granularity: Minimum page granularity to replay notifications for
+ * @is_write: Whether to treat the replay as a translate "write"
+ *     through the iommu
+ */
+void memory_region_iommu_replay(MemoryRegion *mr, Notifier *n,
+                                hwaddr granularity, bool is_write);
+
+/**
  * memory_region_unregister_iommu_notifier: unregister a notifier for
  * changes to IOMMU translation entries.
  *
@@ -1138,7 +1159,7 @@ MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
  * address_space_init: initializes an address space
  *
  * @as: an uninitialized #AddressSpace
- * @root: a #MemoryRegion that routes addesses for the address space
+ * @root: a #MemoryRegion that routes addresses for the address space
  * @name: an address space name.  The name is only used for debugging
  *        output.
  */
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index c113f2114..7115154bc 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -22,6 +22,44 @@
 #ifndef CONFIG_USER_ONLY
 #include "hw/xen/xen.h"
 
+struct RAMBlock {
+    struct rcu_head rcu;
+    struct MemoryRegion *mr;
+    uint8_t *host;
+    ram_addr_t offset;
+    ram_addr_t used_length;
+    ram_addr_t max_length;
+    void (*resized)(const char*, uint64_t length, void *host);
+    uint32_t flags;
+    /* Protected by iothread lock.  */
+    char idstr[256];
+    /* RCU-enabled, writes protected by the ramlist lock */
+    QLIST_ENTRY(RAMBlock) next;
+    int fd;
+};
+
+static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
+{
+    assert(offset < block->used_length);
+    assert(block->host);
+    return (char *)block->host + offset;
+}
+
+typedef struct RAMList {
+    QemuMutex mutex;
+    /* Protected by the iothread lock.  */
+    unsigned long *dirty_memory[DIRTY_MEMORY_NUM];
+    RAMBlock *mru_block;
+    /* RCU-enabled, writes protected by the ramlist lock. */
+    QLIST_HEAD(, RAMBlock) blocks;
+    uint32_t version;
+} RAMList;
+extern RAMList ram_list;
+
+ram_addr_t last_ram_offset(void);
+void qemu_mutex_lock_ramlist(void);
+void qemu_mutex_unlock_ramlist(void);
+
 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
                                     bool share, const char *mem_path,
                                     Error **errp);
@@ -249,5 +287,6 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
     return num_dirty;
 }
 
+void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
 #endif
 #endif
diff --git a/include/exec/softmmu-semi.h b/include/exec/softmmu-semi.h
index 1819cc249..3a58c3f08 100644
--- a/include/exec/softmmu-semi.h
+++ b/include/exec/softmmu-semi.h
@@ -9,6 +9,14 @@
 #ifndef SOFTMMU_SEMI_H
 #define SOFTMMU_SEMI_H 1
 
+static inline uint64_t softmmu_tget64(CPUArchState *env, target_ulong addr)
+{
+    uint64_t val;
+
+    cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 8, 0);
+    return tswap64(val);
+}
+
 static inline uint32_t softmmu_tget32(CPUArchState *env, target_ulong addr)
 {
     uint32_t val;
@@ -16,6 +24,7 @@ static inline uint32_t softmmu_tget32(CPUArchState *env, target_ulong addr)
     cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 4, 0);
     return tswap32(val);
 }
+
 static inline uint32_t softmmu_tget8(CPUArchState *env, target_ulong addr)
 {
     uint8_t val;
@@ -24,16 +33,25 @@ static inline uint32_t softmmu_tget8(CPUArchState *env, target_ulong addr)
     return val;
 }
 
+#define get_user_u64(arg, p) ({ arg = softmmu_tget64(env, p); 0; })
 #define get_user_u32(arg, p) ({ arg = softmmu_tget32(env, p) ; 0; })
 #define get_user_u8(arg, p) ({ arg = softmmu_tget8(env, p) ; 0; })
 #define get_user_ual(arg, p) get_user_u32(arg, p)
 
+static inline void softmmu_tput64(CPUArchState *env,
+                                  target_ulong addr, uint64_t val)
+{
+    val = tswap64(val);
+    cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 8, 1);
+}
+
 static inline void softmmu_tput32(CPUArchState *env,
                                   target_ulong addr, uint32_t val)
 {
     val = tswap32(val);
     cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 4, 1);
 }
+#define put_user_u64(arg, p) ({ softmmu_tput64(env, p, arg) ; 0; })
 #define put_user_u32(arg, p) ({ softmmu_tput32(env, p, arg) ; 0; })
 #define put_user_ual(arg, p) put_user_u32(arg, p)
 
diff --git a/include/exec/spinlock.h b/include/exec/spinlock.h
deleted file mode 100644
index a72edda1d..000000000
--- a/include/exec/spinlock.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-
-/* configure guarantees us that we have pthreads on any host except
- * mingw32, which doesn't support any of the user-only targets.
- * So we can simply assume we have pthread mutexes here.
- */
-#if defined(CONFIG_USER_ONLY)
-
-#include <pthread.h>
-#define spin_lock pthread_mutex_lock
-#define spin_unlock pthread_mutex_unlock
-#define spinlock_t pthread_mutex_t
-#define SPIN_LOCK_UNLOCKED PTHREAD_MUTEX_INITIALIZER
-
-#else
-
-/* Empty implementations, on the theory that system mode emulation
- * is single-threaded. This means that these functions should only
- * be used from code run in the TCG cpu thread, and cannot protect
- * data structures which might also be accessed from the IO thread
- * or from signal handlers.
- */
-typedef int spinlock_t;
-#define SPIN_LOCK_UNLOCKED 0
-
-static inline void spin_lock(spinlock_t *lock)
-{
-}
-
-static inline void spin_unlock(spinlock_t *lock)
-{
-}
-
-#endif
diff --git a/include/glib-compat.h b/include/glib-compat.h
index 318e00036..03d8b1267 100644
--- a/include/glib-compat.h
+++ b/include/glib-compat.h
@@ -165,4 +165,73 @@ static inline GThread *g_thread_new(const char *name,
 #define CompatGCond GCond
 #endif /* glib 2.31 */
 
+#if !GLIB_CHECK_VERSION(2, 32, 0)
+/* Beware, function returns gboolean since 2.39.2, see GLib commit 9101915 */
+static inline void g_hash_table_add(GHashTable *hash_table, gpointer key)
+{
+    g_hash_table_replace(hash_table, key, key);
+}
+#endif
+
+#ifndef g_assert_true
+#define g_assert_true(expr)                                                    \
+    do {                                                                       \
+        if (G_LIKELY(expr)) {                                                  \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be TRUE");                 \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_false
+#define g_assert_false(expr)                                                   \
+    do {                                                                       \
+        if (G_LIKELY(!(expr))) {                                               \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be FALSE");                \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_null
+#define g_assert_null(expr)                                                    \
+    do {                                                                       \
+        if (G_LIKELY((expr) == NULL)) {                                        \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be NULL");                 \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_nonnull
+#define g_assert_nonnull(expr)                                                 \
+    do {                                                                       \
+        if (G_LIKELY((expr) != NULL)) {                                        \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should not be NULL");             \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_cmpmem
+#define g_assert_cmpmem(m1, l1, m2, l2)                                        \
+    do {                                                                       \
+        gconstpointer __m1 = m1, __m2 = m2;                                    \
+        int __l1 = l1, __l2 = l2;                                              \
+        if (__l1 != __l2) {                                                    \
+            g_assertion_message_cmpnum(                                        \
+                G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,                   \
+                #l1 " (len(" #m1 ")) == " #l2 " (len(" #m2 "))", __l1, "==",   \
+                __l2, 'i');                                                    \
+        } else if (memcmp(__m1, __m2, __l1) != 0) {                            \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "assertion failed (" #m1 " == " #m2 ")");      \
+        }                                                                      \
+    } while (0)
+#endif
+
 #endif
diff --git a/include/hw/acpi/acpi-defs.h b/include/hw/acpi/acpi-defs.h
index 2b431e624..c7a03d43b 100644
--- a/include/hw/acpi/acpi-defs.h
+++ b/include/hw/acpi/acpi-defs.h
@@ -384,6 +384,15 @@ struct AcpiMadtGenericMsiFrame {
 
 typedef struct AcpiMadtGenericMsiFrame AcpiMadtGenericMsiFrame;
 
+struct AcpiMadtGenericRedistributor {
+    ACPI_SUB_HEADER_DEF
+    uint16_t reserved;
+    uint64_t base_address;
+    uint32_t range_length;
+} QEMU_PACKED;
+
+typedef struct AcpiMadtGenericRedistributor AcpiMadtGenericRedistributor;
+
 /*
  * Generic Timer Description Table (GTDT)
  */
diff --git a/include/hw/acpi/aml-build.h b/include/hw/acpi/aml-build.h
index e3afa1367..1b632dc54 100644
--- a/include/hw/acpi/aml-build.h
+++ b/include/hw/acpi/aml-build.h
@@ -9,7 +9,6 @@
 /* Reserve RAM space for tables: add another order of magnitude. */
 #define ACPI_BUILD_TABLE_MAX_SIZE         0x200000
 
-#define ACPI_BUILD_APPNAME  "Bochs"
 #define ACPI_BUILD_APPNAME6 "BOCHS "
 #define ACPI_BUILD_APPNAME4 "BXPC"
 
diff --git a/include/hw/arm/allwinner-a10.h b/include/hw/arm/allwinner-a10.h
index 01a189bcd..6b32a99e2 100644
--- a/include/hw/arm/allwinner-a10.h
+++ b/include/hw/arm/allwinner-a10.h
@@ -7,6 +7,8 @@
 #include "hw/timer/allwinner-a10-pit.h"
 #include "hw/intc/allwinner-a10-pic.h"
 #include "hw/net/allwinner_emac.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
 
 #include "sysemu/sysemu.h"
 #include "exec/address-spaces.h"
@@ -16,6 +18,7 @@
 #define AW_A10_PIT_REG_BASE     0x01c20c00
 #define AW_A10_UART0_REG_BASE   0x01c28000
 #define AW_A10_EMAC_BASE        0x01c0b000
+#define AW_A10_SATA_BASE        0x01c18000
 
 #define AW_A10_SDRAM_BASE       0x40000000
 
@@ -32,6 +35,7 @@ typedef struct AwA10State {
     AwA10PITState timer;
     AwA10PICState intc;
     AwEmacState emac;
+    AllwinnerAHCIState sata;
 } AwA10State;
 
 #define ALLWINNER_H_
diff --git a/include/hw/arm/arm.h b/include/hw/arm/arm.h
index 4dcd4f9b6..c26b0e357 100644
--- a/include/hw/arm/arm.h
+++ b/include/hw/arm/arm.h
@@ -17,7 +17,7 @@
 #include "cpu.h"
 
 /* armv7m.c */
-qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
+DeviceState *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
                       const char *kernel_filename, const char *cpu_model);
 
 /*
@@ -87,6 +87,22 @@ struct arm_boot_info {
      * -pflash. It also implies that fw_cfg_find() will succeed.
      */
     bool firmware_loaded;
+
+    /* Address at which board specific loader/setup code exists. If enabled,
+     * this code-blob will run before anything else. It must return to the
+     * caller via the link register. There is no stack set up. Enabled by
+     * defining write_board_setup, which is responsible for loading the blob
+     * to the specified address.
+     */
+    hwaddr board_setup_addr;
+    void (*write_board_setup)(ARMCPU *cpu,
+                              const struct arm_boot_info *info);
+
+    /* If set, the board specific loader/setup blob will be run from secure
+     * mode, regardless of secure_boot. The blob becomes responsible for
+     * changing to non-secure state if implementing a non-secure boot
+     */
+    bool secure_board_setup;
 };
 
 /**
diff --git a/include/hw/arm/fsl-imx25.h b/include/hw/arm/fsl-imx25.h
new file mode 100644
index 000000000..73f50c64d
--- /dev/null
+++ b/include/hw/arm/fsl-imx25.h
@@ -0,0 +1,249 @@
+/*
+ * Freescale i.MX25 SoC emulation
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef FSL_IMX25_H
+#define FSL_IMX25_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/imx_avic.h"
+#include "hw/misc/imx_ccm.h"
+#include "hw/char/imx_serial.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/net/imx_fec.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/gpio/imx_gpio.h"
+#include "exec/memory.h"
+
+#define TYPE_FSL_IMX25 "fsl,imx25"
+#define FSL_IMX25(obj) OBJECT_CHECK(FslIMX25State, (obj), TYPE_FSL_IMX25)
+
+#define FSL_IMX25_NUM_UARTS 5
+#define FSL_IMX25_NUM_GPTS 4
+#define FSL_IMX25_NUM_EPITS 2
+#define FSL_IMX25_NUM_I2CS 3
+#define FSL_IMX25_NUM_GPIOS 4
+
+typedef struct FslIMX25State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU         cpu;
+    IMXAVICState   avic;
+    IMXCCMState    ccm;
+    IMXSerialState uart[FSL_IMX25_NUM_UARTS];
+    IMXGPTState    gpt[FSL_IMX25_NUM_GPTS];
+    IMXEPITState   epit[FSL_IMX25_NUM_EPITS];
+    IMXFECState    fec;
+    IMXI2CState    i2c[FSL_IMX25_NUM_I2CS];
+    IMXGPIOState   gpio[FSL_IMX25_NUM_GPIOS];
+    MemoryRegion   rom[2];
+    MemoryRegion   iram;
+    MemoryRegion   iram_alias;
+} FslIMX25State;
+
+/**
+ * i.MX25 memory map
+ ****************************************************************
+ * 0x0000_0000 0x0000_3FFF 16 Kbytes    ROM (36 Kbytes)
+ * 0x0000_4000 0x0040_3FFF 4 Mbytes     Reserved
+ * 0x0040_4000 0x0040_8FFF 20 Kbytes    ROM (36 Kbytes)
+ * 0x0040_9000 0x0FFF_FFFF 252 Mbytes (minus 36 Kbytes) Reserved
+ * 0x1000_0000 0x1FFF_FFFF 256 Mbytes   Reserved
+ * 0x2000_0000 0x2FFF_FFFF 256 Mbytes   Reserved
+ * 0x3000_0000 0x3FFF_FFFF 256 Mbytes   Reserved
+ * 0x4000_0000 0x43EF_FFFF 63 Mbytes    Reserved
+ * 0x43F0_0000 0x43F0_3FFF 16 Kbytes    AIPS A control registers
+ * 0x43F0_4000 0x43F0_7FFF 16 Kbytes    ARM926 platform MAX
+ * 0x43F0_8000 0x43F0_BFFF 16 Kbytes    ARM926 platform CLKCTL
+ * 0x43F0_C000 0x43F0_FFFF 16 Kbytes    ARM926 platform ETB registers
+ * 0x43F1_0000 0x43F1_3FFF 16 Kbytes    ARM926 platform ETB memory
+ * 0x43F1_4000 0x43F1_7FFF 16 Kbytes    ARM926 platform AAPE registers
+ * 0x43F1_8000 0x43F7_FFFF 416 Kbytes   Reserved
+ * 0x43F8_0000 0x43F8_3FFF 16 Kbytes    I2C-1
+ * 0x43F8_4000 0x43F8_7FFF 16 Kbytes    I2C-3
+ * 0x43F8_8000 0x43F8_BFFF 16 Kbytes    CAN-1
+ * 0x43F8_C000 0x43F8_FFFF 16 Kbytes    CAN-2
+ * 0x43F9_0000 0x43F9_3FFF 16 Kbytes    UART-1
+ * 0x43F9_4000 0x43F9_7FFF 16 Kbytes    UART-2
+ * 0x43F9_8000 0x43F9_BFFF 16 Kbytes    I2C-2
+ * 0x43F9_C000 0x43F9_FFFF 16 Kbytes    1-Wire
+ * 0x43FA_0000 0x43FA_3FFF 16 Kbytes    ATA (CPU side)
+ * 0x43FA_4000 0x43FA_7FFF 16 Kbytes    CSPI-1
+ * 0x43FA_8000 0x43FA_BFFF 16 Kbytes    KPP
+ * 0x43FA_C000 0x43FA_FFFF 16 Kbytes    IOMUXC
+ * 0x43FB_0000 0x43FB_3FFF 16 Kbytes    AUDMUX
+ * 0x43FB_4000 0x43FB_7FFF 16 Kbytes    Reserved
+ * 0x43FB_8000 0x43FB_BFFF 16 Kbytes    ECT (IP BUS A)
+ * 0x43FB_C000 0x43FB_FFFF 16 Kbytes    ECT (IP BUS B)
+ * 0x43FC_0000 0x43FF_FFFF 256 Kbytes   Reserved AIPS A off-platform slots
+ * 0x4400_0000 0x4FFF_FFFF 192 Mbytes   Reserved
+ * 0x5000_0000 0x5000_3FFF 16 Kbytes    SPBA base address
+ * 0x5000_4000 0x5000_7FFF 16 Kbytes    CSPI-3
+ * 0x5000_8000 0x5000_BFFF 16 Kbytes    UART-4
+ * 0x5000_C000 0x5000_FFFF 16 Kbytes    UART-3
+ * 0x5001_0000 0x5001_3FFF 16 Kbytes    CSPI-2
+ * 0x5001_4000 0x5001_7FFF 16 Kbytes    SSI-2
+ * 0x5001_C000 0x5001_FFFF 16 Kbytes    Reserved
+ * 0x5002_0000 0x5002_3FFF 16 Kbytes    ATA
+ * 0x5002_4000 0x5002_7FFF 16 Kbytes    SIM-1
+ * 0x5002_8000 0x5002_BFFF 16 Kbytes    SIM-2
+ * 0x5002_C000 0x5002_FFFF 16 Kbytes    UART-5
+ * 0x5003_0000 0x5003_3FFF 16 Kbytes    TSC
+ * 0x5003_4000 0x5003_7FFF 16 Kbytes    SSI-1
+ * 0x5003_8000 0x5003_BFFF 16 Kbytes    FEC
+ * 0x5003_C000 0x5003_FFFF 16 Kbytes    SPBA registers
+ * 0x5004_0000 0x51FF_FFFF 32 Mbytes (minus 256 Kbytes)
+ * 0x5200_0000 0x53EF_FFFF 31 Mbytes    Reserved
+ * 0x53F0_0000 0x53F0_3FFF 16 Kbytes    AIPS B control registers
+ * 0x53F0_4000 0x53F7_FFFF 496 Kbytes   Reserved
+ * 0x53F8_0000 0x53F8_3FFF 16 Kbytes    CCM
+ * 0x53F8_4000 0x53F8_7FFF 16 Kbytes    GPT-4
+ * 0x53F8_8000 0x53F8_BFFF 16 Kbytes    GPT-3
+ * 0x53F8_C000 0x53F8_FFFF 16 Kbytes    GPT-2
+ * 0x53F9_0000 0x53F9_3FFF 16 Kbytes    GPT-1
+ * 0x53F9_4000 0x53F9_7FFF 16 Kbytes    EPIT-1
+ * 0x53F9_8000 0x53F9_BFFF 16 Kbytes    EPIT-2
+ * 0x53F9_C000 0x53F9_FFFF 16 Kbytes    GPIO-4
+ * 0x53FA_0000 0x53FA_3FFF 16 Kbytes    PWM-2
+ * 0x53FA_4000 0x53FA_7FFF 16 Kbytes    GPIO-3
+ * 0x53FA_8000 0x53FA_BFFF 16 Kbytes    PWM-3
+ * 0x53FA_C000 0x53FA_FFFF 16 Kbytes    SCC
+ * 0x53FB_0000 0x53FB_3FFF 16 Kbytes    RNGB
+ * 0x53FB_4000 0x53FB_7FFF 16 Kbytes    eSDHC-1
+ * 0x53FB_8000 0x53FB_BFFF 16 Kbytes    eSDHC-2
+ * 0x53FB_C000 0x53FB_FFFF 16 Kbytes    LCDC
+ * 0x53FC_0000 0x53FC_3FFF 16 Kbytes    SLCDC
+ * 0x53FC_4000 0x53FC_7FFF 16 Kbytes    Reserved
+ * 0x53FC_8000 0x53FC_BFFF 16 Kbytes    PWM-4
+ * 0x53FC_C000 0x53FC_FFFF 16 Kbytes    GPIO-1
+ * 0x53FD_0000 0x53FD_3FFF 16 Kbytes    GPIO-2
+ * 0x53FD_4000 0x53FD_7FFF 16 Kbytes    SDMA
+ * 0x53FD_8000 0x53FD_BFFF 16 Kbytes    Reserved
+ * 0x53FD_C000 0x53FD_FFFF 16 Kbytes    WDOG
+ * 0x53FE_0000 0x53FE_3FFF 16 Kbytes    PWM-1
+ * 0x53FE_4000 0x53FE_7FFF 16 Kbytes    Reserved
+ * 0x53FE_8000 0x53FE_BFFF 16 Kbytes    Reserved
+ * 0x53FE_C000 0x53FE_FFFF 16 Kbytes    RTICv3
+ * 0x53FF_0000 0x53FF_3FFF 16 Kbytes    IIM
+ * 0x53FF_4000 0x53FF_7FFF 16 Kbytes    USB
+ * 0x53FF_8000 0x53FF_BFFF 16 Kbytes    CSI
+ * 0x53FF_C000 0x53FF_FFFF 16 Kbytes    DryIce
+ * 0x5400_0000 0x5FFF_FFFF 192 Mbytes   Reserved (aliased AIPS B slots)
+ * 0x6000_0000 0x67FF_FFFF 128 Mbytes   ARM926 platform ROMPATCH
+ * 0x6800_0000 0x6FFF_FFFF 128 Mbytes   ARM926 platform ASIC
+ * 0x7000_0000 0x77FF_FFFF 128 Mbytes   Reserved
+ * 0x7800_0000 0x7801_FFFF 128 Kbytes   RAM
+ * 0x7802_0000 0x7FFF_FFFF 128 Mbytes (minus 128 Kbytes)
+ * 0x8000_0000 0x8FFF_FFFF 256 Mbytes   SDRAM bank 0
+ * 0x9000_0000 0x9FFF_FFFF 256 Mbytes   SDRAM bank 1
+ * 0xA000_0000 0xA7FF_FFFF 128 Mbytes   WEIM CS0 (flash 128) 1
+ * 0xA800_0000 0xAFFF_FFFF 128 Mbytes   WEIM CS1 (flash 64) 1
+ * 0xB000_0000 0xB1FF_FFFF 32 Mbytes    WEIM CS2 (SRAM)
+ * 0xB200_0000 0xB3FF_FFFF 32 Mbytes    WEIM CS3 (SRAM)
+ * 0xB400_0000 0xB5FF_FFFF 32 Mbytes    WEIM CS4
+ * 0xB600_0000 0xB7FF_FFFF 32 Mbytes    Reserved
+ * 0xB800_0000 0xB800_0FFF 4 Kbytes     Reserved
+ * 0xB800_1000 0xB800_1FFF 4 Kbytes     SDRAM control registers
+ * 0xB800_2000 0xB800_2FFF 4 Kbytes     WEIM control registers
+ * 0xB800_3000 0xB800_3FFF 4 Kbytes     M3IF control registers
+ * 0xB800_4000 0xB800_4FFF 4 Kbytes     EMI control registers
+ * 0xB800_5000 0xBAFF_FFFF 32 Mbytes (minus 20 Kbytes)
+ * 0xBB00_0000 0xBB00_0FFF 4 Kbytes     NAND flash main area buffer
+ * 0xBB00_1000 0xBB00_11FF 512 B        NAND flash spare area buffer
+ * 0xBB00_1200 0xBB00_1DFF 3 Kbytes     Reserved
+ * 0xBB00_1E00 0xBB00_1FFF 512 B        NAND flash control regisers
+ * 0xBB01_2000 0xBFFF_FFFF 96 Mbytes (minus 8 Kbytes) Reserved
+ * 0xC000_0000 0xFFFF_FFFF 1024 Mbytes  Reserved
+ */
+
+#define FSL_IMX25_ROM0_ADDR     0x00000000
+#define FSL_IMX25_ROM0_SIZE     0x4000
+#define FSL_IMX25_ROM1_ADDR     0x00404000
+#define FSL_IMX25_ROM1_SIZE     0x4000
+#define FSL_IMX25_I2C1_ADDR     0x43F80000
+#define FSL_IMX25_I2C1_SIZE     0x4000
+#define FSL_IMX25_I2C3_ADDR     0x43F84000
+#define FSL_IMX25_I2C3_SIZE     0x4000
+#define FSL_IMX25_UART1_ADDR    0x43F90000
+#define FSL_IMX25_UART1_SIZE    0x4000
+#define FSL_IMX25_UART2_ADDR    0x43F94000
+#define FSL_IMX25_UART2_SIZE    0x4000
+#define FSL_IMX25_I2C2_ADDR     0x43F98000
+#define FSL_IMX25_I2C2_SIZE     0x4000
+#define FSL_IMX25_UART4_ADDR    0x50008000
+#define FSL_IMX25_UART4_SIZE    0x4000
+#define FSL_IMX25_UART3_ADDR    0x5000C000
+#define FSL_IMX25_UART3_SIZE    0x4000
+#define FSL_IMX25_UART5_ADDR    0x5002C000
+#define FSL_IMX25_UART5_SIZE    0x4000
+#define FSL_IMX25_FEC_ADDR      0x50038000
+#define FSL_IMX25_FEC_SIZE      0x4000
+#define FSL_IMX25_CCM_ADDR      0x53F80000
+#define FSL_IMX25_CCM_SIZE      0x4000
+#define FSL_IMX25_GPT4_ADDR     0x53F84000
+#define FSL_IMX25_GPT4_SIZE     0x4000
+#define FSL_IMX25_GPT3_ADDR     0x53F88000
+#define FSL_IMX25_GPT3_SIZE     0x4000
+#define FSL_IMX25_GPT2_ADDR     0x53F8C000
+#define FSL_IMX25_GPT2_SIZE     0x4000
+#define FSL_IMX25_GPT1_ADDR     0x53F90000
+#define FSL_IMX25_GPT1_SIZE     0x4000
+#define FSL_IMX25_EPIT1_ADDR    0x53F94000
+#define FSL_IMX25_EPIT1_SIZE    0x4000
+#define FSL_IMX25_EPIT2_ADDR    0x53F98000
+#define FSL_IMX25_EPIT2_SIZE    0x4000
+#define FSL_IMX25_GPIO4_ADDR    0x53F9C000
+#define FSL_IMX25_GPIO4_SIZE    0x4000
+#define FSL_IMX25_GPIO3_ADDR    0x53FA4000
+#define FSL_IMX25_GPIO3_SIZE    0x4000
+#define FSL_IMX25_GPIO1_ADDR    0x53FCC000
+#define FSL_IMX25_GPIO1_SIZE    0x4000
+#define FSL_IMX25_GPIO2_ADDR    0x53FD0000
+#define FSL_IMX25_GPIO2_SIZE    0x4000
+#define FSL_IMX25_AVIC_ADDR     0x68000000
+#define FSL_IMX25_AVIC_SIZE     0x4000
+#define FSL_IMX25_IRAM_ADDR     0x78000000
+#define FSL_IMX25_IRAM_SIZE     0x20000
+#define FSL_IMX25_IRAM_ALIAS_ADDR     0x78020000
+#define FSL_IMX25_IRAM_ALIAS_SIZE     0x7FE0000
+#define FSL_IMX25_SDRAM0_ADDR   0x80000000
+#define FSL_IMX25_SDRAM0_SIZE   0x10000000
+#define FSL_IMX25_SDRAM1_ADDR   0x90000000
+#define FSL_IMX25_SDRAM1_SIZE   0x10000000
+
+#define FSL_IMX25_UART1_IRQ     45
+#define FSL_IMX25_UART2_IRQ     32
+#define FSL_IMX25_UART3_IRQ     18
+#define FSL_IMX25_UART4_IRQ     5
+#define FSL_IMX25_UART5_IRQ     40
+#define FSL_IMX25_GPT1_IRQ      54
+#define FSL_IMX25_GPT2_IRQ      53
+#define FSL_IMX25_GPT3_IRQ      29
+#define FSL_IMX25_GPT4_IRQ      1
+#define FSL_IMX25_EPIT1_IRQ     28
+#define FSL_IMX25_EPIT2_IRQ     27
+#define FSL_IMX25_FEC_IRQ       57
+#define FSL_IMX25_I2C1_IRQ      3
+#define FSL_IMX25_I2C2_IRQ      4
+#define FSL_IMX25_I2C3_IRQ      10
+#define FSL_IMX25_GPIO1_IRQ     52
+#define FSL_IMX25_GPIO2_IRQ     51
+#define FSL_IMX25_GPIO3_IRQ     16
+#define FSL_IMX25_GPIO4_IRQ     23
+
+#endif /* FSL_IMX25_H */
diff --git a/include/hw/arm/fsl-imx31.h b/include/hw/arm/fsl-imx31.h
new file mode 100644
index 000000000..5e8f795f9
--- /dev/null
+++ b/include/hw/arm/fsl-imx31.h
@@ -0,0 +1,122 @@
+/*
+ * Freescale i.MX31 SoC emulation
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef FSL_IMX31_H
+#define FSL_IMX31_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/imx_avic.h"
+#include "hw/misc/imx_ccm.h"
+#include "hw/char/imx_serial.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/gpio/imx_gpio.h"
+#include "exec/memory.h"
+
+#define TYPE_FSL_IMX31 "fsl,imx31"
+#define FSL_IMX31(obj) OBJECT_CHECK(FslIMX31State, (obj), TYPE_FSL_IMX31)
+
+#define FSL_IMX31_NUM_UARTS 2
+#define FSL_IMX31_NUM_EPITS 2
+#define FSL_IMX31_NUM_I2CS 3
+#define FSL_IMX31_NUM_GPIOS 3
+
+typedef struct FslIMX31State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU         cpu;
+    IMXAVICState   avic;
+    IMXCCMState    ccm;
+    IMXSerialState uart[FSL_IMX31_NUM_UARTS];
+    IMXGPTState    gpt;
+    IMXEPITState   epit[FSL_IMX31_NUM_EPITS];
+    IMXI2CState    i2c[FSL_IMX31_NUM_I2CS];
+    IMXGPIOState   gpio[FSL_IMX31_NUM_GPIOS];
+    MemoryRegion   secure_rom;
+    MemoryRegion   rom;
+    MemoryRegion   iram;
+    MemoryRegion   iram_alias;
+} FslIMX31State;
+
+#define FSL_IMX31_SECURE_ROM_ADDR       0x00000000
+#define FSL_IMX31_SECURE_ROM_SIZE       0x4000
+#define FSL_IMX31_ROM_ADDR              0x00404000
+#define FSL_IMX31_ROM_SIZE              0x4000
+#define FSL_IMX31_IRAM_ALIAS_ADDR       0x10000000
+#define FSL_IMX31_IRAM_ALIAS_SIZE       0xFFC0000
+#define FSL_IMX31_IRAM_ADDR             0x1FFFC000
+#define FSL_IMX31_IRAM_SIZE             0x4000
+#define FSL_IMX31_I2C1_ADDR             0x43F80000
+#define FSL_IMX31_I2C1_SIZE             0x4000
+#define FSL_IMX31_I2C3_ADDR             0x43F84000
+#define FSL_IMX31_I2C3_SIZE             0x4000
+#define FSL_IMX31_UART1_ADDR            0x43F90000
+#define FSL_IMX31_UART1_SIZE            0x4000
+#define FSL_IMX31_UART2_ADDR            0x43F94000
+#define FSL_IMX31_UART2_SIZE            0x4000
+#define FSL_IMX31_I2C2_ADDR             0x43F98000
+#define FSL_IMX31_I2C2_SIZE             0x4000
+#define FSL_IMX31_CCM_ADDR              0x53F80000
+#define FSL_IMX31_CCM_SIZE              0x4000
+#define FSL_IMX31_GPT_ADDR              0x53F90000
+#define FSL_IMX31_GPT_SIZE              0x4000
+#define FSL_IMX31_EPIT1_ADDR            0x53F94000
+#define FSL_IMX31_EPIT1_SIZE            0x4000
+#define FSL_IMX31_EPIT2_ADDR            0x53F98000
+#define FSL_IMX31_EPIT2_SIZE            0x4000
+#define FSL_IMX31_GPIO3_ADDR            0x53FA4000
+#define FSL_IMX31_GPIO3_SIZE            0x4000
+#define FSL_IMX31_GPIO1_ADDR            0x53FCC000
+#define FSL_IMX31_GPIO1_SIZE            0x4000
+#define FSL_IMX31_GPIO2_ADDR            0x53FD0000
+#define FSL_IMX31_GPIO2_SIZE            0x4000
+#define FSL_IMX31_AVIC_ADDR             0x68000000
+#define FSL_IMX31_AVIC_SIZE             0x100
+#define FSL_IMX31_SDRAM0_ADDR           0x80000000
+#define FSL_IMX31_SDRAM0_SIZE           0x10000000
+#define FSL_IMX31_SDRAM1_ADDR           0x90000000
+#define FSL_IMX31_SDRAM1_SIZE           0x10000000
+#define FSL_IMX31_FLASH0_ADDR           0xA0000000
+#define FSL_IMX31_FLASH0_SIZE           0x8000000
+#define FSL_IMX31_FLASH1_ADDR           0xA8000000
+#define FSL_IMX31_FLASH1_SIZE           0x8000000
+#define FSL_IMX31_CS2_ADDR              0xB0000000
+#define FSL_IMX31_CS2_SIZE              0x2000000
+#define FSL_IMX31_CS3_ADDR              0xB2000000
+#define FSL_IMX31_CS3_SIZE              0x2000000
+#define FSL_IMX31_CS4_ADDR              0xB4000000
+#define FSL_IMX31_CS4_SIZE              0x2000000
+#define FSL_IMX31_CS5_ADDR              0xB6000000
+#define FSL_IMX31_CS5_SIZE              0x2000000
+#define FSL_IMX31_NAND_ADDR             0xB8000000
+#define FSL_IMX31_NAND_SIZE             0x1000
+
+#define FSL_IMX31_EPIT2_IRQ             27
+#define FSL_IMX31_EPIT1_IRQ             28
+#define FSL_IMX31_GPT_IRQ               29
+#define FSL_IMX31_UART2_IRQ             32
+#define FSL_IMX31_UART1_IRQ             45
+#define FSL_IMX31_I2C1_IRQ              10
+#define FSL_IMX31_I2C2_IRQ              4
+#define FSL_IMX31_I2C3_IRQ              3
+#define FSL_IMX31_GPIO1_IRQ             52
+#define FSL_IMX31_GPIO2_IRQ             51
+#define FSL_IMX31_GPIO3_IRQ             56
+
+#endif /* FSL_IMX31_H */
diff --git a/include/hw/arm/imx.h b/include/hw/arm/imx.h
deleted file mode 100644
index ea9e09327..000000000
--- a/include/hw/arm/imx.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * i.MX31 emulation
- *
- * Copyright (C) 2012 Peter Chubb
- * NICTA
- *
- * This code is released under the GPL, version 2.0 or later
- * See the file `../COPYING' for details.
- */
-
-#ifndef IMX_H
-#define IMX_H
-
-void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq);
-
-typedef enum  {
-    NOCLK,
-    MCU,
-    HSP,
-    IPG,
-    CLK_32k
-} IMXClk;
-
-uint32_t imx_clock_frequency(DeviceState *s, IMXClk clock);
-
-void imx_timerp_create(const hwaddr addr,
-                      qemu_irq irq,
-                      DeviceState *ccm);
-void imx_timerg_create(const hwaddr addr,
-                      qemu_irq irq,
-                      DeviceState *ccm);
-
-
-#endif /* IMX_H */
diff --git a/include/hw/arm/linux-boot-if.h b/include/hw/arm/linux-boot-if.h
new file mode 100644
index 000000000..aba4479a1
--- /dev/null
+++ b/include/hw/arm/linux-boot-if.h
@@ -0,0 +1,43 @@
+/*
+ * hw/arm/linux-boot-if.h : interface for devices which need to behave
+ * specially for direct boot of an ARM Linux kernel
+ */
+
+#ifndef HW_ARM_LINUX_BOOT_IF_H
+#define HW_ARM_LINUX_BOOT_IF_H
+
+#include "qom/object.h"
+
+#define TYPE_ARM_LINUX_BOOT_IF "arm-linux-boot-if"
+#define ARM_LINUX_BOOT_IF_CLASS(klass) \
+    OBJECT_CLASS_CHECK(ARMLinuxBootIfClass, (klass), TYPE_ARM_LINUX_BOOT_IF)
+#define ARM_LINUX_BOOT_IF_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ARMLinuxBootIfClass, (obj), TYPE_ARM_LINUX_BOOT_IF)
+#define ARM_LINUX_BOOT_IF(obj) \
+    INTERFACE_CHECK(ARMLinuxBootIf, (obj), TYPE_ARM_LINUX_BOOT_IF)
+
+typedef struct ARMLinuxBootIf {
+    /*< private >*/
+    Object parent_obj;
+} ARMLinuxBootIf;
+
+typedef struct ARMLinuxBootIfClass {
+    /*< private >*/
+    InterfaceClass parent_class;
+
+    /*< public >*/
+    /** arm_linux_init: configure the device for a direct boot
+     * of an ARM Linux kernel (so that device reset puts it into
+     * the state the kernel expects after firmware initialization,
+     * rather than the true hardware reset state). This callback is
+     * called once after machine construction is complete (before the
+     * first system reset).
+     *
+     * @obj: the object implementing this interface
+     * @secure_boot: true if we are booting Secure, false for NonSecure
+     * (or for a CPU which doesn't support TrustZone)
+     */
+    void (*arm_linux_init)(ARMLinuxBootIf *obj, bool secure_boot);
+} ARMLinuxBootIfClass;
+
+#endif
diff --git a/include/hw/arm/virt-acpi-build.h b/include/hw/arm/virt-acpi-build.h
index 04f174d52..744b66638 100644
--- a/include/hw/arm/virt-acpi-build.h
+++ b/include/hw/arm/virt-acpi-build.h
@@ -31,6 +31,8 @@ typedef struct VirtGuestInfo {
     FWCfgState *fw_cfg;
     const MemMapEntry *memmap;
     const int *irqmap;
+    bool use_highmem;
+    int gic_version;
 } VirtGuestInfo;
 
 
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index d22fd8e50..f46458630 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -46,6 +46,9 @@ enum {
     VIRT_CPUPERIPHS,
     VIRT_GIC_DIST,
     VIRT_GIC_CPU,
+    VIRT_GIC_V2M,
+    VIRT_GIC_ITS,
+    VIRT_GIC_REDIST,
     VIRT_UART,
     VIRT_MMIO,
     VIRT_RTC,
@@ -54,8 +57,8 @@ enum {
     VIRT_PCIE_MMIO,
     VIRT_PCIE_PIO,
     VIRT_PCIE_ECAM,
-    VIRT_GIC_V2M,
     VIRT_PLATFORM_BUS,
+    VIRT_PCIE_MMIO_HIGH,
 };
 
 typedef struct MemMapEntry {
diff --git a/include/hw/arm/xlnx-zynqmp.h b/include/hw/arm/xlnx-zynqmp.h
index c379632f2..d1160920c 100644
--- a/include/hw/arm/xlnx-zynqmp.h
+++ b/include/hw/arm/xlnx-zynqmp.h
@@ -22,6 +22,9 @@
 #include "hw/intc/arm_gic.h"
 #include "hw/net/cadence_gem.h"
 #include "hw/char/cadence_uart.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
+#include "hw/sd/sdhci.h"
 
 #define TYPE_XLNX_ZYNQMP "xlnx,zynqmp"
 #define XLNX_ZYNQMP(obj) OBJECT_CHECK(XlnxZynqMPState, (obj), \
@@ -31,6 +34,11 @@
 #define XLNX_ZYNQMP_NUM_RPU_CPUS 2
 #define XLNX_ZYNQMP_NUM_GEMS 4
 #define XLNX_ZYNQMP_NUM_UARTS 2
+#define XLNX_ZYNQMP_NUM_SDHCI 2
+
+#define XLNX_ZYNQMP_NUM_OCM_BANKS 4
+#define XLNX_ZYNQMP_OCM_RAM_0_ADDRESS 0xFFFC0000
+#define XLNX_ZYNQMP_OCM_RAM_SIZE 0x10000
 
 #define XLNX_ZYNQMP_GIC_REGIONS 2
 
@@ -40,7 +48,7 @@
  * number of memory region aliases.
  */
 
-#define XLNX_ZYNQMP_GIC_REGION_SIZE 0x4000
+#define XLNX_ZYNQMP_GIC_REGION_SIZE 0x1000
 #define XLNX_ZYNQMP_GIC_ALIASES     (0x10000 / XLNX_ZYNQMP_GIC_REGION_SIZE - 1)
 
 typedef struct XlnxZynqMPState {
@@ -52,8 +60,12 @@ typedef struct XlnxZynqMPState {
     ARMCPU rpu_cpu[XLNX_ZYNQMP_NUM_RPU_CPUS];
     GICState gic;
     MemoryRegion gic_mr[XLNX_ZYNQMP_GIC_REGIONS][XLNX_ZYNQMP_GIC_ALIASES];
+    MemoryRegion ocm_ram[XLNX_ZYNQMP_NUM_OCM_BANKS];
+
     CadenceGEMState gem[XLNX_ZYNQMP_NUM_GEMS];
     CadenceUARTState uart[XLNX_ZYNQMP_NUM_UARTS];
+    SysbusAHCIState sata;
+    SDHCIState sdhci[XLNX_ZYNQMP_NUM_SDHCI];
 
     char *boot_cpu;
     ARMCPU *boot_cpu_ptr;
diff --git a/include/hw/boards.h b/include/hw/boards.h
index 2aec9cbb1..59cac05d1 100644
--- a/include/hw/boards.h
+++ b/include/hw/boards.h
@@ -9,37 +9,17 @@
 #include "hw/qdev.h"
 #include "qom/object.h"
 
-
-typedef void QEMUMachineInitFunc(MachineState *ms);
-
-typedef void QEMUMachineResetFunc(void);
-
-typedef void QEMUMachineHotAddCPUFunc(const int64_t id, Error **errp);
-
-typedef int QEMUMachineGetKvmtypeFunc(const char *arg);
-
-struct QEMUMachine {
-    const char *name;
-    const char *desc;
-    QEMUMachineInitFunc *init;
-    QEMUMachineGetKvmtypeFunc *kvm_type;
-    BlockInterfaceType block_default_type;
-    int max_cpus;
-    unsigned int
-        no_sdcard:1,
-        has_dynamic_sysbus:1;
-    int is_default;
-    const char *default_machine_opts;
-    const char *default_boot_order;
-};
-
 void memory_region_allocate_system_memory(MemoryRegion *mr, Object *owner,
                                           const char *name,
                                           uint64_t ram_size);
 
-int qemu_register_machine(QEMUMachine *m);
-
 #define TYPE_MACHINE_SUFFIX "-machine"
+
+/* Machine class name that needs to be used for class-name-based machine
+ * type lookup to work.
+ */
+#define MACHINE_TYPE_NAME(machinename) (machinename TYPE_MACHINE_SUFFIX)
+
 #define TYPE_MACHINE "machine"
 #undef MACHINE  /* BSD defines it and QEMU does not use it */
 #define MACHINE(obj) \
@@ -53,7 +33,6 @@ MachineClass *find_default_machine(void);
 extern MachineState *current_machine;
 
 bool machine_usb(MachineState *machine);
-bool machine_iommu(MachineState *machine);
 bool machine_kernel_irqchip_allowed(MachineState *machine);
 bool machine_kernel_irqchip_required(MachineState *machine);
 int machine_kvm_shadow_mem(MachineState *machine);
@@ -63,7 +42,6 @@ bool machine_mem_merge(MachineState *machine);
 
 /**
  * MachineClass:
- * @qemu_machine: #QEMUMachine
  * @get_hotplug_handler: this function is called during bus-less
  *    device hotplug. If defined it returns pointer to an instance
  *    of HotplugHandler object, which handles hotplug operation
@@ -73,6 +51,11 @@ bool machine_mem_merge(MachineState *machine);
  *    used to provide @cpu_index to socket number mapping, allowing
  *    a machine to group CPU threads belonging to the same socket/package
  *    Returns: socket number given cpu_index belongs to.
+ * @hw_version:
+ *    Value of QEMU_VERSION when the machine was added to QEMU.
+ *    Set only by old machines because they need to keep
+ *    compatibility on code that exposed QEMU_VERSION to guests in
+ *    the past (and now use qemu_hw_version()).
  */
 struct MachineClass {
     /*< private >*/
@@ -100,7 +83,8 @@ struct MachineClass {
         no_cdrom:1,
         no_sdcard:1,
         has_dynamic_sysbus:1,
-        no_tco:1;
+        no_tco:1,
+        pci_allow_0_address:1;
     int is_default;
     const char *default_machine_opts;
     const char *default_boot_order;
@@ -136,9 +120,11 @@ struct MachineState {
     bool mem_merge;
     bool usb;
     bool usb_disabled;
+    bool igd_gfx_passthru;
     char *firmware;
     bool iommu;
     bool suppress_vmdesc;
+    bool enforce_config_section;
 
     ram_addr_t ram_size;
     ram_addr_t maxram_size;
@@ -151,4 +137,21 @@ struct MachineState {
     AccelState *accelerator;
 };
 
+#define DEFINE_MACHINE(namestr, machine_initfn) \
+    static void machine_initfn##_class_init(ObjectClass *oc, void *data) \
+    { \
+        MachineClass *mc = MACHINE_CLASS(oc); \
+        machine_initfn(mc); \
+    } \
+    static const TypeInfo machine_initfn##_typeinfo = { \
+        .name       = MACHINE_TYPE_NAME(namestr), \
+        .parent     = TYPE_MACHINE, \
+        .class_init = machine_initfn##_class_init, \
+    }; \
+    static void machine_initfn##_register_types(void) \
+    { \
+        type_register_static(&machine_initfn##_typeinfo); \
+    } \
+    machine_init(machine_initfn##_register_types)
+
 #endif
diff --git a/include/hw/bt.h b/include/hw/bt.h
index cb2a7e657..c7c7909a3 100644
--- a/include/hw/bt.h
+++ b/include/hw/bt.h
@@ -504,7 +504,6 @@ typedef struct {
 
 #define OCF_CREATE_CONN_CANCEL		0x0008
 typedef struct {
-    uint8_t	status;
     bdaddr_t	bdaddr;
 } QEMU_PACKED create_conn_cancel_cp;
 #define CREATE_CONN_CANCEL_CP_SIZE 6
@@ -1266,13 +1265,13 @@ typedef struct {
     uint8_t	status;
     uint16_t	handle;
 } QEMU_PACKED reset_failed_contact_counter_rp;
-#define RESET_FAILED_CONTACT_COUNTER_RP_SIZE 4
+#define RESET_FAILED_CONTACT_COUNTER_RP_SIZE 3
 
 #define OCF_READ_LINK_QUALITY		0x0003
 typedef struct {
     uint16_t	handle;
 } QEMU_PACKED read_link_quality_cp;
-#define READ_LINK_QUALITY_CP_SIZE 4
+#define READ_LINK_QUALITY_CP_SIZE 2
 
 typedef struct {
     uint8_t	status;
@@ -1332,7 +1331,7 @@ typedef struct {
     uint8_t	dev_class[3];
     uint16_t	clock_offset;
 } QEMU_PACKED inquiry_info;
-#define INQUIRY_INFO_SIZE 14
+#define INQUIRY_INFO_SIZE 15
 
 #define EVT_CONN_COMPLETE		0x03
 typedef struct {
@@ -1381,7 +1380,7 @@ typedef struct {
     uint16_t	handle;
     uint8_t	encrypt;
 } QEMU_PACKED evt_encrypt_change;
-#define EVT_ENCRYPT_CHANGE_SIZE 5
+#define EVT_ENCRYPT_CHANGE_SIZE 4
 
 #define EVT_CHANGE_CONN_LINK_KEY_COMPLETE	0x09
 typedef struct {
@@ -1629,18 +1628,6 @@ typedef struct {
 } QEMU_PACKED evt_sniff_subrate;
 #define EVT_SNIFF_SUBRATE_SIZE 11
 
-#define EVT_EXTENDED_INQUIRY_RESULT	0x2F
-typedef struct {
-    bdaddr_t	bdaddr;
-    uint8_t	pscan_rep_mode;
-    uint8_t	pscan_period_mode;
-    uint8_t	dev_class[3];
-    uint16_t	clock_offset;
-    int8_t	rssi;
-    uint8_t	data[240];
-} QEMU_PACKED extended_inquiry_info;
-#define EXTENDED_INQUIRY_INFO_SIZE 254
-
 #define EVT_TESTING			0xFE
 
 #define EVT_VENDOR			0xFF
diff --git a/include/hw/char/imx_serial.h b/include/hw/char/imx_serial.h
new file mode 100644
index 000000000..6cd75c0ba
--- /dev/null
+++ b/include/hw/char/imx_serial.h
@@ -0,0 +1,102 @@
+/*
+ * Device model for i.MX UART
+ *
+ * Copyright (c) 2008 OKL
+ * Originally Written by Hans Jiang
+ * Copyright (c) 2011 NICTA Pty Ltd.
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IMX_SERIAL_H
+#define IMX_SERIAL_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_SERIAL "imx.serial"
+#define IMX_SERIAL(obj) OBJECT_CHECK(IMXSerialState, (obj), TYPE_IMX_SERIAL)
+
+#define URXD_CHARRDY    (1<<15)   /* character read is valid */
+#define URXD_ERR        (1<<14)   /* Character has error */
+#define URXD_BRK        (1<<11)   /* Break received */
+
+#define USR1_PARTYER    (1<<15)   /* Parity Error */
+#define USR1_RTSS       (1<<14)   /* RTS pin status */
+#define USR1_TRDY       (1<<13)   /* Tx ready */
+#define USR1_RTSD       (1<<12)   /* RTS delta: pin changed state */
+#define USR1_ESCF       (1<<11)   /* Escape sequence interrupt */
+#define USR1_FRAMERR    (1<<10)   /* Framing error  */
+#define USR1_RRDY       (1<<9)    /* receiver ready */
+#define USR1_AGTIM      (1<<8)    /* Aging timer interrupt */
+#define USR1_DTRD       (1<<7)    /* DTR changed */
+#define USR1_RXDS       (1<<6)    /* Receiver is idle */
+#define USR1_AIRINT     (1<<5)    /* Aysnch IR interrupt */
+#define USR1_AWAKE      (1<<4)    /* Falling edge detected on RXd pin */
+
+#define USR2_ADET       (1<<15)   /* Autobaud complete */
+#define USR2_TXFE       (1<<14)   /* Transmit FIFO empty */
+#define USR2_DTRF       (1<<13)   /* DTR/DSR transition */
+#define USR2_IDLE       (1<<12)   /* UART has been idle for too long */
+#define USR2_ACST       (1<<11)   /* Autobaud counter stopped */
+#define USR2_RIDELT     (1<<10)   /* Ring Indicator delta */
+#define USR2_RIIN       (1<<9)    /* Ring Indicator Input */
+#define USR2_IRINT      (1<<8)    /* Serial Infrared Interrupt */
+#define USR2_WAKE       (1<<7)    /* Start bit detected */
+#define USR2_DCDDELT    (1<<6)    /* Data Carrier Detect delta */
+#define USR2_DCDIN      (1<<5)    /* Data Carrier Detect Input */
+#define USR2_RTSF       (1<<4)    /* RTS transition */
+#define USR2_TXDC       (1<<3)    /* Transmission complete */
+#define USR2_BRCD       (1<<2)    /* Break condition detected */
+#define USR2_ORE        (1<<1)    /* Overrun error */
+#define USR2_RDR        (1<<0)    /* Receive data ready */
+
+#define UCR1_TRDYEN     (1<<13)   /* Tx Ready Interrupt Enable */
+#define UCR1_RRDYEN     (1<<9)    /* Rx Ready Interrupt Enable */
+#define UCR1_TXMPTYEN   (1<<6)    /* Tx Empty Interrupt Enable */
+#define UCR1_UARTEN     (1<<0)    /* UART Enable */
+
+#define UCR2_TXEN       (1<<2)    /* Transmitter enable */
+#define UCR2_RXEN       (1<<1)    /* Receiver enable */
+#define UCR2_SRST       (1<<0)    /* Reset complete */
+
+#define UTS1_TXEMPTY    (1<<6)
+#define UTS1_RXEMPTY    (1<<5)
+#define UTS1_TXFULL     (1<<4)
+#define UTS1_RXFULL     (1<<3)
+
+typedef struct IMXSerialState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    int32_t readbuff;
+
+    uint32_t usr1;
+    uint32_t usr2;
+    uint32_t ucr1;
+    uint32_t ucr2;
+    uint32_t uts1;
+
+    /*
+     * The registers below are implemented just so that the
+     * guest OS sees what it has written
+     */
+    uint32_t onems;
+    uint32_t ufcr;
+    uint32_t ubmr;
+    uint32_t ubrc;
+    uint32_t ucr3;
+
+    qemu_irq irq;
+    CharDriverState *chr;
+} IMXSerialState;
+
+#endif
diff --git a/include/hw/compat.h b/include/hw/compat.h
index 94c8097d0..b7973db5e 100644
--- a/include/hw/compat.h
+++ b/include/hw/compat.h
@@ -1,6 +1,33 @@
 #ifndef HW_COMPAT_H
 #define HW_COMPAT_H
 
+#define HW_COMPAT_2_4 \
+        {\
+            .driver   = "virtio-blk-device",\
+            .property = "scsi",\
+            .value    = "true",\
+        },{\
+            .driver   = "e1000",\
+            .property = "extra_mac_registers",\
+            .value    = "off",\
+        },{\
+            .driver   = "virtio-pci",\
+            .property = "x-disable-pcie",\
+            .value    = "on",\
+        },{\
+            .driver   = "virtio-pci",\
+            .property = "migrate-extra",\
+            .value    = "off",\
+        },{\
+            .driver   = "fw_cfg_mem",\
+            .property = "dma_enabled",\
+            .value    = "off",\
+        },{\
+            .driver   = "fw_cfg_io",\
+            .property = "dma_enabled",\
+            .value    = "off",\
+        },
+
 #define HW_COMPAT_2_3 \
         {\
             .driver   = "virtio-blk-pci",\
diff --git a/include/hw/cpu/icc_bus.h b/include/hw/cpu/icc_bus.h
deleted file mode 100644
index 98a979fa1..000000000
--- a/include/hw/cpu/icc_bus.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* icc_bus.h
- * emulate x86 ICC (Interrupt Controller Communications) bus
- *
- * Copyright (c) 2013 Red Hat, Inc
- *
- * Authors:
- *     Igor Mammedov <imammedo@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-#ifndef ICC_BUS_H
-#define ICC_BUS_H
-
-#include "exec/memory.h"
-#include "hw/qdev-core.h"
-
-#define TYPE_ICC_BUS "icc-bus"
-
-#ifndef CONFIG_USER_ONLY
-
-/**
- * ICCBus:
- *
- * ICC bus
- */
-typedef struct ICCBus {
-    /*< private >*/
-    BusState parent_obj;
-    /*< public >*/
-
-    MemoryRegion *apic_address_space;
-} ICCBus;
-
-#define ICC_BUS(obj) OBJECT_CHECK(ICCBus, (obj), TYPE_ICC_BUS)
-
-/**
- * ICCDevice:
- *
- * ICC device
- */
-typedef struct ICCDevice {
-    /*< private >*/
-    DeviceState qdev;
-    /*< public >*/
-} ICCDevice;
-
-/**
- * ICCDeviceClass:
- * @init: Initialization callback for derived classes.
- *
- * ICC device class
- */
-typedef struct ICCDeviceClass {
-    /*< private >*/
-    DeviceClass parent_class;
-    /*< public >*/
-
-    DeviceRealize realize;
-} ICCDeviceClass;
-
-#define TYPE_ICC_DEVICE "icc-device"
-#define ICC_DEVICE(obj) OBJECT_CHECK(ICCDevice, (obj), TYPE_ICC_DEVICE)
-#define ICC_DEVICE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(ICCDeviceClass, (klass), TYPE_ICC_DEVICE)
-#define ICC_DEVICE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(ICCDeviceClass, (obj), TYPE_ICC_DEVICE)
-
-#define TYPE_ICC_BRIDGE "icc-bridge"
-
-#endif /* CONFIG_USER_ONLY */
-#endif
diff --git a/include/hw/elf_ops.h b/include/hw/elf_ops.h
index bd7196814..0010c441d 100644
--- a/include/hw/elf_ops.h
+++ b/include/hw/elf_ops.h
@@ -282,25 +282,36 @@ static int glue(load_elf, SZ)(const char *name, int fd,
 
     switch (elf_machine) {
         case EM_PPC64:
-            if (EM_PPC64 != ehdr.e_machine)
-                if (EM_PPC != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_PPC64) {
+                if (ehdr.e_machine != EM_PPC) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
             break;
         case EM_X86_64:
-            if (EM_X86_64 != ehdr.e_machine)
-                if (EM_386 != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_X86_64) {
+                if (ehdr.e_machine != EM_386) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
             break;
         case EM_MICROBLAZE:
-            if (EM_MICROBLAZE != ehdr.e_machine)
-                if (EM_MICROBLAZE_OLD != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_MICROBLAZE) {
+                if (ehdr.e_machine != EM_MICROBLAZE_OLD) {
+                    ret = ELF_LOAD_WRONG_ARCH;
+                    goto fail;
+                }
+            }
+            break;
+        case EM_MOXIE:
+            if (ehdr.e_machine != EM_MOXIE) {
+                if (ehdr.e_machine != EM_MOXIE_OLD) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
             break;
         default:
             if (elf_machine != ehdr.e_machine) {
diff --git a/include/hw/gpio/imx_gpio.h b/include/hw/gpio/imx_gpio.h
new file mode 100644
index 000000000..517b2618c
--- /dev/null
+++ b/include/hw/gpio/imx_gpio.h
@@ -0,0 +1,62 @@
+/*
+ * i.MX processors GPIO registers definition.
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __IMX_GPIO_H_
+#define __IMX_GPIO_H_
+
+#include <hw/sysbus.h>
+
+#define TYPE_IMX_GPIO "imx.gpio"
+#define IMX_GPIO(obj) OBJECT_CHECK(IMXGPIOState, (obj), TYPE_IMX_GPIO)
+
+#define IMX_GPIO_MEM_SIZE 0x20
+
+/* i.MX GPIO memory map */
+#define DR_ADDR             0x00 /* DATA REGISTER */
+#define GDIR_ADDR           0x04 /* DIRECTION REGISTER */
+#define PSR_ADDR            0x08 /* PAD STATUS REGISTER */
+#define ICR1_ADDR           0x0c /* INTERRUPT CONFIGURATION REGISTER 1 */
+#define ICR2_ADDR           0x10 /* INTERRUPT CONFIGURATION REGISTER 2 */
+#define IMR_ADDR            0x14 /* INTERRUPT MASK REGISTER */
+#define ISR_ADDR            0x18 /* INTERRUPT STATUS REGISTER */
+#define EDGE_SEL_ADDR       0x1c /* EDGE SEL REGISTER */
+
+#define IMX_GPIO_PIN_COUNT 32
+
+typedef struct IMXGPIOState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t dr;
+    uint32_t gdir;
+    uint32_t psr;
+    uint64_t icr;
+    uint32_t imr;
+    uint32_t isr;
+    bool has_edge_sel;
+    uint32_t edge_sel;
+
+    qemu_irq irq;
+    qemu_irq output[IMX_GPIO_PIN_COUNT];
+} IMXGPIOState;
+
+#endif /* __IMX_GPIO_H_ */
diff --git a/include/hw/i2c/imx_i2c.h b/include/hw/i2c/imx_i2c.h
new file mode 100644
index 000000000..e2ee8eaee
--- /dev/null
+++ b/include/hw/i2c/imx_i2c.h
@@ -0,0 +1,87 @@
+/*
+ *  i.MX I2C Bus Serial Interface registers definition
+ *
+ *  Copyright (C) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef __IMX_I2C_H_
+#define __IMX_I2C_H_
+
+#include <hw/sysbus.h>
+
+#define TYPE_IMX_I2C "imx.i2c"
+#define IMX_I2C(obj) OBJECT_CHECK(IMXI2CState, (obj), TYPE_IMX_I2C)
+
+#define IMX_I2C_MEM_SIZE           0x14
+
+/* i.MX I2C memory map */
+#define IADR_ADDR                  0x00  /* address register */
+#define IFDR_ADDR                  0x04  /* frequency divider register */
+#define I2CR_ADDR                  0x08  /* control register */
+#define I2SR_ADDR                  0x0c  /* status register */
+#define I2DR_ADDR                  0x10  /* data register */
+
+#define IADR_MASK                  0xFE
+#define IADR_RESET                 0
+
+#define IFDR_MASK                  0x3F
+#define IFDR_RESET                 0
+
+#define I2CR_IEN                   (1 << 7)
+#define I2CR_IIEN                  (1 << 6)
+#define I2CR_MSTA                  (1 << 5)
+#define I2CR_MTX                   (1 << 4)
+#define I2CR_TXAK                  (1 << 3)
+#define I2CR_RSTA                  (1 << 2)
+#define I2CR_MASK                  0xFC
+#define I2CR_RESET                 0
+
+#define I2SR_ICF                   (1 << 7)
+#define I2SR_IAAF                  (1 << 6)
+#define I2SR_IBB                   (1 << 5)
+#define I2SR_IAL                   (1 << 4)
+#define I2SR_SRW                   (1 << 2)
+#define I2SR_IIF                   (1 << 1)
+#define I2SR_RXAK                  (1 << 0)
+#define I2SR_MASK                  0xE9
+#define I2SR_RESET                 0x81
+
+#define I2DR_MASK                  0xFF
+#define I2DR_RESET                 0
+
+#define ADDR_RESET                 0xFF00
+
+typedef struct IMXI2CState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    I2CBus *bus;
+    qemu_irq irq;
+
+    uint16_t  address;
+
+    uint16_t iadr;
+    uint16_t ifdr;
+    uint16_t i2cr;
+    uint16_t i2sr;
+    uint16_t i2dr_read;
+    uint16_t i2dr_write;
+} IMXI2CState;
+
+#endif /* __IMX_I2C_H_ */
diff --git a/include/hw/i386/apic_internal.h b/include/hw/i386/apic_internal.h
index dc7a89d98..74fe935e8 100644
--- a/include/hw/i386/apic_internal.h
+++ b/include/hw/i386/apic_internal.h
@@ -20,8 +20,8 @@
 #ifndef QEMU_APIC_INTERNAL_H
 #define QEMU_APIC_INTERNAL_H
 
+#include "cpu.h"
 #include "exec/memory.h"
-#include "hw/cpu/icc_bus.h"
 #include "qemu/timer.h"
 
 /* APIC Local Vector Table */
@@ -49,17 +49,73 @@
 #define APIC_TRIGGER_EDGE               0
 #define APIC_TRIGGER_LEVEL              1
 
-#define APIC_LVT_TIMER_PERIODIC         (1<<17)
-#define APIC_LVT_MASKED                 (1<<16)
-#define APIC_LVT_LEVEL_TRIGGER          (1<<15)
-#define APIC_LVT_REMOTE_IRR             (1<<14)
-#define APIC_INPUT_POLARITY             (1<<13)
-#define APIC_SEND_PENDING               (1<<12)
-
-#define ESR_ILLEGAL_ADDRESS (1 << 7)
-
-#define APIC_SV_DIRECTED_IO             (1<<12)
-#define APIC_SV_ENABLE                  (1<<8)
+#define APIC_VECTOR_MASK                0xff
+#define APIC_DCR_MASK                   0xf
+
+#define APIC_LVT_TIMER_SHIFT            17
+#define APIC_LVT_MASKED_SHIFT           16
+#define APIC_LVT_LEVEL_TRIGGER_SHIFT    15
+#define APIC_LVT_REMOTE_IRR_SHIFT       14
+#define APIC_LVT_INT_POLARITY_SHIFT     13
+#define APIC_LVT_DELIV_STS_SHIFT        12
+#define APIC_LVT_DELIV_MOD_SHIFT        8
+
+#define APIC_LVT_TIMER_TSCDEADLINE      (2 << APIC_LVT_TIMER_SHIFT)
+#define APIC_LVT_TIMER_PERIODIC         (1 << APIC_LVT_TIMER_SHIFT)
+#define APIC_LVT_MASKED                 (1 << APIC_LVT_MASKED_SHIFT)
+#define APIC_LVT_LEVEL_TRIGGER          (1 << APIC_LVT_LEVEL_TRIGGER_SHIFT)
+#define APIC_LVT_REMOTE_IRR             (1 << APIC_LVT_REMOTE_IRR_SHIFT)
+#define APIC_LVT_INT_POLARITY           (1 << APIC_LVT_INT_POLARITY_SHIFT)
+#define APIC_LVT_DELIV_STS              (1 << APIC_LVT_DELIV_STS_SHIFT)
+#define APIC_LVT_DELIV_MOD              (7 << APIC_LVT_DELIV_MOD_SHIFT)
+
+#define APIC_ESR_ILL_ADDRESS_SHIFT      7
+#define APIC_ESR_RECV_ILL_VECT_SHIFT    6
+#define APIC_ESR_SEND_ILL_VECT_SHIFT    5
+#define APIC_ESR_RECV_ACCEPT_SHIFT      3
+#define APIC_ESR_SEND_ACCEPT_SHIFT      2
+#define APIC_ESR_RECV_CHECK_SUM_SHIFT   1
+
+#define APIC_ESR_ILLEGAL_ADDRESS        (1 << APIC_ESR_ILL_ADDRESS_SHIFT)
+#define APIC_ESR_RECV_ILLEGAL_VECT      (1 << APIC_ESR_RECV_ILL_VECT_SHIFT)
+#define APIC_ESR_SEND_ILLEGAL_VECT      (1 << APIC_ESR_SEND_ILL_VECT_SHIFT)
+#define APIC_ESR_RECV_ACCEPT            (1 << APIC_ESR_RECV_ACCEPT_SHIFT)
+#define APIC_ESR_SEND_ACCEPT            (1 << APIC_ESR_SEND_ACCEPT_SHIFT)
+#define APIC_ESR_RECV_CHECK_SUM         (1 << APIC_ESR_RECV_CHECK_SUM_SHIFT)
+#define APIC_ESR_SEND_CHECK_SUM         1
+
+#define APIC_ICR_DEST_SHIFT             24
+#define APIC_ICR_DEST_SHORT_SHIFT       18
+#define APIC_ICR_TRIGGER_MOD_SHIFT      15
+#define APIC_ICR_LEVEL_SHIFT            14
+#define APIC_ICR_DELIV_STS_SHIFT        12
+#define APIC_ICR_DEST_MOD_SHIFT         11
+#define APIC_ICR_DELIV_MOD_SHIFT        8
+
+#define APIC_ICR_DEST_SHORT             (3 << APIC_ICR_DEST_SHORT_SHIFT)
+#define APIC_ICR_TRIGGER_MOD            (1 << APIC_ICR_TRIGGER_MOD_SHIFT)
+#define APIC_ICR_LEVEL                  (1 << APIC_ICR_LEVEL_SHIFT)
+#define APIC_ICR_DELIV_STS              (1 << APIC_ICR_DELIV_STS_SHIFT)
+#define APIC_ICR_DEST_MOD               (1 << APIC_ICR_DEST_MOD_SHIFT)
+#define APIC_ICR_DELIV_MOD              (7 << APIC_ICR_DELIV_MOD_SHIFT)
+
+#define APIC_PR_CLASS_SHIFT             4
+#define APIC_PR_SUB_CLASS               0xf
+
+#define APIC_LOGDEST_XAPIC_SHIFT        4
+#define APIC_LOGDEST_XAPIC_ID           0xf
+
+#define APIC_LOGDEST_X2APIC_SHIFT       16
+#define APIC_LOGDEST_X2APIC_ID          0xffff
+
+#define APIC_SPURIO_FOCUS_SHIFT         9
+#define APIC_SPURIO_ENABLED_SHIFT       8
+
+#define APIC_SPURIO_FOCUS               (1 << APIC_SPURIO_FOCUS_SHIFT)
+#define APIC_SPURIO_ENABLED             (1 << APIC_SPURIO_ENABLED_SHIFT)
+
+#define APIC_SV_DIRECTED_IO             (1 << 12)
+#define APIC_SV_ENABLE                  (1 << 8)
 
 #define VAPIC_ENABLE_BIT                0
 #define VAPIC_ENABLE_MASK               (1 << VAPIC_ENABLE_BIT)
@@ -78,7 +134,7 @@ typedef struct APICCommonState APICCommonState;
 
 typedef struct APICCommonClass
 {
-    ICCDeviceClass parent_class;
+    DeviceClass parent_class;
 
     DeviceRealize realize;
     void (*set_base)(APICCommonState *s, uint64_t val);
@@ -93,7 +149,9 @@ typedef struct APICCommonClass
 } APICCommonClass;
 
 struct APICCommonState {
-    ICCDevice busdev;
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
 
     MemoryRegion io_memory;
     X86CPU *cpu;
@@ -146,4 +204,22 @@ void apic_enable_vapic(DeviceState *d, hwaddr paddr);
 void vapic_report_tpr_access(DeviceState *dev, CPUState *cpu, target_ulong ip,
                              TPRAccess access);
 
+int apic_get_ppr(APICCommonState *s);
+
+static inline void apic_set_bit(uint32_t *tab, int index)
+{
+    int i, mask;
+    i = index >> 5;
+    mask = 1 << (index & 0x1f);
+    tab[i] |= mask;
+}
+
+static inline int apic_get_bit(uint32_t *tab, int index)
+{
+    int i, mask;
+    i = index >> 5;
+    mask = 1 << (index & 0x1f);
+    return !!(tab[i] & mask);
+}
+
 #endif /* !QEMU_APIC_INTERNAL_H */
diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h
index e321ee4fb..5dbadb785 100644
--- a/include/hw/i386/intel_iommu.h
+++ b/include/hw/i386/intel_iommu.h
@@ -49,6 +49,7 @@ typedef struct VTDContextCacheEntry VTDContextCacheEntry;
 typedef struct IntelIOMMUState IntelIOMMUState;
 typedef struct VTDAddressSpace VTDAddressSpace;
 typedef struct VTDIOTLBEntry VTDIOTLBEntry;
+typedef struct VTDBus VTDBus;
 
 /* Context-Entry */
 struct VTDContextEntry {
@@ -65,7 +66,7 @@ struct VTDContextCacheEntry {
 };
 
 struct VTDAddressSpace {
-    uint8_t bus_num;
+    PCIBus *bus;
     uint8_t devfn;
     AddressSpace as;
     MemoryRegion iommu;
@@ -73,6 +74,11 @@ struct VTDAddressSpace {
     VTDContextCacheEntry context_cache_entry;
 };
 
+struct VTDBus {
+    PCIBus* bus;		/* A reference to the bus to provide translation for */
+    VTDAddressSpace *dev_as[0];	/* A table of VTDAddressSpace objects indexed by devfn */
+};
+
 struct VTDIOTLBEntry {
     uint64_t gfn;
     uint16_t domain_id;
@@ -114,7 +120,13 @@ struct IntelIOMMUState {
     GHashTable *iotlb;              /* IOTLB */
 
     MemoryRegionIOMMUOps iommu_ops;
-    VTDAddressSpace **address_spaces[VTD_PCI_BUS_MAX];
+    GHashTable *vtd_as_by_busptr;   /* VTDBus objects indexed by PCIBus* reference */
+    VTDBus *vtd_as_by_bus_num[VTD_PCI_BUS_MAX]; /* VTDBus objects indexed by bus number */
 };
 
+/* Find the VTD Address space associated with the given bus pointer,
+ * create a new one if none exists
+ */
+VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn);
+
 #endif
diff --git a/include/hw/i386/ioapic_internal.h b/include/hw/i386/ioapic_internal.h
index 3be335218..797ed4730 100644
--- a/include/hw/i386/ioapic_internal.h
+++ b/include/hw/i386/ioapic_internal.h
@@ -40,7 +40,12 @@
 #define IOAPIC_LVT_DELIV_MODE_SHIFT     8
 
 #define IOAPIC_LVT_MASKED               (1 << IOAPIC_LVT_MASKED_SHIFT)
+#define IOAPIC_LVT_TRIGGER_MODE         (1 << IOAPIC_LVT_TRIGGER_MODE_SHIFT)
 #define IOAPIC_LVT_REMOTE_IRR           (1 << IOAPIC_LVT_REMOTE_IRR_SHIFT)
+#define IOAPIC_LVT_POLARITY             (1 << IOAPIC_LVT_POLARITY_SHIFT)
+#define IOAPIC_LVT_DELIV_STATUS         (1 << IOAPIC_LVT_DELIV_STATUS_SHIFT)
+#define IOAPIC_LVT_DEST_MODE            (1 << IOAPIC_LVT_DEST_MODE_SHIFT)
+#define IOAPIC_LVT_DELIV_MODE           (7 << IOAPIC_LVT_DELIV_MODE_SHIFT)
 
 #define IOAPIC_TRIGGER_EDGE             0
 #define IOAPIC_TRIGGER_LEVEL            1
@@ -100,4 +105,6 @@ struct IOAPICCommonState {
 
 void ioapic_reset_common(DeviceState *dev);
 
+void ioapic_print_redtbl(Monitor *mon, IOAPICCommonState *s);
+
 #endif /* !QEMU_IOAPIC_INTERNAL_H */
diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
index 954203d22..854c330b6 100644
--- a/include/hw/i386/pc.h
+++ b/include/hw/i386/pc.h
@@ -2,6 +2,7 @@
 #define HW_PC_H
 
 #include "qemu-common.h"
+#include "qemu/typedefs.h"
 #include "exec/memory.h"
 #include "hw/boards.h"
 #include "hw/isa/isa.h"
@@ -39,6 +40,7 @@ struct PCMachineState {
     OnOffAuto vmport;
     OnOffAuto smm;
     bool enforce_aligned_dimm;
+    ram_addr_t below_4g_mem_size, above_4g_mem_size;
 };
 
 #define PC_MACHINE_ACPI_DEVICE_PROP "acpi-device"
@@ -57,13 +59,11 @@ struct PCMachineClass {
     MachineClass parent_class;
 
     /*< public >*/
+    bool broken_reserved_end;
     HotplugHandler *(*get_hotplug_handler)(MachineState *machine,
                                            DeviceState *dev);
 };
 
-typedef struct PCMachineState PCMachineState;
-typedef struct PCMachineClass PCMachineClass;
-
 #define TYPE_PC_MACHINE "generic-pc-machine"
 #define PC_MACHINE(obj) \
     OBJECT_CHECK(PCMachineState, (obj), TYPE_PC_MACHINE)
@@ -123,6 +123,11 @@ int pic_get_output(DeviceState *d);
 void hmp_info_pic(Monitor *mon, const QDict *qdict);
 void hmp_info_irq(Monitor *mon, const QDict *qdict);
 
+/* ioapic.c */
+
+void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict);
+void ioapic_dump_state(Monitor *mon, const QDict *qdict);
+
 /* Global System Interrupts */
 
 #define GSI_NUM_PINS IOAPIC_NUM_PINS
@@ -162,12 +167,11 @@ bool pc_machine_is_smm_enabled(PCMachineState *pcms);
 void pc_register_ferr_irq(qemu_irq irq);
 void pc_acpi_smi_interrupt(void *opaque, int irq, int level);
 
-void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge);
+void pc_cpus_init(PCMachineState *pcms);
 void pc_hot_add_cpu(const int64_t id, Error **errp);
 void pc_acpi_init(const char *default_dsdt);
 
-PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
-                                ram_addr_t above_4g_mem_size);
+PcGuestInfo *pc_guest_info_init(PCMachineState *pcms);
 
 void pc_set_legacy_acpi_data_size(void);
 
@@ -182,15 +186,10 @@ void pc_set_legacy_acpi_data_size(void);
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
                             MemoryRegion *pci_address_space);
 
-FWCfgState *xen_load_linux(const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           ram_addr_t below_4g_mem_size,
+FWCfgState *xen_load_linux(PCMachineState *pcms,
                            PcGuestInfo *guest_info);
-FWCfgState *pc_memory_init(MachineState *machine,
+FWCfgState *pc_memory_init(PCMachineState *pcms,
                            MemoryRegion *system_memory,
-                           ram_addr_t below_4g_mem_size,
-                           ram_addr_t above_4g_mem_size,
                            MemoryRegion *rom_memory,
                            MemoryRegion **ram_memory,
                            PcGuestInfo *guest_info);
@@ -202,8 +201,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           bool no_vmport,
                           uint32 hpet_irqs);
 void pc_init_ne2k_isa(ISABus *bus, NICInfo *nd);
-void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device, MachineState *machine,
+void pc_cmos_init(PCMachineState *pcms,
                   BusState *ide0, BusState *ide1,
                   ISADevice *s);
 void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus);
@@ -227,7 +225,13 @@ extern int no_hpet;
 struct PCII440FXState;
 typedef struct PCII440FXState PCII440FXState;
 
-PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix_devfn,
+#define TYPE_I440FX_PCI_HOST_BRIDGE "i440FX-pcihost"
+#define TYPE_I440FX_PCI_DEVICE "i440FX"
+
+#define TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE "igd-passthrough-i440FX"
+
+PCIBus *i440fx_init(const char *host_type, const char *pci_type,
+                    PCII440FXState **pi440fx_state, int *piix_devfn,
                     ISABus **isa_bus, qemu_irq *pic,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
@@ -292,7 +296,78 @@ int e820_add_entry(uint64_t, uint64_t, uint32_t);
 int e820_get_num_entries(void);
 bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
 
+#define PC_COMPAT_2_4 \
+        HW_COMPAT_2_4 \
+        {\
+            .driver   = "Haswell-" TYPE_X86_CPU,\
+            .property = "abm",\
+            .value    = "off",\
+        },\
+        {\
+            .driver   = "Haswell-noTSX-" TYPE_X86_CPU,\
+            .property = "abm",\
+            .value    = "off",\
+        },\
+        {\
+            .driver   = "Broadwell-" TYPE_X86_CPU,\
+            .property = "abm",\
+            .value    = "off",\
+        },\
+        {\
+            .driver   = "Broadwell-noTSX-" TYPE_X86_CPU,\
+            .property = "abm",\
+            .value    = "off",\
+        },\
+        {\
+            .driver   = "host" "-" TYPE_X86_CPU,\
+            .property = "host-cache-info",\
+            .value    = "on",\
+        },\
+        {\
+            .driver   = TYPE_X86_CPU,\
+            .property = "check",\
+            .value    = "off",\
+        },\
+        {\
+            .driver   = "qemu64" "-" TYPE_X86_CPU,\
+            .property = "sse4a",\
+            .value    = "on",\
+        },\
+        {\
+            .driver   = "qemu64" "-" TYPE_X86_CPU,\
+            .property = "abm",\
+            .value    = "on",\
+        },\
+        {\
+            .driver   = "qemu64" "-" TYPE_X86_CPU,\
+            .property = "popcnt",\
+            .value    = "on",\
+        },\
+        {\
+            .driver   = "qemu32" "-" TYPE_X86_CPU,\
+            .property = "popcnt",\
+            .value    = "on",\
+        },{\
+            .driver   = "Opteron_G2" "-" TYPE_X86_CPU,\
+            .property = "rdtscp",\
+            .value    = "on",\
+        },{\
+            .driver   = "Opteron_G3" "-" TYPE_X86_CPU,\
+            .property = "rdtscp",\
+            .value    = "on",\
+        },{\
+            .driver   = "Opteron_G4" "-" TYPE_X86_CPU,\
+            .property = "rdtscp",\
+            .value    = "on",\
+        },{\
+            .driver   = "Opteron_G5" "-" TYPE_X86_CPU,\
+            .property = "rdtscp",\
+            .value    = "on",\
+        },
+
+
 #define PC_COMPAT_2_3 \
+        PC_COMPAT_2_4 \
         HW_COMPAT_2_3 \
         {\
             .driver   = TYPE_X86_CPU,\
@@ -374,11 +449,111 @@ bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
 
 #define PC_COMPAT_2_2 \
         PC_COMPAT_2_3 \
-        HW_COMPAT_2_2
+        HW_COMPAT_2_2 \
+        {\
+            .driver = "kvm64" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "kvm32" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Conroe" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Penryn" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Nehalem" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Westmere" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "SandyBridge" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Haswell" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Broadwell" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Opteron_G1" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Opteron_G2" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Opteron_G3" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Opteron_G4" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Opteron_G5" "-" TYPE_X86_CPU,\
+            .property = "vme",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Haswell" "-" TYPE_X86_CPU,\
+            .property = "f16c",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Haswell" "-" TYPE_X86_CPU,\
+            .property = "rdrand",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Broadwell" "-" TYPE_X86_CPU,\
+            .property = "f16c",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Broadwell" "-" TYPE_X86_CPU,\
+            .property = "rdrand",\
+            .value = "off",\
+        },
 
 #define PC_COMPAT_2_1 \
         PC_COMPAT_2_2 \
-        HW_COMPAT_2_1
+        HW_COMPAT_2_1 \
+        {\
+            .driver = "coreduo" "-" TYPE_X86_CPU,\
+            .property = "vmx",\
+            .value = "on",\
+        },\
+        {\
+            .driver = "core2duo" "-" TYPE_X86_CPU,\
+            .property = "vmx",\
+            .value = "on",\
+        },
 
 #define PC_COMPAT_2_0 \
         PC_COMPAT_2_1 \
@@ -589,20 +764,18 @@ bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
             .driver   = "486-" TYPE_X86_CPU,\
             .property = "model",\
             .value    = stringify(0),\
+        },\
+        {\
+            .driver = "n270" "-" TYPE_X86_CPU,\
+            .property = "movbe",\
+            .value = "off",\
+        },\
+        {\
+            .driver = "Westmere" "-" TYPE_X86_CPU,\
+            .property = "pclmulqdq",\
+            .value = "off",\
         },
 
-static inline void pc_common_machine_options(MachineClass *m)
-{
-    m->default_boot_order = "cad";
-}
-
-static inline void pc_default_machine_options(MachineClass *m)
-{
-    pc_common_machine_options(m);
-    m->hot_add_cpu = pc_hot_add_cpu;
-    m->max_cpus = 255;
-}
-
 #define DEFINE_PC_MACHINE(suffix, namestr, initfn, optsfn) \
     static void pc_machine_##suffix##_class_init(ObjectClass *oc, void *data) \
     { \
@@ -630,4 +803,5 @@ static inline void pc_default_machine_options(MachineClass *m)
     (m)->compat_props = props; \
 } while (0)
 
+extern void igd_passthrough_isa_bridge_create(PCIBus *bus, uint16_t gpu_dev_id);
 #endif
diff --git a/include/hw/i386/topology.h b/include/hw/i386/topology.h
index 9c6f3a937..148cc1bbc 100644
--- a/include/hw/i386/topology.h
+++ b/include/hw/i386/topology.h
@@ -47,6 +47,12 @@
  */
 typedef uint32_t apic_id_t;
 
+typedef struct X86CPUTopoInfo {
+    unsigned pkg_id;
+    unsigned core_id;
+    unsigned smt_id;
+} X86CPUTopoInfo;
+
 /* Return the bit width needed for 'count' IDs
  */
 static unsigned apicid_bitwidth_for_count(unsigned count)
@@ -92,13 +98,11 @@ static inline unsigned apicid_pkg_offset(unsigned nr_cores, unsigned nr_threads)
  */
 static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
                                              unsigned nr_threads,
-                                             unsigned pkg_id,
-                                             unsigned core_id,
-                                             unsigned smt_id)
+                                             const X86CPUTopoInfo *topo)
 {
-    return (pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
-           (core_id << apicid_core_offset(nr_cores, nr_threads)) |
-           smt_id;
+    return (topo->pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
+           (topo->core_id << apicid_core_offset(nr_cores, nr_threads)) |
+           topo->smt_id;
 }
 
 /* Calculate thread/core/package IDs for a specific topology,
@@ -107,14 +111,12 @@ static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
 static inline void x86_topo_ids_from_idx(unsigned nr_cores,
                                          unsigned nr_threads,
                                          unsigned cpu_index,
-                                         unsigned *pkg_id,
-                                         unsigned *core_id,
-                                         unsigned *smt_id)
+                                         X86CPUTopoInfo *topo)
 {
     unsigned core_index = cpu_index / nr_threads;
-    *smt_id = cpu_index % nr_threads;
-    *core_id = core_index % nr_cores;
-    *pkg_id = core_index / nr_cores;
+    topo->smt_id = cpu_index % nr_threads;
+    topo->core_id = core_index % nr_cores;
+    topo->pkg_id = core_index / nr_cores;
 }
 
 /* Make APIC ID for the CPU 'cpu_index'
@@ -125,10 +127,9 @@ static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_cores,
                                                 unsigned nr_threads,
                                                 unsigned cpu_index)
 {
-    unsigned pkg_id, core_id, smt_id;
-    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index,
-                          &pkg_id, &core_id, &smt_id);
-    return apicid_from_topo_ids(nr_cores, nr_threads, pkg_id, core_id, smt_id);
+    X86CPUTopoInfo topo;
+    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index, &topo);
+    return apicid_from_topo_ids(nr_cores, nr_threads, &topo);
 }
 
 #endif /* HW_I386_TOPOLOGY_H */
diff --git a/include/hw/intc/arm_gic_common.h b/include/hw/intc/arm_gic_common.h
index 899db3d7a..f4c349a2e 100644
--- a/include/hw/intc/arm_gic_common.h
+++ b/include/hw/intc/arm_gic_common.h
@@ -68,7 +68,6 @@ typedef struct GICState {
     uint8_t irq_target[GIC_MAXIRQ];
     uint8_t priority1[GIC_INTERNAL][GIC_NCPU];
     uint8_t priority2[GIC_MAXIRQ - GIC_INTERNAL];
-    uint16_t last_active[GIC_MAXIRQ][GIC_NCPU];
     /* For each SGI on the target CPU, we store 8 bits
      * indicating which source CPUs have made this SGI
      * pending on the target CPU. These correspond to
@@ -78,7 +77,6 @@ typedef struct GICState {
     uint8_t sgi_pending[GIC_NR_SGIS][GIC_NCPU];
 
     uint16_t priority_mask[GIC_NCPU];
-    uint16_t running_irq[GIC_NCPU];
     uint16_t running_priority[GIC_NCPU];
     uint16_t current_pending[GIC_NCPU];
 
@@ -96,16 +94,9 @@ typedef struct GICState {
      * If an interrupt for preemption level X is active, then
      *   APRn[X mod 32] == 0b1,  where n = X / 32
      * otherwise the bit is clear.
-     *
-     * TODO: rewrite the interrupt acknowlege/complete routines to use
-     * the APR registers to track the necessary information to update
-     * s->running_priority[] on interrupt completion (ie completely remove
-     * last_active[][] and running_irq[]). This will be necessary if we ever
-     * want to support TCG<->KVM migration, or TCG guests which can
-     * do power management involving powering down and restarting
-     * the GIC.
      */
     uint32_t apr[GIC_NR_APRS][GIC_NCPU];
+    uint32_t nsapr[GIC_NR_APRS][GIC_NCPU];
 
     uint32_t num_cpu;
 
@@ -118,7 +109,9 @@ typedef struct GICState {
     uint32_t num_irq;
     uint32_t revision;
     bool security_extn;
+    bool irq_reset_nonsecure; /* configure IRQs as group 1 (NS) on reset? */
     int dev_fd; /* kvm device fd if backed by kvm vgic support */
+    Error *migration_blocker;
 } GICState;
 
 #define TYPE_ARM_GIC_COMMON "arm_gic_common"
@@ -138,4 +131,7 @@ typedef struct ARMGICCommonClass {
     void (*post_load)(GICState *s);
 } ARMGICCommonClass;
 
+void gic_init_irqs_and_mmio(GICState *s, qemu_irq_handler handler,
+                            const MemoryRegionOps *ops);
+
 #endif
diff --git a/include/hw/intc/arm_gicv3_common.h b/include/hw/intc/arm_gicv3_common.h
new file mode 100644
index 000000000..c2fd8da4e
--- /dev/null
+++ b/include/hw/intc/arm_gicv3_common.h
@@ -0,0 +1,68 @@
+/*
+ * ARM GIC support
+ *
+ * Copyright (c) 2012 Linaro Limited
+ * Copyright (c) 2015 Huawei.
+ * Written by Peter Maydell
+ * Extended to 64 cores by Shlomo Pongratz
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef HW_ARM_GICV3_COMMON_H
+#define HW_ARM_GICV3_COMMON_H
+
+#include "hw/sysbus.h"
+#include "hw/intc/arm_gic_common.h"
+
+typedef struct GICv3State {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    qemu_irq *parent_irq;
+    qemu_irq *parent_fiq;
+
+    MemoryRegion iomem_dist; /* Distributor */
+    MemoryRegion iomem_redist; /* Redistributors */
+
+    uint32_t num_cpu;
+    uint32_t num_irq;
+    uint32_t revision;
+    bool security_extn;
+
+    int dev_fd; /* kvm device fd if backed by kvm vgic support */
+} GICv3State;
+
+#define TYPE_ARM_GICV3_COMMON "arm-gicv3-common"
+#define ARM_GICV3_COMMON(obj) \
+     OBJECT_CHECK(GICv3State, (obj), TYPE_ARM_GICV3_COMMON)
+#define ARM_GICV3_COMMON_CLASS(klass) \
+     OBJECT_CLASS_CHECK(ARMGICv3CommonClass, (klass), TYPE_ARM_GICV3_COMMON)
+#define ARM_GICV3_COMMON_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(ARMGICv3CommonClass, (obj), TYPE_ARM_GICV3_COMMON)
+
+typedef struct ARMGICv3CommonClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+
+    void (*pre_save)(GICv3State *s);
+    void (*post_load)(GICv3State *s);
+} ARMGICv3CommonClass;
+
+void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler,
+                              const MemoryRegionOps *ops);
+
+#endif
diff --git a/include/hw/intc/imx_avic.h b/include/hw/intc/imx_avic.h
new file mode 100644
index 000000000..1b8076901
--- /dev/null
+++ b/include/hw/intc/imx_avic.h
@@ -0,0 +1,55 @@
+/*
+ * i.MX31 Vectored Interrupt Controller
+ *
+ * Note this is NOT the PL192 provided by ARM, but
+ * a custom implementation by Freescale.
+ *
+ * Copyright (c) 2008 OKL
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ *
+ * TODO: implement vectors.
+ */
+#ifndef IMX_AVIC_H
+#define IMX_AVIC_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_AVIC "imx.avic"
+#define IMX_AVIC(obj) OBJECT_CHECK(IMXAVICState, (obj), TYPE_IMX_AVIC)
+
+#define IMX_AVIC_NUM_IRQS 64
+
+/* Interrupt Control Bits */
+#define ABFLAG (1<<25)
+#define ABFEN  (1<<24)
+#define NIDIS  (1<<22) /* Normal Interrupt disable */
+#define FIDIS  (1<<21) /* Fast interrupt disable */
+#define NIAD   (1<<20) /* Normal Interrupt Arbiter Rise ARM level */
+#define FIAD   (1<<19) /* Fast Interrupt Arbiter Rise ARM level */
+#define NM     (1<<18) /* Normal interrupt mode */
+
+#define PRIO_PER_WORD (sizeof(uint32_t) * 8 / 4)
+#define PRIO_WORDS (IMX_AVIC_NUM_IRQS/PRIO_PER_WORD)
+
+typedef struct IMXAVICState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    uint64_t pending;
+    uint64_t enabled;
+    uint64_t is_fiq;
+    uint32_t intcntl;
+    uint32_t intmask;
+    qemu_irq irq;
+    qemu_irq fiq;
+    uint32_t prio[PRIO_WORDS]; /* Priorities are 4-bits each */
+} IMXAVICState;
+
+#endif /* IMX_AVIC_H */
diff --git a/include/hw/isa/isa.h b/include/hw/isa/isa.h
index f21ceaafc..d758b39a2 100644
--- a/include/hw/isa/isa.h
+++ b/include/hw/isa/isa.h
@@ -112,8 +112,8 @@ int DMA_read_memory (int nchan, void *buf, int pos, int size);
 int DMA_write_memory (int nchan, void *buf, int pos, int size);
 void DMA_hold_DREQ (int nchan);
 void DMA_release_DREQ (int nchan);
-void DMA_schedule(int nchan);
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit);
+void DMA_schedule(void);
+void DMA_init(int high_page_enable);
 void DMA_register_channel (int nchan,
                            DMA_transfer_handler transfer_handler,
                            void *opaque);
diff --git a/include/hw/misc/imx_ccm.h b/include/hw/misc/imx_ccm.h
new file mode 100644
index 000000000..0f2e469b2
--- /dev/null
+++ b/include/hw/misc/imx_ccm.h
@@ -0,0 +1,91 @@
+/*
+ * IMX31 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX_CCM_H
+#define IMX_CCM_H
+
+#include "hw/sysbus.h"
+
+/* CCMR */
+#define CCMR_FPME (1<<0)
+#define CCMR_MPE  (1<<3)
+#define CCMR_MDS  (1<<7)
+#define CCMR_FPMF (1<<26)
+#define CCMR_PRCS (3<<1)
+
+/* PDR0 */
+#define PDR0_MCU_PODF_SHIFT (0)
+#define PDR0_MCU_PODF_MASK (0x7)
+#define PDR0_MAX_PODF_SHIFT (3)
+#define PDR0_MAX_PODF_MASK (0x7)
+#define PDR0_IPG_PODF_SHIFT (6)
+#define PDR0_IPG_PODF_MASK (0x3)
+#define PDR0_NFC_PODF_SHIFT (8)
+#define PDR0_NFC_PODF_MASK (0x7)
+#define PDR0_HSP_PODF_SHIFT (11)
+#define PDR0_HSP_PODF_MASK (0x7)
+#define PDR0_PER_PODF_SHIFT (16)
+#define PDR0_PER_PODF_MASK (0x1f)
+#define PDR0_CSI_PODF_SHIFT (23)
+#define PDR0_CSI_PODF_MASK (0x1ff)
+
+#define EXTRACT(value, name) (((value) >> PDR0_##name##_PODF_SHIFT) \
+                              & PDR0_##name##_PODF_MASK)
+#define INSERT(value, name) (((value) & PDR0_##name##_PODF_MASK) << \
+                             PDR0_##name##_PODF_SHIFT)
+
+/* PLL control registers */
+#define PD(v) (((v) >> 26) & 0xf)
+#define MFD(v) (((v) >> 16) & 0x3ff)
+#define MFI(v) (((v) >> 10) & 0xf);
+#define MFN(v) ((v) & 0x3ff)
+
+#define PLL_PD(x)               (((x) & 0xf) << 26)
+#define PLL_MFD(x)              (((x) & 0x3ff) << 16)
+#define PLL_MFI(x)              (((x) & 0xf) << 10)
+#define PLL_MFN(x)              (((x) & 0x3ff) << 0)
+
+#define TYPE_IMX_CCM "imx.ccm"
+#define IMX_CCM(obj) OBJECT_CHECK(IMXCCMState, (obj), TYPE_IMX_CCM)
+
+typedef struct IMXCCMState {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+
+    uint32_t ccmr;
+    uint32_t pdr0;
+    uint32_t pdr1;
+    uint32_t mpctl;
+    uint32_t spctl;
+    uint32_t cgr[3];
+    uint32_t pmcr0;
+    uint32_t pmcr1;
+
+    /* Frequencies precalculated on register changes */
+    uint32_t pll_refclk_freq;
+    uint32_t mcu_clk_freq;
+    uint32_t hsp_clk_freq;
+    uint32_t ipg_clk_freq;
+} IMXCCMState;
+
+typedef enum  {
+    NOCLK,
+    MCU,
+    HSP,
+    IPG,
+    CLK_32k
+} IMXClk;
+
+uint32_t imx_clock_frequency(DeviceState *s, IMXClk clock);
+
+#endif /* IMX_CCM_H */
diff --git a/include/hw/misc/ivshmem.h b/include/hw/misc/ivshmem.h
new file mode 100644
index 000000000..433ef53d7
--- /dev/null
+++ b/include/hw/misc/ivshmem.h
@@ -0,0 +1,25 @@
+
+/*
+ * Inter-VM Shared Memory PCI device.
+ *
+ * Author:
+ *      Cam Macdonell <cam@cs.ualberta.ca>
+ *
+ * Based On: cirrus_vga.c
+ *          Copyright (c) 2004 Fabrice Bellard
+ *          Copyright (c) 2004 Makoto Suzuki (suzu)
+ *
+ *      and rtl8139.c
+ *          Copyright (c) 2006 Igor Kovalenko
+ *
+ * This code is licensed under the GNU GPL v2.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#ifndef IVSHMEM_H
+#define IVSHMEM_H
+
+#define IVSHMEM_PROTOCOL_VERSION 0
+
+#endif /* IVSHMEM_H */
diff --git a/include/hw/misc/zynq-xadc.h b/include/hw/misc/zynq-xadc.h
new file mode 100644
index 000000000..f1a410a37
--- /dev/null
+++ b/include/hw/misc/zynq-xadc.h
@@ -0,0 +1,46 @@
+/*
+ * Device model for Zynq ADC controller
+ *
+ * Copyright (c) 2015 Guenter Roeck <linux@roeck-us.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef ZYNQ_XADC_H
+#define ZYNQ_XADC_H
+
+#include "hw/sysbus.h"
+
+#define ZYNQ_XADC_MMIO_SIZE     0x0020
+#define ZYNQ_XADC_NUM_IO_REGS   (ZYNQ_XADC_MMIO_SIZE / 4)
+#define ZYNQ_XADC_NUM_ADC_REGS  128
+#define ZYNQ_XADC_FIFO_DEPTH    15
+
+#define TYPE_ZYNQ_XADC          "xlnx,zynq-xadc"
+#define ZYNQ_XADC(obj) \
+    OBJECT_CHECK(ZynqXADCState, (obj), TYPE_ZYNQ_XADC)
+
+typedef struct ZynqXADCState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t regs[ZYNQ_XADC_NUM_IO_REGS];
+    uint16_t xadc_regs[ZYNQ_XADC_NUM_ADC_REGS];
+    uint16_t xadc_read_reg_previous;
+    uint16_t xadc_dfifo[ZYNQ_XADC_FIFO_DEPTH];
+    uint16_t xadc_dfifo_entries;
+
+    struct IRQState *qemu_irq;
+
+} ZynqXADCState;
+
+#endif /* ZYNQ_XADC_H */
diff --git a/include/hw/net/imx_fec.h b/include/hw/net/imx_fec.h
new file mode 100644
index 000000000..cbf86509e
--- /dev/null
+++ b/include/hw/net/imx_fec.h
@@ -0,0 +1,113 @@
+/*
+ * i.MX Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ * Based on Coldfire Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2007 CodeSourcery.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IMX_FEC_H
+#define IMX_FEC_H
+
+#define TYPE_IMX_FEC "imx.fec"
+#define IMX_FEC(obj) OBJECT_CHECK(IMXFECState, (obj), TYPE_IMX_FEC)
+
+#include "hw/sysbus.h"
+#include "net/net.h"
+
+#define FEC_MAX_FRAME_SIZE 2032
+
+#define FEC_INT_HB      (1 << 31)
+#define FEC_INT_BABR    (1 << 30)
+#define FEC_INT_BABT    (1 << 29)
+#define FEC_INT_GRA     (1 << 28)
+#define FEC_INT_TXF     (1 << 27)
+#define FEC_INT_TXB     (1 << 26)
+#define FEC_INT_RXF     (1 << 25)
+#define FEC_INT_RXB     (1 << 24)
+#define FEC_INT_MII     (1 << 23)
+#define FEC_INT_EBERR   (1 << 22)
+#define FEC_INT_LC      (1 << 21)
+#define FEC_INT_RL      (1 << 20)
+#define FEC_INT_UN      (1 << 19)
+
+#define FEC_EN      2
+#define FEC_RESET   1
+
+/* Buffer Descriptor.  */
+typedef struct {
+    uint16_t length;
+    uint16_t flags;
+    uint32_t data;
+} IMXFECBufDesc;
+
+#define FEC_BD_R    (1 << 15)
+#define FEC_BD_E    (1 << 15)
+#define FEC_BD_O1   (1 << 14)
+#define FEC_BD_W    (1 << 13)
+#define FEC_BD_O2   (1 << 12)
+#define FEC_BD_L    (1 << 11)
+#define FEC_BD_TC   (1 << 10)
+#define FEC_BD_ABC  (1 << 9)
+#define FEC_BD_M    (1 << 8)
+#define FEC_BD_BC   (1 << 7)
+#define FEC_BD_MC   (1 << 6)
+#define FEC_BD_LG   (1 << 5)
+#define FEC_BD_NO   (1 << 4)
+#define FEC_BD_CR   (1 << 2)
+#define FEC_BD_OV   (1 << 1)
+#define FEC_BD_TR   (1 << 0)
+
+typedef struct IMXFECState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    NICState *nic;
+    NICConf conf;
+    qemu_irq irq;
+    MemoryRegion iomem;
+
+    uint32_t irq_state;
+    uint32_t eir;
+    uint32_t eimr;
+    uint32_t rx_enabled;
+    uint32_t rx_descriptor;
+    uint32_t tx_descriptor;
+    uint32_t ecr;
+    uint32_t mmfr;
+    uint32_t mscr;
+    uint32_t mibc;
+    uint32_t rcr;
+    uint32_t tcr;
+    uint32_t tfwr;
+    uint32_t frsr;
+    uint32_t erdsr;
+    uint32_t etdsr;
+    uint32_t emrbr;
+    uint32_t miigsk_cfgr;
+    uint32_t miigsk_enr;
+
+    uint32_t phy_status;
+    uint32_t phy_control;
+    uint32_t phy_advertise;
+    uint32_t phy_int;
+    uint32_t phy_int_mask;
+} IMXFECState;
+
+#endif
diff --git a/include/hw/nvram/fw_cfg.h b/include/hw/nvram/fw_cfg.h
index e60d3ca21..ee0cd8adb 100644
--- a/include/hw/nvram/fw_cfg.h
+++ b/include/hw/nvram/fw_cfg.h
@@ -61,6 +61,15 @@ typedef struct FWCfgFiles {
     FWCfgFile f[];
 } FWCfgFiles;
 
+/* Control as first field allows for different structures selected by this
+ * field, which might be useful in the future
+ */
+typedef struct FWCfgDmaAccess {
+    uint32_t control;
+    uint32_t length;
+    uint64_t address;
+} QEMU_PACKED FWCfgDmaAccess;
+
 typedef void (*FWCfgCallback)(void *opaque, uint8_t *data);
 typedef void (*FWCfgReadCallback)(void *opaque, uint32_t offset);
 
@@ -77,10 +86,13 @@ void fw_cfg_add_file_callback(FWCfgState *s, const char *filename,
                               void *data, size_t len);
 void *fw_cfg_modify_file(FWCfgState *s, const char *filename, void *data,
                          size_t len);
+FWCfgState *fw_cfg_init_io_dma(uint32_t iobase, uint32_t dma_iobase,
+                                AddressSpace *dma_as);
 FWCfgState *fw_cfg_init_io(uint32_t iobase);
 FWCfgState *fw_cfg_init_mem(hwaddr ctl_addr, hwaddr data_addr);
-FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr, hwaddr data_addr,
-                                 uint32_t data_width);
+FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr,
+                                 hwaddr data_addr, uint32_t data_width,
+                                 hwaddr dma_addr, AddressSpace *dma_as);
 
 FWCfgState *fw_cfg_find(void);
 
diff --git a/include/hw/pci-host/spapr.h b/include/hw/pci-host/spapr.h
index 5322b560e..7de5e029b 100644
--- a/include/hw/pci-host/spapr.h
+++ b/include/hw/pci-host/spapr.h
@@ -78,6 +78,7 @@ struct sPAPRPHBState {
     MemoryRegion memwindow, iowindow, msiwindow;
 
     uint32_t dma_liobn;
+    hwaddr dma_win_addr, dma_win_size;
     AddressSpace iommu_as;
     MemoryRegion iommu_root;
 
@@ -115,8 +116,6 @@ struct sPAPRPHBVFIOState {
 
 #define SPAPR_PCI_MSI_WINDOW         0x40000000000ULL
 
-#define SPAPR_PCI_DMA32_SIZE         0x40000000
-
 static inline qemu_irq spapr_phb_lsi_qirq(struct sPAPRPHBState *phb, int pin)
 {
     sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
index 954d82b35..72e5f931c 100644
--- a/include/hw/pci/msix.h
+++ b/include/hw/pci/msix.h
@@ -46,12 +46,16 @@ void msix_unset_vector_notifiers(PCIDevice *dev);
 
 extern const VMStateDescription vmstate_msix;
 
-#define VMSTATE_MSIX(_field, _state) {                               \
-    .name       = (stringify(_field)),                               \
-    .size       = sizeof(PCIDevice),                                 \
-    .vmsd       = &vmstate_msix,                                     \
-    .flags      = VMS_STRUCT,                                        \
-    .offset     = vmstate_offset_value(_state, _field, PCIDevice),   \
+#define VMSTATE_MSIX_TEST(_field, _state, _test) {                   \
+    .name         = (stringify(_field)),                             \
+    .size         = sizeof(PCIDevice),                               \
+    .vmsd         = &vmstate_msix,                                   \
+    .flags        = VMS_STRUCT,                                      \
+    .offset       = vmstate_offset_value(_state, _field, PCIDevice), \
+    .field_exists = (_test)                                          \
 }
 
+#define VMSTATE_MSIX(_f, _s)                                         \
+    VMSTATE_MSIX_TEST(_f, _s, NULL)
+
 #endif
diff --git a/include/hw/pci/pci-assign.h b/include/hw/pci/pci-assign.h
new file mode 100644
index 000000000..55f42c56f
--- /dev/null
+++ b/include/hw/pci/pci-assign.h
@@ -0,0 +1,27 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Just split from hw/i386/kvm/pci-assign.c.
+ */
+#ifndef PCI_ASSIGN_H
+#define PCI_ASSIGN_H
+
+#include "hw/pci/pci.h"
+
+//#define DEVICE_ASSIGNMENT_DEBUG
+
+#ifdef DEVICE_ASSIGNMENT_DEBUG
+#define DEBUG(fmt, ...)                                       \
+    do {                                                      \
+        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
+    } while (0)
+#else
+#define DEBUG(fmt, ...)
+#endif
+
+void *pci_assign_dev_load_option_rom(PCIDevice *dev, struct Object *owner,
+                                     int *size, unsigned int domain,
+                                     unsigned int bus, unsigned int slot,
+                                     unsigned int function);
+#endif /* PCI_ASSIGN_H */
diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h
index 551cb3d60..379b6e1a4 100644
--- a/include/hw/pci/pci.h
+++ b/include/hw/pci/pci.h
@@ -397,6 +397,7 @@ void pci_for_each_bus_depth_first(PCIBus *bus,
                                   void *(*begin)(PCIBus *bus, void *parent_state),
                                   void (*end)(PCIBus *bus, void *state),
                                   void *parent_state);
+PCIDevice *pci_get_function_0(PCIDevice *pci_dev);
 
 /* Use this wrapper when specific scan order is not required. */
 static inline
@@ -677,6 +678,11 @@ static inline uint32_t pci_config_size(const PCIDevice *d)
     return pci_is_express(d) ? PCIE_CONFIG_SPACE_SIZE : PCI_CONFIG_SPACE_SIZE;
 }
 
+static inline uint16_t pci_requester_id(PCIDevice *dev)
+{
+    return (pci_bus_num(dev->bus) << 8) | dev->devfn;
+}
+
 /* DMA access functions */
 static inline AddressSpace *pci_get_address_space(PCIDevice *dev)
 {
diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
index 91a61abbc..5baa90683 100644
--- a/include/hw/ppc/spapr.h
+++ b/include/hw/ppc/spapr.h
@@ -5,6 +5,7 @@
 #include "hw/boards.h"
 #include "hw/ppc/xics.h"
 #include "hw/ppc/spapr_drc.h"
+#include "hw/mem/pc-dimm.h"
 
 struct VIOsPAPRBus;
 struct sPAPRPHBState;
@@ -34,6 +35,7 @@ struct sPAPRMachineClass {
     MachineClass parent_class;
 
     /*< public >*/
+    bool dr_lmb_enabled; /* enable dynamic-reconfig/hotplug of LMBs */
 };
 
 /**
@@ -76,6 +78,7 @@ struct sPAPRMachineState {
 
     /*< public >*/
     char *kvm_type;
+    MemoryHotplugState hotplug_memory;
 };
 
 #define H_SUCCESS         0
@@ -331,6 +334,7 @@ struct sPAPRMachineState {
 #define H_SET_MPP               0x2D0
 #define H_GET_MPP               0x2D4
 #define H_XIRR_X                0x2FC
+#define H_RANDOM                0x300
 #define H_SET_MODE              0x31C
 #define MAX_HCALL_OPCODE        H_SET_MODE
 
@@ -353,15 +357,10 @@ typedef struct sPAPRDeviceTreeUpdateHeader {
     uint32_t version_id;
 } sPAPRDeviceTreeUpdateHeader;
 
-/*#define DEBUG_SPAPR_HCALLS*/
-
-#ifdef DEBUG_SPAPR_HCALLS
-#define hcall_dprintf(fmt, ...) \
-    do { fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); } while (0)
-#else
 #define hcall_dprintf(fmt, ...) \
-    do { } while (0)
-#endif
+    do { \
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: " fmt, __func__, ## __VA_ARGS__); \
+    } while (0)
 
 typedef target_ulong (*spapr_hcall_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm,
                                        target_ulong opcode,
@@ -414,6 +413,7 @@ int spapr_allocate_irq_block(int num, bool lsi, bool msi);
 #define RTAS_OUT_BUSY               -2
 #define RTAS_OUT_PARAM_ERROR        -3
 #define RTAS_OUT_NOT_SUPPORTED      -3
+#define RTAS_OUT_NO_SUCH_INDICATOR  -3
 #define RTAS_OUT_NOT_AUTHORIZED     -9002
 
 /* RTAS tokens */
@@ -494,6 +494,11 @@ static inline uint32_t rtas_ld(target_ulong phys, int n)
     return ldl_be_phys(&address_space_memory, ppc64_phys_to_real(phys + 4*n));
 }
 
+static inline uint64_t rtas_ldq(target_ulong phys, int n)
+{
+    return (uint64_t)rtas_ld(phys, n) << 32 | rtas_ld(phys, n + 1);
+}
+
 static inline void rtas_st(target_ulong phys, int n, uint32_t val)
 {
     stl_be_phys(&address_space_memory, ppc64_phys_to_real(phys + 4*n), val);
@@ -558,7 +563,7 @@ struct sPAPRTCETable {
     uint32_t page_shift;
     uint64_t *table;
     bool bypass;
-    bool vfio_accel;
+    bool need_vfio;
     int fd;
     MemoryRegion iommu;
     struct VIOsPAPRDevice *vdev; /* for @bypass migration compatibility only */
@@ -577,20 +582,27 @@ struct sPAPREventLogEntry {
 void spapr_events_init(sPAPRMachineState *sm);
 void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq);
 int spapr_h_cas_compose_response(sPAPRMachineState *sm,
-                                 target_ulong addr, target_ulong size);
+                                 target_ulong addr, target_ulong size,
+                                 bool cpu_update, bool memory_update);
 sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
                                    uint64_t bus_offset,
                                    uint32_t page_shift,
                                    uint32_t nb_table,
-                                   bool vfio_accel);
+                                   bool need_vfio);
+void spapr_tce_set_need_vfio(sPAPRTCETable *tcet, bool need_vfio);
+
 MemoryRegion *spapr_tce_get_iommu(sPAPRTCETable *tcet);
 int spapr_dma_dt(void *fdt, int node_off, const char *propname,
                  uint32_t liobn, uint64_t window, uint32_t size);
 int spapr_tcet_dma_dt(void *fdt, int node_off, const char *propname,
                       sPAPRTCETable *tcet);
 void spapr_pci_switch_vga(bool big_endian);
-void spapr_hotplug_req_add_event(sPAPRDRConnector *drc);
-void spapr_hotplug_req_remove_event(sPAPRDRConnector *drc);
+void spapr_hotplug_req_add_by_index(sPAPRDRConnector *drc);
+void spapr_hotplug_req_remove_by_index(sPAPRDRConnector *drc);
+void spapr_hotplug_req_add_by_count(sPAPRDRConnectorType drc_type,
+                                       uint32_t count);
+void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type,
+                                          uint32_t count);
 
 /* rtas-configure-connector state */
 struct sPAPRConfigureConnectorState {
@@ -603,10 +615,35 @@ struct sPAPRConfigureConnectorState {
 void spapr_ccs_reset_hook(void *opaque);
 
 #define TYPE_SPAPR_RTC "spapr-rtc"
+#define TYPE_SPAPR_RNG "spapr-rng"
 
 void spapr_rtc_read(DeviceState *dev, struct tm *tm, uint32_t *ns);
 int spapr_rtc_import_offset(DeviceState *dev, int64_t legacy_offset);
 
+int spapr_rng_populate_dt(void *fdt);
+
 #define SPAPR_MEMORY_BLOCK_SIZE (1 << 28) /* 256MB */
 
+/*
+ * This defines the maximum number of DIMM slots we can have for sPAPR
+ * guest. This is not defined by sPAPR but we are defining it to 32 slots
+ * based on default number of slots provided by PowerPC kernel.
+ */
+#define SPAPR_MAX_RAM_SLOTS     32
+
+/* 1GB alignment for hotplug memory region */
+#define SPAPR_HOTPLUG_MEM_ALIGN (1ULL << 30)
+
+/*
+ * Number of 32 bit words in each LMB list entry in ibm,dynamic-memory
+ * property under ibm,dynamic-reconfiguration-memory node.
+ */
+#define SPAPR_DR_LMB_LIST_ENTRY_SIZE 6
+
+/*
+ * This flag value defines the LMB as assigned in ibm,dynamic-memory
+ * property under ibm,dynamic-reconfiguration-memory node.
+ */
+#define SPAPR_LMB_FLAGS_ASSIGNED 0x00000008
+
 #endif /* !defined (__HW_SPAPR_H__) */
diff --git a/include/hw/ppc/spapr_drc.h b/include/hw/ppc/spapr_drc.h
index 60cda35ed..7e5634772 100644
--- a/include/hw/ppc/spapr_drc.h
+++ b/include/hw/ppc/spapr_drc.h
@@ -119,13 +119,14 @@ typedef enum {
 } sPAPRDREntitySense;
 
 typedef enum {
-    SPAPR_DR_CC_RESPONSE_NEXT_SIB       = 1, /* currently unused */
-    SPAPR_DR_CC_RESPONSE_NEXT_CHILD     = 2,
-    SPAPR_DR_CC_RESPONSE_NEXT_PROPERTY  = 3,
-    SPAPR_DR_CC_RESPONSE_PREV_PARENT    = 4,
-    SPAPR_DR_CC_RESPONSE_SUCCESS        = 0,
-    SPAPR_DR_CC_RESPONSE_ERROR          = -1,
-    SPAPR_DR_CC_RESPONSE_CONTINUE       = -2,
+    SPAPR_DR_CC_RESPONSE_NEXT_SIB         = 1, /* currently unused */
+    SPAPR_DR_CC_RESPONSE_NEXT_CHILD       = 2,
+    SPAPR_DR_CC_RESPONSE_NEXT_PROPERTY    = 3,
+    SPAPR_DR_CC_RESPONSE_PREV_PARENT      = 4,
+    SPAPR_DR_CC_RESPONSE_SUCCESS          = 0,
+    SPAPR_DR_CC_RESPONSE_ERROR            = -1,
+    SPAPR_DR_CC_RESPONSE_CONTINUE         = -2,
+    SPAPR_DR_CC_RESPONSE_NOT_CONFIGURABLE = -9003,
 } sPAPRDRCCResponse;
 
 typedef void (spapr_drc_detach_cb)(DeviceState *d, void *opaque);
@@ -164,17 +165,17 @@ typedef struct sPAPRDRConnectorClass {
     /*< public >*/
 
     /* accessors for guest-visible (generally via RTAS) DR state */
-    int (*set_isolation_state)(sPAPRDRConnector *drc,
-                               sPAPRDRIsolationState state);
-    int (*set_indicator_state)(sPAPRDRConnector *drc,
-                               sPAPRDRIndicatorState state);
-    int (*set_allocation_state)(sPAPRDRConnector *drc,
-                                sPAPRDRAllocationState state);
+    uint32_t (*set_isolation_state)(sPAPRDRConnector *drc,
+                                    sPAPRDRIsolationState state);
+    uint32_t (*set_indicator_state)(sPAPRDRConnector *drc,
+                                    sPAPRDRIndicatorState state);
+    uint32_t (*set_allocation_state)(sPAPRDRConnector *drc,
+                                     sPAPRDRAllocationState state);
     uint32_t (*get_index)(sPAPRDRConnector *drc);
     uint32_t (*get_type)(sPAPRDRConnector *drc);
     const char *(*get_name)(sPAPRDRConnector *drc);
 
-    sPAPRDREntitySense (*entity_sense)(sPAPRDRConnector *drc);
+    uint32_t (*entity_sense)(sPAPRDRConnector *drc, sPAPRDREntitySense *state);
 
     /* QEMU interfaces for managing FDT/configure-connector */
     const void *(*get_fdt)(sPAPRDRConnector *drc, int *fdt_start_offset);
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index 8057aedaa..c537969f4 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -237,7 +237,7 @@ struct BusState {
 struct Property {
     const char   *name;
     PropertyInfo *info;
-    int          offset;
+    ptrdiff_t    offset;
     uint8_t      bitnr;
     qtype_code   qtype;
     int64_t      defval;
@@ -337,6 +337,7 @@ int qdev_walk_children(DeviceState *dev,
                        void *opaque);
 
 void qdev_reset_all(DeviceState *dev);
+void qdev_reset_all_fn(void *opaque);
 
 /**
  * @qbus_reset_all:
diff --git a/include/hw/s390x/event-facility.h b/include/hw/s390x/event-facility.h
index 6a062b668..dd8881838 100644
--- a/include/hw/s390x/event-facility.h
+++ b/include/hw/s390x/event-facility.h
@@ -47,6 +47,7 @@
      OBJECT_GET_CLASS(SCLPEventClass, (obj), TYPE_SCLP_EVENT)
 
 #define TYPE_SCLP_CPU_HOTPLUG "sclp-cpu-hotplug"
+#define TYPE_SCLP_QUIESCE "sclpquiesce"
 
 typedef struct WriteEventMask {
     SCCBHeader h;
@@ -146,8 +147,10 @@ typedef struct WriteEventData {
 
 typedef struct ReadEventData {
     SCCBHeader h;
-    EventBufferHeader ebh;
-    uint32_t mask;
+    union {
+        uint32_t mask;
+        EventBufferHeader ebh;
+    };
 } QEMU_PACKED ReadEventData;
 
 typedef struct SCLPEvent {
@@ -186,11 +189,8 @@ typedef struct SCLPEventClass {
      OBJECT_GET_CLASS(SCLPEventFacilityClass, (obj), \
                       TYPE_SCLP_EVENT_FACILITY)
 
-typedef struct SCLPEventFacility SCLPEventFacility;
-
 typedef struct SCLPEventFacilityClass {
-    DeviceClass parent_class;
-    int (*init)(SCLPEventFacility *ef);
+    SysBusDeviceClass parent_class;
     void (*command_handler)(SCLPEventFacility *ef, SCCB *sccb, uint64_t code);
     bool (*event_pending)(SCLPEventFacility *ef);
 } SCLPEventFacilityClass;
diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
index e8a64e25b..b0c71b555 100644
--- a/include/hw/s390x/sclp.h
+++ b/include/hw/s390x/sclp.h
@@ -163,6 +163,39 @@ typedef struct SCCB {
     char data[SCCB_DATA_LEN];
  } QEMU_PACKED SCCB;
 
+#define TYPE_SCLP "sclp"
+#define SCLP(obj) OBJECT_CHECK(SCLPDevice, (obj), TYPE_SCLP)
+#define SCLP_CLASS(oc) OBJECT_CLASS_CHECK(SCLPDeviceClass, (oc), TYPE_SCLP)
+#define SCLP_GET_CLASS(obj) OBJECT_GET_CLASS(SCLPDeviceClass, (obj), TYPE_SCLP)
+
+typedef struct SCLPEventFacility SCLPEventFacility;
+
+typedef struct SCLPDevice {
+    /* private */
+    DeviceState parent_obj;
+    SCLPEventFacility *event_facility;
+    int increment_size;
+
+    /* public */
+} SCLPDevice;
+
+typedef struct SCLPDeviceClass {
+    /* private */
+    DeviceClass parent_class;
+    void (*read_SCP_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_storage_element0_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_storage_element1_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*attach_storage_element)(SCLPDevice *sclp, SCCB *sccb,
+                                   uint16_t element);
+    void (*assign_storage)(SCLPDevice *sclp, SCCB *sccb);
+    void (*unassign_storage)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_cpu_info)(SCLPDevice *sclp, SCCB *sccb);
+
+    /* public */
+    void (*execute)(SCLPDevice *sclp, SCCB *sccb, uint32_t code);
+    void (*service_interrupt)(SCLPDevice *sclp, uint32_t sccb);
+} SCLPDeviceClass;
+
 typedef struct sclpMemoryHotplugDev sclpMemoryHotplugDev;
 
 #define TYPE_SCLP_MEMORY_HOTPLUG_DEV "sclp-memory-hotplug-dev"
diff --git a/include/hw/s390x/storage-keys.h b/include/hw/s390x/storage-keys.h
new file mode 100644
index 000000000..72b850cb1
--- /dev/null
+++ b/include/hw/s390x/storage-keys.h
@@ -0,0 +1,60 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef __S390_STORAGE_KEYS_H
+#define __S390_STORAGE_KEYS_H
+
+#include <hw/qdev.h>
+#include "monitor/monitor.h"
+
+#define TYPE_S390_SKEYS "s390-skeys"
+#define S390_SKEYS(obj) \
+    OBJECT_CHECK(S390SKeysState, (obj), TYPE_S390_SKEYS)
+
+typedef struct S390SKeysState {
+    DeviceState parent_obj;
+    bool migration_enabled;
+
+} S390SKeysState;
+
+#define S390_SKEYS_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390SKeysClass, (klass), TYPE_S390_SKEYS)
+#define S390_SKEYS_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(S390SKeysClass, (obj), TYPE_S390_SKEYS)
+
+typedef struct S390SKeysClass {
+    DeviceClass parent_class;
+    int (*skeys_enabled)(S390SKeysState *ks);
+    int (*get_skeys)(S390SKeysState *ks, uint64_t start_gfn, uint64_t count,
+                     uint8_t *keys);
+    int (*set_skeys)(S390SKeysState *ks, uint64_t start_gfn, uint64_t count,
+                     uint8_t *keys);
+} S390SKeysClass;
+
+#define TYPE_KVM_S390_SKEYS "s390-skeys-kvm"
+#define TYPE_QEMU_S390_SKEYS "s390-skeys-qemu"
+#define QEMU_S390_SKEYS(obj) \
+    OBJECT_CHECK(QEMUS390SKeysState, (obj), TYPE_QEMU_S390_SKEYS)
+
+typedef struct QEMUS390SKeysState {
+    S390SKeysState parent_obj;
+    uint8_t *keydata;
+    uint32_t key_count;
+} QEMUS390SKeysState;
+
+void s390_skeys_init(void);
+
+S390SKeysState *s390_get_skeys_device(void);
+
+void hmp_dump_skeys(Monitor *mon, const QDict *qdict);
+void hmp_info_skeys(Monitor *mon, const QDict *qdict);
+
+#endif /* __S390_STORAGE_KEYS_H */
diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h
index cdaf0f8eb..1915a7342 100644
--- a/include/hw/scsi/scsi.h
+++ b/include/hw/scsi/scsi.h
@@ -250,7 +250,6 @@ SCSIRequest *scsi_req_alloc(const SCSIReqOps *reqops, SCSIDevice *d,
 SCSIRequest *scsi_req_new(SCSIDevice *d, uint32_t tag, uint32_t lun,
                           uint8_t *buf, void *hba_private);
 int32_t scsi_req_enqueue(SCSIRequest *req);
-void scsi_req_free(SCSIRequest *req);
 SCSIRequest *scsi_req_ref(SCSIRequest *req);
 void scsi_req_unref(SCSIRequest *req);
 
diff --git a/include/hw/sd.h b/include/hw/sd/sd.h
index 79adb5bb4..79adb5bb4 100644
--- a/include/hw/sd.h
+++ b/include/hw/sd/sd.h
diff --git a/include/hw/sd/sdhci.h b/include/hw/sd/sdhci.h
new file mode 100644
index 000000000..e78d93857
--- /dev/null
+++ b/include/hw/sd/sdhci.h
@@ -0,0 +1,94 @@
+/*
+ * SD Association Host Standard Specification v2.0 controller emulation
+ *
+ * Copyright (c) 2011 Samsung Electronics Co., Ltd.
+ * Mitsyanko Igor <i.mitsyanko@samsung.com>
+ * Peter A.G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ *
+ * Based on MMC controller for Samsung S5PC1xx-based board emulation
+ * by Alexey Merkulov and Vladimir Monakhov.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU _General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef SDHCI_H
+#define SDHCI_H
+
+#include "qemu-common.h"
+#include "hw/block/block.h"
+#include "hw/pci/pci.h"
+#include "hw/sysbus.h"
+#include "hw/sd/sd.h"
+
+/* SD/MMC host controller state */
+typedef struct SDHCIState {
+    union {
+        PCIDevice pcidev;
+        SysBusDevice busdev;
+    };
+    SDState *card;
+    MemoryRegion iomem;
+    BlockBackend *blk;
+
+    QEMUTimer *insert_timer;       /* timer for 'changing' sd card. */
+    QEMUTimer *transfer_timer;
+    qemu_irq eject_cb;
+    qemu_irq ro_cb;
+    qemu_irq irq;
+
+    uint32_t sdmasysad;    /* SDMA System Address register */
+    uint16_t blksize;      /* Host DMA Buff Boundary and Transfer BlkSize Reg */
+    uint16_t blkcnt;       /* Blocks count for current transfer */
+    uint32_t argument;     /* Command Argument Register */
+    uint16_t trnmod;       /* Transfer Mode Setting Register */
+    uint16_t cmdreg;       /* Command Register */
+    uint32_t rspreg[4];    /* Response Registers 0-3 */
+    uint32_t prnsts;       /* Present State Register */
+    uint8_t  hostctl;      /* Host Control Register */
+    uint8_t  pwrcon;       /* Power control Register */
+    uint8_t  blkgap;       /* Block Gap Control Register */
+    uint8_t  wakcon;       /* WakeUp Control Register */
+    uint16_t clkcon;       /* Clock control Register */
+    uint8_t  timeoutcon;   /* Timeout Control Register */
+    uint8_t  admaerr;      /* ADMA Error Status Register */
+    uint16_t norintsts;    /* Normal Interrupt Status Register */
+    uint16_t errintsts;    /* Error Interrupt Status Register */
+    uint16_t norintstsen;  /* Normal Interrupt Status Enable Register */
+    uint16_t errintstsen;  /* Error Interrupt Status Enable Register */
+    uint16_t norintsigen;  /* Normal Interrupt Signal Enable Register */
+    uint16_t errintsigen;  /* Error Interrupt Signal Enable Register */
+    uint16_t acmd12errsts; /* Auto CMD12 error status register */
+    uint64_t admasysaddr;  /* ADMA System Address Register */
+
+    uint32_t capareg;      /* Capabilities Register */
+    uint32_t maxcurr;      /* Maximum Current Capabilities Register */
+    uint8_t  *fifo_buffer; /* SD host i/o FIFO buffer */
+    uint32_t buf_maxsz;
+    uint16_t data_count;   /* current element in FIFO buffer */
+    uint8_t  stopped_state;/* Current SDHC state */
+    /* Buffer Data Port Register - virtual access point to R and W buffers */
+    /* Software Reset Register - always reads as 0 */
+    /* Force Event Auto CMD12 Error Interrupt Reg - write only */
+    /* Force Event Error Interrupt Register- write only */
+    /* RO Host Controller Version Register always reads as 0x2401 */
+} SDHCIState;
+
+#define TYPE_PCI_SDHCI "sdhci-pci"
+#define PCI_SDHCI(obj) OBJECT_CHECK(SDHCIState, (obj), TYPE_PCI_SDHCI)
+
+#define TYPE_SYSBUS_SDHCI "generic-sdhci"
+#define SYSBUS_SDHCI(obj)                               \
+     OBJECT_CHECK(SDHCIState, (obj), TYPE_SYSBUS_SDHCI)
+
+#endif /* SDHCI_H */
diff --git a/include/hw/i386/smbios.h b/include/hw/smbios/smbios.h
index d2850bed2..76ccf7098 100644
--- a/include/hw/i386/smbios.h
+++ b/include/hw/smbios/smbios.h
@@ -17,23 +17,33 @@
 
 #define SMBIOS_MAX_TYPE 127
 
-void smbios_entry_add(QemuOpts *opts);
-void smbios_set_cpuid(uint32_t version, uint32_t features);
-void smbios_set_defaults(const char *manufacturer, const char *product,
-                         const char *version, bool legacy_mode,
-                         bool uuid_encoded);
-uint8_t *smbios_get_table_legacy(size_t *length);
-void smbios_get_tables(uint8_t **tables, size_t *tables_len,
-                       uint8_t **anchor, size_t *anchor_len);
+/* memory area description, used by type 19 table */
+struct smbios_phys_mem_area {
+    uint64_t address;
+    uint64_t length;
+};
 
 /*
  * SMBIOS spec defined tables
  */
+typedef enum SmbiosEntryPointType {
+    SMBIOS_ENTRY_POINT_21,
+    SMBIOS_ENTRY_POINT_30,
+} SmbiosEntryPointType;
 
-/* SMBIOS entry point (anchor).
- * BIOS must place this at a 16-bit-aligned address between 0xf0000 and 0xfffff.
+/* SMBIOS Entry Point
+ * There are two types of entry points defined in the SMBIOS specification
+ * (see below). BIOS must place the entry point(s) at a 16-bit-aligned
+ * address between 0xf0000 and 0xfffff. Note that either entry point type
+ * can be used in a 64-bit target system, except that SMBIOS 2.1 entry point
+ * only allows the SMBIOS struct table to reside below 4GB address space.
  */
-struct smbios_entry_point {
+
+/* SMBIOS 2.1 (32-bit) Entry Point
+ *  - introduced since SMBIOS 2.1
+ *  - supports structure table below 4GB only
+ */
+struct smbios_21_entry_point {
     uint8_t anchor_string[4];
     uint8_t checksum;
     uint8_t length;
@@ -50,6 +60,28 @@ struct smbios_entry_point {
     uint8_t smbios_bcd_revision;
 } QEMU_PACKED;
 
+/* SMBIOS 3.0 (64-bit) Entry Point
+ *  - introduced since SMBIOS 3.0
+ *  - supports structure table at 64-bit address space
+ */
+struct smbios_30_entry_point {
+    uint8_t anchor_string[5];
+    uint8_t checksum;
+    uint8_t length;
+    uint8_t smbios_major_version;
+    uint8_t smbios_minor_version;
+    uint8_t smbios_doc_rev;
+    uint8_t entry_point_revision;
+    uint8_t reserved;
+    uint32_t structure_table_max_size;
+    uint64_t structure_table_address;
+} QEMU_PACKED;
+
+typedef union {
+    struct smbios_21_entry_point ep21;
+    struct smbios_30_entry_point ep30;
+} QEMU_PACKED SmbiosEntryPoint;
+
 /* This goes at the beginning of every SMBIOS structure. */
 struct smbios_structure_header {
     uint8_t type;
@@ -224,4 +256,14 @@ struct smbios_type_127 {
     struct smbios_structure_header header;
 } QEMU_PACKED;
 
+void smbios_entry_add(QemuOpts *opts);
+void smbios_set_cpuid(uint32_t version, uint32_t features);
+void smbios_set_defaults(const char *manufacturer, const char *product,
+                         const char *version, bool legacy_mode,
+                         bool uuid_encoded, SmbiosEntryPointType ep_type);
+uint8_t *smbios_get_table_legacy(size_t *length);
+void smbios_get_tables(const struct smbios_phys_mem_area *mem_array,
+                       const unsigned int mem_array_size,
+                       uint8_t **tables, size_t *tables_len,
+                       uint8_t **anchor, size_t *anchor_len);
 #endif /*QEMU_SMBIOS_H */
diff --git a/include/hw/timer/hpet.h b/include/hw/timer/hpet.h
index 773953be7..d872909ce 100644
--- a/include/hw/timer/hpet.h
+++ b/include/hw/timer/hpet.h
@@ -16,9 +16,9 @@
 #include "qom/object.h"
 
 #define HPET_BASE               0xfed00000
-#define HPET_CLK_PERIOD         10000000ULL /* 10000000 femtoseconds == 10ns*/
+#define HPET_CLK_PERIOD         10 /* 10 ns*/
 
-#define FS_PER_NS 1000000
+#define FS_PER_NS 1000000       /* 1000000 femtoseconds == 1 ns */
 #define HPET_MIN_TIMERS         3
 #define HPET_MAX_TIMERS         32
 
diff --git a/include/hw/timer/imx_epit.h b/include/hw/timer/imx_epit.h
new file mode 100644
index 000000000..c5328aefe
--- /dev/null
+++ b/include/hw/timer/imx_epit.h
@@ -0,0 +1,79 @@
+/*
+ * i.MX EPIT Timer
+ *
+ * Copyright (c) 2008 OK Labs
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Peter Chubb
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef IMX_EPIT_H
+#define IMX_EPIT_H
+
+#include "hw/sysbus.h"
+#include "hw/ptimer.h"
+
+/*
+ * EPIT: Enhanced periodic interrupt timer
+ */
+
+#define CR_EN       (1 << 0)
+#define CR_ENMOD    (1 << 1)
+#define CR_OCIEN    (1 << 2)
+#define CR_RLD      (1 << 3)
+#define CR_PRESCALE_SHIFT (4)
+#define CR_PRESCALE_MASK  (0xfff)
+#define CR_SWR      (1 << 16)
+#define CR_IOVW     (1 << 17)
+#define CR_DBGEN    (1 << 18)
+#define CR_WAITEN   (1 << 19)
+#define CR_DOZEN    (1 << 20)
+#define CR_STOPEN   (1 << 21)
+#define CR_CLKSRC_SHIFT (24)
+#define CR_CLKSRC_MASK  (0x3 << CR_CLKSRC_SHIFT)
+
+#define EPIT_TIMER_MAX  0XFFFFFFFFUL
+
+#define TYPE_IMX_EPIT "imx.epit"
+#define IMX_EPIT(obj) OBJECT_CHECK(IMXEPITState, (obj), TYPE_IMX_EPIT)
+
+typedef struct IMXEPITState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    ptimer_state *timer_reload;
+    ptimer_state *timer_cmp;
+    MemoryRegion iomem;
+    DeviceState *ccm;
+
+    uint32_t cr;
+    uint32_t sr;
+    uint32_t lr;
+    uint32_t cmp;
+    uint32_t cnt;
+
+    uint32_t freq;
+    qemu_irq irq;
+} IMXEPITState;
+
+#endif /* IMX_EPIT_H */
diff --git a/include/hw/timer/imx_gpt.h b/include/hw/timer/imx_gpt.h
new file mode 100644
index 000000000..3f02d3b33
--- /dev/null
+++ b/include/hw/timer/imx_gpt.h
@@ -0,0 +1,107 @@
+/*
+ * i.MX GPT Timer
+ *
+ * Copyright (c) 2008 OK Labs
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Peter Chubb
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef IMX_GPT_H
+#define IMX_GPT_H
+
+#include "hw/sysbus.h"
+#include "hw/ptimer.h"
+
+/*
+ * GPT : General purpose timer
+ *
+ * This timer counts up continuously while it is enabled, resetting itself
+ * to 0 when it reaches GPT_TIMER_MAX (in freerun mode) or when it
+ * reaches the value of one of the ocrX (in periodic mode).
+ */
+
+#define GPT_TIMER_MAX  0XFFFFFFFFUL
+
+/* Control register.  Not all of these bits have any effect (yet) */
+#define GPT_CR_EN     (1 << 0)  /* GPT Enable */
+#define GPT_CR_ENMOD  (1 << 1)  /* GPT Enable Mode */
+#define GPT_CR_DBGEN  (1 << 2)  /* GPT Debug mode enable */
+#define GPT_CR_WAITEN (1 << 3)  /* GPT Wait Mode Enable  */
+#define GPT_CR_DOZEN  (1 << 4)  /* GPT Doze mode enable */
+#define GPT_CR_STOPEN (1 << 5)  /* GPT Stop Mode Enable */
+#define GPT_CR_CLKSRC_SHIFT (6)
+#define GPT_CR_CLKSRC_MASK  (0x7)
+
+#define GPT_CR_FRR    (1 << 9)  /* Freerun or Restart */
+#define GPT_CR_SWR    (1 << 15) /* Software Reset */
+#define GPT_CR_IM1    (3 << 16) /* Input capture channel 1 mode (2 bits) */
+#define GPT_CR_IM2    (3 << 18) /* Input capture channel 2 mode (2 bits) */
+#define GPT_CR_OM1    (7 << 20) /* Output Compare Channel 1 Mode (3 bits) */
+#define GPT_CR_OM2    (7 << 23) /* Output Compare Channel 2 Mode (3 bits) */
+#define GPT_CR_OM3    (7 << 26) /* Output Compare Channel 3 Mode (3 bits) */
+#define GPT_CR_FO1    (1 << 29) /* Force Output Compare Channel 1 */
+#define GPT_CR_FO2    (1 << 30) /* Force Output Compare Channel 2 */
+#define GPT_CR_FO3    (1 << 31) /* Force Output Compare Channel 3 */
+
+#define GPT_SR_OF1  (1 << 0)
+#define GPT_SR_OF2  (1 << 1)
+#define GPT_SR_OF3  (1 << 2)
+#define GPT_SR_ROV  (1 << 5)
+
+#define GPT_IR_OF1IE  (1 << 0)
+#define GPT_IR_OF2IE  (1 << 1)
+#define GPT_IR_OF3IE  (1 << 2)
+#define GPT_IR_ROVIE  (1 << 5)
+
+#define TYPE_IMX_GPT "imx.gpt"
+#define IMX_GPT(obj) OBJECT_CHECK(IMXGPTState, (obj), TYPE_IMX_GPT)
+
+typedef struct IMXGPTState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    ptimer_state *timer;
+    MemoryRegion iomem;
+    DeviceState *ccm;
+
+    uint32_t cr;
+    uint32_t pr;
+    uint32_t sr;
+    uint32_t ir;
+    uint32_t ocr1;
+    uint32_t ocr2;
+    uint32_t ocr3;
+    uint32_t icr1;
+    uint32_t icr2;
+    uint32_t cnt;
+
+    uint32_t next_timeout;
+    uint32_t next_int;
+
+    uint32_t freq;
+
+    qemu_irq irq;
+} IMXGPTState;
+
+#endif /* IMX_GPT_H */
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 59a321d47..f037f3c42 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -35,11 +35,6 @@
     do { } while (0)
 #endif
 
-/* Extra debugging, trap acceleration paths for more logging */
-#define VFIO_ALLOW_KVM_INTX 1
-#define VFIO_ALLOW_KVM_MSI 1
-#define VFIO_ALLOW_KVM_MSIX 1
-
 enum {
     VFIO_DEVICE_TYPE_PCI = 0,
     VFIO_DEVICE_TYPE_PLATFORM = 1,
@@ -64,22 +59,19 @@ typedef struct VFIOAddressSpace {
 
 struct VFIOGroup;
 
-typedef struct VFIOType1 {
-    MemoryListener listener;
-    int error;
-    bool initialized;
-} VFIOType1;
-
 typedef struct VFIOContainer {
     VFIOAddressSpace *space;
     int fd; /* /dev/vfio/vfio, empowered by the attached groups */
-    struct {
-        /* enable abstraction to support various iommu backends */
-        union {
-            VFIOType1 type1;
-        };
-        void (*release)(struct VFIOContainer *);
-    } iommu_data;
+    MemoryListener listener;
+    int error;
+    bool initialized;
+    /*
+     * This assumes the host IOMMU can support only a single
+     * contiguous IOVA window.  We may need to generalize that in
+     * future
+     */
+    hwaddr min_iova, max_iova;
+    uint64_t iova_pgsizes;
     QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
     QLIST_HEAD(, VFIOGroup) group_list;
     QLIST_ENTRY(VFIOContainer) next;
@@ -102,7 +94,7 @@ typedef struct VFIODevice {
     int type;
     bool reset_works;
     bool needs_reset;
-    bool allow_mmap;
+    bool no_mmap;
     VFIODeviceOps *ops;
     unsigned int num_irqs;
     unsigned int num_regions;
diff --git a/include/hw/vfio/vfio-platform.h b/include/hw/vfio/vfio-platform.h
index c5cf1d79f..b468f80b1 100644
--- a/include/hw/vfio/vfio-platform.h
+++ b/include/hw/vfio/vfio-platform.h
@@ -34,8 +34,8 @@ enum {
 typedef struct VFIOINTp {
     QLIST_ENTRY(VFIOINTp) next; /* entry for IRQ list */
     QSIMPLEQ_ENTRY(VFIOINTp) pqnext; /* entry for pending IRQ queue */
-    EventNotifier interrupt; /* eventfd triggered on interrupt */
-    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+    EventNotifier *interrupt; /* eventfd triggered on interrupt */
+    EventNotifier *unmask; /* eventfd for unmask on QEMU bypass */
     qemu_irq qemuirq;
     struct VFIOPlatformDevice *vdev; /* back pointer to device */
     int state; /* inactive, pending, active */
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index e472f2971..a6e5c97e3 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -11,6 +11,8 @@
 #ifndef VHOST_BACKEND_H_
 #define VHOST_BACKEND_H_
 
+#include <stdbool.h>
+
 typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
     VHOST_BACKEND_TYPE_KERNEL = 1,
@@ -19,17 +21,86 @@ typedef enum VhostBackendType {
 } VhostBackendType;
 
 struct vhost_dev;
+struct vhost_log;
+struct vhost_memory;
+struct vhost_vring_file;
+struct vhost_vring_state;
+struct vhost_vring_addr;
+struct vhost_scsi_target;
 
-typedef int (*vhost_call)(struct vhost_dev *dev, unsigned long int request,
-             void *arg);
 typedef int (*vhost_backend_init)(struct vhost_dev *dev, void *opaque);
 typedef int (*vhost_backend_cleanup)(struct vhost_dev *dev);
+typedef int (*vhost_backend_memslots_limit)(struct vhost_dev *dev);
+
+typedef int (*vhost_net_set_backend_op)(struct vhost_dev *dev,
+                                struct vhost_vring_file *file);
+typedef int (*vhost_scsi_set_endpoint_op)(struct vhost_dev *dev,
+                                  struct vhost_scsi_target *target);
+typedef int (*vhost_scsi_clear_endpoint_op)(struct vhost_dev *dev,
+                                    struct vhost_scsi_target *target);
+typedef int (*vhost_scsi_get_abi_version_op)(struct vhost_dev *dev,
+                                             int *version);
+typedef int (*vhost_set_log_base_op)(struct vhost_dev *dev, uint64_t base,
+                                     struct vhost_log *log);
+typedef int (*vhost_set_mem_table_op)(struct vhost_dev *dev,
+                                      struct vhost_memory *mem);
+typedef int (*vhost_set_vring_addr_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_addr *addr);
+typedef int (*vhost_set_vring_endian_op)(struct vhost_dev *dev,
+                                         struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_num_op)(struct vhost_dev *dev,
+                                      struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_base_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+typedef int (*vhost_get_vring_base_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_kick_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+typedef int (*vhost_set_vring_call_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+typedef int (*vhost_set_features_op)(struct vhost_dev *dev,
+                                     uint64_t features);
+typedef int (*vhost_get_features_op)(struct vhost_dev *dev,
+                                     uint64_t *features);
+typedef int (*vhost_set_owner_op)(struct vhost_dev *dev);
+typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
+typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
+typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
+                                         int enable);
+typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
+typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
+                                       char *mac_addr);
+typedef bool (*vhost_backend_can_merge_op)(struct vhost_dev *dev,
+                                           uint64_t start1, uint64_t size1,
+                                           uint64_t start2, uint64_t size2);
 
 typedef struct VhostOps {
     VhostBackendType backend_type;
-    vhost_call vhost_call;
     vhost_backend_init vhost_backend_init;
     vhost_backend_cleanup vhost_backend_cleanup;
+    vhost_backend_memslots_limit vhost_backend_memslots_limit;
+    vhost_net_set_backend_op vhost_net_set_backend;
+    vhost_scsi_set_endpoint_op vhost_scsi_set_endpoint;
+    vhost_scsi_clear_endpoint_op vhost_scsi_clear_endpoint;
+    vhost_scsi_get_abi_version_op vhost_scsi_get_abi_version;
+    vhost_set_log_base_op vhost_set_log_base;
+    vhost_set_mem_table_op vhost_set_mem_table;
+    vhost_set_vring_addr_op vhost_set_vring_addr;
+    vhost_set_vring_endian_op vhost_set_vring_endian;
+    vhost_set_vring_num_op vhost_set_vring_num;
+    vhost_set_vring_base_op vhost_set_vring_base;
+    vhost_get_vring_base_op vhost_get_vring_base;
+    vhost_set_vring_kick_op vhost_set_vring_kick;
+    vhost_set_vring_call_op vhost_set_vring_call;
+    vhost_set_features_op vhost_set_features;
+    vhost_get_features_op vhost_get_features;
+    vhost_set_owner_op vhost_set_owner;
+    vhost_reset_device_op vhost_reset_device;
+    vhost_get_vq_index_op vhost_get_vq_index;
+    vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_requires_shm_log_op vhost_requires_shm_log;
+    vhost_migration_done_op vhost_migration_done;
+    vhost_backend_can_merge_op vhost_backend_can_merge;
 } VhostOps;
 
 extern const VhostOps user_ops;
diff --git a/include/hw/virtio/vhost-scsi.h b/include/hw/virtio/vhost-scsi.h
index 701bfee61..9fd63df12 100644
--- a/include/hw/virtio/vhost-scsi.h
+++ b/include/hw/virtio/vhost-scsi.h
@@ -19,37 +19,12 @@
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/vhost.h"
 
-/*
- * Used by QEMU userspace to ensure a consistent vhost-scsi ABI.
- *
- * ABI Rev 0: July 2012 version starting point for v3.6-rc merge candidate +
- *            RFC-v2 vhost-scsi userspace.  Add GET_ABI_VERSION ioctl usage
- * ABI Rev 1: January 2013. Ignore vhost_tpgt filed in struct vhost_scsi_target.
- * 	      All the targets under vhost_wwpn can be seen and used by guest.
- */
-
-#define VHOST_SCSI_ABI_VERSION 1
-
-/* TODO #include <linux/vhost.h> properly */
-/* For VHOST_SCSI_SET_ENDPOINT/VHOST_SCSI_CLEAR_ENDPOINT ioctl */
-struct vhost_scsi_target {
-    int abi_version;
-    char vhost_wwpn[224];
-    unsigned short vhost_tpgt;
-    unsigned short reserved;
-};
-
 enum vhost_scsi_vq_list {
     VHOST_SCSI_VQ_CONTROL = 0,
     VHOST_SCSI_VQ_EVENT = 1,
     VHOST_SCSI_VQ_NUM_FIXED = 2,
 };
 
-#define VHOST_VIRTIO 0xAF
-#define VHOST_SCSI_SET_ENDPOINT _IOW(VHOST_VIRTIO, 0x40, struct vhost_scsi_target)
-#define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target)
-#define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, int)
-
 #define TYPE_VHOST_SCSI "vhost-scsi"
 #define VHOST_SCSI(obj) \
         OBJECT_CHECK(VHostSCSI, (obj), TYPE_VHOST_SCSI)
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index dd510509e..b60d7585b 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -31,7 +31,8 @@ typedef unsigned long vhost_log_chunk_t;
 struct vhost_log {
     unsigned long long size;
     int refcnt;
-    vhost_log_chunk_t log[0];
+    int fd;
+    vhost_log_chunk_t *log;
 };
 
 struct vhost_memory;
@@ -44,12 +45,14 @@ struct vhost_dev {
     int nvqs;
     /* the first virtqueue which would be used by this vhost dev */
     int vq_index;
-    unsigned long long features;
-    unsigned long long acked_features;
-    unsigned long long backend_features;
+    uint64_t features;
+    uint64_t acked_features;
+    uint64_t backend_features;
+    uint64_t protocol_features;
+    uint64_t max_queues;
     bool started;
     bool log_enabled;
-    unsigned long long log_size;
+    uint64_t log_size;
     Error *migration_blocker;
     bool memory_changed;
     hwaddr mem_changed_start_addr;
@@ -57,12 +60,12 @@ struct vhost_dev {
     const VhostOps *vhost_ops;
     void *opaque;
     struct vhost_log *log;
+    QLIST_ENTRY(vhost_dev) entry;
 };
 
 int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
                    VhostBackendType backend_type);
 void vhost_dev_cleanup(struct vhost_dev *hdev);
-bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev);
 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev);
@@ -81,4 +84,5 @@ uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
                             uint64_t features);
 void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
                         uint64_t features);
+bool vhost_has_free_slot(void);
 #endif
diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h
index 6bf5905c5..ae11a6393 100644
--- a/include/hw/virtio/virtio-blk.h
+++ b/include/hw/virtio/virtio-blk.h
@@ -37,7 +37,6 @@ struct VirtIOBlkConf
     char *serial;
     uint32_t scsi;
     uint32_t config_wce;
-    uint32_t data_plane;
     uint32_t request_merging;
 };
 
diff --git a/include/hw/virtio/virtio-bus.h b/include/hw/virtio/virtio-bus.h
index 8811415fa..3f2c1363d 100644
--- a/include/hw/virtio/virtio-bus.h
+++ b/include/hw/virtio/virtio-bus.h
@@ -44,9 +44,12 @@ typedef struct VirtioBusClass {
     void (*notify)(DeviceState *d, uint16_t vector);
     void (*save_config)(DeviceState *d, QEMUFile *f);
     void (*save_queue)(DeviceState *d, int n, QEMUFile *f);
+    void (*save_extra_state)(DeviceState *d, QEMUFile *f);
     int (*load_config)(DeviceState *d, QEMUFile *f);
     int (*load_queue)(DeviceState *d, int n, QEMUFile *f);
     int (*load_done)(DeviceState *d, QEMUFile *f);
+    int (*load_extra_state)(DeviceState *d, QEMUFile *f);
+    bool (*has_extra_state)(DeviceState *d);
     bool (*query_guest_notifiers)(DeviceState *d);
     int (*set_guest_notifiers)(DeviceState *d, int nvqs, bool assign);
     int (*set_host_notifier)(DeviceState *d, int n, bool assigned);
@@ -57,6 +60,11 @@ typedef struct VirtioBusClass {
      */
     void (*device_plugged)(DeviceState *d, Error **errp);
     /*
+     * Re-evaluate setup after feature bits have been validated
+     * by the device backend.
+     */
+    void (*post_plugged)(DeviceState *d, Error **errp);
+    /*
      * transport independent exit function.
      * This is called by virtio-bus just before the device is unplugged.
      */
diff --git a/include/hw/virtio/virtio-gpu.h b/include/hw/virtio/virtio-gpu.h
index 889676147..9b279d702 100644
--- a/include/hw/virtio/virtio-gpu.h
+++ b/include/hw/virtio/virtio-gpu.h
@@ -56,8 +56,19 @@ struct virtio_gpu_requested_state {
     int x, y;
 };
 
+enum virtio_gpu_conf_flags {
+    VIRTIO_GPU_FLAG_VIRGL_ENABLED = 1,
+    VIRTIO_GPU_FLAG_STATS_ENABLED,
+};
+
+#define virtio_gpu_virgl_enabled(_cfg) \
+    (_cfg.flags & (1 << VIRTIO_GPU_FLAG_VIRGL_ENABLED))
+#define virtio_gpu_stats_enabled(_cfg) \
+    (_cfg.flags & (1 << VIRTIO_GPU_FLAG_STATS_ENABLED))
+
 struct virtio_gpu_conf {
     uint32_t max_outputs;
+    uint32_t flags;
 };
 
 struct virtio_gpu_ctrl_command {
@@ -92,11 +103,13 @@ typedef struct VirtIOGPU {
     int enabled_output_bitmask;
     struct virtio_gpu_config virtio_config;
 
+    bool use_virgl_renderer;
+    bool renderer_inited;
     QEMUTimer *fence_poll;
     QEMUTimer *print_stats;
 
+    uint32_t inflight;
     struct {
-        uint32_t inflight;
         uint32_t max_inflight;
         uint32_t requests;
         uint32_t req_3d;
@@ -139,4 +152,11 @@ int virtio_gpu_create_mapping_iov(struct virtio_gpu_resource_attach_backing *ab,
                                   struct iovec **iov);
 void virtio_gpu_cleanup_mapping_iov(struct iovec *iov, uint32_t count);
 
+/* virtio-gpu-3d.c */
+void virtio_gpu_virgl_process_cmd(VirtIOGPU *g,
+                                  struct virtio_gpu_ctrl_command *cmd);
+void virtio_gpu_virgl_fence_poll(VirtIOGPU *g);
+void virtio_gpu_virgl_reset(VirtIOGPU *g);
+int virtio_gpu_virgl_init(VirtIOGPU *g);
+
 #endif
diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h
index 60b11d5c2..f3cc25fec 100644
--- a/include/hw/virtio/virtio-net.h
+++ b/include/hw/virtio/virtio-net.h
@@ -48,7 +48,6 @@ typedef struct VirtIONetQueue {
     int tx_waiting;
     struct {
         VirtQueueElement elem;
-        ssize_t len;
     } async_tx;
     struct VirtIONet *n;
 } VirtIONetQueue;
diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
index 9d09115fa..205fadf23 100644
--- a/include/hw/virtio/virtio.h
+++ b/include/hw/virtio/virtio.h
@@ -151,8 +151,7 @@ void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len, unsigned int idx);
 
-void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
-    size_t num_sg, int is_write);
+void virtqueue_map(VirtQueueElement *elem);
 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem);
 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
                           unsigned int out_bytes);
diff --git a/include/hw/xen/xen.h b/include/hw/xen/xen.h
index 4356af456..e90931a50 100644
--- a/include/hw/xen/xen.h
+++ b/include/hw/xen/xen.h
@@ -10,6 +10,7 @@
 
 #include "hw/irq.h"
 #include "qemu-common.h"
+#include "qemu/typedefs.h"
 
 /* xen-machine.c */
 enum xen_mode {
@@ -38,8 +39,7 @@ qemu_irq *xen_interrupt_controller_init(void);
 void xenstore_store_pv_console_info(int i, struct CharDriverState *chr);
 
 #if defined(NEED_CPU_H) && !defined(CONFIG_USER_ONLY)
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory);
+int xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory);
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
                    struct MemoryRegion *mr);
 void xen_modified_memory(ram_addr_t start, ram_addr_t length);
diff --git a/include/hw/xen/xen_common.h b/include/hw/xen/xen_common.h
index ed5fd3e1a..4ac0c6f44 100644
--- a/include/hw/xen/xen_common.h
+++ b/include/hw/xen/xen_common.h
@@ -186,6 +186,15 @@ static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
 }
 #endif
 
+/* Xen before 4.6 */
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 460
+
+#ifndef HVM_IOREQSRV_BUFIOREQ_ATOMIC
+#define HVM_IOREQSRV_BUFIOREQ_ATOMIC 2
+#endif
+
+#endif
+
 /* Xen before 4.5 */
 #if CONFIG_XEN_CTRL_INTERFACE_VERSION < 450
 
@@ -370,7 +379,8 @@ static inline void xen_unmap_pcidev(XenXC xc, domid_t dom,
 static inline int xen_create_ioreq_server(XenXC xc, domid_t dom,
                                           ioservid_t *ioservid)
 {
-    int rc = xc_hvm_create_ioreq_server(xc, dom, 1, ioservid);
+    int rc = xc_hvm_create_ioreq_server(xc, dom, HVM_IOREQSRV_BUFIOREQ_ATOMIC,
+                                        ioservid);
 
     if (rc == 0) {
         trace_xen_ioreq_server_create(*ioservid);
@@ -407,4 +417,42 @@ static inline int xen_set_ioreq_server_state(XenXC xc, domid_t dom,
 
 #endif
 
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 460
+static inline int xen_xc_domain_add_to_physmap(XenXC xch, uint32_t domid,
+                                               unsigned int space,
+                                               unsigned long idx,
+                                               xen_pfn_t gpfn)
+{
+    return xc_domain_add_to_physmap(xch, domid, space, idx, gpfn);
+}
+#else
+static inline int xen_xc_domain_add_to_physmap(XenXC xch, uint32_t domid,
+                                               unsigned int space,
+                                               unsigned long idx,
+                                               xen_pfn_t gpfn)
+{
+    /* In Xen 4.6 rc is -1 and errno contains the error value. */
+    int rc = xc_domain_add_to_physmap(xch, domid, space, idx, gpfn);
+    if (rc == -1)
+        return errno;
+    return rc;
+}
+#endif
+
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 470
+static inline int xen_domain_create(XenXC xc, uint32_t ssidref,
+                                    xen_domain_handle_t handle, uint32_t flags,
+                                    uint32_t *pdomid)
+{
+    return xc_domain_create(xc, ssidref, handle, flags, pdomid);
+}
+#else
+static inline int xen_domain_create(XenXC xc, uint32_t ssidref,
+                                    xen_domain_handle_t handle, uint32_t flags,
+                                    uint32_t *pdomid)
+{
+    return xc_domain_create(xc, ssidref, handle, flags, pdomid, NULL);
+}
+#endif
+
 #endif /* QEMU_HW_XEN_COMMON_H */
diff --git a/include/migration/migration.h b/include/migration/migration.h
index 83346210b..fd018b74a 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -35,6 +35,7 @@
 #define QEMU_VM_SUBSECTION           0x05
 #define QEMU_VM_VMDESCRIPTION        0x06
 #define QEMU_VM_CONFIGURATION        0x07
+#define QEMU_VM_COMMAND              0x08
 #define QEMU_VM_SECTION_FOOTER       0x7e
 
 struct MigrationParams {
@@ -42,13 +43,67 @@ struct MigrationParams {
     bool shared;
 };
 
-typedef struct MigrationState MigrationState;
+/* Messages sent on the return path from destination to source */
+enum mig_rp_message_type {
+    MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
+    MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
+    MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
+
+    MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
+    MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
+
+    MIG_RP_MSG_MAX
+};
 
 typedef QLIST_HEAD(, LoadStateEntry) LoadStateEntry_Head;
 
+/* The current postcopy state is read/set by postcopy_state_get/set
+ * which update it atomically.
+ * The state is updated as postcopy messages are received, and
+ * in general only one thread should be writing to the state at any one
+ * time, initially the main thread and then the listen thread;
+ * Corner cases are where either thread finishes early and/or errors.
+ * The state is checked as messages are received to ensure that
+ * the source is sending us messages in the correct order.
+ * The state is also used by the RAM reception code to know if it
+ * has to place pages atomically, and the cleanup code at the end of
+ * the main thread to know if it has to delay cleanup until the end
+ * of postcopy.
+ */
+typedef enum {
+    POSTCOPY_INCOMING_NONE = 0,  /* Initial state - no postcopy */
+    POSTCOPY_INCOMING_ADVISE,
+    POSTCOPY_INCOMING_DISCARD,
+    POSTCOPY_INCOMING_LISTENING,
+    POSTCOPY_INCOMING_RUNNING,
+    POSTCOPY_INCOMING_END
+} PostcopyState;
+
 /* State for the incoming migration */
 struct MigrationIncomingState {
-    QEMUFile *file;
+    QEMUFile *from_src_file;
+
+    /*
+     * Free at the start of the main state load, set as the main thread finishes
+     * loading state.
+     */
+    QemuEvent main_thread_load_event;
+
+    bool           have_fault_thread;
+    QemuThread     fault_thread;
+    QemuSemaphore  fault_thread_sem;
+
+    bool           have_listen_thread;
+    QemuThread     listen_thread;
+    QemuSemaphore  listen_thread_sem;
+
+    /* For the kernel to send us notifications */
+    int       userfault_fd;
+    /* To tell the fault_thread to quit */
+    int       userfault_quit_fd;
+    QEMUFile *to_src_file;
+    QemuMutex rp_mutex;    /* We send replies from multiple threads */
+    void     *postcopy_tmp_page;
 
     /* See savevm.c */
     LoadStateEntry_Head loadvm_handlers;
@@ -58,6 +113,18 @@ MigrationIncomingState *migration_incoming_get_current(void);
 MigrationIncomingState *migration_incoming_state_new(QEMUFile *f);
 void migration_incoming_state_destroy(void);
 
+/*
+ * An outstanding page request, on the source, having been received
+ * and queued
+ */
+struct MigrationSrcPageRequest {
+    RAMBlock *rb;
+    hwaddr    offset;
+    hwaddr    len;
+
+    QSIMPLEQ_ENTRY(MigrationSrcPageRequest) next_req;
+};
+
 struct MigrationState
 {
     int64_t bandwidth_limit;
@@ -70,6 +137,14 @@ struct MigrationState
 
     int state;
     MigrationParams params;
+
+    /* State related to return path */
+    struct {
+        QEMUFile     *from_dst_file;
+        QemuThread    rp_thread;
+        bool          error;
+    } rp_state;
+
     double mbps;
     int64_t total_time;
     int64_t downtime;
@@ -80,6 +155,18 @@ struct MigrationState
     int64_t xbzrle_cache_size;
     int64_t setup_time;
     int64_t dirty_sync_count;
+
+    /* Flag set once the migration has been asked to enter postcopy */
+    bool start_postcopy;
+
+    /* Flag set once the migration thread is running (and needs joining) */
+    bool migration_thread_running;
+
+    /* Queue of outstanding page requests from the destination */
+    QemuMutex src_page_req_mutex;
+    QSIMPLEQ_HEAD(src_page_requests, MigrationSrcPageRequest) src_page_requests;
+    /* The RAMBlock used in the last src_page_request */
+    RAMBlock *last_req_rb;
 };
 
 void process_incoming_migration(QEMUFile *f);
@@ -116,9 +203,12 @@ int migrate_fd_close(MigrationState *s);
 
 void add_migration_state_change_notifier(Notifier *notify);
 void remove_migration_state_change_notifier(Notifier *notify);
+MigrationState *migrate_init(const MigrationParams *params);
 bool migration_in_setup(MigrationState *);
 bool migration_has_finished(MigrationState *);
 bool migration_has_failed(MigrationState *);
+/* True if outgoing migration has entered postcopy phase */
+bool migration_in_postcopy(MigrationState *);
 MigrationState *migrate_get_current(void);
 
 void migrate_compress_threads_create(void);
@@ -145,6 +235,13 @@ uint64_t xbzrle_mig_pages_cache_miss(void);
 double xbzrle_mig_cache_miss_rate(void);
 
 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size);
+void ram_debug_dump_bitmap(unsigned long *todump, bool expected);
+/* For outgoing discard bitmap */
+int ram_postcopy_send_discard_bitmap(MigrationState *ms);
+/* For incoming postcopy discard */
+int ram_discard_range(MigrationIncomingState *mis, const char *block_name,
+                      uint64_t start, size_t length);
+int ram_postcopy_incoming_init(MigrationIncomingState *mis);
 
 /**
  * @migrate_add_blocker - prevent migration from proceeding
@@ -160,6 +257,7 @@ void migrate_add_blocker(Error *reason);
  */
 void migrate_del_blocker(Error *reason);
 
+bool migrate_postcopy_ram(void);
 bool migrate_zero_blocks(void);
 
 bool migrate_auto_converge(void);
@@ -179,6 +277,17 @@ int migrate_compress_threads(void);
 int migrate_decompress_threads(void);
 bool migrate_use_events(void);
 
+/* Sending on the return path - generic and then for each message type */
+void migrate_send_rp_message(MigrationIncomingState *mis,
+                             enum mig_rp_message_type message_type,
+                             uint16_t len, void *data);
+void migrate_send_rp_shut(MigrationIncomingState *mis,
+                          uint32_t value);
+void migrate_send_rp_pong(MigrationIncomingState *mis,
+                          uint32_t value);
+void migrate_send_rp_req_pages(MigrationIncomingState *mis, const char* rbname,
+                              ram_addr_t start, size_t len);
+
 void ram_control_before_iterate(QEMUFile *f, uint64_t flags);
 void ram_control_after_iterate(QEMUFile *f, uint64_t flags);
 void ram_control_load_hook(QEMUFile *f, uint64_t flags, void *data);
@@ -204,4 +313,12 @@ void global_state_set_optional(void);
 void savevm_skip_configuration(void);
 int global_state_store(void);
 void global_state_store_running(void);
+
+void flush_page_queue(MigrationState *ms);
+int ram_save_queue_pages(MigrationState *ms, const char *rbname,
+                         ram_addr_t start, ram_addr_t len);
+
+PostcopyState postcopy_state_get(void);
+/* Set the state and return the old state */
+PostcopyState postcopy_state_set(PostcopyState new_state);
 #endif
diff --git a/include/migration/postcopy-ram.h b/include/migration/postcopy-ram.h
new file mode 100644
index 000000000..b6a7491f2
--- /dev/null
+++ b/include/migration/postcopy-ram.h
@@ -0,0 +1,99 @@
+/*
+ * Postcopy migration for RAM
+ *
+ * Copyright 2013 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Dave Gilbert  <dgilbert@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#ifndef QEMU_POSTCOPY_RAM_H
+#define QEMU_POSTCOPY_RAM_H
+
+/* Return true if the host supports everything we need to do postcopy-ram */
+bool postcopy_ram_supported_by_host(void);
+
+/*
+ * Make all of RAM sensitive to accesses to areas that haven't yet been written
+ * and wire up anything necessary to deal with it.
+ */
+int postcopy_ram_enable_notify(MigrationIncomingState *mis);
+
+/*
+ * Initialise postcopy-ram, setting the RAM to a state where we can go into
+ * postcopy later; must be called prior to any precopy.
+ * called from ram.c's similarly named ram_postcopy_incoming_init
+ */
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages);
+
+/*
+ * At the end of a migration where postcopy_ram_incoming_init was called.
+ */
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis);
+
+/*
+ * Discard the contents of 'length' bytes from 'start'
+ * We can assume that if we've been called postcopy_ram_hosttest returned true
+ */
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length);
+
+/*
+ * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard
+ * however leaving it until after precopy means that most of the precopy
+ * data is still THPd
+ */
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis);
+
+/*
+ * Called at the start of each RAMBlock by the bitmap code.
+ * 'offset' is the bitmap offset of the named RAMBlock in the migration
+ * bitmap.
+ * Returns a new PDS
+ */
+PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms,
+                                                 unsigned long offset,
+                                                 const char *name);
+
+/*
+ * Called by the bitmap code for each chunk to discard.
+ * May send a discard message, may just leave it queued to
+ * be sent later.
+ * @start,@length: a range of pages in the migration bitmap in the
+ *  RAM block passed to postcopy_discard_send_init() (length=1 is one page)
+ */
+void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds,
+                                 unsigned long start, unsigned long length);
+
+/*
+ * Called at the end of each RAMBlock by the bitmap code.
+ * Sends any outstanding discard messages, frees the PDS.
+ */
+void postcopy_discard_send_finish(MigrationState *ms,
+                                  PostcopyDiscardState *pds);
+
+/*
+ * Place a page (from) at (host) efficiently
+ *    There are restrictions on how 'from' must be mapped, in general best
+ *    to use other postcopy_ routines to allocate.
+ * returns 0 on success
+ */
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from);
+
+/*
+ * Place a zero page at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host);
+
+/*
+ * Allocate a page of memory that can be mapped at a later point in time
+ * using postcopy_place_page
+ * Returns: Pointer to allocated page
+ */
+void *postcopy_get_tmp_page(MigrationIncomingState *mis);
+
+#endif
diff --git a/include/migration/qemu-file.h b/include/migration/qemu-file.h
index ea49f33fa..b5d08d217 100644
--- a/include/migration/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -31,15 +31,15 @@
  * The pos argument can be ignored if the file is only being used for
  * streaming.  The handler should try to write all of the data it can.
  */
-typedef int (QEMUFilePutBufferFunc)(void *opaque, const uint8_t *buf,
-                                    int64_t pos, int size);
+typedef ssize_t (QEMUFilePutBufferFunc)(void *opaque, const uint8_t *buf,
+                                        int64_t pos, size_t size);
 
 /* Read a chunk of data from a file at the given position.  The pos argument
  * can be ignored if the file is only be used for streaming.  The number of
  * bytes actually read should be returned.
  */
-typedef int (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
-                                    int64_t pos, int size);
+typedef ssize_t (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
+                                        int64_t pos, size_t size);
 
 /* Close a file
  *
@@ -89,6 +89,11 @@ typedef size_t (QEMURamSaveFunc)(QEMUFile *f, void *opaque,
                                uint64_t *bytes_sent);
 
 /*
+ * Return a QEMUFile for comms in the opposite direction
+ */
+typedef QEMUFile *(QEMURetPathFunc)(void *opaque);
+
+/*
  * Stop any read or write (depending on flags) on the underlying
  * transport on the QEMUFile.
  * Existing blocking reads/writes must be woken
@@ -106,6 +111,7 @@ typedef struct QEMUFileOps {
     QEMURamHookFunc *after_ram_iterate;
     QEMURamHookFunc *hook_ram_load;
     QEMURamSaveFunc *save_page;
+    QEMURetPathFunc *get_return_path;
     QEMUFileShutdownFunc *shut_down;
 } QEMUFileOps;
 
@@ -126,13 +132,13 @@ int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 int64_t qemu_ftell(QEMUFile *f);
 int64_t qemu_ftell_fast(QEMUFile *f);
-void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
+void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size);
 void qemu_put_byte(QEMUFile *f, int v);
 /*
  * put_buffer without copying the buffer.
  * The buffer should be available till it is sent asynchronously.
  */
-void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size);
+void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size);
 bool qemu_file_mode_is_not_valid(const char *mode);
 bool qemu_file_is_writable(QEMUFile *f);
 
@@ -161,11 +167,13 @@ static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
 void qemu_put_be16(QEMUFile *f, unsigned int v);
 void qemu_put_be32(QEMUFile *f, unsigned int v);
 void qemu_put_be64(QEMUFile *f, uint64_t v);
-int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset);
-int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size);
+size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset);
+size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size);
+size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size);
 ssize_t qemu_put_compression_data(QEMUFile *f, const uint8_t *p, size_t size,
                                   int level);
 int qemu_put_qemu_file(QEMUFile *f_des, QEMUFile *f_src);
+
 /*
  * Note that you can only peek continuous bytes from where the current pointer
  * is; you aren't guaranteed to be able to peak to +n bytes unless you've
@@ -194,7 +202,9 @@ int64_t qemu_file_get_rate_limit(QEMUFile *f);
 int qemu_file_get_error(QEMUFile *f);
 void qemu_file_set_error(QEMUFile *f, int ret);
 int qemu_file_shutdown(QEMUFile *f);
+QEMUFile *qemu_file_get_return_path(QEMUFile *f);
 void qemu_fflush(QEMUFile *f);
+void qemu_file_set_blocking(QEMUFile *f, bool block);
 
 static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
 {
@@ -237,7 +247,7 @@ static inline void qemu_get_8s(QEMUFile *f, uint8_t *pv)
 }
 
 // Signed versions for type safety
-static inline void qemu_put_sbuffer(QEMUFile *f, const int8_t *buf, int size)
+static inline void qemu_put_sbuffer(QEMUFile *f, const int8_t *buf, size_t size)
 {
     qemu_put_buffer(f, (const uint8_t *)buf, size);
 }
diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 2e5a97dec..7267e38c1 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -39,8 +39,9 @@ typedef struct SaveVMHandlers {
     void (*set_params)(const MigrationParams *params, void * opaque);
     SaveStateHandler *save_state;
 
-    void (*cancel)(void *opaque);
-    int (*save_live_complete)(QEMUFile *f, void *opaque);
+    void (*cleanup)(void *opaque);
+    int (*save_live_complete_postcopy)(QEMUFile *f, void *opaque);
+    int (*save_live_complete_precopy)(QEMUFile *f, void *opaque);
 
     /* This runs both outside and inside the iothread lock.  */
     bool (*is_active)(void *opaque);
@@ -54,8 +55,9 @@ typedef struct SaveVMHandlers {
 
     /* This runs outside the iothread lock!  */
     int (*save_live_setup)(QEMUFile *f, void *opaque);
-    uint64_t (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size);
-
+    void (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size,
+                              uint64_t *non_postcopiable_pending,
+                              uint64_t *postcopiable_pending);
     LoadStateHandler *load_state;
 } SaveVMHandlers;
 
@@ -754,9 +756,6 @@ extern const VMStateInfo vmstate_info_bitmap;
 #define VMSTATE_UINT32_SUB_ARRAY(_f, _s, _start, _num)                \
     VMSTATE_SUB_ARRAY(_f, _s, _start, _num, 0, vmstate_info_uint32, uint32_t)
 
-#define VMSTATE_UINT32_ARRAY(_f, _s, _n)                              \
-    VMSTATE_UINT32_ARRAY_V(_f, _s, _n, 0)
-
 #define VMSTATE_INT64_ARRAY_V(_f, _s, _n, _v)                         \
     VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_int64, int64_t)
 
diff --git a/ui/vnc-tls.h b/include/monitor/hmp-target.h
index f9829c782..bc2c9c04d 100644
--- a/ui/vnc-tls.h
+++ b/include/monitor/hmp-target.h
@@ -1,9 +1,7 @@
 /*
- * QEMU VNC display driver. TLS helpers
+ * QEMU monitor
  *
- * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2006 Fabrice Bellard
- * Copyright (C) 2009 Red Hat, Inc
+ * Copyright (c) 2003-2004 Fabrice Bellard
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,47 +21,29 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef MONITOR_COMMON_H
+#define MONITOR_COMMON_H
 
+#define MD_TLONG 0
+#define MD_I32   1
 
-#ifndef __QEMU_VNC_TLS_H__
-#define __QEMU_VNC_TLS_H__
-
-#include <gnutls/gnutls.h>
-#include <gnutls/x509.h>
-
-#include "qemu/acl.h"
-
-typedef struct VncDisplayTLS VncDisplayTLS;
-typedef struct VncStateTLS VncStateTLS;
-
-/* Server state */
-struct VncDisplayTLS {
-    int x509verify; /* Non-zero if server requests & validates client cert */
-    qemu_acl *acl;
-
-    /* Paths to x509 certs/keys */
-    char *x509cacert;
-    char *x509cacrl;
-    char *x509cert;
-    char *x509key;
+struct MonitorDef {
+    const char *name;
+    int offset;
+    target_long (*get_value)(const struct MonitorDef *md, int val);
+    int type;
 };
 
-/* Per client state */
-struct VncStateTLS {
-    gnutls_session_t session;
-
-    /* Client's Distinguished Name from the x509 cert */
-    char *dname;
-};
-
-int vnc_tls_client_setup(VncState *vs, int x509Creds);
-void vnc_tls_client_cleanup(VncState *vs);
-
-int vnc_tls_validate_certificate(VncState *vs);
-
-int vnc_tls_set_x509_creds_dir(VncDisplay *vd,
-			       const char *path);
+const MonitorDef *target_monitor_defs(void);
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval);
 
+CPUArchState *mon_get_cpu_env(void);
+CPUState *mon_get_cpu(void);
 
-#endif /* __QEMU_VNC_TLS_H__ */
+void hmp_info_mem(Monitor *mon, const QDict *qdict);
+void hmp_info_tlb(Monitor *mon, const QDict *qdict);
+void hmp_mce(Monitor *mon, const QDict *qdict);
+void hmp_info_local_apic(Monitor *mon, const QDict *qdict);
+void hmp_info_io_apic(Monitor *mon, const QDict *qdict);
 
+#endif /* MONITOR_COMMON */
diff --git a/include/monitor/monitor.h b/include/monitor/monitor.h
index 9aff47e69..91b95ae90 100644
--- a/include/monitor/monitor.h
+++ b/include/monitor/monitor.h
@@ -34,6 +34,7 @@ int monitor_fd_param(Monitor *mon, const char *fdname, Error **errp);
 void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
     GCC_FMT_ATTR(2, 0);
 void monitor_printf(Monitor *mon, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
+int monitor_fprintf(FILE *stream, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
 void monitor_flush(Monitor *mon);
 int monitor_set_cpu(int cpu_index);
 int monitor_get_cpu_index(void);
@@ -42,9 +43,6 @@ void monitor_read_command(Monitor *mon, int show_prompt);
 int monitor_read_password(Monitor *mon, ReadLineFunc *readline_func,
                           void *opaque);
 
-void qmp_qom_set(QDict *qdict, QObject **ret, Error **errp);
-void qmp_qom_get(QDict *qdict, QObject **ret, Error **errp);
-void qmp_object_add(QDict *qdict, QObject **ret, Error **errp);
 void object_add(const char *type, const char *id, const QDict *qdict,
                 Visitor *v, Error **errp);
 
diff --git a/include/net/filter.h b/include/net/filter.h
new file mode 100644
index 000000000..2deda362a
--- /dev/null
+++ b/include/net/filter.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_NET_FILTER_H
+#define QEMU_NET_FILTER_H
+
+#include "qom/object.h"
+#include "qemu-common.h"
+#include "qemu/typedefs.h"
+#include "net/queue.h"
+
+#define TYPE_NETFILTER "netfilter"
+#define NETFILTER(obj) \
+    OBJECT_CHECK(NetFilterState, (obj), TYPE_NETFILTER)
+#define NETFILTER_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(NetFilterClass, (obj), TYPE_NETFILTER)
+#define NETFILTER_CLASS(klass) \
+    OBJECT_CLASS_CHECK(NetFilterClass, (klass), TYPE_NETFILTER)
+
+typedef void (FilterSetup) (NetFilterState *nf, Error **errp);
+typedef void (FilterCleanup) (NetFilterState *nf);
+/*
+ * Return:
+ *   0: finished handling the packet, we should continue
+ *   size: filter stolen this packet, we stop pass this packet further
+ */
+typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
+                                   NetClientState *sender,
+                                   unsigned flags,
+                                   const struct iovec *iov,
+                                   int iovcnt,
+                                   NetPacketSent *sent_cb);
+
+typedef struct NetFilterClass {
+    ObjectClass parent_class;
+
+    /* optional */
+    FilterSetup *setup;
+    FilterCleanup *cleanup;
+    /* mandatory */
+    FilterReceiveIOV *receive_iov;
+} NetFilterClass;
+
+
+struct NetFilterState {
+    /* private */
+    Object parent;
+
+    /* protected */
+    char *netdev_id;
+    NetClientState *netdev;
+    NetFilterDirection direction;
+    char info_str[256];
+    QTAILQ_ENTRY(NetFilterState) next;
+};
+
+ssize_t qemu_netfilter_receive(NetFilterState *nf,
+                               NetFilterDirection direction,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb);
+
+/* pass the packet to the next filter */
+ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
+                                    unsigned flags,
+                                    const struct iovec *iov,
+                                    int iovcnt,
+                                    void *opaque);
+
+#endif /* QEMU_NET_FILTER_H */
diff --git a/include/net/net.h b/include/net/net.h
index 6a6cbef24..7af3e15f8 100644
--- a/include/net/net.h
+++ b/include/net/net.h
@@ -92,6 +92,7 @@ struct NetClientState {
     NetClientDestructor *destructor;
     unsigned int queue_index;
     unsigned rxfilter_notify_enabled:1;
+    QTAILQ_HEAD(, NetFilterState) filters;
 };
 
 typedef struct NICState {
@@ -151,11 +152,6 @@ void qemu_check_nic_model(NICInfo *nd, const char *model);
 int qemu_find_nic_model(NICInfo *nd, const char * const *models,
                         const char *default_model);
 
-ssize_t qemu_deliver_packet(NetClientState *sender,
-                            unsigned flags,
-                            const uint8_t *data,
-                            size_t size,
-                            void *opaque);
 ssize_t qemu_deliver_packet_iov(NetClientState *sender,
                             unsigned flags,
                             const struct iovec *iov,
diff --git a/include/net/queue.h b/include/net/queue.h
index fc02b3391..5469fdbea 100644
--- a/include/net/queue.h
+++ b/include/net/queue.h
@@ -34,7 +34,25 @@ typedef void (NetPacketSent) (NetClientState *sender, ssize_t ret);
 #define QEMU_NET_PACKET_FLAG_NONE  0
 #define QEMU_NET_PACKET_FLAG_RAW  (1<<0)
 
-NetQueue *qemu_new_net_queue(void *opaque);
+/* Returns:
+ *   >0 - success
+ *    0 - queue packet for future redelivery
+ *   <0 - failure (discard packet)
+ */
+typedef ssize_t (NetQueueDeliverFunc)(NetClientState *sender,
+                                      unsigned flags,
+                                      const struct iovec *iov,
+                                      int iovcnt,
+                                      void *opaque);
+
+NetQueue *qemu_new_net_queue(NetQueueDeliverFunc *deliver, void *opaque);
+
+void qemu_net_queue_append_iov(NetQueue *queue,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb);
 
 void qemu_del_net_queue(NetQueue *queue);
 
diff --git a/include/net/vhost_net.h b/include/net/vhost_net.h
index 840d4b16e..3389b410d 100644
--- a/include/net/vhost_net.h
+++ b/include/net/vhost_net.h
@@ -13,6 +13,7 @@ typedef struct VhostNetOptions {
     void *opaque;
 } VhostNetOptions;
 
+uint64_t vhost_net_get_max_queues(VHostNetState *net);
 struct vhost_net *vhost_net_init(VhostNetOptions *options);
 
 int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, int total_queues);
@@ -26,5 +27,8 @@ void vhost_net_ack_features(VHostNetState *net, uint64_t features);
 bool vhost_net_virtqueue_pending(VHostNetState *net, int n);
 void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
                               int idx, bool mask);
+int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
+
+int vhost_set_vring_enable(NetClientState * nc, int enable);
 #endif
diff --git a/include/qapi/error.h b/include/qapi/error.h
index f44c45183..4d42cdc5f 100644
--- a/include/qapi/error.h
+++ b/include/qapi/error.h
@@ -2,13 +2,82 @@
  * QEMU Error Objects
  *
  * Copyright IBM, Corp. 2011
+ * Copyright (C) 2011-2015 Red Hat, Inc.
  *
  * Authors:
  *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Markus Armbruster <armbru@redhat.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.  See
  * the COPYING.LIB file in the top-level directory.
  */
+
+/*
+ * Error reporting system loosely patterned after Glib's GError.
+ *
+ * Create an error:
+ *     error_setg(&err, "situation normal, all fouled up");
+ *
+ * Report an error to stderr:
+ *     error_report_err(err);
+ * This frees the error object.
+ *
+ * Report an error somewhere else:
+ *     const char *msg = error_get_pretty(err);
+ *     do with msg what needs to be done...
+ *     error_free(err);
+ *
+ * Handle an error without reporting it (just for completeness):
+ *     error_free(err);
+ *
+ * Assert that an expected error occurred, but clean it up without
+ * reporting it (primarily useful in testsuites):
+ *     error_free_or_abort(&err);
+ *
+ * Pass an existing error to the caller:
+ *     error_propagate(errp, err);
+ * where Error **errp is a parameter, by convention the last one.
+ *
+ * Create a new error and pass it to the caller:
+ *     error_setg(errp, "situation normal, all fouled up");
+ *
+ * Call a function and receive an error from it:
+ *     Error *err = NULL;
+ *     foo(arg, &err);
+ *     if (err) {
+ *         handle the error...
+ *     }
+ *
+ * Call a function ignoring errors:
+ *     foo(arg, NULL);
+ *
+ * Call a function aborting on errors:
+ *     foo(arg, &error_abort);
+ *
+ * Call a function treating errors as fatal:
+ *     foo(arg, &error_fatal);
+ *
+ * Receive an error and pass it on to the caller:
+ *     Error *err = NULL;
+ *     foo(arg, &err);
+ *     if (err) {
+ *         handle the error...
+ *         error_propagate(errp, err);
+ *     }
+ * where Error **errp is a parameter, by convention the last one.
+ *
+ * Do *not* "optimize" this to
+ *     foo(arg, errp);
+ *     if (*errp) { // WRONG!
+ *         handle the error...
+ *     }
+ * because errp may be NULL!
+ *
+ * But when all you do with the error is pass it on, please use
+ *     foo(arg, errp);
+ * for readability.
+ */
+
 #ifndef ERROR_H
 #define ERROR_H
 
@@ -16,93 +85,145 @@
 #include "qapi-types.h"
 #include <stdbool.h>
 
-/**
- * A class representing internal errors within QEMU.  An error has a ErrorClass
- * code and a human message.
+/*
+ * Opaque error object.
  */
 typedef struct Error Error;
 
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message.  This function is not meant to be used outside
- * of QEMU.
+/*
+ * Get @err's human-readable error message.
  */
-void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
-    GCC_FMT_ATTR(3, 4);
+const char *error_get_pretty(Error *err);
 
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message, followed by a strerror() string if
- * @os_error is not zero.
+/*
+ * Get @err's error class.
+ * Note: use of error classes other than ERROR_CLASS_GENERIC_ERROR is
+ * strongly discouraged.
  */
-void error_set_errno(Error **errp, int os_error, ErrorClass err_class,
-                     const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+ErrorClass error_get_class(const Error *err);
 
-#ifdef _WIN32
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message, followed by a g_win32_error_message() string if
- * @win32_err is not zero.
+/*
+ * Create a new error object and assign it to *@errp.
+ * If @errp is NULL, the error is ignored.  Don't bother creating one
+ * then.
+ * If @errp is &error_abort, print a suitable message and abort().
+ * If @errp is &error_fatal, print a suitable message and exit(1).
+ * If @errp is anything else, *@errp must be NULL.
+ * The new error's class is ERROR_CLASS_GENERIC_ERROR, and its
+ * human-readable error message is made from printf-style @fmt, ...
  */
-void error_set_win32(Error **errp, int win32_err, ErrorClass err_class,
-                     const char *fmt, ...) GCC_FMT_ATTR(4, 5);
-#endif
+#define error_setg(errp, fmt, ...)                              \
+    error_setg_internal((errp), __FILE__, __LINE__, __func__,   \
+                        (fmt), ## __VA_ARGS__)
+void error_setg_internal(Error **errp,
+                         const char *src, int line, const char *func,
+                         const char *fmt, ...)
+    GCC_FMT_ATTR(5, 6);
 
-/**
- * Same as error_set(), but sets a generic error
+/*
+ * Just like error_setg(), with @os_error info added to the message.
+ * If @os_error is non-zero, ": " + strerror(os_error) is appended to
+ * the human-readable error message.
  */
-#define error_setg(errp, fmt, ...) \
-    error_set(errp, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
-#define error_setg_errno(errp, os_error, fmt, ...) \
-    error_set_errno(errp, os_error, ERROR_CLASS_GENERIC_ERROR, \
-                    fmt, ## __VA_ARGS__)
+#define error_setg_errno(errp, os_error, fmt, ...)                      \
+    error_setg_errno_internal((errp), __FILE__, __LINE__, __func__,     \
+                              (os_error), (fmt), ## __VA_ARGS__)
+void error_setg_errno_internal(Error **errp,
+                               const char *fname, int line, const char *func,
+                               int os_error, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
+
 #ifdef _WIN32
-#define error_setg_win32(errp, win32_err, fmt, ...) \
-    error_set_win32(errp, win32_err, ERROR_CLASS_GENERIC_ERROR, \
-                    fmt, ## __VA_ARGS__)
+/*
+ * Just like error_setg(), with @win32_error info added to the message.
+ * If @win32_error is non-zero, ": " + g_win32_error_message(win32_err)
+ * is appended to the human-readable error message.
+ */
+#define error_setg_win32(errp, win32_err, fmt, ...)                     \
+    error_setg_win32_internal((errp), __FILE__, __LINE__, __func__,     \
+                              (win32_err), (fmt), ## __VA_ARGS__)
+void error_setg_win32_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int win32_err, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
 #endif
 
+/*
+ * Propagate error object (if any) from @local_err to @dst_errp.
+ * If @local_err is NULL, do nothing (because there's nothing to
+ * propagate).
+ * Else, if @dst_errp is NULL, errors are being ignored.  Free the
+ * error object.
+ * Else, if @dst_errp is &error_abort, print a suitable message and
+ * abort().
+ * Else, if @dst_errp is &error_fatal, print a suitable message and
+ * exit(1).
+ * Else, if @dst_errp already contains an error, ignore this one: free
+ * the error object.
+ * Else, move the error object from @local_err to *@dst_errp.
+ * On return, @local_err is invalid.
+ */
+void error_propagate(Error **dst_errp, Error *local_err);
+
 /**
- * Helper for open() errors
+ * Append a printf-style human-readable explanation to an existing error.
+ * May be called multiple times, and safe if @errp is NULL.
  */
-void error_setg_file_open(Error **errp, int os_errno, const char *filename);
+void error_append_hint(Error **errp, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
 
 /*
- * Get the error class of an error object.
+ * Convenience function to report open() failure.
  */
-ErrorClass error_get_class(const Error *err);
+#define error_setg_file_open(errp, os_errno, filename)                  \
+    error_setg_file_open_internal((errp), __FILE__, __LINE__, __func__, \
+                                  (os_errno), (filename))
+void error_setg_file_open_internal(Error **errp,
+                                   const char *src, int line, const char *func,
+                                   int os_errno, const char *filename);
 
-/**
- * Returns an exact copy of the error passed as an argument.
+/*
+ * Return an exact copy of @err.
  */
 Error *error_copy(const Error *err);
 
-/**
- * Get a human readable representation of an error object.
+/*
+ * Free @err.
+ * @err may be NULL.
  */
-const char *error_get_pretty(Error *err);
+void error_free(Error *err);
 
-/**
- * Convenience function to error_report() and free an error object.
+/*
+ * Convenience function to assert that *@errp is set, then silently free it.
  */
-void error_report_err(Error *);
+void error_free_or_abort(Error **errp);
 
-/**
- * Propagate an error to an indirect pointer to an error.  This function will
- * always transfer ownership of the error reference and handles the case where
- * dst_err is NULL correctly.  Errors after the first are discarded.
+/*
+ * Convenience function to error_report() and free @err.
  */
-void error_propagate(Error **dst_errp, Error *local_err);
+void error_report_err(Error *);
 
-/**
- * Free an error object.
+/*
+ * Just like error_setg(), except you get to specify the error class.
+ * Note: use of error classes other than ERROR_CLASS_GENERIC_ERROR is
+ * strongly discouraged.
  */
-void error_free(Error *err);
+#define error_set(errp, err_class, fmt, ...)                    \
+    error_set_internal((errp), __FILE__, __LINE__, __func__,    \
+                       (err_class), (fmt), ## __VA_ARGS__)
+void error_set_internal(Error **errp,
+                        const char *src, int line, const char *func,
+                        ErrorClass err_class, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
 
-/**
- * If passed to error_set and friends, abort().
+/*
+ * Pass to error_setg() & friends to abort() on error.
  */
-
 extern Error *error_abort;
 
+/*
+ * Pass to error_setg() & friends to exit(1) on error.
+ */
+extern Error *error_fatal;
+
 #endif
diff --git a/include/qapi/qmp/json-lexer.h b/include/qapi/qmp/json-lexer.h
index cdff0460a..cb456d53e 100644
--- a/include/qapi/qmp/json-lexer.h
+++ b/include/qapi/qmp/json-lexer.h
@@ -14,11 +14,16 @@
 #ifndef QEMU_JSON_LEXER_H
 #define QEMU_JSON_LEXER_H
 
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qlist.h"
+#include "glib-compat.h"
 
 typedef enum json_token_type {
-    JSON_OPERATOR = 100,
+    JSON_MIN = 100,
+    JSON_LCURLY = JSON_MIN,
+    JSON_RCURLY,
+    JSON_LSQUARE,
+    JSON_RSQUARE,
+    JSON_COLON,
+    JSON_COMMA,
     JSON_INTEGER,
     JSON_FLOAT,
     JSON_KEYWORD,
@@ -30,13 +35,14 @@ typedef enum json_token_type {
 
 typedef struct JSONLexer JSONLexer;
 
-typedef void (JSONLexerEmitter)(JSONLexer *, QString *, JSONTokenType, int x, int y);
+typedef void (JSONLexerEmitter)(JSONLexer *, GString *,
+                                JSONTokenType, int x, int y);
 
 struct JSONLexer
 {
     JSONLexerEmitter *emit;
     int state;
-    QString *token;
+    GString *token;
     int x, y;
 };
 
diff --git a/include/qapi/qmp/json-parser.h b/include/qapi/qmp/json-parser.h
index 44d88f346..fea89f873 100644
--- a/include/qapi/qmp/json-parser.h
+++ b/include/qapi/qmp/json-parser.h
@@ -18,7 +18,7 @@
 #include "qapi/qmp/qlist.h"
 #include "qapi/error.h"
 
-QObject *json_parser_parse(QList *tokens, va_list *ap);
-QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp);
+QObject *json_parser_parse(GQueue *tokens, va_list *ap);
+QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp);
 
 #endif
diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
index 823f7d7fa..09b3d3ec1 100644
--- a/include/qapi/qmp/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -14,21 +14,29 @@
 #ifndef QEMU_JSON_STREAMER_H
 #define QEMU_JSON_STREAMER_H
 
-#include "qapi/qmp/qlist.h"
+#include <stdint.h>
+#include "glib-compat.h"
 #include "qapi/qmp/json-lexer.h"
 
+typedef struct JSONToken {
+    int type;
+    int x;
+    int y;
+    char str[];
+} JSONToken;
+
 typedef struct JSONMessageParser
 {
-    void (*emit)(struct JSONMessageParser *parser, QList *tokens);
+    void (*emit)(struct JSONMessageParser *parser, GQueue *tokens);
     JSONLexer lexer;
     int brace_count;
     int bracket_count;
-    QList *tokens;
+    GQueue *tokens;
     uint64_t token_size;
 } JSONMessageParser;
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, QList *));
+                              void (*func)(JSONMessageParser *, GQueue *));
 
 int json_message_parser_feed(JSONMessageParser *parser,
                              const char *buffer, size_t size);
diff --git a/include/qapi/qmp/qbool.h b/include/qapi/qmp/qbool.h
index 4aa6be3b3..d9256e426 100644
--- a/include/qapi/qmp/qbool.h
+++ b/include/qapi/qmp/qbool.h
@@ -18,7 +18,7 @@
 #include "qapi/qmp/qobject.h"
 
 typedef struct QBool {
-    QObject_HEAD;
+    QObject base;
     bool value;
 } QBool;
 
diff --git a/include/qapi/qmp/qdict.h b/include/qapi/qmp/qdict.h
index a37f4c156..787c65896 100644
--- a/include/qapi/qmp/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -28,7 +28,7 @@ typedef struct QDictEntry {
 } QDictEntry;
 
 typedef struct QDict {
-    QObject_HEAD;
+    QObject base;
     size_t size;
     QLIST_HEAD(,QDictEntry) table[QDICT_BUCKET_MAX];
 } QDict;
diff --git a/include/qapi/qmp/qerror.h b/include/qapi/qmp/qerror.h
index 842b27ae1..f60149978 100644
--- a/include/qapi/qmp/qerror.h
+++ b/include/qapi/qmp/qerror.h
@@ -106,4 +106,7 @@
 #define QERR_UNSUPPORTED \
     "this feature or command is not currently supported"
 
+#define QERR_REPLAY_NOT_SUPPORTED \
+    "Record/replay feature is not supported for '%s'"
+
 #endif /* QERROR_H */
diff --git a/include/qapi/qmp/qfloat.h b/include/qapi/qmp/qfloat.h
index a8658443d..46745e50d 100644
--- a/include/qapi/qmp/qfloat.h
+++ b/include/qapi/qmp/qfloat.h
@@ -18,7 +18,7 @@
 #include "qapi/qmp/qobject.h"
 
 typedef struct QFloat {
-    QObject_HEAD;
+    QObject base;
     double value;
 } QFloat;
 
diff --git a/include/qapi/qmp/qint.h b/include/qapi/qmp/qint.h
index 48a41b0f2..339a9abb8 100644
--- a/include/qapi/qmp/qint.h
+++ b/include/qapi/qmp/qint.h
@@ -17,7 +17,7 @@
 #include "qapi/qmp/qobject.h"
 
 typedef struct QInt {
-    QObject_HEAD;
+    QObject base;
     int64_t value;
 } QInt;
 
diff --git a/include/qapi/qmp/qlist.h b/include/qapi/qmp/qlist.h
index 6cc4831df..b1bf7852c 100644
--- a/include/qapi/qmp/qlist.h
+++ b/include/qapi/qmp/qlist.h
@@ -22,7 +22,7 @@ typedef struct QListEntry {
 } QListEntry;
 
 typedef struct QList {
-    QObject_HEAD;
+    QObject base;
     QTAILQ_HEAD(,QListEntry) head;
 } QList;
 
diff --git a/include/qapi/qmp/qobject.h b/include/qapi/qmp/qobject.h
index 260d2ed3c..4b96ed583 100644
--- a/include/qapi/qmp/qobject.h
+++ b/include/qapi/qmp/qobject.h
@@ -59,10 +59,6 @@ typedef struct QObject {
     size_t refcnt;
 } QObject;
 
-/* Objects definitions must include this */
-#define QObject_HEAD  \
-    QObject base
-
 /* Get the 'base' part of an object */
 #define QOBJECT(obj) (&(obj)->base)
 
@@ -94,6 +90,7 @@ static inline void qobject_incref(QObject *obj)
  */
 static inline void qobject_decref(QObject *obj)
 {
+    assert(!obj || obj->refcnt);
     if (obj && --obj->refcnt == 0) {
         assert(obj->type != NULL);
         assert(obj->type->destroy != NULL);
diff --git a/include/qapi/qmp/qstring.h b/include/qapi/qmp/qstring.h
index 1bc366610..34675a7fc 100644
--- a/include/qapi/qmp/qstring.h
+++ b/include/qapi/qmp/qstring.h
@@ -17,7 +17,7 @@
 #include "qapi/qmp/qobject.h"
 
 typedef struct QString {
-    QObject_HEAD;
+    QObject base;
     char *string;
     size_t length;
     size_t capacity;
diff --git a/include/qapi/visitor-impl.h b/include/qapi/visitor-impl.h
index f4a2f746c..8c0ba5729 100644
--- a/include/qapi/visitor-impl.h
+++ b/include/qapi/visitor-impl.h
@@ -40,6 +40,8 @@ struct Visitor
     void (*type_str)(Visitor *v, char **obj, const char *name, Error **errp);
     void (*type_number)(Visitor *v, double *obj, const char *name,
                         Error **errp);
+    void (*type_any)(Visitor *v, QObject **obj, const char *name,
+                     Error **errp);
 
     /* May be NULL */
     void (*optional)(Visitor *v, bool *present, const char *name,
diff --git a/include/qapi/visitor.h b/include/qapi/visitor.h
index 00ba104cd..cfc19a616 100644
--- a/include/qapi/visitor.h
+++ b/include/qapi/visitor.h
@@ -58,6 +58,7 @@ void visit_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp);
 void visit_type_bool(Visitor *v, bool *obj, const char *name, Error **errp);
 void visit_type_str(Visitor *v, char **obj, const char *name, Error **errp);
 void visit_type_number(Visitor *v, double *obj, const char *name, Error **errp);
+void visit_type_any(Visitor *v, QObject **obj, const char *name, Error **errp);
 bool visit_start_union(Visitor *v, bool data_present, Error **errp);
 void visit_end_union(Visitor *v, bool data_present, Error **errp);
 
diff --git a/include/qemu-common.h b/include/qemu-common.h
index fb3da6ca2..405364f2b 100644
--- a/include/qemu-common.h
+++ b/include/qemu-common.h
@@ -12,8 +12,7 @@
 #ifndef QEMU_COMMON_H
 #define QEMU_COMMON_H
 
-#include "qemu/compiler.h"
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu/typedefs.h"
 #include "qemu/fprintf-fn.h"
 
@@ -23,59 +22,9 @@
 
 #define TFR(expr) do { if ((expr) != -1) break; } while (errno == EINTR)
 
-/* we put basic includes here to avoid repeating them in device drivers */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <string.h>
-#include <strings.h>
-#include <inttypes.h>
-#include <limits.h>
-#include <time.h>
-#include <ctype.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <assert.h>
-#include <signal.h>
 #include "glib-compat.h"
 #include "qemu/option.h"
-
-#ifdef _WIN32
-#include "sysemu/os-win32.h"
-#endif
-
-#ifdef CONFIG_POSIX
-#include "sysemu/os-posix.h"
-#endif
-
-#ifndef O_LARGEFILE
-#define O_LARGEFILE 0
-#endif
-#ifndef O_BINARY
-#define O_BINARY 0
-#endif
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-#ifndef ENOMEDIUM
-#define ENOMEDIUM ENODEV
-#endif
-#if !defined(ENOTSUP)
-#define ENOTSUP 4096
-#endif
-#if !defined(ECANCELED)
-#define ECANCELED 4097
-#endif
-#if !defined(EMEDIUMTYPE)
-#define EMEDIUMTYPE 4098
-#endif
-#ifndef TIME_MAX
-#define TIME_MAX LONG_MAX
-#endif
+#include "qemu/host-utils.h"
 
 /* HOST_LONG_BITS is the size of a native pointer in bits. */
 #if UINTPTR_MAX == UINT32_MAX
@@ -86,23 +35,6 @@
 # error Unknown pointer size
 #endif
 
-#ifdef _WIN32
-#define fsync _commit
-#if !defined(lseek)
-# define lseek _lseeki64
-#endif
-int qemu_ftruncate64(int, int64_t);
-#if !defined(ftruncate)
-# define ftruncate qemu_ftruncate64
-#endif
-
-static inline char *realpath(const char *path, char *resolved_path)
-{
-    _fullpath(resolved_path, path, _MAX_PATH);
-    return resolved_path;
-}
-#endif
-
 void cpu_ticks_init(void);
 
 /* icount */
@@ -114,7 +46,6 @@ extern int64_t max_delay;
 extern int64_t max_advance;
 void dump_drift_info(FILE *f, fprintf_function cpu_fprintf);
 
-#include "qemu/osdep.h"
 #include "qemu/bswap.h"
 
 /* FIXME: Remove NEED_CPU_H.  */
@@ -269,32 +200,40 @@ int qemu_strnlen(const char *s, int max_len);
  */
 char *qemu_strsep(char **input, const char *delim);
 time_t mktimegm(struct tm *tm);
-int qemu_fls(int i);
 int qemu_fdatasync(int fd);
 int fcntl_setfl(int fd, int flag);
 int qemu_parse_fd(const char *param);
+int qemu_strtol(const char *nptr, const char **endptr, int base,
+                long *result);
+int qemu_strtoul(const char *nptr, const char **endptr, int base,
+                 unsigned long *result);
+int qemu_strtoll(const char *nptr, const char **endptr, int base,
+                 int64_t *result);
+int qemu_strtoull(const char *nptr, const char **endptr, int base,
+                  uint64_t *result);
 
 int parse_uint(const char *s, unsigned long long *value, char **endptr,
                int base);
 int parse_uint_full(const char *s, unsigned long long *value, int base);
 
 /*
- * strtosz() suffixes used to specify the default treatment of an
- * argument passed to strtosz() without an explicit suffix.
+ * qemu_strtosz() suffixes used to specify the default treatment of an
+ * argument passed to qemu_strtosz() without an explicit suffix.
  * These should be defined using upper case characters in the range
- * A-Z, as strtosz() will use qemu_toupper() on the given argument
+ * A-Z, as qemu_strtosz() will use qemu_toupper() on the given argument
  * prior to comparison.
  */
-#define STRTOSZ_DEFSUFFIX_EB	'E'
-#define STRTOSZ_DEFSUFFIX_PB	'P'
-#define STRTOSZ_DEFSUFFIX_TB	'T'
-#define STRTOSZ_DEFSUFFIX_GB	'G'
-#define STRTOSZ_DEFSUFFIX_MB	'M'
-#define STRTOSZ_DEFSUFFIX_KB	'K'
-#define STRTOSZ_DEFSUFFIX_B	'B'
-int64_t strtosz(const char *nptr, char **end);
-int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix);
-int64_t strtosz_suffix_unit(const char *nptr, char **end,
+#define QEMU_STRTOSZ_DEFSUFFIX_EB 'E'
+#define QEMU_STRTOSZ_DEFSUFFIX_PB 'P'
+#define QEMU_STRTOSZ_DEFSUFFIX_TB 'T'
+#define QEMU_STRTOSZ_DEFSUFFIX_GB 'G'
+#define QEMU_STRTOSZ_DEFSUFFIX_MB 'M'
+#define QEMU_STRTOSZ_DEFSUFFIX_KB 'K'
+#define QEMU_STRTOSZ_DEFSUFFIX_B 'B'
+int64_t qemu_strtosz(const char *nptr, char **end);
+int64_t qemu_strtosz_suffix(const char *nptr, char **end,
+                            const char default_suffix);
+int64_t qemu_strtosz_suffix_unit(const char *nptr, char **end,
                             const char default_suffix, int64_t unit);
 #define K_BYTE     (1ULL << 10)
 #define M_BYTE     (1ULL << 20)
@@ -307,6 +246,14 @@ int64_t strtosz_suffix_unit(const char *nptr, char **end,
 #define STR_OR_NULL(str) ((str) ? (str) : "null")
 
 /* id.c */
+
+typedef enum IdSubSystems {
+    ID_QDEV,
+    ID_BLOCK,
+    ID_MAX      /* last element, used as array size */
+} IdSubSystems;
+
+char *id_generate(IdSubSystems id);
 bool id_wellformed(const char *id);
 
 /* path.c */
@@ -481,58 +428,12 @@ static inline uint8_t from_bcd(uint8_t val)
     return ((val >> 4) * 10) + (val & 0x0f);
 }
 
-/* compute with 96 bit intermediate result: (a*b)/c */
-#ifdef CONFIG_INT128
-static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
-{
-    return (__int128_t)a * b / c;
-}
-#else
-static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
-{
-    union {
-        uint64_t ll;
-        struct {
-#ifdef HOST_WORDS_BIGENDIAN
-            uint32_t high, low;
-#else
-            uint32_t low, high;
-#endif
-        } l;
-    } u, res;
-    uint64_t rl, rh;
-
-    u.ll = a;
-    rl = (uint64_t)u.l.low * (uint64_t)b;
-    rh = (uint64_t)u.l.high * (uint64_t)b;
-    rh += (rl >> 32);
-    res.l.high = rh / c;
-    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
-    return res.ll;
-}
-#endif
-
 /* Round number down to multiple */
 #define QEMU_ALIGN_DOWN(n, m) ((n) / (m) * (m))
 
 /* Round number up to multiple */
 #define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m))
 
-static inline bool is_power_of_2(uint64_t value)
-{
-    if (!value) {
-        return 0;
-    }
-
-    return !(value & (value - 1));
-}
-
-/* round down to the nearest power of 2*/
-int64_t pow2floor(int64_t value);
-
-/* round up to the nearest power of 2 (0 if overflow) */
-uint64_t pow2ceil(uint64_t value);
-
 #include "qemu/module.h"
 
 /*
@@ -598,5 +499,6 @@ size_t buffer_find_nonzero_offset(const void *buf, size_t len);
 int parse_debug_env(const char *name, int max, int initial);
 
 const char *qemu_ether_ntoa(const MACAddr *mac);
+void page_size_init(void);
 
 #endif
diff --git a/include/qemu/buffer.h b/include/qemu/buffer.h
new file mode 100644
index 000000000..dead9b77e
--- /dev/null
+++ b/include/qemu/buffer.h
@@ -0,0 +1,161 @@
+/*
+ * QEMU generic buffers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QEMU_BUFFER_H__
+#define QEMU_BUFFER_H__
+
+#include "qemu-common.h"
+
+typedef struct Buffer Buffer;
+
+/**
+ * Buffer:
+ *
+ * The Buffer object provides a simple dynamically resizing
+ * array, with separate tracking of capacity and usage. This
+ * is typically useful when buffering I/O or processing data.
+ */
+
+struct Buffer {
+    char *name;
+    size_t capacity;
+    size_t offset;
+    uint64_t avg_size;
+    uint8_t *buffer;
+};
+
+/**
+ * buffer_init:
+ * @buffer: the buffer object
+ * @name: buffer name
+ *
+ * Optionally attach a name to the buffer, to make it easier
+ * to identify in debug traces.
+ */
+void buffer_init(Buffer *buffer, const char *name, ...)
+        GCC_FMT_ATTR(2, 3);
+
+/**
+ * buffer_shrink:
+ * @buffer: the buffer object
+ *
+ * Try to shrink the buffer.  Checks current buffer capacity and size
+ * and reduces capacity in case only a fraction of the buffer is
+ * actually used.
+ */
+void buffer_shrink(Buffer *buffer);
+
+/**
+ * buffer_reserve:
+ * @buffer: the buffer object
+ * @len: the minimum required free space
+ *
+ * Ensure that the buffer has space allocated for at least
+ * @len bytes. If the current buffer is too small, it will
+ * be reallocated, possibly to a larger size than requested.
+ */
+void buffer_reserve(Buffer *buffer, size_t len);
+
+/**
+ * buffer_reset:
+ * @buffer: the buffer object
+ *
+ * Reset the length of the stored data to zero, but do
+ * not free / reallocate the memory buffer
+ */
+void buffer_reset(Buffer *buffer);
+
+/**
+ * buffer_free:
+ * @buffer: the buffer object
+ *
+ * Reset the length of the stored data to zero and also
+ * free the internal memory buffer
+ */
+void buffer_free(Buffer *buffer);
+
+/**
+ * buffer_append:
+ * @buffer: the buffer object
+ * @data: the data block to append
+ * @len: the length of @data in bytes
+ *
+ * Append the contents of @data to the end of the buffer.
+ * The caller must ensure that the buffer has sufficient
+ * free space for @len bytes, typically by calling the
+ * buffer_reserve() method prior to appending.
+ */
+void buffer_append(Buffer *buffer, const void *data, size_t len);
+
+/**
+ * buffer_advance:
+ * @buffer: the buffer object
+ * @len: the number of bytes to skip
+ *
+ * Remove @len bytes of data from the head of the buffer.
+ * The internal buffer will not be reallocated, so will
+ * have at least @len bytes of free space after this
+ * call completes
+ */
+void buffer_advance(Buffer *buffer, size_t len);
+
+/**
+ * buffer_end:
+ * @buffer: the buffer object
+ *
+ * Get a pointer to the tail end of the internal buffer
+ * The returned pointer is only valid until the next
+ * call to buffer_reserve().
+ *
+ * Returns: the tail of the buffer
+ */
+uint8_t *buffer_end(Buffer *buffer);
+
+/**
+ * buffer_empty:
+ * @buffer: the buffer object
+ *
+ * Determine if the buffer contains any current data
+ *
+ * Returns: true if the buffer holds data, false otherwise
+ */
+gboolean buffer_empty(Buffer *buffer);
+
+/**
+ * buffer_move_empty:
+ * @to: destination buffer object
+ * @from: source buffer object
+ *
+ * Moves buffer, without copying data.  'to' buffer must be empty.
+ * 'from' buffer is empty and zero-sized on return.
+ */
+void buffer_move_empty(Buffer *to, Buffer *from);
+
+/**
+ * buffer_move:
+ * @to: destination buffer object
+ * @from: source buffer object
+ *
+ * Moves buffer, copying data (unless 'to' buffer happens to be empty).
+ * 'from' buffer is empty and zero-sized on return.
+ */
+void buffer_move(Buffer *to, Buffer *from);
+
+#endif /* QEMU_BUFFER_H__ */
diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h
index df9dd514f..d22eb01be 100644
--- a/include/qemu/compiler.h
+++ b/include/qemu/compiler.h
@@ -42,10 +42,55 @@
 # define QEMU_PACKED __attribute__((packed))
 #endif
 
-#define cat(x,y) x ## y
-#define cat2(x,y) cat(x,y)
+#ifndef glue
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)	tostring(s)
+#define tostring(s)	#s
+#endif
+
+#ifndef likely
+#if __GNUC__ < 3
+#define __builtin_expect(x, n) (x)
+#endif
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x)   __builtin_expect(!!(x), 0)
+#endif
+
+#ifndef container_of
+#define container_of(ptr, type, member) ({                      \
+        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
+        (type *) ((char *) __mptr - offsetof(type, member));})
+#endif
+
+/* Convert from a base type to a parent type, with compile time checking.  */
+#ifdef __GNUC__
+#define DO_UPCAST(type, field, dev) ( __extension__ ( { \
+    char __attribute__((unused)) offset_must_be_zero[ \
+        -offsetof(type, field)]; \
+    container_of(dev, type, field);}))
+#else
+#define DO_UPCAST(type, field, dev) container_of(dev, type, field)
+#endif
+
+#define typeof_field(type, field) typeof(((type *)0)->field)
+#define type_check(t1,t2) ((t1*)0 - (t2*)0)
+
+#ifndef always_inline
+#if !((__GNUC__ < 3) || defined(__APPLE__))
+#ifdef __OPTIMIZE__
+#undef inline
+#define inline __attribute__ (( always_inline )) __inline__
+#endif
+#endif
+#else
+#undef inline
+#define inline always_inline
+#endif
+
 #define QEMU_BUILD_BUG_ON(x) \
-    typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
+    typedef char glue(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
 
 #if defined __GNUC__
 # if !QEMU_GNUC_PREREQ(4, 4)
diff --git a/include/block/coroutine.h b/include/qemu/coroutine.h
index 20c027a7f..20c027a7f 100644
--- a/include/block/coroutine.h
+++ b/include/qemu/coroutine.h
diff --git a/include/block/coroutine_int.h b/include/qemu/coroutine_int.h
index 9aa1aae5d..42d683840 100644
--- a/include/block/coroutine_int.h
+++ b/include/qemu/coroutine_int.h
@@ -26,7 +26,7 @@
 #define QEMU_COROUTINE_INT_H
 
 #include "qemu/queue.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 typedef enum {
     COROUTINE_YIELD = 1,
diff --git a/include/qemu/event_notifier.h b/include/qemu/event_notifier.h
index 88b57af7c..a8f28540c 100644
--- a/include/qemu/event_notifier.h
+++ b/include/qemu/event_notifier.h
@@ -38,7 +38,7 @@ int event_notifier_set_handler(EventNotifier *, EventNotifierHandler *);
 
 #ifdef CONFIG_POSIX
 void event_notifier_init_fd(EventNotifier *, int fd);
-int event_notifier_get_fd(EventNotifier *);
+int event_notifier_get_fd(const EventNotifier *);
 #else
 HANDLE event_notifier_get_handle(EventNotifier *);
 #endif
diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h
index d4f21c947..3ef97d503 100644
--- a/include/qemu/host-utils.h
+++ b/include/qemu/host-utils.h
@@ -26,7 +26,9 @@
 #define HOST_UTILS_H 1
 
 #include "qemu/compiler.h"   /* QEMU_GNUC_PREREQ */
+#include "qemu/bswap.h"
 #include <limits.h>
+#include <stdbool.h>
 
 #ifdef CONFIG_INT128
 static inline void mulu64(uint64_t *plow, uint64_t *phigh,
@@ -45,6 +47,12 @@ static inline void muls64(uint64_t *plow, uint64_t *phigh,
     *phigh = r >> 64;
 }
 
+/* compute with 96 bit intermediate result: (a*b)/c */
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    return (__int128_t)a * b / c;
+}
+
 static inline int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor)
 {
     if (divisor == 0) {
@@ -75,6 +83,29 @@ void muls64(uint64_t *phigh, uint64_t *plow, int64_t a, int64_t b);
 void mulu64(uint64_t *phigh, uint64_t *plow, uint64_t a, uint64_t b);
 int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor);
 int divs128(int64_t *plow, int64_t *phigh, int64_t divisor);
+
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    union {
+        uint64_t ll;
+        struct {
+#ifdef HOST_WORDS_BIGENDIAN
+            uint32_t high, low;
+#else
+            uint32_t low, high;
+#endif
+        } l;
+    } u, res;
+    uint64_t rl, rh;
+
+    u.ll = a;
+    rl = (uint64_t)u.l.low * (uint64_t)b;
+    rh = (uint64_t)u.l.high * (uint64_t)b;
+    rh += (rl >> 32);
+    res.l.high = rh / c;
+    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
+    return res.ll;
+}
 #endif
 
 /**
@@ -361,6 +392,80 @@ static inline int ctpop64(uint64_t val)
 #endif
 }
 
+/**
+ * revbit8 - reverse the bits in an 8-bit value.
+ * @x: The value to modify.
+ */
+static inline uint8_t revbit8(uint8_t x)
+{
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0) >> 4)
+      | ((x & 0x0f) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x88) >> 3)
+      | ((x & 0x44) >> 1)
+      | ((x & 0x22) << 1)
+      | ((x & 0x11) << 3);
+    return x;
+}
+
+/**
+ * revbit16 - reverse the bits in a 16-bit value.
+ * @x: The value to modify.
+ */
+static inline uint16_t revbit16(uint16_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap16(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0) >> 4)
+      | ((x & 0x0f0f) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x8888) >> 3)
+      | ((x & 0x4444) >> 1)
+      | ((x & 0x2222) << 1)
+      | ((x & 0x1111) << 3);
+    return x;
+}
+
+/**
+ * revbit32 - reverse the bits in a 32-bit value.
+ * @x: The value to modify.
+ */
+static inline uint32_t revbit32(uint32_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap32(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0f0f0u) >> 4)
+      | ((x & 0x0f0f0f0fu) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x88888888u) >> 3)
+      | ((x & 0x44444444u) >> 1)
+      | ((x & 0x22222222u) << 1)
+      | ((x & 0x11111111u) << 3);
+    return x;
+}
+
+/**
+ * revbit64 - reverse the bits in a 64-bit value.
+ * @x: The value to modify.
+ */
+static inline uint64_t revbit64(uint64_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap64(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0f0f0f0f0f0f0ull) >> 4)
+      | ((x & 0x0f0f0f0f0f0f0f0full) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x8888888888888888ull) >> 3)
+      | ((x & 0x4444444444444444ull) >> 1)
+      | ((x & 0x2222222222222222ull) << 1)
+      | ((x & 0x1111111111111111ull) << 3);
+    return x;
+}
+
 /* Host type specific sizes of these routines.  */
 
 #if ULONG_MAX == UINT32_MAX
@@ -369,14 +474,48 @@ static inline int ctpop64(uint64_t val)
 # define clol   clo32
 # define ctol   cto32
 # define ctpopl ctpop32
+# define revbitl revbit32
 #elif ULONG_MAX == UINT64_MAX
 # define clzl   clz64
 # define ctzl   ctz64
 # define clol   clo64
 # define ctol   cto64
 # define ctpopl ctpop64
+# define revbitl revbit64
 #else
 # error Unknown sizeof long
 #endif
 
+static inline bool is_power_of_2(uint64_t value)
+{
+    if (!value) {
+        return 0;
+    }
+
+    return !(value & (value - 1));
+}
+
+/* round down to the nearest power of 2*/
+static inline int64_t pow2floor(int64_t value)
+{
+    if (!is_power_of_2(value)) {
+        value = 0x8000000000000000ULL >> clz64(value);
+    }
+    return value;
+}
+
+/* round up to the nearest power of 2 (0 if overflow) */
+static inline uint64_t pow2ceil(uint64_t value)
+{
+    uint8_t nlz = clz64(value);
+
+    if (is_power_of_2(value)) {
+        return value;
+    }
+    if (!nlz) {
+        return 0;
+    }
+    return 1ULL << (64 - nlz);
+}
+
 #endif
diff --git a/include/qemu/log.h b/include/qemu/log.h
index f880e66db..362cbc4e6 100644
--- a/include/qemu/log.h
+++ b/include/qemu/log.h
@@ -35,12 +35,12 @@ static inline bool qemu_log_enabled(void)
 #define CPU_LOG_INT        (1 << 4)
 #define CPU_LOG_EXEC       (1 << 5)
 #define CPU_LOG_PCALL      (1 << 6)
-#define CPU_LOG_IOPORT     (1 << 7)
 #define CPU_LOG_TB_CPU     (1 << 8)
 #define CPU_LOG_RESET      (1 << 9)
 #define LOG_UNIMP          (1 << 10)
 #define LOG_GUEST_ERROR    (1 << 11)
 #define CPU_LOG_MMU        (1 << 12)
+#define CPU_LOG_TB_NOCHAIN (1 << 13)
 
 /* Returns true if a bit is set in the current loglevel mask
  */
diff --git a/include/qemu/main-loop.h b/include/qemu/main-loop.h
index bc18ca30e..99769093f 100644
--- a/include/qemu/main-loop.h
+++ b/include/qemu/main-loop.h
@@ -203,6 +203,7 @@ void qemu_set_fd_handler(int fd,
                          IOHandler *fd_write,
                          void *opaque);
 
+GSource *iohandler_get_g_source(void);
 #ifdef CONFIG_POSIX
 /**
  * qemu_add_child_watch: Register a child process for reaping.
@@ -265,8 +266,6 @@ void qemu_mutex_unlock_iothread(void);
 /* internal interfaces */
 
 void qemu_fd_register(int fd);
-void qemu_iohandler_fill(GArray *pollfds);
-void qemu_iohandler_poll(GArray *pollfds, int rc);
 
 QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
 void qemu_bh_schedule_idle(QEMUBH *bh);
diff --git a/include/qemu/memfd.h b/include/qemu/memfd.h
new file mode 100644
index 000000000..53858ed43
--- /dev/null
+++ b/include/qemu/memfd.h
@@ -0,0 +1,26 @@
+#ifndef QEMU_MEMFD_H
+#define QEMU_MEMFD_H
+
+#include "config-host.h"
+#include <stdbool.h>
+
+#ifndef F_LINUX_SPECIFIC_BASE
+#define F_LINUX_SPECIFIC_BASE 1024
+#endif
+
+#ifndef F_ADD_SEALS
+#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
+#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
+
+#define F_SEAL_SEAL     0x0001  /* prevent further seals from being set */
+#define F_SEAL_SHRINK   0x0002  /* prevent file from shrinking */
+#define F_SEAL_GROW     0x0004  /* prevent file from growing */
+#define F_SEAL_WRITE    0x0008  /* prevent writes */
+#endif
+
+void *qemu_memfd_alloc(const char *name, size_t size, unsigned int seals,
+                       int *fd);
+void qemu_memfd_free(void *ptr, size_t size, int fd);
+bool qemu_memfd_check(void);
+
+#endif /* QEMU_MEMFD_H */
diff --git a/include/qemu/mmap-alloc.h b/include/qemu/mmap-alloc.h
new file mode 100644
index 000000000..0899b2f01
--- /dev/null
+++ b/include/qemu/mmap-alloc.h
@@ -0,0 +1,12 @@
+#ifndef QEMU_MMAP_ALLOC
+#define QEMU_MMAP_ALLOC
+
+#include "qemu-common.h"
+
+size_t qemu_fd_getpagesize(int fd);
+
+void *qemu_ram_mmap(int fd, size_t size, size_t align, bool shared);
+
+void qemu_ram_munmap(void *ptr, size_t size);
+
+#endif
diff --git a/include/qemu/option.h b/include/qemu/option.h
index 57e51c962..71f5f27ec 100644
--- a/include/qemu/option.h
+++ b/include/qemu/option.h
@@ -30,6 +30,7 @@
 #include "qemu/queue.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
+#include "qemu/typedefs.h"
 
 const char *get_opt_name(char *buf, int buf_size, const char *p, char delim);
 const char *get_opt_value(char *buf, int buf_size, const char *p);
@@ -44,10 +45,6 @@ void parse_option_size(const char *name, const char *value,
 bool has_help_option(const char *param);
 bool is_valid_option_list(const char *param);
 
-typedef struct QemuOpt QemuOpt;
-typedef struct QemuOpts QemuOpts;
-typedef struct QemuOptsList QemuOptsList;
-
 enum QemuOptType {
     QEMU_OPT_STRING = 0,  /* no parsing (use string as-is)                        */
     QEMU_OPT_BOOL,        /* on/off                                               */
diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h
index 324736426..84e84ac70 100644
--- a/include/qemu/osdep.h
+++ b/include/qemu/osdep.h
@@ -1,12 +1,55 @@
+/*
+ * OS includes and handling of OS dependencies
+ *
+ * This header exists to pull in some common system headers that
+ * most code in QEMU will want, and to fix up some possible issues with
+ * it (missing defines, Windows weirdness, and so on).
+ *
+ * To avoid getting into possible circular include dependencies, this
+ * file should not include any other QEMU headers, with the exceptions
+ * of config-host.h, compiler.h, os-posix.h and os-win32.h, all of which
+ * are doing a similar job to this file and are under similar constraints.
+ *
+ * This header also contains prototypes for functions defined in
+ * os-*.c and util/oslib-*.c; those would probably be better split
+ * out into separate header files.
+ *
+ * In an ideal world this header would contain only:
+ *  (1) things which everybody needs
+ *  (2) things without which code would work on most platforms but
+ *      fail to compile or misbehave on a minority of host OSes
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
 #ifndef QEMU_OSDEP_H
 #define QEMU_OSDEP_H
 
 #include "config-host.h"
+#include "qemu/compiler.h"
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <sys/types.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <inttypes.h>
+#include <limits.h>
+/* Put unistd.h before time.h as that triggers localtime_r/gmtime_r
+ * function availability on recentish Mingw-w64 platforms. */
+#include <unistd.h>
+#include <time.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <assert.h>
+#include <signal.h>
+
 #ifdef __OpenBSD__
 #include <sys/signal.h>
 #endif
@@ -18,7 +61,15 @@
 #define WEXITSTATUS(x) (x)
 #endif
 
-#include <sys/time.h>
+#ifdef _WIN32
+#include "sysemu/os-win32.h"
+#endif
+
+#ifdef CONFIG_POSIX
+#include "sysemu/os-posix.h"
+#endif
+
+#include "qapi/error.h"
 
 #if defined(CONFIG_SOLARIS) && CONFIG_SOLARIS_VERSION < 10
 /* [u]int_fast*_t not in <sys/int_types.h> */
@@ -27,40 +78,30 @@ typedef unsigned int            uint_fast16_t;
 typedef signed int              int_fast16_t;
 #endif
 
-#ifndef glue
-#define xglue(x, y) x ## y
-#define glue(x, y) xglue(x, y)
-#define stringify(s)	tostring(s)
-#define tostring(s)	#s
+#ifndef O_LARGEFILE
+#define O_LARGEFILE 0
 #endif
-
-#ifndef likely
-#if __GNUC__ < 3
-#define __builtin_expect(x, n) (x)
+#ifndef O_BINARY
+#define O_BINARY 0
 #endif
-
-#define likely(x)   __builtin_expect(!!(x), 1)
-#define unlikely(x)   __builtin_expect(!!(x), 0)
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
 #endif
-
-#ifndef container_of
-#define container_of(ptr, type, member) ({                      \
-        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
-        (type *) ((char *) __mptr - offsetof(type, member));})
+#ifndef ENOMEDIUM
+#define ENOMEDIUM ENODEV
 #endif
-
-/* Convert from a base type to a parent type, with compile time checking.  */
-#ifdef __GNUC__
-#define DO_UPCAST(type, field, dev) ( __extension__ ( { \
-    char __attribute__((unused)) offset_must_be_zero[ \
-        -offsetof(type, field)]; \
-    container_of(dev, type, field);}))
-#else
-#define DO_UPCAST(type, field, dev) container_of(dev, type, field)
+#if !defined(ENOTSUP)
+#define ENOTSUP 4096
+#endif
+#if !defined(ECANCELED)
+#define ECANCELED 4097
+#endif
+#if !defined(EMEDIUMTYPE)
+#define EMEDIUMTYPE 4098
+#endif
+#ifndef TIME_MAX
+#define TIME_MAX LONG_MAX
 #endif
-
-#define typeof_field(type, field) typeof(((type *)0)->field)
-#define type_check(t1,t2) ((t1*)0 - (t2*)0)
 
 #ifndef MIN
 #define MIN(a, b) (((a) < (b)) ? (a) : (b))
@@ -87,20 +128,6 @@ typedef signed int              int_fast16_t;
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 #endif
 
-#ifndef always_inline
-#if !((__GNUC__ < 3) || defined(__APPLE__))
-#ifdef __OPTIMIZE__
-#undef inline
-#define inline __attribute__ (( always_inline )) __inline__
-#endif
-#endif
-#else
-#undef inline
-#define inline always_inline
-#endif
-
-#define qemu_printf printf
-
 int qemu_daemon(int nochdir, int noclose);
 void *qemu_try_memalign(size_t alignment, size_t size);
 void *qemu_memalign(size_t alignment, size_t size);
@@ -112,6 +139,8 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 
 #if defined(CONFIG_MADVISE)
 
+#include <sys/mman.h>
+
 #define QEMU_MADV_WILLNEED  MADV_WILLNEED
 #define QEMU_MADV_DONTNEED  MADV_DONTNEED
 #ifdef MADV_DONTFORK
@@ -144,6 +173,11 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 #else
 #define QEMU_MADV_HUGEPAGE QEMU_MADV_INVALID
 #endif
+#ifdef MADV_NOHUGEPAGE
+#define QEMU_MADV_NOHUGEPAGE MADV_NOHUGEPAGE
+#else
+#define QEMU_MADV_NOHUGEPAGE QEMU_MADV_INVALID
+#endif
 
 #elif defined(CONFIG_POSIX_MADVISE)
 
@@ -155,6 +189,7 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 #define QEMU_MADV_DODUMP QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+#define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
 
 #else /* no-op */
 
@@ -166,6 +201,7 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 #define QEMU_MADV_DODUMP QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+#define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
 
 #endif
 
@@ -220,8 +256,12 @@ static inline void qemu_timersub(const struct timeval *val1,
 
 void qemu_set_cloexec(int fd);
 
-void qemu_set_version(const char *);
-const char *qemu_get_version(void);
+/* QEMU "hardware version" setting. Used to replace code that exposed
+ * QEMU_VERSION to guests in the past and need to keep compatibilty.
+ * Do not use qemu_hw_version() in new code.
+ */
+void qemu_set_hw_version(const char *);
+const char *qemu_hw_version(void);
 
 void fips_set_state(bool requested);
 bool fips_get_state(void);
@@ -261,4 +301,18 @@ void os_mem_prealloc(int fd, char *area, size_t sz);
 
 int qemu_read_password(char *buf, int buf_size);
 
+/**
+ * qemu_fork:
+ *
+ * A version of fork that avoids signal handler race
+ * conditions that can lead to child process getting
+ * signals that are otherwise only expected by the
+ * parent. It also resets all signal handlers to the
+ * default settings.
+ *
+ * Returns 0 to child process, pid number to parent
+ * or -1 on failure.
+ */
+pid_t qemu_fork(Error **errp);
+
 #endif
diff --git a/include/qemu/queue.h b/include/qemu/queue.h
index a8d3cb8e6..f781aa20a 100644
--- a/include/qemu/queue.h
+++ b/include/qemu/queue.h
@@ -117,12 +117,6 @@ struct {                                                                \
         }                                                               \
 } while (/*CONSTCOND*/0)
 
-#define QLIST_FIX_HEAD_PTR(head, field) do {                            \
-        if ((head)->lh_first != NULL) {                                 \
-            (head)->lh_first->field.le_prev = &(head)->lh_first;        \
-        }                                                               \
-} while (/*CONSTCOND*/0)
-
 #define QLIST_INSERT_AFTER(listelm, elm, field) do {                    \
         if (((elm)->field.le_next = (listelm)->field.le_next) != NULL)  \
                 (listelm)->field.le_next->field.le_prev =               \
diff --git a/include/qemu/rcu.h b/include/qemu/rcu.h
index 7df1e8662..f6d1d5637 100644
--- a/include/qemu/rcu.h
+++ b/include/qemu/rcu.h
@@ -71,7 +71,7 @@ struct rcu_reader_data {
     /* Data used by reader only */
     unsigned depth;
 
-    /* Data used for registry, protected by rcu_gp_lock */
+    /* Data used for registry, protected by rcu_registry_lock */
     QLIST_ENTRY(rcu_reader_data) node;
 };
 
diff --git a/include/qemu/seqlock.h b/include/qemu/seqlock.h
index 3ff118a1a..70b01fd60 100644
--- a/include/qemu/seqlock.h
+++ b/include/qemu/seqlock.h
@@ -55,18 +55,18 @@ static inline void seqlock_write_unlock(QemuSeqLock *sl)
 static inline unsigned seqlock_read_begin(QemuSeqLock *sl)
 {
     /* Always fail if a write is in progress.  */
-    unsigned ret = sl->sequence & ~1;
+    unsigned ret = atomic_read(&sl->sequence);
 
     /* Read sequence before reading other fields.  */
     smp_rmb();
-    return ret;
+    return ret & ~1;
 }
 
-static int seqlock_read_retry(const QemuSeqLock *sl, unsigned start)
+static inline int seqlock_read_retry(const QemuSeqLock *sl, unsigned start)
 {
     /* Read other fields before reading final sequence.  */
     smp_rmb();
-    return unlikely(sl->sequence != start);
+    return unlikely(atomic_read(&sl->sequence) != start);
 }
 
 #endif
diff --git a/include/qemu/sockets.h b/include/qemu/sockets.h
index c174b5cbd..5a183c570 100644
--- a/include/qemu/sockets.h
+++ b/include/qemu/sockets.h
@@ -88,4 +88,38 @@ int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp);
 int parse_host_port(struct sockaddr_in *saddr, const char *str);
 int socket_init(void);
 
+/**
+ * socket_local_address:
+ * @fd: the socket file handle
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the local socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *socket_local_address(int fd, Error **errp);
+
+/**
+ * socket_remote_address:
+ * @fd: the socket file handle
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the remote socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *socket_remote_address(int fd, Error **errp);
+
+
+void qapi_copy_SocketAddress(SocketAddress **p_dest,
+                             SocketAddress *src);
+
 #endif /* QEMU_SOCKET_H */
diff --git a/include/qemu/thread-win32.h b/include/qemu/thread-win32.h
index 3d58081be..385ff5f76 100644
--- a/include/qemu/thread-win32.h
+++ b/include/qemu/thread-win32.h
@@ -18,6 +18,7 @@ struct QemuSemaphore {
 };
 
 struct QemuEvent {
+    int value;
     HANDLE event;
 };
 
diff --git a/include/qemu/throttle.h b/include/qemu/throttle.h
index 995b2d595..12faaad95 100644
--- a/include/qemu/throttle.h
+++ b/include/qemu/throttle.h
@@ -114,6 +114,8 @@ bool throttle_conflicting(ThrottleConfig *cfg);
 
 bool throttle_is_valid(ThrottleConfig *cfg);
 
+bool throttle_max_is_missing_limit(ThrottleConfig *cfg);
+
 void throttle_config(ThrottleState *ts,
                      ThrottleTimers *tt,
                      ThrottleConfig *cfg);
diff --git a/include/qemu/timed-average.h b/include/qemu/timed-average.h
new file mode 100644
index 000000000..364bf88f7
--- /dev/null
+++ b/include/qemu/timed-average.h
@@ -0,0 +1,64 @@
+/*
+ * QEMU timed average computation
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) version 3 or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TIMED_AVERAGE_H
+#define TIMED_AVERAGE_H
+
+#include <stdint.h>
+
+#include "qemu/timer.h"
+
+typedef struct TimedAverageWindow TimedAverageWindow;
+typedef struct TimedAverage TimedAverage;
+
+/* All fields of both structures are private */
+
+struct TimedAverageWindow {
+    uint64_t      min;             /* minimum value accounted in the window */
+    uint64_t      max;             /* maximum value accounted in the window */
+    uint64_t      sum;             /* sum of all values */
+    uint64_t      count;           /* number of values */
+    int64_t       expiration;      /* the end of the current window in ns */
+};
+
+struct TimedAverage {
+    uint64_t           period;     /* period in nanoseconds */
+    TimedAverageWindow windows[2]; /* two overlapping windows of with
+                                    * an offset of period / 2 between them */
+    unsigned           current;    /* the current window index: it's also the
+                                    * oldest window index */
+    QEMUClockType      clock_type; /* the clock used */
+};
+
+void timed_average_init(TimedAverage *ta, QEMUClockType clock_type,
+                        uint64_t period);
+
+void timed_average_account(TimedAverage *ta, uint64_t value);
+
+uint64_t timed_average_min(TimedAverage *ta);
+uint64_t timed_average_avg(TimedAverage *ta);
+uint64_t timed_average_max(TimedAverage *ta);
+uint64_t timed_average_sum(TimedAverage *ta, uint64_t *elapsed);
+
+#endif
diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index 5923d600f..d0946cb95 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -4,6 +4,7 @@
 #include "qemu/typedefs.h"
 #include "qemu-common.h"
 #include "qemu/notify.h"
+#include "qemu/host-utils.h"
 
 #define NANOSECONDS_PER_SECOND 1000000000LL
 
@@ -856,7 +857,7 @@ int64_t cpu_icount_to_ns(int64_t icount);
 
 #if defined(_ARCH_PPC)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t retval;
 #ifdef _ARCH_PPC64
@@ -882,7 +883,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__i386__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile ("rdtsc" : "=A" (val));
@@ -891,7 +892,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__x86_64__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     uint32_t low,high;
     int64_t val;
@@ -904,7 +905,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__hppa__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int val;
     asm volatile ("mfctl %%cr16, %0" : "=r"(val));
@@ -913,7 +914,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__ia64)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile ("mov %0 = ar.itc" : "=r"(val) :: "memory");
@@ -922,7 +923,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__s390__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile("stck 0(%1)" : "=m" (val) : "a" (&val) : "cc");
@@ -931,7 +932,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__sparc__)
 
-static inline int64_t cpu_get_real_ticks (void)
+static inline int64_t cpu_get_host_ticks (void)
 {
 #if defined(_LP64)
     uint64_t        rval;
@@ -969,7 +970,7 @@ static inline int64_t cpu_get_real_ticks (void)
                               : "=r" (value));          \
     }
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     /* On kernels >= 2.6.25 rdhwr <reg>, $2 and $3 are emulated */
     uint32_t count;
@@ -985,7 +986,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__alpha__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     uint64_t cc;
     uint32_t cur, ofs;
@@ -1000,7 +1001,7 @@ static inline int64_t cpu_get_real_ticks(void)
 /* The host CPU doesn't have an easily accessible cycle counter.
    Just return a monotonically increasing value.  This will be
    totally wrong, but hopefully better than nothing.  */
-static inline int64_t cpu_get_real_ticks (void)
+static inline int64_t cpu_get_host_ticks (void)
 {
     static int64_t ticks = 0;
     return ticks++;
diff --git a/include/qemu/tls.h b/include/qemu/tls.h
deleted file mode 100644
index b92ea9d7d..000000000
--- a/include/qemu/tls.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Abstraction layer for defining and using TLS variables
- *
- * Copyright (c) 2011 Red Hat, Inc
- * Copyright (c) 2011 Linaro Limited
- *
- * Authors:
- *  Paolo Bonzini <pbonzini@redhat.com>
- *  Peter Maydell <peter.maydell@linaro.org>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef QEMU_TLS_H
-#define QEMU_TLS_H
-
-/* Per-thread variables. Note that we only have implementations
- * which are really thread-local on Linux; the dummy implementations
- * define plain global variables.
- *
- * This means that for the moment use should be restricted to
- * per-VCPU variables, which are OK because:
- *  - the only -user mode supporting multiple VCPU threads is linux-user
- *  - TCG system mode is single-threaded regarding VCPUs
- *  - KVM system mode is multi-threaded but limited to Linux
- *
- * TODO: proper implementations via Win32 .tls sections and
- * POSIX pthread_getspecific.
- */
-#ifdef __linux__
-#define DECLARE_TLS(type, x) extern DEFINE_TLS(type, x)
-#define DEFINE_TLS(type, x)  __thread __typeof__(type) tls__##x
-#define tls_var(x)           tls__##x
-#else
-/* Dummy implementations which define plain global variables */
-#define DECLARE_TLS(type, x) extern DEFINE_TLS(type, x)
-#define DEFINE_TLS(type, x)  __typeof__(type) tls__##x
-#define tls_var(x)           tls__##x
-#endif
-
-#endif
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
index 6fdcbcd52..3eedcf4c8 100644
--- a/include/qemu/typedefs.h
+++ b/include/qemu/typedefs.h
@@ -3,21 +3,23 @@
 
 /* A load of opaque types so that device init declarations don't have to
    pull in all the real definitions.  */
-struct Monitor;
 
 /* Please keep this list in alphabetical order */
 typedef struct AdapterInfo AdapterInfo;
 typedef struct AddressSpace AddressSpace;
 typedef struct AioContext AioContext;
+typedef struct AllwinnerAHCIState AllwinnerAHCIState;
 typedef struct AudioState AudioState;
 typedef struct BlockBackend BlockBackend;
+typedef struct BlockBackendRootState BlockBackendRootState;
 typedef struct BlockDriverState BlockDriverState;
 typedef struct BusClass BusClass;
 typedef struct BusState BusState;
 typedef struct CharDriverState CharDriverState;
 typedef struct CompatProperty CompatProperty;
-typedef struct DeviceState DeviceState;
+typedef struct CPUAddressSpace CPUAddressSpace;
 typedef struct DeviceListener DeviceListener;
+typedef struct DeviceState DeviceState;
 typedef struct DisplayChangeListener DisplayChangeListener;
 typedef struct DisplayState DisplayState;
 typedef struct DisplaySurface DisplaySurface;
@@ -41,10 +43,13 @@ typedef struct MemoryRegion MemoryRegion;
 typedef struct MemoryRegionSection MemoryRegionSection;
 typedef struct MigrationIncomingState MigrationIncomingState;
 typedef struct MigrationParams MigrationParams;
+typedef struct MigrationState MigrationState;
 typedef struct Monitor Monitor;
+typedef struct MonitorDef MonitorDef;
 typedef struct MouseTransformInfo MouseTransformInfo;
 typedef struct MSIMessage MSIMessage;
 typedef struct NetClientState NetClientState;
+typedef struct NetFilterState NetFilterState;
 typedef struct NICInfo NICInfo;
 typedef struct PcGuestInfo PcGuestInfo;
 typedef struct PCIBridge PCIBridge;
@@ -58,18 +63,24 @@ typedef struct PCIESlot PCIESlot;
 typedef struct PCIExpressDevice PCIExpressDevice;
 typedef struct PCIExpressHost PCIExpressHost;
 typedef struct PCIHostState PCIHostState;
+typedef struct PCMachineClass PCMachineClass;
+typedef struct PCMachineState PCMachineState;
 typedef struct PCMCIACardState PCMCIACardState;
 typedef struct PixelFormat PixelFormat;
-typedef struct PropertyInfo PropertyInfo;
+typedef struct PostcopyDiscardState PostcopyDiscardState;
 typedef struct Property Property;
+typedef struct PropertyInfo PropertyInfo;
 typedef struct QEMUBH QEMUBH;
 typedef struct QemuConsole QemuConsole;
 typedef struct QEMUFile QEMUFile;
-typedef struct QEMUMachine QEMUMachine;
+typedef struct QemuOpt QemuOpt;
+typedef struct QemuOpts QemuOpts;
+typedef struct QemuOptsList QemuOptsList;
 typedef struct QEMUSGList QEMUSGList;
 typedef struct QEMUSizedBuffer QEMUSizedBuffer;
-typedef struct QEMUTimerListGroup QEMUTimerListGroup;
 typedef struct QEMUTimer QEMUTimer;
+typedef struct QEMUTimerListGroup QEMUTimerListGroup;
+typedef struct RAMBlock RAMBlock;
 typedef struct Range Range;
 typedef struct SerialState SerialState;
 typedef struct SHPCDevice SHPCDevice;
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index 20aabc9cb..51a1323ea 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -28,7 +28,6 @@
 #include "exec/memattrs.h"
 #include "qemu/queue.h"
 #include "qemu/thread.h"
-#include "qemu/tls.h"
 #include "qemu/typedefs.h"
 
 typedef int (*WriteCoreDumpFunction)(const void *buf, size_t size,
@@ -224,6 +223,7 @@ struct kvm_run;
  * @halted: Nonzero if the CPU is in suspended state.
  * @stop: Indicates a pending stop request.
  * @stopped: Indicates the CPU has been artificially stopped.
+ * @crash_occurred: Indicates the OS reported a crash (panic) for this CPU
  * @tcg_exit_req: Set to force TCG to stop executing linked TBs for this
  *           CPU and return to its top level loop.
  * @singlestep_enabled: Flags for single-stepping.
@@ -231,7 +231,13 @@ struct kvm_run;
  * @icount_decr: Number of cycles left, with interrupt flag in high bit.
  * This allows a single read-compare-cbranch-write sequence to test
  * for both decrementer underflow and exceptions.
- * @can_do_io: Nonzero if memory-mapped IO is safe.
+ * @can_do_io: Nonzero if memory-mapped IO is safe. Deterministic execution
+ * requires that IO only be performed on the last instruction of a TB
+ * so that interrupts take effect immediately.
+ * @cpu_ases: Pointer to array of CPUAddressSpaces (which define the
+ *            AddressSpaces this CPU has)
+ * @as: Pointer to the first AddressSpace, for the convenience of targets which
+ *      only have a single AddressSpace
  * @env_ptr: Pointer to subclass-specific CPUArchState field.
  * @current_tb: Currently executing TB.
  * @gdb_regs: Additional GDB registers.
@@ -242,6 +248,8 @@ struct kvm_run;
  * @mem_io_pc: Host Program Counter at which the memory was accessed.
  * @mem_io_vaddr: Target virtual address at which the memory was accessed.
  * @kvm_fd: vCPU file descriptor for KVM.
+ * @work_mutex: Lock to prevent multiple access to queued_work_*.
+ * @queued_work_first: First asynchronous work pending.
  *
  * State of one CPU core or thread.
  */
@@ -262,20 +270,22 @@ struct CPUState {
     uint32_t host_tid;
     bool running;
     struct QemuCond *halt_cond;
-    struct qemu_work_item *queued_work_first, *queued_work_last;
     bool thread_kicked;
     bool created;
     bool stop;
     bool stopped;
-    volatile sig_atomic_t exit_request;
+    bool crash_occurred;
+    bool exit_request;
     uint32_t interrupt_request;
     int singlestep_enabled;
     int64_t icount_extra;
     sigjmp_buf jmp_env;
 
+    QemuMutex work_mutex;
+    struct qemu_work_item *queued_work_first, *queued_work_last;
+
+    CPUAddressSpace *cpu_ases;
     AddressSpace *as;
-    struct AddressSpaceDispatch *memory_dispatch;
-    MemoryListener *tcg_as_listener;
 
     void *env_ptr; /* CPUArchState */
     struct TranslationBlock *current_tb;
@@ -314,11 +324,16 @@ struct CPUState {
     uint32_t can_do_io;
     int32_t exception_index; /* used by m68k TCG */
 
+    /* Used to keep track of an outstanding cpu throttle thread for migration
+     * autoconverge
+     */
+    bool throttle_thread_scheduled;
+
     /* Note that this is accessed at the start of every TB via a negative
        offset from AREG0.  Leave this field at the end so as to make the
        (absolute value) offset as small as possible.  This reduces code
        size, especially for hosts without large memory offsets.  */
-    volatile sig_atomic_t tcg_exit_req;
+    uint32_t tcg_exit_req;
 };
 
 QTAILQ_HEAD(CPUTailQ, CPUState);
@@ -331,8 +346,7 @@ extern struct CPUTailQ cpus;
     QTAILQ_FOREACH_REVERSE(cpu, &cpus, CPUTailQ, node)
 #define first_cpu QTAILQ_FIRST(&cpus)
 
-DECLARE_TLS(CPUState *, current_cpu);
-#define current_cpu tls_var(current_cpu)
+extern __thread CPUState *current_cpu;
 
 /**
  * cpu_paging_enabled:
@@ -559,6 +573,43 @@ CPUState *qemu_get_cpu(int index);
  */
 bool cpu_exists(int64_t id);
 
+/**
+ * cpu_throttle_set:
+ * @new_throttle_pct: Percent of sleep time. Valid range is 1 to 99.
+ *
+ * Throttles all vcpus by forcing them to sleep for the given percentage of
+ * time. A throttle_percentage of 25 corresponds to a 75% duty cycle roughly.
+ * (example: 10ms sleep for every 30ms awake).
+ *
+ * cpu_throttle_set can be called as needed to adjust new_throttle_pct.
+ * Once the throttling starts, it will remain in effect until cpu_throttle_stop
+ * is called.
+ */
+void cpu_throttle_set(int new_throttle_pct);
+
+/**
+ * cpu_throttle_stop:
+ *
+ * Stops the vcpu throttling started by cpu_throttle_set.
+ */
+void cpu_throttle_stop(void);
+
+/**
+ * cpu_throttle_active:
+ *
+ * Returns: %true if the vcpus are currently being throttled, %false otherwise.
+ */
+bool cpu_throttle_active(void);
+
+/**
+ * cpu_throttle_get_percentage:
+ *
+ * Returns the vcpu throttle percentage. See cpu_throttle_set for details.
+ *
+ * Returns: The throttle percentage in range 1 to 99.
+ */
+int cpu_throttle_get_percentage(void);
+
 #ifndef CONFIG_USER_ONLY
 
 typedef void (*CPUInterruptHandler)(CPUState *, int);
@@ -673,6 +724,7 @@ void cpu_single_step(CPUState *cpu, int enabled);
 /* 0x08 currently unused */
 #define BP_GDB                0x10
 #define BP_CPU                0x20
+#define BP_ANY                (BP_GDB | BP_CPU)
 #define BP_WATCHPOINT_HIT_READ 0x40
 #define BP_WATCHPOINT_HIT_WRITE 0x80
 #define BP_WATCHPOINT_HIT (BP_WATCHPOINT_HIT_READ | BP_WATCHPOINT_HIT_WRITE)
@@ -683,6 +735,21 @@ int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags);
 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint);
 void cpu_breakpoint_remove_all(CPUState *cpu, int mask);
 
+/* Return true if PC matches an installed breakpoint.  */
+static inline bool cpu_breakpoint_test(CPUState *cpu, vaddr pc, int mask)
+{
+    CPUBreakpoint *bp;
+
+    if (unlikely(!QTAILQ_EMPTY(&cpu->breakpoints))) {
+        QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
+            if (bp->pc == pc && (bp->flags & mask)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint);
 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr,
diff --git a/include/qom/object.h b/include/qom/object.h
index 807978eec..4509166f6 100644
--- a/include/qom/object.h
+++ b/include/qom/object.h
@@ -344,8 +344,6 @@ typedef struct ObjectProperty
     ObjectPropertyResolve *resolve;
     ObjectPropertyRelease *release;
     void *opaque;
-
-    QTAILQ_ENTRY(ObjectProperty) node;
 } ObjectProperty;
 
 /**
@@ -396,16 +394,13 @@ struct ObjectClass
  * As a result, #Object contains a reference to the objects type as its
  * first member.  This allows identification of the real type of the object at
  * run time.
- *
- * #Object also contains a list of #Interfaces that this object
- * implements.
  */
 struct Object
 {
     /*< private >*/
     ObjectClass *class;
     ObjectFree *free;
-    QTAILQ_HEAD(, ObjectProperty) properties;
+    GHashTable *properties;
     uint32_t ref;
     Object *parent;
 };
@@ -510,16 +505,16 @@ struct TypeInfo
 
 /**
  * OBJECT_CLASS_CHECK:
- * @class: The C type to use for the return value.
- * @obj: A derivative of @type to cast.
- * @name: the QOM typename of @class.
+ * @class_type: The C type to use for the return value.
+ * @class: A derivative class of @class_type to cast.
+ * @name: the QOM typename of @class_type.
  *
  * A type safe version of @object_class_dynamic_cast_assert.  This macro is
  * typically wrapped by each type to perform type safe casts of a class to a
  * specific class type.
  */
-#define OBJECT_CLASS_CHECK(class, obj, name) \
-    ((class *)object_class_dynamic_cast_assert(OBJECT_CLASS(obj), (name), \
+#define OBJECT_CLASS_CHECK(class_type, class, name) \
+    ((class_type *)object_class_dynamic_cast_assert(OBJECT_CLASS(class), (name), \
                                                __FILE__, __LINE__, __func__))
 
 /**
@@ -960,6 +955,55 @@ void object_property_del(Object *obj, const char *name, Error **errp);
 ObjectProperty *object_property_find(Object *obj, const char *name,
                                      Error **errp);
 
+typedef struct ObjectPropertyIterator ObjectPropertyIterator;
+
+/**
+ * object_property_iter_init:
+ * @obj: the object
+ *
+ * Initializes an iterator for traversing all properties
+ * registered against an object instance.
+ *
+ * It is forbidden to modify the property list while iterating,
+ * whether removing or adding properties.
+ *
+ * Typical usage pattern would be
+ *
+ * <example>
+ *   <title>Using object property iterators</title>
+ *   <programlisting>
+ *   ObjectProperty *prop;
+ *   ObjectPropertyIterator *iter;
+ *
+ *   iter = object_property_iter_init(obj);
+ *   while ((prop = object_property_iter_next(iter))) {
+ *     ... do something with prop ...
+ *   }
+ *   object_property_iter_free(iter);
+ *   </programlisting>
+ * </example>
+ *
+ * Returns: the new iterator
+ */
+ObjectPropertyIterator *object_property_iter_init(Object *obj);
+
+/**
+ * object_property_iter_free:
+ * @iter: the iterator instance
+ *
+ * Releases any resources associated with the iterator.
+ */
+void object_property_iter_free(ObjectPropertyIterator *iter);
+
+/**
+ * object_property_iter_next:
+ * @iter: the iterator instance
+ *
+ * Returns: the next property, or %NULL when all properties
+ * have been traversed.
+ */
+ObjectProperty *object_property_iter_next(ObjectPropertyIterator *iter);
+
 void object_unparent(Object *obj);
 
 /**
@@ -1488,12 +1532,33 @@ void object_property_set_description(Object *obj, const char *name,
  * Call @fn passing each child of @obj and @opaque to it, until @fn returns
  * non-zero.
  *
+ * It is forbidden to add or remove children from @obj from the @fn
+ * callback.
+ *
  * Returns: The last value returned by @fn, or 0 if there is no child.
  */
 int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
                          void *opaque);
 
 /**
+ * object_child_foreach_recursive:
+ * @obj: the object whose children will be navigated
+ * @fn: the iterator function to be called
+ * @opaque: an opaque value that will be passed to the iterator
+ *
+ * Call @fn passing each child of @obj and @opaque to it, until @fn returns
+ * non-zero. Calls recursively, all child nodes of @obj will also be passed
+ * all the way down to the leaf nodes of the tree. Depth first ordering.
+ *
+ * It is forbidden to add or remove children from @obj (or its
+ * child nodes) from the @fn callback.
+ *
+ * Returns: The last value returned by @fn, or 0 if there is no child.
+ */
+int object_child_foreach_recursive(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque);
+/**
  * container_get:
  * @root: root of the #path, e.g., object_get_root()
  * @path: path to the container
diff --git a/include/standard-headers/asm-x86/hyperv.h b/include/standard-headers/asm-x86/hyperv.h
new file mode 100644
index 000000000..c37c14e01
--- /dev/null
+++ b/include/standard-headers/asm-x86/hyperv.h
@@ -0,0 +1,260 @@
+#ifndef _ASM_X86_HYPERV_H
+#define _ASM_X86_HYPERV_H
+
+#include "standard-headers/linux/types.h"
+
+/*
+ * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
+ * is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
+ */
+#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
+#define HYPERV_CPUID_INTERFACE			0x40000001
+#define HYPERV_CPUID_VERSION			0x40000002
+#define HYPERV_CPUID_FEATURES			0x40000003
+#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
+#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
+
+#define HYPERV_HYPERVISOR_PRESENT_BIT		0x80000000
+#define HYPERV_CPUID_MIN			0x40000005
+#define HYPERV_CPUID_MAX			0x4000ffff
+
+/*
+ * Feature identification. EAX indicates which features are available
+ * to the partition based upon the current partition privileges.
+ */
+
+/* VP Runtime (HV_X64_MSR_VP_RUNTIME) available */
+#define HV_X64_MSR_VP_RUNTIME_AVAILABLE		(1 << 0)
+/* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
+#define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
+/* Partition reference TSC MSR is available */
+#define HV_X64_MSR_REFERENCE_TSC_AVAILABLE              (1 << 9)
+
+/* A partition's reference time stamp counter (TSC) page */
+#define HV_X64_MSR_REFERENCE_TSC		0x40000021
+
+/*
+ * There is a single feature flag that signifies the presence of the MSR
+ * that can be used to retrieve both the local APIC Timer frequency as
+ * well as the TSC frequency.
+ */
+
+/* Local APIC timer frequency MSR (HV_X64_MSR_APIC_FREQUENCY) is available */
+#define HV_X64_MSR_APIC_FREQUENCY_AVAILABLE (1 << 11)
+
+/* TSC frequency MSR (HV_X64_MSR_TSC_FREQUENCY) is available */
+#define HV_X64_MSR_TSC_FREQUENCY_AVAILABLE (1 << 11)
+
+/*
+ * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
+ * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available
+ */
+#define HV_X64_MSR_SYNIC_AVAILABLE		(1 << 2)
+/*
+ * Synthetic Timer MSRs (HV_X64_MSR_STIMER0_CONFIG through
+ * HV_X64_MSR_STIMER3_COUNT) available
+ */
+#define HV_X64_MSR_SYNTIMER_AVAILABLE		(1 << 3)
+/*
+ * APIC access MSRs (HV_X64_MSR_EOI, HV_X64_MSR_ICR and HV_X64_MSR_TPR)
+ * are available
+ */
+#define HV_X64_MSR_APIC_ACCESS_AVAILABLE	(1 << 4)
+/* Hypercall MSRs (HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL) available*/
+#define HV_X64_MSR_HYPERCALL_AVAILABLE		(1 << 5)
+/* Access virtual processor index MSR (HV_X64_MSR_VP_INDEX) available*/
+#define HV_X64_MSR_VP_INDEX_AVAILABLE		(1 << 6)
+/* Virtual system reset MSR (HV_X64_MSR_RESET) is available*/
+#define HV_X64_MSR_RESET_AVAILABLE		(1 << 7)
+ /*
+  * Access statistics pages MSRs (HV_X64_MSR_STATS_PARTITION_RETAIL_PAGE,
+  * HV_X64_MSR_STATS_PARTITION_INTERNAL_PAGE, HV_X64_MSR_STATS_VP_RETAIL_PAGE,
+  * HV_X64_MSR_STATS_VP_INTERNAL_PAGE) available
+  */
+#define HV_X64_MSR_STAT_PAGES_AVAILABLE		(1 << 8)
+
+/*
+ * Feature identification: EBX indicates which flags were specified at
+ * partition creation. The format is the same as the partition creation
+ * flag structure defined in section Partition Creation Flags.
+ */
+#define HV_X64_CREATE_PARTITIONS		(1 << 0)
+#define HV_X64_ACCESS_PARTITION_ID		(1 << 1)
+#define HV_X64_ACCESS_MEMORY_POOL		(1 << 2)
+#define HV_X64_ADJUST_MESSAGE_BUFFERS		(1 << 3)
+#define HV_X64_POST_MESSAGES			(1 << 4)
+#define HV_X64_SIGNAL_EVENTS			(1 << 5)
+#define HV_X64_CREATE_PORT			(1 << 6)
+#define HV_X64_CONNECT_PORT			(1 << 7)
+#define HV_X64_ACCESS_STATS			(1 << 8)
+#define HV_X64_DEBUGGING			(1 << 11)
+#define HV_X64_CPU_POWER_MANAGEMENT		(1 << 12)
+#define HV_X64_CONFIGURE_PROFILER		(1 << 13)
+
+/*
+ * Feature identification. EDX indicates which miscellaneous features
+ * are available to the partition.
+ */
+/* The MWAIT instruction is available (per section MONITOR / MWAIT) */
+#define HV_X64_MWAIT_AVAILABLE				(1 << 0)
+/* Guest debugging support is available */
+#define HV_X64_GUEST_DEBUGGING_AVAILABLE		(1 << 1)
+/* Performance Monitor support is available*/
+#define HV_X64_PERF_MONITOR_AVAILABLE			(1 << 2)
+/* Support for physical CPU dynamic partitioning events is available*/
+#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	(1 << 3)
+/*
+ * Support for passing hypercall input parameter block via XMM
+ * registers is available
+ */
+#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE		(1 << 4)
+/* Support for a virtual guest idle state is available */
+#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		(1 << 5)
+/* Guest crash data handler available */
+#define HV_X64_GUEST_CRASH_MSR_AVAILABLE		(1 << 10)
+
+/*
+ * Implementation recommendations. Indicates which behaviors the hypervisor
+ * recommends the OS implement for optimal performance.
+ */
+ /*
+  * Recommend using hypercall for address space switches rather
+  * than MOV to CR3 instruction
+  */
+#define HV_X64_MWAIT_RECOMMENDED		(1 << 0)
+/* Recommend using hypercall for local TLB flushes rather
+ * than INVLPG or MOV to CR3 instructions */
+#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED	(1 << 1)
+/*
+ * Recommend using hypercall for remote TLB flushes rather
+ * than inter-processor interrupts
+ */
+#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED	(1 << 2)
+/*
+ * Recommend using MSRs for accessing APIC registers
+ * EOI, ICR and TPR rather than their memory-mapped counterparts
+ */
+#define HV_X64_APIC_ACCESS_RECOMMENDED		(1 << 3)
+/* Recommend using the hypervisor-provided MSR to initiate a system RESET */
+#define HV_X64_SYSTEM_RESET_RECOMMENDED		(1 << 4)
+/*
+ * Recommend using relaxed timing for this partition. If used,
+ * the VM should disable any watchdog timeouts that rely on the
+ * timely delivery of external interrupts
+ */
+#define HV_X64_RELAXED_TIMING_RECOMMENDED	(1 << 5)
+
+/* MSR used to identify the guest OS. */
+#define HV_X64_MSR_GUEST_OS_ID			0x40000000
+
+/* MSR used to setup pages used to communicate with the hypervisor. */
+#define HV_X64_MSR_HYPERCALL			0x40000001
+
+/* MSR used to provide vcpu index */
+#define HV_X64_MSR_VP_INDEX			0x40000002
+
+/* MSR used to reset the guest OS. */
+#define HV_X64_MSR_RESET			0x40000003
+
+/* MSR used to provide vcpu runtime in 100ns units */
+#define HV_X64_MSR_VP_RUNTIME			0x40000010
+
+/* MSR used to read the per-partition time reference counter */
+#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
+
+/* MSR used to retrieve the TSC frequency */
+#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
+
+/* MSR used to retrieve the local APIC timer frequency */
+#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
+
+/* Define the virtual APIC registers */
+#define HV_X64_MSR_EOI				0x40000070
+#define HV_X64_MSR_ICR				0x40000071
+#define HV_X64_MSR_TPR				0x40000072
+#define HV_X64_MSR_APIC_ASSIST_PAGE		0x40000073
+
+/* Define synthetic interrupt controller model specific registers. */
+#define HV_X64_MSR_SCONTROL			0x40000080
+#define HV_X64_MSR_SVERSION			0x40000081
+#define HV_X64_MSR_SIEFP			0x40000082
+#define HV_X64_MSR_SIMP				0x40000083
+#define HV_X64_MSR_EOM				0x40000084
+#define HV_X64_MSR_SINT0			0x40000090
+#define HV_X64_MSR_SINT1			0x40000091
+#define HV_X64_MSR_SINT2			0x40000092
+#define HV_X64_MSR_SINT3			0x40000093
+#define HV_X64_MSR_SINT4			0x40000094
+#define HV_X64_MSR_SINT5			0x40000095
+#define HV_X64_MSR_SINT6			0x40000096
+#define HV_X64_MSR_SINT7			0x40000097
+#define HV_X64_MSR_SINT8			0x40000098
+#define HV_X64_MSR_SINT9			0x40000099
+#define HV_X64_MSR_SINT10			0x4000009A
+#define HV_X64_MSR_SINT11			0x4000009B
+#define HV_X64_MSR_SINT12			0x4000009C
+#define HV_X64_MSR_SINT13			0x4000009D
+#define HV_X64_MSR_SINT14			0x4000009E
+#define HV_X64_MSR_SINT15			0x4000009F
+
+/*
+ * Synthetic Timer MSRs. Four timers per vcpu.
+ */
+#define HV_X64_MSR_STIMER0_CONFIG		0x400000B0
+#define HV_X64_MSR_STIMER0_COUNT		0x400000B1
+#define HV_X64_MSR_STIMER1_CONFIG		0x400000B2
+#define HV_X64_MSR_STIMER1_COUNT		0x400000B3
+#define HV_X64_MSR_STIMER2_CONFIG		0x400000B4
+#define HV_X64_MSR_STIMER2_COUNT		0x400000B5
+#define HV_X64_MSR_STIMER3_CONFIG		0x400000B6
+#define HV_X64_MSR_STIMER3_COUNT		0x400000B7
+
+/* Hyper-V guest crash notification MSR's */
+#define HV_X64_MSR_CRASH_P0			0x40000100
+#define HV_X64_MSR_CRASH_P1			0x40000101
+#define HV_X64_MSR_CRASH_P2			0x40000102
+#define HV_X64_MSR_CRASH_P3			0x40000103
+#define HV_X64_MSR_CRASH_P4			0x40000104
+#define HV_X64_MSR_CRASH_CTL			0x40000105
+#define HV_X64_MSR_CRASH_CTL_NOTIFY		(1ULL << 63)
+#define HV_X64_MSR_CRASH_PARAMS		\
+		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
+
+#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
+		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
+
+/* Declare the various hypercall operations. */
+#define HV_X64_HV_NOTIFY_LONG_SPIN_WAIT		0x0008
+
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE		0x00000001
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT	12
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
+		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
+
+#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
+#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
+
+#define HV_PROCESSOR_POWER_STATE_C0		0
+#define HV_PROCESSOR_POWER_STATE_C1		1
+#define HV_PROCESSOR_POWER_STATE_C2		2
+#define HV_PROCESSOR_POWER_STATE_C3		3
+
+/* hypercall status code */
+#define HV_STATUS_SUCCESS			0
+#define HV_STATUS_INVALID_HYPERCALL_CODE	2
+#define HV_STATUS_INVALID_HYPERCALL_INPUT	3
+#define HV_STATUS_INVALID_ALIGNMENT		4
+#define HV_STATUS_INSUFFICIENT_MEMORY		11
+#define HV_STATUS_INVALID_CONNECTION_ID		18
+#define HV_STATUS_INSUFFICIENT_BUFFERS		19
+
+typedef struct _HV_REFERENCE_TSC_PAGE {
+	uint32_t tsc_sequence;
+	uint32_t res1;
+	uint64_t tsc_scale;
+	int64_t tsc_offset;
+} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
+
+#endif
diff --git a/include/standard-headers/linux/pci_regs.h b/include/standard-headers/linux/pci_regs.h
index 57e8c80c3..413417f37 100644
--- a/include/standard-headers/linux/pci_regs.h
+++ b/include/standard-headers/linux/pci_regs.h
@@ -13,10 +13,10 @@
  *	PCI to PCI Bridge Specification
  *	PCI System Design Guide
  *
- * 	For hypertransport information, please consult the following manuals
- * 	from http://www.hypertransport.org
+ *	For HyperTransport information, please consult the following manuals
+ *	from http://www.hypertransport.org
  *
- *	The Hypertransport I/O Link Specification
+ *	The HyperTransport I/O Link Specification
  */
 
 #ifndef LINUX_PCI_REGS_H
@@ -26,6 +26,7 @@
  * Under PCI, each device has 256 bytes of configuration address space,
  * of which the first 64 bytes are standardized as follows:
  */
+#define PCI_STD_HEADER_SIZEOF	64
 #define PCI_VENDOR_ID		0x00	/* 16 bits */
 #define PCI_DEVICE_ID		0x02	/* 16 bits */
 #define PCI_COMMAND		0x04	/* 16 bits */
@@ -36,7 +37,7 @@
 #define  PCI_COMMAND_INVALIDATE	0x10	/* Use memory write and invalidate */
 #define  PCI_COMMAND_VGA_PALETTE 0x20	/* Enable palette snooping */
 #define  PCI_COMMAND_PARITY	0x40	/* Enable parity checking */
-#define  PCI_COMMAND_WAIT 	0x80	/* Enable address/data stepping */
+#define  PCI_COMMAND_WAIT	0x80	/* Enable address/data stepping */
 #define  PCI_COMMAND_SERR	0x100	/* Enable SERR */
 #define  PCI_COMMAND_FAST_BACK	0x200	/* Enable back-to-back writes */
 #define  PCI_COMMAND_INTX_DISABLE 0x400 /* INTx Emulation Disable */
@@ -44,7 +45,7 @@
 #define PCI_STATUS		0x06	/* 16 bits */
 #define  PCI_STATUS_INTERRUPT	0x08	/* Interrupt status */
 #define  PCI_STATUS_CAP_LIST	0x10	/* Support Capability List */
-#define  PCI_STATUS_66MHZ	0x20	/* Support 66 Mhz PCI 2.1 bus */
+#define  PCI_STATUS_66MHZ	0x20	/* Support 66 MHz PCI 2.1 bus */
 #define  PCI_STATUS_UDF		0x40	/* Support User Definable Features [obsolete] */
 #define  PCI_STATUS_FAST_BACK	0x80	/* Accept fast-back to back */
 #define  PCI_STATUS_PARITY	0x100	/* Detected parity error */
@@ -125,7 +126,8 @@
 #define  PCI_IO_RANGE_TYPE_MASK	0x0fUL	/* I/O bridging type */
 #define  PCI_IO_RANGE_TYPE_16	0x00
 #define  PCI_IO_RANGE_TYPE_32	0x01
-#define  PCI_IO_RANGE_MASK	(~0x0fUL)
+#define  PCI_IO_RANGE_MASK	(~0x0fUL) /* Standard 4K I/O windows */
+#define  PCI_IO_1K_RANGE_MASK	(~0x03UL) /* Intel 1K I/O windows */
 #define PCI_SEC_STATUS		0x1e	/* Secondary status register, only bit 14 used */
 #define PCI_MEMORY_BASE		0x20	/* Memory range behind */
 #define PCI_MEMORY_LIMIT	0x22
@@ -203,16 +205,18 @@
 #define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
 #define  PCI_CAP_ID_PCIX	0x07	/* PCI-X */
 #define  PCI_CAP_ID_HT		0x08	/* HyperTransport */
-#define  PCI_CAP_ID_VNDR	0x09	/* Vendor specific */
+#define  PCI_CAP_ID_VNDR	0x09	/* Vendor-Specific */
 #define  PCI_CAP_ID_DBG		0x0A	/* Debug port */
 #define  PCI_CAP_ID_CCRC	0x0B	/* CompactPCI Central Resource Control */
-#define  PCI_CAP_ID_SHPC 	0x0C	/* PCI Standard Hot-Plug Controller */
+#define  PCI_CAP_ID_SHPC	0x0C	/* PCI Standard Hot-Plug Controller */
 #define  PCI_CAP_ID_SSVID	0x0D	/* Bridge subsystem vendor/device ID */
 #define  PCI_CAP_ID_AGP3	0x0E	/* AGP Target PCI-PCI bridge */
-#define  PCI_CAP_ID_EXP 	0x10	/* PCI Express */
+#define  PCI_CAP_ID_SECDEV	0x0F	/* Secure Device */
+#define  PCI_CAP_ID_EXP		0x10	/* PCI Express */
 #define  PCI_CAP_ID_MSIX	0x11	/* MSI-X */
-#define  PCI_CAP_ID_SATA	0x12	/* Serial ATA */
+#define  PCI_CAP_ID_SATA	0x12	/* SATA Data/Index Conf. */
 #define  PCI_CAP_ID_AF		0x13	/* PCI Advanced Features */
+#define  PCI_CAP_ID_MAX		PCI_CAP_ID_AF
 #define PCI_CAP_LIST_NEXT	1	/* Next capability in the list */
 #define PCI_CAP_FLAGS		2	/* Capability defined flags (16 bits) */
 #define PCI_CAP_SIZEOF		4
@@ -264,8 +268,8 @@
 #define  PCI_AGP_COMMAND_RQ_MASK 0xff000000  /* Master: Maximum number of requests */
 #define  PCI_AGP_COMMAND_SBA	0x0200	/* Sideband addressing enabled */
 #define  PCI_AGP_COMMAND_AGP	0x0100	/* Allow processing of AGP transactions */
-#define  PCI_AGP_COMMAND_64BIT	0x0020 	/* Allow processing of 64-bit addresses */
-#define  PCI_AGP_COMMAND_FW	0x0010 	/* Force FW transfers */
+#define  PCI_AGP_COMMAND_64BIT	0x0020	/* Allow processing of 64-bit addresses */
+#define  PCI_AGP_COMMAND_FW	0x0010	/* Force FW transfers */
 #define  PCI_AGP_COMMAND_RATE4	0x0004	/* Use 4x rate */
 #define  PCI_AGP_COMMAND_RATE2	0x0002	/* Use 2x rate */
 #define  PCI_AGP_COMMAND_RATE1	0x0001	/* Use 1x rate */
@@ -277,6 +281,7 @@
 #define  PCI_VPD_ADDR_MASK	0x7fff	/* Address mask */
 #define  PCI_VPD_ADDR_F		0x8000	/* Write 0, 1 indicates completion */
 #define PCI_VPD_DATA		4	/* 32-bits of data returned here */
+#define PCI_CAP_VPD_SIZEOF	8
 
 /* Slot Identification */
 
@@ -287,32 +292,37 @@
 
 /* Message Signalled Interrupts registers */
 
-#define PCI_MSI_FLAGS		2	/* Various flags */
-#define  PCI_MSI_FLAGS_64BIT	0x80	/* 64-bit addresses allowed */
-#define  PCI_MSI_FLAGS_QSIZE	0x70	/* Message queue size configured */
-#define  PCI_MSI_FLAGS_QMASK	0x0e	/* Maximum queue size available */
-#define  PCI_MSI_FLAGS_ENABLE	0x01	/* MSI feature enabled */
-#define  PCI_MSI_FLAGS_MASKBIT	0x100	/* 64-bit mask bits allowed */
+#define PCI_MSI_FLAGS		2	/* Message Control */
+#define  PCI_MSI_FLAGS_ENABLE	0x0001	/* MSI feature enabled */
+#define  PCI_MSI_FLAGS_QMASK	0x000e	/* Maximum queue size available */
+#define  PCI_MSI_FLAGS_QSIZE	0x0070	/* Message queue size configured */
+#define  PCI_MSI_FLAGS_64BIT	0x0080	/* 64-bit addresses allowed */
+#define  PCI_MSI_FLAGS_MASKBIT	0x0100	/* Per-vector masking capable */
 #define PCI_MSI_RFU		3	/* Rest of capability flags */
 #define PCI_MSI_ADDRESS_LO	4	/* Lower 32 bits */
 #define PCI_MSI_ADDRESS_HI	8	/* Upper 32 bits (if PCI_MSI_FLAGS_64BIT set) */
 #define PCI_MSI_DATA_32		8	/* 16 bits of data for 32-bit devices */
 #define PCI_MSI_MASK_32		12	/* Mask bits register for 32-bit devices */
-#define PCI_MSI_PENDING_32	16	/* Pending bits register for 32-bit devices */
+#define PCI_MSI_PENDING_32	16	/* Pending intrs for 32-bit devices */
 #define PCI_MSI_DATA_64		12	/* 16 bits of data for 64-bit devices */
 #define PCI_MSI_MASK_64		16	/* Mask bits register for 64-bit devices */
-#define PCI_MSI_PENDING_64	20	/* Pending bits register for 32-bit devices */
+#define PCI_MSI_PENDING_64	20	/* Pending intrs for 64-bit devices */
 
 /* MSI-X registers */
-#define PCI_MSIX_FLAGS		2
-#define  PCI_MSIX_FLAGS_QSIZE	0x7FF
-#define  PCI_MSIX_FLAGS_ENABLE	(1 << 15)
-#define  PCI_MSIX_FLAGS_MASKALL	(1 << 14)
-#define PCI_MSIX_TABLE		4
-#define PCI_MSIX_PBA		8
-#define  PCI_MSIX_FLAGS_BIRMASK	(7 << 0)
-
-/* MSI-X entry's format */
+#define PCI_MSIX_FLAGS		2	/* Message Control */
+#define  PCI_MSIX_FLAGS_QSIZE	0x07FF	/* Table size */
+#define  PCI_MSIX_FLAGS_MASKALL	0x4000	/* Mask all vectors for this function */
+#define  PCI_MSIX_FLAGS_ENABLE	0x8000	/* MSI-X enable */
+#define PCI_MSIX_TABLE		4	/* Table offset */
+#define  PCI_MSIX_TABLE_BIR	0x00000007 /* BAR index */
+#define  PCI_MSIX_TABLE_OFFSET	0xfffffff8 /* Offset into specified BAR */
+#define PCI_MSIX_PBA		8	/* Pending Bit Array offset */
+#define  PCI_MSIX_PBA_BIR	0x00000007 /* BAR index */
+#define  PCI_MSIX_PBA_OFFSET	0xfffffff8 /* Offset into specified BAR */
+#define PCI_MSIX_FLAGS_BIRMASK	PCI_MSIX_PBA_BIR /* deprecated */
+#define PCI_CAP_MSIX_SIZEOF	12	/* size of MSIX registers */
+
+/* MSI-X Table entry format */
 #define PCI_MSIX_ENTRY_SIZE		16
 #define  PCI_MSIX_ENTRY_LOWER_ADDR	0
 #define  PCI_MSIX_ENTRY_UPPER_ADDR	4
@@ -341,8 +351,9 @@
 #define  PCI_AF_CTRL_FLR	0x01
 #define PCI_AF_STATUS		5
 #define  PCI_AF_STATUS_TP	0x01
+#define PCI_CAP_AF_SIZEOF	6	/* size of AF registers */
 
-/* PCI-X registers */
+/* PCI-X registers (Type 0 (non-bridge) devices) */
 
 #define PCI_X_CMD		2	/* Modes & Features */
 #define  PCI_X_CMD_DPERR_E	0x0001	/* Data Parity Error Recovery Enable */
@@ -362,7 +373,7 @@
 #define  PCI_X_CMD_SPLIT_16	0x0060	/* Max 16 */
 #define  PCI_X_CMD_SPLIT_32	0x0070	/* Max 32 */
 #define  PCI_X_CMD_MAX_SPLIT	0x0070	/* Max Outstanding Split Transactions */
-#define  PCI_X_CMD_VERSION(x) 	(((x) >> 12) & 3) /* Version */
+#define  PCI_X_CMD_VERSION(x)	(((x) >> 12) & 3) /* Version */
 #define PCI_X_STATUS		4	/* PCI-X capabilities */
 #define  PCI_X_STATUS_DEVFN	0x000000ff	/* A copy of devfn */
 #define  PCI_X_STATUS_BUS	0x0000ff00	/* A copy of bus nr */
@@ -377,11 +388,28 @@
 #define  PCI_X_STATUS_SPL_ERR	0x20000000	/* Rcvd Split Completion Error Msg */
 #define  PCI_X_STATUS_266MHZ	0x40000000	/* 266 MHz capable */
 #define  PCI_X_STATUS_533MHZ	0x80000000	/* 533 MHz capable */
+#define PCI_X_ECC_CSR		8	/* ECC control and status */
+#define PCI_CAP_PCIX_SIZEOF_V0	8	/* size of registers for Version 0 */
+#define PCI_CAP_PCIX_SIZEOF_V1	24	/* size for Version 1 */
+#define PCI_CAP_PCIX_SIZEOF_V2	PCI_CAP_PCIX_SIZEOF_V1	/* Same for v2 */
+
+/* PCI-X registers (Type 1 (bridge) devices) */
+
+#define PCI_X_BRIDGE_SSTATUS	2	/* Secondary Status */
+#define  PCI_X_SSTATUS_64BIT	0x0001	/* Secondary AD interface is 64 bits */
+#define  PCI_X_SSTATUS_133MHZ	0x0002	/* 133 MHz capable */
+#define  PCI_X_SSTATUS_FREQ	0x03c0	/* Secondary Bus Mode and Frequency */
+#define  PCI_X_SSTATUS_VERS	0x3000	/* PCI-X Capability Version */
+#define  PCI_X_SSTATUS_V1	0x1000	/* Mode 2, not Mode 1 */
+#define  PCI_X_SSTATUS_V2	0x2000	/* Mode 1 or Modes 1 and 2 */
+#define  PCI_X_SSTATUS_266MHZ	0x4000	/* 266 MHz capable */
+#define  PCI_X_SSTATUS_533MHZ	0x8000	/* 533 MHz capable */
+#define PCI_X_BRIDGE_STATUS	4	/* Bridge Status */
 
 /* PCI Bridge Subsystem ID registers */
 
-#define PCI_SSVID_VENDOR_ID     4	/* PCI-Bridge subsystem vendor id register */
-#define PCI_SSVID_DEVICE_ID     6	/* PCI-Bridge subsystem device id register */
+#define PCI_SSVID_VENDOR_ID     4	/* PCI Bridge subsystem vendor ID */
+#define PCI_SSVID_DEVICE_ID     6	/* PCI Bridge subsystem device ID */
 
 /* PCI Express capability registers */
 
@@ -393,24 +421,24 @@
 #define  PCI_EXP_TYPE_ROOT_PORT 0x4	/* Root Port */
 #define  PCI_EXP_TYPE_UPSTREAM	0x5	/* Upstream Port */
 #define  PCI_EXP_TYPE_DOWNSTREAM 0x6	/* Downstream Port */
-#define  PCI_EXP_TYPE_PCI_BRIDGE 0x7	/* PCI/PCI-X Bridge */
-#define  PCI_EXP_TYPE_PCIE_BRIDGE 0x8   /* PCI/PCI-X to PCIE Bridge */
+#define  PCI_EXP_TYPE_PCI_BRIDGE 0x7	/* PCIe to PCI/PCI-X Bridge */
+#define  PCI_EXP_TYPE_PCIE_BRIDGE 0x8	/* PCI/PCI-X to PCIe Bridge */
 #define  PCI_EXP_TYPE_RC_END	0x9	/* Root Complex Integrated Endpoint */
-#define  PCI_EXP_TYPE_RC_EC     0xa     /* Root Complex Event Collector */
+#define  PCI_EXP_TYPE_RC_EC	0xa	/* Root Complex Event Collector */
 #define PCI_EXP_FLAGS_SLOT	0x0100	/* Slot implemented */
 #define PCI_EXP_FLAGS_IRQ	0x3e00	/* Interrupt message number */
 #define PCI_EXP_DEVCAP		4	/* Device capabilities */
-#define  PCI_EXP_DEVCAP_PAYLOAD	0x07	/* Max_Payload_Size */
-#define  PCI_EXP_DEVCAP_PHANTOM	0x18	/* Phantom functions */
-#define  PCI_EXP_DEVCAP_EXT_TAG	0x20	/* Extended tags */
-#define  PCI_EXP_DEVCAP_L0S	0x1c0	/* L0s Acceptable Latency */
-#define  PCI_EXP_DEVCAP_L1	0xe00	/* L1 Acceptable Latency */
-#define  PCI_EXP_DEVCAP_ATN_BUT	0x1000	/* Attention Button Present */
-#define  PCI_EXP_DEVCAP_ATN_IND	0x2000	/* Attention Indicator Present */
-#define  PCI_EXP_DEVCAP_PWR_IND	0x4000	/* Power Indicator Present */
-#define  PCI_EXP_DEVCAP_RBER	0x8000	/* Role-Based Error Reporting */
-#define  PCI_EXP_DEVCAP_PWR_VAL	0x3fc0000 /* Slot Power Limit Value */
-#define  PCI_EXP_DEVCAP_PWR_SCL	0xc000000 /* Slot Power Limit Scale */
+#define  PCI_EXP_DEVCAP_PAYLOAD	0x00000007 /* Max_Payload_Size */
+#define  PCI_EXP_DEVCAP_PHANTOM	0x00000018 /* Phantom functions */
+#define  PCI_EXP_DEVCAP_EXT_TAG	0x00000020 /* Extended tags */
+#define  PCI_EXP_DEVCAP_L0S	0x000001c0 /* L0s Acceptable Latency */
+#define  PCI_EXP_DEVCAP_L1	0x00000e00 /* L1 Acceptable Latency */
+#define  PCI_EXP_DEVCAP_ATN_BUT	0x00001000 /* Attention Button Present */
+#define  PCI_EXP_DEVCAP_ATN_IND	0x00002000 /* Attention Indicator Present */
+#define  PCI_EXP_DEVCAP_PWR_IND	0x00004000 /* Power Indicator Present */
+#define  PCI_EXP_DEVCAP_RBER	0x00008000 /* Role-Based Error Reporting */
+#define  PCI_EXP_DEVCAP_PWR_VAL	0x03fc0000 /* Slot Power Limit Value */
+#define  PCI_EXP_DEVCAP_PWR_SCL	0x0c000000 /* Slot Power Limit Scale */
 #define  PCI_EXP_DEVCAP_FLR     0x10000000 /* Function Level Reset */
 #define PCI_EXP_DEVCTL		8	/* Device Control */
 #define  PCI_EXP_DEVCTL_CERE	0x0001	/* Correctable Error Reporting En. */
@@ -424,47 +452,61 @@
 #define  PCI_EXP_DEVCTL_AUX_PME	0x0400	/* Auxiliary Power PM Enable */
 #define  PCI_EXP_DEVCTL_NOSNOOP_EN 0x0800  /* Enable No Snoop */
 #define  PCI_EXP_DEVCTL_READRQ	0x7000	/* Max_Read_Request_Size */
+#define  PCI_EXP_DEVCTL_READRQ_128B  0x0000 /* 128 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_256B  0x1000 /* 256 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_512B  0x2000 /* 512 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_1024B 0x3000 /* 1024 Bytes */
 #define  PCI_EXP_DEVCTL_BCR_FLR 0x8000  /* Bridge Configuration Retry / FLR */
 #define PCI_EXP_DEVSTA		10	/* Device Status */
-#define  PCI_EXP_DEVSTA_CED	0x01	/* Correctable Error Detected */
-#define  PCI_EXP_DEVSTA_NFED	0x02	/* Non-Fatal Error Detected */
-#define  PCI_EXP_DEVSTA_FED	0x04	/* Fatal Error Detected */
-#define  PCI_EXP_DEVSTA_URD	0x08	/* Unsupported Request Detected */
-#define  PCI_EXP_DEVSTA_AUXPD	0x10	/* AUX Power Detected */
-#define  PCI_EXP_DEVSTA_TRPND	0x20	/* Transactions Pending */
+#define  PCI_EXP_DEVSTA_CED	0x0001	/* Correctable Error Detected */
+#define  PCI_EXP_DEVSTA_NFED	0x0002	/* Non-Fatal Error Detected */
+#define  PCI_EXP_DEVSTA_FED	0x0004	/* Fatal Error Detected */
+#define  PCI_EXP_DEVSTA_URD	0x0008	/* Unsupported Request Detected */
+#define  PCI_EXP_DEVSTA_AUXPD	0x0010	/* AUX Power Detected */
+#define  PCI_EXP_DEVSTA_TRPND	0x0020	/* Transactions Pending */
 #define PCI_EXP_LNKCAP		12	/* Link Capabilities */
 #define  PCI_EXP_LNKCAP_SLS	0x0000000f /* Supported Link Speeds */
+#define  PCI_EXP_LNKCAP_SLS_2_5GB 0x00000001 /* LNKCAP2 SLS Vector bit 0 */
+#define  PCI_EXP_LNKCAP_SLS_5_0GB 0x00000002 /* LNKCAP2 SLS Vector bit 1 */
 #define  PCI_EXP_LNKCAP_MLW	0x000003f0 /* Maximum Link Width */
 #define  PCI_EXP_LNKCAP_ASPMS	0x00000c00 /* ASPM Support */
 #define  PCI_EXP_LNKCAP_L0SEL	0x00007000 /* L0s Exit Latency */
 #define  PCI_EXP_LNKCAP_L1EL	0x00038000 /* L1 Exit Latency */
-#define  PCI_EXP_LNKCAP_CLKPM	0x00040000 /* L1 Clock Power Management */
+#define  PCI_EXP_LNKCAP_CLKPM	0x00040000 /* Clock Power Management */
 #define  PCI_EXP_LNKCAP_SDERC	0x00080000 /* Surprise Down Error Reporting Capable */
 #define  PCI_EXP_LNKCAP_DLLLARC	0x00100000 /* Data Link Layer Link Active Reporting Capable */
 #define  PCI_EXP_LNKCAP_LBNC	0x00200000 /* Link Bandwidth Notification Capability */
 #define  PCI_EXP_LNKCAP_PN	0xff000000 /* Port Number */
 #define PCI_EXP_LNKCTL		16	/* Link Control */
 #define  PCI_EXP_LNKCTL_ASPMC	0x0003	/* ASPM Control */
+#define  PCI_EXP_LNKCTL_ASPM_L0S 0x0001	/* L0s Enable */
+#define  PCI_EXP_LNKCTL_ASPM_L1  0x0002	/* L1 Enable */
 #define  PCI_EXP_LNKCTL_RCB	0x0008	/* Read Completion Boundary */
 #define  PCI_EXP_LNKCTL_LD	0x0010	/* Link Disable */
 #define  PCI_EXP_LNKCTL_RL	0x0020	/* Retrain Link */
 #define  PCI_EXP_LNKCTL_CCC	0x0040	/* Common Clock Configuration */
 #define  PCI_EXP_LNKCTL_ES	0x0080	/* Extended Synch */
-#define  PCI_EXP_LNKCTL_CLKREQ_EN 0x100	/* Enable clkreq */
+#define  PCI_EXP_LNKCTL_CLKREQ_EN 0x0100 /* Enable clkreq */
 #define  PCI_EXP_LNKCTL_HAWD	0x0200	/* Hardware Autonomous Width Disable */
 #define  PCI_EXP_LNKCTL_LBMIE	0x0400	/* Link Bandwidth Management Interrupt Enable */
-#define  PCI_EXP_LNKCTL_LABIE	0x0800	/* Lnk Autonomous Bandwidth Interrupt Enable */
+#define  PCI_EXP_LNKCTL_LABIE	0x0800	/* Link Autonomous Bandwidth Interrupt Enable */
 #define PCI_EXP_LNKSTA		18	/* Link Status */
 #define  PCI_EXP_LNKSTA_CLS	0x000f	/* Current Link Speed */
-#define  PCI_EXP_LNKSTA_CLS_2_5GB 0x01	/* Current Link Speed 2.5GT/s */
-#define  PCI_EXP_LNKSTA_CLS_5_0GB 0x02	/* Current Link Speed 5.0GT/s */
-#define  PCI_EXP_LNKSTA_NLW	0x03f0	/* Nogotiated Link Width */
+#define  PCI_EXP_LNKSTA_CLS_2_5GB 0x0001 /* Current Link Speed 2.5GT/s */
+#define  PCI_EXP_LNKSTA_CLS_5_0GB 0x0002 /* Current Link Speed 5.0GT/s */
+#define  PCI_EXP_LNKSTA_CLS_8_0GB 0x0003 /* Current Link Speed 8.0GT/s */
+#define  PCI_EXP_LNKSTA_NLW	0x03f0	/* Negotiated Link Width */
+#define  PCI_EXP_LNKSTA_NLW_X1	0x0010	/* Current Link Width x1 */
+#define  PCI_EXP_LNKSTA_NLW_X2	0x0020	/* Current Link Width x2 */
+#define  PCI_EXP_LNKSTA_NLW_X4	0x0040	/* Current Link Width x4 */
+#define  PCI_EXP_LNKSTA_NLW_X8	0x0080	/* Current Link Width x8 */
 #define  PCI_EXP_LNKSTA_NLW_SHIFT 4	/* start of NLW mask in link status */
 #define  PCI_EXP_LNKSTA_LT	0x0800	/* Link Training */
 #define  PCI_EXP_LNKSTA_SLC	0x1000	/* Slot Clock Configuration */
 #define  PCI_EXP_LNKSTA_DLLLA	0x2000	/* Data Link Layer Link Active */
 #define  PCI_EXP_LNKSTA_LBMS	0x4000	/* Link Bandwidth Management Status */
 #define  PCI_EXP_LNKSTA_LABS	0x8000	/* Link Autonomous Bandwidth Status */
+#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V1	20	/* v1 endpoints end here */
 #define PCI_EXP_SLTCAP		20	/* Slot Capabilities */
 #define  PCI_EXP_SLTCAP_ABP	0x00000001 /* Attention Button Present */
 #define  PCI_EXP_SLTCAP_PCP	0x00000002 /* Power Controller Present */
@@ -486,8 +528,16 @@
 #define  PCI_EXP_SLTCTL_CCIE	0x0010	/* Command Completed Interrupt Enable */
 #define  PCI_EXP_SLTCTL_HPIE	0x0020	/* Hot-Plug Interrupt Enable */
 #define  PCI_EXP_SLTCTL_AIC	0x00c0	/* Attention Indicator Control */
+#define  PCI_EXP_SLTCTL_ATTN_IND_ON    0x0040 /* Attention Indicator on */
+#define  PCI_EXP_SLTCTL_ATTN_IND_BLINK 0x0080 /* Attention Indicator blinking */
+#define  PCI_EXP_SLTCTL_ATTN_IND_OFF   0x00c0 /* Attention Indicator off */
 #define  PCI_EXP_SLTCTL_PIC	0x0300	/* Power Indicator Control */
+#define  PCI_EXP_SLTCTL_PWR_IND_ON     0x0100 /* Power Indicator on */
+#define  PCI_EXP_SLTCTL_PWR_IND_BLINK  0x0200 /* Power Indicator blinking */
+#define  PCI_EXP_SLTCTL_PWR_IND_OFF    0x0300 /* Power Indicator off */
 #define  PCI_EXP_SLTCTL_PCC	0x0400	/* Power Controller Control */
+#define  PCI_EXP_SLTCTL_PWR_ON         0x0000 /* Power On */
+#define  PCI_EXP_SLTCTL_PWR_OFF        0x0400 /* Power Off */
 #define  PCI_EXP_SLTCTL_EIC	0x0800	/* Electromechanical Interlock Control */
 #define  PCI_EXP_SLTCTL_DLLSCE	0x1000	/* Data Link Layer State Changed Enable */
 #define PCI_EXP_SLTSTA		26	/* Slot Status */
@@ -501,52 +551,94 @@
 #define  PCI_EXP_SLTSTA_EIS	0x0080	/* Electromechanical Interlock Status */
 #define  PCI_EXP_SLTSTA_DLLSC	0x0100	/* Data Link Layer State Changed */
 #define PCI_EXP_RTCTL		28	/* Root Control */
-#define  PCI_EXP_RTCTL_SECEE	0x01	/* System Error on Correctable Error */
-#define  PCI_EXP_RTCTL_SENFEE	0x02	/* System Error on Non-Fatal Error */
-#define  PCI_EXP_RTCTL_SEFEE	0x04	/* System Error on Fatal Error */
-#define  PCI_EXP_RTCTL_PMEIE	0x08	/* PME Interrupt Enable */
-#define  PCI_EXP_RTCTL_CRSSVE	0x10	/* CRS Software Visibility Enable */
+#define  PCI_EXP_RTCTL_SECEE	0x0001	/* System Error on Correctable Error */
+#define  PCI_EXP_RTCTL_SENFEE	0x0002	/* System Error on Non-Fatal Error */
+#define  PCI_EXP_RTCTL_SEFEE	0x0004	/* System Error on Fatal Error */
+#define  PCI_EXP_RTCTL_PMEIE	0x0008	/* PME Interrupt Enable */
+#define  PCI_EXP_RTCTL_CRSSVE	0x0010	/* CRS Software Visibility Enable */
 #define PCI_EXP_RTCAP		30	/* Root Capabilities */
+#define  PCI_EXP_RTCAP_CRSVIS	0x0001	/* CRS Software Visibility capability */
 #define PCI_EXP_RTSTA		32	/* Root Status */
-#define PCI_EXP_RTSTA_PME	0x10000 /* PME status */
-#define PCI_EXP_RTSTA_PENDING	0x20000 /* PME pending */
+#define PCI_EXP_RTSTA_PME	0x00010000 /* PME status */
+#define PCI_EXP_RTSTA_PENDING	0x00020000 /* PME pending */
+/*
+ * The Device Capabilities 2, Device Status 2, Device Control 2,
+ * Link Capabilities 2, Link Status 2, Link Control 2,
+ * Slot Capabilities 2, Slot Status 2, and Slot Control 2 registers
+ * are only present on devices with PCIe Capability version 2.
+ * Use pcie_capability_read_word() and similar interfaces to use them
+ * safely.
+ */
 #define PCI_EXP_DEVCAP2		36	/* Device Capabilities 2 */
-#define  PCI_EXP_DEVCAP2_ARI	0x20	/* Alternative Routing-ID */
-#define  PCI_EXP_DEVCAP2_LTR	0x800	/* Latency tolerance reporting */
-#define  PCI_EXP_OBFF_MASK	0xc0000 /* OBFF support mechanism */
-#define  PCI_EXP_OBFF_MSG	0x40000 /* New message signaling */
-#define  PCI_EXP_OBFF_WAKE	0x80000 /* Re-use WAKE# for OBFF */
+#define  PCI_EXP_DEVCAP2_ARI		0x00000020 /* Alternative Routing-ID */
+#define  PCI_EXP_DEVCAP2_LTR		0x00000800 /* Latency tolerance reporting */
+#define  PCI_EXP_DEVCAP2_OBFF_MASK	0x000c0000 /* OBFF support mechanism */
+#define  PCI_EXP_DEVCAP2_OBFF_MSG	0x00040000 /* New message signaling */
+#define  PCI_EXP_DEVCAP2_OBFF_WAKE	0x00080000 /* Re-use WAKE# for OBFF */
 #define PCI_EXP_DEVCTL2		40	/* Device Control 2 */
-#define  PCI_EXP_DEVCTL2_ARI	0x20	/* Alternative Routing-ID */
-#define  PCI_EXP_IDO_REQ_EN	0x100	/* ID-based ordering request enable */
-#define  PCI_EXP_IDO_CMP_EN	0x200	/* ID-based ordering completion enable */
-#define  PCI_EXP_LTR_EN		0x400	/* Latency tolerance reporting */
-#define  PCI_EXP_OBFF_MSGA_EN	0x2000	/* OBFF enable with Message type A */
-#define  PCI_EXP_OBFF_MSGB_EN	0x4000	/* OBFF enable with Message type B */
-#define  PCI_EXP_OBFF_WAKE_EN	0x6000	/* OBFF using WAKE# signaling */
+#define  PCI_EXP_DEVCTL2_COMP_TIMEOUT	0x000f	/* Completion Timeout Value */
+#define  PCI_EXP_DEVCTL2_ARI		0x0020	/* Alternative Routing-ID */
+#define  PCI_EXP_DEVCTL2_IDO_REQ_EN	0x0100	/* Allow IDO for requests */
+#define  PCI_EXP_DEVCTL2_IDO_CMP_EN	0x0200	/* Allow IDO for completions */
+#define  PCI_EXP_DEVCTL2_LTR_EN		0x0400	/* Enable LTR mechanism */
+#define  PCI_EXP_DEVCTL2_OBFF_MSGA_EN	0x2000	/* Enable OBFF Message type A */
+#define  PCI_EXP_DEVCTL2_OBFF_MSGB_EN	0x4000	/* Enable OBFF Message type B */
+#define  PCI_EXP_DEVCTL2_OBFF_WAKE_EN	0x6000	/* OBFF using WAKE# signaling */
+#define PCI_EXP_DEVSTA2		42	/* Device Status 2 */
+#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2	44	/* v2 endpoints end here */
+#define PCI_EXP_LNKCAP2		44	/* Link Capabilities 2 */
+#define  PCI_EXP_LNKCAP2_SLS_2_5GB	0x00000002 /* Supported Speed 2.5GT/s */
+#define  PCI_EXP_LNKCAP2_SLS_5_0GB	0x00000004 /* Supported Speed 5.0GT/s */
+#define  PCI_EXP_LNKCAP2_SLS_8_0GB	0x00000008 /* Supported Speed 8.0GT/s */
+#define  PCI_EXP_LNKCAP2_CROSSLINK	0x00000100 /* Crosslink supported */
 #define PCI_EXP_LNKCTL2		48	/* Link Control 2 */
+#define PCI_EXP_LNKSTA2		50	/* Link Status 2 */
+#define PCI_EXP_SLTCAP2		52	/* Slot Capabilities 2 */
 #define PCI_EXP_SLTCTL2		56	/* Slot Control 2 */
+#define PCI_EXP_SLTSTA2		58	/* Slot Status 2 */
 
 /* Extended Capabilities (PCI-X 2.0 and Express) */
 #define PCI_EXT_CAP_ID(header)		(header & 0x0000ffff)
 #define PCI_EXT_CAP_VER(header)		((header >> 16) & 0xf)
 #define PCI_EXT_CAP_NEXT(header)	((header >> 20) & 0xffc)
 
-#define PCI_EXT_CAP_ID_ERR	1
-#define PCI_EXT_CAP_ID_VC	2
-#define PCI_EXT_CAP_ID_DSN	3
-#define PCI_EXT_CAP_ID_PWR	4
-#define PCI_EXT_CAP_ID_VNDR	11
-#define PCI_EXT_CAP_ID_ACS	13
-#define PCI_EXT_CAP_ID_ARI	14
-#define PCI_EXT_CAP_ID_ATS	15
-#define PCI_EXT_CAP_ID_SRIOV	16
-#define PCI_EXT_CAP_ID_LTR	24
+#define PCI_EXT_CAP_ID_ERR	0x01	/* Advanced Error Reporting */
+#define PCI_EXT_CAP_ID_VC	0x02	/* Virtual Channel Capability */
+#define PCI_EXT_CAP_ID_DSN	0x03	/* Device Serial Number */
+#define PCI_EXT_CAP_ID_PWR	0x04	/* Power Budgeting */
+#define PCI_EXT_CAP_ID_RCLD	0x05	/* Root Complex Link Declaration */
+#define PCI_EXT_CAP_ID_RCILC	0x06	/* Root Complex Internal Link Control */
+#define PCI_EXT_CAP_ID_RCEC	0x07	/* Root Complex Event Collector */
+#define PCI_EXT_CAP_ID_MFVC	0x08	/* Multi-Function VC Capability */
+#define PCI_EXT_CAP_ID_VC9	0x09	/* same as _VC */
+#define PCI_EXT_CAP_ID_RCRB	0x0A	/* Root Complex RB? */
+#define PCI_EXT_CAP_ID_VNDR	0x0B	/* Vendor-Specific */
+#define PCI_EXT_CAP_ID_CAC	0x0C	/* Config Access - obsolete */
+#define PCI_EXT_CAP_ID_ACS	0x0D	/* Access Control Services */
+#define PCI_EXT_CAP_ID_ARI	0x0E	/* Alternate Routing ID */
+#define PCI_EXT_CAP_ID_ATS	0x0F	/* Address Translation Services */
+#define PCI_EXT_CAP_ID_SRIOV	0x10	/* Single Root I/O Virtualization */
+#define PCI_EXT_CAP_ID_MRIOV	0x11	/* Multi Root I/O Virtualization */
+#define PCI_EXT_CAP_ID_MCAST	0x12	/* Multicast */
+#define PCI_EXT_CAP_ID_PRI	0x13	/* Page Request Interface */
+#define PCI_EXT_CAP_ID_AMD_XXX	0x14	/* Reserved for AMD */
+#define PCI_EXT_CAP_ID_REBAR	0x15	/* Resizable BAR */
+#define PCI_EXT_CAP_ID_DPA	0x16	/* Dynamic Power Allocation */
+#define PCI_EXT_CAP_ID_TPH	0x17	/* TPH Requester */
+#define PCI_EXT_CAP_ID_LTR	0x18	/* Latency Tolerance Reporting */
+#define PCI_EXT_CAP_ID_SECPCI	0x19	/* Secondary PCIe Capability */
+#define PCI_EXT_CAP_ID_PMUX	0x1A	/* Protocol Multiplexing */
+#define PCI_EXT_CAP_ID_PASID	0x1B	/* Process Address Space ID */
+#define PCI_EXT_CAP_ID_MAX	PCI_EXT_CAP_ID_PASID
+
+#define PCI_EXT_CAP_DSN_SIZEOF	12
+#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
 
 /* Advanced Error Reporting */
 #define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */
-#define  PCI_ERR_UNC_TRAIN	0x00000001	/* Training */
+#define  PCI_ERR_UNC_UND	0x00000001	/* Undefined */
 #define  PCI_ERR_UNC_DLP	0x00000010	/* Data Link Protocol */
+#define  PCI_ERR_UNC_SURPDN	0x00000020	/* Surprise Down */
 #define  PCI_ERR_UNC_POISON_TLP	0x00001000	/* Poisoned TLP */
 #define  PCI_ERR_UNC_FCP	0x00002000	/* Flow Control Protocol */
 #define  PCI_ERR_UNC_COMP_TIME	0x00004000	/* Completion Timeout */
@@ -556,6 +648,11 @@
 #define  PCI_ERR_UNC_MALF_TLP	0x00040000	/* Malformed TLP */
 #define  PCI_ERR_UNC_ECRC	0x00080000	/* ECRC Error Status */
 #define  PCI_ERR_UNC_UNSUP	0x00100000	/* Unsupported Request */
+#define  PCI_ERR_UNC_ACSV	0x00200000	/* ACS Violation */
+#define  PCI_ERR_UNC_INTN	0x00400000	/* internal error */
+#define  PCI_ERR_UNC_MCBTLP	0x00800000	/* MC blocked TLP */
+#define  PCI_ERR_UNC_ATOMEG	0x01000000	/* Atomic egress blocked */
+#define  PCI_ERR_UNC_TLPPRE	0x02000000	/* TLP prefix blocked */
 #define PCI_ERR_UNCOR_MASK	8	/* Uncorrectable Error Mask */
 	/* Same bits as above */
 #define PCI_ERR_UNCOR_SEVER	12	/* Uncorrectable Error Severity */
@@ -566,6 +663,9 @@
 #define  PCI_ERR_COR_BAD_DLLP	0x00000080	/* Bad DLLP Status */
 #define  PCI_ERR_COR_REP_ROLL	0x00000100	/* REPLAY_NUM Rollover */
 #define  PCI_ERR_COR_REP_TIMER	0x00001000	/* Replay Timer Timeout */
+#define  PCI_ERR_COR_ADV_NFAT	0x00002000	/* Advisory Non-Fatal */
+#define  PCI_ERR_COR_INTERNAL	0x00004000	/* Corrected Internal */
+#define  PCI_ERR_COR_LOG_OVER	0x00008000	/* Header Log Overflow */
 #define PCI_ERR_COR_MASK	20	/* Correctable Error Mask */
 	/* Same bits as above */
 #define PCI_ERR_CAP		24	/* Advanced Error Capabilities */
@@ -586,9 +686,9 @@
 #define PCI_ERR_ROOT_COR_RCV		0x00000001	/* ERR_COR Received */
 /* Multi ERR_COR Received */
 #define PCI_ERR_ROOT_MULTI_COR_RCV	0x00000002
-/* ERR_FATAL/NONFATAL Recevied */
+/* ERR_FATAL/NONFATAL Received */
 #define PCI_ERR_ROOT_UNCOR_RCV		0x00000004
-/* Multi ERR_FATAL/NONFATAL Recevied */
+/* Multi ERR_FATAL/NONFATAL Received */
 #define PCI_ERR_ROOT_MULTI_UNCOR_RCV	0x00000008
 #define PCI_ERR_ROOT_FIRST_FATAL	0x00000010	/* First Fatal */
 #define PCI_ERR_ROOT_NONFATAL_RCV	0x00000020	/* Non-Fatal Received */
@@ -596,13 +696,36 @@
 #define PCI_ERR_ROOT_ERR_SRC	52	/* Error Source Identification */
 
 /* Virtual Channel */
-#define PCI_VC_PORT_REG1	4
-#define PCI_VC_PORT_REG2	8
+#define PCI_VC_PORT_CAP1	4
+#define  PCI_VC_CAP1_EVCC	0x00000007	/* extended VC count */
+#define  PCI_VC_CAP1_LPEVCC	0x00000070	/* low prio extended VC count */
+#define  PCI_VC_CAP1_ARB_SIZE	0x00000c00
+#define PCI_VC_PORT_CAP2	8
+#define  PCI_VC_CAP2_32_PHASE		0x00000002
+#define  PCI_VC_CAP2_64_PHASE		0x00000004
+#define  PCI_VC_CAP2_128_PHASE		0x00000008
+#define  PCI_VC_CAP2_ARB_OFF		0xff000000
 #define PCI_VC_PORT_CTRL	12
+#define  PCI_VC_PORT_CTRL_LOAD_TABLE	0x00000001
 #define PCI_VC_PORT_STATUS	14
+#define  PCI_VC_PORT_STATUS_TABLE	0x00000001
 #define PCI_VC_RES_CAP		16
+#define  PCI_VC_RES_CAP_32_PHASE	0x00000002
+#define  PCI_VC_RES_CAP_64_PHASE	0x00000004
+#define  PCI_VC_RES_CAP_128_PHASE	0x00000008
+#define  PCI_VC_RES_CAP_128_PHASE_TB	0x00000010
+#define  PCI_VC_RES_CAP_256_PHASE	0x00000020
+#define  PCI_VC_RES_CAP_ARB_OFF		0xff000000
 #define PCI_VC_RES_CTRL		20
+#define  PCI_VC_RES_CTRL_LOAD_TABLE	0x00010000
+#define  PCI_VC_RES_CTRL_ARB_SELECT	0x000e0000
+#define  PCI_VC_RES_CTRL_ID		0x07000000
+#define  PCI_VC_RES_CTRL_ENABLE		0x80000000
 #define PCI_VC_RES_STATUS	26
+#define  PCI_VC_RES_STATUS_TABLE	0x00000001
+#define  PCI_VC_RES_STATUS_NEGO		0x00000002
+#define PCI_CAP_VC_BASE_SIZEOF		0x10
+#define PCI_CAP_VC_PER_VC_SIZEOF	0x0C
 
 /* Power Budgeting */
 #define PCI_PWR_DSR		4	/* Data Select Register */
@@ -615,9 +738,16 @@
 #define  PCI_PWR_DATA_RAIL(x)	(((x) >> 18) & 7)   /* Power Rail */
 #define PCI_PWR_CAP		12	/* Capability */
 #define  PCI_PWR_CAP_BUDGET(x)	((x) & 1)	/* Included in system budget */
+#define PCI_EXT_CAP_PWR_SIZEOF	16
+
+/* Vendor-Specific (VSEC, PCI_EXT_CAP_ID_VNDR) */
+#define PCI_VNDR_HEADER		4	/* Vendor-Specific Header */
+#define  PCI_VNDR_HEADER_ID(x)	((x) & 0xffff)
+#define  PCI_VNDR_HEADER_REV(x)	(((x) >> 16) & 0xf)
+#define  PCI_VNDR_HEADER_LEN(x)	(((x) >> 20) & 0xfff)
 
 /*
- * Hypertransport sub capability types
+ * HyperTransport sub capability types
  *
  * Unfortunately there are both 3 bit and 5 bit capability types defined
  * in the HT spec, catering for that is a little messy. You probably don't
@@ -645,8 +775,10 @@
 #define HT_CAPTYPE_DIRECT_ROUTE	0xB0	/* Direct routing configuration */
 #define HT_CAPTYPE_VCSET	0xB8	/* Virtual Channel configuration */
 #define HT_CAPTYPE_ERROR_RETRY	0xC0	/* Retry on error configuration */
-#define HT_CAPTYPE_GEN3		0xD0	/* Generation 3 hypertransport configuration */
-#define HT_CAPTYPE_PM		0xE0	/* Hypertransport powermanagement configuration */
+#define HT_CAPTYPE_GEN3		0xD0	/* Generation 3 HyperTransport configuration */
+#define HT_CAPTYPE_PM		0xE0	/* HyperTransport power management configuration */
+#define HT_CAP_SIZEOF_LONG	28	/* slave & primary */
+#define HT_CAP_SIZEOF_SHORT	24	/* host & secondary */
 
 /* Alternative Routing-ID Interpretation */
 #define PCI_ARI_CAP		0x04	/* ARI Capability Register */
@@ -657,6 +789,7 @@
 #define  PCI_ARI_CTRL_MFVC	0x0001	/* MFVC Function Groups Enable */
 #define  PCI_ARI_CTRL_ACS	0x0002	/* ACS Function Groups Enable */
 #define  PCI_ARI_CTRL_FG(x)	(((x) >> 4) & 7) /* Function Group */
+#define PCI_EXT_CAP_ARI_SIZEOF	8
 
 /* Address Translation Service */
 #define PCI_ATS_CAP		0x04	/* ATS Capability Register */
@@ -666,6 +799,29 @@
 #define  PCI_ATS_CTRL_ENABLE	0x8000	/* ATS Enable */
 #define  PCI_ATS_CTRL_STU(x)	((x) & 0x1f)	/* Smallest Translation Unit */
 #define  PCI_ATS_MIN_STU	12	/* shift of minimum STU block */
+#define PCI_EXT_CAP_ATS_SIZEOF	8
+
+/* Page Request Interface */
+#define PCI_PRI_CTRL		0x04	/* PRI control register */
+#define  PCI_PRI_CTRL_ENABLE	0x01	/* Enable */
+#define  PCI_PRI_CTRL_RESET	0x02	/* Reset */
+#define PCI_PRI_STATUS		0x06	/* PRI status register */
+#define  PCI_PRI_STATUS_RF	0x001	/* Response Failure */
+#define  PCI_PRI_STATUS_UPRGI	0x002	/* Unexpected PRG index */
+#define  PCI_PRI_STATUS_STOPPED	0x100	/* PRI Stopped */
+#define PCI_PRI_MAX_REQ		0x08	/* PRI max reqs supported */
+#define PCI_PRI_ALLOC_REQ	0x0c	/* PRI max reqs allowed */
+#define PCI_EXT_CAP_PRI_SIZEOF	16
+
+/* Process Address Space ID */
+#define PCI_PASID_CAP		0x04    /* PASID feature register */
+#define  PCI_PASID_CAP_EXEC	0x02	/* Exec permissions Supported */
+#define  PCI_PASID_CAP_PRIV	0x04	/* Privilege Mode Supported */
+#define PCI_PASID_CTRL		0x06    /* PASID control register */
+#define  PCI_PASID_CTRL_ENABLE	0x01	/* Enable bit */
+#define  PCI_PASID_CTRL_EXEC	0x02	/* Exec permissions Enable */
+#define  PCI_PASID_CTRL_PRIV	0x04	/* Privilege Mode Enable */
+#define PCI_EXT_CAP_PASID_SIZEOF	8
 
 /* Single Root I/O Virtualization */
 #define PCI_SRIOV_CAP		0x04	/* SR-IOV Capabilities */
@@ -697,12 +853,14 @@
 #define  PCI_SRIOV_VFM_MI	0x1	/* Dormant.MigrateIn */
 #define  PCI_SRIOV_VFM_MO	0x2	/* Active.MigrateOut */
 #define  PCI_SRIOV_VFM_AV	0x3	/* Active.Available */
+#define PCI_EXT_CAP_SRIOV_SIZEOF 64
 
 #define PCI_LTR_MAX_SNOOP_LAT	0x4
 #define PCI_LTR_MAX_NOSNOOP_LAT	0x6
 #define  PCI_LTR_VALUE_MASK	0x000003ff
 #define  PCI_LTR_SCALE_MASK	0x00001c00
 #define  PCI_LTR_SCALE_SHIFT	10
+#define PCI_EXT_CAP_LTR_SIZEOF	8
 
 /* Access Control Service */
 #define PCI_ACS_CAP		0x04	/* ACS Capability Register */
@@ -713,7 +871,38 @@
 #define  PCI_ACS_UF		0x10	/* Upstream Forwarding */
 #define  PCI_ACS_EC		0x20	/* P2P Egress Control */
 #define  PCI_ACS_DT		0x40	/* Direct Translated P2P */
+#define PCI_ACS_EGRESS_BITS	0x05	/* ACS Egress Control Vector Size */
 #define PCI_ACS_CTRL		0x06	/* ACS Control Register */
 #define PCI_ACS_EGRESS_CTL_V	0x08	/* ACS Egress Control Vector */
 
+#define PCI_VSEC_HDR		4	/* extended cap - vendor-specific */
+#define  PCI_VSEC_HDR_LEN_SHIFT	20	/* shift for length field */
+
+/* SATA capability */
+#define PCI_SATA_REGS		4	/* SATA REGs specifier */
+#define  PCI_SATA_REGS_MASK	0xF	/* location - BAR#/inline */
+#define  PCI_SATA_REGS_INLINE	0xF	/* REGS in config space */
+#define PCI_SATA_SIZEOF_SHORT	8
+#define PCI_SATA_SIZEOF_LONG	16
+
+/* Resizable BARs */
+#define PCI_REBAR_CTRL		8	/* control register */
+#define  PCI_REBAR_CTRL_NBAR_MASK	(7 << 5)	/* mask for # bars */
+#define  PCI_REBAR_CTRL_NBAR_SHIFT	5	/* shift for # bars */
+
+/* Dynamic Power Allocation */
+#define PCI_DPA_CAP		4	/* capability register */
+#define  PCI_DPA_CAP_SUBSTATE_MASK	0x1F	/* # substates - 1 */
+#define PCI_DPA_BASE_SIZEOF	16	/* size with 0 substates */
+
+/* TPH Requester */
+#define PCI_TPH_CAP		4	/* capability register */
+#define  PCI_TPH_CAP_LOC_MASK	0x600	/* location mask */
+#define   PCI_TPH_LOC_NONE	0x000	/* no location */
+#define   PCI_TPH_LOC_CAP	0x200	/* in capability */
+#define   PCI_TPH_LOC_MSIX	0x400	/* in MSI-X */
+#define PCI_TPH_CAP_ST_MASK	0x07FF0000	/* st table mask */
+#define PCI_TPH_CAP_ST_SHIFT	16	/* st table shift */
+#define PCI_TPH_BASE_SIZEOF	12	/* size with no st table */
+
 #endif /* LINUX_PCI_REGS_H */
diff --git a/include/standard-headers/linux/virtio_gpu.h b/include/standard-headers/linux/virtio_gpu.h
index 72ef815f5..76e5e5292 100644
--- a/include/standard-headers/linux/virtio_gpu.h
+++ b/include/standard-headers/linux/virtio_gpu.h
@@ -40,6 +40,8 @@
 
 #include "standard-headers/linux/types.h"
 
+#define VIRTIO_GPU_FEATURE_VIRGL 0
+
 enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_UNDEFINED = 0,
 
@@ -52,6 +54,18 @@ enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D,
 	VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING,
 	VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING,
+	VIRTIO_GPU_CMD_GET_CAPSET_INFO,
+	VIRTIO_GPU_CMD_GET_CAPSET,
+
+	/* 3d commands */
+	VIRTIO_GPU_CMD_CTX_CREATE = 0x0200,
+	VIRTIO_GPU_CMD_CTX_DESTROY,
+	VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE,
+	VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE,
+	VIRTIO_GPU_CMD_RESOURCE_CREATE_3D,
+	VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D,
+	VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D,
+	VIRTIO_GPU_CMD_SUBMIT_3D,
 
 	/* cursor commands */
 	VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
@@ -60,6 +74,8 @@ enum virtio_gpu_ctrl_type {
 	/* success responses */
 	VIRTIO_GPU_RESP_OK_NODATA = 0x1100,
 	VIRTIO_GPU_RESP_OK_DISPLAY_INFO,
+	VIRTIO_GPU_RESP_OK_CAPSET_INFO,
+	VIRTIO_GPU_RESP_OK_CAPSET,
 
 	/* error responses */
 	VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
@@ -180,13 +196,107 @@ struct virtio_gpu_resp_display_info {
 	} pmodes[VIRTIO_GPU_MAX_SCANOUTS];
 };
 
+/* data passed in the control vq, 3d related */
+
+struct virtio_gpu_box {
+	uint32_t x, y, z;
+	uint32_t w, h, d;
+};
+
+/* VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D, VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D */
+struct virtio_gpu_transfer_host_3d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_box box;
+	uint64_t offset;
+	uint32_t resource_id;
+	uint32_t level;
+	uint32_t stride;
+	uint32_t layer_stride;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_CREATE_3D */
+#define VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP (1 << 0)
+struct virtio_gpu_resource_create_3d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t target;
+	uint32_t format;
+	uint32_t bind;
+	uint32_t width;
+	uint32_t height;
+	uint32_t depth;
+	uint32_t array_size;
+	uint32_t last_level;
+	uint32_t nr_samples;
+	uint32_t flags;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_CTX_CREATE */
+struct virtio_gpu_ctx_create {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t nlen;
+	uint32_t padding;
+	char debug_name[64];
+};
+
+/* VIRTIO_GPU_CMD_CTX_DESTROY */
+struct virtio_gpu_ctx_destroy {
+	struct virtio_gpu_ctrl_hdr hdr;
+};
+
+/* VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE, VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE */
+struct virtio_gpu_ctx_resource {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_SUBMIT_3D */
+struct virtio_gpu_cmd_submit {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t size;
+	uint32_t padding;
+};
+
+#define VIRTIO_GPU_CAPSET_VIRGL 1
+
+/* VIRTIO_GPU_CMD_GET_CAPSET_INFO */
+struct virtio_gpu_get_capset_info {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_index;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET_INFO */
+struct virtio_gpu_resp_capset_info {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_id;
+	uint32_t capset_max_version;
+	uint32_t capset_max_size;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_GET_CAPSET */
+struct virtio_gpu_get_capset {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_id;
+	uint32_t capset_version;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET */
+struct virtio_gpu_resp_capset {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint8_t capset_data[];
+};
+
 #define VIRTIO_GPU_EVENT_DISPLAY (1 << 0)
 
 struct virtio_gpu_config {
 	uint32_t events_read;
 	uint32_t events_clear;
 	uint32_t num_scanouts;
-	uint32_t reserved;
+	uint32_t num_capsets;
 };
 
 /* simple formats for fbcon/X use */
diff --git a/include/standard-headers/linux/virtio_ring.h b/include/standard-headers/linux/virtio_ring.h
index 6fe276faf..023c6db04 100644
--- a/include/standard-headers/linux/virtio_ring.h
+++ b/include/standard-headers/linux/virtio_ring.h
@@ -31,6 +31,7 @@
  * SUCH DAMAGE.
  *
  * Copyright Rusty Russell IBM Corporation 2007. */
+#include <stdint.h>
 #include "standard-headers/linux/types.h"
 #include "standard-headers/linux/virtio_types.h"
 
@@ -143,7 +144,7 @@ static inline void vring_init(struct vring *vr, unsigned int num, void *p,
 	vr->num = num;
 	vr->desc = p;
 	vr->avail = p + num*sizeof(struct vring_desc);
-	vr->used = (void *)(((unsigned long)&vr->avail->ring[num] + sizeof(__virtio16)
+	vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(__virtio16)
 		+ align-1) & ~(align - 1));
 }
 
diff --git a/include/sysemu/balloon.h b/include/sysemu/balloon.h
index 17fe30070..3f976b49e 100644
--- a/include/sysemu/balloon.h
+++ b/include/sysemu/balloon.h
@@ -22,5 +22,7 @@ typedef void (QEMUBalloonStatus)(void *opaque, BalloonInfo *info);
 int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
 			     QEMUBalloonStatus *stat_func, void *opaque);
 void qemu_remove_balloon_handler(void *opaque);
+bool qemu_balloon_is_inhibited(void);
+void qemu_balloon_inhibit(bool state);
 
 #endif
diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h
index 8fc960fcb..fb068ea47 100644
--- a/include/sysemu/block-backend.h
+++ b/include/sysemu/block-backend.h
@@ -65,6 +65,7 @@ BlockBackend *blk_new_with_bs(const char *name, Error **errp);
 BlockBackend *blk_new_open(const char *name, const char *filename,
                            const char *reference, QDict *options, int flags,
                            Error **errp);
+int blk_get_refcnt(BlockBackend *blk);
 void blk_ref(BlockBackend *blk);
 void blk_unref(BlockBackend *blk);
 const char *blk_name(BlockBackend *blk);
@@ -72,10 +73,17 @@ BlockBackend *blk_by_name(const char *name);
 BlockBackend *blk_next(BlockBackend *blk);
 
 BlockDriverState *blk_bs(BlockBackend *blk);
+void blk_remove_bs(BlockBackend *blk);
+void blk_insert_bs(BlockBackend *blk, BlockDriverState *bs);
 
 void blk_hide_on_behalf_of_hmp_drive_del(BlockBackend *blk);
 
 void blk_iostatus_enable(BlockBackend *blk);
+bool blk_iostatus_is_enabled(const BlockBackend *blk);
+BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk);
+void blk_iostatus_disable(BlockBackend *blk);
+void blk_iostatus_reset(BlockBackend *blk);
+void blk_iostatus_set_err(BlockBackend *blk, int error);
 int blk_attach_dev(BlockBackend *blk, void *dev);
 void blk_attach_dev_nofail(BlockBackend *blk, void *dev);
 void blk_detach_dev(BlockBackend *blk, void *dev);
@@ -120,6 +128,8 @@ int blk_flush(BlockBackend *blk);
 int blk_flush_all(void);
 void blk_drain(BlockBackend *blk);
 void blk_drain_all(void);
+void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
+                      BlockdevOnError on_write_error);
 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read);
 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
                                       int error);
@@ -130,7 +140,8 @@ int blk_is_sg(BlockBackend *blk);
 int blk_enable_write_cache(BlockBackend *blk);
 void blk_set_enable_write_cache(BlockBackend *blk, bool wce);
 void blk_invalidate_cache(BlockBackend *blk, Error **errp);
-int blk_is_inserted(BlockBackend *blk);
+bool blk_is_inserted(BlockBackend *blk);
+bool blk_is_available(BlockBackend *blk);
 void blk_lock_medium(BlockBackend *blk, bool locked);
 void blk_eject(BlockBackend *blk, bool eject_flag);
 int blk_get_flags(BlockBackend *blk);
@@ -155,6 +166,10 @@ void blk_add_close_notifier(BlockBackend *blk, Notifier *notify);
 void blk_io_plug(BlockBackend *blk);
 void blk_io_unplug(BlockBackend *blk);
 BlockAcctStats *blk_get_stats(BlockBackend *blk);
+BlockBackendRootState *blk_get_root_state(BlockBackend *blk);
+void blk_update_root_state(BlockBackend *blk);
+void blk_apply_root_state(BlockBackend *blk, BlockDriverState *bs);
+int blk_get_open_flags_from_root_state(BlockBackend *blk);
 
 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
                   BlockCompletionFunc *cb, void *opaque);
@@ -169,5 +184,8 @@ int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
 int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size);
 int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz);
 int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo);
+BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque, int ret);
 
 #endif
diff --git a/include/sysemu/blockdev.h b/include/sysemu/blockdev.h
index 310415025..b06a0607a 100644
--- a/include/sysemu/blockdev.h
+++ b/include/sysemu/blockdev.h
@@ -20,7 +20,7 @@ void blockdev_auto_del(BlockBackend *blk);
 typedef enum {
     IF_DEFAULT = -1,            /* for use with drive_add() only */
     /*
-     * IF_IDE must be zero, because we want QEMUMachine member
+     * IF_IDE must be zero, because we want MachineClass member
      * block_default_type to default-initialize to IF_IDE
      */
     IF_IDE = 0,
@@ -63,8 +63,6 @@ DriveInfo *drive_new(QemuOpts *arg, BlockInterfaceType block_default_type);
 
 /* device-hotplug */
 
-void qmp_change_blockdev(const char *device, const char *filename,
-                         const char *format, Error **errp);
 void hmp_commit(Monitor *mon, const QDict *qdict);
 void hmp_drive_del(Monitor *mon, const QDict *qdict);
 #endif
diff --git a/include/sysemu/char.h b/include/sysemu/char.h
index 832b7fead..aff193f08 100644
--- a/include/sysemu/char.h
+++ b/include/sysemu/char.h
@@ -128,11 +128,19 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename,
 /**
  * @qemu_chr_delete:
  *
- * Destroy a character backend.
+ * Destroy a character backend and remove it from the list of
+ * identified character backends.
  */
 void qemu_chr_delete(CharDriverState *chr);
 
 /**
+ * @qemu_chr_free:
+ *
+ * Destroy a character backend.
+ */
+void qemu_chr_free(CharDriverState *chr);
+
+/**
  * @qemu_chr_fe_set_echo:
  *
  * Ask the backend to override its normal echo setting.  This only really
@@ -345,7 +353,9 @@ bool chr_is_ringbuf(const CharDriverState *chr);
 QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename);
 
 void register_char_driver(const char *name, ChardevBackendKind kind,
-        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp));
+        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp),
+        CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                                   ChardevReturn *ret, Error **errp));
 
 /* add an eventfd to the qemu devices that are polled */
 CharDriverState *qemu_chr_open_eventfd(int eventfd);
@@ -354,18 +364,8 @@ extern int term_escape_char;
 
 CharDriverState *qemu_char_get_next_serial(void);
 
-/* msmouse */
-CharDriverState *qemu_chr_open_msmouse(void);
-
-/* testdev.c */
-CharDriverState *chr_testdev_init(void);
-
-/* baum.c */
-CharDriverState *chr_baum_init(void);
-
 /* console.c */
-typedef CharDriverState *(VcHandler)(ChardevVC *vc);
-
+typedef CharDriverState *(VcHandler)(ChardevVC *vc, Error **errp);
 void register_vc_handler(VcHandler *handler);
-CharDriverState *vc_init(ChardevVC *vc);
+
 #endif
diff --git a/include/sysemu/cpus.h b/include/sysemu/cpus.h
index 3f162a9e0..3d1e5ba1e 100644
--- a/include/sysemu/cpus.h
+++ b/include/sysemu/cpus.h
@@ -2,6 +2,7 @@
 #define QEMU_CPUS_H
 
 /* cpus.c */
+bool qemu_in_vcpu_thread(void);
 void qemu_init_cpu_loop(void);
 void resume_all_vcpus(void);
 void pause_all_vcpus(void);
@@ -10,7 +11,6 @@ void cpu_stop_current(void);
 void cpu_synchronize_all_states(void);
 void cpu_synchronize_all_post_reset(void);
 void cpu_synchronize_all_post_init(void);
-void cpu_clean_all_dirty(void);
 
 void qtest_clock_warp(int64_t dest);
 
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 983e99e1e..b31f325fa 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -52,6 +52,8 @@ extern bool kvm_msi_via_irqfd_allowed;
 extern bool kvm_gsi_routing_allowed;
 extern bool kvm_gsi_direct_mapping;
 extern bool kvm_readonly_mem_allowed;
+extern bool kvm_direct_msi_allowed;
+extern bool kvm_ioeventfd_any_length_allowed;
 
 #if defined CONFIG_KVM || !defined NEED_CPU_H
 #define kvm_enabled()           (kvm_allowed)
@@ -145,6 +147,19 @@ extern bool kvm_readonly_mem_allowed;
  */
 #define kvm_readonly_mem_enabled() (kvm_readonly_mem_allowed)
 
+/**
+ * kvm_direct_msi_enabled:
+ *
+ * Returns: true if KVM allows direct MSI injection.
+ */
+#define kvm_direct_msi_enabled() (kvm_direct_msi_allowed)
+
+/**
+ * kvm_ioeventfd_any_length_enabled:
+ * Returns: true if KVM allows any length io eventfd.
+ */
+#define kvm_ioeventfd_any_length_enabled() (kvm_ioeventfd_any_length_allowed)
+
 #else
 #define kvm_enabled()           (0)
 #define kvm_irqchip_in_kernel() (false)
@@ -157,6 +172,8 @@ extern bool kvm_readonly_mem_allowed;
 #define kvm_gsi_routing_allowed() (false)
 #define kvm_gsi_direct_mapping() (false)
 #define kvm_readonly_mem_enabled() (false)
+#define kvm_direct_msi_enabled() (false)
+#define kvm_ioeventfd_any_length_enabled() (false)
 #endif
 
 struct kvm_run;
@@ -182,8 +199,6 @@ int kvm_has_sync_mmu(void);
 int kvm_has_vcpu_events(void);
 int kvm_has_robust_singlestep(void);
 int kvm_has_debugregs(void);
-int kvm_has_xsave(void);
-int kvm_has_xcrs(void);
 int kvm_has_pit_state2(void);
 int kvm_has_many_ioeventfds(void);
 int kvm_has_gsi_routing(void);
@@ -210,6 +225,10 @@ int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset);
 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr);
 int kvm_on_sigbus(int code, void *addr);
 
+/* interface with exec.c */
+
+void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align));
+
 /* internal API */
 
 int kvm_ioctl(KVMState *s, int type, ...);
@@ -240,6 +259,32 @@ int kvm_device_ioctl(int fd, int type, ...);
 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr);
 
 /**
+ * kvm_device_check_attr - check for existence of a specific device attribute
+ * @fd: The device file descriptor
+ * @group: the group
+ * @attr: the attribute of that group to query for
+ *
+ * Returns: 1 if the attribute exists
+ *          0 if the attribute either does not exist or if the vm device
+ *            interface is unavailable
+ */
+int kvm_device_check_attr(int fd, uint32_t group, uint64_t attr);
+
+/**
+ * kvm_device_access - set or get value of a specific vm attribute
+ * @fd: The device file descriptor
+ * @group: the group
+ * @attr: the attribute of that group to set or get
+ * @val: pointer to a storage area for the value
+ * @write: true for set and false for get operation
+ *
+ * This function is not allowed to fail. Use kvm_device_check_attr()
+ * in order to check for the availability of optional attributes.
+ */
+void kvm_device_access(int fd, int group, uint64_t attr,
+                       void *val, bool write);
+
+/**
  * kvm_create_device - create a KVM device for the device control API
  * @KVMState: The KVMState pointer
  * @type: The KVM device type (see Documentation/virtual/kvm/devices in the
@@ -287,7 +332,7 @@ int kvm_arch_on_sigbus(int code, void *addr);
 void kvm_arch_init_irq_routing(KVMState *s);
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data);
+                             uint64_t address, uint32_t data, PCIDevice *dev);
 
 int kvm_arch_msi_data_to_gsi(uint32_t data);
 
@@ -380,7 +425,6 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
 void kvm_cpu_synchronize_state(CPUState *cpu);
 void kvm_cpu_synchronize_post_reset(CPUState *cpu);
 void kvm_cpu_synchronize_post_init(CPUState *cpu);
-void kvm_cpu_clean_state(CPUState *cpu);
 
 /* generic hooks - to be moved/refactored once there are more users */
 
@@ -405,15 +449,9 @@ static inline void cpu_synchronize_post_init(CPUState *cpu)
     }
 }
 
-static inline void cpu_clean_state(CPUState *cpu)
-{
-    if (kvm_enabled()) {
-        kvm_cpu_clean_state(cpu);
-    }
-}
-
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev);
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev);
 void kvm_irqchip_release_virq(KVMState *s, int virq);
 
 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter);
diff --git a/include/sysemu/os-win32.h b/include/sysemu/os-win32.h
index 4035c4fe5..400e09860 100644
--- a/include/sysemu/os-win32.h
+++ b/include/sysemu/os-win32.h
@@ -73,10 +73,12 @@
 #define siglongjmp(env, val) longjmp(env, val)
 
 /* Missing POSIX functions. Don't use MinGW-w64 macros. */
+#ifndef CONFIG_LOCALTIME_R
 #undef gmtime_r
 struct tm *gmtime_r(const time_t *timep, struct tm *result);
 #undef localtime_r
 struct tm *localtime_r(const time_t *timep, struct tm *result);
+#endif /* CONFIG_LOCALTIME_R */
 
 
 static inline void os_setup_signal_handling(void) {}
@@ -85,7 +87,7 @@ static inline void os_setup_post(void) {}
 void os_set_line_buffering(void);
 static inline void os_set_proc_name(const char *dummy) {}
 
-size_t getpagesize(void);
+int getpagesize(void);
 
 #if !defined(EPROTONOSUPPORT)
 # define EPROTONOSUPPORT EINVAL
@@ -109,4 +111,22 @@ static inline int os_mlock(void)
     return -ENOSYS;
 }
 
+#define fsync _commit
+
+#if !defined(lseek)
+# define lseek _lseeki64
+#endif
+
+int qemu_ftruncate64(int, int64_t);
+
+#if !defined(ftruncate)
+# define ftruncate qemu_ftruncate64
+#endif
+
+static inline char *realpath(const char *path, char *resolved_path)
+{
+    _fullpath(resolved_path, path, _MAX_PATH);
+    return resolved_path;
+}
+
 #endif
diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h
new file mode 100644
index 000000000..abb468820
--- /dev/null
+++ b/include/sysemu/replay.h
@@ -0,0 +1,120 @@
+#ifndef REPLAY_H
+#define REPLAY_H
+
+/*
+ * replay.h
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "qapi-types.h"
+#include "qapi/error.h"
+#include "qemu/typedefs.h"
+
+/* replay clock kinds */
+enum ReplayClockKind {
+    /* host_clock */
+    REPLAY_CLOCK_HOST,
+    /* virtual_rt_clock */
+    REPLAY_CLOCK_VIRTUAL_RT,
+    REPLAY_CLOCK_COUNT
+};
+typedef enum ReplayClockKind ReplayClockKind;
+
+/* IDs of the checkpoints */
+enum ReplayCheckpoint {
+    CHECKPOINT_CLOCK_WARP,
+    CHECKPOINT_RESET_REQUESTED,
+    CHECKPOINT_SUSPEND_REQUESTED,
+    CHECKPOINT_CLOCK_VIRTUAL,
+    CHECKPOINT_CLOCK_HOST,
+    CHECKPOINT_CLOCK_VIRTUAL_RT,
+    CHECKPOINT_INIT,
+    CHECKPOINT_RESET,
+    CHECKPOINT_COUNT
+};
+typedef enum ReplayCheckpoint ReplayCheckpoint;
+
+extern ReplayMode replay_mode;
+
+/* Replay process control functions */
+
+/*! Enables recording or saving event log with specified parameters */
+void replay_configure(struct QemuOpts *opts);
+/*! Initializes timers used for snapshotting and enables events recording */
+void replay_start(void);
+/*! Closes replay log file and frees other resources. */
+void replay_finish(void);
+/*! Adds replay blocker with the specified error description */
+void replay_add_blocker(Error *reason);
+
+/* Processing the instructions */
+
+/*! Returns number of executed instructions. */
+uint64_t replay_get_current_step(void);
+/*! Returns number of instructions to execute in replay mode. */
+int replay_get_instructions(void);
+/*! Updates instructions counter in replay mode. */
+void replay_account_executed_instructions(void);
+
+/* Interrupts and exceptions */
+
+/*! Called by exception handler to write or read
+    exception processing events. */
+bool replay_exception(void);
+/*! Used to determine that exception is pending.
+    Does not proceed to the next event in the log. */
+bool replay_has_exception(void);
+/*! Called by interrupt handlers to write or read
+    interrupt processing events.
+    \return true if interrupt should be processed */
+bool replay_interrupt(void);
+/*! Tries to read interrupt event from the file.
+    Returns true, when interrupt request is pending */
+bool replay_has_interrupt(void);
+
+/* Processing clocks and other time sources */
+
+/*! Save the specified clock */
+int64_t replay_save_clock(ReplayClockKind kind, int64_t clock);
+/*! Read the specified clock from the log or return cached data */
+int64_t replay_read_clock(ReplayClockKind kind);
+/*! Saves or reads the clock depending on the current replay mode. */
+#define REPLAY_CLOCK(clock, value)                                      \
+    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
+        : replay_mode == REPLAY_MODE_RECORD                             \
+            ? replay_save_clock((clock), (value))                       \
+        : (value))
+
+/* Events */
+
+/*! Called when qemu shutdown is requested. */
+void replay_shutdown_request(void);
+/*! Should be called at check points in the execution.
+    These check points are skipped, if they were not met.
+    Saves checkpoint in the SAVE mode and validates in the PLAY mode.
+    Returns 0 in PLAY mode if checkpoint was not found.
+    Returns 1 in all other cases. */
+bool replay_checkpoint(ReplayCheckpoint checkpoint);
+
+/* Asynchronous events queue */
+
+/*! Disables storing events in the queue */
+void replay_disable_events(void);
+/*! Returns true when saving events is enabled */
+bool replay_events_enabled(void);
+/*! Adds bottom half event to the queue */
+void replay_bh_schedule_event(QEMUBH *bh);
+/*! Adds input event to the queue */
+void replay_input_event(QemuConsole *src, InputEvent *evt);
+/*! Adds input sync event to the queue */
+void replay_input_sync_event(void);
+
+#endif
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 44570d17e..3bb889772 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -69,6 +69,8 @@ int qemu_reset_requested_get(void);
 void qemu_system_killed(int signal, pid_t pid);
 void qemu_devices_reset(void);
 void qemu_system_reset(bool report);
+void qemu_system_guest_panicked(void);
+size_t qemu_target_page_bits(void);
 
 void qemu_add_exit_notifier(Notifier *notify);
 void qemu_remove_exit_notifier(Notifier *notify);
@@ -82,14 +84,52 @@ void hmp_info_snapshots(Monitor *mon, const QDict *qdict);
 
 void qemu_announce_self(void);
 
+/* Subcommands for QEMU_VM_COMMAND */
+enum qemu_vm_cmd {
+    MIG_CMD_INVALID = 0,   /* Must be 0 */
+    MIG_CMD_OPEN_RETURN_PATH,  /* Tell the dest to open the Return path */
+    MIG_CMD_PING,              /* Request a PONG on the RP */
+
+    MIG_CMD_POSTCOPY_ADVISE,       /* Prior to any page transfers, just
+                                      warn we might want to do PC */
+    MIG_CMD_POSTCOPY_LISTEN,       /* Start listening for incoming
+                                      pages as it's running. */
+    MIG_CMD_POSTCOPY_RUN,          /* Start execution */
+
+    MIG_CMD_POSTCOPY_RAM_DISCARD,  /* A list of pages to discard that
+                                      were previously sent during
+                                      precopy but are dirty. */
+    MIG_CMD_PACKAGED,          /* Send a wrapped stream within this stream */
+    MIG_CMD_MAX
+};
+
+#define MAX_VM_CMD_PACKAGED_SIZE (1ul << 24)
+
 bool qemu_savevm_state_blocked(Error **errp);
 void qemu_savevm_state_begin(QEMUFile *f,
                              const MigrationParams *params);
 void qemu_savevm_state_header(QEMUFile *f);
-int qemu_savevm_state_iterate(QEMUFile *f);
-void qemu_savevm_state_complete(QEMUFile *f);
-void qemu_savevm_state_cancel(void);
-uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size);
+int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy);
+void qemu_savevm_state_cleanup(void);
+void qemu_savevm_state_complete_postcopy(QEMUFile *f);
+void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only);
+void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
+                               uint64_t *res_non_postcopiable,
+                               uint64_t *res_postcopiable);
+void qemu_savevm_command_send(QEMUFile *f, enum qemu_vm_cmd command,
+                              uint16_t len, uint8_t *data);
+void qemu_savevm_send_ping(QEMUFile *f, uint32_t value);
+void qemu_savevm_send_open_return_path(QEMUFile *f);
+int qemu_savevm_send_packaged(QEMUFile *f, const QEMUSizedBuffer *qsb);
+void qemu_savevm_send_postcopy_advise(QEMUFile *f);
+void qemu_savevm_send_postcopy_listen(QEMUFile *f);
+void qemu_savevm_send_postcopy_run(QEMUFile *f);
+
+void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
+                                           uint16_t len,
+                                           uint64_t *start_list,
+                                           uint64_t *length_list);
+
 int qemu_loadvm_state(QEMUFile *f);
 
 typedef enum DisplayType
@@ -132,6 +172,7 @@ extern int boot_menu;
 extern bool boot_strict;
 extern uint8_t *boot_splash_filedata;
 extern size_t boot_splash_filedata_size;
+extern bool enable_mlock;
 extern uint8_t qemu_extra_params_fw[2];
 extern QEMUClockType rtc_clock;
 extern const char *mem_path;
@@ -194,7 +235,7 @@ void device_add_bootindex_property(Object *obj, int32_t *bootindex,
 void restore_boot_order(void *opaque);
 void validate_bootdevices(const char *devices, Error **errp);
 
-/* handler to set the boot_device order for a specific type of QEMUMachine */
+/* handler to set the boot_device order for a specific type of MachineClass */
 typedef void QEMUBootSetHandler(void *opaque, const char *boot_order,
                                 Error **errp);
 void qemu_register_boot_set(QEMUBootSetHandler *func, void *opaque);
diff --git a/include/ui/console.h b/include/ui/console.h
index 047a2b464..5739bdd61 100644
--- a/include/ui/console.h
+++ b/include/ui/console.h
@@ -157,6 +157,14 @@ void cursor_set_mono(QEMUCursor *c,
 void cursor_get_mono_image(QEMUCursor *c, int foreground, uint8_t *mask);
 void cursor_get_mono_mask(QEMUCursor *c, int transparent, uint8_t *mask);
 
+typedef void *QEMUGLContext;
+typedef struct QEMUGLParams QEMUGLParams;
+
+struct QEMUGLParams {
+    int major_ver;
+    int minor_ver;
+};
+
 typedef struct DisplayChangeListenerOps {
     const char *dpy_name;
 
@@ -183,6 +191,21 @@ typedef struct DisplayChangeListenerOps {
                           int x, int y, int on);
     void (*dpy_cursor_define)(DisplayChangeListener *dcl,
                               QEMUCursor *cursor);
+
+    QEMUGLContext (*dpy_gl_ctx_create)(DisplayChangeListener *dcl,
+                                       QEMUGLParams *params);
+    void (*dpy_gl_ctx_destroy)(DisplayChangeListener *dcl,
+                               QEMUGLContext ctx);
+    int (*dpy_gl_ctx_make_current)(DisplayChangeListener *dcl,
+                                   QEMUGLContext ctx);
+    QEMUGLContext (*dpy_gl_ctx_get_current)(DisplayChangeListener *dcl);
+
+    void (*dpy_gl_scanout)(DisplayChangeListener *dcl,
+                           uint32_t backing_id, bool backing_y_0_top,
+                           uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+    void (*dpy_gl_update)(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
 } DisplayChangeListenerOps;
 
 struct DisplayChangeListener {
@@ -244,6 +267,20 @@ bool dpy_cursor_define_supported(QemuConsole *con);
 bool dpy_gfx_check_format(QemuConsole *con,
                           pixman_format_code_t format);
 
+void dpy_gl_scanout(QemuConsole *con,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+void dpy_gl_update(QemuConsole *con,
+                   uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
+QEMUGLContext dpy_gl_ctx_create(QemuConsole *con,
+                                QEMUGLParams *params);
+void dpy_gl_ctx_destroy(QemuConsole *con, QEMUGLContext ctx);
+int dpy_gl_ctx_make_current(QemuConsole *con, QEMUGLContext ctx);
+QEMUGLContext dpy_gl_ctx_get_current(QemuConsole *con);
+
+bool console_has_gl(QemuConsole *con);
+
 static inline int surface_stride(DisplaySurface *s)
 {
     return pixman_image_get_stride(s->image);
@@ -284,13 +321,23 @@ static inline pixman_format_code_t surface_format(DisplaySurface *s)
 #ifdef CONFIG_CURSES
 #include <curses.h>
 typedef chtype console_ch_t;
+extern chtype vga_to_curses[];
 #else
 typedef unsigned long console_ch_t;
 #endif
 static inline void console_write_ch(console_ch_t *dest, uint32_t ch)
 {
-    if (!(ch & 0xff))
+    uint8_t c = ch;
+#ifdef CONFIG_CURSES
+    if (vga_to_curses[c]) {
+        ch &= ~(console_ch_t)0xff;
+        ch |= vga_to_curses[c];
+    }
+#else
+    if (c == '\0') {
         ch |= ' ';
+    }
+#endif
     *dest = ch;
 }
 
@@ -386,7 +433,7 @@ static inline int vnc_display_pw_expire(const char *id, time_t expires)
 void curses_display_init(DisplayState *ds, int full_screen);
 
 /* input.c */
-int index_from_key(const char *key);
+int index_from_key(const char *key, size_t key_length);
 
 /* gtk.c */
 void early_gtk_display_init(int opengl);
diff --git a/include/ui/egl-context.h b/include/ui/egl-context.h
new file mode 100644
index 000000000..f004ce11a
--- /dev/null
+++ b/include/ui/egl-context.h
@@ -0,0 +1,14 @@
+#ifndef EGL_CONTEXT_H
+#define EGL_CONTEXT_H
+
+#include "ui/console.h"
+#include "ui/egl-helpers.h"
+
+QEMUGLContext qemu_egl_create_context(DisplayChangeListener *dcl,
+                                      QEMUGLParams *params);
+void qemu_egl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx);
+int qemu_egl_make_context_current(DisplayChangeListener *dcl,
+                                  QEMUGLContext ctx);
+QEMUGLContext qemu_egl_get_current_context(DisplayChangeListener *dcl);
+
+#endif /* EGL_CONTEXT_H */
diff --git a/include/ui/gtk.h b/include/ui/gtk.h
index ee6dffd30..bf289cff4 100644
--- a/include/ui/gtk.h
+++ b/include/ui/gtk.h
@@ -1,10 +1,6 @@
 #ifndef UI_GTK_H
 #define UI_GTK_H
 
-#ifdef _WIN32
-# define _WIN32_WINNT 0x0601 /* needed to get definition of MAPVK_VK_TO_VSC */
-#endif
-
 #ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
 /* Work around an -Wstrict-prototypes warning in GTK headers */
 #pragma GCC diagnostic push
@@ -24,6 +20,7 @@
 
 #if defined(CONFIG_OPENGL)
 #include "ui/egl-helpers.h"
+#include "ui/egl-context.h"
 #endif
 
 /* Compatibility define to let us build on both Gtk2 and Gtk3 */
@@ -50,6 +47,11 @@ typedef struct VirtualGfxConsole {
     EGLContext ectx;
     EGLSurface esurface;
     int glupdates;
+    int x, y, w, h;
+    GLuint tex_id;
+    GLuint fbo_id;
+    bool y0_top;
+    bool scanout_mode;
 #endif
 } VirtualGfxConsole;
 
@@ -94,6 +96,39 @@ void gd_egl_update(DisplayChangeListener *dcl,
 void gd_egl_refresh(DisplayChangeListener *dcl);
 void gd_egl_switch(DisplayChangeListener *dcl,
                    DisplaySurface *surface);
+QEMUGLContext gd_egl_create_context(DisplayChangeListener *dcl,
+                                    QEMUGLParams *params);
+void gd_egl_scanout(DisplayChangeListener *dcl,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y,
+                    uint32_t w, uint32_t h);
+void gd_egl_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h);
 void gtk_egl_init(void);
+int gd_egl_make_current(DisplayChangeListener *dcl,
+                        QEMUGLContext ctx);
+
+/* ui/gtk-gl-area.c */
+void gd_gl_area_init(VirtualConsole *vc);
+void gd_gl_area_draw(VirtualConsole *vc);
+void gd_gl_area_update(DisplayChangeListener *dcl,
+                       int x, int y, int w, int h);
+void gd_gl_area_refresh(DisplayChangeListener *dcl);
+void gd_gl_area_switch(DisplayChangeListener *dcl,
+                       DisplaySurface *surface);
+QEMUGLContext gd_gl_area_create_context(DisplayChangeListener *dcl,
+                                        QEMUGLParams *params);
+void gd_gl_area_destroy_context(DisplayChangeListener *dcl,
+                                QEMUGLContext ctx);
+void gd_gl_area_scanout(DisplayChangeListener *dcl,
+                        uint32_t backing_id, bool backing_y_0_top,
+                        uint32_t x, uint32_t y,
+                        uint32_t w, uint32_t h);
+void gd_gl_area_scanout_flush(DisplayChangeListener *dcl,
+                              uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+void gtk_gl_area_init(void);
+QEMUGLContext gd_gl_area_get_current_context(DisplayChangeListener *dcl);
+int gd_gl_area_make_current(DisplayChangeListener *dcl,
+                            QEMUGLContext ctx);
 
 #endif /* UI_GTK_H */
diff --git a/include/ui/input.h b/include/ui/input.h
index 5d5ac0066..d06a12dd4 100644
--- a/include/ui/input.h
+++ b/include/ui/input.h
@@ -33,7 +33,9 @@ void qemu_input_handler_bind(QemuInputHandlerState *s,
                              const char *device_id, int head,
                              Error **errp);
 void qemu_input_event_send(QemuConsole *src, InputEvent *evt);
+void qemu_input_event_send_impl(QemuConsole *src, InputEvent *evt);
 void qemu_input_event_sync(void);
+void qemu_input_event_sync_impl(void);
 
 InputEvent *qemu_input_event_new_key(KeyValue *key, bool down);
 void qemu_input_event_send_key(QemuConsole *src, KeyValue *key, bool down);
diff --git a/include/ui/qemu-spice.h b/include/ui/qemu-spice.h
index 0dff4229f..f9ce357ff 100644
--- a/include/ui/qemu-spice.h
+++ b/include/ui/qemu-spice.h
@@ -43,9 +43,7 @@ int qemu_spice_set_pw_expire(time_t expires);
 int qemu_spice_migrate_info(const char *hostname, int port, int tls_port,
                             const char *subject);
 
-CharDriverState *qemu_chr_open_spice_vmc(const char *type);
 #if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name);
 void qemu_spice_register_ports(void);
 #else
 static inline CharDriverState *qemu_chr_open_spice_port(const char *name)
diff --git a/include/ui/shader.h b/include/ui/shader.h
index 8509596ac..f7d86188b 100644
--- a/include/ui/shader.h
+++ b/include/ui/shader.h
@@ -3,7 +3,9 @@
 
 #include <epoxy/gl.h>
 
-void qemu_gl_run_texture_blit(GLint texture_blit_prog);
+GLuint qemu_gl_init_texture_blit(GLint texture_blit_prog);
+void qemu_gl_run_texture_blit(GLint texture_blit_prog,
+                              GLint texture_blit_vao);
 
 GLuint qemu_gl_create_compile_shader(GLenum type, const GLchar *src);
 GLuint qemu_gl_create_link_program(GLuint vert, GLuint frag);
diff --git a/iohandler.c b/iohandler.c
index 826f713e9..eb6808355 100644
--- a/iohandler.c
+++ b/iohandler.c
@@ -32,111 +32,31 @@
 #include <sys/wait.h>
 #endif
 
-typedef struct IOHandlerRecord {
-    IOHandler *fd_read;
-    IOHandler *fd_write;
-    void *opaque;
-    QLIST_ENTRY(IOHandlerRecord) next;
-    int fd;
-    int pollfds_idx;
-    bool deleted;
-} IOHandlerRecord;
-
-static QLIST_HEAD(, IOHandlerRecord) io_handlers =
-    QLIST_HEAD_INITIALIZER(io_handlers);
+/* This context runs on top of main loop. We can't reuse qemu_aio_context
+ * because iohandlers mustn't be polled by aio_poll(qemu_aio_context). */
+static AioContext *iohandler_ctx;
 
-void qemu_set_fd_handler(int fd,
-                         IOHandler *fd_read,
-                         IOHandler *fd_write,
-                         void *opaque)
+static void iohandler_init(void)
 {
-    IOHandlerRecord *ioh;
-
-    assert(fd >= 0);
-
-    if (!fd_read && !fd_write) {
-        QLIST_FOREACH(ioh, &io_handlers, next) {
-            if (ioh->fd == fd) {
-                ioh->deleted = 1;
-                break;
-            }
-        }
-    } else {
-        QLIST_FOREACH(ioh, &io_handlers, next) {
-            if (ioh->fd == fd)
-                goto found;
-        }
-        ioh = g_malloc0(sizeof(IOHandlerRecord));
-        QLIST_INSERT_HEAD(&io_handlers, ioh, next);
-    found:
-        ioh->fd = fd;
-        ioh->fd_read = fd_read;
-        ioh->fd_write = fd_write;
-        ioh->opaque = opaque;
-        ioh->pollfds_idx = -1;
-        ioh->deleted = 0;
-        qemu_notify_event();
+    if (!iohandler_ctx) {
+        iohandler_ctx = aio_context_new(&error_abort);
     }
 }
 
-void qemu_iohandler_fill(GArray *pollfds)
+GSource *iohandler_get_g_source(void)
 {
-    IOHandlerRecord *ioh;
-
-    QLIST_FOREACH(ioh, &io_handlers, next) {
-        int events = 0;
-
-        if (ioh->deleted)
-            continue;
-        if (ioh->fd_read) {
-            events |= G_IO_IN | G_IO_HUP | G_IO_ERR;
-        }
-        if (ioh->fd_write) {
-            events |= G_IO_OUT | G_IO_ERR;
-        }
-        if (events) {
-            GPollFD pfd = {
-                .fd = ioh->fd,
-                .events = events,
-            };
-            ioh->pollfds_idx = pollfds->len;
-            g_array_append_val(pollfds, pfd);
-        } else {
-            ioh->pollfds_idx = -1;
-        }
-    }
+    iohandler_init();
+    return aio_get_g_source(iohandler_ctx);
 }
 
-void qemu_iohandler_poll(GArray *pollfds, int ret)
+void qemu_set_fd_handler(int fd,
+                         IOHandler *fd_read,
+                         IOHandler *fd_write,
+                         void *opaque)
 {
-    if (ret > 0) {
-        IOHandlerRecord *pioh, *ioh;
-
-        QLIST_FOREACH_SAFE(ioh, &io_handlers, next, pioh) {
-            int revents = 0;
-
-            if (!ioh->deleted && ioh->pollfds_idx != -1) {
-                GPollFD *pfd = &g_array_index(pollfds, GPollFD,
-                                              ioh->pollfds_idx);
-                revents = pfd->revents;
-            }
-
-            if (!ioh->deleted && ioh->fd_read &&
-                (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR))) {
-                ioh->fd_read(ioh->opaque);
-            }
-            if (!ioh->deleted && ioh->fd_write &&
-                (revents & (G_IO_OUT | G_IO_ERR))) {
-                ioh->fd_write(ioh->opaque);
-            }
-
-            /* Do this last in case read/write handlers marked it for deletion */
-            if (ioh->deleted) {
-                QLIST_REMOVE(ioh, next);
-                g_free(ioh);
-            }
-        }
-    }
+    iohandler_init();
+    aio_set_fd_handler(iohandler_ctx, fd, false,
+                       fd_read, fd_write, opaque);
 }
 
 /* reaping of zombies.  right now we're not passing the status to
diff --git a/ioport.c b/ioport.c
index e39093edb..193ef7685 100644
--- a/ioport.c
+++ b/ioport.c
@@ -30,14 +30,6 @@
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 
-//#define DEBUG_IOPORT
-
-#ifdef DEBUG_IOPORT
-#  define LOG_IOPORT(...) qemu_log_mask(CPU_LOG_IOPORT, ## __VA_ARGS__)
-#else
-#  define LOG_IOPORT(...) do { } while (0)
-#endif
-
 typedef struct MemoryRegionPortioList {
     MemoryRegion mr;
     void *portio_opaque;
@@ -62,8 +54,7 @@ const MemoryRegionOps unassigned_io_ops = {
 
 void cpu_outb(pio_addr_t addr, uint8_t val)
 {
-    LOG_IOPORT("outb: %04"FMT_pioaddr" %02"PRIx8"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'b', val);
     address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                         &val, 1);
 }
@@ -72,8 +63,7 @@ void cpu_outw(pio_addr_t addr, uint16_t val)
 {
     uint8_t buf[2];
 
-    LOG_IOPORT("outw: %04"FMT_pioaddr" %04"PRIx16"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'w', val);
     stw_p(buf, val);
     address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                         buf, 2);
@@ -83,8 +73,7 @@ void cpu_outl(pio_addr_t addr, uint32_t val)
 {
     uint8_t buf[4];
 
-    LOG_IOPORT("outl: %04"FMT_pioaddr" %08"PRIx32"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'l', val);
     stl_p(buf, val);
     address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                         buf, 4);
@@ -96,8 +85,7 @@ uint8_t cpu_inb(pio_addr_t addr)
 
     address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                        &val, 1);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inb : %04"FMT_pioaddr" %02"PRIx8"\n", addr, val);
+    trace_cpu_in(addr, 'b', val);
     return val;
 }
 
@@ -108,8 +96,7 @@ uint16_t cpu_inw(pio_addr_t addr)
 
     address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED, buf, 2);
     val = lduw_p(buf);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inw : %04"FMT_pioaddr" %04"PRIx16"\n", addr, val);
+    trace_cpu_in(addr, 'w', val);
     return val;
 }
 
@@ -120,8 +107,7 @@ uint32_t cpu_inl(pio_addr_t addr)
 
     address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED, buf, 4);
     val = ldl_p(buf);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inl : %04"FMT_pioaddr" %08"PRIx32"\n", addr, val);
+    trace_cpu_in(addr, 'l', val);
     return val;
 }
 
diff --git a/iothread.c b/iothread.c
index da6ce7b30..1b8c2bbec 100644
--- a/iothread.c
+++ b/iothread.c
@@ -72,6 +72,7 @@ static void iothread_complete(UserCreatable *obj, Error **errp)
 {
     Error *local_error = NULL;
     IOThread *iothread = IOTHREAD(obj);
+    char *name, *thread_name;
 
     iothread->stopping = false;
     iothread->thread_id = -1;
@@ -87,8 +88,12 @@ static void iothread_complete(UserCreatable *obj, Error **errp)
     /* This assumes we are called from a thread with useful CPU affinity for us
      * to inherit.
      */
-    qemu_thread_create(&iothread->thread, "iothread", iothread_run,
+    name = object_get_canonical_path_component(OBJECT(obj));
+    thread_name = g_strdup_printf("IO %s", name);
+    qemu_thread_create(&iothread->thread, thread_name, iothread_run,
                        iothread, QEMU_THREAD_JOINABLE);
+    g_free(thread_name);
+    g_free(name);
 
     /* Wait for initialization to complete */
     qemu_mutex_lock(&iothread->init_done_lock);
diff --git a/kvm-all.c b/kvm-all.c
index 06e06f2b3..c648b814e 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -24,6 +24,7 @@
 #include "qemu/atomic.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "hw/hw.h"
 #include "hw/pci/msi.h"
 #include "hw/s390x/adapter.h"
@@ -76,8 +77,6 @@ struct KVMState
 #ifdef KVM_CAP_SET_GUEST_DEBUG
     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
 #endif
-    int pit_state2;
-    int xsave, xcrs;
     int many_ioeventfds;
     int intx_set_mask;
     /* The man page (and posix) say ioctl numbers are signed int, but
@@ -92,7 +91,6 @@ struct KVMState
     uint32_t *used_gsi_bitmap;
     unsigned int gsi_count;
     QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
-    bool direct_msi;
 #endif
     KVMMemoryListener memory_listener;
 };
@@ -110,6 +108,8 @@ bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
 bool kvm_vm_attributes_allowed;
+bool kvm_direct_msi_allowed;
+bool kvm_ioeventfd_any_length_allowed;
 
 static const KVMCapabilityInfo kvm_required_capabilites[] = {
     KVM_CAP_INFO(USER_MEMORY),
@@ -641,15 +641,15 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml,
     /* kvm works in page size chunks, but the function may be called
        with sub-page size and unaligned start address. Pad the start
        address to next and truncate size to previous page boundary. */
-    delta = (TARGET_PAGE_SIZE - (start_addr & ~TARGET_PAGE_MASK));
-    delta &= ~TARGET_PAGE_MASK;
+    delta = qemu_real_host_page_size - (start_addr & ~qemu_real_host_page_mask);
+    delta &= ~qemu_real_host_page_mask;
     if (delta > size) {
         return;
     }
     start_addr += delta;
     size -= delta;
-    size &= TARGET_PAGE_MASK;
-    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
+    size &= qemu_real_host_page_mask;
+    if (!size || (start_addr & ~qemu_real_host_page_mask)) {
         return;
     }
 
@@ -978,7 +978,7 @@ void kvm_init_irq_routing(KVMState *s)
     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
     s->nr_allocated_irq_routes = 0;
 
-    if (!s->direct_msi) {
+    if (!kvm_direct_msi_allowed) {
         for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
             QTAILQ_INIT(&s->msi_hashtab[i]);
         }
@@ -1112,7 +1112,7 @@ static int kvm_irqchip_get_virq(KVMState *s)
      * number can succeed even though a new route entry cannot be added.
      * When this happens, flush dynamic MSI entries to free IRQ route entries.
      */
-    if (!s->direct_msi && s->irq_routes->nr == s->gsi_count) {
+    if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {
         kvm_flush_dynamic_msi_routes(s);
     }
 
@@ -1149,7 +1149,7 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
     struct kvm_msi msi;
     KVMMSIRoute *route;
 
-    if (s->direct_msi) {
+    if (kvm_direct_msi_allowed) {
         msi.address_lo = (uint32_t)msg.address;
         msi.address_hi = msg.address >> 32;
         msi.data = le32_to_cpu(msg.data);
@@ -1188,7 +1188,7 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
     return kvm_set_irq(s, route->kroute.gsi, 1);
 }
 
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev)
 {
     struct kvm_irq_routing_entry kroute = {};
     int virq;
@@ -1212,7 +1212,7 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
     kroute.u.msi.address_lo = (uint32_t)msg.address;
     kroute.u.msi.address_hi = msg.address >> 32;
     kroute.u.msi.data = le32_to_cpu(msg.data);
-    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
+    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
         kvm_irqchip_release_virq(s, virq);
         return -EINVAL;
     }
@@ -1223,7 +1223,8 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
     return virq;
 }
 
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev)
 {
     struct kvm_irq_routing_entry kroute = {};
 
@@ -1241,7 +1242,7 @@ int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
     kroute.u.msi.address_lo = (uint32_t)msg.address;
     kroute.u.msi.address_hi = msg.address >> 32;
     kroute.u.msi.data = le32_to_cpu(msg.data);
-    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
+    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
         return -EINVAL;
     }
 
@@ -1293,7 +1294,6 @@ int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
     kroute.u.adapter.adapter_id = adapter->adapter_id;
 
     kvm_add_routing_entry(s, &kroute);
-    kvm_irqchip_commit_routes(s);
 
     return virq;
 }
@@ -1462,7 +1462,6 @@ static int kvm_init(MachineState *ms)
      * page size for the system though.
      */
     assert(TARGET_PAGE_SIZE <= getpagesize());
-    page_size_init();
 
     s->sigmask_len = 8;
 
@@ -1585,20 +1584,8 @@ static int kvm_init(MachineState *ms)
     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 #endif
 
-#ifdef KVM_CAP_XSAVE
-    s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
-#endif
-
-#ifdef KVM_CAP_XCRS
-    s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
-#endif
-
-#ifdef KVM_CAP_PIT_STATE2
-    s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
-#endif
-
 #ifdef KVM_CAP_IRQ_ROUTING
-    s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
+    kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
 #endif
 
     s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
@@ -1625,6 +1612,9 @@ static int kvm_init(MachineState *ms)
     kvm_vm_attributes_allowed =
         (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
 
+    kvm_ioeventfd_any_length_allowed =
+        (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0);
+
     ret = kvm_arch_init(ms, s);
     if (ret < 0) {
         goto err;
@@ -1779,11 +1769,6 @@ void kvm_cpu_synchronize_post_init(CPUState *cpu)
     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
 }
 
-void kvm_cpu_clean_state(CPUState *cpu)
-{
-    cpu->kvm_vcpu_dirty = false;
-}
-
 int kvm_cpu_exec(CPUState *cpu)
 {
     struct kvm_run *run = cpu->kvm_run;
@@ -1890,6 +1875,12 @@ int kvm_cpu_exec(CPUState *cpu)
                 qemu_system_reset_request();
                 ret = EXCP_INTERRUPT;
                 break;
+            case KVM_SYSTEM_EVENT_CRASH:
+                qemu_mutex_lock_iothread();
+                qemu_system_guest_panicked();
+                qemu_mutex_unlock_iothread();
+                ret = 0;
+                break;
             default:
                 DPRINTF("kvm_arch_handle_exit\n");
                 ret = kvm_arch_handle_exit(cpu, run);
@@ -2003,6 +1994,39 @@ int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
     return ret ? 0 : 1;
 }
 
+int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
+{
+    struct kvm_device_attr attribute = {
+        .group = group,
+        .attr = attr,
+        .flags = 0,
+    };
+
+    return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
+}
+
+void kvm_device_access(int fd, int group, uint64_t attr,
+                       void *val, bool write)
+{
+    struct kvm_device_attr kvmattr;
+    int err;
+
+    kvmattr.flags = 0;
+    kvmattr.group = group;
+    kvmattr.attr = attr;
+    kvmattr.addr = (uintptr_t)val;
+
+    err = kvm_device_ioctl(fd,
+                           write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
+                           &kvmattr);
+    if (err < 0) {
+        error_report("KVM_%s_DEVICE_ATTR failed: %s\n"
+                     "Group %d attr 0x%016" PRIx64, write ? "SET" : "GET",
+                     strerror(-err), group, attr);
+        abort();
+    }
+}
+
 int kvm_has_sync_mmu(void)
 {
     return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
@@ -2023,21 +2047,6 @@ int kvm_has_debugregs(void)
     return kvm_state->debugregs;
 }
 
-int kvm_has_xsave(void)
-{
-    return kvm_state->xsave;
-}
-
-int kvm_has_xcrs(void)
-{
-    return kvm_state->xcrs;
-}
-
-int kvm_has_pit_state2(void)
-{
-    return kvm_state->pit_state2;
-}
-
 int kvm_has_many_ioeventfds(void)
 {
     if (!kvm_enabled()) {
diff --git a/kvm-stub.c b/kvm-stub.c
index d9ad624ee..dc97a5edf 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -30,6 +30,7 @@ bool kvm_gsi_routing_allowed;
 bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
+bool kvm_ioeventfd_any_length_allowed;
 
 int kvm_init_vcpu(CPUState *cpu)
 {
@@ -67,11 +68,6 @@ int kvm_has_many_ioeventfds(void)
     return 0;
 }
 
-int kvm_has_pit_state2(void)
-{
-    return 0;
-}
-
 void kvm_setup_guest_memory(void *start, size_t size)
 {
 }
@@ -115,7 +111,7 @@ int kvm_on_sigbus(int code, void *addr)
 }
 
 #ifndef CONFIG_USER_ONLY
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev)
 {
     return -ENOSYS;
 }
@@ -128,7 +124,8 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
 {
 }
 
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev)
 {
     return -ENOSYS;
 }
diff --git a/libcacard/Makefile b/libcacard/Makefile
deleted file mode 100644
index b5eddff11..000000000
--- a/libcacard/Makefile
+++ /dev/null
@@ -1,45 +0,0 @@
-libcacard_includedir=$(includedir)/cacard
-
-TOOLS += vscclient$(EXESUF)
-
-# objects linked into a shared library, built with libtool with -fPIC if required
-libcacard-obj-y = $(libcacard-y)
-libcacard-lobj-y=$(patsubst %.o,%.lo,$(libcacard-obj-y))
-
-# libtool will build the .o files, too
-$(libcacard-obj-y): | $(libcacard-lobj-y)
-
-all: libcacard.la libcacard.pc
-
-vscclient$(EXESUF): libcacard/vscclient.o libcacard.la
-	$(call LINK,$^)
-
-#########################################################################
-# Rules for building libcacard standalone library
-
-libcacard.la: LDFLAGS += -rpath $(libdir) -no-undefined \
-	-export-symbols $(SRC_PATH)/libcacard/libcacard.syms
-# Prevent libcacard.so linking against the entire world of 3rd party libs
-libcacard.la: LIBS =
-libcacard.la: $(libcacard-lobj-y)
-	$(call LINK,$^)
-
-libcacard.pc: $(SRC_PATH)/libcacard/libcacard.pc.in
-	$(call quiet-command,sed -e 's|@LIBDIR@|$(libdir)|' \
-		-e 's|@INCLUDEDIR@|$(libcacard_includedir)|' \
-	    -e 's|@VERSION@|$(shell cat $(SRC_PATH)/VERSION)|' \
-		-e 's|@PREFIX@|$(prefix)|' $< > libcacard.pc,\
-	"  GEN   $@")
-
-.PHONY: install-libcacard
-
-install: install-libcacard
-install-libcacard: libcacard.pc libcacard.la
-	$(INSTALL_DIR) "$(DESTDIR)$(libdir)"
-	$(INSTALL_DIR) "$(DESTDIR)$(libdir)/pkgconfig"
-	$(INSTALL_DIR) "$(DESTDIR)$(libcacard_includedir)"
-	$(INSTALL_LIB) libcacard.la "$(DESTDIR)$(libdir)"
-	$(INSTALL_DATA) libcacard.pc "$(DESTDIR)$(libdir)/pkgconfig"
-	for inc in $(SRC_PATH)/libcacard/*.h; do \
-		$(INSTALL_DATA) $$inc "$(DESTDIR)$(libcacard_includedir)"; \
-	done
diff --git a/libcacard/cac.c b/libcacard/cac.c
deleted file mode 100644
index bc84534f9..000000000
--- a/libcacard/cac.c
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
- * implement the applets for the CAC card.
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include <string.h>
-#include <stdbool.h>
-
-#include "cac.h"
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-
-/* private data for PKI applets */
-typedef struct CACPKIAppletDataStruct {
-    unsigned char *cert;
-    int cert_len;
-    unsigned char *cert_buffer;
-    int cert_buffer_len;
-    unsigned char *sign_buffer;
-    int sign_buffer_len;
-    VCardKey *key;
-} CACPKIAppletData;
-
-/*
- * CAC applet private data
- */
-struct VCardAppletPrivateStruct {
-    union {
-        CACPKIAppletData pki_data;
-        void *reserved;
-    } u;
-};
-
-/*
- * handle all the APDU's that are common to all CAC applets
- */
-static VCardStatus
-cac_common_process_apdu(VCard *card, VCardAPDU *apdu, VCardResponse **response)
-{
-    int ef;
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case VCARD7816_INS_SELECT_FILE:
-        if (apdu->a_p1 != 0x02) {
-            /* let the 7816 code handle applet switches */
-            ret = VCARD_NEXT;
-            break;
-        }
-        /* handle file id setting */
-        if (apdu->a_Lc != 2) {
-            *response = vcard_make_response(
-                VCARD7816_STATUS_ERROR_DATA_INVALID);
-            ret = VCARD_DONE;
-            break;
-        }
-        /* CAC 1.0 only supports ef = 0 */
-        ef = apdu->a_body[0] | (apdu->a_body[1] << 8);
-        if (ef != 0) {
-            *response = vcard_make_response(
-                VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-            ret = VCARD_DONE;
-            break;
-        }
-        *response = vcard_make_response(VCARD7816_STATUS_SUCCESS);
-        ret = VCARD_DONE;
-        break;
-    case VCARD7816_INS_GET_RESPONSE:
-    case VCARD7816_INS_VERIFY:
-        /* let the 7816 code handle these */
-        ret = VCARD_NEXT;
-        break;
-    case CAC_GET_PROPERTIES:
-    case CAC_GET_ACR:
-        /* skip these for now, this will probably be needed */
-        *response = vcard_make_response(VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-        ret = VCARD_DONE;
-        break;
-    default:
-        *response = vcard_make_response(
-            VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    }
-    return ret;
-}
-
-/*
- *  reset the inter call state between applet selects
- */
-static VCardStatus
-cac_applet_pki_reset(VCard *card, int channel)
-{
-    VCardAppletPrivate *applet_private;
-    CACPKIAppletData *pki_applet;
-    applet_private = vcard_get_current_applet_private(card, channel);
-    assert(applet_private);
-    pki_applet = &(applet_private->u.pki_data);
-
-    pki_applet->cert_buffer = NULL;
-    g_free(pki_applet->sign_buffer);
-    pki_applet->sign_buffer = NULL;
-    pki_applet->cert_buffer_len = 0;
-    pki_applet->sign_buffer_len = 0;
-    return VCARD_DONE;
-}
-
-static VCardStatus
-cac_applet_pki_process_apdu(VCard *card, VCardAPDU *apdu,
-                            VCardResponse **response)
-{
-    CACPKIAppletData *pki_applet;
-    VCardAppletPrivate *applet_private;
-    int size, next;
-    unsigned char *sign_buffer;
-    bool retain_sign_buffer = FALSE;
-    vcard_7816_status_t status;
-    VCardStatus ret = VCARD_FAIL;
-
-    applet_private = vcard_get_current_applet_private(card, apdu->a_channel);
-    assert(applet_private);
-    pki_applet = &(applet_private->u.pki_data);
-
-    switch (apdu->a_ins) {
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-            VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-        ret = VCARD_DONE;
-        break;
-    case CAC_GET_CERTIFICATE:
-        if ((apdu->a_p2 != 0) || (apdu->a_p1 != 0)) {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        assert(pki_applet->cert != NULL);
-        size = apdu->a_Le;
-        if (pki_applet->cert_buffer == NULL) {
-            pki_applet->cert_buffer = pki_applet->cert;
-            pki_applet->cert_buffer_len = pki_applet->cert_len;
-        }
-        size = MIN(size, pki_applet->cert_buffer_len);
-        next = MIN(255, pki_applet->cert_buffer_len - size);
-        *response = vcard_response_new_bytes(
-                        card, pki_applet->cert_buffer, size,
-                        apdu->a_Le, next ?
-                        VCARD7816_SW1_WARNING_CHANGE :
-                        VCARD7816_SW1_SUCCESS,
-                        next);
-        pki_applet->cert_buffer += size;
-        pki_applet->cert_buffer_len -= size;
-        if ((*response == NULL) || (next == 0)) {
-            pki_applet->cert_buffer = NULL;
-        }
-        if (*response == NULL) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        ret = VCARD_DONE;
-        break;
-    case CAC_SIGN_DECRYPT:
-        if (apdu->a_p2 != 0) {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        size = apdu->a_Lc;
-
-        sign_buffer = g_realloc(pki_applet->sign_buffer,
-                                pki_applet->sign_buffer_len + size);
-        memcpy(sign_buffer+pki_applet->sign_buffer_len, apdu->a_body, size);
-        size += pki_applet->sign_buffer_len;
-        switch (apdu->a_p1) {
-        case  0x80:
-            /* p1 == 0x80 means we haven't yet sent the whole buffer, wait for
-             * the rest */
-            pki_applet->sign_buffer = sign_buffer;
-            pki_applet->sign_buffer_len = size;
-            *response = vcard_make_response(VCARD7816_STATUS_SUCCESS);
-            retain_sign_buffer = TRUE;
-            break;
-        case 0x00:
-            /* we now have the whole buffer, do the operation, result will be
-             * in the sign_buffer */
-            status = vcard_emul_rsa_op(card, pki_applet->key,
-                                       sign_buffer, size);
-            if (status != VCARD7816_STATUS_SUCCESS) {
-                *response = vcard_make_response(status);
-                break;
-            }
-            *response = vcard_response_new(card, sign_buffer, size, apdu->a_Le,
-                                                     VCARD7816_STATUS_SUCCESS);
-            if (*response == NULL) {
-                *response = vcard_make_response(
-                                VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-            }
-            break;
-        default:
-           *response = vcard_make_response(
-                                VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        if (!retain_sign_buffer) {
-            g_free(sign_buffer);
-            pki_applet->sign_buffer = NULL;
-            pki_applet->sign_buffer_len = 0;
-        }
-        ret = VCARD_DONE;
-        break;
-    case CAC_READ_BUFFER:
-        /* new CAC call, go ahead and use the old version for now */
-        /* TODO: implement */
-        *response = vcard_make_response(
-                                VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-
-static VCardStatus
-cac_applet_id_process_apdu(VCard *card, VCardAPDU *apdu,
-                           VCardResponse **response)
-{
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-        ret = VCARD_DONE;
-        break;
-    case CAC_READ_BUFFER:
-        /* new CAC call, go ahead and use the old version for now */
-        /* TODO: implement */
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-
-/*
- * TODO: if we ever want to support general CAC middleware, we will need to
- * implement the various containers.
- */
-static VCardStatus
-cac_applet_container_process_apdu(VCard *card, VCardAPDU *apdu,
-                                  VCardResponse **response)
-{
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case CAC_READ_BUFFER:
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-/*
- * utilities for creating and destroying the private applet data
- */
-static void
-cac_delete_pki_applet_private(VCardAppletPrivate *applet_private)
-{
-    CACPKIAppletData *pki_applet_data;
-
-    if (applet_private == NULL) {
-        return;
-    }
-    pki_applet_data = &(applet_private->u.pki_data);
-    g_free(pki_applet_data->cert);
-    g_free(pki_applet_data->sign_buffer);
-    if (pki_applet_data->key != NULL) {
-        vcard_emul_delete_key(pki_applet_data->key);
-    }
-    g_free(applet_private);
-}
-
-static VCardAppletPrivate *
-cac_new_pki_applet_private(const unsigned char *cert,
-                           int cert_len, VCardKey *key)
-{
-    CACPKIAppletData *pki_applet_data;
-    VCardAppletPrivate *applet_private;
-
-    applet_private = g_new0(VCardAppletPrivate, 1);
-    pki_applet_data = &(applet_private->u.pki_data);
-    pki_applet_data->cert = (unsigned char *)g_malloc(cert_len+1);
-    /*
-     * if we want to support compression, then we simply change the 0 to a 1
-     * and compress the cert data with libz
-     */
-    pki_applet_data->cert[0] = 0; /* not compressed */
-    memcpy(&pki_applet_data->cert[1], cert, cert_len);
-    pki_applet_data->cert_len = cert_len+1;
-
-    pki_applet_data->key = key;
-    return applet_private;
-}
-
-
-/*
- * create a new cac applet which links to a given cert
- */
-static VCardApplet *
-cac_new_pki_applet(int i, const unsigned char *cert,
-                   int cert_len, VCardKey *key)
-{
-    VCardAppletPrivate *applet_private;
-    VCardApplet *applet;
-    unsigned char pki_aid[] = { 0xa0, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00 };
-    int pki_aid_len = sizeof(pki_aid);
-
-    pki_aid[pki_aid_len-1] = i;
-
-    applet_private = cac_new_pki_applet_private(cert, cert_len, key);
-    if (applet_private == NULL) {
-        goto failure;
-    }
-    applet = vcard_new_applet(cac_applet_pki_process_apdu, cac_applet_pki_reset,
-                              pki_aid, pki_aid_len);
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_set_applet_private(applet, applet_private,
-                             cac_delete_pki_applet_private);
-    applet_private = NULL;
-
-    return applet;
-
-failure:
-    if (applet_private != NULL) {
-        cac_delete_pki_applet_private(applet_private);
-    }
-    return NULL;
-}
-
-
-static unsigned char cac_default_container_aid[] = {
-    0xa0, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00 };
-static unsigned char cac_id_aid[] = {
-    0xa0, 0x00, 0x00, 0x00, 0x79, 0x03, 0x00 };
-/*
- * Initialize the cac card. This is the only public function in this file. All
- * the rest are connected through function pointers.
- */
-VCardStatus
-cac_card_init(VReader *reader, VCard *card,
-              const char *params,
-              unsigned char * const *cert,
-              int cert_len[],
-              VCardKey *key[] /* adopt the keys*/,
-              int cert_count)
-{
-    int i;
-    VCardApplet *applet;
-
-    /* CAC Cards are VM Cards */
-    vcard_set_type(card, VCARD_VM);
-
-    /* create one PKI applet for each cert */
-    for (i = 0; i < cert_count; i++) {
-        applet = cac_new_pki_applet(i, cert[i], cert_len[i], key[i]);
-        if (applet == NULL) {
-            goto failure;
-        }
-        vcard_add_applet(card, applet);
-    }
-
-    /* create a default blank container applet */
-    applet = vcard_new_applet(cac_applet_container_process_apdu,
-                              NULL, cac_default_container_aid,
-                              sizeof(cac_default_container_aid));
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_add_applet(card, applet);
-
-    /* create a default blank container applet */
-    applet = vcard_new_applet(cac_applet_id_process_apdu,
-                              NULL, cac_id_aid,
-                              sizeof(cac_id_aid));
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_add_applet(card, applet);
-    return VCARD_DONE;
-
-failure:
-    return VCARD_FAIL;
-}
-
diff --git a/libcacard/cac.h b/libcacard/cac.h
deleted file mode 100644
index d24a2a846..000000000
--- a/libcacard/cac.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * defines the entry point for the cac card. Only used by cac.c anc
- * vcard_emul_type.c
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CAC_H
-#define CAC_H 1
-#include "vcard.h"
-#include "vreader.h"
-
-#define CAC_GET_PROPERTIES  0x56
-#define CAC_GET_ACR         0x4c
-#define CAC_READ_BUFFER     0x52
-#define CAC_UPDATE_BUFFER   0x58
-#define CAC_SIGN_DECRYPT    0x42
-#define CAC_GET_CERTIFICATE 0x36
-
-/*
- * Initialize the cac card. This is the only public function in this file. All
- * the rest are connected through function pointers.
- */
-VCardStatus cac_card_init(VReader *reader, VCard *card, const char *params,
-              unsigned char * const *cert, int cert_len[],
-              VCardKey *key[] /* adopt the keys*/,
-              int cert_count);
-
-/* not yet implemented */
-VCardStatus cac_is_cac_card(VReader *reader);
-#endif
diff --git a/libcacard/card_7816.c b/libcacard/card_7816.c
deleted file mode 100644
index 22fd334d1..000000000
--- a/libcacard/card_7816.c
+++ /dev/null
@@ -1,757 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include <string.h>
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-
-/*
- * set the status bytes based on the status word
- */
-static void
-vcard_response_set_status(VCardResponse *response, vcard_7816_status_t status)
-{
-    unsigned char sw1, sw2;
-    response->b_status = status; /* make sure the status and swX representations
-                                  * are consistent */
-    sw1 = (status >> 8) & 0xff;
-    sw2 = status & 0xff;
-    response->b_sw1 = sw1;
-    response->b_sw2 = sw2;
-    response->b_data[response->b_len] = sw1;
-    response->b_data[response->b_len+1] = sw2;
-}
-
-/*
- * set the status bytes in a response buffer
- */
-static void
-vcard_response_set_status_bytes(VCardResponse *response,
-                               unsigned char sw1, unsigned char sw2)
-{
-    response->b_status = sw1 << 8 | sw2;
-    response->b_sw1 = sw1;
-    response->b_sw2 = sw2;
-    response->b_data[response->b_len] = sw1;
-    response->b_data[response->b_len+1] = sw2;
-}
-
-/*
- * allocate a VCardResponse structure, plus space for the data buffer, and
- * set up everything but the resonse bytes.
- */
-VCardResponse *
-vcard_response_new_data(unsigned char *buf, int len)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = g_malloc(len + 2);
-    memcpy(new_response->b_data, buf, len);
-    new_response->b_total_len = len+2;
-    new_response->b_len = len;
-    new_response->b_type = VCARD_MALLOC;
-    return new_response;
-}
-
-static VCardResponse *
-vcard_init_buffer_response(VCard *card, unsigned char *buf, int len)
-{
-    VCardResponse *response;
-    VCardBufferResponse *buffer_response;
-
-    buffer_response = vcard_get_buffer_response(card);
-    if (buffer_response) {
-        vcard_set_buffer_response(card, NULL);
-        vcard_buffer_response_delete(buffer_response);
-    }
-    buffer_response = vcard_buffer_response_new(buf, len);
-    if (buffer_response == NULL) {
-        return NULL;
-    }
-    response = vcard_response_new_status_bytes(VCARD7816_SW1_RESPONSE_BYTES,
-                                               len > 255 ? 0 : len);
-    if (response == NULL) {
-        return NULL;
-    }
-    vcard_set_buffer_response(card, buffer_response);
-    return response;
-}
-
-/*
- * general buffer to hold results from APDU calls
- */
-VCardResponse *
-vcard_response_new(VCard *card, unsigned char *buf,
-                   int len, int Le, vcard_7816_status_t status)
-{
-    VCardResponse *new_response;
-
-    if (len > Le) {
-        return vcard_init_buffer_response(card, buf, len);
-    }
-    new_response = vcard_response_new_data(buf, len);
-    if (new_response == NULL) {
-        return NULL;
-    }
-    vcard_response_set_status(new_response, status);
-    return new_response;
-}
-
-/*
- * general buffer to hold results from APDU calls
- */
-VCardResponse *
-vcard_response_new_bytes(VCard *card, unsigned char *buf, int len, int Le,
-                         unsigned char sw1, unsigned char sw2)
-{
-    VCardResponse *new_response;
-
-    if (len > Le) {
-        return vcard_init_buffer_response(card, buf, len);
-    }
-    new_response = vcard_response_new_data(buf, len);
-    if (new_response == NULL) {
-        return NULL;
-    }
-    vcard_response_set_status_bytes(new_response, sw1, sw2);
-    return new_response;
-}
-
-/*
- * get a new Response buffer that only has a status.
- */
-static VCardResponse *
-vcard_response_new_status(vcard_7816_status_t status)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = &new_response->b_sw1;
-    new_response->b_len = 0;
-    new_response->b_total_len = 2;
-    new_response->b_type = VCARD_MALLOC_STRUCT;
-    vcard_response_set_status(new_response, status);
-    return new_response;
-}
-
-/*
- * same as above, but specify the status as separate bytes
- */
-VCardResponse *
-vcard_response_new_status_bytes(unsigned char sw1, unsigned char sw2)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = &new_response->b_sw1;
-    new_response->b_len = 0;
-    new_response->b_total_len = 2;
-    new_response->b_type = VCARD_MALLOC_STRUCT;
-    vcard_response_set_status_bytes(new_response, sw1, sw2);
-    return new_response;
-}
-
-
-/*
- * free the response buffer. The Buffer has a type to handle the buffer
- * allocated in other ways than through malloc.
- */
-void
-vcard_response_delete(VCardResponse *response)
-{
-    if (response == NULL) {
-        return;
-    }
-    switch (response->b_type) {
-    case VCARD_MALLOC:
-        /* everything was malloc'ed */
-        g_free(response->b_data);
-        g_free(response);
-        break;
-    case VCARD_MALLOC_DATA:
-        /* only the data buffer was malloc'ed */
-        g_free(response->b_data);
-        break;
-    case VCARD_MALLOC_STRUCT:
-        /* only the structure was malloc'ed */
-        g_free(response);
-        break;
-    case VCARD_STATIC:
-        break;
-    }
-}
-
-/*
- * decode the class bit and set our generic type field, channel, and
- * secure messaging values.
- */
-static vcard_7816_status_t
-vcard_apdu_set_class(VCardAPDU *apdu) {
-    apdu->a_channel = 0;
-    apdu->a_secure_messaging = 0;
-    apdu->a_type = apdu->a_cla & 0xf0;
-    apdu->a_gen_type = VCARD_7816_ISO;
-
-    /* parse the class  tables 8 & 9 of the 7816-4 Part 4 spec */
-    switch (apdu->a_type) {
-        /* we only support the basic types */
-    case 0x00:
-    case 0x80:
-    case 0x90:
-    case 0xa0:
-        apdu->a_channel = apdu->a_cla & 3;
-        apdu->a_secure_messaging = apdu->a_cla & 0xe;
-        break;
-    case 0xb0:
-    case 0xc0:
-        break;
-
-    case 0x10:
-    case 0x20:
-    case 0x30:
-    case 0x40:
-    case 0x50:
-    case 0x60:
-    case 0x70:
-        /* Reserved for future use */
-        apdu->a_gen_type = VCARD_7816_RFU;
-        break;
-    case 0xd0:
-    case 0xe0:
-    case 0xf0:
-    default:
-        apdu->a_gen_type =
-            (apdu->a_cla == 0xff) ? VCARD_7816_PTS : VCARD_7816_PROPRIETARY;
-        break;
-    }
-    return VCARD7816_STATUS_SUCCESS;
-}
-
-/*
- * set the Le and Lc fields according to table 5 of the
- * 7816-4 part 4 spec
- */
-static vcard_7816_status_t
-vcard_apdu_set_length(VCardAPDU *apdu)
-{
-    int L, Le;
-
-    /* process according to table 5 of the 7816-4 Part 4 spec.
-     * variable names match the variables in the spec */
-    L = apdu->a_len-4; /* fixed APDU header */
-    apdu->a_Lc = 0;
-    apdu->a_Le = 0;
-    apdu->a_body = NULL;
-    switch (L) {
-    case 0:
-        /* 1 minimal apdu */
-        return VCARD7816_STATUS_SUCCESS;
-    case 1:
-        /* 2S only return values apdu */
-        /*   zero maps to 256 here */
-        apdu->a_Le = apdu->a_header->ah_Le ?
-                         apdu->a_header->ah_Le : 256;
-        return VCARD7816_STATUS_SUCCESS;
-    default:
-        /* if the ah_Le byte is zero and we have more than
-         * 1 byte in the header, then we must be using extended Le and Lc.
-         * process the extended now. */
-        if (apdu->a_header->ah_Le == 0) {
-            if (L < 3) {
-                /* coding error, need at least 3 bytes */
-                return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-            }
-            /* calculate the first extended value. Could be either Le or Lc */
-            Le = (apdu->a_header->ah_body[0] << 8)
-               || apdu->a_header->ah_body[1];
-            if (L == 3) {
-                /* 2E extended, return data only */
-                /*   zero maps to 65536 */
-                apdu->a_Le = Le ? Le : 65536;
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            if (Le == 0) {
-                /* reserved for future use, probably for next time we need
-                 * to extend the lengths */
-                return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-            }
-            /* we know that the first extended value is Lc now */
-            apdu->a_Lc = Le;
-            apdu->a_body = &apdu->a_header->ah_body[2];
-            if (L == Le+3) {
-                /* 3E extended, only body parameters */
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            if (L == Le+5) {
-                /* 4E extended, parameters and return data */
-                Le = (apdu->a_data[apdu->a_len-2] << 8)
-                   || apdu->a_data[apdu->a_len-1];
-                apdu->a_Le = Le ? Le : 65536;
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-        }
-        /* not extended */
-        apdu->a_Lc = apdu->a_header->ah_Le;
-        apdu->a_body = &apdu->a_header->ah_body[0];
-        if (L ==  apdu->a_Lc + 1) {
-            /* 3S only body parameters */
-            return VCARD7816_STATUS_SUCCESS;
-        }
-        if (L ==  apdu->a_Lc + 2) {
-            /* 4S parameters and return data */
-            Le = apdu->a_data[apdu->a_len-1];
-            apdu->a_Le = Le ?  Le : 256;
-            return VCARD7816_STATUS_SUCCESS;
-        }
-        break;
-    }
-    return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-}
-
-/*
- * create a new APDU from a raw set of bytes. This will decode all the
- * above fields. users of VCARDAPDU's can then depend on the already decoded
- * values.
- */
-VCardAPDU *
-vcard_apdu_new(unsigned char *raw_apdu, int len, vcard_7816_status_t *status)
-{
-    VCardAPDU *new_apdu;
-
-    *status = VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE;
-    if (len < 4) {
-        *status = VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-        return NULL;
-    }
-
-    new_apdu = g_new(VCardAPDU, 1);
-    new_apdu->a_data = g_memdup(raw_apdu, len);
-    new_apdu->a_len = len;
-    *status = vcard_apdu_set_class(new_apdu);
-    if (*status != VCARD7816_STATUS_SUCCESS) {
-        g_free(new_apdu);
-        return NULL;
-    }
-    *status = vcard_apdu_set_length(new_apdu);
-    if (*status != VCARD7816_STATUS_SUCCESS) {
-        g_free(new_apdu);
-        new_apdu = NULL;
-    }
-    return new_apdu;
-}
-
-void
-vcard_apdu_delete(VCardAPDU *apdu)
-{
-    if (apdu == NULL) {
-        return;
-    }
-    g_free(apdu->a_data);
-    g_free(apdu);
-}
-
-
-/*
- * declare response buffers for all the 7816 defined error codes
- */
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_SUCCESS)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_RET_CORUPT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_CHANGE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_FILE_FILLED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR_CHANGE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_LENGTH)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(
-                    VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_NO_EF)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_PARAMETERS)
-VCARD_RESPONSE_NEW_STATIC_STATUS(
-                            VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_FILE_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_P1_P2_INCORRECT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_INS_CODE_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CLA_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_GENERAL)
-
-/*
- * return a single response code. This function cannot fail. It will always
- * return a response.
- */
-VCardResponse *
-vcard_make_response(vcard_7816_status_t status)
-{
-    VCardResponse *response;
-
-    switch (status) {
-    /* known 7816 response codes */
-    case VCARD7816_STATUS_SUCCESS:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_SUCCESS);
-    case VCARD7816_STATUS_WARNING:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING);
-    case VCARD7816_STATUS_WARNING_RET_CORUPT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_RET_CORUPT);
-    case VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE);
-    case VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED);
-    case VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID);
-    case VCARD7816_STATUS_WARNING_CHANGE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_CHANGE);
-    case VCARD7816_STATUS_WARNING_FILE_FILLED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_FILE_FILLED);
-    case VCARD7816_STATUS_EXC_ERROR:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR);
-    case VCARD7816_STATUS_EXC_ERROR_CHANGE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR_CHANGE);
-    case VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-    case VCARD7816_STATUS_ERROR_WRONG_LENGTH:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_LENGTH);
-    case VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE);
-    case VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED);
-    case VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED);
-    case VCARD7816_STATUS_ERROR_DATA_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_INVALID);
-    case VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-    case VCARD7816_STATUS_ERROR_DATA_NO_EF:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_NO_EF);
-    case VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING);
-    case VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA);
-    case VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_FILE_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE);
-    case VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT);
-    case VCARD7816_STATUS_ERROR_P1_P2_INCORRECT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-    case VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT);
-    case VCARD7816_STATUS_ERROR_DATA_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2);
-    case VCARD7816_STATUS_ERROR_INS_CODE_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_INS_CODE_INVALID);
-    case VCARD7816_STATUS_ERROR_CLA_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CLA_INVALID);
-    case VCARD7816_STATUS_ERROR_GENERAL:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_GENERAL);
-    default:
-        /* we don't know this status code, create a response buffer to
-         * hold it */
-        response = vcard_response_new_status(status);
-        if (response == NULL) {
-            /* couldn't allocate the buffer, return memmory error */
-            return VCARD_RESPONSE_GET_STATIC(
-                        VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        return response;
-    }
-}
-
-/*
- * Add File card support here if you need it.
- */
-static VCardStatus
-vcard7816_file_system_process_apdu(VCard *card, VCardAPDU *apdu,
-                                   VCardResponse **response)
-{
-    /* TODO: if we want to support a virtual file system card, we do it here.
-     * It would probably be a pkcs #15 card type */
-    *response = vcard_make_response(
-                    VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    return VCARD_DONE;
-}
-
-/*
- * VM card (including java cards)
- */
-static VCardStatus
-vcard7816_vm_process_apdu(VCard *card, VCardAPDU *apdu,
-                          VCardResponse **response)
-{
-    int bytes_to_copy, next_byte_count, count;
-    VCardApplet *current_applet;
-    VCardBufferResponse *buffer_response;
-    vcard_7816_status_t status;
-
-    /* parse the class first */
-    if (apdu->a_gen_type !=  VCARD_7816_ISO) {
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        return VCARD_DONE;
-    }
-
-    /* use a switch so that if we need to support secure channel stuff later,
-     * we know where to put it */
-    switch (apdu->a_secure_messaging) {
-    case 0x0: /* no SM */
-        break;
-    case 0x4: /* proprietary SM */
-    case 0x8: /* header not authenticated */
-    case 0xc: /* header authenticated */
-    default:
-        /* for now, don't try to support secure channel stuff in the
-         * virtual card. */
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED);
-        return VCARD_DONE;
-    }
-
-    /* now parse the instruction */
-    switch (apdu->a_ins) {
-    case  VCARD7816_INS_MANAGE_CHANNEL: /* secure channel op */
-    case  VCARD7816_INS_EXTERNAL_AUTHENTICATE: /* secure channel op */
-    case  VCARD7816_INS_GET_CHALLENGE: /* secure channel op */
-    case  VCARD7816_INS_INTERNAL_AUTHENTICATE: /* secure channel op */
-    case  VCARD7816_INS_ERASE_BINARY: /* applet control op */
-    case  VCARD7816_INS_READ_BINARY: /* applet control op */
-    case  VCARD7816_INS_WRITE_BINARY: /* applet control op */
-    case  VCARD7816_INS_UPDATE_BINARY: /* applet control op */
-    case  VCARD7816_INS_READ_RECORD: /* file op */
-    case  VCARD7816_INS_WRITE_RECORD: /* file op */
-    case  VCARD7816_INS_UPDATE_RECORD: /* file op */
-    case  VCARD7816_INS_APPEND_RECORD: /* file op */
-    case  VCARD7816_INS_ENVELOPE:
-    case  VCARD7816_INS_PUT_DATA:
-        *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-
-    case  VCARD7816_INS_SELECT_FILE:
-        if (apdu->a_p1 != 0x04) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED);
-            break;
-        }
-
-        /* side effect, deselect the current applet if no applet has been found
-         * */
-        current_applet = vcard_find_applet(card, apdu->a_body, apdu->a_Lc);
-        vcard_select_applet(card, apdu->a_channel, current_applet);
-        if (current_applet) {
-            unsigned char *aid;
-            int aid_len;
-            aid = vcard_applet_get_aid(current_applet, &aid_len);
-            *response = vcard_response_new(card, aid, aid_len, apdu->a_Le,
-                                          VCARD7816_STATUS_SUCCESS);
-        } else {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-        }
-        break;
-
-    case  VCARD7816_INS_VERIFY:
-        if ((apdu->a_p1 != 0x00) || (apdu->a_p2 != 0x00)) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_WRONG_PARAMETERS);
-        } else {
-            if (apdu->a_Lc == 0) {
-                /* handle pin count if possible */
-                count = vcard_emul_get_login_count(card);
-                if (count < 0) {
-                    *response = vcard_make_response(
-                                    VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-                } else {
-                    if (count > 0xf) {
-                        count = 0xf;
-                    }
-                    *response = vcard_response_new_status_bytes(
-                                                VCARD7816_SW1_WARNING_CHANGE,
-                                                                0xc0 | count);
-                    if (*response == NULL) {
-                        *response = vcard_make_response(
-                                    VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-                    }
-                }
-            } else {
-                    status = vcard_emul_login(card, apdu->a_body, apdu->a_Lc);
-                *response = vcard_make_response(status);
-            }
-        }
-        break;
-
-    case VCARD7816_INS_GET_RESPONSE:
-        buffer_response = vcard_get_buffer_response(card);
-        if (!buffer_response) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-            /* handle error */
-            break;
-        }
-        bytes_to_copy = MIN(buffer_response->len, apdu->a_Le);
-        next_byte_count = MIN(256, buffer_response->len - bytes_to_copy);
-        *response = vcard_response_new_bytes(
-                        card, buffer_response->current, bytes_to_copy,
-                        apdu->a_Le,
-                        next_byte_count ?
-                        VCARD7816_SW1_RESPONSE_BYTES : VCARD7816_SW1_SUCCESS,
-                        next_byte_count);
-        buffer_response->current += bytes_to_copy;
-        buffer_response->len -= bytes_to_copy;
-        if (*response == NULL || (next_byte_count == 0)) {
-            vcard_set_buffer_response(card, NULL);
-            vcard_buffer_response_delete(buffer_response);
-        }
-        if (*response == NULL) {
-            *response =
-                vcard_make_response(VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        break;
-
-    case VCARD7816_INS_GET_DATA:
-        *response =
-            vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-
-    default:
-        *response =
-            vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-    }
-
-    /* response should have been set somewhere */
-    assert(*response != NULL);
-    return VCARD_DONE;
-}
-
-
-/*
- * APDU processing starts here. This routes the card processing stuff to the
- * right location.
- */
-VCardStatus
-vcard_process_apdu(VCard *card, VCardAPDU *apdu, VCardResponse **response)
-{
-    VCardStatus status;
-    VCardBufferResponse *buffer_response;
-
-    /* first handle any PTS commands, which aren't really APDU's */
-    if (apdu->a_type == VCARD_7816_PTS) {
-        /* the PTS responses aren't really responses either */
-        *response = vcard_response_new_data(apdu->a_data, apdu->a_len);
-        /* PTS responses have no status bytes */
-        (*response)->b_total_len = (*response)->b_len;
-        return VCARD_DONE;
-    }
-    buffer_response = vcard_get_buffer_response(card);
-    if (buffer_response && apdu->a_ins != VCARD7816_INS_GET_RESPONSE) {
-        /* clear out buffer_response, return an error */
-        vcard_set_buffer_response(card, NULL);
-        vcard_buffer_response_delete(buffer_response);
-        *response = vcard_make_response(VCARD7816_STATUS_EXC_ERROR);
-        return VCARD_DONE;
-    }
-
-    status = vcard_process_applet_apdu(card, apdu, response);
-    if (status != VCARD_NEXT) {
-        return status;
-    }
-    switch (vcard_get_type(card)) {
-    case VCARD_FILE_SYSTEM:
-        return vcard7816_file_system_process_apdu(card, apdu, response);
-    case VCARD_VM:
-        return vcard7816_vm_process_apdu(card, apdu, response);
-    case VCARD_DIRECT:
-        /* if we are type direct, then the applet should handle everything */
-        assert(!"VCARD_DIRECT: applet failure");
-        break;
-    }
-    *response =
-        vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    return VCARD_DONE;
-}
diff --git a/libcacard/card_7816.h b/libcacard/card_7816.h
deleted file mode 100644
index 4a01993d2..000000000
--- a/libcacard/card_7816.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CARD_7816_H
-#define CARD_7816_H  1
-
-#include "card_7816t.h"
-#include "vcardt.h"
-
-/*
- * constructors for VCardResponse's
- */
-/* response from a return buffer and a status */
-VCardResponse *vcard_response_new(VCard *card, unsigned char *buf, int len,
-                                  int Le, vcard_7816_status_t status);
-/* response from a return buffer and status bytes */
-VCardResponse *vcard_response_new_bytes(VCard *card, unsigned char *buf,
-                                        int len, int Le,
-                                        unsigned char sw1, unsigned char sw2);
-/* response from just status bytes */
-VCardResponse *vcard_response_new_status_bytes(unsigned char sw1,
-                                               unsigned char sw2);
-/* response from just status: NOTE this cannot fail, it will always return a
- * valid response, if it can't allocate memory, the response will be
- * VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE */
-VCardResponse *vcard_make_response(vcard_7816_status_t status);
-
-/* create a raw response (status has already been encoded */
-VCardResponse *vcard_response_new_data(unsigned char *buf, int len);
-
-
-
-
-/*
- * destructor for VCardResponse.
- *  Can be called with a NULL response
- */
-void vcard_response_delete(VCardResponse *response);
-
-/*
- * constructor for VCardAPDU
- */
-VCardAPDU *vcard_apdu_new(unsigned char *raw_apdu, int len,
-                          unsigned short *status);
-
-/*
- * destructor for VCardAPDU
- *  Can be called with a NULL apdu
- */
-void vcard_apdu_delete(VCardAPDU *apdu);
-
-/*
- * APDU processing starts here. This routes the card processing stuff to the
- * right location. Always returns a valid response.
- */
-VCardStatus vcard_process_apdu(VCard *card, VCardAPDU *apdu,
-                               VCardResponse **response);
-
-#endif
diff --git a/libcacard/card_7816t.h b/libcacard/card_7816t.h
deleted file mode 100644
index 8eef0ce42..000000000
--- a/libcacard/card_7816t.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CARD_7816T_H
-#define CARD_7816T_H 1
-
-typedef unsigned short vcard_7816_status_t;
-
-struct VCardResponseStruct {
-    unsigned char *b_data;
-    vcard_7816_status_t b_status;
-    unsigned char b_sw1;
-    unsigned char b_sw2;
-    int b_len;
-    int b_total_len;
-    enum VCardResponseBufferType {
-        VCARD_MALLOC,
-        VCARD_MALLOC_DATA,
-        VCARD_MALLOC_STRUCT,
-        VCARD_STATIC
-    } b_type;
-};
-
-#define VCARD_RESPONSE_NEW_STATIC_STATUS(stat) \
-static const VCardResponse VCardResponse##stat = \
-        {(unsigned char *)&VCardResponse##stat.b_sw1, (stat), ((stat) >> 8), \
-         ((stat) & 0xff), 0, 2, VCARD_STATIC};
-
-#define VCARD_RESPONSE_NEW_STATIC_STATUS_BYTES(sw1, sw2) \
-static const VCardResponse VCARDResponse##sw1 = \
-        {(unsigned char *)&VCardResponse##name.b_sw1, ((sw1) << 8 | (sw2)), \
-         (sw1), (sw2), 0, 2, VCARD_STATIC};
-
-/* cast away the const, callers need may need to 'free' the
- * result, and const implies that they don't */
-#define VCARD_RESPONSE_GET_STATIC(name) \
-        ((VCardResponse *)(&VCardResponse##name))
-
-typedef enum {
-    VCARD_7816_ISO,
-    VCARD_7816_RFU,
-    VCARD_7816_PTS,
-    VCARD_7816_PROPRIETARY
-} VCardAPDUType;
-
-
-/*
- * 7816 header. All APDU's have this header.
- * They must be laid out in this order.
- */
-struct VCardAPDUHeader {
-    unsigned char ah_cla;
-    unsigned char ah_ins;
-    unsigned char ah_p1;
-    unsigned char ah_p2;
-    unsigned char ah_Le;
-    unsigned char ah_body[1]; /* indefinate length */
-};
-
-/*
- * 7816 APDU structure. The raw bytes are stored in the union and can be
- * accessed directly through u.data (which is aliased as a_data).
- *
- * Names of the fields match the 7816 documentation.
- */
-struct VCardAPDUStruct {
-    int a_len;                /* length of the whole buffer, including header */
-    int a_Lc;                 /* 7816 Lc (parameter length) value */
-    int a_Le;                 /* 7816 Le (expected result length) value */
-    unsigned char *a_body;    /* pointer to the parameter */
-    int a_channel;            /* decoded channel */
-    int a_secure_messaging;   /* decoded secure messaging type */
-    int a_type;               /* decoded type from cla (top nibble of class) */
-    VCardAPDUType a_gen_type; /* generic type (7816, PROPRIETARY, RFU, etc) */
-    union {
-        struct VCardAPDUHeader *header;
-        unsigned char   *data;
-    } u;
-/* give the subfields a unified look */
-#define a_header u.header
-#define a_data u.data
-#define a_cla a_header->ah_cla /* class */
-#define a_ins a_header->ah_ins /* instruction */
-#define a_p1 a_header->ah_p1   /* parameter 1 */
-#define a_p2 a_header->ah_p2   /* parameter 2 */
-};
-
-/* 7816 status codes */
-#define VCARD7816_STATUS_SUCCESS                              0x9000
-#define VCARD7816_STATUS_WARNING                              0x6200
-#define VCARD7816_STATUS_WARNING_RET_CORUPT                   0x6281
-#define VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE            0x6282
-#define VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED        0x6283
-#define VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID           0x6284
-#define VCARD7816_STATUS_WARNING_CHANGE                       0x6300
-#define VCARD7816_STATUS_WARNING_FILE_FILLED                  0x6381
-#define VCARD7816_STATUS_EXC_ERROR                            0x6400
-#define VCARD7816_STATUS_EXC_ERROR_CHANGE                     0x6500
-#define VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE             0x6581
-#define VCARD7816_STATUS_ERROR_WRONG_LENGTH                   0x6700
-#define VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED              0x6800
-#define VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED          0x6881
-#define VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED           0x6882
-#define VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED          0x6900
-#define VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE 0x6981
-#define VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED         0x6982
-#define VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED         0x6983
-#define VCARD7816_STATUS_ERROR_DATA_INVALID                   0x6984
-#define VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED        0x6985
-#define VCARD7816_STATUS_ERROR_DATA_NO_EF                     0x6986
-#define VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING              0x6987
-#define VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT            0x6988
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS               0x6a00
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA       0x6a80
-#define VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED         0x6a81
-#define VCARD7816_STATUS_ERROR_FILE_NOT_FOUND                 0x6a82
-#define VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND               0x6a83
-#define VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE              0x6a84
-#define VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT            0x6a85
-#define VCARD7816_STATUS_ERROR_P1_P2_INCORRECT                0x6a86
-#define VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT          0x6a87
-#define VCARD7816_STATUS_ERROR_DATA_NOT_FOUND                 0x6a88
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2             0x6b00
-#define VCARD7816_STATUS_ERROR_INS_CODE_INVALID               0x6d00
-#define VCARD7816_STATUS_ERROR_CLA_INVALID                    0x6e00
-#define VCARD7816_STATUS_ERROR_GENERAL                        0x6f00
-/* 7816 sw1 codes */
-#define VCARD7816_SW1_SUCCESS               0x90
-#define VCARD7816_SW1_RESPONSE_BYTES        0x61
-#define VCARD7816_SW1_WARNING               0x62
-#define VCARD7816_SW1_WARNING_CHANGE        0x63
-#define VCARD7816_SW1_EXC_ERROR             0x64
-#define VCARD7816_SW1_EXC_ERROR_CHANGE      0x65
-#define VCARD7816_SW1_ERROR_WRONG_LENGTH    0x67
-#define VCARD7816_SW1_CLA_ERROR             0x68
-#define VCARD7816_SW1_COMMAND_ERROR         0x69
-#define VCARD7816_SW1_P1_P2_ERROR           0x6a
-#define VCARD7816_SW1_LE_ERROR              0x6c
-#define VCARD7816_SW1_INS_ERROR             0x6d
-#define VCARD7816_SW1_CLA_NOT_SUPPORTED     0x6e
-
-/* 7816 Instructions */
-#define VCARD7816_INS_MANAGE_CHANNEL        0x70
-#define VCARD7816_INS_EXTERNAL_AUTHENTICATE 0x82
-#define VCARD7816_INS_GET_CHALLENGE         0x84
-#define VCARD7816_INS_INTERNAL_AUTHENTICATE 0x88
-#define VCARD7816_INS_ERASE_BINARY          0x0e
-#define VCARD7816_INS_READ_BINARY           0xb0
-#define VCARD7816_INS_WRITE_BINARY          0xd0
-#define VCARD7816_INS_UPDATE_BINARY         0xd6
-#define VCARD7816_INS_READ_RECORD           0xb2
-#define VCARD7816_INS_WRITE_RECORD          0xd2
-#define VCARD7816_INS_UPDATE_RECORD         0xdc
-#define VCARD7816_INS_APPEND_RECORD         0xe2
-#define VCARD7816_INS_ENVELOPE              0xc2
-#define VCARD7816_INS_PUT_DATA              0xda
-#define VCARD7816_INS_GET_DATA              0xca
-#define VCARD7816_INS_SELECT_FILE           0xa4
-#define VCARD7816_INS_VERIFY                0x20
-#define VCARD7816_INS_GET_RESPONSE          0xc0
-
-#endif
diff --git a/libcacard/event.c b/libcacard/event.c
deleted file mode 100644
index 63f4057fe..000000000
--- a/libcacard/event.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * event queue implementation.
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include "vcard.h"
-#include "vreader.h"
-#include "vevent.h"
-
-VEvent *
-vevent_new(VEventType type, VReader *reader, VCard *card)
-{
-    VEvent *new_vevent;
-
-    new_vevent = g_new(VEvent, 1);
-    new_vevent->next = NULL;
-    new_vevent->type = type;
-    new_vevent->reader = vreader_reference(reader);
-    new_vevent->card = vcard_reference(card);
-
-    return new_vevent;
-}
-
-void
-vevent_delete(VEvent *vevent)
-{
-    if (vevent == NULL) {
-        return;
-    }
-    vreader_free(vevent->reader);
-    vcard_free(vevent->card);
-    g_free(vevent);
-}
-
-/*
- * VEvent queue management
- */
-
-static VEvent *vevent_queue_head;
-static VEvent *vevent_queue_tail;
-static CompatGMutex vevent_queue_lock;
-static CompatGCond vevent_queue_condition;
-
-void vevent_queue_init(void)
-{
-    vevent_queue_head = vevent_queue_tail = NULL;
-}
-
-void
-vevent_queue_vevent(VEvent *vevent)
-{
-    vevent->next = NULL;
-    g_mutex_lock(&vevent_queue_lock);
-    if (vevent_queue_head) {
-        assert(vevent_queue_tail);
-        vevent_queue_tail->next = vevent;
-    } else {
-        vevent_queue_head = vevent;
-    }
-    vevent_queue_tail = vevent;
-    g_cond_signal(&vevent_queue_condition);
-    g_mutex_unlock(&vevent_queue_lock);
-}
-
-/* must have lock */
-static VEvent *
-vevent_dequeue_vevent(void)
-{
-    VEvent *vevent = NULL;
-    if (vevent_queue_head) {
-        vevent = vevent_queue_head;
-        vevent_queue_head = vevent->next;
-        vevent->next = NULL;
-    }
-    return vevent;
-}
-
-VEvent *vevent_wait_next_vevent(void)
-{
-    VEvent *vevent;
-
-    g_mutex_lock(&vevent_queue_lock);
-    while ((vevent = vevent_dequeue_vevent()) == NULL) {
-        g_cond_wait(&vevent_queue_condition, &vevent_queue_lock);
-    }
-    g_mutex_unlock(&vevent_queue_lock);
-    return vevent;
-}
-
-VEvent *vevent_get_next_vevent(void)
-{
-    VEvent *vevent;
-
-    g_mutex_lock(&vevent_queue_lock);
-    vevent = vevent_dequeue_vevent();
-    g_mutex_unlock(&vevent_queue_lock);
-    return vevent;
-}
-
diff --git a/libcacard/eventt.h b/libcacard/eventt.h
deleted file mode 100644
index 0dc7bd468..000000000
--- a/libcacard/eventt.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef EVENTT_H
-#define EVENTT_H 1
-#include "vreadert.h"
-#include "vcardt.h"
-
-typedef struct VEventStruct VEvent;
-
-typedef enum {
-    VEVENT_READER_INSERT,
-    VEVENT_READER_REMOVE,
-    VEVENT_CARD_INSERT,
-    VEVENT_CARD_REMOVE,
-    VEVENT_LAST,
-} VEventType;
-
-struct VEventStruct {
-    VEvent *next;
-    VEventType type;
-    VReader *reader;
-    VCard *card;
-};
-#endif
-
-
diff --git a/libcacard/libcacard.pc.in b/libcacard/libcacard.pc.in
deleted file mode 100644
index 4b60023ce..000000000
--- a/libcacard/libcacard.pc.in
+++ /dev/null
@@ -1,13 +0,0 @@
-prefix=@PREFIX@
-exec_prefix=${prefix}
-libdir=@LIBDIR@
-includedir=@INCLUDEDIR@
-
-Name: cacard
-Description: CA Card library
-Version: @VERSION@
-
-Requires.private: nss glib-2.0
-Libs: -L${libdir} -lcacard
-Libs.private:
-Cflags: -I${includedir}
diff --git a/libcacard/libcacard.syms b/libcacard/libcacard.syms
deleted file mode 100644
index 1697515a7..000000000
--- a/libcacard/libcacard.syms
+++ /dev/null
@@ -1,77 +0,0 @@
-cac_card_init
-cac_is_cac_card
-vcard_add_applet
-vcard_apdu_delete
-vcard_apdu_new
-vcard_applet_get_aid
-vcard_buffer_response_delete
-vcard_buffer_response_new
-vcard_delete_applet
-vcard_emul_delete_key
-vcard_emul_force_card_insert
-vcard_emul_force_card_remove
-vcard_emul_get_atr
-vcard_emul_get_login_count
-vcard_emul_init
-vcard_emul_login
-vcard_emul_options
-vcard_emul_replay_insertion_events
-vcard_emul_reset
-vcard_emul_rsa_op
-vcard_emul_type_from_string
-vcard_emul_type_select
-vcard_emul_usage
-vcard_find_applet
-vcard_free
-vcard_get_atr
-vcard_get_buffer_response
-vcard_get_current_applet_private
-vcard_get_private
-vcard_get_type
-vcard_init
-vcard_make_response
-vcard_new
-vcard_new_applet
-vcard_process_apdu
-vcard_process_applet_apdu
-vcard_reference
-vcard_reset
-vcard_response_delete
-vcard_response_new
-vcard_response_new_bytes
-vcard_response_new_data
-vcard_response_new_status_bytes
-vcard_select_applet
-vcard_set_applet_private
-vcard_set_atr_func
-vcard_set_buffer_response
-vcard_set_type
-vevent_delete
-vevent_get_next_vevent
-vevent_new
-vevent_queue_init
-vevent_queue_vevent
-vevent_wait_next_vevent
-vreader_add_reader
-vreader_card_is_present
-vreader_free
-vreader_get_id
-vreader_get_name
-vreader_get_private
-vreader_get_reader_by_id
-vreader_get_reader_by_name
-vreader_get_reader_list
-vreader_init
-vreader_insert_card
-vreader_list_delete
-vreader_list_get_first
-vreader_list_get_next
-vreader_list_get_reader
-vreader_new
-vreader_power_off
-vreader_power_on
-vreader_queue_card_event
-vreader_reference
-vreader_remove_reader
-vreader_set_id
-vreader_xfr_bytes
diff --git a/libcacard/link_test.c b/libcacard/link_test.c
deleted file mode 100644
index 6f67a23d9..000000000
--- a/libcacard/link_test.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <stdio.h>
-#include "vcard.h"
-
-VCardStatus cac_card_init(const char *flags, VCard *card,
-                const unsigned char *cert[],
-                int cert_len[], VCardKey *key[] /* adopt the keys*/,
-                int cert_count);
-/*
- * this will crash... just test the linkage right now
- */
-
-main(int argc, char **argv)
-{
-    VCard *card; /* no constructor yet */
-    cac_card_init("", card, NULL, 0, NULL, 0);
-}
-
diff --git a/libcacard/vcard.c b/libcacard/vcard.c
deleted file mode 100644
index 1a87208f3..000000000
--- a/libcacard/vcard.c
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * implement the Java card standard.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include <string.h>
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816t.h"
-
-struct VCardAppletStruct {
-    VCardApplet   *next;
-    VCardProcessAPDU process_apdu;
-    VCardResetApplet reset_applet;
-    unsigned char *aid;
-    int aid_len;
-    void *applet_private;
-    VCardAppletPrivateFree applet_private_free;
-};
-
-struct VCardStruct {
-    int reference_count;
-    VCardApplet *applet_list;
-    VCardApplet *current_applet[MAX_CHANNEL];
-    VCardBufferResponse *vcard_buffer_response;
-    VCardType type;
-    VCardEmul *vcard_private;
-    VCardEmulFree vcard_private_free;
-    VCardGetAtr vcard_get_atr;
-};
-
-VCardBufferResponse *
-vcard_buffer_response_new(unsigned char *buffer, int size)
-{
-    VCardBufferResponse *new_buffer;
-
-    new_buffer = g_new(VCardBufferResponse, 1);
-    new_buffer->buffer = (unsigned char *)g_memdup(buffer, size);
-    new_buffer->buffer_len = size;
-    new_buffer->current = new_buffer->buffer;
-    new_buffer->len = size;
-    return new_buffer;
-}
-
-void
-vcard_buffer_response_delete(VCardBufferResponse *buffer_response)
-{
-    if (buffer_response == NULL) {
-        return;
-    }
-    g_free(buffer_response->buffer);
-    g_free(buffer_response);
-}
-
-
-/*
- * clean up state after a reset
- */
-void
-vcard_reset(VCard *card, VCardPower power)
-{
-    int i;
-    VCardApplet *applet = NULL;
-
-    if (card->type ==  VCARD_DIRECT) {
-        /* select the last applet */
-        VCardApplet *current_applet = NULL;
-        for (current_applet = card->applet_list; current_applet;
-                                       current_applet = current_applet->next) {
-            applet = current_applet;
-        }
-    }
-    for (i = 0; i < MAX_CHANNEL; i++) {
-        card->current_applet[i] = applet;
-    }
-    if (card->vcard_buffer_response) {
-        vcard_buffer_response_delete(card->vcard_buffer_response);
-        card->vcard_buffer_response = NULL;
-    }
-    vcard_emul_reset(card, power);
-    if (applet) {
-        applet->reset_applet(card, 0);
-    }
-}
-
-/* applet utilities */
-
-/*
- * applet utilities
- */
-/* constructor */
-VCardApplet *
-vcard_new_applet(VCardProcessAPDU applet_process_function,
-                 VCardResetApplet applet_reset_function,
-                 unsigned char *aid, int aid_len)
-{
-    VCardApplet *applet;
-
-    applet = g_new0(VCardApplet, 1);
-    applet->process_apdu = applet_process_function;
-    applet->reset_applet = applet_reset_function;
-
-    applet->aid = g_memdup(aid, aid_len);
-    applet->aid_len = aid_len;
-    return applet;
-}
-
-/* destructor */
-void
-vcard_delete_applet(VCardApplet *applet)
-{
-    if (applet == NULL) {
-        return;
-    }
-    if (applet->applet_private_free) {
-        applet->applet_private_free(applet->applet_private);
-    }
-    g_free(applet->aid);
-    g_free(applet);
-}
-
-/* accessor */
-void
-vcard_set_applet_private(VCardApplet *applet, VCardAppletPrivate *private,
-                         VCardAppletPrivateFree private_free)
-{
-    if (applet->applet_private_free) {
-        applet->applet_private_free(applet->applet_private);
-    }
-    applet->applet_private = private;
-    applet->applet_private_free = private_free;
-}
-
-VCard *
-vcard_new(VCardEmul *private, VCardEmulFree private_free)
-{
-    VCard *new_card;
-
-    new_card = g_new0(VCard, 1);
-    new_card->type = VCARD_VM;
-    new_card->vcard_private = private;
-    new_card->vcard_private_free = private_free;
-    new_card->reference_count = 1;
-    return new_card;
-}
-
-VCard *
-vcard_reference(VCard *vcard)
-{
-    if (vcard == NULL) {
-        return NULL;
-    }
-    vcard->reference_count++;
-    return vcard;
-}
-
-void
-vcard_free(VCard *vcard)
-{
-    VCardApplet *current_applet;
-    VCardApplet *next_applet;
-
-    if (vcard == NULL) {
-        return;
-    }
-    vcard->reference_count--;
-    if (vcard->reference_count != 0) {
-        return;
-    }
-    if (vcard->vcard_private_free) {
-        (*vcard->vcard_private_free)(vcard->vcard_private);
-    }
-    for (current_applet = vcard->applet_list; current_applet;
-                                        current_applet = next_applet) {
-        next_applet = current_applet->next;
-        vcard_delete_applet(current_applet);
-    }
-    vcard_buffer_response_delete(vcard->vcard_buffer_response);
-    g_free(vcard);
-}
-
-void
-vcard_get_atr(VCard *vcard, unsigned char *atr, int *atr_len)
-{
-    if (vcard->vcard_get_atr) {
-        (*vcard->vcard_get_atr)(vcard, atr, atr_len);
-        return;
-    }
-    vcard_emul_get_atr(vcard, atr, atr_len);
-}
-
-void
-vcard_set_atr_func(VCard *card, VCardGetAtr vcard_get_atr)
-{
-    card->vcard_get_atr = vcard_get_atr;
-}
-
-
-VCardStatus
-vcard_add_applet(VCard *card, VCardApplet *applet)
-{
-    applet->next = card->applet_list;
-    card->applet_list = applet;
-    /* if our card-type is direct, always call the applet */
-    if (card->type ==  VCARD_DIRECT) {
-        int i;
-
-        for (i = 0; i < MAX_CHANNEL; i++) {
-            card->current_applet[i] = applet;
-        }
-    }
-    return VCARD_DONE;
-}
-
-/*
- * manage applets
- */
-VCardApplet *
-vcard_find_applet(VCard *card, unsigned char *aid, int aid_len)
-{
-    VCardApplet *current_applet;
-
-    for (current_applet = card->applet_list; current_applet;
-                                        current_applet = current_applet->next) {
-        if (current_applet->aid_len != aid_len) {
-            continue;
-        }
-        if (memcmp(current_applet->aid, aid, aid_len) == 0) {
-            break;
-        }
-    }
-    return current_applet;
-}
-
-unsigned char *
-vcard_applet_get_aid(VCardApplet *applet, int *aid_len)
-{
-    if (applet == NULL) {
-        return NULL;
-    }
-    *aid_len = applet->aid_len;
-    return applet->aid;
-}
-
-
-void
-vcard_select_applet(VCard *card, int channel, VCardApplet *applet)
-{
-    assert(channel < MAX_CHANNEL);
-
-    /* If using an emulated card, make sure to log out of any already logged in
-     * session. */
-    vcard_emul_logout(card);
-
-    card->current_applet[channel] = applet;
-    /* reset the applet */
-    if (applet && applet->reset_applet) {
-        applet->reset_applet(card, channel);
-    }
-}
-
-VCardAppletPrivate *
-vcard_get_current_applet_private(VCard *card, int channel)
-{
-    VCardApplet *applet = card->current_applet[channel];
-
-    if (applet == NULL) {
-        return NULL;
-    }
-    return applet->applet_private;
-}
-
-VCardStatus
-vcard_process_applet_apdu(VCard *card, VCardAPDU *apdu,
-                          VCardResponse **response)
-{
-    if (card->current_applet[apdu->a_channel]) {
-        return card->current_applet[apdu->a_channel]->process_apdu(
-                                                        card, apdu, response);
-    }
-    return VCARD_NEXT;
-}
-
-/*
- * Accessor functions
- */
-/* accessor functions for the response buffer */
-VCardBufferResponse *
-vcard_get_buffer_response(VCard *card)
-{
-    return card->vcard_buffer_response;
-}
-
-void
-vcard_set_buffer_response(VCard *card, VCardBufferResponse *buffer)
-{
-    card->vcard_buffer_response = buffer;
-}
-
-
-/* accessor functions for the type */
-VCardType
-vcard_get_type(VCard *card)
-{
-    return card->type;
-}
-
-void
-vcard_set_type(VCard *card, VCardType type)
-{
-    card->type = type;
-}
-
-/* accessor for private data */
-VCardEmul *
-vcard_get_private(VCard *vcard)
-{
-    return vcard->vcard_private;
-}
-
diff --git a/libcacard/vcard.h b/libcacard/vcard.h
deleted file mode 100644
index 47dc70382..000000000
--- a/libcacard/vcard.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef VCARD_H
-#define VCARD_H 1
-
-#include "vcardt.h"
-
-/*
- * response buffer constructors and destructors.
- *
- * response buffers are used when we need to return more data than will fit in
- * a normal APDU response (nominally 254 bytes).
- */
-VCardBufferResponse *vcard_buffer_response_new(unsigned char *buffer, int size);
-void vcard_buffer_response_delete(VCardBufferResponse *buffer_response);
-
-
-/*
- * clean up state on reset
- */
-void vcard_reset(VCard *card, VCardPower power);
-
-/*
- * applet utilities
- */
-/*
- * Constructor for a VCardApplet
- */
-VCardApplet *vcard_new_applet(VCardProcessAPDU applet_process_function,
-                              VCardResetApplet applet_reset_function,
-                              unsigned char *aid, int aid_len);
-
-/*
- * destructor for a VCardApplet
- *  Can be called with a NULL applet
- */
-void vcard_delete_applet(VCardApplet *applet);
-
-/* accessor - set the card type specific private data */
-void vcard_set_applet_private(VCardApplet *applet, VCardAppletPrivate *_private,
-                              VCardAppletPrivateFree private_free);
-
-/* set type of vcard */
-void vcard_set_type(VCard *card, VCardType type);
-
-/*
- * utilities interacting with the current applet
- */
-/* add a new applet to a card */
-VCardStatus vcard_add_applet(VCard *card, VCardApplet *applet);
-/* find the applet on the card with the given aid */
-VCardApplet *vcard_find_applet(VCard *card, unsigned char *aid, int aid_len);
-/* set the following applet to be current on the given channel */
-void vcard_select_applet(VCard *card, int channel, VCardApplet *applet);
-/* get the card type specific private data on the given channel */
-VCardAppletPrivate *vcard_get_current_applet_private(VCard *card, int channel);
-/* fetch the applet's id */
-unsigned char *vcard_applet_get_aid(VCardApplet *applet, int *aid_len);
-
-/* process the apdu for the current selected applet/file */
-VCardStatus vcard_process_applet_apdu(VCard *card, VCardAPDU *apdu,
-                                      VCardResponse **response);
-/*
- * VCard utilities
- */
-/* constructor */
-VCard *vcard_new(VCardEmul *_private, VCardEmulFree private_free);
-/* get a reference */
-VCard *vcard_reference(VCard *);
-/* destructor (reference counted) */
-void vcard_free(VCard *);
-/* get the atr from the card */
-void vcard_get_atr(VCard *card, unsigned char *atr, int *atr_len);
-void vcard_set_atr_func(VCard *card, VCardGetAtr vcard_get_atr);
-
-/* accessor functions for the response buffer */
-VCardBufferResponse *vcard_get_buffer_response(VCard *card);
-void vcard_set_buffer_response(VCard *card, VCardBufferResponse *buffer);
-/* accessor functions for the type */
-VCardType vcard_get_type(VCard *card);
-/* get the private data */
-VCardEmul *vcard_get_private(VCard *card);
-
-#endif
diff --git a/libcacard/vcard_emul.h b/libcacard/vcard_emul.h
deleted file mode 100644
index f09ee98dc..000000000
--- a/libcacard/vcard_emul.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * This is the actual card emulator.
- *
- * These functions can be implemented in different ways on different platforms
- * using the underlying system primitives. For Linux it uses NSS, though direct
- * to PKCS #11, openssl+pkcs11, or even gnu crypto libraries+pkcs #11 could be
- * used. On Windows CAPI could be used.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VCARD_EMUL_H
-#define VCARD_EMUL_H 1
-
-#include "card_7816t.h"
-#include "vcard.h"
-#include "vcard_emul_type.h"
-
-/*
- * types
- */
-typedef enum {
-    VCARD_EMUL_OK = 0,
-    VCARD_EMUL_FAIL,
-    /* return values by vcard_emul_init */
-    VCARD_EMUL_INIT_ALREADY_INITED,
-} VCardEmulError;
-
-/* options are emul specific. call card_emul_parse_args to change a string
- * To an options struct */
-typedef struct VCardEmulOptionsStruct VCardEmulOptions;
-
-/*
- * Login functions
- */
-/* return the number of login attempts still possible on the card. if unknown,
- * return -1 */
-int vcard_emul_get_login_count(VCard *card);
-/* login into the card, return the 7816 status word (sw2 || sw1) */
-vcard_7816_status_t vcard_emul_login(VCard *card, unsigned char *pin,
-                                     int pin_len);
-void vcard_emul_logout(VCard *card);
-
-/*
- * key functions
- */
-/* delete a key */
-void vcard_emul_delete_key(VCardKey *key);
-/* RSA sign/decrypt with the key, signature happens 'in place' */
-vcard_7816_status_t vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                                  unsigned char *buffer, int buffer_size);
-
-void vcard_emul_reset(VCard *card, VCardPower power);
-void vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len);
-
-/* Re-insert of a card that has been removed by force removal */
-VCardEmulError vcard_emul_force_card_insert(VReader *vreader);
-/* Force a card removal even if the card is not physically removed */
-VCardEmulError vcard_emul_force_card_remove(VReader *vreader);
-
-VCardEmulOptions *vcard_emul_options(const char *args);
-VCardEmulError vcard_emul_init(const VCardEmulOptions *options);
-void vcard_emul_replay_insertion_events(void);
-void vcard_emul_usage(void);
-#endif
diff --git a/libcacard/vcard_emul_nss.c b/libcacard/vcard_emul_nss.c
deleted file mode 100644
index d9761eedc..000000000
--- a/libcacard/vcard_emul_nss.c
+++ /dev/null
@@ -1,1274 +0,0 @@
-/*
- * This is the actual card emulator.
- *
- * These functions can be implemented in different ways on different platforms
- * using the underlying system primitives. For Linux it uses NSS, though direct
- * to PKCS #11, openssl+pkcs11, or even gnu crypto libraries+pkcs #11 could be
- * used. On Windows CAPI could be used.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-/*
- * NSS headers
- */
-
-/* avoid including prototypes.h that redefines uint32 */
-#define NO_NSPR_10_SUPPORT
-
-#include <nss.h>
-#include <pk11pub.h>
-#include <cert.h>
-#include <key.h>
-#include <secmod.h>
-#include <prthread.h>
-#include <secerr.h>
-
-#include "glib-compat.h"
-
-#include "vcard.h"
-#include "card_7816t.h"
-#include "vcard_emul.h"
-#include "vreader.h"
-#include "vevent.h"
-
-#include "vcardt_internal.h"
-
-
-typedef enum {
-    VCardEmulUnknown = -1,
-    VCardEmulFalse = 0,
-    VCardEmulTrue = 1
-} VCardEmulTriState;
-
-struct VCardKeyStruct {
-    CERTCertificate *cert;
-    PK11SlotInfo *slot;
-    SECKEYPrivateKey *key;
-    VCardEmulTriState failedX509;
-};
-
-
-typedef struct VirtualReaderOptionsStruct VirtualReaderOptions;
-
-struct VReaderEmulStruct {
-    PK11SlotInfo *slot;
-    VCardEmulType default_type;
-    char *type_params;
-    PRBool present;
-    int     series;
-    VCard *saved_vcard;
-};
-
-/*
- *  NSS Specific options
- */
-struct VirtualReaderOptionsStruct {
-    char *name;
-    char *vname;
-    VCardEmulType card_type;
-    char *type_params;
-    char **cert_name;
-    int cert_count;
-};
-
-struct VCardEmulOptionsStruct {
-    void *nss_db;
-    VirtualReaderOptions *vreader;
-    int vreader_count;
-    VCardEmulType hw_card_type;
-    const char *hw_type_params;
-    PRBool use_hw;
-};
-
-static int nss_emul_init;
-
-/* if we have more that just the slot, define
- * VCardEmulStruct here */
-
-/*
- * allocate the set of arrays for certs, cert_len, key
- */
-static void
-vcard_emul_alloc_arrays(unsigned char ***certsp, int **cert_lenp,
-                        VCardKey ***keysp, int cert_count)
-{
-    *certsp = g_new(unsigned char *, cert_count);
-    *cert_lenp = g_new(int, cert_count);
-    *keysp = g_new(VCardKey *, cert_count);
-}
-
-/*
- * Emulator specific card information
- */
-typedef struct CardEmulCardStruct CardEmulPrivate;
-
-static VCardEmul *
-vcard_emul_new_card(PK11SlotInfo *slot)
-{
-    PK11_ReferenceSlot(slot);
-    /* currently we don't need anything other than the slot */
-    return (VCardEmul *)slot;
-}
-
-static void
-vcard_emul_delete_card(VCardEmul *vcard_emul)
-{
-    PK11SlotInfo *slot = (PK11SlotInfo *)vcard_emul;
-    if (slot == NULL) {
-        return;
-    }
-    PK11_FreeSlot(slot);
-}
-
-static PK11SlotInfo *
-vcard_emul_card_get_slot(VCard *card)
-{
-    /* note, the card is holding the reference, no need to get another one */
-    return (PK11SlotInfo *)vcard_get_private(card);
-}
-
-
-/*
- * key functions
- */
-/* private constructure */
-static VCardKey *
-vcard_emul_make_key(PK11SlotInfo *slot, CERTCertificate *cert)
-{
-    VCardKey *key;
-
-    key = g_new(VCardKey, 1);
-    key->slot = PK11_ReferenceSlot(slot);
-    key->cert = CERT_DupCertificate(cert);
-    /* NOTE: if we aren't logged into the token, this could return NULL */
-    /* NOTE: the cert is a temp cert, not necessarily the cert in the token,
-     * use the DER version of this function */
-    key->key = PK11_FindKeyByDERCert(slot, cert, NULL);
-    key->failedX509 = VCardEmulUnknown;
-    return key;
-}
-
-/* destructor */
-void
-vcard_emul_delete_key(VCardKey *key)
-{
-    if (!nss_emul_init || (key == NULL)) {
-        return;
-    }
-    if (key->key) {
-        SECKEY_DestroyPrivateKey(key->key);
-        key->key = NULL;
-    }
-    if (key->cert) {
-        CERT_DestroyCertificate(key->cert);
-    }
-    if (key->slot) {
-        PK11_FreeSlot(key->slot);
-    }
-}
-
-/*
- * grab the nss key from a VCardKey. If it doesn't exist, try to look it up
- */
-static SECKEYPrivateKey *
-vcard_emul_get_nss_key(VCardKey *key)
-{
-    if (key->key) {
-        return key->key;
-    }
-    /* NOTE: if we aren't logged into the token, this could return NULL */
-    key->key = PK11_FindPrivateKeyFromCert(key->slot, key->cert, NULL);
-    return key->key;
-}
-
-/*
- * Map NSS errors to 7816 errors
- */
-static vcard_7816_status_t
-vcard_emul_map_error(int error)
-{
-    switch (error) {
-    case SEC_ERROR_TOKEN_NOT_LOGGED_IN:
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    case SEC_ERROR_BAD_DATA:
-    case SEC_ERROR_OUTPUT_LEN:
-    case SEC_ERROR_INPUT_LEN:
-    case SEC_ERROR_INVALID_ARGS:
-    case SEC_ERROR_INVALID_ALGORITHM:
-    case SEC_ERROR_NO_KEY:
-    case SEC_ERROR_INVALID_KEY:
-    case SEC_ERROR_DECRYPTION_DISALLOWED:
-        return VCARD7816_STATUS_ERROR_DATA_INVALID;
-    case SEC_ERROR_NO_MEMORY:
-        return VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE;
-    }
-    return VCARD7816_STATUS_EXC_ERROR_CHANGE;
-}
-
-/* RSA sign/decrypt with the key, signature happens 'in place' */
-vcard_7816_status_t
-vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                  unsigned char *buffer, int buffer_size)
-{
-    SECKEYPrivateKey *priv_key;
-    unsigned signature_len;
-    PK11SlotInfo *slot;
-    SECStatus rv;
-    unsigned char buf[2048];
-    unsigned char *bp = NULL;
-    int pad_len;
-    vcard_7816_status_t ret = VCARD7816_STATUS_SUCCESS;
-
-    if ((!nss_emul_init) || (key == NULL)) {
-        /* couldn't get the key, indicate that we aren't logged in */
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    priv_key = vcard_emul_get_nss_key(key);
-    if (priv_key == NULL) {
-        /* couldn't get the key, indicate that we aren't logged in */
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    slot = vcard_emul_card_get_slot(card);
-
-    /*
-     * this is only true of the rsa signature
-     */
-    signature_len = PK11_SignatureLen(priv_key);
-    if (buffer_size != signature_len) {
-        return  VCARD7816_STATUS_ERROR_DATA_INVALID;
-    }
-    /* be able to handle larger keys if necessariy */
-    bp = &buf[0];
-    if (sizeof(buf) < signature_len) {
-        bp = g_malloc(signature_len);
-    }
-
-    /*
-     * do the raw operations. Some tokens claim to do CKM_RSA_X_509, but then
-     * choke when they try to do the actual operations. Try to detect
-     * those cases and treat them as if the token didn't claim support for
-     * X_509.
-     */
-    if (key->failedX509 != VCardEmulTrue
-                              && PK11_DoesMechanism(slot, CKM_RSA_X_509)) {
-        rv = PK11_PrivDecryptRaw(priv_key, bp, &signature_len, signature_len,
-                                 buffer, buffer_size);
-        if (rv == SECSuccess) {
-            assert(buffer_size == signature_len);
-            memcpy(buffer, bp, signature_len);
-            key->failedX509 = VCardEmulFalse;
-            goto cleanup;
-        }
-        /*
-         * we've had a successful X509 operation, this failure must be
-         * somethine else
-         */
-        if (key->failedX509 == VCardEmulFalse) {
-            ret = vcard_emul_map_error(PORT_GetError());
-            goto cleanup;
-        }
-        /*
-         * key->failedX509 must be Unknown at this point, try the
-         * non-x_509 case
-         */
-    }
-    /* token does not support CKM_RSA_X509, emulate that with CKM_RSA_PKCS */
-    /* is this a PKCS #1 formatted signature? */
-    if ((buffer[0] == 0) && (buffer[1] == 1)) {
-        int i;
-
-        for (i = 2; i < buffer_size; i++) {
-            /* rsa signature pad */
-            if (buffer[i] != 0xff) {
-                break;
-            }
-        }
-        if ((i < buffer_size) && (buffer[i] == 0)) {
-            /* yes, we have a properly formatted PKCS #1 signature */
-            /*
-             * NOTE: even if we accidentally got an encrypt buffer, which
-             * through sheer luck started with 00, 01, ff, 00, it won't matter
-             * because the resulting Sign operation will effectively decrypt
-             * the real buffer.
-             */
-            SECItem signature;
-            SECItem hash;
-
-            i++;
-            hash.data = &buffer[i];
-            hash.len = buffer_size - i;
-            signature.data = bp;
-            signature.len = signature_len;
-            rv = PK11_Sign(priv_key,  &signature, &hash);
-            if (rv != SECSuccess) {
-                ret = vcard_emul_map_error(PORT_GetError());
-                goto cleanup;
-            }
-            assert(buffer_size == signature.len);
-            memcpy(buffer, bp, signature.len);
-            /*
-             * we got here because either the X509 attempt failed, or the
-             * token couldn't do the X509 operation, in either case stay
-             * with the PKCS version for future operations on this key
-             */
-            key->failedX509 = VCardEmulTrue;
-            goto cleanup;
-        }
-    }
-    pad_len = buffer_size - signature_len;
-    assert(pad_len < 4);
-    /*
-     * OK now we've decrypted the payload, package it up in PKCS #1 for the
-     * upper layer.
-     */
-    buffer[0] = 0;
-    buffer[1] = 2; /* RSA_encrypt  */
-    pad_len -= 3; /* format is 0 || 2 || pad || 0 || data */
-    /*
-     * padding for PKCS #1 encrypted data is a string of random bytes. The
-     * random butes protect against potential decryption attacks against RSA.
-     * Since PrivDecrypt has already stripped those bytes, we can't reconstruct
-     * them. This shouldn't matter to the upper level code which should just
-     * strip this code out anyway, so We'll pad with a constant 3.
-     */
-    memset(&buffer[2], 0x03, pad_len);
-    pad_len += 2; /* index to the end of the pad */
-    buffer[pad_len] = 0;
-    pad_len++; /* index to the start of the data */
-    memcpy(&buffer[pad_len], bp, signature_len);
-    /*
-     * we got here because either the X509 attempt failed, or the
-     * token couldn't do the X509 operation, in either case stay
-     * with the PKCS version for future operations on this key
-     */
-    key->failedX509 = VCardEmulTrue;
-cleanup:
-    if (bp != buf) {
-        g_free(bp);
-    }
-    return ret;
-}
-
-/*
- * Login functions
- */
-/* return the number of login attempts still possible on the card. if unknown,
- * return -1 */
-int
-vcard_emul_get_login_count(VCard *card)
-{
-    return -1;
-}
-
-/* login into the card, return the 7816 status word (sw2 || sw1) */
-vcard_7816_status_t
-vcard_emul_login(VCard *card, unsigned char *pin, int pin_len)
-{
-    PK11SlotInfo *slot;
-    unsigned char *pin_string;
-    int i;
-    SECStatus rv;
-
-    if (!nss_emul_init) {
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    slot = vcard_emul_card_get_slot(card);
-     /* We depend on the PKCS #11 module internal login state here because we
-      * create a separate process to handle each guest instance. If we needed
-      * to handle multiple guests from one process, then we would need to keep
-      * a lot of extra state in our card structure
-      * */
-    pin_string = g_malloc(pin_len+1);
-    memcpy(pin_string, pin, pin_len);
-    pin_string[pin_len] = 0;
-
-    /* handle CAC expanded pins correctly */
-    for (i = pin_len-1; i >= 0 && (pin_string[i] == 0xff); i--) {
-        pin_string[i] = 0;
-    }
-
-    rv = PK11_Authenticate(slot, PR_FALSE, pin_string);
-    memset(pin_string, 0, pin_len);  /* don't let the pin hang around in memory
-                                        to be snooped */
-    g_free(pin_string);
-    if (rv == SECSuccess) {
-        return VCARD7816_STATUS_SUCCESS;
-    }
-    /* map the error from port get error */
-    return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-}
-
-void
-vcard_emul_logout(VCard *card)
-{
-    PK11SlotInfo *slot;
-
-    if (!nss_emul_init) {
-        return;
-    }
-
-    slot = vcard_emul_card_get_slot(card);
-    if (PK11_IsLoggedIn(slot, NULL)) {
-        PK11_Logout(slot); /* NOTE: ignoring SECStatus return value */
-    }
-}
-
-void
-vcard_emul_reset(VCard *card, VCardPower power)
-{
-    /*
-     * if we reset the card (either power on or power off), we lose our login
-     * state
-     */
-    vcard_emul_logout(card);
-
-    /* TODO: we may also need to send insertion/removal events? */
-}
-
-static VReader *
-vcard_emul_find_vreader_from_slot(PK11SlotInfo *slot)
-{
-    VReaderList *reader_list = vreader_get_reader_list();
-    VReaderListEntry *current_entry;
-
-    if (reader_list == NULL) {
-        return NULL;
-    }
-    for (current_entry = vreader_list_get_first(reader_list); current_entry;
-                        current_entry = vreader_list_get_next(current_entry)) {
-        VReader *reader = vreader_list_get_reader(current_entry);
-        VReaderEmul *reader_emul = vreader_get_private(reader);
-        if (reader_emul->slot == slot) {
-            vreader_list_delete(reader_list);
-            return reader;
-        }
-        vreader_free(reader);
-    }
-
-    vreader_list_delete(reader_list);
-    return NULL;
-}
-
-/*
- * create a new reader emul
- */
-static VReaderEmul *
-vreader_emul_new(PK11SlotInfo *slot, VCardEmulType type, const char *params)
-{
-    VReaderEmul *new_reader_emul;
-
-    new_reader_emul = g_new(VReaderEmul, 1);
-
-    new_reader_emul->slot = PK11_ReferenceSlot(slot);
-    new_reader_emul->default_type = type;
-    new_reader_emul->type_params = g_strdup(params);
-    new_reader_emul->present = PR_FALSE;
-    new_reader_emul->series = 0;
-    new_reader_emul->saved_vcard = NULL;
-    return new_reader_emul;
-}
-
-static void
-vreader_emul_delete(VReaderEmul *vreader_emul)
-{
-    if (vreader_emul == NULL) {
-        return;
-    }
-    if (vreader_emul->slot) {
-        PK11_FreeSlot(vreader_emul->slot);
-    }
-    g_free(vreader_emul->type_params);
-    g_free(vreader_emul);
-}
-
-/*
- *  TODO: move this to emulater non-specific file
- */
-static VCardEmulType
-vcard_emul_get_type(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-
-    vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul && vreader_emul->default_type != VCARD_EMUL_NONE) {
-        return vreader_emul->default_type;
-    }
-
-    return vcard_emul_type_select(vreader);
-}
-/*
- *  TODO: move this to emulater non-specific file
- */
-static const char *
-vcard_emul_get_type_params(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-
-    vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul && vreader_emul->type_params) {
-        return vreader_emul->type_params;
-    }
-
-    return "";
-}
-
-/* pull the slot out of the reader private data */
-static PK11SlotInfo *
-vcard_emul_reader_get_slot(VReader *vreader)
-{
-    VReaderEmul *vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul == NULL) {
-        return NULL;
-    }
-    return vreader_emul->slot;
-}
-
-/*
- *  Card ATR's map to physical cards. vcard_alloc_atr will set appropriate
- *  historical bytes for any software emulated card. The remaining bytes can be
- *  used to indicate the actual emulator
- */
-static unsigned char *nss_atr;
-static int nss_atr_len;
-
-void
-vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len)
-{
-    int len;
-    assert(atr != NULL);
-
-    if (nss_atr == NULL) {
-        nss_atr = vcard_alloc_atr("NSS", &nss_atr_len);
-    }
-    len = MIN(nss_atr_len, *atr_len);
-    memcpy(atr, nss_atr, len);
-    *atr_len = len;
-}
-
-/*
- * create a new card from certs and keys
- */
-static VCard *
-vcard_emul_make_card(VReader *reader,
-                     unsigned char * const *certs, int *cert_len,
-                     VCardKey *keys[], int cert_count)
-{
-    VCardEmul *vcard_emul;
-    VCard *vcard;
-    PK11SlotInfo *slot;
-    VCardEmulType type;
-    const char *params;
-
-    type = vcard_emul_get_type(reader);
-
-    /* ignore the inserted card */
-    if (type == VCARD_EMUL_NONE) {
-        return NULL;
-    }
-    slot = vcard_emul_reader_get_slot(reader);
-    if (slot == NULL) {
-        return NULL;
-    }
-
-    params = vcard_emul_get_type_params(reader);
-    /* params these can be NULL */
-
-    vcard_emul = vcard_emul_new_card(slot);
-    if (vcard_emul == NULL) {
-        return NULL;
-    }
-    vcard = vcard_new(vcard_emul, vcard_emul_delete_card);
-    if (vcard == NULL) {
-        vcard_emul_delete_card(vcard_emul);
-        return NULL;
-    }
-    vcard_init(reader, vcard, type, params, certs, cert_len, keys, cert_count);
-    return vcard;
-}
-
-
-/*
- * 'clone' a physical card as a virtual card
- */
-static VCard *
-vcard_emul_mirror_card(VReader *vreader)
-{
-    /*
-     * lookup certs using the C_FindObjects. The Stan Cert handle won't give
-     * us the real certs until we log in.
-     */
-    PK11GenericObject *firstObj, *thisObj;
-    int cert_count;
-    unsigned char **certs;
-    int *cert_len;
-    VCardKey **keys;
-    PK11SlotInfo *slot;
-    VCard *card;
-
-    slot = vcard_emul_reader_get_slot(vreader);
-    if (slot == NULL) {
-        return NULL;
-    }
-
-    firstObj = PK11_FindGenericObjects(slot, CKO_CERTIFICATE);
-    if (firstObj == NULL) {
-        return NULL;
-    }
-
-    /* count the certs */
-    cert_count = 0;
-    for (thisObj = firstObj; thisObj;
-                             thisObj = PK11_GetNextGenericObject(thisObj)) {
-        cert_count++;
-    }
-
-    /* allocate the arrays */
-    vcard_emul_alloc_arrays(&certs, &cert_len, &keys, cert_count);
-
-    /* fill in the arrays */
-    cert_count = 0;
-    for (thisObj = firstObj; thisObj;
-                             thisObj = PK11_GetNextGenericObject(thisObj)) {
-        SECItem derCert;
-        CERTCertificate *cert;
-        SECStatus rv;
-
-        rv = PK11_ReadRawAttribute(PK11_TypeGeneric, thisObj,
-                                   CKA_VALUE, &derCert);
-        if (rv != SECSuccess) {
-            continue;
-        }
-        /* create floating temp cert. This gives us a cert structure even if
-         * the token isn't logged in */
-        cert = CERT_NewTempCertificate(CERT_GetDefaultCertDB(), &derCert,
-                                       NULL, PR_FALSE, PR_TRUE);
-        SECITEM_FreeItem(&derCert, PR_FALSE);
-        if (cert == NULL) {
-            continue;
-        }
-
-        certs[cert_count] = cert->derCert.data;
-        cert_len[cert_count] = cert->derCert.len;
-        keys[cert_count] = vcard_emul_make_key(slot, cert);
-        cert_count++;
-        CERT_DestroyCertificate(cert); /* key obj still has a reference */
-    }
-
-    /* now create the card */
-    card = vcard_emul_make_card(vreader, certs, cert_len, keys, cert_count);
-    g_free(certs);
-    g_free(cert_len);
-    g_free(keys);
-
-    return card;
-}
-
-static VCardEmulType default_card_type = VCARD_EMUL_NONE;
-static const char *default_type_params = "";
-
-/*
- * This thread looks for card and reader insertions and puts events on the
- * event queue
- */
-static void
-vcard_emul_event_thread(void *arg)
-{
-    PK11SlotInfo *slot;
-    VReader *vreader;
-    VReaderEmul *vreader_emul;
-    VCard *vcard;
-    SECMODModule *module = (SECMODModule *)arg;
-
-    do {
-        /*
-         * XXX - the latency value doesn't matter one bit. you only get no
-         * blocking (flags |= CKF_DONT_BLOCK) or PKCS11_WAIT_LATENCY (==500),
-         * hard coded in coolkey.  And it isn't coolkey's fault - the timeout
-         * value we pass get's dropped on the floor before C_WaitForSlotEvent
-         * is called.
-         */
-        slot = SECMOD_WaitForAnyTokenEvent(module, 0, 500);
-        if (slot == NULL) {
-            /* this could be just a no event indication */
-            if (PORT_GetError() == SEC_ERROR_NO_EVENT) {
-                continue;
-            }
-            break;
-        }
-        vreader = vcard_emul_find_vreader_from_slot(slot);
-        if (vreader == NULL) {
-            /* new vreader */
-            vreader_emul = vreader_emul_new(slot, default_card_type,
-                                            default_type_params);
-            vreader = vreader_new(PK11_GetSlotName(slot), vreader_emul,
-                                  vreader_emul_delete);
-            PK11_FreeSlot(slot);
-            slot = NULL;
-            vreader_add_reader(vreader);
-            vreader_free(vreader);
-            continue;
-        }
-        /* card remove/insert */
-        vreader_emul = vreader_get_private(vreader);
-        if (PK11_IsPresent(slot)) {
-            int series = PK11_GetSlotSeries(slot);
-            if (series != vreader_emul->series) {
-                if (vreader_emul->present) {
-                    vreader_insert_card(vreader, NULL);
-                }
-                vcard = vcard_emul_mirror_card(vreader);
-                vreader_insert_card(vreader, vcard);
-                vcard_free(vcard);
-            }
-            vreader_emul->series = series;
-            vreader_emul->present = 1;
-            vreader_free(vreader);
-            PK11_FreeSlot(slot);
-            continue;
-        }
-        if (vreader_emul->present) {
-            vreader_insert_card(vreader, NULL);
-        }
-        vreader_emul->series = 0;
-        vreader_emul->present = 0;
-        PK11_FreeSlot(slot);
-        vreader_free(vreader);
-    } while (1);
-}
-
-/* if the card is inserted when we start up, make sure our state is correct */
-static void
-vcard_emul_init_series(VReader *vreader, VCard *vcard)
-{
-    VReaderEmul *vreader_emul = vreader_get_private(vreader);
-    PK11SlotInfo *slot = vreader_emul->slot;
-
-    vreader_emul->present = PK11_IsPresent(slot);
-    vreader_emul->series = PK11_GetSlotSeries(slot);
-    if (vreader_emul->present == 0) {
-        vreader_insert_card(vreader, NULL);
-    }
-}
-
-/*
- * each module has a separate wait call, create a thread for each module that
- * we are using.
- */
-static void
-vcard_emul_new_event_thread(SECMODModule *module)
-{
-    PR_CreateThread(PR_SYSTEM_THREAD, vcard_emul_event_thread,
-                     module, PR_PRIORITY_HIGH, PR_GLOBAL_THREAD,
-                     PR_UNJOINABLE_THREAD, 0);
-}
-
-static const VCardEmulOptions default_options = {
-    .nss_db = NULL,
-    .vreader = NULL,
-    .vreader_count = 0,
-    .hw_card_type = VCARD_EMUL_CAC,
-    .hw_type_params = "",
-    .use_hw = PR_TRUE
-};
-
-
-/*
- *  NSS needs the app to supply a password prompt. In our case the only time
- *  the password is supplied is as part of the Login APDU. The actual password
- *  is passed in the pw_arg in that case. In all other cases pw_arg should be
- *  NULL.
- */
-static char *
-vcard_emul_get_password(PK11SlotInfo *slot, PRBool retries, void *pw_arg)
-{
-    /* if it didn't work the first time, don't keep trying */
-    if (retries) {
-        return NULL;
-    }
-    /* we are looking up a password when we don't have one in hand */
-    if (pw_arg == NULL) {
-        return NULL;
-    }
-    /* TODO: we really should verify that were are using the right slot */
-    return PORT_Strdup(pw_arg);
-}
-
-/* Force a card removal even if the card is not physically removed */
-VCardEmulError
-vcard_emul_force_card_remove(VReader *vreader)
-{
-    if (!nss_emul_init || (vreader_card_is_present(vreader) != VREADER_OK)) {
-        return VCARD_EMUL_FAIL; /* card is already removed */
-    }
-
-    /* OK, remove it */
-    vreader_insert_card(vreader, NULL);
-    return VCARD_EMUL_OK;
-}
-
-/* Re-insert of a card that has been removed by force removal */
-VCardEmulError
-vcard_emul_force_card_insert(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-    VCard *vcard;
-
-    if (!nss_emul_init || (vreader_card_is_present(vreader) == VREADER_OK)) {
-        return VCARD_EMUL_FAIL; /* card is already removed */
-    }
-    vreader_emul = vreader_get_private(vreader);
-
-    /* if it's a softcard, get the saved vcard from the reader emul structure */
-    if (vreader_emul->saved_vcard) {
-        vcard = vcard_reference(vreader_emul->saved_vcard);
-    } else {
-        /* it must be a physical card, rebuild it */
-        if (!PK11_IsPresent(vreader_emul->slot)) {
-            /* physical card has been removed, not way to reinsert it */
-            return VCARD_EMUL_FAIL;
-        }
-        vcard = vcard_emul_mirror_card(vreader);
-    }
-    vreader_insert_card(vreader, vcard);
-    vcard_free(vcard);
-
-    return VCARD_EMUL_OK;
-}
-
-
-static PRBool
-module_has_removable_hw_slots(SECMODModule *mod)
-{
-    int i;
-    PRBool ret = PR_FALSE;
-    SECMODListLock *moduleLock = SECMOD_GetDefaultModuleListLock();
-
-    if (!moduleLock) {
-        PORT_SetError(SEC_ERROR_NOT_INITIALIZED);
-        return ret;
-    }
-    SECMOD_GetReadLock(moduleLock);
-    for (i = 0; i < mod->slotCount; i++) {
-        PK11SlotInfo *slot = mod->slots[i];
-        if (PK11_IsRemovable(slot) && PK11_IsHW(slot)) {
-            ret = PR_TRUE;
-            break;
-        }
-    }
-    SECMOD_ReleaseReadLock(moduleLock);
-    return ret;
-}
-
-/* Previously we returned FAIL if no readers found. This makes
- * no sense when using hardware, since there may be no readers connected
- * at the time vcard_emul_init is called, but they will be properly
- * recognized later. So Instead return FAIL only if no_hw==1 and no
- * vcards can be created (indicates error with certificates provided
- * or db), or if any other higher level error (NSS error, missing coolkey). */
-static int vcard_emul_init_called;
-
-VCardEmulError
-vcard_emul_init(const VCardEmulOptions *options)
-{
-    SECStatus rv;
-    PRBool has_readers = PR_FALSE;
-    VReader *vreader;
-    VReaderEmul *vreader_emul;
-    SECMODListLock *module_lock;
-    SECMODModuleList *module_list;
-    SECMODModuleList *mlp;
-    int i;
-
-    if (vcard_emul_init_called) {
-        return VCARD_EMUL_INIT_ALREADY_INITED;
-    }
-    vcard_emul_init_called = 1;
-    vreader_init();
-    vevent_queue_init();
-
-    if (options == NULL) {
-        options = &default_options;
-    }
-
-    /* first initialize NSS */
-    if (options->nss_db) {
-        rv = NSS_Init(options->nss_db);
-    } else {
-        gchar *path;
-#ifndef _WIN32
-        path = g_strdup("/etc/pki/nssdb");
-#else
-        if (g_get_system_config_dirs() == NULL ||
-            g_get_system_config_dirs()[0] == NULL) {
-            return VCARD_EMUL_FAIL;
-        }
-
-        path = g_build_filename(
-            g_get_system_config_dirs()[0], "pki", "nssdb", NULL);
-#endif
-
-        rv = NSS_Init(path);
-        g_free(path);
-    }
-    if (rv != SECSuccess) {
-        return VCARD_EMUL_FAIL;
-    }
-    /* Set password callback function */
-    PK11_SetPasswordFunc(vcard_emul_get_password);
-
-    /* set up soft cards emulated by software certs rather than physical cards
-     * */
-    for (i = 0; i < options->vreader_count; i++) {
-        int j;
-        int cert_count;
-        unsigned char **certs;
-        int *cert_len;
-        VCardKey **keys;
-        PK11SlotInfo *slot;
-
-        slot = PK11_FindSlotByName(options->vreader[i].name);
-        if (slot == NULL) {
-            continue;
-        }
-        vreader_emul = vreader_emul_new(slot, options->vreader[i].card_type,
-                                        options->vreader[i].type_params);
-        vreader = vreader_new(options->vreader[i].vname, vreader_emul,
-                              vreader_emul_delete);
-        vreader_add_reader(vreader);
-
-        vcard_emul_alloc_arrays(&certs, &cert_len, &keys,
-                                options->vreader[i].cert_count);
-
-        cert_count = 0;
-        for (j = 0; j < options->vreader[i].cert_count; j++) {
-            /* we should have a better way of identifying certs than by
-             * nickname here */
-            CERTCertificate *cert = PK11_FindCertFromNickname(
-                                        options->vreader[i].cert_name[j],
-                                        NULL);
-            if (cert == NULL) {
-                continue;
-            }
-            certs[cert_count] = cert->derCert.data;
-            cert_len[cert_count] = cert->derCert.len;
-            keys[cert_count] = vcard_emul_make_key(slot, cert);
-            /* this is safe because the key is still holding a cert reference */
-            CERT_DestroyCertificate(cert);
-            cert_count++;
-        }
-        if (cert_count) {
-            VCard *vcard = vcard_emul_make_card(vreader, certs, cert_len,
-                                                keys, cert_count);
-            vreader_insert_card(vreader, vcard);
-            vcard_emul_init_series(vreader, vcard);
-            /* allow insertion and removal of soft cards */
-            vreader_emul->saved_vcard = vcard_reference(vcard);
-            vcard_free(vcard);
-            vreader_free(vreader);
-            has_readers = PR_TRUE;
-        }
-        g_free(certs);
-        g_free(cert_len);
-        g_free(keys);
-    }
-
-    /* if we aren't suppose to use hw, skip looking up hardware tokens */
-    if (!options->use_hw) {
-        nss_emul_init = has_readers;
-        return has_readers ? VCARD_EMUL_OK : VCARD_EMUL_FAIL;
-    }
-
-    /* make sure we have some PKCS #11 module loaded */
-    module_lock = SECMOD_GetDefaultModuleListLock();
-    module_list = SECMOD_GetDefaultModuleList();
-    SECMOD_GetReadLock(module_lock);
-    for (mlp = module_list; mlp; mlp = mlp->next) {
-        SECMODModule *module = mlp->module;
-        if (module_has_removable_hw_slots(module)) {
-            break;
-        }
-    }
-    SECMOD_ReleaseReadLock(module_lock);
-
-    /* now examine all the slots, finding which should be readers */
-    /* We should control this with options. For now we mirror out any
-     * removable hardware slot */
-    default_card_type = options->hw_card_type;
-    default_type_params = g_strdup(options->hw_type_params);
-
-    SECMOD_GetReadLock(module_lock);
-    for (mlp = module_list; mlp; mlp = mlp->next) {
-        SECMODModule *module = mlp->module;
-
-        /* Ignore the internal module */
-        if (module == NULL || module == SECMOD_GetInternalModule()) {
-            continue;
-        }
-
-        for (i = 0; i < module->slotCount; i++) {
-            PK11SlotInfo *slot = module->slots[i];
-
-            /* only map removable HW slots */
-            if (slot == NULL || !PK11_IsRemovable(slot) || !PK11_IsHW(slot)) {
-                continue;
-            }
-            if (strcmp("E-Gate 0 0", PK11_GetSlotName(slot)) == 0) {
-                /*
-                 * coolkey <= 1.1.0-20 emulates this reader if it can't find
-                 * any hardware readers. This causes problems, warn user of
-                 * problems.
-                 */
-                fprintf(stderr, "known bad coolkey version - see "
-                        "https://bugzilla.redhat.com/show_bug.cgi?id=802435\n");
-                continue;
-            }
-            vreader_emul = vreader_emul_new(slot, options->hw_card_type,
-                                            options->hw_type_params);
-            vreader = vreader_new(PK11_GetSlotName(slot), vreader_emul,
-                                  vreader_emul_delete);
-            vreader_add_reader(vreader);
-
-            if (PK11_IsPresent(slot)) {
-                VCard *vcard;
-                vcard = vcard_emul_mirror_card(vreader);
-                vreader_insert_card(vreader, vcard);
-                vcard_emul_init_series(vreader, vcard);
-                vcard_free(vcard);
-            }
-        }
-        vcard_emul_new_event_thread(module);
-    }
-    SECMOD_ReleaseReadLock(module_lock);
-    nss_emul_init = PR_TRUE;
-
-    return VCARD_EMUL_OK;
-}
-
-/* Recreate card insert events for all readers (user should
- * deduce implied reader insert. perhaps do a reader insert as well?)
- */
-void
-vcard_emul_replay_insertion_events(void)
-{
-    VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry;
-    VReaderList *list = vreader_get_reader_list();
-
-    for (current_entry = vreader_list_get_first(list); current_entry;
-            current_entry = next_entry) {
-        VReader *vreader = vreader_list_get_reader(current_entry);
-        next_entry = vreader_list_get_next(current_entry);
-        vreader_queue_card_event(vreader);
-    }
-
-    vreader_list_delete(list);
-}
-
-/*
- *  Silly little functions to help parsing our argument string
- */
-static int
-count_tokens(const char *str, char token, char token_end)
-{
-    int count = 0;
-
-    for (; *str; str++) {
-        if (*str == token) {
-            count++;
-        }
-        if (*str == token_end) {
-            break;
-        }
-    }
-    return count;
-}
-
-static const char *
-strip(const char *str)
-{
-    for (; *str && isspace(*str); str++) {
-    }
-    return str;
-}
-
-static const char *
-find_blank(const char *str)
-{
-    for (; *str && !isspace(*str); str++) {
-    }
-    return str;
-}
-
-
-/*
- *  We really want to use some existing argument parsing library here. That
- *  would give us a consistent look */
-static VCardEmulOptions options;
-#define READER_STEP 4
-
-/* Expects "args" to be at the beginning of a token (ie right after the ','
- * ending the previous token), and puts the next token start in "token",
- * and its length in "token_length". "token" will not be nul-terminated.
- * After calling the macro, "args" will be advanced to the beginning of
- * the next token.
- * This macro may call continue or break.
- */
-#define NEXT_TOKEN(token) \
-            (token) = args; \
-            args = strpbrk(args, ",)"); \
-            if (*args == 0) { \
-                break; \
-            } \
-            if (*args == ')') { \
-                args++; \
-                continue; \
-            } \
-            (token##_length) = args - (token); \
-            args = strip(args+1);
-
-VCardEmulOptions *
-vcard_emul_options(const char *args)
-{
-    int reader_count = 0;
-    VCardEmulOptions *opts;
-
-    /* Allow the future use of allocating the options structure on the fly */
-    memcpy(&options, &default_options, sizeof(options));
-    opts = &options;
-
-    do {
-        args = strip(args); /* strip off the leading spaces */
-        if (*args == ',') {
-            continue;
-        }
-        /* soft=(slot_name,virt_name,emul_type,emul_flags,cert_1, (no eol)
-         *       cert_2,cert_3...) */
-        if (strncmp(args, "soft=", 5) == 0) {
-            const char *name;
-            size_t name_length;
-            const char *vname;
-            size_t vname_length;
-            const char *type_params;
-            size_t type_params_length;
-            char type_str[100];
-            VCardEmulType type;
-            int count, i;
-            VirtualReaderOptions *vreaderOpt;
-
-            args = strip(args + 5);
-            if (*args != '(') {
-                continue;
-            }
-            args = strip(args+1);
-
-            NEXT_TOKEN(name)
-            NEXT_TOKEN(vname)
-            NEXT_TOKEN(type_params)
-            type_params_length = MIN(type_params_length, sizeof(type_str)-1);
-            memcpy(type_str, type_params, type_params_length);
-            type_str[type_params_length] = '\0';
-            type = vcard_emul_type_from_string(type_str);
-
-            NEXT_TOKEN(type_params)
-
-            if (*args == 0) {
-                break;
-            }
-
-            if (opts->vreader_count >= reader_count) {
-                reader_count += READER_STEP;
-                opts->vreader = g_renew(VirtualReaderOptions, opts->vreader,
-                                        reader_count);
-            }
-            vreaderOpt = &opts->vreader[opts->vreader_count];
-            vreaderOpt->name = g_strndup(name, name_length);
-            vreaderOpt->vname = g_strndup(vname, vname_length);
-            vreaderOpt->card_type = type;
-            vreaderOpt->type_params =
-                g_strndup(type_params, type_params_length);
-            count = count_tokens(args, ',', ')') + 1;
-            vreaderOpt->cert_count = count;
-            vreaderOpt->cert_name = g_new(char *, count);
-            for (i = 0; i < count; i++) {
-                const char *cert = args;
-                args = strpbrk(args, ",)");
-                vreaderOpt->cert_name[i] = g_strndup(cert, args - cert);
-                args = strip(args+1);
-            }
-            if (*args == ')') {
-                args++;
-            }
-            opts->vreader_count++;
-        /* use_hw= */
-        } else if (strncmp(args, "use_hw=", 7) == 0) {
-            args = strip(args+7);
-            if (*args == '0' || *args == 'N' || *args == 'n' || *args == 'F') {
-                opts->use_hw = PR_FALSE;
-            } else {
-                opts->use_hw = PR_TRUE;
-            }
-            args = find_blank(args);
-        /* hw_type= */
-        } else if (strncmp(args, "hw_type=", 8) == 0) {
-            args = strip(args+8);
-            opts->hw_card_type = vcard_emul_type_from_string(args);
-            args = find_blank(args);
-        /* hw_params= */
-        } else if (strncmp(args, "hw_params=", 10) == 0) {
-            const char *params;
-            args = strip(args+10);
-            params = args;
-            args = find_blank(args);
-            opts->hw_type_params = g_strndup(params, args-params);
-        /* db="/data/base/path" */
-        } else if (strncmp(args, "db=", 3) == 0) {
-            const char *db;
-            args = strip(args+3);
-            if (*args != '"') {
-                continue;
-            }
-            args++;
-            db = args;
-            args = strpbrk(args, "\"\n");
-            opts->nss_db = g_strndup(db, args-db);
-            if (*args != 0) {
-                args++;
-            }
-        } else {
-            args = find_blank(args);
-        }
-    } while (*args != 0);
-
-    return opts;
-}
-
-void
-vcard_emul_usage(void)
-{
-   fprintf(stderr,
-"emul args: comma separated list of the following arguments\n"
-" db={nss_database}               (default sql:/etc/pki/nssdb)\n"
-" use_hw=[yes|no]                 (default yes)\n"
-" hw_type={card_type_to_emulate}  (default CAC)\n"
-" hw_param={param_for_card}       (default \"\")\n"
-" soft=({slot_name},{vreader_name},{card_type_to_emulate},{params_for_card},\n"
-"       {cert1},{cert2},{cert3}    (default none)\n"
-"\n"
-"  {nss_database}          The location of the NSS cert & key database\n"
-"  {card_type_to_emulate}  What card interface to present to the guest\n"
-"  {param_for_card}        Card interface specific parameters\n"
-"  {slot_name}             NSS slot that contains the certs\n"
-"  {vreader_name}          Virtual reader name to present to the guest\n"
-"  {certN}                 Nickname of the certificate n on the virtual card\n"
-"\n"
-"These parameters come as a single string separated by blanks or newlines."
-"\n"
-"Unless use_hw is set to no, all tokens that look like removable hardware\n"
-"tokens will be presented to the guest using the emulator specified by\n"
-"hw_type, and parameters of hw_param.\n"
-"\n"
-"If more one or more soft= parameters are specified, these readers will be\n"
-"presented to the guest\n");
-}
diff --git a/libcacard/vcard_emul_type.c b/libcacard/vcard_emul_type.c
deleted file mode 100644
index 59a145820..000000000
--- a/libcacard/vcard_emul_type.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- *  This file contains utility functions which abstract the different card
- *  types.  The goal is that new card types can easily be added by simply
- *  changing this file and vcard_emul_type.h. It is currently not a requirement
- *  to dynamically add new card types.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <strings.h>
-#include "vcardt.h"
-#include "vcard_emul_type.h"
-#include "cac.h"
-
-VCardStatus vcard_init(VReader *vreader, VCard *vcard,
-                       VCardEmulType type, const char *params,
-                       unsigned char *const *cert, int cert_len[],
-                       VCardKey *key[], int cert_count)
-{
-    switch (type) {
-    case VCARD_EMUL_NONE:
-        break;
-    case VCARD_EMUL_CAC:
-        return cac_card_init(vreader, vcard, params,
-                             cert, cert_len, key,  cert_count);
-    /* add new ones here */
-    default:
-        break;
-    }
-    return VCARD_FAIL;
-}
-
-VCardEmulType vcard_emul_type_select(VReader *vreader)
-{
-#ifdef notdef
-    /* since there is only one emulator no need to call this function */
-    if (cac_is_cac_card(vreader) == VCARD_DONE) {
-        return VCARD_EMUL_CAC;
-    }
-#endif
-    /* return the default */
-    return VCARD_EMUL_CAC;
-}
-
-VCardEmulType vcard_emul_type_from_string(const char *type_string)
-{
-     if (strcasecmp(type_string, "CAC") == 0) {
-        return VCARD_EMUL_CAC;
-     }
-#ifdef USE_PASSTHRU
-     if (strcasecmp(type_string, "PASSTHRU") == 0) {
-        return VCARD_EMUL_PASSTHRU;
-     }
-#endif
-     return VCARD_EMUL_NONE;
-}
diff --git a/libcacard/vcard_emul_type.h b/libcacard/vcard_emul_type.h
deleted file mode 100644
index 0242f40eb..000000000
--- a/libcacard/vcard_emul_type.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *  This header file abstracts the different card types. The goal is new card
- *  types can easily be added by simply changing this file and
- *  vcard_emul_type.c. It is currently not a requirement to dynamically add new
- *  card types.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VCARD_EMUL_TYPE_H
-#define VCARD_EMUL_TYPE_H 1
-#include "vcardt.h"
-#include "vreadert.h"
-
-/*
- * types
- */
-typedef enum {
-     VCARD_EMUL_NONE = 0,
-     VCARD_EMUL_CAC,
-     VCARD_EMUL_PASSTHRU
-} VCardEmulType;
-
-/* functions used by the rest of the emulator */
-VCardStatus vcard_init(VReader *vreader, VCard *vcard, VCardEmulType type,
-                       const char *params, unsigned char * const *cert,
-                       int cert_len[], VCardKey *key[], int cert_count);
-VCardEmulType vcard_emul_type_select(VReader *vreader);
-VCardEmulType vcard_emul_type_from_string(const char *type_string);
-
-#endif
diff --git a/libcacard/vcardt.c b/libcacard/vcardt.c
deleted file mode 100644
index c67de2f3c..000000000
--- a/libcacard/vcardt.c
+++ /dev/null
@@ -1,40 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <glib.h>
-
-#include "vcardt.h"
-
-#include "vcardt_internal.h"
-
-/* create an ATR with appropriate historical bytes */
-#define ATR_TS_DIRECT_CONVENTION 0x3b
-#define ATR_TA_PRESENT 0x10
-#define ATR_TB_PRESENT 0x20
-#define ATR_TC_PRESENT 0x40
-#define ATR_TD_PRESENT 0x80
-
-unsigned char *vcard_alloc_atr(const char *postfix, int *atr_len)
-{
-    int postfix_len;
-    const char prefix[] = "VCARD_";
-    const char default_postfix[] = "DEFAULT";
-    const int prefix_len = sizeof(prefix) - 1;
-    int total_len;
-    unsigned char *atr;
-
-    if (postfix == NULL) {
-        postfix = default_postfix;
-    }
-    postfix_len = strlen(postfix);
-    total_len = 3 + prefix_len + postfix_len;
-    atr = g_malloc(total_len);
-    atr[0] = ATR_TS_DIRECT_CONVENTION;
-    atr[1] = ATR_TD_PRESENT + prefix_len + postfix_len;
-    atr[2] = 0x00;
-    memcpy(&atr[3], prefix, prefix_len);
-    memcpy(&atr[3 + prefix_len], postfix, postfix_len);
-    if (atr_len) {
-        *atr_len = total_len;
-    }
-    return atr;
-}
diff --git a/libcacard/vcardt.h b/libcacard/vcardt.h
deleted file mode 100644
index 795e26530..000000000
--- a/libcacard/vcardt.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef VCARDT_H
-#define VCARDT_H 1
-
-/*
- * these should come from some common spice header file
- */
-#include <assert.h>
-#ifndef MIN
-#define MIN(x, y) ((x) > (y) ? (y) : (x))
-#define MAX(x, y) ((x) > (y) ? (x) : (y))
-#endif
-
-typedef struct VCardStruct VCard;
-typedef struct VCardAPDUStruct VCardAPDU;
-typedef struct VCardResponseStruct VCardResponse;
-typedef struct VCardBufferResponseStruct VCardBufferResponse;
-typedef struct VCardAppletStruct VCardApplet;
-typedef struct VCardAppletPrivateStruct VCardAppletPrivate;
-typedef struct VCardKeyStruct VCardKey;  /* opaque */
-typedef struct VCardEmulStruct VCardEmul;
-
-#define MAX_CHANNEL 4
-
-typedef enum {
-    VCARD_DONE,
-    VCARD_NEXT,
-    VCARD_FAIL
-} VCardStatus;
-
-typedef enum {
-    VCARD_FILE_SYSTEM,
-    VCARD_VM,
-    VCARD_DIRECT
-} VCardType;
-
-typedef enum {
-    VCARD_POWER_ON,
-    VCARD_POWER_OFF
-} VCardPower;
-
-typedef VCardStatus (*VCardProcessAPDU)(VCard *card, VCardAPDU *apdu,
-                                        VCardResponse **response);
-typedef VCardStatus (*VCardResetApplet)(VCard *card, int channel);
-typedef void (*VCardAppletPrivateFree) (VCardAppletPrivate *);
-typedef void (*VCardEmulFree) (VCardEmul *);
-typedef void (*VCardGetAtr) (VCard *, unsigned char *atr, int *atr_len);
-
-struct VCardBufferResponseStruct {
-    unsigned char *buffer;
-    int buffer_len;
-    unsigned char *current;
-    int len;
-};
-
-#endif
diff --git a/libcacard/vcardt_internal.h b/libcacard/vcardt_internal.h
deleted file mode 100644
index e5c8d2dd3..000000000
--- a/libcacard/vcardt_internal.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef VCARDT_INTERNAL_H
-#define VCARDT_INTERNAL_H
-
-unsigned char *vcard_alloc_atr(const char *postfix, int *atr_len);
-
-#endif
diff --git a/libcacard/vevent.h b/libcacard/vevent.h
deleted file mode 100644
index 38c3482c3..000000000
--- a/libcacard/vevent.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef EVENT_H
-#define EVENT_H 1
-#include "eventt.h"
-#include "vreadert.h"
-#include "vcardt.h"
-
-VEvent *vevent_new(VEventType type, VReader *reader, VCard *card);
-void vevent_delete(VEvent *);
-
-/*
- * VEvent queueing services
- */
-void vevent_queue_vevent(VEvent *);
-void vevent_queue_init(void);
-
-/*
- *  VEvent dequeing services
- */
-VEvent *vevent_wait_next_vevent(void);
-VEvent *vevent_get_next_vevent(void);
-
-
-#endif
diff --git a/libcacard/vreader.c b/libcacard/vreader.c
deleted file mode 100644
index 9725f46a7..000000000
--- a/libcacard/vreader.c
+++ /dev/null
@@ -1,578 +0,0 @@
-/*
- * emulate the reader
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifdef G_LOG_DOMAIN
-#undef G_LOG_DOMAIN
-#endif
-#define G_LOG_DOMAIN "libcacard"
-
-#include "glib-compat.h"
-
-#include <string.h>
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-#include "vreader.h"
-#include "vevent.h"
-#include "cac.h" /* just for debugging defines */
-
-#define LIBCACARD_LOG_DOMAIN "libcacard"
-
-struct VReaderStruct {
-    int    reference_count;
-    VCard *card;
-    char *name;
-    vreader_id_t id;
-    CompatGMutex lock;
-    VReaderEmul  *reader_private;
-    VReaderEmulFree reader_private_free;
-};
-
-/*
- * Debug helpers
- */
-
-static const char *
-apdu_ins_to_string(int ins)
-{
-    switch (ins) {
-    case VCARD7816_INS_MANAGE_CHANNEL:
-        return "manage channel";
-    case VCARD7816_INS_EXTERNAL_AUTHENTICATE:
-        return "external authenticate";
-    case VCARD7816_INS_GET_CHALLENGE:
-        return "get challenge";
-    case VCARD7816_INS_INTERNAL_AUTHENTICATE:
-        return "internal authenticate";
-    case VCARD7816_INS_ERASE_BINARY:
-        return "erase binary";
-    case VCARD7816_INS_READ_BINARY:
-        return "read binary";
-    case VCARD7816_INS_WRITE_BINARY:
-        return "write binary";
-    case VCARD7816_INS_UPDATE_BINARY:
-        return "update binary";
-    case VCARD7816_INS_READ_RECORD:
-        return "read record";
-    case VCARD7816_INS_WRITE_RECORD:
-        return "write record";
-    case VCARD7816_INS_UPDATE_RECORD:
-        return "update record";
-    case VCARD7816_INS_APPEND_RECORD:
-        return "append record";
-    case VCARD7816_INS_ENVELOPE:
-        return "envelope";
-    case VCARD7816_INS_PUT_DATA:
-        return "put data";
-    case VCARD7816_INS_GET_DATA:
-        return "get data";
-    case VCARD7816_INS_SELECT_FILE:
-        return "select file";
-    case VCARD7816_INS_VERIFY:
-        return "verify";
-    case VCARD7816_INS_GET_RESPONSE:
-        return "get response";
-    case CAC_GET_PROPERTIES:
-        return "get properties";
-    case CAC_GET_ACR:
-        return "get acr";
-    case CAC_READ_BUFFER:
-        return "read buffer";
-    case CAC_UPDATE_BUFFER:
-        return "update buffer";
-    case CAC_SIGN_DECRYPT:
-        return "sign decrypt";
-    case CAC_GET_CERTIFICATE:
-        return "get certificate";
-    }
-    return "unknown";
-}
-
-/* manage locking */
-static inline void
-vreader_lock(VReader *reader)
-{
-    g_mutex_lock(&reader->lock);
-}
-
-static inline void
-vreader_unlock(VReader *reader)
-{
-    g_mutex_unlock(&reader->lock);
-}
-
-/*
- * vreader constructor
- */
-VReader *
-vreader_new(const char *name, VReaderEmul *private,
-            VReaderEmulFree private_free)
-{
-    VReader *reader;
-
-    reader = g_new(VReader, 1);
-    g_mutex_init(&reader->lock);
-    reader->reference_count = 1;
-    reader->name = g_strdup(name);
-    reader->card = NULL;
-    reader->id = (vreader_id_t)-1;
-    reader->reader_private = private;
-    reader->reader_private_free = private_free;
-    return reader;
-}
-
-/* get a reference */
-VReader*
-vreader_reference(VReader *reader)
-{
-    if (reader == NULL) {
-        return NULL;
-    }
-    vreader_lock(reader);
-    reader->reference_count++;
-    vreader_unlock(reader);
-    return reader;
-}
-
-/* free a reference */
-void
-vreader_free(VReader *reader)
-{
-    if (reader == NULL) {
-        return;
-    }
-    vreader_lock(reader);
-    if (reader->reference_count-- > 1) {
-        vreader_unlock(reader);
-        return;
-    }
-    vreader_unlock(reader);
-    g_mutex_clear(&reader->lock);
-    if (reader->card) {
-        vcard_free(reader->card);
-    }
-    g_free(reader->name);
-    if (reader->reader_private_free) {
-        reader->reader_private_free(reader->reader_private);
-    }
-    g_free(reader);
-}
-
-static VCard *
-vreader_get_card(VReader *reader)
-{
-    VCard *card;
-
-    vreader_lock(reader);
-    card = vcard_reference(reader->card);
-    vreader_unlock(reader);
-    return card;
-}
-
-VReaderStatus
-vreader_card_is_present(VReader *reader)
-{
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-    vcard_free(card);
-    return VREADER_OK;
-}
-
-vreader_id_t
-vreader_get_id(VReader *reader)
-{
-    if (reader == NULL) {
-        return (vreader_id_t)-1;
-    }
-    return reader->id;
-}
-
-VReaderStatus
-vreader_set_id(VReader *reader, vreader_id_t id)
-{
-    if (reader == NULL) {
-        return VREADER_NO_CARD;
-    }
-    reader->id = id;
-    return VREADER_OK;
-}
-
-const char *
-vreader_get_name(VReader *reader)
-{
-    if (reader == NULL) {
-        return NULL;
-    }
-    return reader->name;
-}
-
-VReaderEmul *
-vreader_get_private(VReader *reader)
-{
-    return reader->reader_private;
-}
-
-static VReaderStatus
-vreader_reset(VReader *reader, VCardPower power, unsigned char *atr, int *len)
-{
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-    /*
-     * clean up our state
-     */
-    vcard_reset(card, power);
-    if (atr) {
-        vcard_get_atr(card, atr, len);
-    }
-    vcard_free(card); /* free our reference */
-    return VREADER_OK;
-}
-
-VReaderStatus
-vreader_power_on(VReader *reader, unsigned char *atr, int *len)
-{
-    return vreader_reset(reader, VCARD_POWER_ON, atr, len);
-}
-
-VReaderStatus
-vreader_power_off(VReader *reader)
-{
-    return vreader_reset(reader, VCARD_POWER_OFF, NULL, 0);
-}
-
-
-VReaderStatus
-vreader_xfr_bytes(VReader *reader,
-                  unsigned char *send_buf, int send_buf_len,
-                  unsigned char *receive_buf, int *receive_buf_len)
-{
-    VCardAPDU *apdu;
-    VCardResponse *response = NULL;
-    VCardStatus card_status;
-    unsigned short status;
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-
-    apdu = vcard_apdu_new(send_buf, send_buf_len, &status);
-    if (apdu == NULL) {
-        response = vcard_make_response(status);
-        card_status = VCARD_DONE;
-    } else {
-        g_debug("%s: CLS=0x%x,INS=0x%x,P1=0x%x,P2=0x%x,Lc=%d,Le=%d %s",
-              __func__, apdu->a_cla, apdu->a_ins, apdu->a_p1, apdu->a_p2,
-              apdu->a_Lc, apdu->a_Le, apdu_ins_to_string(apdu->a_ins));
-        card_status = vcard_process_apdu(card, apdu, &response);
-        if (response) {
-            g_debug("%s: status=%d sw1=0x%x sw2=0x%x len=%d (total=%d)",
-                  __func__, response->b_status, response->b_sw1,
-                  response->b_sw2, response->b_len, response->b_total_len);
-        }
-    }
-    assert(card_status == VCARD_DONE && response);
-    int size = MIN(*receive_buf_len, response->b_total_len);
-    memcpy(receive_buf, response->b_data, size);
-    *receive_buf_len = size;
-    vcard_response_delete(response);
-    vcard_apdu_delete(apdu);
-    vcard_free(card); /* free our reference */
-    return VREADER_OK;
-}
-
-struct VReaderListStruct {
-    VReaderListEntry *head;
-    VReaderListEntry *tail;
-};
-
-struct VReaderListEntryStruct {
-    VReaderListEntry *next;
-    VReaderListEntry *prev;
-    VReader *reader;
-};
-
-
-static VReaderListEntry *
-vreader_list_entry_new(VReader *reader)
-{
-    VReaderListEntry *new_reader_list_entry;
-
-    new_reader_list_entry = g_new0(VReaderListEntry, 1);
-    new_reader_list_entry->reader = vreader_reference(reader);
-    return new_reader_list_entry;
-}
-
-static void
-vreader_list_entry_delete(VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    vreader_free(entry->reader);
-    g_free(entry);
-}
-
-
-static VReaderList *
-vreader_list_new(void)
-{
-    VReaderList *new_reader_list;
-
-    new_reader_list = g_new0(VReaderList, 1);
-    return new_reader_list;
-}
-
-void
-vreader_list_delete(VReaderList *list)
-{
-    VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry;
-    for (current_entry = vreader_list_get_first(list); current_entry;
-         current_entry = next_entry) {
-        next_entry = vreader_list_get_next(current_entry);
-        vreader_list_entry_delete(current_entry);
-    }
-    g_free(list);
-}
-
-
-VReaderListEntry *
-vreader_list_get_first(VReaderList *list)
-{
-    return list ? list->head : NULL;
-}
-
-VReaderListEntry *
-vreader_list_get_next(VReaderListEntry *current)
-{
-    return current ? current->next : NULL;
-}
-
-VReader *
-vreader_list_get_reader(VReaderListEntry *entry)
-{
-    return entry ? vreader_reference(entry->reader) : NULL;
-}
-
-static void
-vreader_queue(VReaderList *list, VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    entry->next = NULL;
-    entry->prev = list->tail;
-    if (list->head) {
-        list->tail->next = entry;
-    } else {
-        list->head = entry;
-    }
-    list->tail = entry;
-}
-
-static void
-vreader_dequeue(VReaderList *list, VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    if (entry->next == NULL) {
-        list->tail = entry->prev;
-    } else if (entry->prev == NULL) {
-        list->head = entry->next;
-    } else {
-        entry->prev->next = entry->next;
-        entry->next->prev = entry->prev;
-    }
-    if ((list->tail == NULL) || (list->head == NULL)) {
-        list->head = list->tail = NULL;
-    }
-    entry->next = entry->prev = NULL;
-}
-
-static VReaderList *vreader_list;
-static CompatGMutex vreader_list_mutex;
-
-static void
-vreader_list_init(void)
-{
-    vreader_list = vreader_list_new();
-}
-
-static void
-vreader_list_lock(void)
-{
-    g_mutex_lock(&vreader_list_mutex);
-}
-
-static void
-vreader_list_unlock(void)
-{
-    g_mutex_unlock(&vreader_list_mutex);
-}
-
-static VReaderList *
-vreader_copy_list(VReaderList *list)
-{
-    VReaderList *new_list;
-    VReaderListEntry *current_entry;
-
-    new_list = vreader_list_new();
-    if (new_list == NULL) {
-        return NULL;
-    }
-    for (current_entry = vreader_list_get_first(list); current_entry;
-         current_entry = vreader_list_get_next(current_entry)) {
-        VReader *reader = vreader_list_get_reader(current_entry);
-        VReaderListEntry *new_entry = vreader_list_entry_new(reader);
-
-        vreader_free(reader);
-        vreader_queue(new_list, new_entry);
-    }
-    return new_list;
-}
-
-VReaderList *
-vreader_get_reader_list(void)
-{
-    VReaderList *new_reader_list;
-
-    vreader_list_lock();
-    new_reader_list = vreader_copy_list(vreader_list);
-    vreader_list_unlock();
-    return new_reader_list;
-}
-
-VReader *
-vreader_get_reader_by_id(vreader_id_t id)
-{
-    VReader *reader = NULL;
-    VReaderListEntry *current_entry;
-
-    if (id == (vreader_id_t) -1) {
-        return NULL;
-    }
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-            current_entry = vreader_list_get_next(current_entry)) {
-        VReader *creader = vreader_list_get_reader(current_entry);
-        if (creader->id == id) {
-            reader = creader;
-            break;
-        }
-        vreader_free(creader);
-    }
-    vreader_list_unlock();
-    return reader;
-}
-
-VReader *
-vreader_get_reader_by_name(const char *name)
-{
-    VReader *reader = NULL;
-    VReaderListEntry *current_entry;
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-            current_entry = vreader_list_get_next(current_entry)) {
-        VReader *creader = vreader_list_get_reader(current_entry);
-        if (strcmp(creader->name, name) == 0) {
-            reader = creader;
-            break;
-        }
-        vreader_free(creader);
-    }
-    vreader_list_unlock();
-    return reader;
-}
-
-/* called from card_emul to initialize the readers */
-VReaderStatus
-vreader_add_reader(VReader *reader)
-{
-    VReaderListEntry *reader_entry;
-
-    reader_entry = vreader_list_entry_new(reader);
-    if (reader_entry == NULL) {
-        return VREADER_OUT_OF_MEMORY;
-    }
-    vreader_list_lock();
-    vreader_queue(vreader_list, reader_entry);
-    vreader_list_unlock();
-    vevent_queue_vevent(vevent_new(VEVENT_READER_INSERT, reader, NULL));
-    return VREADER_OK;
-}
-
-
-VReaderStatus
-vreader_remove_reader(VReader *reader)
-{
-    VReaderListEntry *current_entry;
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-         current_entry = vreader_list_get_next(current_entry)) {
-        if (current_entry->reader == reader) {
-            break;
-        }
-    }
-    vreader_dequeue(vreader_list, current_entry);
-    vreader_list_unlock();
-    vreader_list_entry_delete(current_entry);
-    vevent_queue_vevent(vevent_new(VEVENT_READER_REMOVE, reader, NULL));
-    return VREADER_OK;
-}
-
-/*
- * Generate VEVENT_CARD_INSERT or VEVENT_CARD_REMOVE based on vreader
- * state. Separated from vreader_insert_card to allow replaying events
- * for a given state.
- */
-void
-vreader_queue_card_event(VReader *reader)
-{
-    vevent_queue_vevent(vevent_new(
-        reader->card ? VEVENT_CARD_INSERT : VEVENT_CARD_REMOVE, reader,
-        reader->card));
-}
-
-/*
- * insert/remove a new card. for removal, card == NULL
- */
-VReaderStatus
-vreader_insert_card(VReader *reader, VCard *card)
-{
-    vreader_lock(reader);
-    if (reader->card) {
-        /* decrement reference count */
-        vcard_free(reader->card);
-        reader->card = NULL;
-    }
-    reader->card = vcard_reference(card);
-    vreader_unlock(reader);
-    vreader_queue_card_event(reader);
-    return VREADER_OK;
-}
-
-/*
- * initialize all the static reader structures
- */
-void
-vreader_init(void)
-{
-    vreader_list_init();
-}
-
diff --git a/libcacard/vreader.h b/libcacard/vreader.h
deleted file mode 100644
index ec2042136..000000000
--- a/libcacard/vreader.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VREADER_H
-#define VREADER_H 1
-
-#include "eventt.h"
-#include "vreadert.h"
-#include "vcardt.h"
-
-/*
- * calls for reader front end
- */
-VReaderStatus vreader_power_on(VReader *reader, unsigned char *atr, int *len);
-VReaderStatus vreader_power_off(VReader *reader);
-VReaderStatus vreader_xfr_bytes(VReader *reader, unsigned char *send_buf,
-                                int send_buf_len, unsigned char *receive_buf,
-                                int *receive_buf_len);
-
-/* constructor */
-VReader *vreader_new(const char *readerName, VReaderEmul *emul_private,
-                     VReaderEmulFree private_free);
-/* get a new reference to a reader */
-VReader *vreader_reference(VReader *reader);
-/* "destructor" (readers are reference counted) */
-void vreader_free(VReader *reader);
-
-/* accessors */
-VReaderEmul *vreader_get_private(VReader *);
-VReaderStatus vreader_card_is_present(VReader *reader);
-void vreader_queue_card_event(VReader *reader);
-const char *vreader_get_name(VReader *reader);
-vreader_id_t vreader_get_id(VReader *reader);
-VReaderStatus vreader_set_id(VReader *reader, vreader_id_t id);
-
-/* list operations */
-VReaderList *vreader_get_reader_list(void);
-void vreader_list_delete(VReaderList *list);
-VReader *vreader_list_get_reader(VReaderListEntry *entry);
-VReaderListEntry *vreader_list_get_first(VReaderList *list);
-VReaderListEntry *vreader_list_get_next(VReaderListEntry *list);
-VReader *vreader_get_reader_by_id(vreader_id_t id);
-VReader *vreader_get_reader_by_name(const char *name);
-
-/*
- * list tools for vcard_emul
- */
-void vreader_init(void);
-VReaderStatus vreader_add_reader(VReader *reader);
-VReaderStatus vreader_remove_reader(VReader *reader);
-VReaderStatus vreader_insert_card(VReader *reader, VCard *card);
-
-#endif
diff --git a/libcacard/vreadert.h b/libcacard/vreadert.h
deleted file mode 100644
index f97e0a79e..000000000
--- a/libcacard/vreadert.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VREADERT_H
-#define VREADERT_H 1
-
-typedef enum {
-    VREADER_OK = 0,
-    VREADER_NO_CARD,
-    VREADER_OUT_OF_MEMORY
-} VReaderStatus;
-
-typedef unsigned int vreader_id_t;
-typedef struct VReaderStruct VReader;
-typedef struct VReaderListStruct VReaderList;
-typedef struct VReaderListEntryStruct VReaderListEntry;
-
-typedef struct VReaderEmulStruct VReaderEmul;
-typedef void (*VReaderEmulFree)(VReaderEmul *);
-
-#endif
-
diff --git a/libcacard/vscard_common.h b/libcacard/vscard_common.h
deleted file mode 100644
index 08f68e4dd..000000000
--- a/libcacard/vscard_common.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/* Virtual Smart Card protocol definition
- *
- * This protocol is between a host using virtual smart card readers,
- * and a client providing the smart cards, perhaps by emulating them or by
- * access to real cards.
- *
- * Definitions for this protocol:
- *  Host   - user of the card
- *  Client - owner of the card
- *
- * The current implementation passes the raw APDU's from 7816 and additionally
- * contains messages to setup and teardown readers, handle insertion and
- * removal of cards, negotiate the protocol via capabilities and provide
- * for error responses.
- *
- * Copyright (c) 2011 Red Hat.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VSCARD_COMMON_H
-#define VSCARD_COMMON_H
-
-#include <stdint.h>
-
-#define VERSION_MAJOR_BITS 11
-#define VERSION_MIDDLE_BITS 11
-#define VERSION_MINOR_BITS 10
-
-#define MAKE_VERSION(major, middle, minor) \
-     ((major  << (VERSION_MINOR_BITS + VERSION_MIDDLE_BITS)) \
-      | (middle <<  VERSION_MINOR_BITS) \
-      | (minor))
-
-/*
- * IMPORTANT NOTE on VERSION
- *
- * The version below MUST be changed whenever a change in this file is made.
- *
- * The last digit, the minor, is for bug fix changes only.
- *
- * The middle digit is for backward / forward compatible changes, updates
- * to the existing messages, addition of fields.
- *
- * The major digit is for a breaking change of protocol, presumably
- * something that cannot be accommodated with the existing protocol.
- */
-
-#define VSCARD_VERSION MAKE_VERSION(0, 0, 2)
-
-typedef enum VSCMsgType {
-    VSC_Init = 1,
-    VSC_Error,
-    VSC_ReaderAdd,
-    VSC_ReaderRemove,
-    VSC_ATR,
-    VSC_CardRemove,
-    VSC_APDU,
-    VSC_Flush,
-    VSC_FlushComplete
-} VSCMsgType;
-
-typedef enum VSCErrorCode {
-    VSC_SUCCESS = 0,
-    VSC_GENERAL_ERROR = 1,
-    VSC_CANNOT_ADD_MORE_READERS,
-    VSC_CARD_ALREAY_INSERTED,
-} VSCErrorCode;
-
-#define VSCARD_UNDEFINED_READER_ID  0xffffffff
-#define VSCARD_MINIMAL_READER_ID    0
-
-#define VSCARD_MAGIC (*(uint32_t *)"VSCD")
-
-/*
- * Header
- * Each message starts with the header.
- * type - message type
- * reader_id - used by messages that are reader specific
- * length - length of payload (not including header, i.e. zero for
- *  messages containing empty payloads)
- */
-typedef struct VSCMsgHeader {
-    uint32_t   type;
-    uint32_t   reader_id;
-    uint32_t   length;
-    uint8_t    data[0];
-} VSCMsgHeader;
-
-/*
- * VSCMsgInit               Client <-> Host
- * Client sends it on connection, with its own capabilities.
- * Host replies with VSCMsgInit filling in its capabilities.
- *
- * It is not meant to be used for negotiation, i.e. sending more then
- * once from any side, but could be used for that in the future.
- */
-typedef struct VSCMsgInit {
-    uint32_t   magic;
-    uint32_t   version;
-    uint32_t   capabilities[1]; /* receiver must check length,
-                                   array may grow in the future*/
-} VSCMsgInit;
-
-/*
- * VSCMsgError              Client <-> Host
- * This message is a response to any of:
- *  Reader Add
- *  Reader Remove
- *  Card Remove
- * If the operation was successful then VSC_SUCCESS
- * is returned, other wise a specific error code.
- */
-typedef struct VSCMsgError {
-    uint32_t   code;
-} VSCMsgError;
-
-/*
- * VSCMsgReaderAdd          Client -> Host
- * Host replies with allocated reader id in VSCMsgError with code==SUCCESS.
- *
- * name - name of the reader on client side, UTF-8 encoded. Only used
- *  for client presentation (may be translated to the device presented to the
- *  guest), protocol wise only reader_id is important.
- */
-typedef struct VSCMsgReaderAdd {
-    uint8_t    name[0];
-} VSCMsgReaderAdd;
-
-/*
- * VSCMsgReaderRemove       Client -> Host
- * The client's reader has been removed.
- */
-typedef struct VSCMsgReaderRemove {
-} VSCMsgReaderRemove;
-
-/*
- * VSCMsgATR                Client -> Host
- * Answer to reset. Sent for card insertion or card reset. The reset/insertion
- * happens on the client side, they do not require any action from the host.
- */
-typedef struct VSCMsgATR {
-    uint8_t     atr[0];
-} VSCMsgATR;
-
-/*
- * VSCMsgCardRemove         Client -> Host
- * The client card has been removed.
- */
-typedef struct VSCMsgCardRemove {
-} VSCMsgCardRemove;
-
-/*
- * VSCMsgAPDU               Client <-> Host
- * Main reason of existence. Transfer a single APDU in either direction.
- */
-typedef struct VSCMsgAPDU {
-    uint8_t    data[0];
-} VSCMsgAPDU;
-
-/*
- * VSCMsgFlush               Host -> Client
- * Request client to send a FlushComplete message when it is done
- * servicing all outstanding APDUs
- */
-typedef struct VSCMsgFlush {
-} VSCMsgFlush;
-
-/*
- * VSCMsgFlush               Client -> Host
- * Client response to Flush after all APDUs have been processed and
- * responses sent.
- */
-typedef struct VSCMsgFlushComplete {
-} VSCMsgFlushComplete;
-
-#endif /* VSCARD_COMMON_H */
diff --git a/libcacard/vscclient.c b/libcacard/vscclient.c
deleted file mode 100644
index 065268443..000000000
--- a/libcacard/vscclient.c
+++ /dev/null
@@ -1,785 +0,0 @@
-/*
- * Tester for VSCARD protocol, client side.
- *
- * Can be used with ccid-card-passthru.
- *
- * Copyright (c) 2011 Red Hat.
- * Written by Alon Levy.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#ifndef _WIN32
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netdb.h>
-#include <unistd.h>
-#define closesocket(x) close(x)
-#else
-#include <getopt.h>
-#endif
-
-#include "glib-compat.h"
-
-#include "vscard_common.h"
-
-#include "vreader.h"
-#include "vcard_emul.h"
-#include "vevent.h"
-
-static int verbose;
-
-static void
-print_byte_array(
-    uint8_t *arrBytes,
-    unsigned int nSize
-) {
-    int i;
-    for (i = 0; i < nSize; i++) {
-        printf("%02X ", arrBytes[i]);
-    }
-    printf("\n");
-}
-
-static void
-print_usage(void) {
-    printf("vscclient [-c <certname> .. -e <emul_args> -d <level>%s] "
-            "<host> <port>\n",
-#ifdef USE_PASSTHRU
-    " -p");
-    printf(" -p use passthrough mode\n");
-#else
-   "");
-#endif
-    vcard_emul_usage();
-}
-
-static GIOChannel *channel_socket;
-static GByteArray *socket_to_send;
-static CompatGMutex socket_to_send_lock;
-static guint socket_tag;
-
-static void
-update_socket_watch(void);
-
-static gboolean
-do_socket_send(GIOChannel *source,
-               GIOCondition condition,
-               gpointer data)
-{
-    gsize bw;
-    GError *err = NULL;
-
-    g_return_val_if_fail(socket_to_send->len != 0, FALSE);
-    g_return_val_if_fail(condition & G_IO_OUT, FALSE);
-
-    g_io_channel_write_chars(channel_socket,
-        (gchar *)socket_to_send->data, socket_to_send->len, &bw, &err);
-    if (err != NULL) {
-        g_error("Error while sending socket %s", err->message);
-        return FALSE;
-    }
-    g_byte_array_remove_range(socket_to_send, 0, bw);
-
-    if (socket_to_send->len == 0) {
-        update_socket_watch();
-        return FALSE;
-    }
-    return TRUE;
-}
-
-static gboolean
-socket_prepare_sending(gpointer user_data)
-{
-    update_socket_watch();
-
-    return FALSE;
-}
-
-static int
-send_msg(
-    VSCMsgType type,
-    uint32_t reader_id,
-    const void *msg,
-    unsigned int length
-) {
-    VSCMsgHeader mhHeader;
-
-    g_mutex_lock(&socket_to_send_lock);
-
-    if (verbose > 10) {
-        printf("sending type=%d id=%u, len =%u (0x%x)\n",
-               type, reader_id, length, length);
-    }
-
-    mhHeader.type = htonl(type);
-    mhHeader.reader_id = 0;
-    mhHeader.length = htonl(length);
-    g_byte_array_append(socket_to_send, (guint8 *)&mhHeader, sizeof(mhHeader));
-    g_byte_array_append(socket_to_send, (guint8 *)msg, length);
-    g_idle_add(socket_prepare_sending, NULL);
-
-    g_mutex_unlock(&socket_to_send_lock);
-
-    return 0;
-}
-
-static VReader *pending_reader;
-static CompatGMutex pending_reader_lock;
-static CompatGCond pending_reader_condition;
-
-#define MAX_ATR_LEN 40
-static gpointer
-event_thread(gpointer arg)
-{
-    unsigned char atr[MAX_ATR_LEN];
-    int atr_len;
-    VEvent *event;
-    unsigned int reader_id;
-
-
-    while (1) {
-        const char *reader_name;
-
-        event = vevent_wait_next_vevent();
-        if (event == NULL) {
-            break;
-        }
-        reader_id = vreader_get_id(event->reader);
-        if (reader_id == VSCARD_UNDEFINED_READER_ID &&
-            event->type != VEVENT_READER_INSERT) {
-            /* ignore events from readers qemu has rejected */
-            /* if qemu is still deciding on this reader, wait to see if need to
-             * forward this event */
-            g_mutex_lock(&pending_reader_lock);
-            if (!pending_reader || (pending_reader != event->reader)) {
-                /* wasn't for a pending reader, this reader has already been
-                 * rejected by qemu */
-                g_mutex_unlock(&pending_reader_lock);
-                vevent_delete(event);
-                continue;
-            }
-            /* this reader hasn't been told its status from qemu yet, wait for
-             * that status */
-            while (pending_reader != NULL) {
-                g_cond_wait(&pending_reader_condition, &pending_reader_lock);
-            }
-            g_mutex_unlock(&pending_reader_lock);
-            /* now recheck the id */
-            reader_id = vreader_get_id(event->reader);
-            if (reader_id == VSCARD_UNDEFINED_READER_ID) {
-                /* this reader was rejected */
-                vevent_delete(event);
-                continue;
-            }
-            /* reader was accepted, now forward the event */
-        }
-        switch (event->type) {
-        case VEVENT_READER_INSERT:
-            /* tell qemu to insert a new CCID reader */
-            /* wait until qemu has responded to our first reader insert
-             * before we send a second. That way we won't confuse the responses
-             * */
-            g_mutex_lock(&pending_reader_lock);
-            while (pending_reader != NULL) {
-                g_cond_wait(&pending_reader_condition, &pending_reader_lock);
-            }
-            pending_reader = vreader_reference(event->reader);
-            g_mutex_unlock(&pending_reader_lock);
-            reader_name = vreader_get_name(event->reader);
-            if (verbose > 10) {
-                printf(" READER INSERT: %s\n", reader_name);
-            }
-            send_msg(VSC_ReaderAdd,
-                reader_id, /* currerntly VSCARD_UNDEFINED_READER_ID */
-                NULL, 0 /* TODO reader_name, strlen(reader_name) */);
-            break;
-        case VEVENT_READER_REMOVE:
-            /* future, tell qemu that an old CCID reader has been removed */
-            if (verbose > 10) {
-                printf(" READER REMOVE: %u\n", reader_id);
-            }
-            send_msg(VSC_ReaderRemove, reader_id, NULL, 0);
-            break;
-        case VEVENT_CARD_INSERT:
-            /* get the ATR (intended as a response to a power on from the
-             * reader */
-            atr_len = MAX_ATR_LEN;
-            vreader_power_on(event->reader, atr, &atr_len);
-            /* ATR call functions as a Card Insert event */
-            if (verbose > 10) {
-                printf(" CARD INSERT %u: ", reader_id);
-                print_byte_array(atr, atr_len);
-            }
-            send_msg(VSC_ATR, reader_id, atr, atr_len);
-            break;
-        case VEVENT_CARD_REMOVE:
-            /* Card removed */
-            if (verbose > 10) {
-                printf(" CARD REMOVE %u:\n", reader_id);
-            }
-            send_msg(VSC_CardRemove, reader_id, NULL, 0);
-            break;
-        default:
-            break;
-        }
-        vevent_delete(event);
-    }
-    return NULL;
-}
-
-
-static unsigned int
-get_id_from_string(char *string, unsigned int default_id)
-{
-    unsigned int id = atoi(string);
-
-    /* don't accidentally swith to zero because no numbers have been supplied */
-    if ((id == 0) && *string != '0') {
-        return default_id;
-    }
-    return id;
-}
-
-static int
-on_host_init(VSCMsgHeader *mhHeader, VSCMsgInit *incoming)
-{
-    uint32_t *capabilities = (incoming->capabilities);
-    int num_capabilities =
-        1 + ((mhHeader->length - sizeof(VSCMsgInit)) / sizeof(uint32_t));
-    int i;
-
-    incoming->version = ntohl(incoming->version);
-    if (incoming->version != VSCARD_VERSION) {
-        if (verbose > 0) {
-            printf("warning: host has version %d, we have %d\n",
-                verbose, VSCARD_VERSION);
-        }
-    }
-    if (incoming->magic != VSCARD_MAGIC) {
-        printf("unexpected magic: got %d, expected %d\n",
-            incoming->magic, VSCARD_MAGIC);
-        return -1;
-    }
-    for (i = 0 ; i < num_capabilities; ++i) {
-        capabilities[i] = ntohl(capabilities[i]);
-    }
-    /* Future: check capabilities */
-    /* remove whatever reader might be left in qemu,
-     * in case of an unclean previous exit. */
-    send_msg(VSC_ReaderRemove, VSCARD_MINIMAL_READER_ID, NULL, 0);
-    /* launch the event_thread. This will trigger reader adds for all the
-     * existing readers */
-    g_thread_new("vsc/event", event_thread, NULL);
-    return 0;
-}
-
-
-enum {
-    STATE_HEADER,
-    STATE_MESSAGE,
-};
-
-#define APDUBufSize 270
-
-static gboolean
-do_socket_read(GIOChannel *source,
-               GIOCondition condition,
-               gpointer data)
-{
-    int rv;
-    int dwSendLength;
-    int dwRecvLength;
-    uint8_t pbRecvBuffer[APDUBufSize];
-    static uint8_t pbSendBuffer[APDUBufSize];
-    VReaderStatus reader_status;
-    VReader *reader = NULL;
-    static VSCMsgHeader mhHeader;
-    VSCMsgError *error_msg;
-    GError *err = NULL;
-
-    static gchar *buf;
-    static gsize br, to_read;
-    static int state = STATE_HEADER;
-
-    if (state == STATE_HEADER && to_read == 0) {
-        buf = (gchar *)&mhHeader;
-        to_read = sizeof(mhHeader);
-    }
-
-    if (to_read > 0) {
-        g_io_channel_read_chars(source, (gchar *)buf, to_read, &br, &err);
-        if (err != NULL) {
-            g_error("error while reading: %s", err->message);
-        }
-        buf += br;
-        to_read -= br;
-        if (to_read != 0) {
-            return TRUE;
-        }
-    }
-
-    if (state == STATE_HEADER) {
-        mhHeader.type = ntohl(mhHeader.type);
-        mhHeader.reader_id = ntohl(mhHeader.reader_id);
-        mhHeader.length = ntohl(mhHeader.length);
-        if (verbose) {
-            printf("Header: type=%d, reader_id=%u length=%d (0x%x)\n",
-                   mhHeader.type, mhHeader.reader_id, mhHeader.length,
-                   mhHeader.length);
-        }
-        switch (mhHeader.type) {
-        case VSC_APDU:
-        case VSC_Flush:
-        case VSC_Error:
-        case VSC_Init:
-            buf = (gchar *)pbSendBuffer;
-            to_read = mhHeader.length;
-            state = STATE_MESSAGE;
-            return TRUE;
-        default:
-            fprintf(stderr, "Unexpected message of type 0x%X\n", mhHeader.type);
-            return FALSE;
-        }
-    }
-
-    if (state == STATE_MESSAGE) {
-        switch (mhHeader.type) {
-        case VSC_APDU:
-            if (verbose) {
-                printf(" recv APDU: ");
-                print_byte_array(pbSendBuffer, mhHeader.length);
-            }
-            /* Transmit received APDU */
-            dwSendLength = mhHeader.length;
-            dwRecvLength = sizeof(pbRecvBuffer);
-            reader = vreader_get_reader_by_id(mhHeader.reader_id);
-            reader_status = vreader_xfr_bytes(reader,
-                                              pbSendBuffer, dwSendLength,
-                                              pbRecvBuffer, &dwRecvLength);
-            if (reader_status == VREADER_OK) {
-                mhHeader.length = dwRecvLength;
-                if (verbose) {
-                    printf(" send response: ");
-                    print_byte_array(pbRecvBuffer, mhHeader.length);
-                }
-                send_msg(VSC_APDU, mhHeader.reader_id,
-                         pbRecvBuffer, dwRecvLength);
-            } else {
-                rv = reader_status; /* warning: not meaningful */
-                send_msg(VSC_Error, mhHeader.reader_id, &rv, sizeof(uint32_t));
-            }
-            vreader_free(reader);
-            reader = NULL; /* we've freed it, don't use it by accident
-                              again */
-            break;
-        case VSC_Flush:
-            /* TODO: actually flush */
-            send_msg(VSC_FlushComplete, mhHeader.reader_id, NULL, 0);
-            break;
-        case VSC_Error:
-            error_msg = (VSCMsgError *) pbSendBuffer;
-            if (error_msg->code == VSC_SUCCESS) {
-                g_mutex_lock(&pending_reader_lock);
-                if (pending_reader) {
-                    vreader_set_id(pending_reader, mhHeader.reader_id);
-                    vreader_free(pending_reader);
-                    pending_reader = NULL;
-                    g_cond_signal(&pending_reader_condition);
-                }
-                g_mutex_unlock(&pending_reader_lock);
-                break;
-            }
-            printf("warning: qemu refused to add reader\n");
-            if (error_msg->code == VSC_CANNOT_ADD_MORE_READERS) {
-                /* clear pending reader, qemu can't handle any more */
-                g_mutex_lock(&pending_reader_lock);
-                if (pending_reader) {
-                    pending_reader = NULL;
-                    /* make sure the event loop doesn't hang */
-                    g_cond_signal(&pending_reader_condition);
-                }
-                g_mutex_unlock(&pending_reader_lock);
-            }
-            break;
-        case VSC_Init:
-            if (on_host_init(&mhHeader, (VSCMsgInit *)pbSendBuffer) < 0) {
-                return FALSE;
-            }
-            break;
-        default:
-            g_assert_not_reached();
-            return FALSE;
-        }
-
-        state = STATE_HEADER;
-    }
-
-
-    return TRUE;
-}
-
-static gboolean
-do_socket(GIOChannel *source,
-          GIOCondition condition,
-          gpointer data)
-{
-    /* not sure if two watches work well with a single win32 sources */
-    if (condition & G_IO_OUT) {
-        if (!do_socket_send(source, condition, data)) {
-            return FALSE;
-        }
-    }
-
-    if (condition & G_IO_IN) {
-        if (!do_socket_read(source, condition, data)) {
-            return FALSE;
-        }
-    }
-
-    return TRUE;
-}
-
-static void
-update_socket_watch(void)
-{
-    gboolean out = socket_to_send->len > 0;
-
-    if (socket_tag != 0) {
-        g_source_remove(socket_tag);
-    }
-
-    socket_tag = g_io_add_watch(channel_socket,
-        G_IO_IN | (out ? G_IO_OUT : 0), do_socket, NULL);
-}
-
-static gboolean
-do_command(GIOChannel *source,
-           GIOCondition condition,
-           gpointer data)
-{
-    char *string;
-    VCardEmulError error;
-    static unsigned int default_reader_id;
-    unsigned int reader_id;
-    VReader *reader = NULL;
-    GError *err = NULL;
-
-    g_assert(condition & G_IO_IN);
-
-    reader_id = default_reader_id;
-    g_io_channel_read_line(source, &string, NULL, NULL, &err);
-    if (err != NULL) {
-        g_error("Error while reading command: %s", err->message);
-    }
-
-    if (string != NULL) {
-        if (strncmp(string, "exit", 4) == 0) {
-            /* remove all the readers */
-            VReaderList *list = vreader_get_reader_list();
-            VReaderListEntry *reader_entry;
-            printf("Active Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id == -1) {
-                    continue;
-                }
-                /* be nice and signal card removal first (qemu probably should
-                 * do this itself) */
-                if (vreader_card_is_present(reader) == VREADER_OK) {
-                    send_msg(VSC_CardRemove, reader_id, NULL, 0);
-                }
-                send_msg(VSC_ReaderRemove, reader_id, NULL, 0);
-            }
-            exit(0);
-        } else if (strncmp(string, "insert", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7], reader_id);
-            }
-            reader = vreader_get_reader_by_id(reader_id);
-            if (reader != NULL) {
-                error = vcard_emul_force_card_insert(reader);
-                printf("insert %s, returned %d\n",
-                       vreader_get_name(reader), error);
-            } else {
-                printf("no reader by id %u found\n", reader_id);
-            }
-        } else if (strncmp(string, "remove", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7], reader_id);
-            }
-            reader = vreader_get_reader_by_id(reader_id);
-            if (reader != NULL) {
-                error = vcard_emul_force_card_remove(reader);
-                printf("remove %s, returned %d\n",
-                       vreader_get_name(reader), error);
-            } else {
-                printf("no reader by id %u found\n", reader_id);
-            }
-        } else if (strncmp(string, "select", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7],
-                                               VSCARD_UNDEFINED_READER_ID);
-            }
-            if (reader_id != VSCARD_UNDEFINED_READER_ID) {
-                reader = vreader_get_reader_by_id(reader_id);
-            }
-            if (reader) {
-                printf("Selecting reader %u, %s\n", reader_id,
-                        vreader_get_name(reader));
-                default_reader_id = reader_id;
-            } else {
-                printf("Reader with id %u not found\n", reader_id);
-            }
-        } else if (strncmp(string, "debug", 5) == 0) {
-            if (string[5] == ' ') {
-                verbose = get_id_from_string(&string[6], 0);
-            }
-            printf("debug level = %d\n", verbose);
-        } else if (strncmp(string, "list", 4) == 0) {
-            VReaderList *list = vreader_get_reader_list();
-            VReaderListEntry *reader_entry;
-            printf("Active Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id == -1) {
-                    continue;
-                }
-                printf("%3u %s %s\n", reader_id,
-                       vreader_card_is_present(reader) == VREADER_OK ?
-                       "CARD_PRESENT" : "            ",
-                       vreader_get_name(reader));
-            }
-            printf("Inactive Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id != -1) {
-                    continue;
-                }
-
-                printf("INA %s %s\n",
-                       vreader_card_is_present(reader) == VREADER_OK ?
-                       "CARD_PRESENT" : "            ",
-                       vreader_get_name(reader));
-            }
-            vreader_list_delete(list);
-        } else if (*string != 0) {
-            printf("valid commands:\n");
-            printf("insert [reader_id]\n");
-            printf("remove [reader_id]\n");
-            printf("select reader_id\n");
-            printf("list\n");
-            printf("debug [level]\n");
-            printf("exit\n");
-        }
-    }
-    vreader_free(reader);
-    printf("> ");
-    fflush(stdout);
-
-    return TRUE;
-}
-
-
-/* just for ease of parsing command line arguments. */
-#define MAX_CERTS 100
-
-static int
-connect_to_qemu(
-    const char *host,
-    const char *port
-) {
-    struct addrinfo hints;
-    struct addrinfo *server = NULL;
-    int ret, sock;
-
-    sock = socket(AF_INET, SOCK_STREAM, 0);
-    if (sock < 0) {
-        /* Error */
-        fprintf(stderr, "Error opening socket!\n");
-        return -1;
-    }
-
-    memset(&hints, 0, sizeof(struct addrinfo));
-    hints.ai_family = AF_UNSPEC;
-    hints.ai_socktype = SOCK_STREAM;
-    hints.ai_flags = 0;
-    hints.ai_protocol = 0;          /* Any protocol */
-
-    ret = getaddrinfo(host, port, &hints, &server);
-
-    if (ret != 0) {
-        /* Error */
-        fprintf(stderr, "getaddrinfo failed\n");
-        goto cleanup_socket;
-    }
-
-    if (connect(sock, server->ai_addr, server->ai_addrlen) < 0) {
-        /* Error */
-        fprintf(stderr, "Could not connect\n");
-        goto cleanup_socket;
-    }
-    if (verbose) {
-        printf("Connected (sizeof Header=%zd)!\n", sizeof(VSCMsgHeader));
-    }
-
-    freeaddrinfo(server);
-    return sock;
-
-cleanup_socket:
-    if (server) {
-        freeaddrinfo(server);
-    }
-    closesocket(sock);
-    return -1;
-}
-
-int
-main(
-    int argc,
-    char *argv[]
-) {
-    GMainLoop *loop;
-    GIOChannel *channel_stdin;
-    char *qemu_host;
-    char *qemu_port;
-
-    VCardEmulOptions *command_line_options = NULL;
-
-    char *cert_names[MAX_CERTS];
-    char *emul_args = NULL;
-    int cert_count = 0;
-    int c, sock;
-
-#ifdef _WIN32
-    WSADATA Data;
-
-    if (WSAStartup(MAKEWORD(2, 2), &Data) != 0) {
-        c = WSAGetLastError();
-        fprintf(stderr, "WSAStartup: %d\n", c);
-        return 1;
-    }
-#endif
-#if !GLIB_CHECK_VERSION(2, 31, 0)
-    if (!g_thread_supported()) {
-         g_thread_init(NULL);
-    }
-#endif
-
-    while ((c = getopt(argc, argv, "c:e:pd:")) != -1) {
-        switch (c) {
-        case 'c':
-            if (cert_count >= MAX_CERTS) {
-                printf("too many certificates (max = %d)\n", MAX_CERTS);
-                exit(5);
-            }
-            cert_names[cert_count++] = optarg;
-            break;
-        case 'e':
-            emul_args = optarg;
-            break;
-        case 'p':
-            print_usage();
-            exit(4);
-            break;
-        case 'd':
-            verbose = get_id_from_string(optarg, 1);
-            break;
-        }
-    }
-
-    if (argc - optind != 2) {
-        print_usage();
-        exit(4);
-    }
-
-    if (cert_count > 0) {
-        char *new_args;
-        int len, i;
-        /* if we've given some -c options, we clearly we want do so some
-         * software emulation.  add that emulation now. this is NSS Emulator
-         * specific */
-        if (emul_args == NULL) {
-            emul_args = (char *)"db=\"/etc/pki/nssdb\"";
-        }
-#define SOFT_STRING ",soft=(,Virtual Reader,CAC,,"
-             /* 2 == close paren & null */
-        len = strlen(emul_args) + strlen(SOFT_STRING) + 2;
-        for (i = 0; i < cert_count; i++) {
-            len += strlen(cert_names[i])+1; /* 1 == comma */
-        }
-        new_args = g_malloc(len);
-        strcpy(new_args, emul_args);
-        strcat(new_args, SOFT_STRING);
-        for (i = 0; i < cert_count; i++) {
-            strcat(new_args, cert_names[i]);
-            strcat(new_args, ",");
-        }
-        strcat(new_args, ")");
-        emul_args = new_args;
-    }
-    if (emul_args) {
-        command_line_options = vcard_emul_options(emul_args);
-    }
-
-    qemu_host = g_strdup(argv[argc - 2]);
-    qemu_port = g_strdup(argv[argc - 1]);
-    sock = connect_to_qemu(qemu_host, qemu_port);
-    if (sock == -1) {
-        fprintf(stderr, "error opening socket, exiting.\n");
-        exit(5);
-    }
-
-    socket_to_send = g_byte_array_new();
-    vcard_emul_init(command_line_options);
-    loop = g_main_loop_new(NULL, TRUE);
-
-    printf("> ");
-    fflush(stdout);
-
-#ifdef _WIN32
-    channel_stdin = g_io_channel_win32_new_fd(STDIN_FILENO);
-#else
-    channel_stdin = g_io_channel_unix_new(STDIN_FILENO);
-#endif
-    g_io_add_watch(channel_stdin, G_IO_IN, do_command, NULL);
-#ifdef _WIN32
-    channel_socket = g_io_channel_win32_new_socket(sock);
-#else
-    channel_socket = g_io_channel_unix_new(sock);
-#endif
-    g_io_channel_set_encoding(channel_socket, NULL, NULL);
-    /* we buffer ourself for thread safety reasons */
-    g_io_channel_set_buffered(channel_socket, FALSE);
-
-    /* Send init message, Host responds (and then we send reader attachments) */
-    VSCMsgInit init = {
-        .version = htonl(VSCARD_VERSION),
-        .magic = VSCARD_MAGIC,
-        .capabilities = {0}
-    };
-    send_msg(VSC_Init, 0, &init, sizeof(init));
-
-    g_main_loop_run(loop);
-    g_main_loop_unref(loop);
-
-    g_io_channel_unref(channel_stdin);
-    g_io_channel_unref(channel_socket);
-    g_byte_array_free(socket_to_send, TRUE);
-
-    closesocket(sock);
-    return 0;
-}
diff --git a/libdecnumber/decContext.c b/libdecnumber/decContext.c
index 8b6ae21be..68e6f60e9 100644
--- a/libdecnumber/decContext.c
+++ b/libdecnumber/decContext.c
@@ -186,7 +186,7 @@ uInt decContextGetStatus(decContext *context) {
 /*  newstatus is the source for the bits to be restored		      */
 /*  mask indicates the bits to be restored (the status bit that	      */
 /*    corresponds to each 1 bit in the mask is set to the value of    */
-/*    the correspnding bit in newstatus)			      */
+/*    the corresponding bit in newstatus)			      */
 /*  returns context						      */
 /*								      */
 /* No error is possible.					      */
diff --git a/libdecnumber/decNumber.c b/libdecnumber/decNumber.c
index 58211e7af..ca1412f30 100644
--- a/libdecnumber/decNumber.c
+++ b/libdecnumber/decNumber.c
@@ -2238,7 +2238,7 @@ decNumber * decNumberPower(decNumber *res, const decNumber *lhs,
       /* if a negative power the constant 1 is needed, and if not subset */
       /* invert the lhs now rather than inverting the result later */
       if (decNumberIsNegative(rhs)) {	/* was a **-n [hence digits>0] */
-	decNumber *inv=invbuff;		/* asssume use fixed buffer */
+	decNumber *inv=invbuff;		/* assume use fixed buffer */
 	decNumberCopy(&dnOne, dac);	/* dnOne=1;  [needed now or later] */
 	#if DECSUBSET
 	if (set->extended) {		/* need to calculate 1/lhs */
@@ -3849,7 +3849,7 @@ static void decToString(const decNumber *dn, char *string, Flag eng) {
 /*								      */
 /* Addition, especially x=x+1, is speed-critical.		      */
 /* The static buffer is larger than might be expected to allow for    */
-/* calls from higher-level funtions (notable exp).		      */
+/* calls from higher-level functions (notably exp).		      */
 /* ------------------------------------------------------------------ */
 static decNumber * decAddOp(decNumber *res, const decNumber *lhs,
 			    const decNumber *rhs, decContext *set,
@@ -4263,7 +4263,7 @@ static decNumber * decAddOp(decNumber *res, const decNumber *lhs,
 /* long subtractions.  These are acc and var1 respectively.	      */
 /* var1 is a copy of the lhs coefficient, var2 is the rhs coefficient.*/
 /* The static buffers may be larger than might be expected to allow   */
-/* for calls from higher-level funtions (notable exp).		      */
+/* for calls from higher-level functions (notably exp).		      */
 /* ------------------------------------------------------------------ */
 static decNumber * decDivideOp(decNumber *res,
 			       const decNumber *lhs, const decNumber *rhs,
@@ -5254,7 +5254,7 @@ static decNumber * decMultiplyOp(decNumber *res, const decNumber *lhs,
 /*    exp(-x) where x can be the tiniest number (Ntiny).	      */
 /*								      */
 /* 2. Normalizing x to be <=0.1 (instead of <=1) reduces loop	      */
-/*    iterations by appoximately a third with additional (although    */
+/*    iterations by approximately a third with additional (although    */
 /*    diminishing) returns as the range is reduced to even smaller    */
 /*    fractions.  However, h (the power of 10 used to correct the     */
 /*    result at the end, see below) must be kept <=8 as otherwise     */
diff --git a/linux-headers/asm-arm/unistd.h b/linux-headers/asm-arm/unistd.h
new file mode 100644
index 000000000..0a1376c35
--- /dev/null
+++ b/linux-headers/asm-arm/unistd.h
@@ -0,0 +1,448 @@
+/*
+ *  arch/arm/include/asm/unistd.h
+ *
+ *  Copyright (C) 2001-2005 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Please forward _all_ changes to this file to rmk@arm.linux.org.uk,
+ * no matter what the change is.  Thanks!
+ */
+#ifndef __ASM_ARM_UNISTD_H
+#define __ASM_ARM_UNISTD_H
+
+#define __NR_OABI_SYSCALL_BASE	0x900000
+
+#if defined(__thumb__) || defined(__ARM_EABI__)
+#define __NR_SYSCALL_BASE	0
+#else
+#define __NR_SYSCALL_BASE	__NR_OABI_SYSCALL_BASE
+#endif
+
+/*
+ * This file contains the system call numbers.
+ */
+
+#define __NR_restart_syscall		(__NR_SYSCALL_BASE+  0)
+#define __NR_exit			(__NR_SYSCALL_BASE+  1)
+#define __NR_fork			(__NR_SYSCALL_BASE+  2)
+#define __NR_read			(__NR_SYSCALL_BASE+  3)
+#define __NR_write			(__NR_SYSCALL_BASE+  4)
+#define __NR_open			(__NR_SYSCALL_BASE+  5)
+#define __NR_close			(__NR_SYSCALL_BASE+  6)
+					/* 7 was sys_waitpid */
+#define __NR_creat			(__NR_SYSCALL_BASE+  8)
+#define __NR_link			(__NR_SYSCALL_BASE+  9)
+#define __NR_unlink			(__NR_SYSCALL_BASE+ 10)
+#define __NR_execve			(__NR_SYSCALL_BASE+ 11)
+#define __NR_chdir			(__NR_SYSCALL_BASE+ 12)
+#define __NR_time			(__NR_SYSCALL_BASE+ 13)
+#define __NR_mknod			(__NR_SYSCALL_BASE+ 14)
+#define __NR_chmod			(__NR_SYSCALL_BASE+ 15)
+#define __NR_lchown			(__NR_SYSCALL_BASE+ 16)
+					/* 17 was sys_break */
+					/* 18 was sys_stat */
+#define __NR_lseek			(__NR_SYSCALL_BASE+ 19)
+#define __NR_getpid			(__NR_SYSCALL_BASE+ 20)
+#define __NR_mount			(__NR_SYSCALL_BASE+ 21)
+#define __NR_umount			(__NR_SYSCALL_BASE+ 22)
+#define __NR_setuid			(__NR_SYSCALL_BASE+ 23)
+#define __NR_getuid			(__NR_SYSCALL_BASE+ 24)
+#define __NR_stime			(__NR_SYSCALL_BASE+ 25)
+#define __NR_ptrace			(__NR_SYSCALL_BASE+ 26)
+#define __NR_alarm			(__NR_SYSCALL_BASE+ 27)
+					/* 28 was sys_fstat */
+#define __NR_pause			(__NR_SYSCALL_BASE+ 29)
+#define __NR_utime			(__NR_SYSCALL_BASE+ 30)
+					/* 31 was sys_stty */
+					/* 32 was sys_gtty */
+#define __NR_access			(__NR_SYSCALL_BASE+ 33)
+#define __NR_nice			(__NR_SYSCALL_BASE+ 34)
+					/* 35 was sys_ftime */
+#define __NR_sync			(__NR_SYSCALL_BASE+ 36)
+#define __NR_kill			(__NR_SYSCALL_BASE+ 37)
+#define __NR_rename			(__NR_SYSCALL_BASE+ 38)
+#define __NR_mkdir			(__NR_SYSCALL_BASE+ 39)
+#define __NR_rmdir			(__NR_SYSCALL_BASE+ 40)
+#define __NR_dup			(__NR_SYSCALL_BASE+ 41)
+#define __NR_pipe			(__NR_SYSCALL_BASE+ 42)
+#define __NR_times			(__NR_SYSCALL_BASE+ 43)
+					/* 44 was sys_prof */
+#define __NR_brk			(__NR_SYSCALL_BASE+ 45)
+#define __NR_setgid			(__NR_SYSCALL_BASE+ 46)
+#define __NR_getgid			(__NR_SYSCALL_BASE+ 47)
+					/* 48 was sys_signal */
+#define __NR_geteuid			(__NR_SYSCALL_BASE+ 49)
+#define __NR_getegid			(__NR_SYSCALL_BASE+ 50)
+#define __NR_acct			(__NR_SYSCALL_BASE+ 51)
+#define __NR_umount2			(__NR_SYSCALL_BASE+ 52)
+					/* 53 was sys_lock */
+#define __NR_ioctl			(__NR_SYSCALL_BASE+ 54)
+#define __NR_fcntl			(__NR_SYSCALL_BASE+ 55)
+					/* 56 was sys_mpx */
+#define __NR_setpgid			(__NR_SYSCALL_BASE+ 57)
+					/* 58 was sys_ulimit */
+					/* 59 was sys_olduname */
+#define __NR_umask			(__NR_SYSCALL_BASE+ 60)
+#define __NR_chroot			(__NR_SYSCALL_BASE+ 61)
+#define __NR_ustat			(__NR_SYSCALL_BASE+ 62)
+#define __NR_dup2			(__NR_SYSCALL_BASE+ 63)
+#define __NR_getppid			(__NR_SYSCALL_BASE+ 64)
+#define __NR_getpgrp			(__NR_SYSCALL_BASE+ 65)
+#define __NR_setsid			(__NR_SYSCALL_BASE+ 66)
+#define __NR_sigaction			(__NR_SYSCALL_BASE+ 67)
+					/* 68 was sys_sgetmask */
+					/* 69 was sys_ssetmask */
+#define __NR_setreuid			(__NR_SYSCALL_BASE+ 70)
+#define __NR_setregid			(__NR_SYSCALL_BASE+ 71)
+#define __NR_sigsuspend			(__NR_SYSCALL_BASE+ 72)
+#define __NR_sigpending			(__NR_SYSCALL_BASE+ 73)
+#define __NR_sethostname		(__NR_SYSCALL_BASE+ 74)
+#define __NR_setrlimit			(__NR_SYSCALL_BASE+ 75)
+#define __NR_getrlimit			(__NR_SYSCALL_BASE+ 76)	/* Back compat 2GB limited rlimit */
+#define __NR_getrusage			(__NR_SYSCALL_BASE+ 77)
+#define __NR_gettimeofday		(__NR_SYSCALL_BASE+ 78)
+#define __NR_settimeofday		(__NR_SYSCALL_BASE+ 79)
+#define __NR_getgroups			(__NR_SYSCALL_BASE+ 80)
+#define __NR_setgroups			(__NR_SYSCALL_BASE+ 81)
+#define __NR_select			(__NR_SYSCALL_BASE+ 82)
+#define __NR_symlink			(__NR_SYSCALL_BASE+ 83)
+					/* 84 was sys_lstat */
+#define __NR_readlink			(__NR_SYSCALL_BASE+ 85)
+#define __NR_uselib			(__NR_SYSCALL_BASE+ 86)
+#define __NR_swapon			(__NR_SYSCALL_BASE+ 87)
+#define __NR_reboot			(__NR_SYSCALL_BASE+ 88)
+#define __NR_readdir			(__NR_SYSCALL_BASE+ 89)
+#define __NR_mmap			(__NR_SYSCALL_BASE+ 90)
+#define __NR_munmap			(__NR_SYSCALL_BASE+ 91)
+#define __NR_truncate			(__NR_SYSCALL_BASE+ 92)
+#define __NR_ftruncate			(__NR_SYSCALL_BASE+ 93)
+#define __NR_fchmod			(__NR_SYSCALL_BASE+ 94)
+#define __NR_fchown			(__NR_SYSCALL_BASE+ 95)
+#define __NR_getpriority		(__NR_SYSCALL_BASE+ 96)
+#define __NR_setpriority		(__NR_SYSCALL_BASE+ 97)
+					/* 98 was sys_profil */
+#define __NR_statfs			(__NR_SYSCALL_BASE+ 99)
+#define __NR_fstatfs			(__NR_SYSCALL_BASE+100)
+					/* 101 was sys_ioperm */
+#define __NR_socketcall			(__NR_SYSCALL_BASE+102)
+#define __NR_syslog			(__NR_SYSCALL_BASE+103)
+#define __NR_setitimer			(__NR_SYSCALL_BASE+104)
+#define __NR_getitimer			(__NR_SYSCALL_BASE+105)
+#define __NR_stat			(__NR_SYSCALL_BASE+106)
+#define __NR_lstat			(__NR_SYSCALL_BASE+107)
+#define __NR_fstat			(__NR_SYSCALL_BASE+108)
+					/* 109 was sys_uname */
+					/* 110 was sys_iopl */
+#define __NR_vhangup			(__NR_SYSCALL_BASE+111)
+					/* 112 was sys_idle */
+#define __NR_syscall			(__NR_SYSCALL_BASE+113) /* syscall to call a syscall! */
+#define __NR_wait4			(__NR_SYSCALL_BASE+114)
+#define __NR_swapoff			(__NR_SYSCALL_BASE+115)
+#define __NR_sysinfo			(__NR_SYSCALL_BASE+116)
+#define __NR_ipc			(__NR_SYSCALL_BASE+117)
+#define __NR_fsync			(__NR_SYSCALL_BASE+118)
+#define __NR_sigreturn			(__NR_SYSCALL_BASE+119)
+#define __NR_clone			(__NR_SYSCALL_BASE+120)
+#define __NR_setdomainname		(__NR_SYSCALL_BASE+121)
+#define __NR_uname			(__NR_SYSCALL_BASE+122)
+					/* 123 was sys_modify_ldt */
+#define __NR_adjtimex			(__NR_SYSCALL_BASE+124)
+#define __NR_mprotect			(__NR_SYSCALL_BASE+125)
+#define __NR_sigprocmask		(__NR_SYSCALL_BASE+126)
+					/* 127 was sys_create_module */
+#define __NR_init_module		(__NR_SYSCALL_BASE+128)
+#define __NR_delete_module		(__NR_SYSCALL_BASE+129)
+					/* 130 was sys_get_kernel_syms */
+#define __NR_quotactl			(__NR_SYSCALL_BASE+131)
+#define __NR_getpgid			(__NR_SYSCALL_BASE+132)
+#define __NR_fchdir			(__NR_SYSCALL_BASE+133)
+#define __NR_bdflush			(__NR_SYSCALL_BASE+134)
+#define __NR_sysfs			(__NR_SYSCALL_BASE+135)
+#define __NR_personality		(__NR_SYSCALL_BASE+136)
+					/* 137 was sys_afs_syscall */
+#define __NR_setfsuid			(__NR_SYSCALL_BASE+138)
+#define __NR_setfsgid			(__NR_SYSCALL_BASE+139)
+#define __NR__llseek			(__NR_SYSCALL_BASE+140)
+#define __NR_getdents			(__NR_SYSCALL_BASE+141)
+#define __NR__newselect			(__NR_SYSCALL_BASE+142)
+#define __NR_flock			(__NR_SYSCALL_BASE+143)
+#define __NR_msync			(__NR_SYSCALL_BASE+144)
+#define __NR_readv			(__NR_SYSCALL_BASE+145)
+#define __NR_writev			(__NR_SYSCALL_BASE+146)
+#define __NR_getsid			(__NR_SYSCALL_BASE+147)
+#define __NR_fdatasync			(__NR_SYSCALL_BASE+148)
+#define __NR__sysctl			(__NR_SYSCALL_BASE+149)
+#define __NR_mlock			(__NR_SYSCALL_BASE+150)
+#define __NR_munlock			(__NR_SYSCALL_BASE+151)
+#define __NR_mlockall			(__NR_SYSCALL_BASE+152)
+#define __NR_munlockall			(__NR_SYSCALL_BASE+153)
+#define __NR_sched_setparam		(__NR_SYSCALL_BASE+154)
+#define __NR_sched_getparam		(__NR_SYSCALL_BASE+155)
+#define __NR_sched_setscheduler		(__NR_SYSCALL_BASE+156)
+#define __NR_sched_getscheduler		(__NR_SYSCALL_BASE+157)
+#define __NR_sched_yield		(__NR_SYSCALL_BASE+158)
+#define __NR_sched_get_priority_max	(__NR_SYSCALL_BASE+159)
+#define __NR_sched_get_priority_min	(__NR_SYSCALL_BASE+160)
+#define __NR_sched_rr_get_interval	(__NR_SYSCALL_BASE+161)
+#define __NR_nanosleep			(__NR_SYSCALL_BASE+162)
+#define __NR_mremap			(__NR_SYSCALL_BASE+163)
+#define __NR_setresuid			(__NR_SYSCALL_BASE+164)
+#define __NR_getresuid			(__NR_SYSCALL_BASE+165)
+					/* 166 was sys_vm86 */
+					/* 167 was sys_query_module */
+#define __NR_poll			(__NR_SYSCALL_BASE+168)
+#define __NR_nfsservctl			(__NR_SYSCALL_BASE+169)
+#define __NR_setresgid			(__NR_SYSCALL_BASE+170)
+#define __NR_getresgid			(__NR_SYSCALL_BASE+171)
+#define __NR_prctl			(__NR_SYSCALL_BASE+172)
+#define __NR_rt_sigreturn		(__NR_SYSCALL_BASE+173)
+#define __NR_rt_sigaction		(__NR_SYSCALL_BASE+174)
+#define __NR_rt_sigprocmask		(__NR_SYSCALL_BASE+175)
+#define __NR_rt_sigpending		(__NR_SYSCALL_BASE+176)
+#define __NR_rt_sigtimedwait		(__NR_SYSCALL_BASE+177)
+#define __NR_rt_sigqueueinfo		(__NR_SYSCALL_BASE+178)
+#define __NR_rt_sigsuspend		(__NR_SYSCALL_BASE+179)
+#define __NR_pread64			(__NR_SYSCALL_BASE+180)
+#define __NR_pwrite64			(__NR_SYSCALL_BASE+181)
+#define __NR_chown			(__NR_SYSCALL_BASE+182)
+#define __NR_getcwd			(__NR_SYSCALL_BASE+183)
+#define __NR_capget			(__NR_SYSCALL_BASE+184)
+#define __NR_capset			(__NR_SYSCALL_BASE+185)
+#define __NR_sigaltstack		(__NR_SYSCALL_BASE+186)
+#define __NR_sendfile			(__NR_SYSCALL_BASE+187)
+					/* 188 reserved */
+					/* 189 reserved */
+#define __NR_vfork			(__NR_SYSCALL_BASE+190)
+#define __NR_ugetrlimit			(__NR_SYSCALL_BASE+191)	/* SuS compliant getrlimit */
+#define __NR_mmap2			(__NR_SYSCALL_BASE+192)
+#define __NR_truncate64			(__NR_SYSCALL_BASE+193)
+#define __NR_ftruncate64		(__NR_SYSCALL_BASE+194)
+#define __NR_stat64			(__NR_SYSCALL_BASE+195)
+#define __NR_lstat64			(__NR_SYSCALL_BASE+196)
+#define __NR_fstat64			(__NR_SYSCALL_BASE+197)
+#define __NR_lchown32			(__NR_SYSCALL_BASE+198)
+#define __NR_getuid32			(__NR_SYSCALL_BASE+199)
+#define __NR_getgid32			(__NR_SYSCALL_BASE+200)
+#define __NR_geteuid32			(__NR_SYSCALL_BASE+201)
+#define __NR_getegid32			(__NR_SYSCALL_BASE+202)
+#define __NR_setreuid32			(__NR_SYSCALL_BASE+203)
+#define __NR_setregid32			(__NR_SYSCALL_BASE+204)
+#define __NR_getgroups32		(__NR_SYSCALL_BASE+205)
+#define __NR_setgroups32		(__NR_SYSCALL_BASE+206)
+#define __NR_fchown32			(__NR_SYSCALL_BASE+207)
+#define __NR_setresuid32		(__NR_SYSCALL_BASE+208)
+#define __NR_getresuid32		(__NR_SYSCALL_BASE+209)
+#define __NR_setresgid32		(__NR_SYSCALL_BASE+210)
+#define __NR_getresgid32		(__NR_SYSCALL_BASE+211)
+#define __NR_chown32			(__NR_SYSCALL_BASE+212)
+#define __NR_setuid32			(__NR_SYSCALL_BASE+213)
+#define __NR_setgid32			(__NR_SYSCALL_BASE+214)
+#define __NR_setfsuid32			(__NR_SYSCALL_BASE+215)
+#define __NR_setfsgid32			(__NR_SYSCALL_BASE+216)
+#define __NR_getdents64			(__NR_SYSCALL_BASE+217)
+#define __NR_pivot_root			(__NR_SYSCALL_BASE+218)
+#define __NR_mincore			(__NR_SYSCALL_BASE+219)
+#define __NR_madvise			(__NR_SYSCALL_BASE+220)
+#define __NR_fcntl64			(__NR_SYSCALL_BASE+221)
+					/* 222 for tux */
+					/* 223 is unused */
+#define __NR_gettid			(__NR_SYSCALL_BASE+224)
+#define __NR_readahead			(__NR_SYSCALL_BASE+225)
+#define __NR_setxattr			(__NR_SYSCALL_BASE+226)
+#define __NR_lsetxattr			(__NR_SYSCALL_BASE+227)
+#define __NR_fsetxattr			(__NR_SYSCALL_BASE+228)
+#define __NR_getxattr			(__NR_SYSCALL_BASE+229)
+#define __NR_lgetxattr			(__NR_SYSCALL_BASE+230)
+#define __NR_fgetxattr			(__NR_SYSCALL_BASE+231)
+#define __NR_listxattr			(__NR_SYSCALL_BASE+232)
+#define __NR_llistxattr			(__NR_SYSCALL_BASE+233)
+#define __NR_flistxattr			(__NR_SYSCALL_BASE+234)
+#define __NR_removexattr		(__NR_SYSCALL_BASE+235)
+#define __NR_lremovexattr		(__NR_SYSCALL_BASE+236)
+#define __NR_fremovexattr		(__NR_SYSCALL_BASE+237)
+#define __NR_tkill			(__NR_SYSCALL_BASE+238)
+#define __NR_sendfile64			(__NR_SYSCALL_BASE+239)
+#define __NR_futex			(__NR_SYSCALL_BASE+240)
+#define __NR_sched_setaffinity		(__NR_SYSCALL_BASE+241)
+#define __NR_sched_getaffinity		(__NR_SYSCALL_BASE+242)
+#define __NR_io_setup			(__NR_SYSCALL_BASE+243)
+#define __NR_io_destroy			(__NR_SYSCALL_BASE+244)
+#define __NR_io_getevents		(__NR_SYSCALL_BASE+245)
+#define __NR_io_submit			(__NR_SYSCALL_BASE+246)
+#define __NR_io_cancel			(__NR_SYSCALL_BASE+247)
+#define __NR_exit_group			(__NR_SYSCALL_BASE+248)
+#define __NR_lookup_dcookie		(__NR_SYSCALL_BASE+249)
+#define __NR_epoll_create		(__NR_SYSCALL_BASE+250)
+#define __NR_epoll_ctl			(__NR_SYSCALL_BASE+251)
+#define __NR_epoll_wait			(__NR_SYSCALL_BASE+252)
+#define __NR_remap_file_pages		(__NR_SYSCALL_BASE+253)
+					/* 254 for set_thread_area */
+					/* 255 for get_thread_area */
+#define __NR_set_tid_address		(__NR_SYSCALL_BASE+256)
+#define __NR_timer_create		(__NR_SYSCALL_BASE+257)
+#define __NR_timer_settime		(__NR_SYSCALL_BASE+258)
+#define __NR_timer_gettime		(__NR_SYSCALL_BASE+259)
+#define __NR_timer_getoverrun		(__NR_SYSCALL_BASE+260)
+#define __NR_timer_delete		(__NR_SYSCALL_BASE+261)
+#define __NR_clock_settime		(__NR_SYSCALL_BASE+262)
+#define __NR_clock_gettime		(__NR_SYSCALL_BASE+263)
+#define __NR_clock_getres		(__NR_SYSCALL_BASE+264)
+#define __NR_clock_nanosleep		(__NR_SYSCALL_BASE+265)
+#define __NR_statfs64			(__NR_SYSCALL_BASE+266)
+#define __NR_fstatfs64			(__NR_SYSCALL_BASE+267)
+#define __NR_tgkill			(__NR_SYSCALL_BASE+268)
+#define __NR_utimes			(__NR_SYSCALL_BASE+269)
+#define __NR_arm_fadvise64_64		(__NR_SYSCALL_BASE+270)
+#define __NR_pciconfig_iobase		(__NR_SYSCALL_BASE+271)
+#define __NR_pciconfig_read		(__NR_SYSCALL_BASE+272)
+#define __NR_pciconfig_write		(__NR_SYSCALL_BASE+273)
+#define __NR_mq_open			(__NR_SYSCALL_BASE+274)
+#define __NR_mq_unlink			(__NR_SYSCALL_BASE+275)
+#define __NR_mq_timedsend		(__NR_SYSCALL_BASE+276)
+#define __NR_mq_timedreceive		(__NR_SYSCALL_BASE+277)
+#define __NR_mq_notify			(__NR_SYSCALL_BASE+278)
+#define __NR_mq_getsetattr		(__NR_SYSCALL_BASE+279)
+#define __NR_waitid			(__NR_SYSCALL_BASE+280)
+#define __NR_socket			(__NR_SYSCALL_BASE+281)
+#define __NR_bind			(__NR_SYSCALL_BASE+282)
+#define __NR_connect			(__NR_SYSCALL_BASE+283)
+#define __NR_listen			(__NR_SYSCALL_BASE+284)
+#define __NR_accept			(__NR_SYSCALL_BASE+285)
+#define __NR_getsockname		(__NR_SYSCALL_BASE+286)
+#define __NR_getpeername		(__NR_SYSCALL_BASE+287)
+#define __NR_socketpair			(__NR_SYSCALL_BASE+288)
+#define __NR_send			(__NR_SYSCALL_BASE+289)
+#define __NR_sendto			(__NR_SYSCALL_BASE+290)
+#define __NR_recv			(__NR_SYSCALL_BASE+291)
+#define __NR_recvfrom			(__NR_SYSCALL_BASE+292)
+#define __NR_shutdown			(__NR_SYSCALL_BASE+293)
+#define __NR_setsockopt			(__NR_SYSCALL_BASE+294)
+#define __NR_getsockopt			(__NR_SYSCALL_BASE+295)
+#define __NR_sendmsg			(__NR_SYSCALL_BASE+296)
+#define __NR_recvmsg			(__NR_SYSCALL_BASE+297)
+#define __NR_semop			(__NR_SYSCALL_BASE+298)
+#define __NR_semget			(__NR_SYSCALL_BASE+299)
+#define __NR_semctl			(__NR_SYSCALL_BASE+300)
+#define __NR_msgsnd			(__NR_SYSCALL_BASE+301)
+#define __NR_msgrcv			(__NR_SYSCALL_BASE+302)
+#define __NR_msgget			(__NR_SYSCALL_BASE+303)
+#define __NR_msgctl			(__NR_SYSCALL_BASE+304)
+#define __NR_shmat			(__NR_SYSCALL_BASE+305)
+#define __NR_shmdt			(__NR_SYSCALL_BASE+306)
+#define __NR_shmget			(__NR_SYSCALL_BASE+307)
+#define __NR_shmctl			(__NR_SYSCALL_BASE+308)
+#define __NR_add_key			(__NR_SYSCALL_BASE+309)
+#define __NR_request_key		(__NR_SYSCALL_BASE+310)
+#define __NR_keyctl			(__NR_SYSCALL_BASE+311)
+#define __NR_semtimedop			(__NR_SYSCALL_BASE+312)
+#define __NR_vserver			(__NR_SYSCALL_BASE+313)
+#define __NR_ioprio_set			(__NR_SYSCALL_BASE+314)
+#define __NR_ioprio_get			(__NR_SYSCALL_BASE+315)
+#define __NR_inotify_init		(__NR_SYSCALL_BASE+316)
+#define __NR_inotify_add_watch		(__NR_SYSCALL_BASE+317)
+#define __NR_inotify_rm_watch		(__NR_SYSCALL_BASE+318)
+#define __NR_mbind			(__NR_SYSCALL_BASE+319)
+#define __NR_get_mempolicy		(__NR_SYSCALL_BASE+320)
+#define __NR_set_mempolicy		(__NR_SYSCALL_BASE+321)
+#define __NR_openat			(__NR_SYSCALL_BASE+322)
+#define __NR_mkdirat			(__NR_SYSCALL_BASE+323)
+#define __NR_mknodat			(__NR_SYSCALL_BASE+324)
+#define __NR_fchownat			(__NR_SYSCALL_BASE+325)
+#define __NR_futimesat			(__NR_SYSCALL_BASE+326)
+#define __NR_fstatat64			(__NR_SYSCALL_BASE+327)
+#define __NR_unlinkat			(__NR_SYSCALL_BASE+328)
+#define __NR_renameat			(__NR_SYSCALL_BASE+329)
+#define __NR_linkat			(__NR_SYSCALL_BASE+330)
+#define __NR_symlinkat			(__NR_SYSCALL_BASE+331)
+#define __NR_readlinkat			(__NR_SYSCALL_BASE+332)
+#define __NR_fchmodat			(__NR_SYSCALL_BASE+333)
+#define __NR_faccessat			(__NR_SYSCALL_BASE+334)
+#define __NR_pselect6			(__NR_SYSCALL_BASE+335)
+#define __NR_ppoll			(__NR_SYSCALL_BASE+336)
+#define __NR_unshare			(__NR_SYSCALL_BASE+337)
+#define __NR_set_robust_list		(__NR_SYSCALL_BASE+338)
+#define __NR_get_robust_list		(__NR_SYSCALL_BASE+339)
+#define __NR_splice			(__NR_SYSCALL_BASE+340)
+#define __NR_arm_sync_file_range	(__NR_SYSCALL_BASE+341)
+#define __NR_sync_file_range2		__NR_arm_sync_file_range
+#define __NR_tee			(__NR_SYSCALL_BASE+342)
+#define __NR_vmsplice			(__NR_SYSCALL_BASE+343)
+#define __NR_move_pages			(__NR_SYSCALL_BASE+344)
+#define __NR_getcpu			(__NR_SYSCALL_BASE+345)
+#define __NR_epoll_pwait		(__NR_SYSCALL_BASE+346)
+#define __NR_kexec_load			(__NR_SYSCALL_BASE+347)
+#define __NR_utimensat			(__NR_SYSCALL_BASE+348)
+#define __NR_signalfd			(__NR_SYSCALL_BASE+349)
+#define __NR_timerfd_create		(__NR_SYSCALL_BASE+350)
+#define __NR_eventfd			(__NR_SYSCALL_BASE+351)
+#define __NR_fallocate			(__NR_SYSCALL_BASE+352)
+#define __NR_timerfd_settime		(__NR_SYSCALL_BASE+353)
+#define __NR_timerfd_gettime		(__NR_SYSCALL_BASE+354)
+#define __NR_signalfd4			(__NR_SYSCALL_BASE+355)
+#define __NR_eventfd2			(__NR_SYSCALL_BASE+356)
+#define __NR_epoll_create1		(__NR_SYSCALL_BASE+357)
+#define __NR_dup3			(__NR_SYSCALL_BASE+358)
+#define __NR_pipe2			(__NR_SYSCALL_BASE+359)
+#define __NR_inotify_init1		(__NR_SYSCALL_BASE+360)
+#define __NR_preadv			(__NR_SYSCALL_BASE+361)
+#define __NR_pwritev			(__NR_SYSCALL_BASE+362)
+#define __NR_rt_tgsigqueueinfo		(__NR_SYSCALL_BASE+363)
+#define __NR_perf_event_open		(__NR_SYSCALL_BASE+364)
+#define __NR_recvmmsg			(__NR_SYSCALL_BASE+365)
+#define __NR_accept4			(__NR_SYSCALL_BASE+366)
+#define __NR_fanotify_init		(__NR_SYSCALL_BASE+367)
+#define __NR_fanotify_mark		(__NR_SYSCALL_BASE+368)
+#define __NR_prlimit64			(__NR_SYSCALL_BASE+369)
+#define __NR_name_to_handle_at		(__NR_SYSCALL_BASE+370)
+#define __NR_open_by_handle_at		(__NR_SYSCALL_BASE+371)
+#define __NR_clock_adjtime		(__NR_SYSCALL_BASE+372)
+#define __NR_syncfs			(__NR_SYSCALL_BASE+373)
+#define __NR_sendmmsg			(__NR_SYSCALL_BASE+374)
+#define __NR_setns			(__NR_SYSCALL_BASE+375)
+#define __NR_process_vm_readv		(__NR_SYSCALL_BASE+376)
+#define __NR_process_vm_writev		(__NR_SYSCALL_BASE+377)
+#define __NR_kcmp			(__NR_SYSCALL_BASE+378)
+#define __NR_finit_module		(__NR_SYSCALL_BASE+379)
+#define __NR_sched_setattr		(__NR_SYSCALL_BASE+380)
+#define __NR_sched_getattr		(__NR_SYSCALL_BASE+381)
+#define __NR_renameat2			(__NR_SYSCALL_BASE+382)
+#define __NR_seccomp			(__NR_SYSCALL_BASE+383)
+#define __NR_getrandom			(__NR_SYSCALL_BASE+384)
+#define __NR_memfd_create		(__NR_SYSCALL_BASE+385)
+#define __NR_bpf			(__NR_SYSCALL_BASE+386)
+#define __NR_execveat			(__NR_SYSCALL_BASE+387)
+#define __NR_userfaultfd		(__NR_SYSCALL_BASE+388)
+#define __NR_membarrier			(__NR_SYSCALL_BASE+389)
+
+/*
+ * The following SWIs are ARM private.
+ */
+#define __ARM_NR_BASE			(__NR_SYSCALL_BASE+0x0f0000)
+#define __ARM_NR_breakpoint		(__ARM_NR_BASE+1)
+#define __ARM_NR_cacheflush		(__ARM_NR_BASE+2)
+#define __ARM_NR_usr26			(__ARM_NR_BASE+3)
+#define __ARM_NR_usr32			(__ARM_NR_BASE+4)
+#define __ARM_NR_set_tls		(__ARM_NR_BASE+5)
+
+/*
+ * The following syscalls are obsolete and no longer available for EABI.
+ */
+#if defined(__ARM_EABI__)
+#undef __NR_time
+#undef __NR_umount
+#undef __NR_stime
+#undef __NR_alarm
+#undef __NR_utime
+#undef __NR_getrlimit
+#undef __NR_select
+#undef __NR_readdir
+#undef __NR_mmap
+#undef __NR_socketcall
+#undef __NR_syscall
+#undef __NR_ipc
+#endif
+
+#endif /* __ASM_ARM_UNISTD_H */
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index c8abf257c..d3714c0aa 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -53,14 +53,20 @@ struct kvm_regs {
 	struct user_fpsimd_state fp_regs;
 };
 
-/* Supported Processor Types */
+/*
+ * Supported CPU Targets - Adding a new target type is not recommended,
+ * unless there are some special registers not supported by the
+ * genericv8 syreg table.
+ */
 #define KVM_ARM_TARGET_AEM_V8		0
 #define KVM_ARM_TARGET_FOUNDATION_V8	1
 #define KVM_ARM_TARGET_CORTEX_A57	2
 #define KVM_ARM_TARGET_XGENE_POTENZA	3
 #define KVM_ARM_TARGET_CORTEX_A53	4
+/* Generic ARM v8 target */
+#define KVM_ARM_TARGET_GENERIC_V8	5
 
-#define KVM_ARM_NUM_TARGETS		5
+#define KVM_ARM_NUM_TARGETS		6
 
 /* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */
 #define KVM_ARM_DEVICE_TYPE_SHIFT	0
@@ -100,12 +106,39 @@ struct kvm_sregs {
 struct kvm_fpu {
 };
 
+/*
+ * See v8 ARM ARM D7.3: Debug Registers
+ *
+ * The architectural limit is 16 debug registers of each type although
+ * in practice there are usually less (see ID_AA64DFR0_EL1).
+ *
+ * Although the control registers are architecturally defined as 32
+ * bits wide we use a 64 bit structure here to keep parity with
+ * KVM_GET/SET_ONE_REG behaviour which treats all system registers as
+ * 64 bit values. It also allows for the possibility of the
+ * architecture expanding the control registers without having to
+ * change the userspace ABI.
+ */
+#define KVM_ARM_MAX_DBG_REGS 16
 struct kvm_guest_debug_arch {
+	__u64 dbg_bcr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_bvr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_wcr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_wvr[KVM_ARM_MAX_DBG_REGS];
 };
 
 struct kvm_debug_exit_arch {
+	__u32 hsr;
+	__u64 far;	/* used for watchpoints */
 };
 
+/*
+ * Architecture specific defines for kvm_guest_debug->control
+ */
+
+#define KVM_GUESTDBG_USE_SW_BP		(1 << 16)
+#define KVM_GUESTDBG_USE_HW		(1 << 17)
+
 struct kvm_sync_regs {
 };
 
diff --git a/linux-headers/asm-arm64/unistd.h b/linux-headers/asm-arm64/unistd.h
new file mode 100644
index 000000000..1caadc24e
--- /dev/null
+++ b/linux-headers/asm-arm64/unistd.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2012 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <asm-generic/unistd.h>
diff --git a/linux-headers/asm-mips/unistd.h b/linux-headers/asm-mips/unistd.h
new file mode 100644
index 000000000..2a2020938
--- /dev/null
+++ b/linux-headers/asm-mips/unistd.h
@@ -0,0 +1,1063 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 1995, 96, 97, 98, 99, 2000 by Ralf Baechle
+ * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
+ *
+ * Changed system calls macros _syscall5 - _syscall7 to push args 5 to 7 onto
+ * the stack. Robin Farine for ACN S.A, Copyright (C) 1996 by ACN S.A
+ */
+#ifndef _ASM_UNISTD_H
+#define _ASM_UNISTD_H
+
+#include <asm/sgidefs.h>
+
+#if _MIPS_SIM == _MIPS_SIM_ABI32
+
+/*
+ * Linux o32 style syscalls are in the range from 4000 to 4999.
+ */
+#define __NR_Linux			4000
+#define __NR_syscall			(__NR_Linux +	0)
+#define __NR_exit			(__NR_Linux +	1)
+#define __NR_fork			(__NR_Linux +	2)
+#define __NR_read			(__NR_Linux +	3)
+#define __NR_write			(__NR_Linux +	4)
+#define __NR_open			(__NR_Linux +	5)
+#define __NR_close			(__NR_Linux +	6)
+#define __NR_waitpid			(__NR_Linux +	7)
+#define __NR_creat			(__NR_Linux +	8)
+#define __NR_link			(__NR_Linux +	9)
+#define __NR_unlink			(__NR_Linux +  10)
+#define __NR_execve			(__NR_Linux +  11)
+#define __NR_chdir			(__NR_Linux +  12)
+#define __NR_time			(__NR_Linux +  13)
+#define __NR_mknod			(__NR_Linux +  14)
+#define __NR_chmod			(__NR_Linux +  15)
+#define __NR_lchown			(__NR_Linux +  16)
+#define __NR_break			(__NR_Linux +  17)
+#define __NR_unused18			(__NR_Linux +  18)
+#define __NR_lseek			(__NR_Linux +  19)
+#define __NR_getpid			(__NR_Linux +  20)
+#define __NR_mount			(__NR_Linux +  21)
+#define __NR_umount			(__NR_Linux +  22)
+#define __NR_setuid			(__NR_Linux +  23)
+#define __NR_getuid			(__NR_Linux +  24)
+#define __NR_stime			(__NR_Linux +  25)
+#define __NR_ptrace			(__NR_Linux +  26)
+#define __NR_alarm			(__NR_Linux +  27)
+#define __NR_unused28			(__NR_Linux +  28)
+#define __NR_pause			(__NR_Linux +  29)
+#define __NR_utime			(__NR_Linux +  30)
+#define __NR_stty			(__NR_Linux +  31)
+#define __NR_gtty			(__NR_Linux +  32)
+#define __NR_access			(__NR_Linux +  33)
+#define __NR_nice			(__NR_Linux +  34)
+#define __NR_ftime			(__NR_Linux +  35)
+#define __NR_sync			(__NR_Linux +  36)
+#define __NR_kill			(__NR_Linux +  37)
+#define __NR_rename			(__NR_Linux +  38)
+#define __NR_mkdir			(__NR_Linux +  39)
+#define __NR_rmdir			(__NR_Linux +  40)
+#define __NR_dup			(__NR_Linux +  41)
+#define __NR_pipe			(__NR_Linux +  42)
+#define __NR_times			(__NR_Linux +  43)
+#define __NR_prof			(__NR_Linux +  44)
+#define __NR_brk			(__NR_Linux +  45)
+#define __NR_setgid			(__NR_Linux +  46)
+#define __NR_getgid			(__NR_Linux +  47)
+#define __NR_signal			(__NR_Linux +  48)
+#define __NR_geteuid			(__NR_Linux +  49)
+#define __NR_getegid			(__NR_Linux +  50)
+#define __NR_acct			(__NR_Linux +  51)
+#define __NR_umount2			(__NR_Linux +  52)
+#define __NR_lock			(__NR_Linux +  53)
+#define __NR_ioctl			(__NR_Linux +  54)
+#define __NR_fcntl			(__NR_Linux +  55)
+#define __NR_mpx			(__NR_Linux +  56)
+#define __NR_setpgid			(__NR_Linux +  57)
+#define __NR_ulimit			(__NR_Linux +  58)
+#define __NR_unused59			(__NR_Linux +  59)
+#define __NR_umask			(__NR_Linux +  60)
+#define __NR_chroot			(__NR_Linux +  61)
+#define __NR_ustat			(__NR_Linux +  62)
+#define __NR_dup2			(__NR_Linux +  63)
+#define __NR_getppid			(__NR_Linux +  64)
+#define __NR_getpgrp			(__NR_Linux +  65)
+#define __NR_setsid			(__NR_Linux +  66)
+#define __NR_sigaction			(__NR_Linux +  67)
+#define __NR_sgetmask			(__NR_Linux +  68)
+#define __NR_ssetmask			(__NR_Linux +  69)
+#define __NR_setreuid			(__NR_Linux +  70)
+#define __NR_setregid			(__NR_Linux +  71)
+#define __NR_sigsuspend			(__NR_Linux +  72)
+#define __NR_sigpending			(__NR_Linux +  73)
+#define __NR_sethostname		(__NR_Linux +  74)
+#define __NR_setrlimit			(__NR_Linux +  75)
+#define __NR_getrlimit			(__NR_Linux +  76)
+#define __NR_getrusage			(__NR_Linux +  77)
+#define __NR_gettimeofday		(__NR_Linux +  78)
+#define __NR_settimeofday		(__NR_Linux +  79)
+#define __NR_getgroups			(__NR_Linux +  80)
+#define __NR_setgroups			(__NR_Linux +  81)
+#define __NR_reserved82			(__NR_Linux +  82)
+#define __NR_symlink			(__NR_Linux +  83)
+#define __NR_unused84			(__NR_Linux +  84)
+#define __NR_readlink			(__NR_Linux +  85)
+#define __NR_uselib			(__NR_Linux +  86)
+#define __NR_swapon			(__NR_Linux +  87)
+#define __NR_reboot			(__NR_Linux +  88)
+#define __NR_readdir			(__NR_Linux +  89)
+#define __NR_mmap			(__NR_Linux +  90)
+#define __NR_munmap			(__NR_Linux +  91)
+#define __NR_truncate			(__NR_Linux +  92)
+#define __NR_ftruncate			(__NR_Linux +  93)
+#define __NR_fchmod			(__NR_Linux +  94)
+#define __NR_fchown			(__NR_Linux +  95)
+#define __NR_getpriority		(__NR_Linux +  96)
+#define __NR_setpriority		(__NR_Linux +  97)
+#define __NR_profil			(__NR_Linux +  98)
+#define __NR_statfs			(__NR_Linux +  99)
+#define __NR_fstatfs			(__NR_Linux + 100)
+#define __NR_ioperm			(__NR_Linux + 101)
+#define __NR_socketcall			(__NR_Linux + 102)
+#define __NR_syslog			(__NR_Linux + 103)
+#define __NR_setitimer			(__NR_Linux + 104)
+#define __NR_getitimer			(__NR_Linux + 105)
+#define __NR_stat			(__NR_Linux + 106)
+#define __NR_lstat			(__NR_Linux + 107)
+#define __NR_fstat			(__NR_Linux + 108)
+#define __NR_unused109			(__NR_Linux + 109)
+#define __NR_iopl			(__NR_Linux + 110)
+#define __NR_vhangup			(__NR_Linux + 111)
+#define __NR_idle			(__NR_Linux + 112)
+#define __NR_vm86			(__NR_Linux + 113)
+#define __NR_wait4			(__NR_Linux + 114)
+#define __NR_swapoff			(__NR_Linux + 115)
+#define __NR_sysinfo			(__NR_Linux + 116)
+#define __NR_ipc			(__NR_Linux + 117)
+#define __NR_fsync			(__NR_Linux + 118)
+#define __NR_sigreturn			(__NR_Linux + 119)
+#define __NR_clone			(__NR_Linux + 120)
+#define __NR_setdomainname		(__NR_Linux + 121)
+#define __NR_uname			(__NR_Linux + 122)
+#define __NR_modify_ldt			(__NR_Linux + 123)
+#define __NR_adjtimex			(__NR_Linux + 124)
+#define __NR_mprotect			(__NR_Linux + 125)
+#define __NR_sigprocmask		(__NR_Linux + 126)
+#define __NR_create_module		(__NR_Linux + 127)
+#define __NR_init_module		(__NR_Linux + 128)
+#define __NR_delete_module		(__NR_Linux + 129)
+#define __NR_get_kernel_syms		(__NR_Linux + 130)
+#define __NR_quotactl			(__NR_Linux + 131)
+#define __NR_getpgid			(__NR_Linux + 132)
+#define __NR_fchdir			(__NR_Linux + 133)
+#define __NR_bdflush			(__NR_Linux + 134)
+#define __NR_sysfs			(__NR_Linux + 135)
+#define __NR_personality		(__NR_Linux + 136)
+#define __NR_afs_syscall		(__NR_Linux + 137) /* Syscall for Andrew File System */
+#define __NR_setfsuid			(__NR_Linux + 138)
+#define __NR_setfsgid			(__NR_Linux + 139)
+#define __NR__llseek			(__NR_Linux + 140)
+#define __NR_getdents			(__NR_Linux + 141)
+#define __NR__newselect			(__NR_Linux + 142)
+#define __NR_flock			(__NR_Linux + 143)
+#define __NR_msync			(__NR_Linux + 144)
+#define __NR_readv			(__NR_Linux + 145)
+#define __NR_writev			(__NR_Linux + 146)
+#define __NR_cacheflush			(__NR_Linux + 147)
+#define __NR_cachectl			(__NR_Linux + 148)
+#define __NR_sysmips			(__NR_Linux + 149)
+#define __NR_unused150			(__NR_Linux + 150)
+#define __NR_getsid			(__NR_Linux + 151)
+#define __NR_fdatasync			(__NR_Linux + 152)
+#define __NR__sysctl			(__NR_Linux + 153)
+#define __NR_mlock			(__NR_Linux + 154)
+#define __NR_munlock			(__NR_Linux + 155)
+#define __NR_mlockall			(__NR_Linux + 156)
+#define __NR_munlockall			(__NR_Linux + 157)
+#define __NR_sched_setparam		(__NR_Linux + 158)
+#define __NR_sched_getparam		(__NR_Linux + 159)
+#define __NR_sched_setscheduler		(__NR_Linux + 160)
+#define __NR_sched_getscheduler		(__NR_Linux + 161)
+#define __NR_sched_yield		(__NR_Linux + 162)
+#define __NR_sched_get_priority_max	(__NR_Linux + 163)
+#define __NR_sched_get_priority_min	(__NR_Linux + 164)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 165)
+#define __NR_nanosleep			(__NR_Linux + 166)
+#define __NR_mremap			(__NR_Linux + 167)
+#define __NR_accept			(__NR_Linux + 168)
+#define __NR_bind			(__NR_Linux + 169)
+#define __NR_connect			(__NR_Linux + 170)
+#define __NR_getpeername		(__NR_Linux + 171)
+#define __NR_getsockname		(__NR_Linux + 172)
+#define __NR_getsockopt			(__NR_Linux + 173)
+#define __NR_listen			(__NR_Linux + 174)
+#define __NR_recv			(__NR_Linux + 175)
+#define __NR_recvfrom			(__NR_Linux + 176)
+#define __NR_recvmsg			(__NR_Linux + 177)
+#define __NR_send			(__NR_Linux + 178)
+#define __NR_sendmsg			(__NR_Linux + 179)
+#define __NR_sendto			(__NR_Linux + 180)
+#define __NR_setsockopt			(__NR_Linux + 181)
+#define __NR_shutdown			(__NR_Linux + 182)
+#define __NR_socket			(__NR_Linux + 183)
+#define __NR_socketpair			(__NR_Linux + 184)
+#define __NR_setresuid			(__NR_Linux + 185)
+#define __NR_getresuid			(__NR_Linux + 186)
+#define __NR_query_module		(__NR_Linux + 187)
+#define __NR_poll			(__NR_Linux + 188)
+#define __NR_nfsservctl			(__NR_Linux + 189)
+#define __NR_setresgid			(__NR_Linux + 190)
+#define __NR_getresgid			(__NR_Linux + 191)
+#define __NR_prctl			(__NR_Linux + 192)
+#define __NR_rt_sigreturn		(__NR_Linux + 193)
+#define __NR_rt_sigaction		(__NR_Linux + 194)
+#define __NR_rt_sigprocmask		(__NR_Linux + 195)
+#define __NR_rt_sigpending		(__NR_Linux + 196)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 197)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 198)
+#define __NR_rt_sigsuspend		(__NR_Linux + 199)
+#define __NR_pread64			(__NR_Linux + 200)
+#define __NR_pwrite64			(__NR_Linux + 201)
+#define __NR_chown			(__NR_Linux + 202)
+#define __NR_getcwd			(__NR_Linux + 203)
+#define __NR_capget			(__NR_Linux + 204)
+#define __NR_capset			(__NR_Linux + 205)
+#define __NR_sigaltstack		(__NR_Linux + 206)
+#define __NR_sendfile			(__NR_Linux + 207)
+#define __NR_getpmsg			(__NR_Linux + 208)
+#define __NR_putpmsg			(__NR_Linux + 209)
+#define __NR_mmap2			(__NR_Linux + 210)
+#define __NR_truncate64			(__NR_Linux + 211)
+#define __NR_ftruncate64		(__NR_Linux + 212)
+#define __NR_stat64			(__NR_Linux + 213)
+#define __NR_lstat64			(__NR_Linux + 214)
+#define __NR_fstat64			(__NR_Linux + 215)
+#define __NR_pivot_root			(__NR_Linux + 216)
+#define __NR_mincore			(__NR_Linux + 217)
+#define __NR_madvise			(__NR_Linux + 218)
+#define __NR_getdents64			(__NR_Linux + 219)
+#define __NR_fcntl64			(__NR_Linux + 220)
+#define __NR_reserved221		(__NR_Linux + 221)
+#define __NR_gettid			(__NR_Linux + 222)
+#define __NR_readahead			(__NR_Linux + 223)
+#define __NR_setxattr			(__NR_Linux + 224)
+#define __NR_lsetxattr			(__NR_Linux + 225)
+#define __NR_fsetxattr			(__NR_Linux + 226)
+#define __NR_getxattr			(__NR_Linux + 227)
+#define __NR_lgetxattr			(__NR_Linux + 228)
+#define __NR_fgetxattr			(__NR_Linux + 229)
+#define __NR_listxattr			(__NR_Linux + 230)
+#define __NR_llistxattr			(__NR_Linux + 231)
+#define __NR_flistxattr			(__NR_Linux + 232)
+#define __NR_removexattr		(__NR_Linux + 233)
+#define __NR_lremovexattr		(__NR_Linux + 234)
+#define __NR_fremovexattr		(__NR_Linux + 235)
+#define __NR_tkill			(__NR_Linux + 236)
+#define __NR_sendfile64			(__NR_Linux + 237)
+#define __NR_futex			(__NR_Linux + 238)
+#define __NR_sched_setaffinity		(__NR_Linux + 239)
+#define __NR_sched_getaffinity		(__NR_Linux + 240)
+#define __NR_io_setup			(__NR_Linux + 241)
+#define __NR_io_destroy			(__NR_Linux + 242)
+#define __NR_io_getevents		(__NR_Linux + 243)
+#define __NR_io_submit			(__NR_Linux + 244)
+#define __NR_io_cancel			(__NR_Linux + 245)
+#define __NR_exit_group			(__NR_Linux + 246)
+#define __NR_lookup_dcookie		(__NR_Linux + 247)
+#define __NR_epoll_create		(__NR_Linux + 248)
+#define __NR_epoll_ctl			(__NR_Linux + 249)
+#define __NR_epoll_wait			(__NR_Linux + 250)
+#define __NR_remap_file_pages		(__NR_Linux + 251)
+#define __NR_set_tid_address		(__NR_Linux + 252)
+#define __NR_restart_syscall		(__NR_Linux + 253)
+#define __NR_fadvise64			(__NR_Linux + 254)
+#define __NR_statfs64			(__NR_Linux + 255)
+#define __NR_fstatfs64			(__NR_Linux + 256)
+#define __NR_timer_create		(__NR_Linux + 257)
+#define __NR_timer_settime		(__NR_Linux + 258)
+#define __NR_timer_gettime		(__NR_Linux + 259)
+#define __NR_timer_getoverrun		(__NR_Linux + 260)
+#define __NR_timer_delete		(__NR_Linux + 261)
+#define __NR_clock_settime		(__NR_Linux + 262)
+#define __NR_clock_gettime		(__NR_Linux + 263)
+#define __NR_clock_getres		(__NR_Linux + 264)
+#define __NR_clock_nanosleep		(__NR_Linux + 265)
+#define __NR_tgkill			(__NR_Linux + 266)
+#define __NR_utimes			(__NR_Linux + 267)
+#define __NR_mbind			(__NR_Linux + 268)
+#define __NR_get_mempolicy		(__NR_Linux + 269)
+#define __NR_set_mempolicy		(__NR_Linux + 270)
+#define __NR_mq_open			(__NR_Linux + 271)
+#define __NR_mq_unlink			(__NR_Linux + 272)
+#define __NR_mq_timedsend		(__NR_Linux + 273)
+#define __NR_mq_timedreceive		(__NR_Linux + 274)
+#define __NR_mq_notify			(__NR_Linux + 275)
+#define __NR_mq_getsetattr		(__NR_Linux + 276)
+#define __NR_vserver			(__NR_Linux + 277)
+#define __NR_waitid			(__NR_Linux + 278)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 279) */
+#define __NR_add_key			(__NR_Linux + 280)
+#define __NR_request_key		(__NR_Linux + 281)
+#define __NR_keyctl			(__NR_Linux + 282)
+#define __NR_set_thread_area		(__NR_Linux + 283)
+#define __NR_inotify_init		(__NR_Linux + 284)
+#define __NR_inotify_add_watch		(__NR_Linux + 285)
+#define __NR_inotify_rm_watch		(__NR_Linux + 286)
+#define __NR_migrate_pages		(__NR_Linux + 287)
+#define __NR_openat			(__NR_Linux + 288)
+#define __NR_mkdirat			(__NR_Linux + 289)
+#define __NR_mknodat			(__NR_Linux + 290)
+#define __NR_fchownat			(__NR_Linux + 291)
+#define __NR_futimesat			(__NR_Linux + 292)
+#define __NR_fstatat64			(__NR_Linux + 293)
+#define __NR_unlinkat			(__NR_Linux + 294)
+#define __NR_renameat			(__NR_Linux + 295)
+#define __NR_linkat			(__NR_Linux + 296)
+#define __NR_symlinkat			(__NR_Linux + 297)
+#define __NR_readlinkat			(__NR_Linux + 298)
+#define __NR_fchmodat			(__NR_Linux + 299)
+#define __NR_faccessat			(__NR_Linux + 300)
+#define __NR_pselect6			(__NR_Linux + 301)
+#define __NR_ppoll			(__NR_Linux + 302)
+#define __NR_unshare			(__NR_Linux + 303)
+#define __NR_splice			(__NR_Linux + 304)
+#define __NR_sync_file_range		(__NR_Linux + 305)
+#define __NR_tee			(__NR_Linux + 306)
+#define __NR_vmsplice			(__NR_Linux + 307)
+#define __NR_move_pages			(__NR_Linux + 308)
+#define __NR_set_robust_list		(__NR_Linux + 309)
+#define __NR_get_robust_list		(__NR_Linux + 310)
+#define __NR_kexec_load			(__NR_Linux + 311)
+#define __NR_getcpu			(__NR_Linux + 312)
+#define __NR_epoll_pwait		(__NR_Linux + 313)
+#define __NR_ioprio_set			(__NR_Linux + 314)
+#define __NR_ioprio_get			(__NR_Linux + 315)
+#define __NR_utimensat			(__NR_Linux + 316)
+#define __NR_signalfd			(__NR_Linux + 317)
+#define __NR_timerfd			(__NR_Linux + 318)
+#define __NR_eventfd			(__NR_Linux + 319)
+#define __NR_fallocate			(__NR_Linux + 320)
+#define __NR_timerfd_create		(__NR_Linux + 321)
+#define __NR_timerfd_gettime		(__NR_Linux + 322)
+#define __NR_timerfd_settime		(__NR_Linux + 323)
+#define __NR_signalfd4			(__NR_Linux + 324)
+#define __NR_eventfd2			(__NR_Linux + 325)
+#define __NR_epoll_create1		(__NR_Linux + 326)
+#define __NR_dup3			(__NR_Linux + 327)
+#define __NR_pipe2			(__NR_Linux + 328)
+#define __NR_inotify_init1		(__NR_Linux + 329)
+#define __NR_preadv			(__NR_Linux + 330)
+#define __NR_pwritev			(__NR_Linux + 331)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 332)
+#define __NR_perf_event_open		(__NR_Linux + 333)
+#define __NR_accept4			(__NR_Linux + 334)
+#define __NR_recvmmsg			(__NR_Linux + 335)
+#define __NR_fanotify_init		(__NR_Linux + 336)
+#define __NR_fanotify_mark		(__NR_Linux + 337)
+#define __NR_prlimit64			(__NR_Linux + 338)
+#define __NR_name_to_handle_at		(__NR_Linux + 339)
+#define __NR_open_by_handle_at		(__NR_Linux + 340)
+#define __NR_clock_adjtime		(__NR_Linux + 341)
+#define __NR_syncfs			(__NR_Linux + 342)
+#define __NR_sendmmsg			(__NR_Linux + 343)
+#define __NR_setns			(__NR_Linux + 344)
+#define __NR_process_vm_readv		(__NR_Linux + 345)
+#define __NR_process_vm_writev		(__NR_Linux + 346)
+#define __NR_kcmp			(__NR_Linux + 347)
+#define __NR_finit_module		(__NR_Linux + 348)
+#define __NR_sched_setattr		(__NR_Linux + 349)
+#define __NR_sched_getattr		(__NR_Linux + 350)
+#define __NR_renameat2			(__NR_Linux + 351)
+#define __NR_seccomp			(__NR_Linux + 352)
+#define __NR_getrandom			(__NR_Linux + 353)
+#define __NR_memfd_create		(__NR_Linux + 354)
+#define __NR_bpf			(__NR_Linux + 355)
+#define __NR_execveat			(__NR_Linux + 356)
+
+/*
+ * Offset of the last Linux o32 flavoured syscall
+ */
+#define __NR_Linux_syscalls		356
+
+#endif /* _MIPS_SIM == _MIPS_SIM_ABI32 */
+
+#define __NR_O32_Linux			4000
+#define __NR_O32_Linux_syscalls		356
+
+#if _MIPS_SIM == _MIPS_SIM_ABI64
+
+/*
+ * Linux 64-bit syscalls are in the range from 5000 to 5999.
+ */
+#define __NR_Linux			5000
+#define __NR_read			(__NR_Linux +	0)
+#define __NR_write			(__NR_Linux +	1)
+#define __NR_open			(__NR_Linux +	2)
+#define __NR_close			(__NR_Linux +	3)
+#define __NR_stat			(__NR_Linux +	4)
+#define __NR_fstat			(__NR_Linux +	5)
+#define __NR_lstat			(__NR_Linux +	6)
+#define __NR_poll			(__NR_Linux +	7)
+#define __NR_lseek			(__NR_Linux +	8)
+#define __NR_mmap			(__NR_Linux +	9)
+#define __NR_mprotect			(__NR_Linux +  10)
+#define __NR_munmap			(__NR_Linux +  11)
+#define __NR_brk			(__NR_Linux +  12)
+#define __NR_rt_sigaction		(__NR_Linux +  13)
+#define __NR_rt_sigprocmask		(__NR_Linux +  14)
+#define __NR_ioctl			(__NR_Linux +  15)
+#define __NR_pread64			(__NR_Linux +  16)
+#define __NR_pwrite64			(__NR_Linux +  17)
+#define __NR_readv			(__NR_Linux +  18)
+#define __NR_writev			(__NR_Linux +  19)
+#define __NR_access			(__NR_Linux +  20)
+#define __NR_pipe			(__NR_Linux +  21)
+#define __NR__newselect			(__NR_Linux +  22)
+#define __NR_sched_yield		(__NR_Linux +  23)
+#define __NR_mremap			(__NR_Linux +  24)
+#define __NR_msync			(__NR_Linux +  25)
+#define __NR_mincore			(__NR_Linux +  26)
+#define __NR_madvise			(__NR_Linux +  27)
+#define __NR_shmget			(__NR_Linux +  28)
+#define __NR_shmat			(__NR_Linux +  29)
+#define __NR_shmctl			(__NR_Linux +  30)
+#define __NR_dup			(__NR_Linux +  31)
+#define __NR_dup2			(__NR_Linux +  32)
+#define __NR_pause			(__NR_Linux +  33)
+#define __NR_nanosleep			(__NR_Linux +  34)
+#define __NR_getitimer			(__NR_Linux +  35)
+#define __NR_setitimer			(__NR_Linux +  36)
+#define __NR_alarm			(__NR_Linux +  37)
+#define __NR_getpid			(__NR_Linux +  38)
+#define __NR_sendfile			(__NR_Linux +  39)
+#define __NR_socket			(__NR_Linux +  40)
+#define __NR_connect			(__NR_Linux +  41)
+#define __NR_accept			(__NR_Linux +  42)
+#define __NR_sendto			(__NR_Linux +  43)
+#define __NR_recvfrom			(__NR_Linux +  44)
+#define __NR_sendmsg			(__NR_Linux +  45)
+#define __NR_recvmsg			(__NR_Linux +  46)
+#define __NR_shutdown			(__NR_Linux +  47)
+#define __NR_bind			(__NR_Linux +  48)
+#define __NR_listen			(__NR_Linux +  49)
+#define __NR_getsockname		(__NR_Linux +  50)
+#define __NR_getpeername		(__NR_Linux +  51)
+#define __NR_socketpair			(__NR_Linux +  52)
+#define __NR_setsockopt			(__NR_Linux +  53)
+#define __NR_getsockopt			(__NR_Linux +  54)
+#define __NR_clone			(__NR_Linux +  55)
+#define __NR_fork			(__NR_Linux +  56)
+#define __NR_execve			(__NR_Linux +  57)
+#define __NR_exit			(__NR_Linux +  58)
+#define __NR_wait4			(__NR_Linux +  59)
+#define __NR_kill			(__NR_Linux +  60)
+#define __NR_uname			(__NR_Linux +  61)
+#define __NR_semget			(__NR_Linux +  62)
+#define __NR_semop			(__NR_Linux +  63)
+#define __NR_semctl			(__NR_Linux +  64)
+#define __NR_shmdt			(__NR_Linux +  65)
+#define __NR_msgget			(__NR_Linux +  66)
+#define __NR_msgsnd			(__NR_Linux +  67)
+#define __NR_msgrcv			(__NR_Linux +  68)
+#define __NR_msgctl			(__NR_Linux +  69)
+#define __NR_fcntl			(__NR_Linux +  70)
+#define __NR_flock			(__NR_Linux +  71)
+#define __NR_fsync			(__NR_Linux +  72)
+#define __NR_fdatasync			(__NR_Linux +  73)
+#define __NR_truncate			(__NR_Linux +  74)
+#define __NR_ftruncate			(__NR_Linux +  75)
+#define __NR_getdents			(__NR_Linux +  76)
+#define __NR_getcwd			(__NR_Linux +  77)
+#define __NR_chdir			(__NR_Linux +  78)
+#define __NR_fchdir			(__NR_Linux +  79)
+#define __NR_rename			(__NR_Linux +  80)
+#define __NR_mkdir			(__NR_Linux +  81)
+#define __NR_rmdir			(__NR_Linux +  82)
+#define __NR_creat			(__NR_Linux +  83)
+#define __NR_link			(__NR_Linux +  84)
+#define __NR_unlink			(__NR_Linux +  85)
+#define __NR_symlink			(__NR_Linux +  86)
+#define __NR_readlink			(__NR_Linux +  87)
+#define __NR_chmod			(__NR_Linux +  88)
+#define __NR_fchmod			(__NR_Linux +  89)
+#define __NR_chown			(__NR_Linux +  90)
+#define __NR_fchown			(__NR_Linux +  91)
+#define __NR_lchown			(__NR_Linux +  92)
+#define __NR_umask			(__NR_Linux +  93)
+#define __NR_gettimeofday		(__NR_Linux +  94)
+#define __NR_getrlimit			(__NR_Linux +  95)
+#define __NR_getrusage			(__NR_Linux +  96)
+#define __NR_sysinfo			(__NR_Linux +  97)
+#define __NR_times			(__NR_Linux +  98)
+#define __NR_ptrace			(__NR_Linux +  99)
+#define __NR_getuid			(__NR_Linux + 100)
+#define __NR_syslog			(__NR_Linux + 101)
+#define __NR_getgid			(__NR_Linux + 102)
+#define __NR_setuid			(__NR_Linux + 103)
+#define __NR_setgid			(__NR_Linux + 104)
+#define __NR_geteuid			(__NR_Linux + 105)
+#define __NR_getegid			(__NR_Linux + 106)
+#define __NR_setpgid			(__NR_Linux + 107)
+#define __NR_getppid			(__NR_Linux + 108)
+#define __NR_getpgrp			(__NR_Linux + 109)
+#define __NR_setsid			(__NR_Linux + 110)
+#define __NR_setreuid			(__NR_Linux + 111)
+#define __NR_setregid			(__NR_Linux + 112)
+#define __NR_getgroups			(__NR_Linux + 113)
+#define __NR_setgroups			(__NR_Linux + 114)
+#define __NR_setresuid			(__NR_Linux + 115)
+#define __NR_getresuid			(__NR_Linux + 116)
+#define __NR_setresgid			(__NR_Linux + 117)
+#define __NR_getresgid			(__NR_Linux + 118)
+#define __NR_getpgid			(__NR_Linux + 119)
+#define __NR_setfsuid			(__NR_Linux + 120)
+#define __NR_setfsgid			(__NR_Linux + 121)
+#define __NR_getsid			(__NR_Linux + 122)
+#define __NR_capget			(__NR_Linux + 123)
+#define __NR_capset			(__NR_Linux + 124)
+#define __NR_rt_sigpending		(__NR_Linux + 125)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 126)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 127)
+#define __NR_rt_sigsuspend		(__NR_Linux + 128)
+#define __NR_sigaltstack		(__NR_Linux + 129)
+#define __NR_utime			(__NR_Linux + 130)
+#define __NR_mknod			(__NR_Linux + 131)
+#define __NR_personality		(__NR_Linux + 132)
+#define __NR_ustat			(__NR_Linux + 133)
+#define __NR_statfs			(__NR_Linux + 134)
+#define __NR_fstatfs			(__NR_Linux + 135)
+#define __NR_sysfs			(__NR_Linux + 136)
+#define __NR_getpriority		(__NR_Linux + 137)
+#define __NR_setpriority		(__NR_Linux + 138)
+#define __NR_sched_setparam		(__NR_Linux + 139)
+#define __NR_sched_getparam		(__NR_Linux + 140)
+#define __NR_sched_setscheduler		(__NR_Linux + 141)
+#define __NR_sched_getscheduler		(__NR_Linux + 142)
+#define __NR_sched_get_priority_max	(__NR_Linux + 143)
+#define __NR_sched_get_priority_min	(__NR_Linux + 144)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 145)
+#define __NR_mlock			(__NR_Linux + 146)
+#define __NR_munlock			(__NR_Linux + 147)
+#define __NR_mlockall			(__NR_Linux + 148)
+#define __NR_munlockall			(__NR_Linux + 149)
+#define __NR_vhangup			(__NR_Linux + 150)
+#define __NR_pivot_root			(__NR_Linux + 151)
+#define __NR__sysctl			(__NR_Linux + 152)
+#define __NR_prctl			(__NR_Linux + 153)
+#define __NR_adjtimex			(__NR_Linux + 154)
+#define __NR_setrlimit			(__NR_Linux + 155)
+#define __NR_chroot			(__NR_Linux + 156)
+#define __NR_sync			(__NR_Linux + 157)
+#define __NR_acct			(__NR_Linux + 158)
+#define __NR_settimeofday		(__NR_Linux + 159)
+#define __NR_mount			(__NR_Linux + 160)
+#define __NR_umount2			(__NR_Linux + 161)
+#define __NR_swapon			(__NR_Linux + 162)
+#define __NR_swapoff			(__NR_Linux + 163)
+#define __NR_reboot			(__NR_Linux + 164)
+#define __NR_sethostname		(__NR_Linux + 165)
+#define __NR_setdomainname		(__NR_Linux + 166)
+#define __NR_create_module		(__NR_Linux + 167)
+#define __NR_init_module		(__NR_Linux + 168)
+#define __NR_delete_module		(__NR_Linux + 169)
+#define __NR_get_kernel_syms		(__NR_Linux + 170)
+#define __NR_query_module		(__NR_Linux + 171)
+#define __NR_quotactl			(__NR_Linux + 172)
+#define __NR_nfsservctl			(__NR_Linux + 173)
+#define __NR_getpmsg			(__NR_Linux + 174)
+#define __NR_putpmsg			(__NR_Linux + 175)
+#define __NR_afs_syscall		(__NR_Linux + 176)
+#define __NR_reserved177		(__NR_Linux + 177)
+#define __NR_gettid			(__NR_Linux + 178)
+#define __NR_readahead			(__NR_Linux + 179)
+#define __NR_setxattr			(__NR_Linux + 180)
+#define __NR_lsetxattr			(__NR_Linux + 181)
+#define __NR_fsetxattr			(__NR_Linux + 182)
+#define __NR_getxattr			(__NR_Linux + 183)
+#define __NR_lgetxattr			(__NR_Linux + 184)
+#define __NR_fgetxattr			(__NR_Linux + 185)
+#define __NR_listxattr			(__NR_Linux + 186)
+#define __NR_llistxattr			(__NR_Linux + 187)
+#define __NR_flistxattr			(__NR_Linux + 188)
+#define __NR_removexattr		(__NR_Linux + 189)
+#define __NR_lremovexattr		(__NR_Linux + 190)
+#define __NR_fremovexattr		(__NR_Linux + 191)
+#define __NR_tkill			(__NR_Linux + 192)
+#define __NR_reserved193		(__NR_Linux + 193)
+#define __NR_futex			(__NR_Linux + 194)
+#define __NR_sched_setaffinity		(__NR_Linux + 195)
+#define __NR_sched_getaffinity		(__NR_Linux + 196)
+#define __NR_cacheflush			(__NR_Linux + 197)
+#define __NR_cachectl			(__NR_Linux + 198)
+#define __NR_sysmips			(__NR_Linux + 199)
+#define __NR_io_setup			(__NR_Linux + 200)
+#define __NR_io_destroy			(__NR_Linux + 201)
+#define __NR_io_getevents		(__NR_Linux + 202)
+#define __NR_io_submit			(__NR_Linux + 203)
+#define __NR_io_cancel			(__NR_Linux + 204)
+#define __NR_exit_group			(__NR_Linux + 205)
+#define __NR_lookup_dcookie		(__NR_Linux + 206)
+#define __NR_epoll_create		(__NR_Linux + 207)
+#define __NR_epoll_ctl			(__NR_Linux + 208)
+#define __NR_epoll_wait			(__NR_Linux + 209)
+#define __NR_remap_file_pages		(__NR_Linux + 210)
+#define __NR_rt_sigreturn		(__NR_Linux + 211)
+#define __NR_set_tid_address		(__NR_Linux + 212)
+#define __NR_restart_syscall		(__NR_Linux + 213)
+#define __NR_semtimedop			(__NR_Linux + 214)
+#define __NR_fadvise64			(__NR_Linux + 215)
+#define __NR_timer_create		(__NR_Linux + 216)
+#define __NR_timer_settime		(__NR_Linux + 217)
+#define __NR_timer_gettime		(__NR_Linux + 218)
+#define __NR_timer_getoverrun		(__NR_Linux + 219)
+#define __NR_timer_delete		(__NR_Linux + 220)
+#define __NR_clock_settime		(__NR_Linux + 221)
+#define __NR_clock_gettime		(__NR_Linux + 222)
+#define __NR_clock_getres		(__NR_Linux + 223)
+#define __NR_clock_nanosleep		(__NR_Linux + 224)
+#define __NR_tgkill			(__NR_Linux + 225)
+#define __NR_utimes			(__NR_Linux + 226)
+#define __NR_mbind			(__NR_Linux + 227)
+#define __NR_get_mempolicy		(__NR_Linux + 228)
+#define __NR_set_mempolicy		(__NR_Linux + 229)
+#define __NR_mq_open			(__NR_Linux + 230)
+#define __NR_mq_unlink			(__NR_Linux + 231)
+#define __NR_mq_timedsend		(__NR_Linux + 232)
+#define __NR_mq_timedreceive		(__NR_Linux + 233)
+#define __NR_mq_notify			(__NR_Linux + 234)
+#define __NR_mq_getsetattr		(__NR_Linux + 235)
+#define __NR_vserver			(__NR_Linux + 236)
+#define __NR_waitid			(__NR_Linux + 237)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 238) */
+#define __NR_add_key			(__NR_Linux + 239)
+#define __NR_request_key		(__NR_Linux + 240)
+#define __NR_keyctl			(__NR_Linux + 241)
+#define __NR_set_thread_area		(__NR_Linux + 242)
+#define __NR_inotify_init		(__NR_Linux + 243)
+#define __NR_inotify_add_watch		(__NR_Linux + 244)
+#define __NR_inotify_rm_watch		(__NR_Linux + 245)
+#define __NR_migrate_pages		(__NR_Linux + 246)
+#define __NR_openat			(__NR_Linux + 247)
+#define __NR_mkdirat			(__NR_Linux + 248)
+#define __NR_mknodat			(__NR_Linux + 249)
+#define __NR_fchownat			(__NR_Linux + 250)
+#define __NR_futimesat			(__NR_Linux + 251)
+#define __NR_newfstatat			(__NR_Linux + 252)
+#define __NR_unlinkat			(__NR_Linux + 253)
+#define __NR_renameat			(__NR_Linux + 254)
+#define __NR_linkat			(__NR_Linux + 255)
+#define __NR_symlinkat			(__NR_Linux + 256)
+#define __NR_readlinkat			(__NR_Linux + 257)
+#define __NR_fchmodat			(__NR_Linux + 258)
+#define __NR_faccessat			(__NR_Linux + 259)
+#define __NR_pselect6			(__NR_Linux + 260)
+#define __NR_ppoll			(__NR_Linux + 261)
+#define __NR_unshare			(__NR_Linux + 262)
+#define __NR_splice			(__NR_Linux + 263)
+#define __NR_sync_file_range		(__NR_Linux + 264)
+#define __NR_tee			(__NR_Linux + 265)
+#define __NR_vmsplice			(__NR_Linux + 266)
+#define __NR_move_pages			(__NR_Linux + 267)
+#define __NR_set_robust_list		(__NR_Linux + 268)
+#define __NR_get_robust_list		(__NR_Linux + 269)
+#define __NR_kexec_load			(__NR_Linux + 270)
+#define __NR_getcpu			(__NR_Linux + 271)
+#define __NR_epoll_pwait		(__NR_Linux + 272)
+#define __NR_ioprio_set			(__NR_Linux + 273)
+#define __NR_ioprio_get			(__NR_Linux + 274)
+#define __NR_utimensat			(__NR_Linux + 275)
+#define __NR_signalfd			(__NR_Linux + 276)
+#define __NR_timerfd			(__NR_Linux + 277)
+#define __NR_eventfd			(__NR_Linux + 278)
+#define __NR_fallocate			(__NR_Linux + 279)
+#define __NR_timerfd_create		(__NR_Linux + 280)
+#define __NR_timerfd_gettime		(__NR_Linux + 281)
+#define __NR_timerfd_settime		(__NR_Linux + 282)
+#define __NR_signalfd4			(__NR_Linux + 283)
+#define __NR_eventfd2			(__NR_Linux + 284)
+#define __NR_epoll_create1		(__NR_Linux + 285)
+#define __NR_dup3			(__NR_Linux + 286)
+#define __NR_pipe2			(__NR_Linux + 287)
+#define __NR_inotify_init1		(__NR_Linux + 288)
+#define __NR_preadv			(__NR_Linux + 289)
+#define __NR_pwritev			(__NR_Linux + 290)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 291)
+#define __NR_perf_event_open		(__NR_Linux + 292)
+#define __NR_accept4			(__NR_Linux + 293)
+#define __NR_recvmmsg			(__NR_Linux + 294)
+#define __NR_fanotify_init		(__NR_Linux + 295)
+#define __NR_fanotify_mark		(__NR_Linux + 296)
+#define __NR_prlimit64			(__NR_Linux + 297)
+#define __NR_name_to_handle_at		(__NR_Linux + 298)
+#define __NR_open_by_handle_at		(__NR_Linux + 299)
+#define __NR_clock_adjtime		(__NR_Linux + 300)
+#define __NR_syncfs			(__NR_Linux + 301)
+#define __NR_sendmmsg			(__NR_Linux + 302)
+#define __NR_setns			(__NR_Linux + 303)
+#define __NR_process_vm_readv		(__NR_Linux + 304)
+#define __NR_process_vm_writev		(__NR_Linux + 305)
+#define __NR_kcmp			(__NR_Linux + 306)
+#define __NR_finit_module		(__NR_Linux + 307)
+#define __NR_getdents64			(__NR_Linux + 308)
+#define __NR_sched_setattr		(__NR_Linux + 309)
+#define __NR_sched_getattr		(__NR_Linux + 310)
+#define __NR_renameat2			(__NR_Linux + 311)
+#define __NR_seccomp			(__NR_Linux + 312)
+#define __NR_getrandom			(__NR_Linux + 313)
+#define __NR_memfd_create		(__NR_Linux + 314)
+#define __NR_bpf			(__NR_Linux + 315)
+#define __NR_execveat			(__NR_Linux + 316)
+
+/*
+ * Offset of the last Linux 64-bit flavoured syscall
+ */
+#define __NR_Linux_syscalls		316
+
+#endif /* _MIPS_SIM == _MIPS_SIM_ABI64 */
+
+#define __NR_64_Linux			5000
+#define __NR_64_Linux_syscalls		316
+
+#if _MIPS_SIM == _MIPS_SIM_NABI32
+
+/*
+ * Linux N32 syscalls are in the range from 6000 to 6999.
+ */
+#define __NR_Linux			6000
+#define __NR_read			(__NR_Linux +	0)
+#define __NR_write			(__NR_Linux +	1)
+#define __NR_open			(__NR_Linux +	2)
+#define __NR_close			(__NR_Linux +	3)
+#define __NR_stat			(__NR_Linux +	4)
+#define __NR_fstat			(__NR_Linux +	5)
+#define __NR_lstat			(__NR_Linux +	6)
+#define __NR_poll			(__NR_Linux +	7)
+#define __NR_lseek			(__NR_Linux +	8)
+#define __NR_mmap			(__NR_Linux +	9)
+#define __NR_mprotect			(__NR_Linux +  10)
+#define __NR_munmap			(__NR_Linux +  11)
+#define __NR_brk			(__NR_Linux +  12)
+#define __NR_rt_sigaction		(__NR_Linux +  13)
+#define __NR_rt_sigprocmask		(__NR_Linux +  14)
+#define __NR_ioctl			(__NR_Linux +  15)
+#define __NR_pread64			(__NR_Linux +  16)
+#define __NR_pwrite64			(__NR_Linux +  17)
+#define __NR_readv			(__NR_Linux +  18)
+#define __NR_writev			(__NR_Linux +  19)
+#define __NR_access			(__NR_Linux +  20)
+#define __NR_pipe			(__NR_Linux +  21)
+#define __NR__newselect			(__NR_Linux +  22)
+#define __NR_sched_yield		(__NR_Linux +  23)
+#define __NR_mremap			(__NR_Linux +  24)
+#define __NR_msync			(__NR_Linux +  25)
+#define __NR_mincore			(__NR_Linux +  26)
+#define __NR_madvise			(__NR_Linux +  27)
+#define __NR_shmget			(__NR_Linux +  28)
+#define __NR_shmat			(__NR_Linux +  29)
+#define __NR_shmctl			(__NR_Linux +  30)
+#define __NR_dup			(__NR_Linux +  31)
+#define __NR_dup2			(__NR_Linux +  32)
+#define __NR_pause			(__NR_Linux +  33)
+#define __NR_nanosleep			(__NR_Linux +  34)
+#define __NR_getitimer			(__NR_Linux +  35)
+#define __NR_setitimer			(__NR_Linux +  36)
+#define __NR_alarm			(__NR_Linux +  37)
+#define __NR_getpid			(__NR_Linux +  38)
+#define __NR_sendfile			(__NR_Linux +  39)
+#define __NR_socket			(__NR_Linux +  40)
+#define __NR_connect			(__NR_Linux +  41)
+#define __NR_accept			(__NR_Linux +  42)
+#define __NR_sendto			(__NR_Linux +  43)
+#define __NR_recvfrom			(__NR_Linux +  44)
+#define __NR_sendmsg			(__NR_Linux +  45)
+#define __NR_recvmsg			(__NR_Linux +  46)
+#define __NR_shutdown			(__NR_Linux +  47)
+#define __NR_bind			(__NR_Linux +  48)
+#define __NR_listen			(__NR_Linux +  49)
+#define __NR_getsockname		(__NR_Linux +  50)
+#define __NR_getpeername		(__NR_Linux +  51)
+#define __NR_socketpair			(__NR_Linux +  52)
+#define __NR_setsockopt			(__NR_Linux +  53)
+#define __NR_getsockopt			(__NR_Linux +  54)
+#define __NR_clone			(__NR_Linux +  55)
+#define __NR_fork			(__NR_Linux +  56)
+#define __NR_execve			(__NR_Linux +  57)
+#define __NR_exit			(__NR_Linux +  58)
+#define __NR_wait4			(__NR_Linux +  59)
+#define __NR_kill			(__NR_Linux +  60)
+#define __NR_uname			(__NR_Linux +  61)
+#define __NR_semget			(__NR_Linux +  62)
+#define __NR_semop			(__NR_Linux +  63)
+#define __NR_semctl			(__NR_Linux +  64)
+#define __NR_shmdt			(__NR_Linux +  65)
+#define __NR_msgget			(__NR_Linux +  66)
+#define __NR_msgsnd			(__NR_Linux +  67)
+#define __NR_msgrcv			(__NR_Linux +  68)
+#define __NR_msgctl			(__NR_Linux +  69)
+#define __NR_fcntl			(__NR_Linux +  70)
+#define __NR_flock			(__NR_Linux +  71)
+#define __NR_fsync			(__NR_Linux +  72)
+#define __NR_fdatasync			(__NR_Linux +  73)
+#define __NR_truncate			(__NR_Linux +  74)
+#define __NR_ftruncate			(__NR_Linux +  75)
+#define __NR_getdents			(__NR_Linux +  76)
+#define __NR_getcwd			(__NR_Linux +  77)
+#define __NR_chdir			(__NR_Linux +  78)
+#define __NR_fchdir			(__NR_Linux +  79)
+#define __NR_rename			(__NR_Linux +  80)
+#define __NR_mkdir			(__NR_Linux +  81)
+#define __NR_rmdir			(__NR_Linux +  82)
+#define __NR_creat			(__NR_Linux +  83)
+#define __NR_link			(__NR_Linux +  84)
+#define __NR_unlink			(__NR_Linux +  85)
+#define __NR_symlink			(__NR_Linux +  86)
+#define __NR_readlink			(__NR_Linux +  87)
+#define __NR_chmod			(__NR_Linux +  88)
+#define __NR_fchmod			(__NR_Linux +  89)
+#define __NR_chown			(__NR_Linux +  90)
+#define __NR_fchown			(__NR_Linux +  91)
+#define __NR_lchown			(__NR_Linux +  92)
+#define __NR_umask			(__NR_Linux +  93)
+#define __NR_gettimeofday		(__NR_Linux +  94)
+#define __NR_getrlimit			(__NR_Linux +  95)
+#define __NR_getrusage			(__NR_Linux +  96)
+#define __NR_sysinfo			(__NR_Linux +  97)
+#define __NR_times			(__NR_Linux +  98)
+#define __NR_ptrace			(__NR_Linux +  99)
+#define __NR_getuid			(__NR_Linux + 100)
+#define __NR_syslog			(__NR_Linux + 101)
+#define __NR_getgid			(__NR_Linux + 102)
+#define __NR_setuid			(__NR_Linux + 103)
+#define __NR_setgid			(__NR_Linux + 104)
+#define __NR_geteuid			(__NR_Linux + 105)
+#define __NR_getegid			(__NR_Linux + 106)
+#define __NR_setpgid			(__NR_Linux + 107)
+#define __NR_getppid			(__NR_Linux + 108)
+#define __NR_getpgrp			(__NR_Linux + 109)
+#define __NR_setsid			(__NR_Linux + 110)
+#define __NR_setreuid			(__NR_Linux + 111)
+#define __NR_setregid			(__NR_Linux + 112)
+#define __NR_getgroups			(__NR_Linux + 113)
+#define __NR_setgroups			(__NR_Linux + 114)
+#define __NR_setresuid			(__NR_Linux + 115)
+#define __NR_getresuid			(__NR_Linux + 116)
+#define __NR_setresgid			(__NR_Linux + 117)
+#define __NR_getresgid			(__NR_Linux + 118)
+#define __NR_getpgid			(__NR_Linux + 119)
+#define __NR_setfsuid			(__NR_Linux + 120)
+#define __NR_setfsgid			(__NR_Linux + 121)
+#define __NR_getsid			(__NR_Linux + 122)
+#define __NR_capget			(__NR_Linux + 123)
+#define __NR_capset			(__NR_Linux + 124)
+#define __NR_rt_sigpending		(__NR_Linux + 125)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 126)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 127)
+#define __NR_rt_sigsuspend		(__NR_Linux + 128)
+#define __NR_sigaltstack		(__NR_Linux + 129)
+#define __NR_utime			(__NR_Linux + 130)
+#define __NR_mknod			(__NR_Linux + 131)
+#define __NR_personality		(__NR_Linux + 132)
+#define __NR_ustat			(__NR_Linux + 133)
+#define __NR_statfs			(__NR_Linux + 134)
+#define __NR_fstatfs			(__NR_Linux + 135)
+#define __NR_sysfs			(__NR_Linux + 136)
+#define __NR_getpriority		(__NR_Linux + 137)
+#define __NR_setpriority		(__NR_Linux + 138)
+#define __NR_sched_setparam		(__NR_Linux + 139)
+#define __NR_sched_getparam		(__NR_Linux + 140)
+#define __NR_sched_setscheduler		(__NR_Linux + 141)
+#define __NR_sched_getscheduler		(__NR_Linux + 142)
+#define __NR_sched_get_priority_max	(__NR_Linux + 143)
+#define __NR_sched_get_priority_min	(__NR_Linux + 144)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 145)
+#define __NR_mlock			(__NR_Linux + 146)
+#define __NR_munlock			(__NR_Linux + 147)
+#define __NR_mlockall			(__NR_Linux + 148)
+#define __NR_munlockall			(__NR_Linux + 149)
+#define __NR_vhangup			(__NR_Linux + 150)
+#define __NR_pivot_root			(__NR_Linux + 151)
+#define __NR__sysctl			(__NR_Linux + 152)
+#define __NR_prctl			(__NR_Linux + 153)
+#define __NR_adjtimex			(__NR_Linux + 154)
+#define __NR_setrlimit			(__NR_Linux + 155)
+#define __NR_chroot			(__NR_Linux + 156)
+#define __NR_sync			(__NR_Linux + 157)
+#define __NR_acct			(__NR_Linux + 158)
+#define __NR_settimeofday		(__NR_Linux + 159)
+#define __NR_mount			(__NR_Linux + 160)
+#define __NR_umount2			(__NR_Linux + 161)
+#define __NR_swapon			(__NR_Linux + 162)
+#define __NR_swapoff			(__NR_Linux + 163)
+#define __NR_reboot			(__NR_Linux + 164)
+#define __NR_sethostname		(__NR_Linux + 165)
+#define __NR_setdomainname		(__NR_Linux + 166)
+#define __NR_create_module		(__NR_Linux + 167)
+#define __NR_init_module		(__NR_Linux + 168)
+#define __NR_delete_module		(__NR_Linux + 169)
+#define __NR_get_kernel_syms		(__NR_Linux + 170)
+#define __NR_query_module		(__NR_Linux + 171)
+#define __NR_quotactl			(__NR_Linux + 172)
+#define __NR_nfsservctl			(__NR_Linux + 173)
+#define __NR_getpmsg			(__NR_Linux + 174)
+#define __NR_putpmsg			(__NR_Linux + 175)
+#define __NR_afs_syscall		(__NR_Linux + 176)
+#define __NR_reserved177		(__NR_Linux + 177)
+#define __NR_gettid			(__NR_Linux + 178)
+#define __NR_readahead			(__NR_Linux + 179)
+#define __NR_setxattr			(__NR_Linux + 180)
+#define __NR_lsetxattr			(__NR_Linux + 181)
+#define __NR_fsetxattr			(__NR_Linux + 182)
+#define __NR_getxattr			(__NR_Linux + 183)
+#define __NR_lgetxattr			(__NR_Linux + 184)
+#define __NR_fgetxattr			(__NR_Linux + 185)
+#define __NR_listxattr			(__NR_Linux + 186)
+#define __NR_llistxattr			(__NR_Linux + 187)
+#define __NR_flistxattr			(__NR_Linux + 188)
+#define __NR_removexattr		(__NR_Linux + 189)
+#define __NR_lremovexattr		(__NR_Linux + 190)
+#define __NR_fremovexattr		(__NR_Linux + 191)
+#define __NR_tkill			(__NR_Linux + 192)
+#define __NR_reserved193		(__NR_Linux + 193)
+#define __NR_futex			(__NR_Linux + 194)
+#define __NR_sched_setaffinity		(__NR_Linux + 195)
+#define __NR_sched_getaffinity		(__NR_Linux + 196)
+#define __NR_cacheflush			(__NR_Linux + 197)
+#define __NR_cachectl			(__NR_Linux + 198)
+#define __NR_sysmips			(__NR_Linux + 199)
+#define __NR_io_setup			(__NR_Linux + 200)
+#define __NR_io_destroy			(__NR_Linux + 201)
+#define __NR_io_getevents		(__NR_Linux + 202)
+#define __NR_io_submit			(__NR_Linux + 203)
+#define __NR_io_cancel			(__NR_Linux + 204)
+#define __NR_exit_group			(__NR_Linux + 205)
+#define __NR_lookup_dcookie		(__NR_Linux + 206)
+#define __NR_epoll_create		(__NR_Linux + 207)
+#define __NR_epoll_ctl			(__NR_Linux + 208)
+#define __NR_epoll_wait			(__NR_Linux + 209)
+#define __NR_remap_file_pages		(__NR_Linux + 210)
+#define __NR_rt_sigreturn		(__NR_Linux + 211)
+#define __NR_fcntl64			(__NR_Linux + 212)
+#define __NR_set_tid_address		(__NR_Linux + 213)
+#define __NR_restart_syscall		(__NR_Linux + 214)
+#define __NR_semtimedop			(__NR_Linux + 215)
+#define __NR_fadvise64			(__NR_Linux + 216)
+#define __NR_statfs64			(__NR_Linux + 217)
+#define __NR_fstatfs64			(__NR_Linux + 218)
+#define __NR_sendfile64			(__NR_Linux + 219)
+#define __NR_timer_create		(__NR_Linux + 220)
+#define __NR_timer_settime		(__NR_Linux + 221)
+#define __NR_timer_gettime		(__NR_Linux + 222)
+#define __NR_timer_getoverrun		(__NR_Linux + 223)
+#define __NR_timer_delete		(__NR_Linux + 224)
+#define __NR_clock_settime		(__NR_Linux + 225)
+#define __NR_clock_gettime		(__NR_Linux + 226)
+#define __NR_clock_getres		(__NR_Linux + 227)
+#define __NR_clock_nanosleep		(__NR_Linux + 228)
+#define __NR_tgkill			(__NR_Linux + 229)
+#define __NR_utimes			(__NR_Linux + 230)
+#define __NR_mbind			(__NR_Linux + 231)
+#define __NR_get_mempolicy		(__NR_Linux + 232)
+#define __NR_set_mempolicy		(__NR_Linux + 233)
+#define __NR_mq_open			(__NR_Linux + 234)
+#define __NR_mq_unlink			(__NR_Linux + 235)
+#define __NR_mq_timedsend		(__NR_Linux + 236)
+#define __NR_mq_timedreceive		(__NR_Linux + 237)
+#define __NR_mq_notify			(__NR_Linux + 238)
+#define __NR_mq_getsetattr		(__NR_Linux + 239)
+#define __NR_vserver			(__NR_Linux + 240)
+#define __NR_waitid			(__NR_Linux + 241)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 242) */
+#define __NR_add_key			(__NR_Linux + 243)
+#define __NR_request_key		(__NR_Linux + 244)
+#define __NR_keyctl			(__NR_Linux + 245)
+#define __NR_set_thread_area		(__NR_Linux + 246)
+#define __NR_inotify_init		(__NR_Linux + 247)
+#define __NR_inotify_add_watch		(__NR_Linux + 248)
+#define __NR_inotify_rm_watch		(__NR_Linux + 249)
+#define __NR_migrate_pages		(__NR_Linux + 250)
+#define __NR_openat			(__NR_Linux + 251)
+#define __NR_mkdirat			(__NR_Linux + 252)
+#define __NR_mknodat			(__NR_Linux + 253)
+#define __NR_fchownat			(__NR_Linux + 254)
+#define __NR_futimesat			(__NR_Linux + 255)
+#define __NR_newfstatat			(__NR_Linux + 256)
+#define __NR_unlinkat			(__NR_Linux + 257)
+#define __NR_renameat			(__NR_Linux + 258)
+#define __NR_linkat			(__NR_Linux + 259)
+#define __NR_symlinkat			(__NR_Linux + 260)
+#define __NR_readlinkat			(__NR_Linux + 261)
+#define __NR_fchmodat			(__NR_Linux + 262)
+#define __NR_faccessat			(__NR_Linux + 263)
+#define __NR_pselect6			(__NR_Linux + 264)
+#define __NR_ppoll			(__NR_Linux + 265)
+#define __NR_unshare			(__NR_Linux + 266)
+#define __NR_splice			(__NR_Linux + 267)
+#define __NR_sync_file_range		(__NR_Linux + 268)
+#define __NR_tee			(__NR_Linux + 269)
+#define __NR_vmsplice			(__NR_Linux + 270)
+#define __NR_move_pages			(__NR_Linux + 271)
+#define __NR_set_robust_list		(__NR_Linux + 272)
+#define __NR_get_robust_list		(__NR_Linux + 273)
+#define __NR_kexec_load			(__NR_Linux + 274)
+#define __NR_getcpu			(__NR_Linux + 275)
+#define __NR_epoll_pwait		(__NR_Linux + 276)
+#define __NR_ioprio_set			(__NR_Linux + 277)
+#define __NR_ioprio_get			(__NR_Linux + 278)
+#define __NR_utimensat			(__NR_Linux + 279)
+#define __NR_signalfd			(__NR_Linux + 280)
+#define __NR_timerfd			(__NR_Linux + 281)
+#define __NR_eventfd			(__NR_Linux + 282)
+#define __NR_fallocate			(__NR_Linux + 283)
+#define __NR_timerfd_create		(__NR_Linux + 284)
+#define __NR_timerfd_gettime		(__NR_Linux + 285)
+#define __NR_timerfd_settime		(__NR_Linux + 286)
+#define __NR_signalfd4			(__NR_Linux + 287)
+#define __NR_eventfd2			(__NR_Linux + 288)
+#define __NR_epoll_create1		(__NR_Linux + 289)
+#define __NR_dup3			(__NR_Linux + 290)
+#define __NR_pipe2			(__NR_Linux + 291)
+#define __NR_inotify_init1		(__NR_Linux + 292)
+#define __NR_preadv			(__NR_Linux + 293)
+#define __NR_pwritev			(__NR_Linux + 294)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 295)
+#define __NR_perf_event_open		(__NR_Linux + 296)
+#define __NR_accept4			(__NR_Linux + 297)
+#define __NR_recvmmsg			(__NR_Linux + 298)
+#define __NR_getdents64			(__NR_Linux + 299)
+#define __NR_fanotify_init		(__NR_Linux + 300)
+#define __NR_fanotify_mark		(__NR_Linux + 301)
+#define __NR_prlimit64			(__NR_Linux + 302)
+#define __NR_name_to_handle_at		(__NR_Linux + 303)
+#define __NR_open_by_handle_at		(__NR_Linux + 304)
+#define __NR_clock_adjtime		(__NR_Linux + 305)
+#define __NR_syncfs			(__NR_Linux + 306)
+#define __NR_sendmmsg			(__NR_Linux + 307)
+#define __NR_setns			(__NR_Linux + 308)
+#define __NR_process_vm_readv		(__NR_Linux + 309)
+#define __NR_process_vm_writev		(__NR_Linux + 310)
+#define __NR_kcmp			(__NR_Linux + 311)
+#define __NR_finit_module		(__NR_Linux + 312)
+#define __NR_sched_setattr		(__NR_Linux + 313)
+#define __NR_sched_getattr		(__NR_Linux + 314)
+#define __NR_renameat2			(__NR_Linux + 315)
+#define __NR_seccomp			(__NR_Linux + 316)
+#define __NR_getrandom			(__NR_Linux + 317)
+#define __NR_memfd_create		(__NR_Linux + 318)
+#define __NR_bpf			(__NR_Linux + 319)
+#define __NR_execveat			(__NR_Linux + 320)
+
+/*
+ * Offset of the last N32 flavoured syscall
+ */
+#define __NR_Linux_syscalls		320
+
+#endif /* _MIPS_SIM == _MIPS_SIM_NABI32 */
+
+#define __NR_N32_Linux			6000
+#define __NR_N32_Linux_syscalls		320
+
+#endif /* _ASM_UNISTD_H */
diff --git a/linux-headers/asm-powerpc/unistd.h b/linux-headers/asm-powerpc/unistd.h
new file mode 100644
index 000000000..28deee06c
--- /dev/null
+++ b/linux-headers/asm-powerpc/unistd.h
@@ -0,0 +1,392 @@
+/*
+ * This file contains the system call numbers.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#ifndef _ASM_POWERPC_UNISTD_H_
+#define _ASM_POWERPC_UNISTD_H_
+
+
+#define __NR_restart_syscall	  0
+#define __NR_exit		  1
+#define __NR_fork		  2
+#define __NR_read		  3
+#define __NR_write		  4
+#define __NR_open		  5
+#define __NR_close		  6
+#define __NR_waitpid		  7
+#define __NR_creat		  8
+#define __NR_link		  9
+#define __NR_unlink		 10
+#define __NR_execve		 11
+#define __NR_chdir		 12
+#define __NR_time		 13
+#define __NR_mknod		 14
+#define __NR_chmod		 15
+#define __NR_lchown		 16
+#define __NR_break		 17
+#define __NR_oldstat		 18
+#define __NR_lseek		 19
+#define __NR_getpid		 20
+#define __NR_mount		 21
+#define __NR_umount		 22
+#define __NR_setuid		 23
+#define __NR_getuid		 24
+#define __NR_stime		 25
+#define __NR_ptrace		 26
+#define __NR_alarm		 27
+#define __NR_oldfstat		 28
+#define __NR_pause		 29
+#define __NR_utime		 30
+#define __NR_stty		 31
+#define __NR_gtty		 32
+#define __NR_access		 33
+#define __NR_nice		 34
+#define __NR_ftime		 35
+#define __NR_sync		 36
+#define __NR_kill		 37
+#define __NR_rename		 38
+#define __NR_mkdir		 39
+#define __NR_rmdir		 40
+#define __NR_dup		 41
+#define __NR_pipe		 42
+#define __NR_times		 43
+#define __NR_prof		 44
+#define __NR_brk		 45
+#define __NR_setgid		 46
+#define __NR_getgid		 47
+#define __NR_signal		 48
+#define __NR_geteuid		 49
+#define __NR_getegid		 50
+#define __NR_acct		 51
+#define __NR_umount2		 52
+#define __NR_lock		 53
+#define __NR_ioctl		 54
+#define __NR_fcntl		 55
+#define __NR_mpx		 56
+#define __NR_setpgid		 57
+#define __NR_ulimit		 58
+#define __NR_oldolduname	 59
+#define __NR_umask		 60
+#define __NR_chroot		 61
+#define __NR_ustat		 62
+#define __NR_dup2		 63
+#define __NR_getppid		 64
+#define __NR_getpgrp		 65
+#define __NR_setsid		 66
+#define __NR_sigaction		 67
+#define __NR_sgetmask		 68
+#define __NR_ssetmask		 69
+#define __NR_setreuid		 70
+#define __NR_setregid		 71
+#define __NR_sigsuspend		 72
+#define __NR_sigpending		 73
+#define __NR_sethostname	 74
+#define __NR_setrlimit		 75
+#define __NR_getrlimit		 76
+#define __NR_getrusage		 77
+#define __NR_gettimeofday	 78
+#define __NR_settimeofday	 79
+#define __NR_getgroups		 80
+#define __NR_setgroups		 81
+#define __NR_select		 82
+#define __NR_symlink		 83
+#define __NR_oldlstat		 84
+#define __NR_readlink		 85
+#define __NR_uselib		 86
+#define __NR_swapon		 87
+#define __NR_reboot		 88
+#define __NR_readdir		 89
+#define __NR_mmap		 90
+#define __NR_munmap		 91
+#define __NR_truncate		 92
+#define __NR_ftruncate		 93
+#define __NR_fchmod		 94
+#define __NR_fchown		 95
+#define __NR_getpriority	 96
+#define __NR_setpriority	 97
+#define __NR_profil		 98
+#define __NR_statfs		 99
+#define __NR_fstatfs		100
+#define __NR_ioperm		101
+#define __NR_socketcall		102
+#define __NR_syslog		103
+#define __NR_setitimer		104
+#define __NR_getitimer		105
+#define __NR_stat		106
+#define __NR_lstat		107
+#define __NR_fstat		108
+#define __NR_olduname		109
+#define __NR_iopl		110
+#define __NR_vhangup		111
+#define __NR_idle		112
+#define __NR_vm86		113
+#define __NR_wait4		114
+#define __NR_swapoff		115
+#define __NR_sysinfo		116
+#define __NR_ipc		117
+#define __NR_fsync		118
+#define __NR_sigreturn		119
+#define __NR_clone		120
+#define __NR_setdomainname	121
+#define __NR_uname		122
+#define __NR_modify_ldt		123
+#define __NR_adjtimex		124
+#define __NR_mprotect		125
+#define __NR_sigprocmask	126
+#define __NR_create_module	127
+#define __NR_init_module	128
+#define __NR_delete_module	129
+#define __NR_get_kernel_syms	130
+#define __NR_quotactl		131
+#define __NR_getpgid		132
+#define __NR_fchdir		133
+#define __NR_bdflush		134
+#define __NR_sysfs		135
+#define __NR_personality	136
+#define __NR_afs_syscall	137 /* Syscall for Andrew File System */
+#define __NR_setfsuid		138
+#define __NR_setfsgid		139
+#define __NR__llseek		140
+#define __NR_getdents		141
+#define __NR__newselect		142
+#define __NR_flock		143
+#define __NR_msync		144
+#define __NR_readv		145
+#define __NR_writev		146
+#define __NR_getsid		147
+#define __NR_fdatasync		148
+#define __NR__sysctl		149
+#define __NR_mlock		150
+#define __NR_munlock		151
+#define __NR_mlockall		152
+#define __NR_munlockall		153
+#define __NR_sched_setparam		154
+#define __NR_sched_getparam		155
+#define __NR_sched_setscheduler		156
+#define __NR_sched_getscheduler		157
+#define __NR_sched_yield		158
+#define __NR_sched_get_priority_max	159
+#define __NR_sched_get_priority_min	160
+#define __NR_sched_rr_get_interval	161
+#define __NR_nanosleep		162
+#define __NR_mremap		163
+#define __NR_setresuid		164
+#define __NR_getresuid		165
+#define __NR_query_module	166
+#define __NR_poll		167
+#define __NR_nfsservctl		168
+#define __NR_setresgid		169
+#define __NR_getresgid		170
+#define __NR_prctl		171
+#define __NR_rt_sigreturn	172
+#define __NR_rt_sigaction	173
+#define __NR_rt_sigprocmask	174
+#define __NR_rt_sigpending	175
+#define __NR_rt_sigtimedwait	176
+#define __NR_rt_sigqueueinfo	177
+#define __NR_rt_sigsuspend	178
+#define __NR_pread64		179
+#define __NR_pwrite64		180
+#define __NR_chown		181
+#define __NR_getcwd		182
+#define __NR_capget		183
+#define __NR_capset		184
+#define __NR_sigaltstack	185
+#define __NR_sendfile		186
+#define __NR_getpmsg		187	/* some people actually want streams */
+#define __NR_putpmsg		188	/* some people actually want streams */
+#define __NR_vfork		189
+#define __NR_ugetrlimit		190	/* SuS compliant getrlimit */
+#define __NR_readahead		191
+#ifndef __powerpc64__			/* these are 32-bit only */
+#define __NR_mmap2		192
+#define __NR_truncate64		193
+#define __NR_ftruncate64	194
+#define __NR_stat64		195
+#define __NR_lstat64		196
+#define __NR_fstat64		197
+#endif
+#define __NR_pciconfig_read	198
+#define __NR_pciconfig_write	199
+#define __NR_pciconfig_iobase	200
+#define __NR_multiplexer	201
+#define __NR_getdents64		202
+#define __NR_pivot_root		203
+#ifndef __powerpc64__
+#define __NR_fcntl64		204
+#endif
+#define __NR_madvise		205
+#define __NR_mincore		206
+#define __NR_gettid		207
+#define __NR_tkill		208
+#define __NR_setxattr		209
+#define __NR_lsetxattr		210
+#define __NR_fsetxattr		211
+#define __NR_getxattr		212
+#define __NR_lgetxattr		213
+#define __NR_fgetxattr		214
+#define __NR_listxattr		215
+#define __NR_llistxattr		216
+#define __NR_flistxattr		217
+#define __NR_removexattr	218
+#define __NR_lremovexattr	219
+#define __NR_fremovexattr	220
+#define __NR_futex		221
+#define __NR_sched_setaffinity	222
+#define __NR_sched_getaffinity	223
+/* 224 currently unused */
+#define __NR_tuxcall		225
+#ifndef __powerpc64__
+#define __NR_sendfile64		226
+#endif
+#define __NR_io_setup		227
+#define __NR_io_destroy		228
+#define __NR_io_getevents	229
+#define __NR_io_submit		230
+#define __NR_io_cancel		231
+#define __NR_set_tid_address	232
+#define __NR_fadvise64		233
+#define __NR_exit_group		234
+#define __NR_lookup_dcookie	235
+#define __NR_epoll_create	236
+#define __NR_epoll_ctl		237
+#define __NR_epoll_wait		238
+#define __NR_remap_file_pages	239
+#define __NR_timer_create	240
+#define __NR_timer_settime	241
+#define __NR_timer_gettime	242
+#define __NR_timer_getoverrun	243
+#define __NR_timer_delete	244
+#define __NR_clock_settime	245
+#define __NR_clock_gettime	246
+#define __NR_clock_getres	247
+#define __NR_clock_nanosleep	248
+#define __NR_swapcontext	249
+#define __NR_tgkill		250
+#define __NR_utimes		251
+#define __NR_statfs64		252
+#define __NR_fstatfs64		253
+#ifndef __powerpc64__
+#define __NR_fadvise64_64	254
+#endif
+#define __NR_rtas		255
+#define __NR_sys_debug_setcontext 256
+/* Number 257 is reserved for vserver */
+#define __NR_migrate_pages	258
+#define __NR_mbind		259
+#define __NR_get_mempolicy	260
+#define __NR_set_mempolicy	261
+#define __NR_mq_open		262
+#define __NR_mq_unlink		263
+#define __NR_mq_timedsend	264
+#define __NR_mq_timedreceive	265
+#define __NR_mq_notify		266
+#define __NR_mq_getsetattr	267
+#define __NR_kexec_load		268
+#define __NR_add_key		269
+#define __NR_request_key	270
+#define __NR_keyctl		271
+#define __NR_waitid		272
+#define __NR_ioprio_set		273
+#define __NR_ioprio_get		274
+#define __NR_inotify_init	275
+#define __NR_inotify_add_watch	276
+#define __NR_inotify_rm_watch	277
+#define __NR_spu_run		278
+#define __NR_spu_create		279
+#define __NR_pselect6		280
+#define __NR_ppoll		281
+#define __NR_unshare		282
+#define __NR_splice		283
+#define __NR_tee		284
+#define __NR_vmsplice		285
+#define __NR_openat		286
+#define __NR_mkdirat		287
+#define __NR_mknodat		288
+#define __NR_fchownat		289
+#define __NR_futimesat		290
+#ifdef __powerpc64__
+#define __NR_newfstatat		291
+#else
+#define __NR_fstatat64		291
+#endif
+#define __NR_unlinkat		292
+#define __NR_renameat		293
+#define __NR_linkat		294
+#define __NR_symlinkat		295
+#define __NR_readlinkat		296
+#define __NR_fchmodat		297
+#define __NR_faccessat		298
+#define __NR_get_robust_list	299
+#define __NR_set_robust_list	300
+#define __NR_move_pages		301
+#define __NR_getcpu		302
+#define __NR_epoll_pwait	303
+#define __NR_utimensat		304
+#define __NR_signalfd		305
+#define __NR_timerfd_create	306
+#define __NR_eventfd		307
+#define __NR_sync_file_range2	308
+#define __NR_fallocate		309
+#define __NR_subpage_prot	310
+#define __NR_timerfd_settime	311
+#define __NR_timerfd_gettime	312
+#define __NR_signalfd4		313
+#define __NR_eventfd2		314
+#define __NR_epoll_create1	315
+#define __NR_dup3		316
+#define __NR_pipe2		317
+#define __NR_inotify_init1	318
+#define __NR_perf_event_open	319
+#define __NR_preadv		320
+#define __NR_pwritev		321
+#define __NR_rt_tgsigqueueinfo	322
+#define __NR_fanotify_init	323
+#define __NR_fanotify_mark	324
+#define __NR_prlimit64		325
+#define __NR_socket		326
+#define __NR_bind		327
+#define __NR_connect		328
+#define __NR_listen		329
+#define __NR_accept		330
+#define __NR_getsockname	331
+#define __NR_getpeername	332
+#define __NR_socketpair		333
+#define __NR_send		334
+#define __NR_sendto		335
+#define __NR_recv		336
+#define __NR_recvfrom		337
+#define __NR_shutdown		338
+#define __NR_setsockopt		339
+#define __NR_getsockopt		340
+#define __NR_sendmsg		341
+#define __NR_recvmsg		342
+#define __NR_recvmmsg		343
+#define __NR_accept4		344
+#define __NR_name_to_handle_at	345
+#define __NR_open_by_handle_at	346
+#define __NR_clock_adjtime	347
+#define __NR_syncfs		348
+#define __NR_sendmmsg		349
+#define __NR_setns		350
+#define __NR_process_vm_readv	351
+#define __NR_process_vm_writev	352
+#define __NR_finit_module	353
+#define __NR_kcmp		354
+#define __NR_sched_setattr	355
+#define __NR_sched_getattr	356
+#define __NR_renameat2		357
+#define __NR_seccomp		358
+#define __NR_getrandom		359
+#define __NR_memfd_create	360
+#define __NR_bpf		361
+#define __NR_execveat		362
+#define __NR_switch_endian	363
+#define __NR_userfaultfd	364
+#define __NR_membarrier		365
+
+#endif /* _ASM_POWERPC_UNISTD_H_ */
diff --git a/linux-headers/asm-s390/unistd.h b/linux-headers/asm-s390/unistd.h
new file mode 100644
index 000000000..04b43b129
--- /dev/null
+++ b/linux-headers/asm-s390/unistd.h
@@ -0,0 +1,404 @@
+/*
+ *  S390 version
+ *
+ *  Derived from "include/asm-i386/unistd.h"
+ */
+
+#ifndef _ASM_S390_UNISTD_H_
+#define _ASM_S390_UNISTD_H_
+
+/*
+ * This file contains the system call numbers.
+ */
+
+#define __NR_exit                 1
+#define __NR_fork                 2
+#define __NR_read                 3
+#define __NR_write                4
+#define __NR_open                 5
+#define __NR_close                6
+#define __NR_restart_syscall	  7
+#define __NR_creat                8
+#define __NR_link                 9
+#define __NR_unlink              10
+#define __NR_execve              11
+#define __NR_chdir               12
+#define __NR_mknod               14
+#define __NR_chmod               15
+#define __NR_lseek               19
+#define __NR_getpid              20
+#define __NR_mount               21
+#define __NR_umount              22
+#define __NR_ptrace              26
+#define __NR_alarm               27
+#define __NR_pause               29
+#define __NR_utime               30
+#define __NR_access              33
+#define __NR_nice                34
+#define __NR_sync                36
+#define __NR_kill                37
+#define __NR_rename              38
+#define __NR_mkdir               39
+#define __NR_rmdir               40
+#define __NR_dup                 41
+#define __NR_pipe                42
+#define __NR_times               43
+#define __NR_brk                 45
+#define __NR_signal              48
+#define __NR_acct                51
+#define __NR_umount2             52
+#define __NR_ioctl               54
+#define __NR_fcntl               55
+#define __NR_setpgid             57
+#define __NR_umask               60
+#define __NR_chroot              61
+#define __NR_ustat               62
+#define __NR_dup2                63
+#define __NR_getppid             64
+#define __NR_getpgrp             65
+#define __NR_setsid              66
+#define __NR_sigaction           67
+#define __NR_sigsuspend          72
+#define __NR_sigpending          73
+#define __NR_sethostname         74
+#define __NR_setrlimit           75
+#define __NR_getrusage           77
+#define __NR_gettimeofday        78
+#define __NR_settimeofday        79
+#define __NR_symlink             83
+#define __NR_readlink            85
+#define __NR_uselib              86
+#define __NR_swapon              87
+#define __NR_reboot              88
+#define __NR_readdir             89
+#define __NR_mmap                90
+#define __NR_munmap              91
+#define __NR_truncate            92
+#define __NR_ftruncate           93
+#define __NR_fchmod              94
+#define __NR_getpriority         96
+#define __NR_setpriority         97
+#define __NR_statfs              99
+#define __NR_fstatfs            100
+#define __NR_socketcall         102
+#define __NR_syslog             103
+#define __NR_setitimer          104
+#define __NR_getitimer          105
+#define __NR_stat               106
+#define __NR_lstat              107
+#define __NR_fstat              108
+#define __NR_lookup_dcookie     110
+#define __NR_vhangup            111
+#define __NR_idle               112
+#define __NR_wait4              114
+#define __NR_swapoff            115
+#define __NR_sysinfo            116
+#define __NR_ipc                117
+#define __NR_fsync              118
+#define __NR_sigreturn          119
+#define __NR_clone              120
+#define __NR_setdomainname      121
+#define __NR_uname              122
+#define __NR_adjtimex           124
+#define __NR_mprotect           125
+#define __NR_sigprocmask        126
+#define __NR_create_module      127
+#define __NR_init_module        128
+#define __NR_delete_module      129
+#define __NR_get_kernel_syms    130
+#define __NR_quotactl           131
+#define __NR_getpgid            132
+#define __NR_fchdir             133
+#define __NR_bdflush            134
+#define __NR_sysfs              135
+#define __NR_personality        136
+#define __NR_afs_syscall        137 /* Syscall for Andrew File System */
+#define __NR_getdents           141
+#define __NR_flock              143
+#define __NR_msync              144
+#define __NR_readv              145
+#define __NR_writev             146
+#define __NR_getsid             147
+#define __NR_fdatasync          148
+#define __NR__sysctl            149
+#define __NR_mlock              150
+#define __NR_munlock            151
+#define __NR_mlockall           152
+#define __NR_munlockall         153
+#define __NR_sched_setparam             154
+#define __NR_sched_getparam             155
+#define __NR_sched_setscheduler         156
+#define __NR_sched_getscheduler         157
+#define __NR_sched_yield                158
+#define __NR_sched_get_priority_max     159
+#define __NR_sched_get_priority_min     160
+#define __NR_sched_rr_get_interval      161
+#define __NR_nanosleep          162
+#define __NR_mremap             163
+#define __NR_query_module       167
+#define __NR_poll               168
+#define __NR_nfsservctl         169
+#define __NR_prctl              172
+#define __NR_rt_sigreturn       173
+#define __NR_rt_sigaction       174
+#define __NR_rt_sigprocmask     175
+#define __NR_rt_sigpending      176
+#define __NR_rt_sigtimedwait    177
+#define __NR_rt_sigqueueinfo    178
+#define __NR_rt_sigsuspend      179
+#define __NR_pread64            180
+#define __NR_pwrite64           181
+#define __NR_getcwd             183
+#define __NR_capget             184
+#define __NR_capset             185
+#define __NR_sigaltstack        186
+#define __NR_sendfile           187
+#define __NR_getpmsg		188
+#define __NR_putpmsg		189
+#define __NR_vfork		190
+#define __NR_pivot_root         217
+#define __NR_mincore            218
+#define __NR_madvise            219
+#define __NR_getdents64		220
+#define __NR_readahead		222
+#define __NR_setxattr		224
+#define __NR_lsetxattr		225
+#define __NR_fsetxattr		226
+#define __NR_getxattr		227
+#define __NR_lgetxattr		228
+#define __NR_fgetxattr		229
+#define __NR_listxattr		230
+#define __NR_llistxattr		231
+#define __NR_flistxattr		232
+#define __NR_removexattr	233
+#define __NR_lremovexattr	234
+#define __NR_fremovexattr	235
+#define __NR_gettid		236
+#define __NR_tkill		237
+#define __NR_futex		238
+#define __NR_sched_setaffinity	239
+#define __NR_sched_getaffinity	240
+#define __NR_tgkill		241
+/* Number 242 is reserved for tux */
+#define __NR_io_setup		243
+#define __NR_io_destroy		244
+#define __NR_io_getevents	245
+#define __NR_io_submit		246
+#define __NR_io_cancel		247
+#define __NR_exit_group		248
+#define __NR_epoll_create	249
+#define __NR_epoll_ctl		250
+#define __NR_epoll_wait		251
+#define __NR_set_tid_address	252
+#define __NR_fadvise64		253
+#define __NR_timer_create	254
+#define __NR_timer_settime	(__NR_timer_create+1)
+#define __NR_timer_gettime	(__NR_timer_create+2)
+#define __NR_timer_getoverrun	(__NR_timer_create+3)
+#define __NR_timer_delete	(__NR_timer_create+4)
+#define __NR_clock_settime	(__NR_timer_create+5)
+#define __NR_clock_gettime	(__NR_timer_create+6)
+#define __NR_clock_getres	(__NR_timer_create+7)
+#define __NR_clock_nanosleep	(__NR_timer_create+8)
+/* Number 263 is reserved for vserver */
+#define __NR_statfs64		265
+#define __NR_fstatfs64		266
+#define __NR_remap_file_pages	267
+#define __NR_mbind		268
+#define __NR_get_mempolicy	269
+#define __NR_set_mempolicy	270
+#define __NR_mq_open		271
+#define __NR_mq_unlink		272
+#define __NR_mq_timedsend	273
+#define __NR_mq_timedreceive	274
+#define __NR_mq_notify		275
+#define __NR_mq_getsetattr	276
+#define __NR_kexec_load		277
+#define __NR_add_key		278
+#define __NR_request_key	279
+#define __NR_keyctl		280
+#define __NR_waitid		281
+#define __NR_ioprio_set		282
+#define __NR_ioprio_get		283
+#define __NR_inotify_init	284
+#define __NR_inotify_add_watch	285
+#define __NR_inotify_rm_watch	286
+#define __NR_migrate_pages	287
+#define __NR_openat		288
+#define __NR_mkdirat		289
+#define __NR_mknodat		290
+#define __NR_fchownat		291
+#define __NR_futimesat		292
+#define __NR_unlinkat		294
+#define __NR_renameat		295
+#define __NR_linkat		296
+#define __NR_symlinkat		297
+#define __NR_readlinkat		298
+#define __NR_fchmodat		299
+#define __NR_faccessat		300
+#define __NR_pselect6		301
+#define __NR_ppoll		302
+#define __NR_unshare		303
+#define __NR_set_robust_list	304
+#define __NR_get_robust_list	305
+#define __NR_splice		306
+#define __NR_sync_file_range	307
+#define __NR_tee		308
+#define __NR_vmsplice		309
+#define __NR_move_pages		310
+#define __NR_getcpu		311
+#define __NR_epoll_pwait	312
+#define __NR_utimes		313
+#define __NR_fallocate		314
+#define __NR_utimensat		315
+#define __NR_signalfd		316
+#define __NR_timerfd		317
+#define __NR_eventfd		318
+#define __NR_timerfd_create	319
+#define __NR_timerfd_settime	320
+#define __NR_timerfd_gettime	321
+#define __NR_signalfd4		322
+#define __NR_eventfd2		323
+#define __NR_inotify_init1	324
+#define __NR_pipe2		325
+#define __NR_dup3		326
+#define __NR_epoll_create1	327
+#define	__NR_preadv		328
+#define	__NR_pwritev		329
+#define __NR_rt_tgsigqueueinfo	330
+#define __NR_perf_event_open	331
+#define __NR_fanotify_init	332
+#define __NR_fanotify_mark	333
+#define __NR_prlimit64		334
+#define __NR_name_to_handle_at	335
+#define __NR_open_by_handle_at	336
+#define __NR_clock_adjtime	337
+#define __NR_syncfs		338
+#define __NR_setns		339
+#define __NR_process_vm_readv	340
+#define __NR_process_vm_writev	341
+#define __NR_s390_runtime_instr 342
+#define __NR_kcmp		343
+#define __NR_finit_module	344
+#define __NR_sched_setattr	345
+#define __NR_sched_getattr	346
+#define __NR_renameat2		347
+#define __NR_seccomp		348
+#define __NR_getrandom		349
+#define __NR_memfd_create	350
+#define __NR_bpf		351
+#define __NR_s390_pci_mmio_write	352
+#define __NR_s390_pci_mmio_read		353
+#define __NR_execveat		354
+#define __NR_userfaultfd	355
+#define __NR_membarrier		356
+#define __NR_recvmmsg		357
+#define __NR_sendmmsg		358
+#define __NR_socket		359
+#define __NR_socketpair		360
+#define __NR_bind		361
+#define __NR_connect		362
+#define __NR_listen		363
+#define __NR_accept4		364
+#define __NR_getsockopt		365
+#define __NR_setsockopt		366
+#define __NR_getsockname	367
+#define __NR_getpeername	368
+#define __NR_sendto		369
+#define __NR_sendmsg		370
+#define __NR_recvfrom		371
+#define __NR_recvmsg		372
+#define __NR_shutdown		373
+#define NR_syscalls 374
+
+/* 
+ * There are some system calls that are not present on 64 bit, some
+ * have a different name although they do the same (e.g. __NR_chown32
+ * is __NR_chown on 64 bit).
+ */
+#ifndef __s390x__
+
+#define __NR_time		 13
+#define __NR_lchown		 16
+#define __NR_setuid		 23
+#define __NR_getuid		 24
+#define __NR_stime		 25
+#define __NR_setgid		 46
+#define __NR_getgid		 47
+#define __NR_geteuid		 49
+#define __NR_getegid		 50
+#define __NR_setreuid		 70
+#define __NR_setregid		 71
+#define __NR_getrlimit		 76
+#define __NR_getgroups		 80
+#define __NR_setgroups		 81
+#define __NR_fchown		 95
+#define __NR_ioperm		101
+#define __NR_setfsuid		138
+#define __NR_setfsgid		139
+#define __NR__llseek		140
+#define __NR__newselect 	142
+#define __NR_setresuid		164
+#define __NR_getresuid		165
+#define __NR_setresgid		170
+#define __NR_getresgid		171
+#define __NR_chown		182
+#define __NR_ugetrlimit		191	/* SuS compliant getrlimit */
+#define __NR_mmap2		192
+#define __NR_truncate64		193
+#define __NR_ftruncate64	194
+#define __NR_stat64		195
+#define __NR_lstat64		196
+#define __NR_fstat64		197
+#define __NR_lchown32		198
+#define __NR_getuid32		199
+#define __NR_getgid32		200
+#define __NR_geteuid32		201
+#define __NR_getegid32		202
+#define __NR_setreuid32		203
+#define __NR_setregid32		204
+#define __NR_getgroups32	205
+#define __NR_setgroups32	206
+#define __NR_fchown32		207
+#define __NR_setresuid32	208
+#define __NR_getresuid32	209
+#define __NR_setresgid32	210
+#define __NR_getresgid32	211
+#define __NR_chown32		212
+#define __NR_setuid32		213
+#define __NR_setgid32		214
+#define __NR_setfsuid32		215
+#define __NR_setfsgid32		216
+#define __NR_fcntl64		221
+#define __NR_sendfile64		223
+#define __NR_fadvise64_64	264
+#define __NR_fstatat64		293
+
+#else
+
+#define __NR_select		142
+#define __NR_getrlimit		191	/* SuS compliant getrlimit */
+#define __NR_lchown  		198
+#define __NR_getuid  		199
+#define __NR_getgid  		200
+#define __NR_geteuid  		201
+#define __NR_getegid  		202
+#define __NR_setreuid  		203
+#define __NR_setregid  		204
+#define __NR_getgroups  	205
+#define __NR_setgroups  	206
+#define __NR_fchown  		207
+#define __NR_setresuid  	208
+#define __NR_getresuid  	209
+#define __NR_setresgid  	210
+#define __NR_getresgid  	211
+#define __NR_chown  		212
+#define __NR_setuid  		213
+#define __NR_setgid  		214
+#define __NR_setfsuid  		215
+#define __NR_setfsgid  		216
+#define __NR_newfstatat		293
+
+#endif
+
+#endif /* _ASM_S390_UNISTD_H_ */
diff --git a/linux-headers/asm-x86/hyperv.h b/linux-headers/asm-x86/hyperv.h
index 8fba544e9..01af4d859 100644
--- a/linux-headers/asm-x86/hyperv.h
+++ b/linux-headers/asm-x86/hyperv.h
@@ -1,250 +1 @@
-#ifndef _ASM_X86_HYPERV_H
-#define _ASM_X86_HYPERV_H
-
-#include <linux/types.h>
-
-/*
- * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
- * is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
- */
-#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
-#define HYPERV_CPUID_INTERFACE			0x40000001
-#define HYPERV_CPUID_VERSION			0x40000002
-#define HYPERV_CPUID_FEATURES			0x40000003
-#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
-#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
-
-#define HYPERV_HYPERVISOR_PRESENT_BIT		0x80000000
-#define HYPERV_CPUID_MIN			0x40000005
-#define HYPERV_CPUID_MAX			0x4000ffff
-
-/*
- * Feature identification. EAX indicates which features are available
- * to the partition based upon the current partition privileges.
- */
-
-/* VP Runtime (HV_X64_MSR_VP_RUNTIME) available */
-#define HV_X64_MSR_VP_RUNTIME_AVAILABLE		(1 << 0)
-/* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
-#define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
-
-/* A partition's reference time stamp counter (TSC) page */
-#define HV_X64_MSR_REFERENCE_TSC		0x40000021
-
-/*
- * There is a single feature flag that signifies the presence of the MSR
- * that can be used to retrieve both the local APIC Timer frequency as
- * well as the TSC frequency.
- */
-
-/* Local APIC timer frequency MSR (HV_X64_MSR_APIC_FREQUENCY) is available */
-#define HV_X64_MSR_APIC_FREQUENCY_AVAILABLE (1 << 11)
-
-/* TSC frequency MSR (HV_X64_MSR_TSC_FREQUENCY) is available */
-#define HV_X64_MSR_TSC_FREQUENCY_AVAILABLE (1 << 11)
-
-/*
- * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
- * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available
- */
-#define HV_X64_MSR_SYNIC_AVAILABLE		(1 << 2)
-/*
- * Synthetic Timer MSRs (HV_X64_MSR_STIMER0_CONFIG through
- * HV_X64_MSR_STIMER3_COUNT) available
- */
-#define HV_X64_MSR_SYNTIMER_AVAILABLE		(1 << 3)
-/*
- * APIC access MSRs (HV_X64_MSR_EOI, HV_X64_MSR_ICR and HV_X64_MSR_TPR)
- * are available
- */
-#define HV_X64_MSR_APIC_ACCESS_AVAILABLE	(1 << 4)
-/* Hypercall MSRs (HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL) available*/
-#define HV_X64_MSR_HYPERCALL_AVAILABLE		(1 << 5)
-/* Access virtual processor index MSR (HV_X64_MSR_VP_INDEX) available*/
-#define HV_X64_MSR_VP_INDEX_AVAILABLE		(1 << 6)
-/* Virtual system reset MSR (HV_X64_MSR_RESET) is available*/
-#define HV_X64_MSR_RESET_AVAILABLE		(1 << 7)
- /*
-  * Access statistics pages MSRs (HV_X64_MSR_STATS_PARTITION_RETAIL_PAGE,
-  * HV_X64_MSR_STATS_PARTITION_INTERNAL_PAGE, HV_X64_MSR_STATS_VP_RETAIL_PAGE,
-  * HV_X64_MSR_STATS_VP_INTERNAL_PAGE) available
-  */
-#define HV_X64_MSR_STAT_PAGES_AVAILABLE		(1 << 8)
-
-/*
- * Feature identification: EBX indicates which flags were specified at
- * partition creation. The format is the same as the partition creation
- * flag structure defined in section Partition Creation Flags.
- */
-#define HV_X64_CREATE_PARTITIONS		(1 << 0)
-#define HV_X64_ACCESS_PARTITION_ID		(1 << 1)
-#define HV_X64_ACCESS_MEMORY_POOL		(1 << 2)
-#define HV_X64_ADJUST_MESSAGE_BUFFERS		(1 << 3)
-#define HV_X64_POST_MESSAGES			(1 << 4)
-#define HV_X64_SIGNAL_EVENTS			(1 << 5)
-#define HV_X64_CREATE_PORT			(1 << 6)
-#define HV_X64_CONNECT_PORT			(1 << 7)
-#define HV_X64_ACCESS_STATS			(1 << 8)
-#define HV_X64_DEBUGGING			(1 << 11)
-#define HV_X64_CPU_POWER_MANAGEMENT		(1 << 12)
-#define HV_X64_CONFIGURE_PROFILER		(1 << 13)
-
-/*
- * Feature identification. EDX indicates which miscellaneous features
- * are available to the partition.
- */
-/* The MWAIT instruction is available (per section MONITOR / MWAIT) */
-#define HV_X64_MWAIT_AVAILABLE				(1 << 0)
-/* Guest debugging support is available */
-#define HV_X64_GUEST_DEBUGGING_AVAILABLE		(1 << 1)
-/* Performance Monitor support is available*/
-#define HV_X64_PERF_MONITOR_AVAILABLE			(1 << 2)
-/* Support for physical CPU dynamic partitioning events is available*/
-#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	(1 << 3)
-/*
- * Support for passing hypercall input parameter block via XMM
- * registers is available
- */
-#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE		(1 << 4)
-/* Support for a virtual guest idle state is available */
-#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		(1 << 5)
-
-/*
- * Implementation recommendations. Indicates which behaviors the hypervisor
- * recommends the OS implement for optimal performance.
- */
- /*
-  * Recommend using hypercall for address space switches rather
-  * than MOV to CR3 instruction
-  */
-#define HV_X64_MWAIT_RECOMMENDED		(1 << 0)
-/* Recommend using hypercall for local TLB flushes rather
- * than INVLPG or MOV to CR3 instructions */
-#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED	(1 << 1)
-/*
- * Recommend using hypercall for remote TLB flushes rather
- * than inter-processor interrupts
- */
-#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED	(1 << 2)
-/*
- * Recommend using MSRs for accessing APIC registers
- * EOI, ICR and TPR rather than their memory-mapped counterparts
- */
-#define HV_X64_APIC_ACCESS_RECOMMENDED		(1 << 3)
-/* Recommend using the hypervisor-provided MSR to initiate a system RESET */
-#define HV_X64_SYSTEM_RESET_RECOMMENDED		(1 << 4)
-/*
- * Recommend using relaxed timing for this partition. If used,
- * the VM should disable any watchdog timeouts that rely on the
- * timely delivery of external interrupts
- */
-#define HV_X64_RELAXED_TIMING_RECOMMENDED	(1 << 5)
-
-/* MSR used to identify the guest OS. */
-#define HV_X64_MSR_GUEST_OS_ID			0x40000000
-
-/* MSR used to setup pages used to communicate with the hypervisor. */
-#define HV_X64_MSR_HYPERCALL			0x40000001
-
-/* MSR used to provide vcpu index */
-#define HV_X64_MSR_VP_INDEX			0x40000002
-
-/* MSR used to read the per-partition time reference counter */
-#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
-
-/* MSR used to retrieve the TSC frequency */
-#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
-
-/* MSR used to retrieve the local APIC timer frequency */
-#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
-
-/* Define the virtual APIC registers */
-#define HV_X64_MSR_EOI				0x40000070
-#define HV_X64_MSR_ICR				0x40000071
-#define HV_X64_MSR_TPR				0x40000072
-#define HV_X64_MSR_APIC_ASSIST_PAGE		0x40000073
-
-/* Define synthetic interrupt controller model specific registers. */
-#define HV_X64_MSR_SCONTROL			0x40000080
-#define HV_X64_MSR_SVERSION			0x40000081
-#define HV_X64_MSR_SIEFP			0x40000082
-#define HV_X64_MSR_SIMP				0x40000083
-#define HV_X64_MSR_EOM				0x40000084
-#define HV_X64_MSR_SINT0			0x40000090
-#define HV_X64_MSR_SINT1			0x40000091
-#define HV_X64_MSR_SINT2			0x40000092
-#define HV_X64_MSR_SINT3			0x40000093
-#define HV_X64_MSR_SINT4			0x40000094
-#define HV_X64_MSR_SINT5			0x40000095
-#define HV_X64_MSR_SINT6			0x40000096
-#define HV_X64_MSR_SINT7			0x40000097
-#define HV_X64_MSR_SINT8			0x40000098
-#define HV_X64_MSR_SINT9			0x40000099
-#define HV_X64_MSR_SINT10			0x4000009A
-#define HV_X64_MSR_SINT11			0x4000009B
-#define HV_X64_MSR_SINT12			0x4000009C
-#define HV_X64_MSR_SINT13			0x4000009D
-#define HV_X64_MSR_SINT14			0x4000009E
-#define HV_X64_MSR_SINT15			0x4000009F
-
-/*
- * Synthetic Timer MSRs. Four timers per vcpu.
- */
-#define HV_X64_MSR_STIMER0_CONFIG		0x400000B0
-#define HV_X64_MSR_STIMER0_COUNT		0x400000B1
-#define HV_X64_MSR_STIMER1_CONFIG		0x400000B2
-#define HV_X64_MSR_STIMER1_COUNT		0x400000B3
-#define HV_X64_MSR_STIMER2_CONFIG		0x400000B4
-#define HV_X64_MSR_STIMER2_COUNT		0x400000B5
-#define HV_X64_MSR_STIMER3_CONFIG		0x400000B6
-#define HV_X64_MSR_STIMER3_COUNT		0x400000B7
-
-/* Hyper-V guest crash notification MSR's */
-#define HV_X64_MSR_CRASH_P0			0x40000100
-#define HV_X64_MSR_CRASH_P1			0x40000101
-#define HV_X64_MSR_CRASH_P2			0x40000102
-#define HV_X64_MSR_CRASH_P3			0x40000103
-#define HV_X64_MSR_CRASH_P4			0x40000104
-#define HV_X64_MSR_CRASH_CTL			0x40000105
-#define HV_X64_MSR_CRASH_CTL_NOTIFY		(1ULL << 63)
-#define HV_X64_MSR_CRASH_PARAMS		\
-		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
-
-#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
-
-/* Declare the various hypercall operations. */
-#define HV_X64_HV_NOTIFY_LONG_SPIN_WAIT		0x0008
-
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE		0x00000001
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
-
-#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
-#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
-
-#define HV_PROCESSOR_POWER_STATE_C0		0
-#define HV_PROCESSOR_POWER_STATE_C1		1
-#define HV_PROCESSOR_POWER_STATE_C2		2
-#define HV_PROCESSOR_POWER_STATE_C3		3
-
-/* hypercall status code */
-#define HV_STATUS_SUCCESS			0
-#define HV_STATUS_INVALID_HYPERCALL_CODE	2
-#define HV_STATUS_INVALID_HYPERCALL_INPUT	3
-#define HV_STATUS_INVALID_ALIGNMENT		4
-#define HV_STATUS_INSUFFICIENT_MEMORY		11
-#define HV_STATUS_INVALID_CONNECTION_ID		18
-#define HV_STATUS_INSUFFICIENT_BUFFERS		19
-
-typedef struct _HV_REFERENCE_TSC_PAGE {
-	__u32 tsc_sequence;
-	__u32 res1;
-	__u64 tsc_scale;
-	__s64 tsc_offset;
-} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
-
-#endif
+#include "standard-headers/asm-x86/hyperv.h"
diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index a4ae82eb8..cd54147cb 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -354,7 +354,7 @@ struct kvm_xcrs {
 struct kvm_sync_regs {
 };
 
-#define KVM_QUIRK_LINT0_REENABLED	(1 << 0)
-#define KVM_QUIRK_CD_NW_CLEARED		(1 << 1)
+#define KVM_X86_QUIRK_LINT0_REENABLED	(1 << 0)
+#define KVM_X86_QUIRK_CD_NW_CLEARED	(1 << 1)
 
 #endif /* _ASM_X86_KVM_H */
diff --git a/linux-headers/asm-x86/unistd.h b/linux-headers/asm-x86/unistd.h
new file mode 100644
index 000000000..1f99b1284
--- /dev/null
+++ b/linux-headers/asm-x86/unistd.h
@@ -0,0 +1,15 @@
+#ifndef _ASM_X86_UNISTD_H
+#define _ASM_X86_UNISTD_H
+
+/* x32 syscall flag bit */
+#define __X32_SYSCALL_BIT	0x40000000
+
+# ifdef __i386__
+#  include <asm/unistd_32.h>
+# elif defined(__ILP32__)
+#  include <asm/unistd_x32.h>
+# else
+#  include <asm/unistd_64.h>
+# endif
+
+#endif /* _ASM_X86_UNISTD_H */
diff --git a/linux-headers/asm-x86/unistd_32.h b/linux-headers/asm-x86/unistd_32.h
new file mode 100644
index 000000000..7570c80c4
--- /dev/null
+++ b/linux-headers/asm-x86/unistd_32.h
@@ -0,0 +1,377 @@
+#ifndef _ASM_X86_UNISTD_32_H
+#define _ASM_X86_UNISTD_32_H 1
+
+#define __NR_restart_syscall 0
+#define __NR_exit 1
+#define __NR_fork 2
+#define __NR_read 3
+#define __NR_write 4
+#define __NR_open 5
+#define __NR_close 6
+#define __NR_waitpid 7
+#define __NR_creat 8
+#define __NR_link 9
+#define __NR_unlink 10
+#define __NR_execve 11
+#define __NR_chdir 12
+#define __NR_time 13
+#define __NR_mknod 14
+#define __NR_chmod 15
+#define __NR_lchown 16
+#define __NR_break 17
+#define __NR_oldstat 18
+#define __NR_lseek 19
+#define __NR_getpid 20
+#define __NR_mount 21
+#define __NR_umount 22
+#define __NR_setuid 23
+#define __NR_getuid 24
+#define __NR_stime 25
+#define __NR_ptrace 26
+#define __NR_alarm 27
+#define __NR_oldfstat 28
+#define __NR_pause 29
+#define __NR_utime 30
+#define __NR_stty 31
+#define __NR_gtty 32
+#define __NR_access 33
+#define __NR_nice 34
+#define __NR_ftime 35
+#define __NR_sync 36
+#define __NR_kill 37
+#define __NR_rename 38
+#define __NR_mkdir 39
+#define __NR_rmdir 40
+#define __NR_dup 41
+#define __NR_pipe 42
+#define __NR_times 43
+#define __NR_prof 44
+#define __NR_brk 45
+#define __NR_setgid 46
+#define __NR_getgid 47
+#define __NR_signal 48
+#define __NR_geteuid 49
+#define __NR_getegid 50
+#define __NR_acct 51
+#define __NR_umount2 52
+#define __NR_lock 53
+#define __NR_ioctl 54
+#define __NR_fcntl 55
+#define __NR_mpx 56
+#define __NR_setpgid 57
+#define __NR_ulimit 58
+#define __NR_oldolduname 59
+#define __NR_umask 60
+#define __NR_chroot 61
+#define __NR_ustat 62
+#define __NR_dup2 63
+#define __NR_getppid 64
+#define __NR_getpgrp 65
+#define __NR_setsid 66
+#define __NR_sigaction 67
+#define __NR_sgetmask 68
+#define __NR_ssetmask 69
+#define __NR_setreuid 70
+#define __NR_setregid 71
+#define __NR_sigsuspend 72
+#define __NR_sigpending 73
+#define __NR_sethostname 74
+#define __NR_setrlimit 75
+#define __NR_getrlimit 76
+#define __NR_getrusage 77
+#define __NR_gettimeofday 78
+#define __NR_settimeofday 79
+#define __NR_getgroups 80
+#define __NR_setgroups 81
+#define __NR_select 82
+#define __NR_symlink 83
+#define __NR_oldlstat 84
+#define __NR_readlink 85
+#define __NR_uselib 86
+#define __NR_swapon 87
+#define __NR_reboot 88
+#define __NR_readdir 89
+#define __NR_mmap 90
+#define __NR_munmap 91
+#define __NR_truncate 92
+#define __NR_ftruncate 93
+#define __NR_fchmod 94
+#define __NR_fchown 95
+#define __NR_getpriority 96
+#define __NR_setpriority 97
+#define __NR_profil 98
+#define __NR_statfs 99
+#define __NR_fstatfs 100
+#define __NR_ioperm 101
+#define __NR_socketcall 102
+#define __NR_syslog 103
+#define __NR_setitimer 104
+#define __NR_getitimer 105
+#define __NR_stat 106
+#define __NR_lstat 107
+#define __NR_fstat 108
+#define __NR_olduname 109
+#define __NR_iopl 110
+#define __NR_vhangup 111
+#define __NR_idle 112
+#define __NR_vm86old 113
+#define __NR_wait4 114
+#define __NR_swapoff 115
+#define __NR_sysinfo 116
+#define __NR_ipc 117
+#define __NR_fsync 118
+#define __NR_sigreturn 119
+#define __NR_clone 120
+#define __NR_setdomainname 121
+#define __NR_uname 122
+#define __NR_modify_ldt 123
+#define __NR_adjtimex 124
+#define __NR_mprotect 125
+#define __NR_sigprocmask 126
+#define __NR_create_module 127
+#define __NR_init_module 128
+#define __NR_delete_module 129
+#define __NR_get_kernel_syms 130
+#define __NR_quotactl 131
+#define __NR_getpgid 132
+#define __NR_fchdir 133
+#define __NR_bdflush 134
+#define __NR_sysfs 135
+#define __NR_personality 136
+#define __NR_afs_syscall 137
+#define __NR_setfsuid 138
+#define __NR_setfsgid 139
+#define __NR__llseek 140
+#define __NR_getdents 141
+#define __NR__newselect 142
+#define __NR_flock 143
+#define __NR_msync 144
+#define __NR_readv 145
+#define __NR_writev 146
+#define __NR_getsid 147
+#define __NR_fdatasync 148
+#define __NR__sysctl 149
+#define __NR_mlock 150
+#define __NR_munlock 151
+#define __NR_mlockall 152
+#define __NR_munlockall 153
+#define __NR_sched_setparam 154
+#define __NR_sched_getparam 155
+#define __NR_sched_setscheduler 156
+#define __NR_sched_getscheduler 157
+#define __NR_sched_yield 158
+#define __NR_sched_get_priority_max 159
+#define __NR_sched_get_priority_min 160
+#define __NR_sched_rr_get_interval 161
+#define __NR_nanosleep 162
+#define __NR_mremap 163
+#define __NR_setresuid 164
+#define __NR_getresuid 165
+#define __NR_vm86 166
+#define __NR_query_module 167
+#define __NR_poll 168
+#define __NR_nfsservctl 169
+#define __NR_setresgid 170
+#define __NR_getresgid 171
+#define __NR_prctl 172
+#define __NR_rt_sigreturn 173
+#define __NR_rt_sigaction 174
+#define __NR_rt_sigprocmask 175
+#define __NR_rt_sigpending 176
+#define __NR_rt_sigtimedwait 177
+#define __NR_rt_sigqueueinfo 178
+#define __NR_rt_sigsuspend 179
+#define __NR_pread64 180
+#define __NR_pwrite64 181
+#define __NR_chown 182
+#define __NR_getcwd 183
+#define __NR_capget 184
+#define __NR_capset 185
+#define __NR_sigaltstack 186
+#define __NR_sendfile 187
+#define __NR_getpmsg 188
+#define __NR_putpmsg 189
+#define __NR_vfork 190
+#define __NR_ugetrlimit 191
+#define __NR_mmap2 192
+#define __NR_truncate64 193
+#define __NR_ftruncate64 194
+#define __NR_stat64 195
+#define __NR_lstat64 196
+#define __NR_fstat64 197
+#define __NR_lchown32 198
+#define __NR_getuid32 199
+#define __NR_getgid32 200
+#define __NR_geteuid32 201
+#define __NR_getegid32 202
+#define __NR_setreuid32 203
+#define __NR_setregid32 204
+#define __NR_getgroups32 205
+#define __NR_setgroups32 206
+#define __NR_fchown32 207
+#define __NR_setresuid32 208
+#define __NR_getresuid32 209
+#define __NR_setresgid32 210
+#define __NR_getresgid32 211
+#define __NR_chown32 212
+#define __NR_setuid32 213
+#define __NR_setgid32 214
+#define __NR_setfsuid32 215
+#define __NR_setfsgid32 216
+#define __NR_pivot_root 217
+#define __NR_mincore 218
+#define __NR_madvise 219
+#define __NR_getdents64 220
+#define __NR_fcntl64 221
+#define __NR_gettid 224
+#define __NR_readahead 225
+#define __NR_setxattr 226
+#define __NR_lsetxattr 227
+#define __NR_fsetxattr 228
+#define __NR_getxattr 229
+#define __NR_lgetxattr 230
+#define __NR_fgetxattr 231
+#define __NR_listxattr 232
+#define __NR_llistxattr 233
+#define __NR_flistxattr 234
+#define __NR_removexattr 235
+#define __NR_lremovexattr 236
+#define __NR_fremovexattr 237
+#define __NR_tkill 238
+#define __NR_sendfile64 239
+#define __NR_futex 240
+#define __NR_sched_setaffinity 241
+#define __NR_sched_getaffinity 242
+#define __NR_set_thread_area 243
+#define __NR_get_thread_area 244
+#define __NR_io_setup 245
+#define __NR_io_destroy 246
+#define __NR_io_getevents 247
+#define __NR_io_submit 248
+#define __NR_io_cancel 249
+#define __NR_fadvise64 250
+#define __NR_exit_group 252
+#define __NR_lookup_dcookie 253
+#define __NR_epoll_create 254
+#define __NR_epoll_ctl 255
+#define __NR_epoll_wait 256
+#define __NR_remap_file_pages 257
+#define __NR_set_tid_address 258
+#define __NR_timer_create 259
+#define __NR_timer_settime 260
+#define __NR_timer_gettime 261
+#define __NR_timer_getoverrun 262
+#define __NR_timer_delete 263
+#define __NR_clock_settime 264
+#define __NR_clock_gettime 265
+#define __NR_clock_getres 266
+#define __NR_clock_nanosleep 267
+#define __NR_statfs64 268
+#define __NR_fstatfs64 269
+#define __NR_tgkill 270
+#define __NR_utimes 271
+#define __NR_fadvise64_64 272
+#define __NR_vserver 273
+#define __NR_mbind 274
+#define __NR_get_mempolicy 275
+#define __NR_set_mempolicy 276
+#define __NR_mq_open 277
+#define __NR_mq_unlink 278
+#define __NR_mq_timedsend 279
+#define __NR_mq_timedreceive 280
+#define __NR_mq_notify 281
+#define __NR_mq_getsetattr 282
+#define __NR_kexec_load 283
+#define __NR_waitid 284
+#define __NR_add_key 286
+#define __NR_request_key 287
+#define __NR_keyctl 288
+#define __NR_ioprio_set 289
+#define __NR_ioprio_get 290
+#define __NR_inotify_init 291
+#define __NR_inotify_add_watch 292
+#define __NR_inotify_rm_watch 293
+#define __NR_migrate_pages 294
+#define __NR_openat 295
+#define __NR_mkdirat 296
+#define __NR_mknodat 297
+#define __NR_fchownat 298
+#define __NR_futimesat 299
+#define __NR_fstatat64 300
+#define __NR_unlinkat 301
+#define __NR_renameat 302
+#define __NR_linkat 303
+#define __NR_symlinkat 304
+#define __NR_readlinkat 305
+#define __NR_fchmodat 306
+#define __NR_faccessat 307
+#define __NR_pselect6 308
+#define __NR_ppoll 309
+#define __NR_unshare 310
+#define __NR_set_robust_list 311
+#define __NR_get_robust_list 312
+#define __NR_splice 313
+#define __NR_sync_file_range 314
+#define __NR_tee 315
+#define __NR_vmsplice 316
+#define __NR_move_pages 317
+#define __NR_getcpu 318
+#define __NR_epoll_pwait 319
+#define __NR_utimensat 320
+#define __NR_signalfd 321
+#define __NR_timerfd_create 322
+#define __NR_eventfd 323
+#define __NR_fallocate 324
+#define __NR_timerfd_settime 325
+#define __NR_timerfd_gettime 326
+#define __NR_signalfd4 327
+#define __NR_eventfd2 328
+#define __NR_epoll_create1 329
+#define __NR_dup3 330
+#define __NR_pipe2 331
+#define __NR_inotify_init1 332
+#define __NR_preadv 333
+#define __NR_pwritev 334
+#define __NR_rt_tgsigqueueinfo 335
+#define __NR_perf_event_open 336
+#define __NR_recvmmsg 337
+#define __NR_fanotify_init 338
+#define __NR_fanotify_mark 339
+#define __NR_prlimit64 340
+#define __NR_name_to_handle_at 341
+#define __NR_open_by_handle_at 342
+#define __NR_clock_adjtime 343
+#define __NR_syncfs 344
+#define __NR_sendmmsg 345
+#define __NR_setns 346
+#define __NR_process_vm_readv 347
+#define __NR_process_vm_writev 348
+#define __NR_kcmp 349
+#define __NR_finit_module 350
+#define __NR_sched_setattr 351
+#define __NR_sched_getattr 352
+#define __NR_renameat2 353
+#define __NR_seccomp 354
+#define __NR_getrandom 355
+#define __NR_memfd_create 356
+#define __NR_bpf 357
+#define __NR_execveat 358
+#define __NR_socket 359
+#define __NR_socketpair 360
+#define __NR_bind 361
+#define __NR_connect 362
+#define __NR_listen 363
+#define __NR_accept4 364
+#define __NR_getsockopt 365
+#define __NR_setsockopt 366
+#define __NR_getsockname 367
+#define __NR_getpeername 368
+#define __NR_sendto 369
+#define __NR_sendmsg 370
+#define __NR_recvfrom 371
+#define __NR_recvmsg 372
+#define __NR_shutdown 373
+#define __NR_userfaultfd 374
+#define __NR_membarrier 375
+
+#endif /* _ASM_X86_UNISTD_32_H */
diff --git a/linux-headers/asm-x86/unistd_64.h b/linux-headers/asm-x86/unistd_64.h
new file mode 100644
index 000000000..fdc7afb05
--- /dev/null
+++ b/linux-headers/asm-x86/unistd_64.h
@@ -0,0 +1,330 @@
+#ifndef _ASM_X86_UNISTD_64_H
+#define _ASM_X86_UNISTD_64_H 1
+
+#define __NR_read 0
+#define __NR_write 1
+#define __NR_open 2
+#define __NR_close 3
+#define __NR_stat 4
+#define __NR_fstat 5
+#define __NR_lstat 6
+#define __NR_poll 7
+#define __NR_lseek 8
+#define __NR_mmap 9
+#define __NR_mprotect 10
+#define __NR_munmap 11
+#define __NR_brk 12
+#define __NR_rt_sigaction 13
+#define __NR_rt_sigprocmask 14
+#define __NR_rt_sigreturn 15
+#define __NR_ioctl 16
+#define __NR_pread64 17
+#define __NR_pwrite64 18
+#define __NR_readv 19
+#define __NR_writev 20
+#define __NR_access 21
+#define __NR_pipe 22
+#define __NR_select 23
+#define __NR_sched_yield 24
+#define __NR_mremap 25
+#define __NR_msync 26
+#define __NR_mincore 27
+#define __NR_madvise 28
+#define __NR_shmget 29
+#define __NR_shmat 30
+#define __NR_shmctl 31
+#define __NR_dup 32
+#define __NR_dup2 33
+#define __NR_pause 34
+#define __NR_nanosleep 35
+#define __NR_getitimer 36
+#define __NR_alarm 37
+#define __NR_setitimer 38
+#define __NR_getpid 39
+#define __NR_sendfile 40
+#define __NR_socket 41
+#define __NR_connect 42
+#define __NR_accept 43
+#define __NR_sendto 44
+#define __NR_recvfrom 45
+#define __NR_sendmsg 46
+#define __NR_recvmsg 47
+#define __NR_shutdown 48
+#define __NR_bind 49
+#define __NR_listen 50
+#define __NR_getsockname 51
+#define __NR_getpeername 52
+#define __NR_socketpair 53
+#define __NR_setsockopt 54
+#define __NR_getsockopt 55
+#define __NR_clone 56
+#define __NR_fork 57
+#define __NR_vfork 58
+#define __NR_execve 59
+#define __NR_exit 60
+#define __NR_wait4 61
+#define __NR_kill 62
+#define __NR_uname 63
+#define __NR_semget 64
+#define __NR_semop 65
+#define __NR_semctl 66
+#define __NR_shmdt 67
+#define __NR_msgget 68
+#define __NR_msgsnd 69
+#define __NR_msgrcv 70
+#define __NR_msgctl 71
+#define __NR_fcntl 72
+#define __NR_flock 73
+#define __NR_fsync 74
+#define __NR_fdatasync 75
+#define __NR_truncate 76
+#define __NR_ftruncate 77
+#define __NR_getdents 78
+#define __NR_getcwd 79
+#define __NR_chdir 80
+#define __NR_fchdir 81
+#define __NR_rename 82
+#define __NR_mkdir 83
+#define __NR_rmdir 84
+#define __NR_creat 85
+#define __NR_link 86
+#define __NR_unlink 87
+#define __NR_symlink 88
+#define __NR_readlink 89
+#define __NR_chmod 90
+#define __NR_fchmod 91
+#define __NR_chown 92
+#define __NR_fchown 93
+#define __NR_lchown 94
+#define __NR_umask 95
+#define __NR_gettimeofday 96
+#define __NR_getrlimit 97
+#define __NR_getrusage 98
+#define __NR_sysinfo 99
+#define __NR_times 100
+#define __NR_ptrace 101
+#define __NR_getuid 102
+#define __NR_syslog 103
+#define __NR_getgid 104
+#define __NR_setuid 105
+#define __NR_setgid 106
+#define __NR_geteuid 107
+#define __NR_getegid 108
+#define __NR_setpgid 109
+#define __NR_getppid 110
+#define __NR_getpgrp 111
+#define __NR_setsid 112
+#define __NR_setreuid 113
+#define __NR_setregid 114
+#define __NR_getgroups 115
+#define __NR_setgroups 116
+#define __NR_setresuid 117
+#define __NR_getresuid 118
+#define __NR_setresgid 119
+#define __NR_getresgid 120
+#define __NR_getpgid 121
+#define __NR_setfsuid 122
+#define __NR_setfsgid 123
+#define __NR_getsid 124
+#define __NR_capget 125
+#define __NR_capset 126
+#define __NR_rt_sigpending 127
+#define __NR_rt_sigtimedwait 128
+#define __NR_rt_sigqueueinfo 129
+#define __NR_rt_sigsuspend 130
+#define __NR_sigaltstack 131
+#define __NR_utime 132
+#define __NR_mknod 133
+#define __NR_uselib 134
+#define __NR_personality 135
+#define __NR_ustat 136
+#define __NR_statfs 137
+#define __NR_fstatfs 138
+#define __NR_sysfs 139
+#define __NR_getpriority 140
+#define __NR_setpriority 141
+#define __NR_sched_setparam 142
+#define __NR_sched_getparam 143
+#define __NR_sched_setscheduler 144
+#define __NR_sched_getscheduler 145
+#define __NR_sched_get_priority_max 146
+#define __NR_sched_get_priority_min 147
+#define __NR_sched_rr_get_interval 148
+#define __NR_mlock 149
+#define __NR_munlock 150
+#define __NR_mlockall 151
+#define __NR_munlockall 152
+#define __NR_vhangup 153
+#define __NR_modify_ldt 154
+#define __NR_pivot_root 155
+#define __NR__sysctl 156
+#define __NR_prctl 157
+#define __NR_arch_prctl 158
+#define __NR_adjtimex 159
+#define __NR_setrlimit 160
+#define __NR_chroot 161
+#define __NR_sync 162
+#define __NR_acct 163
+#define __NR_settimeofday 164
+#define __NR_mount 165
+#define __NR_umount2 166
+#define __NR_swapon 167
+#define __NR_swapoff 168
+#define __NR_reboot 169
+#define __NR_sethostname 170
+#define __NR_setdomainname 171
+#define __NR_iopl 172
+#define __NR_ioperm 173
+#define __NR_create_module 174
+#define __NR_init_module 175
+#define __NR_delete_module 176
+#define __NR_get_kernel_syms 177
+#define __NR_query_module 178
+#define __NR_quotactl 179
+#define __NR_nfsservctl 180
+#define __NR_getpmsg 181
+#define __NR_putpmsg 182
+#define __NR_afs_syscall 183
+#define __NR_tuxcall 184
+#define __NR_security 185
+#define __NR_gettid 186
+#define __NR_readahead 187
+#define __NR_setxattr 188
+#define __NR_lsetxattr 189
+#define __NR_fsetxattr 190
+#define __NR_getxattr 191
+#define __NR_lgetxattr 192
+#define __NR_fgetxattr 193
+#define __NR_listxattr 194
+#define __NR_llistxattr 195
+#define __NR_flistxattr 196
+#define __NR_removexattr 197
+#define __NR_lremovexattr 198
+#define __NR_fremovexattr 199
+#define __NR_tkill 200
+#define __NR_time 201
+#define __NR_futex 202
+#define __NR_sched_setaffinity 203
+#define __NR_sched_getaffinity 204
+#define __NR_set_thread_area 205
+#define __NR_io_setup 206
+#define __NR_io_destroy 207
+#define __NR_io_getevents 208
+#define __NR_io_submit 209
+#define __NR_io_cancel 210
+#define __NR_get_thread_area 211
+#define __NR_lookup_dcookie 212
+#define __NR_epoll_create 213
+#define __NR_epoll_ctl_old 214
+#define __NR_epoll_wait_old 215
+#define __NR_remap_file_pages 216
+#define __NR_getdents64 217
+#define __NR_set_tid_address 218
+#define __NR_restart_syscall 219
+#define __NR_semtimedop 220
+#define __NR_fadvise64 221
+#define __NR_timer_create 222
+#define __NR_timer_settime 223
+#define __NR_timer_gettime 224
+#define __NR_timer_getoverrun 225
+#define __NR_timer_delete 226
+#define __NR_clock_settime 227
+#define __NR_clock_gettime 228
+#define __NR_clock_getres 229
+#define __NR_clock_nanosleep 230
+#define __NR_exit_group 231
+#define __NR_epoll_wait 232
+#define __NR_epoll_ctl 233
+#define __NR_tgkill 234
+#define __NR_utimes 235
+#define __NR_vserver 236
+#define __NR_mbind 237
+#define __NR_set_mempolicy 238
+#define __NR_get_mempolicy 239
+#define __NR_mq_open 240
+#define __NR_mq_unlink 241
+#define __NR_mq_timedsend 242
+#define __NR_mq_timedreceive 243
+#define __NR_mq_notify 244
+#define __NR_mq_getsetattr 245
+#define __NR_kexec_load 246
+#define __NR_waitid 247
+#define __NR_add_key 248
+#define __NR_request_key 249
+#define __NR_keyctl 250
+#define __NR_ioprio_set 251
+#define __NR_ioprio_get 252
+#define __NR_inotify_init 253
+#define __NR_inotify_add_watch 254
+#define __NR_inotify_rm_watch 255
+#define __NR_migrate_pages 256
+#define __NR_openat 257
+#define __NR_mkdirat 258
+#define __NR_mknodat 259
+#define __NR_fchownat 260
+#define __NR_futimesat 261
+#define __NR_newfstatat 262
+#define __NR_unlinkat 263
+#define __NR_renameat 264
+#define __NR_linkat 265
+#define __NR_symlinkat 266
+#define __NR_readlinkat 267
+#define __NR_fchmodat 268
+#define __NR_faccessat 269
+#define __NR_pselect6 270
+#define __NR_ppoll 271
+#define __NR_unshare 272
+#define __NR_set_robust_list 273
+#define __NR_get_robust_list 274
+#define __NR_splice 275
+#define __NR_tee 276
+#define __NR_sync_file_range 277
+#define __NR_vmsplice 278
+#define __NR_move_pages 279
+#define __NR_utimensat 280
+#define __NR_epoll_pwait 281
+#define __NR_signalfd 282
+#define __NR_timerfd_create 283
+#define __NR_eventfd 284
+#define __NR_fallocate 285
+#define __NR_timerfd_settime 286
+#define __NR_timerfd_gettime 287
+#define __NR_accept4 288
+#define __NR_signalfd4 289
+#define __NR_eventfd2 290
+#define __NR_epoll_create1 291
+#define __NR_dup3 292
+#define __NR_pipe2 293
+#define __NR_inotify_init1 294
+#define __NR_preadv 295
+#define __NR_pwritev 296
+#define __NR_rt_tgsigqueueinfo 297
+#define __NR_perf_event_open 298
+#define __NR_recvmmsg 299
+#define __NR_fanotify_init 300
+#define __NR_fanotify_mark 301
+#define __NR_prlimit64 302
+#define __NR_name_to_handle_at 303
+#define __NR_open_by_handle_at 304
+#define __NR_clock_adjtime 305
+#define __NR_syncfs 306
+#define __NR_sendmmsg 307
+#define __NR_setns 308
+#define __NR_getcpu 309
+#define __NR_process_vm_readv 310
+#define __NR_process_vm_writev 311
+#define __NR_kcmp 312
+#define __NR_finit_module 313
+#define __NR_sched_setattr 314
+#define __NR_sched_getattr 315
+#define __NR_renameat2 316
+#define __NR_seccomp 317
+#define __NR_getrandom 318
+#define __NR_memfd_create 319
+#define __NR_kexec_file_load 320
+#define __NR_bpf 321
+#define __NR_execveat 322
+#define __NR_userfaultfd 323
+#define __NR_membarrier 324
+
+#endif /* _ASM_X86_UNISTD_64_H */
diff --git a/linux-headers/asm-x86/unistd_x32.h b/linux-headers/asm-x86/unistd_x32.h
new file mode 100644
index 000000000..ac6d1980b
--- /dev/null
+++ b/linux-headers/asm-x86/unistd_x32.h
@@ -0,0 +1,319 @@
+#ifndef _ASM_X86_UNISTD_X32_H
+#define _ASM_X86_UNISTD_X32_H 1
+
+#define __NR_read (__X32_SYSCALL_BIT + 0)
+#define __NR_write (__X32_SYSCALL_BIT + 1)
+#define __NR_open (__X32_SYSCALL_BIT + 2)
+#define __NR_close (__X32_SYSCALL_BIT + 3)
+#define __NR_stat (__X32_SYSCALL_BIT + 4)
+#define __NR_fstat (__X32_SYSCALL_BIT + 5)
+#define __NR_lstat (__X32_SYSCALL_BIT + 6)
+#define __NR_poll (__X32_SYSCALL_BIT + 7)
+#define __NR_lseek (__X32_SYSCALL_BIT + 8)
+#define __NR_mmap (__X32_SYSCALL_BIT + 9)
+#define __NR_mprotect (__X32_SYSCALL_BIT + 10)
+#define __NR_munmap (__X32_SYSCALL_BIT + 11)
+#define __NR_brk (__X32_SYSCALL_BIT + 12)
+#define __NR_rt_sigprocmask (__X32_SYSCALL_BIT + 14)
+#define __NR_pread64 (__X32_SYSCALL_BIT + 17)
+#define __NR_pwrite64 (__X32_SYSCALL_BIT + 18)
+#define __NR_access (__X32_SYSCALL_BIT + 21)
+#define __NR_pipe (__X32_SYSCALL_BIT + 22)
+#define __NR_select (__X32_SYSCALL_BIT + 23)
+#define __NR_sched_yield (__X32_SYSCALL_BIT + 24)
+#define __NR_mremap (__X32_SYSCALL_BIT + 25)
+#define __NR_msync (__X32_SYSCALL_BIT + 26)
+#define __NR_mincore (__X32_SYSCALL_BIT + 27)
+#define __NR_madvise (__X32_SYSCALL_BIT + 28)
+#define __NR_shmget (__X32_SYSCALL_BIT + 29)
+#define __NR_shmat (__X32_SYSCALL_BIT + 30)
+#define __NR_shmctl (__X32_SYSCALL_BIT + 31)
+#define __NR_dup (__X32_SYSCALL_BIT + 32)
+#define __NR_dup2 (__X32_SYSCALL_BIT + 33)
+#define __NR_pause (__X32_SYSCALL_BIT + 34)
+#define __NR_nanosleep (__X32_SYSCALL_BIT + 35)
+#define __NR_getitimer (__X32_SYSCALL_BIT + 36)
+#define __NR_alarm (__X32_SYSCALL_BIT + 37)
+#define __NR_setitimer (__X32_SYSCALL_BIT + 38)
+#define __NR_getpid (__X32_SYSCALL_BIT + 39)
+#define __NR_sendfile (__X32_SYSCALL_BIT + 40)
+#define __NR_socket (__X32_SYSCALL_BIT + 41)
+#define __NR_connect (__X32_SYSCALL_BIT + 42)
+#define __NR_accept (__X32_SYSCALL_BIT + 43)
+#define __NR_sendto (__X32_SYSCALL_BIT + 44)
+#define __NR_shutdown (__X32_SYSCALL_BIT + 48)
+#define __NR_bind (__X32_SYSCALL_BIT + 49)
+#define __NR_listen (__X32_SYSCALL_BIT + 50)
+#define __NR_getsockname (__X32_SYSCALL_BIT + 51)
+#define __NR_getpeername (__X32_SYSCALL_BIT + 52)
+#define __NR_socketpair (__X32_SYSCALL_BIT + 53)
+#define __NR_clone (__X32_SYSCALL_BIT + 56)
+#define __NR_fork (__X32_SYSCALL_BIT + 57)
+#define __NR_vfork (__X32_SYSCALL_BIT + 58)
+#define __NR_exit (__X32_SYSCALL_BIT + 60)
+#define __NR_wait4 (__X32_SYSCALL_BIT + 61)
+#define __NR_kill (__X32_SYSCALL_BIT + 62)
+#define __NR_uname (__X32_SYSCALL_BIT + 63)
+#define __NR_semget (__X32_SYSCALL_BIT + 64)
+#define __NR_semop (__X32_SYSCALL_BIT + 65)
+#define __NR_semctl (__X32_SYSCALL_BIT + 66)
+#define __NR_shmdt (__X32_SYSCALL_BIT + 67)
+#define __NR_msgget (__X32_SYSCALL_BIT + 68)
+#define __NR_msgsnd (__X32_SYSCALL_BIT + 69)
+#define __NR_msgrcv (__X32_SYSCALL_BIT + 70)
+#define __NR_msgctl (__X32_SYSCALL_BIT + 71)
+#define __NR_fcntl (__X32_SYSCALL_BIT + 72)
+#define __NR_flock (__X32_SYSCALL_BIT + 73)
+#define __NR_fsync (__X32_SYSCALL_BIT + 74)
+#define __NR_fdatasync (__X32_SYSCALL_BIT + 75)
+#define __NR_truncate (__X32_SYSCALL_BIT + 76)
+#define __NR_ftruncate (__X32_SYSCALL_BIT + 77)
+#define __NR_getdents (__X32_SYSCALL_BIT + 78)
+#define __NR_getcwd (__X32_SYSCALL_BIT + 79)
+#define __NR_chdir (__X32_SYSCALL_BIT + 80)
+#define __NR_fchdir (__X32_SYSCALL_BIT + 81)
+#define __NR_rename (__X32_SYSCALL_BIT + 82)
+#define __NR_mkdir (__X32_SYSCALL_BIT + 83)
+#define __NR_rmdir (__X32_SYSCALL_BIT + 84)
+#define __NR_creat (__X32_SYSCALL_BIT + 85)
+#define __NR_link (__X32_SYSCALL_BIT + 86)
+#define __NR_unlink (__X32_SYSCALL_BIT + 87)
+#define __NR_symlink (__X32_SYSCALL_BIT + 88)
+#define __NR_readlink (__X32_SYSCALL_BIT + 89)
+#define __NR_chmod (__X32_SYSCALL_BIT + 90)
+#define __NR_fchmod (__X32_SYSCALL_BIT + 91)
+#define __NR_chown (__X32_SYSCALL_BIT + 92)
+#define __NR_fchown (__X32_SYSCALL_BIT + 93)
+#define __NR_lchown (__X32_SYSCALL_BIT + 94)
+#define __NR_umask (__X32_SYSCALL_BIT + 95)
+#define __NR_gettimeofday (__X32_SYSCALL_BIT + 96)
+#define __NR_getrlimit (__X32_SYSCALL_BIT + 97)
+#define __NR_getrusage (__X32_SYSCALL_BIT + 98)
+#define __NR_sysinfo (__X32_SYSCALL_BIT + 99)
+#define __NR_times (__X32_SYSCALL_BIT + 100)
+#define __NR_getuid (__X32_SYSCALL_BIT + 102)
+#define __NR_syslog (__X32_SYSCALL_BIT + 103)
+#define __NR_getgid (__X32_SYSCALL_BIT + 104)
+#define __NR_setuid (__X32_SYSCALL_BIT + 105)
+#define __NR_setgid (__X32_SYSCALL_BIT + 106)
+#define __NR_geteuid (__X32_SYSCALL_BIT + 107)
+#define __NR_getegid (__X32_SYSCALL_BIT + 108)
+#define __NR_setpgid (__X32_SYSCALL_BIT + 109)
+#define __NR_getppid (__X32_SYSCALL_BIT + 110)
+#define __NR_getpgrp (__X32_SYSCALL_BIT + 111)
+#define __NR_setsid (__X32_SYSCALL_BIT + 112)
+#define __NR_setreuid (__X32_SYSCALL_BIT + 113)
+#define __NR_setregid (__X32_SYSCALL_BIT + 114)
+#define __NR_getgroups (__X32_SYSCALL_BIT + 115)
+#define __NR_setgroups (__X32_SYSCALL_BIT + 116)
+#define __NR_setresuid (__X32_SYSCALL_BIT + 117)
+#define __NR_getresuid (__X32_SYSCALL_BIT + 118)
+#define __NR_setresgid (__X32_SYSCALL_BIT + 119)
+#define __NR_getresgid (__X32_SYSCALL_BIT + 120)
+#define __NR_getpgid (__X32_SYSCALL_BIT + 121)
+#define __NR_setfsuid (__X32_SYSCALL_BIT + 122)
+#define __NR_setfsgid (__X32_SYSCALL_BIT + 123)
+#define __NR_getsid (__X32_SYSCALL_BIT + 124)
+#define __NR_capget (__X32_SYSCALL_BIT + 125)
+#define __NR_capset (__X32_SYSCALL_BIT + 126)
+#define __NR_rt_sigsuspend (__X32_SYSCALL_BIT + 130)
+#define __NR_utime (__X32_SYSCALL_BIT + 132)
+#define __NR_mknod (__X32_SYSCALL_BIT + 133)
+#define __NR_personality (__X32_SYSCALL_BIT + 135)
+#define __NR_ustat (__X32_SYSCALL_BIT + 136)
+#define __NR_statfs (__X32_SYSCALL_BIT + 137)
+#define __NR_fstatfs (__X32_SYSCALL_BIT + 138)
+#define __NR_sysfs (__X32_SYSCALL_BIT + 139)
+#define __NR_getpriority (__X32_SYSCALL_BIT + 140)
+#define __NR_setpriority (__X32_SYSCALL_BIT + 141)
+#define __NR_sched_setparam (__X32_SYSCALL_BIT + 142)
+#define __NR_sched_getparam (__X32_SYSCALL_BIT + 143)
+#define __NR_sched_setscheduler (__X32_SYSCALL_BIT + 144)
+#define __NR_sched_getscheduler (__X32_SYSCALL_BIT + 145)
+#define __NR_sched_get_priority_max (__X32_SYSCALL_BIT + 146)
+#define __NR_sched_get_priority_min (__X32_SYSCALL_BIT + 147)
+#define __NR_sched_rr_get_interval (__X32_SYSCALL_BIT + 148)
+#define __NR_mlock (__X32_SYSCALL_BIT + 149)
+#define __NR_munlock (__X32_SYSCALL_BIT + 150)
+#define __NR_mlockall (__X32_SYSCALL_BIT + 151)
+#define __NR_munlockall (__X32_SYSCALL_BIT + 152)
+#define __NR_vhangup (__X32_SYSCALL_BIT + 153)
+#define __NR_modify_ldt (__X32_SYSCALL_BIT + 154)
+#define __NR_pivot_root (__X32_SYSCALL_BIT + 155)
+#define __NR_prctl (__X32_SYSCALL_BIT + 157)
+#define __NR_arch_prctl (__X32_SYSCALL_BIT + 158)
+#define __NR_adjtimex (__X32_SYSCALL_BIT + 159)
+#define __NR_setrlimit (__X32_SYSCALL_BIT + 160)
+#define __NR_chroot (__X32_SYSCALL_BIT + 161)
+#define __NR_sync (__X32_SYSCALL_BIT + 162)
+#define __NR_acct (__X32_SYSCALL_BIT + 163)
+#define __NR_settimeofday (__X32_SYSCALL_BIT + 164)
+#define __NR_mount (__X32_SYSCALL_BIT + 165)
+#define __NR_umount2 (__X32_SYSCALL_BIT + 166)
+#define __NR_swapon (__X32_SYSCALL_BIT + 167)
+#define __NR_swapoff (__X32_SYSCALL_BIT + 168)
+#define __NR_reboot (__X32_SYSCALL_BIT + 169)
+#define __NR_sethostname (__X32_SYSCALL_BIT + 170)
+#define __NR_setdomainname (__X32_SYSCALL_BIT + 171)
+#define __NR_iopl (__X32_SYSCALL_BIT + 172)
+#define __NR_ioperm (__X32_SYSCALL_BIT + 173)
+#define __NR_init_module (__X32_SYSCALL_BIT + 175)
+#define __NR_delete_module (__X32_SYSCALL_BIT + 176)
+#define __NR_quotactl (__X32_SYSCALL_BIT + 179)
+#define __NR_getpmsg (__X32_SYSCALL_BIT + 181)
+#define __NR_putpmsg (__X32_SYSCALL_BIT + 182)
+#define __NR_afs_syscall (__X32_SYSCALL_BIT + 183)
+#define __NR_tuxcall (__X32_SYSCALL_BIT + 184)
+#define __NR_security (__X32_SYSCALL_BIT + 185)
+#define __NR_gettid (__X32_SYSCALL_BIT + 186)
+#define __NR_readahead (__X32_SYSCALL_BIT + 187)
+#define __NR_setxattr (__X32_SYSCALL_BIT + 188)
+#define __NR_lsetxattr (__X32_SYSCALL_BIT + 189)
+#define __NR_fsetxattr (__X32_SYSCALL_BIT + 190)
+#define __NR_getxattr (__X32_SYSCALL_BIT + 191)
+#define __NR_lgetxattr (__X32_SYSCALL_BIT + 192)
+#define __NR_fgetxattr (__X32_SYSCALL_BIT + 193)
+#define __NR_listxattr (__X32_SYSCALL_BIT + 194)
+#define __NR_llistxattr (__X32_SYSCALL_BIT + 195)
+#define __NR_flistxattr (__X32_SYSCALL_BIT + 196)
+#define __NR_removexattr (__X32_SYSCALL_BIT + 197)
+#define __NR_lremovexattr (__X32_SYSCALL_BIT + 198)
+#define __NR_fremovexattr (__X32_SYSCALL_BIT + 199)
+#define __NR_tkill (__X32_SYSCALL_BIT + 200)
+#define __NR_time (__X32_SYSCALL_BIT + 201)
+#define __NR_futex (__X32_SYSCALL_BIT + 202)
+#define __NR_sched_setaffinity (__X32_SYSCALL_BIT + 203)
+#define __NR_sched_getaffinity (__X32_SYSCALL_BIT + 204)
+#define __NR_io_destroy (__X32_SYSCALL_BIT + 207)
+#define __NR_io_getevents (__X32_SYSCALL_BIT + 208)
+#define __NR_io_cancel (__X32_SYSCALL_BIT + 210)
+#define __NR_lookup_dcookie (__X32_SYSCALL_BIT + 212)
+#define __NR_epoll_create (__X32_SYSCALL_BIT + 213)
+#define __NR_remap_file_pages (__X32_SYSCALL_BIT + 216)
+#define __NR_getdents64 (__X32_SYSCALL_BIT + 217)
+#define __NR_set_tid_address (__X32_SYSCALL_BIT + 218)
+#define __NR_restart_syscall (__X32_SYSCALL_BIT + 219)
+#define __NR_semtimedop (__X32_SYSCALL_BIT + 220)
+#define __NR_fadvise64 (__X32_SYSCALL_BIT + 221)
+#define __NR_timer_settime (__X32_SYSCALL_BIT + 223)
+#define __NR_timer_gettime (__X32_SYSCALL_BIT + 224)
+#define __NR_timer_getoverrun (__X32_SYSCALL_BIT + 225)
+#define __NR_timer_delete (__X32_SYSCALL_BIT + 226)
+#define __NR_clock_settime (__X32_SYSCALL_BIT + 227)
+#define __NR_clock_gettime (__X32_SYSCALL_BIT + 228)
+#define __NR_clock_getres (__X32_SYSCALL_BIT + 229)
+#define __NR_clock_nanosleep (__X32_SYSCALL_BIT + 230)
+#define __NR_exit_group (__X32_SYSCALL_BIT + 231)
+#define __NR_epoll_wait (__X32_SYSCALL_BIT + 232)
+#define __NR_epoll_ctl (__X32_SYSCALL_BIT + 233)
+#define __NR_tgkill (__X32_SYSCALL_BIT + 234)
+#define __NR_utimes (__X32_SYSCALL_BIT + 235)
+#define __NR_mbind (__X32_SYSCALL_BIT + 237)
+#define __NR_set_mempolicy (__X32_SYSCALL_BIT + 238)
+#define __NR_get_mempolicy (__X32_SYSCALL_BIT + 239)
+#define __NR_mq_open (__X32_SYSCALL_BIT + 240)
+#define __NR_mq_unlink (__X32_SYSCALL_BIT + 241)
+#define __NR_mq_timedsend (__X32_SYSCALL_BIT + 242)
+#define __NR_mq_timedreceive (__X32_SYSCALL_BIT + 243)
+#define __NR_mq_getsetattr (__X32_SYSCALL_BIT + 245)
+#define __NR_add_key (__X32_SYSCALL_BIT + 248)
+#define __NR_request_key (__X32_SYSCALL_BIT + 249)
+#define __NR_keyctl (__X32_SYSCALL_BIT + 250)
+#define __NR_ioprio_set (__X32_SYSCALL_BIT + 251)
+#define __NR_ioprio_get (__X32_SYSCALL_BIT + 252)
+#define __NR_inotify_init (__X32_SYSCALL_BIT + 253)
+#define __NR_inotify_add_watch (__X32_SYSCALL_BIT + 254)
+#define __NR_inotify_rm_watch (__X32_SYSCALL_BIT + 255)
+#define __NR_migrate_pages (__X32_SYSCALL_BIT + 256)
+#define __NR_openat (__X32_SYSCALL_BIT + 257)
+#define __NR_mkdirat (__X32_SYSCALL_BIT + 258)
+#define __NR_mknodat (__X32_SYSCALL_BIT + 259)
+#define __NR_fchownat (__X32_SYSCALL_BIT + 260)
+#define __NR_futimesat (__X32_SYSCALL_BIT + 261)
+#define __NR_newfstatat (__X32_SYSCALL_BIT + 262)
+#define __NR_unlinkat (__X32_SYSCALL_BIT + 263)
+#define __NR_renameat (__X32_SYSCALL_BIT + 264)
+#define __NR_linkat (__X32_SYSCALL_BIT + 265)
+#define __NR_symlinkat (__X32_SYSCALL_BIT + 266)
+#define __NR_readlinkat (__X32_SYSCALL_BIT + 267)
+#define __NR_fchmodat (__X32_SYSCALL_BIT + 268)
+#define __NR_faccessat (__X32_SYSCALL_BIT + 269)
+#define __NR_pselect6 (__X32_SYSCALL_BIT + 270)
+#define __NR_ppoll (__X32_SYSCALL_BIT + 271)
+#define __NR_unshare (__X32_SYSCALL_BIT + 272)
+#define __NR_splice (__X32_SYSCALL_BIT + 275)
+#define __NR_tee (__X32_SYSCALL_BIT + 276)
+#define __NR_sync_file_range (__X32_SYSCALL_BIT + 277)
+#define __NR_utimensat (__X32_SYSCALL_BIT + 280)
+#define __NR_epoll_pwait (__X32_SYSCALL_BIT + 281)
+#define __NR_signalfd (__X32_SYSCALL_BIT + 282)
+#define __NR_timerfd_create (__X32_SYSCALL_BIT + 283)
+#define __NR_eventfd (__X32_SYSCALL_BIT + 284)
+#define __NR_fallocate (__X32_SYSCALL_BIT + 285)
+#define __NR_timerfd_settime (__X32_SYSCALL_BIT + 286)
+#define __NR_timerfd_gettime (__X32_SYSCALL_BIT + 287)
+#define __NR_accept4 (__X32_SYSCALL_BIT + 288)
+#define __NR_signalfd4 (__X32_SYSCALL_BIT + 289)
+#define __NR_eventfd2 (__X32_SYSCALL_BIT + 290)
+#define __NR_epoll_create1 (__X32_SYSCALL_BIT + 291)
+#define __NR_dup3 (__X32_SYSCALL_BIT + 292)
+#define __NR_pipe2 (__X32_SYSCALL_BIT + 293)
+#define __NR_inotify_init1 (__X32_SYSCALL_BIT + 294)
+#define __NR_perf_event_open (__X32_SYSCALL_BIT + 298)
+#define __NR_fanotify_init (__X32_SYSCALL_BIT + 300)
+#define __NR_fanotify_mark (__X32_SYSCALL_BIT + 301)
+#define __NR_prlimit64 (__X32_SYSCALL_BIT + 302)
+#define __NR_name_to_handle_at (__X32_SYSCALL_BIT + 303)
+#define __NR_open_by_handle_at (__X32_SYSCALL_BIT + 304)
+#define __NR_clock_adjtime (__X32_SYSCALL_BIT + 305)
+#define __NR_syncfs (__X32_SYSCALL_BIT + 306)
+#define __NR_setns (__X32_SYSCALL_BIT + 308)
+#define __NR_getcpu (__X32_SYSCALL_BIT + 309)
+#define __NR_kcmp (__X32_SYSCALL_BIT + 312)
+#define __NR_finit_module (__X32_SYSCALL_BIT + 313)
+#define __NR_sched_setattr (__X32_SYSCALL_BIT + 314)
+#define __NR_sched_getattr (__X32_SYSCALL_BIT + 315)
+#define __NR_renameat2 (__X32_SYSCALL_BIT + 316)
+#define __NR_seccomp (__X32_SYSCALL_BIT + 317)
+#define __NR_getrandom (__X32_SYSCALL_BIT + 318)
+#define __NR_memfd_create (__X32_SYSCALL_BIT + 319)
+#define __NR_kexec_file_load (__X32_SYSCALL_BIT + 320)
+#define __NR_bpf (__X32_SYSCALL_BIT + 321)
+#define __NR_userfaultfd (__X32_SYSCALL_BIT + 323)
+#define __NR_membarrier (__X32_SYSCALL_BIT + 324)
+#define __NR_rt_sigaction (__X32_SYSCALL_BIT + 512)
+#define __NR_rt_sigreturn (__X32_SYSCALL_BIT + 513)
+#define __NR_ioctl (__X32_SYSCALL_BIT + 514)
+#define __NR_readv (__X32_SYSCALL_BIT + 515)
+#define __NR_writev (__X32_SYSCALL_BIT + 516)
+#define __NR_recvfrom (__X32_SYSCALL_BIT + 517)
+#define __NR_sendmsg (__X32_SYSCALL_BIT + 518)
+#define __NR_recvmsg (__X32_SYSCALL_BIT + 519)
+#define __NR_execve (__X32_SYSCALL_BIT + 520)
+#define __NR_ptrace (__X32_SYSCALL_BIT + 521)
+#define __NR_rt_sigpending (__X32_SYSCALL_BIT + 522)
+#define __NR_rt_sigtimedwait (__X32_SYSCALL_BIT + 523)
+#define __NR_rt_sigqueueinfo (__X32_SYSCALL_BIT + 524)
+#define __NR_sigaltstack (__X32_SYSCALL_BIT + 525)
+#define __NR_timer_create (__X32_SYSCALL_BIT + 526)
+#define __NR_mq_notify (__X32_SYSCALL_BIT + 527)
+#define __NR_kexec_load (__X32_SYSCALL_BIT + 528)
+#define __NR_waitid (__X32_SYSCALL_BIT + 529)
+#define __NR_set_robust_list (__X32_SYSCALL_BIT + 530)
+#define __NR_get_robust_list (__X32_SYSCALL_BIT + 531)
+#define __NR_vmsplice (__X32_SYSCALL_BIT + 532)
+#define __NR_move_pages (__X32_SYSCALL_BIT + 533)
+#define __NR_preadv (__X32_SYSCALL_BIT + 534)
+#define __NR_pwritev (__X32_SYSCALL_BIT + 535)
+#define __NR_rt_tgsigqueueinfo (__X32_SYSCALL_BIT + 536)
+#define __NR_recvmmsg (__X32_SYSCALL_BIT + 537)
+#define __NR_sendmmsg (__X32_SYSCALL_BIT + 538)
+#define __NR_process_vm_readv (__X32_SYSCALL_BIT + 539)
+#define __NR_process_vm_writev (__X32_SYSCALL_BIT + 540)
+#define __NR_setsockopt (__X32_SYSCALL_BIT + 541)
+#define __NR_getsockopt (__X32_SYSCALL_BIT + 542)
+#define __NR_io_setup (__X32_SYSCALL_BIT + 543)
+#define __NR_io_submit (__X32_SYSCALL_BIT + 544)
+#define __NR_execveat (__X32_SYSCALL_BIT + 545)
+
+#endif /* _ASM_X86_UNISTD_X32_H */
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 3bac8736d..dcc410efe 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -183,6 +183,7 @@ struct kvm_s390_skeys {
 #define KVM_EXIT_EPR              23
 #define KVM_EXIT_SYSTEM_EVENT     24
 #define KVM_EXIT_S390_STSI        25
+#define KVM_EXIT_IOAPIC_EOI       26
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -237,6 +238,7 @@ struct kvm_run {
 			__u32 count;
 			__u64 data_offset; /* relative to kvm_run start */
 		} io;
+		/* KVM_EXIT_DEBUG */
 		struct {
 			struct kvm_debug_exit_arch arch;
 		} debug;
@@ -285,6 +287,7 @@ struct kvm_run {
 			__u32 data;
 			__u8  is_write;
 		} dcr;
+		/* KVM_EXIT_INTERNAL_ERROR */
 		struct {
 			__u32 suberror;
 			/* Available with KVM_CAP_INTERNAL_ERROR_DATA: */
@@ -295,6 +298,7 @@ struct kvm_run {
 		struct {
 			__u64 gprs[32];
 		} osi;
+		/* KVM_EXIT_PAPR_HCALL */
 		struct {
 			__u64 nr;
 			__u64 ret;
@@ -317,6 +321,7 @@ struct kvm_run {
 		struct {
 #define KVM_SYSTEM_EVENT_SHUTDOWN       1
 #define KVM_SYSTEM_EVENT_RESET          2
+#define KVM_SYSTEM_EVENT_CRASH          3
 			__u32 type;
 			__u64 flags;
 		} system_event;
@@ -329,6 +334,10 @@ struct kvm_run {
 			__u8 sel1;
 			__u16 sel2;
 		} s390_stsi;
+		/* KVM_EXIT_IOAPIC_EOI */
+		struct {
+			__u8 vector;
+		} eoi;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -481,6 +490,7 @@ struct kvm_s390_psw {
 	 ((ai) << 26))
 #define KVM_S390_INT_IO_MIN		0x00000000u
 #define KVM_S390_INT_IO_MAX		0xfffdffffu
+#define KVM_S390_INT_IO_AI_MASK		0x04000000u
 
 
 struct kvm_s390_interrupt {
@@ -817,6 +827,10 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_DISABLE_QUIRKS 116
 #define KVM_CAP_X86_SMM 117
 #define KVM_CAP_MULTI_ADDRESS_SPACE 118
+#define KVM_CAP_GUEST_DEBUG_HW_BPS 119
+#define KVM_CAP_GUEST_DEBUG_HW_WPS 120
+#define KVM_CAP_SPLIT_IRQCHIP 121
+#define KVM_CAP_IOEVENTFD_ANY_LENGTH 122
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
diff --git a/linux-headers/linux/userfaultfd.h b/linux-headers/linux/userfaultfd.h
new file mode 100644
index 000000000..9057d7af3
--- /dev/null
+++ b/linux-headers/linux/userfaultfd.h
@@ -0,0 +1,167 @@
+/*
+ *  include/linux/userfaultfd.h
+ *
+ *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
+ *  Copyright (C) 2015  Red Hat, Inc.
+ *
+ */
+
+#ifndef _LINUX_USERFAULTFD_H
+#define _LINUX_USERFAULTFD_H
+
+#include <linux/types.h>
+
+#define UFFD_API ((__u64)0xAA)
+/*
+ * After implementing the respective features it will become:
+ * #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \
+ *			      UFFD_FEATURE_EVENT_FORK)
+ */
+#define UFFD_API_FEATURES (0)
+#define UFFD_API_IOCTLS				\
+	((__u64)1 << _UFFDIO_REGISTER |		\
+	 (__u64)1 << _UFFDIO_UNREGISTER |	\
+	 (__u64)1 << _UFFDIO_API)
+#define UFFD_API_RANGE_IOCTLS			\
+	((__u64)1 << _UFFDIO_WAKE |		\
+	 (__u64)1 << _UFFDIO_COPY |		\
+	 (__u64)1 << _UFFDIO_ZEROPAGE)
+
+/*
+ * Valid ioctl command number range with this API is from 0x00 to
+ * 0x3F.  UFFDIO_API is the fixed number, everything else can be
+ * changed by implementing a different UFFD_API. If sticking to the
+ * same UFFD_API more ioctl can be added and userland will be aware of
+ * which ioctl the running kernel implements through the ioctl command
+ * bitmask written by the UFFDIO_API.
+ */
+#define _UFFDIO_REGISTER		(0x00)
+#define _UFFDIO_UNREGISTER		(0x01)
+#define _UFFDIO_WAKE			(0x02)
+#define _UFFDIO_COPY			(0x03)
+#define _UFFDIO_ZEROPAGE		(0x04)
+#define _UFFDIO_API			(0x3F)
+
+/* userfaultfd ioctl ids */
+#define UFFDIO 0xAA
+#define UFFDIO_API		_IOWR(UFFDIO, _UFFDIO_API,	\
+				      struct uffdio_api)
+#define UFFDIO_REGISTER		_IOWR(UFFDIO, _UFFDIO_REGISTER, \
+				      struct uffdio_register)
+#define UFFDIO_UNREGISTER	_IOR(UFFDIO, _UFFDIO_UNREGISTER,	\
+				     struct uffdio_range)
+#define UFFDIO_WAKE		_IOR(UFFDIO, _UFFDIO_WAKE,	\
+				     struct uffdio_range)
+#define UFFDIO_COPY		_IOWR(UFFDIO, _UFFDIO_COPY,	\
+				      struct uffdio_copy)
+#define UFFDIO_ZEROPAGE		_IOWR(UFFDIO, _UFFDIO_ZEROPAGE,	\
+				      struct uffdio_zeropage)
+
+/* read() structure */
+struct uffd_msg {
+	__u8	event;
+
+	__u8	reserved1;
+	__u16	reserved2;
+	__u32	reserved3;
+
+	union {
+		struct {
+			__u64	flags;
+			__u64	address;
+		} pagefault;
+
+		struct {
+			/* unused reserved fields */
+			__u64	reserved1;
+			__u64	reserved2;
+			__u64	reserved3;
+		} reserved;
+	} arg;
+} __packed;
+
+/*
+ * Start at 0x12 and not at 0 to be more strict against bugs.
+ */
+#define UFFD_EVENT_PAGEFAULT	0x12
+#if 0 /* not available yet */
+#define UFFD_EVENT_FORK		0x13
+#endif
+
+/* flags for UFFD_EVENT_PAGEFAULT */
+#define UFFD_PAGEFAULT_FLAG_WRITE	(1<<0)	/* If this was a write fault */
+#define UFFD_PAGEFAULT_FLAG_WP		(1<<1)	/* If reason is VM_UFFD_WP */
+
+struct uffdio_api {
+	/* userland asks for an API number and the features to enable */
+	__u64 api;
+	/*
+	 * Kernel answers below with the all available features for
+	 * the API, this notifies userland of which events and/or
+	 * which flags for each event are enabled in the current
+	 * kernel.
+	 *
+	 * Note: UFFD_EVENT_PAGEFAULT and UFFD_PAGEFAULT_FLAG_WRITE
+	 * are to be considered implicitly always enabled in all kernels as
+	 * long as the uffdio_api.api requested matches UFFD_API.
+	 */
+#if 0 /* not available yet */
+#define UFFD_FEATURE_PAGEFAULT_FLAG_WP		(1<<0)
+#define UFFD_FEATURE_EVENT_FORK			(1<<1)
+#endif
+	__u64 features;
+
+	__u64 ioctls;
+};
+
+struct uffdio_range {
+	__u64 start;
+	__u64 len;
+};
+
+struct uffdio_register {
+	struct uffdio_range range;
+#define UFFDIO_REGISTER_MODE_MISSING	((__u64)1<<0)
+#define UFFDIO_REGISTER_MODE_WP		((__u64)1<<1)
+	__u64 mode;
+
+	/*
+	 * kernel answers which ioctl commands are available for the
+	 * range, keep at the end as the last 8 bytes aren't read.
+	 */
+	__u64 ioctls;
+};
+
+struct uffdio_copy {
+	__u64 dst;
+	__u64 src;
+	__u64 len;
+	/*
+	 * There will be a wrprotection flag later that allows to map
+	 * pages wrprotected on the fly. And such a flag will be
+	 * available if the wrprotection ioctl are implemented for the
+	 * range according to the uffdio_register.ioctls.
+	 */
+#define UFFDIO_COPY_MODE_DONTWAKE		((__u64)1<<0)
+	__u64 mode;
+
+	/*
+	 * "copy" is written by the ioctl and must be at the end: the
+	 * copy_from_user will not read the last 8 bytes.
+	 */
+	__s64 copy;
+};
+
+struct uffdio_zeropage {
+	struct uffdio_range range;
+#define UFFDIO_ZEROPAGE_MODE_DONTWAKE		((__u64)1<<0)
+	__u64 mode;
+
+	/*
+	 * "zeropage" is written by the ioctl and must be at the end:
+	 * the copy_from_user will not read the last 8 bytes.
+	 */
+	__s64 zeropage;
+};
+
+#endif /* _LINUX_USERFAULTFD_H */
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 17883686f..8b17c0e94 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -145,7 +145,6 @@ static uint32_t get_elf_hwcap(void)
 
 #ifdef TARGET_X86_64
 #define ELF_START_MMAP 0x2aaaaab000ULL
-#define elf_check_arch(x) ( ((x) == ELF_ARCH) )
 
 #define ELF_CLASS      ELFCLASS64
 #define ELF_ARCH       EM_X86_64
@@ -273,9 +272,7 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUX86State *en
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ((x) == ELF_MACHINE)
-
-#define ELF_ARCH        ELF_MACHINE
+#define ELF_ARCH        EM_ARM
 #define ELF_CLASS       ELFCLASS32
 
 static inline void init_thread(struct target_pt_regs *regs,
@@ -481,9 +478,7 @@ static uint32_t get_elf_hwcap2(void)
 /* 64 bit ARM definitions */
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ((x) == ELF_MACHINE)
-
-#define ELF_ARCH        ELF_MACHINE
+#define ELF_ARCH        EM_AARCH64
 #define ELF_CLASS       ELFCLASS64
 #define ELF_PLATFORM    "aarch64"
 
@@ -556,8 +551,6 @@ static uint32_t get_elf_hwcap(void)
 
 #define ELF_START_MMAP          0x80000000
 
-#define elf_check_arch(x)       ((x) == EM_UNICORE32)
-
 #define ELF_CLASS               ELFCLASS32
 #define ELF_DATA                ELFDATA2LSB
 #define ELF_ARCH                EM_UNICORE32
@@ -666,7 +659,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 #define ELF_START_MMAP 0x80000000
 #define ELF_HWCAP  (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | HWCAP_SPARC_SWAP \
                     | HWCAP_SPARC_MULDIV)
-#define elf_check_arch(x) ( (x) == EM_SPARC )
 
 #define ELF_CLASS   ELFCLASS32
 #define ELF_ARCH    EM_SPARC
@@ -686,6 +678,7 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #ifdef TARGET_PPC
 
+#define ELF_MACHINE    PPC_ELF_MACHINE
 #define ELF_START_MMAP 0x80000000
 
 #if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
@@ -696,8 +689,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #else
 
-#define elf_check_arch(x) ( (x) == EM_PPC )
-
 #define ELF_CLASS       ELFCLASS32
 
 #endif
@@ -875,8 +866,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUPPCState *en
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_MIPS )
-
 #ifdef TARGET_MIPS64
 #define ELF_CLASS   ELFCLASS64
 #else
@@ -985,8 +974,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUMBState *env
 
 #define ELF_START_MMAP 0x08000000
 
-#define elf_check_arch(x) ((x) == EM_OPENRISC)
-
 #define ELF_ARCH EM_OPENRISC
 #define ELF_CLASS ELFCLASS32
 #define ELF_DATA  ELFDATA2MSB
@@ -1026,8 +1013,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs,
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_SH )
-
 #define ELF_CLASS ELFCLASS32
 #define ELF_ARCH  EM_SH
 
@@ -1110,8 +1095,6 @@ static uint32_t get_elf_hwcap(void)
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_CRIS )
-
 #define ELF_CLASS ELFCLASS32
 #define ELF_ARCH  EM_CRIS
 
@@ -1129,8 +1112,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_68K )
-
 #define ELF_CLASS       ELFCLASS32
 #define ELF_ARCH        EM_68K
 
@@ -1182,8 +1163,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUM68KState *e
 
 #define ELF_START_MMAP (0x30000000000ULL)
 
-#define elf_check_arch(x) ( (x) == ELF_ARCH )
-
 #define ELF_CLASS      ELFCLASS64
 #define ELF_ARCH       EM_ALPHA
 
@@ -1203,8 +1182,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #define ELF_START_MMAP (0x20000000000ULL)
 
-#define elf_check_arch(x) ( (x) == ELF_ARCH )
-
 #define ELF_CLASS	ELFCLASS64
 #define ELF_DATA	ELFDATA2MSB
 #define ELF_ARCH	EM_S390
@@ -1218,10 +1195,41 @@ static inline void init_thread(struct target_pt_regs *regs, struct image_info *i
 
 #endif /* TARGET_S390X */
 
+#ifdef TARGET_TILEGX
+
+/* 42 bits real used address, a half for user mode */
+#define ELF_START_MMAP (0x00000020000000000ULL)
+
+#define elf_check_arch(x) ((x) == EM_TILEGX)
+
+#define ELF_CLASS   ELFCLASS64
+#define ELF_DATA    ELFDATA2LSB
+#define ELF_ARCH    EM_TILEGX
+
+static inline void init_thread(struct target_pt_regs *regs,
+                               struct image_info *infop)
+{
+    regs->pc = infop->entry;
+    regs->sp = infop->start_stack;
+
+}
+
+#define ELF_EXEC_PAGESIZE        65536 /* TILE-Gx page size is 64KB */
+
+#endif /* TARGET_TILEGX */
+
 #ifndef ELF_PLATFORM
 #define ELF_PLATFORM (NULL)
 #endif
 
+#ifndef ELF_MACHINE
+#define ELF_MACHINE ELF_ARCH
+#endif
+
+#ifndef elf_check_arch
+#define elf_check_arch(x) ((x) == ELF_ARCH)
+#endif
+
 #ifndef ELF_HWCAP
 #define ELF_HWCAP 0
 #endif
@@ -1365,66 +1373,69 @@ static bool elf_check_ehdr(struct elfhdr *ehdr)
  * to be put directly into the top of new user memory.
  *
  */
-static abi_ulong copy_elf_strings(int argc,char ** argv, void **page,
-                                  abi_ulong p)
+static abi_ulong copy_elf_strings(int argc, char **argv, char *scratch,
+                                  abi_ulong p, abi_ulong stack_limit)
 {
-    char *tmp, *tmp1, *pag = NULL;
-    int len, offset = 0;
+    char *tmp;
+    int len, offset;
+    abi_ulong top = p;
 
     if (!p) {
         return 0;       /* bullet-proofing */
     }
+
+    offset = ((p - 1) % TARGET_PAGE_SIZE) + 1;
+
     while (argc-- > 0) {
         tmp = argv[argc];
         if (!tmp) {
             fprintf(stderr, "VFS: argc is wrong");
             exit(-1);
         }
-        tmp1 = tmp;
-        while (*tmp++);
-        len = tmp - tmp1;
-        if (p < len) {  /* this shouldn't happen - 128kB */
+        len = strlen(tmp) + 1;
+        tmp += len;
+
+        if (len > (p - stack_limit)) {
             return 0;
         }
         while (len) {
-            --p; --tmp; --len;
-            if (--offset < 0) {
-                offset = p % TARGET_PAGE_SIZE;
-                pag = (char *)page[p/TARGET_PAGE_SIZE];
-                if (!pag) {
-                    pag = g_try_malloc0(TARGET_PAGE_SIZE);
-                    page[p/TARGET_PAGE_SIZE] = pag;
-                    if (!pag)
-                        return 0;
-                }
-            }
-            if (len == 0 || offset == 0) {
-                *(pag + offset) = *tmp;
-            }
-            else {
-                int bytes_to_copy = (len > offset) ? offset : len;
-                tmp -= bytes_to_copy;
-                p -= bytes_to_copy;
-                offset -= bytes_to_copy;
-                len -= bytes_to_copy;
-                memcpy_fromfs(pag + offset, tmp, bytes_to_copy + 1);
+            int bytes_to_copy = (len > offset) ? offset : len;
+            tmp -= bytes_to_copy;
+            p -= bytes_to_copy;
+            offset -= bytes_to_copy;
+            len -= bytes_to_copy;
+
+            memcpy_fromfs(scratch + offset, tmp, bytes_to_copy);
+
+            if (offset == 0) {
+                memcpy_to_target(p, scratch, top - p);
+                top = p;
+                offset = TARGET_PAGE_SIZE;
             }
         }
     }
+    if (offset) {
+        memcpy_to_target(p, scratch + offset, top - p);
+    }
+
     return p;
 }
 
-static abi_ulong setup_arg_pages(abi_ulong p, struct linux_binprm *bprm,
+/* Older linux kernels provide up to MAX_ARG_PAGES (default: 32) of
+ * argument/environment space. Newer kernels (>2.6.33) allow more,
+ * dependent on stack size, but guarantee at least 32 pages for
+ * backwards compatibility.
+ */
+#define STACK_LOWER_LIMIT (32 * TARGET_PAGE_SIZE)
+
+static abi_ulong setup_arg_pages(struct linux_binprm *bprm,
                                  struct image_info *info)
 {
-    abi_ulong stack_base, size, error, guard;
-    int i;
+    abi_ulong size, error, guard;
 
-    /* Create enough stack to hold everything.  If we don't use
-       it for args, we'll use it for something else.  */
     size = guest_stack_size;
-    if (size < MAX_ARG_PAGES*TARGET_PAGE_SIZE) {
-        size = MAX_ARG_PAGES*TARGET_PAGE_SIZE;
+    if (size < STACK_LOWER_LIMIT) {
+        size = STACK_LOWER_LIMIT;
     }
     guard = TARGET_PAGE_SIZE;
     if (guard < qemu_real_host_page_size) {
@@ -1442,19 +1453,8 @@ static abi_ulong setup_arg_pages(abi_ulong p, struct linux_binprm *bprm,
     target_mprotect(error, guard, PROT_NONE);
 
     info->stack_limit = error + guard;
-    stack_base = info->stack_limit + size - MAX_ARG_PAGES*TARGET_PAGE_SIZE;
-    p += stack_base;
-
-    for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
-        if (bprm->page[i]) {
-            info->rss++;
-            /* FIXME - check return value of memcpy_to_target() for failure */
-            memcpy_to_target(stack_base, bprm->page[i], TARGET_PAGE_SIZE);
-            g_free(bprm->page[i]);
-        }
-        stack_base += TARGET_PAGE_SIZE;
-    }
-    return p;
+
+    return info->stack_limit + size - sizeof(void *);
 }
 
 /* Map and zero the bss.  We need to explicitly zero any fractional pages
@@ -1478,8 +1478,7 @@ static void zero_bss(abi_ulong elf_bss, abi_ulong last_bss, int prot)
 
     host_start = (uintptr_t) g2h(elf_bss);
     host_end = (uintptr_t) g2h(last_bss);
-    host_map_start = (host_start + qemu_real_host_page_size - 1);
-    host_map_start &= -qemu_real_host_page_size;
+    host_map_start = REAL_HOST_PAGE_ALIGN(host_start);
 
     if (host_map_start < host_end) {
         void *p = mmap((void *)host_map_start, host_end - host_map_start,
@@ -1756,7 +1755,6 @@ static void probe_guest_base(const char *image_name,
      * it explicitly, and set guest_base appropriately.
      * In case of error we will print a suitable message and exit.
      */
-#if defined(CONFIG_USE_GUEST_BASE)
     const char *errmsg;
     if (!have_guest_base && !reserved_va) {
         unsigned long host_start, real_start, host_size;
@@ -1795,7 +1793,6 @@ static void probe_guest_base(const char *image_name,
 exit_errmsg:
     fprintf(stderr, "%s: %s\n", image_name, errmsg);
     exit(-1);
-#endif
 }
 
 
@@ -2198,10 +2195,9 @@ int load_elf_binary(struct linux_binprm *bprm, struct image_info *info)
     struct image_info interp_info;
     struct elfhdr elf_ex;
     char *elf_interpreter = NULL;
+    char *scratch;
 
     info->start_mmap = (abi_ulong)ELF_START_MMAP;
-    info->mmap = 0;
-    info->rss = 0;
 
     load_elf_image(bprm->filename, bprm->fd, info,
                    &elf_interpreter, bprm->buf);
@@ -2211,18 +2207,24 @@ int load_elf_binary(struct linux_binprm *bprm, struct image_info *info)
        when we load the interpreter.  */
     elf_ex = *(struct elfhdr *)bprm->buf;
 
-    bprm->p = copy_elf_strings(1, &bprm->filename, bprm->page, bprm->p);
-    bprm->p = copy_elf_strings(bprm->envc,bprm->envp,bprm->page,bprm->p);
-    bprm->p = copy_elf_strings(bprm->argc,bprm->argv,bprm->page,bprm->p);
+    /* Do this so that we can load the interpreter, if need be.  We will
+       change some of these later */
+    bprm->p = setup_arg_pages(bprm, info);
+
+    scratch = g_new0(char, TARGET_PAGE_SIZE);
+    bprm->p = copy_elf_strings(1, &bprm->filename, scratch,
+                               bprm->p, info->stack_limit);
+    bprm->p = copy_elf_strings(bprm->envc, bprm->envp, scratch,
+                               bprm->p, info->stack_limit);
+    bprm->p = copy_elf_strings(bprm->argc, bprm->argv, scratch,
+                               bprm->p, info->stack_limit);
+    g_free(scratch);
+
     if (!bprm->p) {
         fprintf(stderr, "%s: %s\n", bprm->filename, strerror(E2BIG));
         exit(-1);
     }
 
-    /* Do this so that we can load the interpreter, if need be.  We will
-       change some of these later */
-    bprm->p = setup_arg_pages(bprm->p, bprm, info);
-
     if (elf_interpreter) {
         load_elf_interp(elf_interpreter, &interp_info, bprm->buf);
 
@@ -2850,7 +2852,7 @@ static int fill_note_info(struct elf_note_info *info,
     TaskState *ts = (TaskState *)cpu->opaque;
     int i;
 
-    info->notes = g_malloc0(NUMNOTES * sizeof (struct memelfnote));
+    info->notes = g_new0(struct memelfnote, NUMNOTES);
     if (info->notes == NULL)
         return (-ENOMEM);
     info->prstatus = g_malloc0(sizeof (*info->prstatus));
@@ -3016,7 +3018,7 @@ static int elf_core_dump(int signr, const CPUArchState *env)
     if (dump_write(fd, &elf, sizeof (elf)) != 0)
         goto out;
 
-    /* fill in in-memory version of notes */
+    /* fill in the in-memory version of notes */
     if (fill_note_info(&info, signr, env) < 0)
         goto out;
 
diff --git a/linux-user/flatload.c b/linux-user/flatload.c
index 566a7a87a..ceacb9844 100644
--- a/linux-user/flatload.c
+++ b/linux-user/flatload.c
@@ -707,7 +707,7 @@ static int load_flat_shared_library(int id, struct lib_info *libs)
 int load_flt_binary(struct linux_binprm *bprm, struct image_info *info)
 {
     struct lib_info libinfo[MAX_SHARED_LIBS];
-    abi_ulong p = bprm->p;
+    abi_ulong p;
     abi_ulong stack_len;
     abi_ulong start_addr;
     abi_ulong sp;
diff --git a/linux-user/linuxload.c b/linux-user/linuxload.c
index 506e837ae..dbaf0ec58 100644
--- a/linux-user/linuxload.c
+++ b/linux-user/linuxload.c
@@ -135,10 +135,7 @@ int loader_exec(int fdexec, const char *filename, char **argv, char **envp,
              struct linux_binprm *bprm)
 {
     int retval;
-    int i;
 
-    bprm->p = TARGET_PAGE_SIZE*MAX_ARG_PAGES-sizeof(unsigned int);
-    memset(bprm->page, 0, sizeof(bprm->page));
     bprm->fd = fdexec;
     bprm->filename = (char *)filename;
     bprm->argc = count(argv);
@@ -172,9 +169,5 @@ int loader_exec(int fdexec, const char *filename, char **argv, char **envp,
         return retval;
     }
 
-    /* Something went wrong, return the inode and free the argument pages*/
-    for (i=0 ; i<MAX_ARG_PAGES ; i++) {
-        g_free(bprm->page[i]);
-    }
     return(retval);
 }
diff --git a/linux-user/main.c b/linux-user/main.c
index fdee98135..8acfe0fdf 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -37,13 +37,12 @@
 char *exec_path;
 
 int singlestep;
-const char *filename;
-const char *argv0;
-int gdbstub_port;
-envlist_t *envlist;
+static const char *filename;
+static const char *argv0;
+static int gdbstub_port;
+static envlist_t *envlist;
 static const char *cpu_model;
 unsigned long mmap_min_addr;
-#if defined(CONFIG_USE_GUEST_BASE)
 unsigned long guest_base;
 int have_guest_base;
 #if (TARGET_LONG_BITS == 32) && (HOST_LONG_BITS == 64)
@@ -63,9 +62,8 @@ unsigned long reserved_va = 0xf7000000;
 #else
 unsigned long reserved_va;
 #endif
-#endif
 
-static void usage(void);
+static void usage(int exitcode);
 
 static const char *interp_prefix = CONFIG_QEMU_INTERP_PREFIX;
 const char *qemu_uname_release;
@@ -107,7 +105,7 @@ static int pending_cpus;
 /* Make sure everything is in a consistent state for calling fork().  */
 void fork_start(void)
 {
-    pthread_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
+    qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
     pthread_mutex_lock(&exclusive_lock);
     mmap_fork_start();
 }
@@ -129,11 +127,11 @@ void fork_end(int child)
         pthread_mutex_init(&cpu_list_mutex, NULL);
         pthread_cond_init(&exclusive_cond, NULL);
         pthread_cond_init(&exclusive_resume, NULL);
-        pthread_mutex_init(&tcg_ctx.tb_ctx.tb_lock, NULL);
+        qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
         gdbserver_fork(thread_cpu);
     } else {
         pthread_mutex_unlock(&exclusive_lock);
-        pthread_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+        qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
     }
 }
 
@@ -217,7 +215,7 @@ void cpu_list_unlock(void)
 
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 static void write_dt(void *ptr, unsigned long addr, unsigned long limit,
@@ -1054,6 +1052,9 @@ void cpu_loop(CPUARMState *env)
                 queue_signal(env, info.si_signo, &info);
             }
             break;
+        case EXCP_SEMIHOST:
+            env->xregs[0] = do_arm_semihosting(env);
+            break;
         default:
             fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
                     trapnr);
@@ -1413,7 +1414,7 @@ void cpu_loop (CPUSPARCState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -1424,7 +1425,7 @@ void cpu_loop (CPUSPARCState *env)
 #ifdef TARGET_PPC
 static inline uint64_t cpu_ppc_get_tb(CPUPPCState *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 uint64_t cpu_ppc_load_tbl(CPUPPCState *env)
@@ -1649,7 +1650,7 @@ void cpu_loop(CPUPPCState *env)
             info.si_signo = TARGET_SIGBUS;
             info.si_errno = 0;
             info.si_code = TARGET_BUS_ADRALN;
-            info._sifields._sigfault._addr = env->nip - 4;
+            info._sifields._sigfault._addr = env->nip;
             queue_signal(env, info.si_signo, &info);
             break;
         case POWERPC_EXCP_PROGRAM:  /* Program exception                     */
@@ -2661,7 +2662,7 @@ void cpu_loop(CPUOpenRISCState *env)
         switch (trapnr) {
         case EXCP_RESET:
             qemu_log("\nReset request, exit, pc is %#x\n", env->pc);
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_BUSERR:
             qemu_log("\nBus error, exit, pc is %#x\n", env->pc);
@@ -2725,7 +2726,7 @@ void cpu_loop(CPUOpenRISCState *env)
         if (gdbsig) {
             gdb_handlesig(cs, gdbsig);
             if (gdbsig != TARGET_SIGTRAP) {
-                exit(1);
+                exit(EXIT_FAILURE);
             }
         }
 
@@ -2790,7 +2791,7 @@ void cpu_loop(CPUSH4State *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -2851,7 +2852,7 @@ void cpu_loop(CPUCRISState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -2932,7 +2933,7 @@ void cpu_loop(CPUMBState *env)
                     printf ("Unhandled hw-exception: 0x%x\n",
                             env->sregs[SR_ESR] & ESR_EC_MASK);
                     cpu_dump_state(cs, stderr, fprintf, 0);
-                    exit (1);
+                    exit(EXIT_FAILURE);
                     break;
             }
             break;
@@ -2953,7 +2954,7 @@ void cpu_loop(CPUMBState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3122,17 +3123,17 @@ void cpu_loop(CPUAlphaState *env)
         switch (trapnr) {
         case EXCP_RESET:
             fprintf(stderr, "Reset requested. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_MCHK:
             fprintf(stderr, "Machine check exception. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_SMP_INTERRUPT:
         case EXCP_CLK_INTERRUPT:
         case EXCP_DEV_INTERRUPT:
             fprintf(stderr, "External interrupt. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_MMFAULT:
             env->lock_addr = -1;
@@ -3282,7 +3283,7 @@ void cpu_loop(CPUAlphaState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3386,7 +3387,7 @@ void cpu_loop(CPUS390XState *env)
             default:
                 fprintf(stderr, "Unhandled program exception: %#x\n", n);
                 cpu_dump_state(cs, stderr, fprintf, 0);
-                exit(1);
+                exit(EXIT_FAILURE);
             }
             break;
 
@@ -3403,7 +3404,7 @@ void cpu_loop(CPUS390XState *env)
         default:
             fprintf(stderr, "Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3411,6 +3412,252 @@ void cpu_loop(CPUS390XState *env)
 
 #endif /* TARGET_S390X */
 
+#ifdef TARGET_TILEGX
+
+static void gen_sigill_reg(CPUTLGState *env)
+{
+    target_siginfo_t info;
+
+    info.si_signo = TARGET_SIGILL;
+    info.si_errno = 0;
+    info.si_code = TARGET_ILL_PRVREG;
+    info._sifields._sigfault._addr = env->pc;
+    queue_signal(env, info.si_signo, &info);
+}
+
+static void do_signal(CPUTLGState *env, int signo, int sigcode)
+{
+    target_siginfo_t info;
+
+    info.si_signo = signo;
+    info.si_errno = 0;
+    info._sifields._sigfault._addr = env->pc;
+
+    if (signo == TARGET_SIGSEGV) {
+        /* The passed in sigcode is a dummy; check for a page mapping
+           and pass either MAPERR or ACCERR.  */
+        target_ulong addr = env->excaddr;
+        info._sifields._sigfault._addr = addr;
+        if (page_check_range(addr, 1, PAGE_VALID) < 0) {
+            sigcode = TARGET_SEGV_MAPERR;
+        } else {
+            sigcode = TARGET_SEGV_ACCERR;
+        }
+    }
+    info.si_code = sigcode;
+
+    queue_signal(env, info.si_signo, &info);
+}
+
+static void gen_sigsegv_maperr(CPUTLGState *env, target_ulong addr)
+{
+    env->excaddr = addr;
+    do_signal(env, TARGET_SIGSEGV, 0);
+}
+
+static void set_regval(CPUTLGState *env, uint8_t reg, uint64_t val)
+{
+    if (unlikely(reg >= TILEGX_R_COUNT)) {
+        switch (reg) {
+        case TILEGX_R_SN:
+        case TILEGX_R_ZERO:
+            return;
+        case TILEGX_R_IDN0:
+        case TILEGX_R_IDN1:
+        case TILEGX_R_UDN0:
+        case TILEGX_R_UDN1:
+        case TILEGX_R_UDN2:
+        case TILEGX_R_UDN3:
+            gen_sigill_reg(env);
+            return;
+        default:
+            g_assert_not_reached();
+        }
+    }
+    env->regs[reg] = val;
+}
+
+/*
+ * Compare the 8-byte contents of the CmpValue SPR with the 8-byte value in
+ * memory at the address held in the first source register. If the values are
+ * not equal, then no memory operation is performed. If the values are equal,
+ * the 8-byte quantity from the second source register is written into memory
+ * at the address held in the first source register. In either case, the result
+ * of the instruction is the value read from memory. The compare and write to
+ * memory are atomic and thus can be used for synchronization purposes. This
+ * instruction only operates for addresses aligned to a 8-byte boundary.
+ * Unaligned memory access causes an Unaligned Data Reference interrupt.
+ *
+ * Functional Description (64-bit)
+ *       uint64_t memVal = memoryReadDoubleWord (rf[SrcA]);
+ *       rf[Dest] = memVal;
+ *       if (memVal == SPR[CmpValueSPR])
+ *           memoryWriteDoubleWord (rf[SrcA], rf[SrcB]);
+ *
+ * Functional Description (32-bit)
+ *       uint64_t memVal = signExtend32 (memoryReadWord (rf[SrcA]));
+ *       rf[Dest] = memVal;
+ *       if (memVal == signExtend32 (SPR[CmpValueSPR]))
+ *           memoryWriteWord (rf[SrcA], rf[SrcB]);
+ *
+ *
+ * This function also processes exch and exch4 which need not process SPR.
+ */
+static void do_exch(CPUTLGState *env, bool quad, bool cmp)
+{
+    target_ulong addr;
+    target_long val, sprval;
+
+    start_exclusive();
+
+    addr = env->atomic_srca;
+    if (quad ? get_user_s64(val, addr) : get_user_s32(val, addr)) {
+        goto sigsegv_maperr;
+    }
+
+    if (cmp) {
+        if (quad) {
+            sprval = env->spregs[TILEGX_SPR_CMPEXCH];
+        } else {
+            sprval = sextract64(env->spregs[TILEGX_SPR_CMPEXCH], 0, 32);
+        }
+    }
+
+    if (!cmp || val == sprval) {
+        target_long valb = env->atomic_srcb;
+        if (quad ? put_user_u64(valb, addr) : put_user_u32(valb, addr)) {
+            goto sigsegv_maperr;
+        }
+    }
+
+    set_regval(env, env->atomic_dstr, val);
+    end_exclusive();
+    return;
+
+ sigsegv_maperr:
+    end_exclusive();
+    gen_sigsegv_maperr(env, addr);
+}
+
+static void do_fetch(CPUTLGState *env, int trapnr, bool quad)
+{
+    int8_t write = 1;
+    target_ulong addr;
+    target_long val, valb;
+
+    start_exclusive();
+
+    addr = env->atomic_srca;
+    valb = env->atomic_srcb;
+    if (quad ? get_user_s64(val, addr) : get_user_s32(val, addr)) {
+        goto sigsegv_maperr;
+    }
+
+    switch (trapnr) {
+    case TILEGX_EXCP_OPCODE_FETCHADD:
+    case TILEGX_EXCP_OPCODE_FETCHADD4:
+        valb += val;
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHADDGEZ:
+        valb += val;
+        if (valb < 0) {
+            write = 0;
+        }
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHADDGEZ4:
+        valb += val;
+        if ((int32_t)valb < 0) {
+            write = 0;
+        }
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHAND:
+    case TILEGX_EXCP_OPCODE_FETCHAND4:
+        valb &= val;
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHOR:
+    case TILEGX_EXCP_OPCODE_FETCHOR4:
+        valb |= val;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if (write) {
+        if (quad ? put_user_u64(valb, addr) : put_user_u32(valb, addr)) {
+            goto sigsegv_maperr;
+        }
+    }
+
+    set_regval(env, env->atomic_dstr, val);
+    end_exclusive();
+    return;
+
+ sigsegv_maperr:
+    end_exclusive();
+    gen_sigsegv_maperr(env, addr);
+}
+
+void cpu_loop(CPUTLGState *env)
+{
+    CPUState *cs = CPU(tilegx_env_get_cpu(env));
+    int trapnr;
+
+    while (1) {
+        cpu_exec_start(cs);
+        trapnr = cpu_tilegx_exec(cs);
+        cpu_exec_end(cs);
+        switch (trapnr) {
+        case TILEGX_EXCP_SYSCALL:
+            env->regs[TILEGX_R_RE] = do_syscall(env, env->regs[TILEGX_R_NR],
+                                                env->regs[0], env->regs[1],
+                                                env->regs[2], env->regs[3],
+                                                env->regs[4], env->regs[5],
+                                                env->regs[6], env->regs[7]);
+            env->regs[TILEGX_R_ERR] = TILEGX_IS_ERRNO(env->regs[TILEGX_R_RE])
+                                                      ? - env->regs[TILEGX_R_RE]
+                                                      : 0;
+            break;
+        case TILEGX_EXCP_OPCODE_EXCH:
+            do_exch(env, true, false);
+            break;
+        case TILEGX_EXCP_OPCODE_EXCH4:
+            do_exch(env, false, false);
+            break;
+        case TILEGX_EXCP_OPCODE_CMPEXCH:
+            do_exch(env, true, true);
+            break;
+        case TILEGX_EXCP_OPCODE_CMPEXCH4:
+            do_exch(env, false, true);
+            break;
+        case TILEGX_EXCP_OPCODE_FETCHADD:
+        case TILEGX_EXCP_OPCODE_FETCHADDGEZ:
+        case TILEGX_EXCP_OPCODE_FETCHAND:
+        case TILEGX_EXCP_OPCODE_FETCHOR:
+            do_fetch(env, trapnr, true);
+            break;
+        case TILEGX_EXCP_OPCODE_FETCHADD4:
+        case TILEGX_EXCP_OPCODE_FETCHADDGEZ4:
+        case TILEGX_EXCP_OPCODE_FETCHAND4:
+        case TILEGX_EXCP_OPCODE_FETCHOR4:
+            do_fetch(env, trapnr, false);
+            break;
+        case TILEGX_EXCP_SIGNAL:
+            do_signal(env, env->signo, env->sigcode);
+            break;
+        case TILEGX_EXCP_REG_IDN_ACCESS:
+        case TILEGX_EXCP_REG_UDN_ACCESS:
+            gen_sigill_reg(env);
+            break;
+        default:
+            fprintf(stderr, "trapnr is %d[0x%x].\n", trapnr, trapnr);
+            g_assert_not_reached();
+        }
+        process_pending_signals(env);
+    }
+}
+
+#endif
+
 THREAD CPUState *thread_cpu;
 
 void task_settid(TaskState *ts)
@@ -3472,7 +3719,7 @@ CPUArchState *cpu_copy(CPUArchState *env)
 
 static void handle_arg_help(const char *arg)
 {
-    usage();
+    usage(EXIT_SUCCESS);
 }
 
 static void handle_arg_log(const char *arg)
@@ -3482,7 +3729,7 @@ static void handle_arg_log(const char *arg)
     mask = qemu_str_to_log_mask(arg);
     if (!mask) {
         qemu_print_log_usage(stdout);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     qemu_set_log(mask);
 }
@@ -3498,7 +3745,7 @@ static void handle_arg_set_env(const char *arg)
     r = p = strdup(arg);
     while ((token = strsep(&p, ",")) != NULL) {
         if (envlist_setenv(envlist, token) != 0) {
-            usage();
+            usage(EXIT_FAILURE);
         }
     }
     free(r);
@@ -3510,7 +3757,7 @@ static void handle_arg_unset_env(const char *arg)
     r = p = strdup(arg);
     while ((token = strsep(&p, ",")) != NULL) {
         if (envlist_unsetenv(envlist, token) != 0) {
-            usage();
+            usage(EXIT_FAILURE);
         }
     }
     free(r);
@@ -3526,7 +3773,7 @@ static void handle_arg_stack_size(const char *arg)
     char *p;
     guest_stack_size = strtoul(arg, &p, 0);
     if (guest_stack_size == 0) {
-        usage();
+        usage(EXIT_FAILURE);
     }
 
     if (*p == 'M') {
@@ -3547,7 +3794,7 @@ static void handle_arg_pagesize(const char *arg)
     if (qemu_host_page_size == 0 ||
         (qemu_host_page_size & (qemu_host_page_size - 1)) != 0) {
         fprintf(stderr, "page size must be a power of two\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
 
@@ -3557,7 +3804,7 @@ static void handle_arg_randseed(const char *arg)
 
     if (parse_uint_full(arg, &seed, 0) != 0 || seed > UINT_MAX) {
         fprintf(stderr, "Invalid seed number: %s\n", arg);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     srand(seed);
 }
@@ -3580,11 +3827,10 @@ static void handle_arg_cpu(const char *arg)
 #if defined(cpu_list)
         cpu_list(stdout, &fprintf);
 #endif
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
 
-#if defined(CONFIG_USE_GUEST_BASE)
 static void handle_arg_guest_base(const char *arg)
 {
     guest_base = strtol(arg, NULL, 0);
@@ -3618,15 +3864,14 @@ static void handle_arg_reserved_va(const char *arg)
 #endif
             ) {
             fprintf(stderr, "Reserved virtual address too big\n");
-            exit(1);
+            exit(EXIT_FAILURE);
         }
     }
     if (*p) {
         fprintf(stderr, "Unrecognised -R size suffix '%s'\n", p);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
-#endif
 
 static void handle_arg_singlestep(const char *arg)
 {
@@ -3642,7 +3887,7 @@ static void handle_arg_version(const char *arg)
 {
     printf("qemu-" TARGET_NAME " version " QEMU_VERSION QEMU_PKGVERSION
            ", Copyright (c) 2003-2008 Fabrice Bellard\n");
-    exit(0);
+    exit(EXIT_SUCCESS);
 }
 
 struct qemu_argument {
@@ -3657,6 +3902,8 @@ struct qemu_argument {
 static const struct qemu_argument arg_table[] = {
     {"h",          "",                 false, handle_arg_help,
      "",           "print this help"},
+    {"help",       "",                 false, handle_arg_help,
+     "",           ""},
     {"g",          "QEMU_GDB",         true,  handle_arg_gdb,
      "port",       "wait gdb connection to 'port'"},
     {"L",          "QEMU_LD_PREFIX",   true,  handle_arg_ld_prefix,
@@ -3673,12 +3920,10 @@ static const struct qemu_argument arg_table[] = {
      "argv0",      "forces target process argv[0] to be 'argv0'"},
     {"r",          "QEMU_UNAME",       true,  handle_arg_uname,
      "uname",      "set qemu uname release string to 'uname'"},
-#if defined(CONFIG_USE_GUEST_BASE)
     {"B",          "QEMU_GUEST_BASE",  true,  handle_arg_guest_base,
      "address",    "set guest_base address to 'address'"},
     {"R",          "QEMU_RESERVED_VA", true,  handle_arg_reserved_va,
      "size",       "reserve 'size' bytes for guest virtual address space"},
-#endif
     {"d",          "QEMU_LOG",         true,  handle_arg_log,
      "item[,...]", "enable logging of specified items "
      "(use '-d help' for a list of items)"},
@@ -3697,7 +3942,7 @@ static const struct qemu_argument arg_table[] = {
     {NULL, NULL, false, NULL, NULL, NULL}
 };
 
-static void usage(void)
+static void usage(int exitcode)
 {
     const struct qemu_argument *arginfo;
     int maxarglen;
@@ -3764,7 +4009,7 @@ static void usage(void)
            "Note that if you provide several changes to a single variable\n"
            "the last change will stay in effect.\n");
 
-    exit(1);
+    exit(exitcode);
 }
 
 static int parse_args(int argc, char **argv)
@@ -3798,12 +4043,18 @@ static int parse_args(int argc, char **argv)
         if (!strcmp(r, "-")) {
             break;
         }
+        /* Treat --foo the same as -foo.  */
+        if (r[0] == '-') {
+            r++;
+        }
 
         for (arginfo = arg_table; arginfo->handle_opt != NULL; arginfo++) {
             if (!strcmp(r, arginfo->argv)) {
                 if (arginfo->has_arg) {
                     if (optind >= argc) {
-                        usage();
+                        (void) fprintf(stderr,
+                            "qemu: missing argument for option '%s'\n", r);
+                        exit(EXIT_FAILURE);
                     }
                     arginfo->handle_opt(argv[optind]);
                     optind++;
@@ -3816,12 +4067,14 @@ static int parse_args(int argc, char **argv)
 
         /* no option matched the current argv */
         if (arginfo->handle_opt == NULL) {
-            usage();
+            (void) fprintf(stderr, "qemu: unknown option '%s'\n", r);
+            exit(EXIT_FAILURE);
         }
     }
 
     if (optind >= argc) {
-        usage();
+        (void) fprintf(stderr, "qemu: no user program specified\n");
+        exit(EXIT_FAILURE);
     }
 
     filename = argv[optind];
@@ -3850,7 +4103,7 @@ int main(int argc, char **argv, char **envp)
 
     if ((envlist = envlist_create()) == NULL) {
         (void) fprintf(stderr, "Unable to allocate envlist\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 
     /* add current environment into the list */
@@ -3936,7 +4189,7 @@ int main(int argc, char **argv, char **envp)
     cpu = cpu_init(cpu_model);
     if (!cpu) {
         fprintf(stderr, "Unable to find CPU definition\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     env = cpu->env_ptr;
     cpu_reset(cpu);
@@ -3954,7 +4207,6 @@ int main(int argc, char **argv, char **envp)
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * Now that page sizes are configured in cpu_init() we can do
      * proper page alignment for guest_base.
@@ -3969,14 +4221,13 @@ int main(int argc, char **argv, char **envp)
                     "space for use as guest address space (check your virtual "
                     "memory ulimit setting or reserve less using -R option)\n",
                     reserved_va);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
 
         if (reserved_va) {
             mmap_next_start = reserved_va;
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     /*
      * Read in mmap_min_addr kernel parameter.  This value is used
@@ -4003,7 +4254,7 @@ int main(int argc, char **argv, char **envp)
     target_argv = calloc(target_argc + 1, sizeof (char *));
     if (target_argv == NULL) {
 	(void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
-	exit(1);
+	exit(EXIT_FAILURE);
     }
 
     /*
@@ -4019,7 +4270,7 @@ int main(int argc, char **argv, char **envp)
     }
     target_argv[target_argc] = NULL;
 
-    ts = g_malloc0 (sizeof(TaskState));
+    ts = g_new0(TaskState, 1);
     init_task_state(ts);
     /* build Task State */
     ts->info = info;
@@ -4032,7 +4283,7 @@ int main(int argc, char **argv, char **envp)
         execfd = open(filename, O_RDONLY);
         if (execfd < 0) {
             printf("Error while loading %s: %s\n", filename, strerror(errno));
-            _exit(1);
+            _exit(EXIT_FAILURE);
         }
     }
 
@@ -4040,7 +4291,7 @@ int main(int argc, char **argv, char **envp)
         info, &bprm);
     if (ret != 0) {
         printf("Error while loading %s: %s\n", filename, strerror(-ret));
-        _exit(1);
+        _exit(EXIT_FAILURE);
     }
 
     for (wrk = target_environ; *wrk; wrk++) {
@@ -4050,9 +4301,7 @@ int main(int argc, char **argv, char **envp)
     free(target_environ);
 
     if (qemu_log_enabled()) {
-#if defined(CONFIG_USE_GUEST_BASE)
         qemu_log("guest_base  0x%lx\n", guest_base);
-#endif
         log_page_dump();
 
         qemu_log("start_brk   0x" TARGET_ABI_FMT_lx "\n", info->start_brk);
@@ -4072,12 +4321,10 @@ int main(int argc, char **argv, char **envp)
     syscall_init();
     signal_init();
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /* Now that we've loaded the binary, GUEST_BASE is fixed.  Delay
        generating the prologue until now so that the prologue can take
        the real value of GUEST_BASE into account.  */
     tcg_prologue_init(&tcg_ctx);
-#endif
 
 #if defined(TARGET_I386)
     env->cr[0] = CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK;
@@ -4090,7 +4337,7 @@ int main(int argc, char **argv, char **envp)
     /* enable 64 bit mode if possible */
     if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_LM)) {
         fprintf(stderr, "The selected x86 CPU does not support 64 bit mode\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     env->cr[4] |= CR4_PAE_MASK;
     env->efer |= MSR_EFER_LMA | MSR_EFER_LME;
@@ -4200,7 +4447,7 @@ int main(int argc, char **argv, char **envp)
         if (!(arm_feature(env, ARM_FEATURE_AARCH64))) {
             fprintf(stderr,
                     "The selected ARM CPU does not support 64 bit mode\n");
-            exit(1);
+            exit(EXIT_FAILURE);
         }
 
         for (i = 0; i < 31; i++) {
@@ -4386,6 +4633,17 @@ int main(int argc, char **argv, char **envp)
             env->psw.mask = regs->psw.mask;
             env->psw.addr = regs->psw.addr;
     }
+#elif defined(TARGET_TILEGX)
+    {
+        int i;
+        for (i = 0; i < TILEGX_R_COUNT; i++) {
+            env->regs[i] = regs->regs[i];
+        }
+        for (i = 0; i < TILEGX_SPR_COUNT; i++) {
+            env->spregs[i] = 0;
+        }
+        env->pc = regs->pc;
+    }
 #else
 #error unsupported target CPU
 #endif
@@ -4401,7 +4659,7 @@ int main(int argc, char **argv, char **envp)
         if (gdbserver_start(gdbstub_port) < 0) {
             fprintf(stderr, "qemu: could not open gdbserver on port %d\n",
                     gdbstub_port);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
         gdb_handlesig(cpu, 0);
     }
diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index 78e1b2df4..7b459d510 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -206,7 +206,6 @@ abi_ulong mmap_next_start = TASK_UNMAPPED_BASE;
 
 unsigned long last_brk;
 
-#ifdef CONFIG_USE_GUEST_BASE
 /* Subroutine of mmap_find_vma, used when we have pre-allocated a chunk
    of guest address space.  */
 static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
@@ -216,14 +215,14 @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
     int prot;
     int looped = 0;
 
-    if (size > RESERVED_VA) {
+    if (size > reserved_va) {
         return (abi_ulong)-1;
     }
 
     size = HOST_PAGE_ALIGN(size);
     end_addr = start + size;
-    if (end_addr > RESERVED_VA) {
-        end_addr = RESERVED_VA;
+    if (end_addr > reserved_va) {
+        end_addr = reserved_va;
     }
     addr = end_addr - qemu_host_page_size;
 
@@ -232,7 +231,7 @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
             if (looped) {
                 return (abi_ulong)-1;
             }
-            end_addr = RESERVED_VA;
+            end_addr = reserved_va;
             addr = end_addr - qemu_host_page_size;
             looped = 1;
             continue;
@@ -253,7 +252,6 @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
 
     return addr;
 }
-#endif
 
 /*
  * Find and reserve a free memory area of size 'size'. The search
@@ -276,11 +274,9 @@ abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size)
 
     size = HOST_PAGE_ALIGN(size);
 
-#ifdef CONFIG_USE_GUEST_BASE
-    if (RESERVED_VA) {
+    if (reserved_va) {
         return mmap_find_vma_reserved(start, size);
     }
-#endif
 
     addr = start;
     wrapped = repeat = 0;
@@ -448,9 +444,7 @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
            /* If so, truncate the file map at eof aligned with 
               the hosts real pagesize. Additional anonymous maps
               will be created beyond EOF.  */
-           len = (sb.st_size - offset);
-           len += qemu_real_host_page_size - 1;
-           len &= ~(qemu_real_host_page_size - 1);
+           len = REAL_HOST_PAGE_ALIGN(sb.st_size - offset);
        }
     }
 
@@ -518,10 +512,7 @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
                 goto fail;
             if (!(prot & PROT_WRITE)) {
                 ret = target_mprotect(start, len, prot);
-                if (ret != 0) {
-                    start = ret;
-                    goto the_end;
-                }
+                assert(ret == 0);
             }
             goto the_end;
         }
@@ -671,7 +662,7 @@ int target_munmap(abi_ulong start, abi_ulong len)
     ret = 0;
     /* unmap what we can */
     if (real_start < real_end) {
-        if (RESERVED_VA) {
+        if (reserved_va) {
             mmap_reserve(real_start, real_end - real_start);
         } else {
             ret = munmap(g2h(real_start), real_end - real_start);
@@ -701,7 +692,7 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
                                      flags,
                                      g2h(new_addr));
 
-        if (RESERVED_VA && host_addr != MAP_FAILED) {
+        if (reserved_va && host_addr != MAP_FAILED) {
             /* If new and old addresses overlap then the above mremap will
                already have failed with EINVAL.  */
             mmap_reserve(old_addr, old_size);
@@ -719,13 +710,13 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
                                          old_size, new_size,
                                          flags | MREMAP_FIXED,
                                          g2h(mmap_start));
-            if ( RESERVED_VA ) {
+            if (reserved_va) {
                 mmap_reserve(old_addr, old_size);
             }
         }
     } else {
         int prot = 0;
-        if (RESERVED_VA && old_size < new_size) {
+        if (reserved_va && old_size < new_size) {
             abi_ulong addr;
             for (addr = old_addr + old_size;
                  addr < old_addr + new_size;
@@ -735,7 +726,7 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
         }
         if (prot == 0) {
             host_addr = mremap(g2h(old_addr), old_size, new_size, flags);
-            if (host_addr != MAP_FAILED && RESERVED_VA && old_size > new_size) {
+            if (host_addr != MAP_FAILED && reserved_va && old_size > new_size) {
                 mmap_reserve(old_addr + old_size, new_size - old_size);
             }
         } else {
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 8012cc2f5..bd90cc379 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -36,8 +36,6 @@ struct image_info {
         abi_ulong       start_brk;
         abi_ulong       brk;
         abi_ulong       start_mmap;
-        abi_ulong       mmap;
-        abi_ulong       rss;
         abi_ulong       start_stack;
         abi_ulong       stack_limit;
         abi_ulong       entry;
@@ -145,12 +143,6 @@ extern const char *qemu_uname_release;
 extern unsigned long mmap_min_addr;
 
 /* ??? See if we can avoid exposing so much of the loader internals.  */
-/*
- * MAX_ARG_PAGES defines the number of pages allocated for arguments
- * and envelope for the new program. 32 should suffice, this gives
- * a maximum env+arg of 128kB w/4KB pages!
- */
-#define MAX_ARG_PAGES 33
 
 /* Read a good amount of data initially, to hopefully get all the
    program headers loaded.  */
@@ -162,7 +154,6 @@ extern unsigned long mmap_min_addr;
  */
 struct linux_binprm {
         char buf[BPRM_BUF_SIZE] __attribute__((aligned));
-        void *page[MAX_ARG_PAGES];
         abi_ulong p;
 	int fd;
         int e_uid, e_gid;
@@ -261,8 +252,6 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
 int target_msync(abi_ulong start, abi_ulong len, int flags);
 extern unsigned long last_brk;
 extern abi_ulong mmap_next_start;
-void mmap_lock(void);
-void mmap_unlock(void);
 abi_ulong mmap_find_vma(abi_ulong, abi_ulong);
 void cpu_list_lock(void);
 void cpu_list_unlock(void);
diff --git a/linux-user/signal.c b/linux-user/signal.c
index 9d4cef409..9d62e027e 100644
--- a/linux-user/signal.c
+++ b/linux-user/signal.c
@@ -22,7 +22,6 @@
 #include <stdarg.h>
 #include <unistd.h>
 #include <errno.h>
-#include <assert.h>
 #include <sys/ucontext.h>
 #include <sys/resource.h>
 
@@ -3901,12 +3900,6 @@ static inline abi_ulong get_sigframe(struct target_sigaction *ka,
     return sp;
 }
 
-static void setup_frame(int sig, struct target_sigaction *ka,
-                        target_sigset_t *set, CPUOpenRISCState *env)
-{
-    qemu_log("Not implement.\n");
-}
-
 static void setup_rt_frame(int sig, struct target_sigaction *ka,
                            target_siginfo_t *info,
                            target_sigset_t *set, CPUOpenRISCState *env)
@@ -5544,6 +5537,163 @@ long do_rt_sigreturn(CPUAlphaState *env)
     force_sig(TARGET_SIGSEGV);
 }
 
+#elif defined(TARGET_TILEGX)
+
+struct target_sigcontext {
+    union {
+        /* General-purpose registers.  */
+        abi_ulong gregs[56];
+        struct {
+            abi_ulong __gregs[53];
+            abi_ulong tp;        /* Aliases gregs[TREG_TP].  */
+            abi_ulong sp;        /* Aliases gregs[TREG_SP].  */
+            abi_ulong lr;        /* Aliases gregs[TREG_LR].  */
+        };
+    };
+    abi_ulong pc;        /* Program counter.  */
+    abi_ulong ics;       /* In Interrupt Critical Section?  */
+    abi_ulong faultnum;  /* Fault number.  */
+    abi_ulong pad[5];
+};
+
+struct target_ucontext {
+    abi_ulong tuc_flags;
+    abi_ulong tuc_link;
+    target_stack_t tuc_stack;
+    struct target_sigcontext tuc_mcontext;
+    target_sigset_t tuc_sigmask;   /* mask last for extensibility */
+};
+
+struct target_rt_sigframe {
+    unsigned char save_area[16]; /* caller save area */
+    struct target_siginfo info;
+    struct target_ucontext uc;
+};
+
+static void setup_sigcontext(struct target_sigcontext *sc,
+                             CPUArchState *env, int signo)
+{
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; ++i) {
+        __put_user(env->regs[i], &sc->gregs[i]);
+    }
+
+    __put_user(env->pc, &sc->pc);
+    __put_user(0, &sc->ics);
+    __put_user(signo, &sc->faultnum);
+}
+
+static void restore_sigcontext(CPUTLGState *env, struct target_sigcontext *sc)
+{
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; ++i) {
+        __get_user(env->regs[i], &sc->gregs[i]);
+    }
+
+    __get_user(env->pc, &sc->pc);
+}
+
+static abi_ulong get_sigframe(struct target_sigaction *ka, CPUArchState *env,
+                              size_t frame_size)
+{
+    unsigned long sp = env->regs[TILEGX_R_SP];
+
+    if (on_sig_stack(sp) && !likely(on_sig_stack(sp - frame_size))) {
+        return -1UL;
+    }
+
+    if ((ka->sa_flags & SA_ONSTACK) && !sas_ss_flags(sp)) {
+        sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
+    }
+
+    sp -= frame_size;
+    sp &= -16UL;
+    return sp;
+}
+
+static void setup_rt_frame(int sig, struct target_sigaction *ka,
+                           target_siginfo_t *info,
+                           target_sigset_t *set, CPUArchState *env)
+{
+    abi_ulong frame_addr;
+    struct target_rt_sigframe *frame;
+    unsigned long restorer;
+
+    frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
+
+    /* Always write at least the signal number for the stack backtracer. */
+    if (ka->sa_flags & TARGET_SA_SIGINFO) {
+        /* At sigreturn time, restore the callee-save registers too. */
+        tswap_siginfo(&frame->info, info);
+        /* regs->flags |= PT_FLAGS_RESTORE_REGS; FIXME: we can skip it? */
+    } else {
+        __put_user(info->si_signo, &frame->info.si_signo);
+    }
+
+    /* Create the ucontext.  */
+    __put_user(0, &frame->uc.tuc_flags);
+    __put_user(0, &frame->uc.tuc_link);
+    __put_user(target_sigaltstack_used.ss_sp, &frame->uc.tuc_stack.ss_sp);
+    __put_user(sas_ss_flags(env->regs[TILEGX_R_SP]),
+               &frame->uc.tuc_stack.ss_flags);
+    __put_user(target_sigaltstack_used.ss_size, &frame->uc.tuc_stack.ss_size);
+    setup_sigcontext(&frame->uc.tuc_mcontext, env, info->si_signo);
+
+    restorer = (unsigned long) do_rt_sigreturn;
+    if (ka->sa_flags & TARGET_SA_RESTORER) {
+            restorer = (unsigned long) ka->sa_restorer;
+    }
+    env->pc = (unsigned long) ka->_sa_handler;
+    env->regs[TILEGX_R_SP] = (unsigned long) frame;
+    env->regs[TILEGX_R_LR] = restorer;
+    env->regs[0] = (unsigned long) sig;
+    env->regs[1] = (unsigned long) &frame->info;
+    env->regs[2] = (unsigned long) &frame->uc;
+    /* regs->flags |= PT_FLAGS_CALLER_SAVES; FIXME: we can skip it? */
+
+    unlock_user_struct(frame, frame_addr, 1);
+    return;
+
+give_sigsegv:
+    if (sig == TARGET_SIGSEGV) {
+        ka->_sa_handler = TARGET_SIG_DFL;
+    }
+    force_sig(TARGET_SIGSEGV /* , current */);
+}
+
+long do_rt_sigreturn(CPUTLGState *env)
+{
+    abi_ulong frame_addr = env->regs[TILEGX_R_SP];
+    struct target_rt_sigframe *frame;
+    sigset_t set;
+
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
+    target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
+    do_sigprocmask(SIG_SETMASK, &set, NULL);
+
+    restore_sigcontext(env, &frame->uc.tuc_mcontext);
+    if (do_sigaltstack(frame_addr + offsetof(struct target_rt_sigframe,
+                                             uc.tuc_stack),
+                       0, env->regs[TILEGX_R_SP]) == -EFAULT) {
+        goto badframe;
+    }
+
+    unlock_user_struct(frame, frame_addr, 0);
+    return env->regs[TILEGX_R_RE];
+
+
+ badframe:
+    unlock_user_struct(frame, frame_addr, 0);
+    force_sig(TARGET_SIGSEGV);
+}
+
 #else
 
 static void setup_frame(int sig, struct target_sigaction *ka,
@@ -5663,7 +5813,8 @@ void process_pending_signals(CPUArchState *cpu_env)
         }
 #endif
         /* prepare the stack frame of the virtual CPU */
-#if defined(TARGET_ABI_MIPSN32) || defined(TARGET_ABI_MIPSN64)
+#if defined(TARGET_ABI_MIPSN32) || defined(TARGET_ABI_MIPSN64) \
+    || defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
         /* These targets do not have traditional signals.  */
         setup_rt_frame(sig, sa, &q->info, &target_old_set, cpu_env);
 #else
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index f62c69894..6c64ba63d 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -213,7 +213,7 @@ static int gettid(void) {
     return -ENOSYS;
 }
 #endif
-#ifdef __NR_getdents
+#if defined(TARGET_NR_getdents) && defined(__NR_getdents)
 _syscall3(int, sys_getdents, uint, fd, struct linux_dirent *, dirp, uint, count);
 #endif
 #if !defined(__NR_getdents) || \
@@ -457,6 +457,7 @@ static uint16_t target_to_host_errno_table[ERRNO_TABLE_SIZE] = {
  * minus the errnos that are not actually generic to all archs.
  */
 static uint16_t host_to_target_errno_table[ERRNO_TABLE_SIZE] = {
+    [EAGAIN]		= TARGET_EAGAIN,
     [EIDRM]		= TARGET_EIDRM,
     [ECHRNG]		= TARGET_ECHRNG,
     [EL2NSYNC]		= TARGET_EL2NSYNC,
@@ -1181,7 +1182,7 @@ static inline abi_long target_to_host_cmsg(struct msghdr *msgh,
     struct cmsghdr *cmsg = CMSG_FIRSTHDR(msgh);
     abi_long msg_controllen;
     abi_ulong target_cmsg_addr;
-    struct target_cmsghdr *target_cmsg;
+    struct target_cmsghdr *target_cmsg, *target_cmsg_start;
     socklen_t space = 0;
     
     msg_controllen = tswapal(target_msgh->msg_controllen);
@@ -1189,6 +1190,7 @@ static inline abi_long target_to_host_cmsg(struct msghdr *msgh,
         goto the_end;
     target_cmsg_addr = tswapal(target_msgh->msg_control);
     target_cmsg = lock_user(VERIFY_READ, target_cmsg_addr, msg_controllen, 1);
+    target_cmsg_start = target_cmsg;
     if (!target_cmsg)
         return -TARGET_EFAULT;
 
@@ -1247,7 +1249,8 @@ static inline abi_long target_to_host_cmsg(struct msghdr *msgh,
         }
 
         cmsg = CMSG_NXTHDR(msgh, cmsg);
-        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg);
+        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg,
+                                         target_cmsg_start);
     }
     unlock_user(target_cmsg, target_cmsg_addr, 0);
  the_end:
@@ -1261,7 +1264,7 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
     struct cmsghdr *cmsg = CMSG_FIRSTHDR(msgh);
     abi_long msg_controllen;
     abi_ulong target_cmsg_addr;
-    struct target_cmsghdr *target_cmsg;
+    struct target_cmsghdr *target_cmsg, *target_cmsg_start;
     socklen_t space = 0;
 
     msg_controllen = tswapal(target_msgh->msg_controllen);
@@ -1269,6 +1272,7 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
         goto the_end;
     target_cmsg_addr = tswapal(target_msgh->msg_control);
     target_cmsg = lock_user(VERIFY_WRITE, target_cmsg_addr, msg_controllen, 0);
+    target_cmsg_start = target_cmsg;
     if (!target_cmsg)
         return -TARGET_EFAULT;
 
@@ -1382,14 +1386,15 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
         }
 
         target_cmsg->cmsg_len = tswapal(tgt_len);
-        tgt_space = TARGET_CMSG_SPACE(tgt_len);
+        tgt_space = TARGET_CMSG_SPACE(len);
         if (msg_controllen < tgt_space) {
             tgt_space = msg_controllen;
         }
         msg_controllen -= tgt_space;
         space += tgt_space;
         cmsg = CMSG_NXTHDR(msgh, cmsg);
-        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg);
+        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg,
+                                         target_cmsg_start);
     }
     unlock_user(target_cmsg, target_cmsg_addr, space);
  the_end:
@@ -2723,8 +2728,9 @@ static inline abi_long host_to_target_semarray(int semid, abi_ulong target_addr,
 }
 
 static inline abi_long do_semctl(int semid, int semnum, int cmd,
-                                 union target_semun target_su)
+                                 abi_ulong target_arg)
 {
+    union target_semun target_su = { .buf = target_arg };
     union semun arg;
     struct semid_ds dsarg;
     unsigned short *array = NULL;
@@ -3246,8 +3252,7 @@ static abi_long do_ipc(unsigned int call, abi_long first,
          * ptr argument. */
         abi_ulong atptr;
         get_user_ual(atptr, ptr);
-        ret = do_semctl(first, second, third,
-                (union target_semun) atptr);
+        ret = do_semctl(first, second, third, atptr);
         break;
     }
 
@@ -3939,7 +3944,6 @@ static abi_long do_ioctl(int fd, int cmd, abi_long arg)
         break;
     case TYPE_PTRVOID:
     case TYPE_INT:
-        /* int argment */
         ret = get_errno(ioctl(fd, ie->host_cmd, arg));
         break;
     case TYPE_PTR:
@@ -4513,6 +4517,7 @@ static void *clone_func(void *arg)
     CPUState *cpu;
     TaskState *ts;
 
+    rcu_register_thread();
     env = info->env;
     cpu = ENV_GET_CPU(env);
     thread_cpu = cpu;
@@ -4561,7 +4566,7 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp,
         new_thread_info info;
         pthread_attr_t attr;
 
-        ts = g_malloc0(sizeof(TaskState));
+        ts = g_new0(TaskState, 1);
         init_task_state(ts);
         /* we create a new CPU instance. */
         new_env = cpu_copy(env);
@@ -4622,8 +4627,9 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp,
         pthread_mutex_unlock(&clone_lock);
     } else {
         /* if no CLONE_VM, we consider it is a fork */
-        if ((flags & ~(CSIGNAL | CLONE_NPTL_FLAGS2)) != 0)
-            return -EINVAL;
+        if ((flags & ~(CSIGNAL | CLONE_NPTL_FLAGS2)) != 0) {
+            return -TARGET_EINVAL;
+        }
         fork_start();
         ret = fork();
         if (ret == 0) {
@@ -5246,6 +5252,93 @@ static int do_futex(target_ulong uaddr, int op, int val, target_ulong timeout,
         return -TARGET_ENOSYS;
     }
 }
+#if defined(TARGET_NR_name_to_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+static abi_long do_name_to_handle_at(abi_long dirfd, abi_long pathname,
+                                     abi_long handle, abi_long mount_id,
+                                     abi_long flags)
+{
+    struct file_handle *target_fh;
+    struct file_handle *fh;
+    int mid = 0;
+    abi_long ret;
+    char *name;
+    unsigned int size, total_size;
+
+    if (get_user_s32(size, handle)) {
+        return -TARGET_EFAULT;
+    }
+
+    name = lock_user_string(pathname);
+    if (!name) {
+        return -TARGET_EFAULT;
+    }
+
+    total_size = sizeof(struct file_handle) + size;
+    target_fh = lock_user(VERIFY_WRITE, handle, total_size, 0);
+    if (!target_fh) {
+        unlock_user(name, pathname, 0);
+        return -TARGET_EFAULT;
+    }
+
+    fh = g_malloc0(total_size);
+    fh->handle_bytes = size;
+
+    ret = get_errno(name_to_handle_at(dirfd, path(name), fh, &mid, flags));
+    unlock_user(name, pathname, 0);
+
+    /* man name_to_handle_at(2):
+     * Other than the use of the handle_bytes field, the caller should treat
+     * the file_handle structure as an opaque data type
+     */
+
+    memcpy(target_fh, fh, total_size);
+    target_fh->handle_bytes = tswap32(fh->handle_bytes);
+    target_fh->handle_type = tswap32(fh->handle_type);
+    g_free(fh);
+    unlock_user(target_fh, handle, total_size);
+
+    if (put_user_s32(mid, mount_id)) {
+        return -TARGET_EFAULT;
+    }
+
+    return ret;
+
+}
+#endif
+
+#if defined(TARGET_NR_open_by_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+static abi_long do_open_by_handle_at(abi_long mount_fd, abi_long handle,
+                                     abi_long flags)
+{
+    struct file_handle *target_fh;
+    struct file_handle *fh;
+    unsigned int size, total_size;
+    abi_long ret;
+
+    if (get_user_s32(size, handle)) {
+        return -TARGET_EFAULT;
+    }
+
+    total_size = sizeof(struct file_handle) + size;
+    target_fh = lock_user(VERIFY_READ, handle, total_size, 1);
+    if (!target_fh) {
+        return -TARGET_EFAULT;
+    }
+
+    fh = g_memdup(target_fh, total_size);
+    fh->handle_bytes = size;
+    fh->handle_type = tswap32(target_fh->handle_type);
+
+    ret = get_errno(open_by_handle_at(mount_fd, fh,
+                    target_to_host_bitmask(flags, fcntl_flags_tbl)));
+
+    g_free(fh);
+
+    unlock_user(target_fh, handle, total_size);
+
+    return ret;
+}
+#endif
 
 /* Map host to target signal numbers for the wait family of syscalls.
    Assume all other status bits are the same.  */
@@ -5614,6 +5707,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             thread_cpu = NULL;
             object_unref(OBJECT(cpu));
             g_free(ts);
+            rcu_unregister_thread();
             pthread_exit(NULL);
         }
 #ifdef TARGET_GPROF
@@ -5639,6 +5733,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(write(arg1, p, arg3));
         unlock_user(p, arg2, 0);
         break;
+#ifdef TARGET_NR_open
     case TARGET_NR_open:
         if (!(p = lock_user_string(arg1)))
             goto efault;
@@ -5647,6 +5742,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
                                   arg3));
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_openat:
         if (!(p = lock_user_string(arg2)))
             goto efault;
@@ -5655,15 +5751,27 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
                                   arg4));
         unlock_user(p, arg2, 0);
         break;
+#if defined(TARGET_NR_name_to_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+    case TARGET_NR_name_to_handle_at:
+        ret = do_name_to_handle_at(arg1, arg2, arg3, arg4, arg5);
+        break;
+#endif
+#if defined(TARGET_NR_open_by_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+    case TARGET_NR_open_by_handle_at:
+        ret = do_open_by_handle_at(arg1, arg2, arg3);
+        break;
+#endif
     case TARGET_NR_close:
         ret = get_errno(close(arg1));
         break;
     case TARGET_NR_brk:
         ret = do_brk(arg1);
         break;
+#ifdef TARGET_NR_fork
     case TARGET_NR_fork:
         ret = get_errno(do_fork(cpu_env, SIGCHLD, 0, 0, 0, 0));
         break;
+#endif
 #ifdef TARGET_NR_waitpid
     case TARGET_NR_waitpid:
         {
@@ -5698,6 +5806,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         unlock_user(p, arg1, 0);
         break;
 #endif
+#ifdef TARGET_NR_link
     case TARGET_NR_link:
         {
             void * p2;
@@ -5711,6 +5820,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_linkat)
     case TARGET_NR_linkat:
         {
@@ -5728,12 +5838,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_unlink
     case TARGET_NR_unlink:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(unlink(p));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_unlinkat)
     case TARGET_NR_unlinkat:
         if (!(p = lock_user_string(arg2)))
@@ -5799,12 +5911,6 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             }
             *q = NULL;
 
-            /* This case will not be caught by the host's execve() if its
-               page size is bigger than the target's. */
-            if (total_size > MAX_ARG_PAGES * TARGET_PAGE_SIZE) {
-                ret = -TARGET_E2BIG;
-                goto execve_end;
-            }
             if (!(p = lock_user_string(arg1)))
                 goto execve_efault;
             ret = get_errno(execve(p, argp, envp));
@@ -5850,12 +5956,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_mknod
     case TARGET_NR_mknod:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(mknod(p, arg2, arg3));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_mknodat)
     case TARGET_NR_mknodat:
         if (!(p = lock_user_string(arg2)))
@@ -5864,12 +5972,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         unlock_user(p, arg2, 0);
         break;
 #endif
+#ifdef TARGET_NR_chmod
     case TARGET_NR_chmod:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(chmod(p, arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #ifdef TARGET_NR_break
     case TARGET_NR_break:
         goto unimplemented;
@@ -6004,6 +6114,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_utimes
     case TARGET_NR_utimes:
         {
             struct timeval *tvp, tv[2];
@@ -6022,6 +6133,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_futimesat)
     case TARGET_NR_futimesat:
         {
@@ -6050,12 +6162,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_gtty:
         goto unimplemented;
 #endif
+#ifdef TARGET_NR_access
     case TARGET_NR_access:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(access(path(p), arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_faccessat) && defined(__NR_faccessat)
     case TARGET_NR_faccessat:
         if (!(p = lock_user_string(arg2)))
@@ -6080,6 +6194,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_kill:
         ret = get_errno(kill(arg1, target_to_host_signal(arg2)));
         break;
+#ifdef TARGET_NR_rename
     case TARGET_NR_rename:
         {
             void *p2;
@@ -6093,6 +6208,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_renameat)
     case TARGET_NR_renameat:
         {
@@ -6108,12 +6224,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_mkdir
     case TARGET_NR_mkdir:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(mkdir(p, arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_mkdirat)
     case TARGET_NR_mkdirat:
         if (!(p = lock_user_string(arg2)))
@@ -6122,18 +6240,22 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         unlock_user(p, arg2, 0);
         break;
 #endif
+#ifdef TARGET_NR_rmdir
     case TARGET_NR_rmdir:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(rmdir(p));
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_dup:
         ret = get_errno(dup(arg1));
         break;
+#ifdef TARGET_NR_pipe
     case TARGET_NR_pipe:
         ret = do_pipe(cpu_env, arg1, 0, 0);
         break;
+#endif
 #ifdef TARGET_NR_pipe2
     case TARGET_NR_pipe2:
         ret = do_pipe(cpu_env, arg1,
@@ -6218,11 +6340,15 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(chroot(p));
         unlock_user(p, arg1, 0);
         break;
+#ifdef TARGET_NR_ustat
     case TARGET_NR_ustat:
         goto unimplemented;
+#endif
+#ifdef TARGET_NR_dup2
     case TARGET_NR_dup2:
         ret = get_errno(dup2(arg1, arg2));
         break;
+#endif
 #if defined(CONFIG_DUP3) && defined(TARGET_NR_dup3)
     case TARGET_NR_dup3:
         ret = get_errno(dup3(arg1, arg2, arg3));
@@ -6233,9 +6359,11 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(getppid());
         break;
 #endif
+#ifdef TARGET_NR_getpgrp
     case TARGET_NR_getpgrp:
         ret = get_errno(getpgrp());
         break;
+#endif
     case TARGET_NR_setsid:
         ret = get_errno(setsid());
         break;
@@ -6811,6 +6939,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_symlink
     case TARGET_NR_symlink:
         {
             void *p2;
@@ -6824,6 +6953,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_symlinkat)
     case TARGET_NR_symlinkat:
         {
@@ -6843,6 +6973,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_oldlstat:
         goto unimplemented;
 #endif
+#ifdef TARGET_NR_readlink
     case TARGET_NR_readlink:
         {
             void *p2;
@@ -6873,6 +7004,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_readlinkat)
     case TARGET_NR_readlinkat:
         {
@@ -7272,22 +7404,28 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             }
         }
         break;
+#ifdef TARGET_NR_stat
     case TARGET_NR_stat:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(stat(path(p), &st));
         unlock_user(p, arg1, 0);
         goto do_stat;
+#endif
+#ifdef TARGET_NR_lstat
     case TARGET_NR_lstat:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(lstat(path(p), &st));
         unlock_user(p, arg1, 0);
         goto do_stat;
+#endif
     case TARGET_NR_fstat:
         {
             ret = get_errno(fstat(arg1, &st));
+#if defined(TARGET_NR_stat) || defined(TARGET_NR_lstat)
         do_stat:
+#endif
             if (!is_error(ret)) {
                 struct target_stat *target_st;
 
@@ -7411,7 +7549,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #endif
 #ifdef TARGET_NR_semctl
     case TARGET_NR_semctl:
-        ret = do_semctl(arg1, arg2, arg3, (union target_semun)(abi_ulong)arg4);
+        ret = do_semctl(arg1, arg2, arg3, arg4);
         break;
 #endif
 #ifdef TARGET_NR_msgctl
@@ -7575,6 +7713,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_getdents
     case TARGET_NR_getdents:
 #ifdef __NR_getdents
 #if TARGET_ABI_BITS == 32 && HOST_LONG_BITS == 64
@@ -7705,6 +7844,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
 #endif
         break;
+#endif /* TARGET_NR_getdents */
 #if defined(TARGET_NR_getdents64) && defined(__NR_getdents64)
     case TARGET_NR_getdents64:
         {
@@ -7844,11 +7984,13 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(fdatasync(arg1));
         break;
 #endif
+#ifdef TARGET_NR__sysctl
     case TARGET_NR__sysctl:
         /* We don't implement this, but ENOTDIR is always a safe
            return value. */
         ret = -TARGET_ENOTDIR;
         break;
+#endif
     case TARGET_NR_sched_getaffinity:
         {
             unsigned int mask_size;
@@ -8295,12 +8437,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             ret = host_to_target_stat64(cpu_env, arg3, &st);
         break;
 #endif
+#ifdef TARGET_NR_lchown
     case TARGET_NR_lchown:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(lchown(p, low2highuid(arg2), low2highgid(arg3)));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #ifdef TARGET_NR_getuid
     case TARGET_NR_getuid:
         ret = get_errno(high2lowuid(getuid()));
@@ -8423,12 +8567,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_chown
     case TARGET_NR_chown:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(chown(p, low2highuid(arg2), low2highgid(arg3)));
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_setuid:
         ret = get_errno(setuid(low2highuid(arg1)));
         break;
diff --git a/linux-user/syscall_defs.h b/linux-user/syscall_defs.h
index edd5f3c80..f996acf94 100644
--- a/linux-user/syscall_defs.h
+++ b/linux-user/syscall_defs.h
@@ -64,8 +64,9 @@
 #endif
 
 #if defined(TARGET_I386) || defined(TARGET_ARM) || defined(TARGET_SH4) \
-    || defined(TARGET_M68K) || defined(TARGET_CRIS) || defined(TARGET_UNICORE32) \
-    || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
+    || defined(TARGET_M68K) || defined(TARGET_CRIS) \
+    || defined(TARGET_UNICORE32) || defined(TARGET_S390X) \
+    || defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
 
 #define TARGET_IOC_SIZEBITS	14
 #define TARGET_IOC_DIRBITS	2
@@ -234,7 +235,8 @@ struct target_cmsghdr {
 };
 
 #define TARGET_CMSG_DATA(cmsg) ((unsigned char *) ((struct target_cmsghdr *) (cmsg) + 1))
-#define TARGET_CMSG_NXTHDR(mhdr, cmsg) __target_cmsg_nxthdr (mhdr, cmsg)
+#define TARGET_CMSG_NXTHDR(mhdr, cmsg, cmsg_start) \
+                               __target_cmsg_nxthdr(mhdr, cmsg, cmsg_start)
 #define TARGET_CMSG_ALIGN(len) (((len) + sizeof (abi_long) - 1) \
                                & (size_t) ~(sizeof (abi_long) - 1))
 #define TARGET_CMSG_SPACE(len) (TARGET_CMSG_ALIGN (len) \
@@ -242,17 +244,20 @@ struct target_cmsghdr {
 #define TARGET_CMSG_LEN(len)   (TARGET_CMSG_ALIGN (sizeof (struct target_cmsghdr)) + (len))
 
 static __inline__ struct target_cmsghdr *
-__target_cmsg_nxthdr (struct target_msghdr *__mhdr, struct target_cmsghdr *__cmsg)
+__target_cmsg_nxthdr(struct target_msghdr *__mhdr,
+                     struct target_cmsghdr *__cmsg,
+                     struct target_cmsghdr *__cmsg_start)
 {
   struct target_cmsghdr *__ptr;
 
   __ptr = (struct target_cmsghdr *)((unsigned char *) __cmsg
                                     + TARGET_CMSG_ALIGN (tswapal(__cmsg->cmsg_len)));
-  if ((unsigned long)((char *)(__ptr+1) - (char *)(size_t)tswapal(__mhdr->msg_control))
-      > tswapal(__mhdr->msg_controllen))
+  if ((unsigned long)((char *)(__ptr+1) - (char *)__cmsg_start)
+      > tswapal(__mhdr->msg_controllen)) {
     /* No more entries.  */
     return (struct target_cmsghdr *)0;
-  return __cmsg;
+  }
+  return __ptr;
 }
 
 struct target_mmsghdr {
@@ -365,7 +370,8 @@ int do_sigaction(int sig, const struct target_sigaction *act,
     || defined(TARGET_PPC) || defined(TARGET_MIPS) || defined(TARGET_SH4) \
     || defined(TARGET_M68K) || defined(TARGET_ALPHA) || defined(TARGET_CRIS) \
     || defined(TARGET_MICROBLAZE) || defined(TARGET_UNICORE32) \
-    || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
+    || defined(TARGET_S390X) || defined(TARGET_OPENRISC) \
+    || defined(TARGET_TILEGX)
 
 #if defined(TARGET_SPARC)
 #define TARGET_SA_NOCLDSTOP    8u
@@ -742,6 +748,10 @@ typedef struct target_siginfo {
 #define TARGET_ILL_PRVREG	(6)	/* privileged register */
 #define TARGET_ILL_COPROC	(7)	/* coprocessor error */
 #define TARGET_ILL_BADSTK	(8)	/* internal stack error */
+#ifdef TARGET_TILEGX
+#define TARGET_ILL_DBLFLT       (9)     /* double fault */
+#define TARGET_ILL_HARDWALL     (10)    /* user networks hardwall violation */
+#endif
 
 /*
  * SIGFPE si_codes
@@ -761,6 +771,7 @@ typedef struct target_siginfo {
  */
 #define TARGET_SEGV_MAPERR     (1)  /* address not mapped to object */
 #define TARGET_SEGV_ACCERR     (2)  /* invalid permissions for mapped object */
+#define TARGET_SEGV_BNDERR     (3)  /* failed address bound checks */
 
 /*
  * SIGBUS si_codes
@@ -768,12 +779,18 @@ typedef struct target_siginfo {
 #define TARGET_BUS_ADRALN       (1)	/* invalid address alignment */
 #define TARGET_BUS_ADRERR       (2)	/* non-existent physical address */
 #define TARGET_BUS_OBJERR       (3)	/* object specific hardware error */
+/* hardware memory error consumed on a machine check: action required */
+#define TARGET_BUS_MCEERR_AR    (4)
+/* hardware memory error detected in process but not consumed: action optional*/
+#define TARGET_BUS_MCEERR_AO    (5)
 
 /*
  * SIGTRAP si_codes
  */
 #define TARGET_TRAP_BRKPT	(1)	/* process breakpoint */
 #define TARGET_TRAP_TRACE	(2)	/* process trace trap */
+#define TARGET_TRAP_BRANCH      (3)     /* process taken branch trap */
+#define TARGET_TRAP_HWBKPT      (4)     /* hardware breakpoint/watchpoint */
 
 #endif /* defined(TARGET_I386) || defined(TARGET_ARM) */
 
@@ -969,7 +986,7 @@ struct target_pollfd {
                                            (struct cdrom_multisession) */
 #define TARGET_CDROM_GET_MCN		0x5311 /* Obtain the "Universal Product Code"
                                            if available (struct cdrom_mcn) */
-#define TARGET_CDROM_GET_UPC		TARGET_CDROM_GET_MCN  /* This one is depricated,
+#define TARGET_CDROM_GET_UPC		TARGET_CDROM_GET_MCN  /* This one is deprecated,
                                           but here anyway for compatibility */
 #define TARGET_CDROMRESET		0x5312 /* hard-reset the drive */
 #define TARGET_CDROMVOLREAD		0x5313 /* Get the drive's volume setting
@@ -1871,7 +1888,7 @@ struct target_stat {
     abi_ulong  target_st_ctime_nsec;
     unsigned int __unused[2];
 };
-#elif defined(TARGET_OPENRISC)
+#elif defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
 
 /* These are the asm-generic versions of the stat and stat64 structures */
 
@@ -2264,7 +2281,9 @@ struct target_flock {
 struct target_flock64 {
 	short  l_type;
 	short  l_whence;
-#if defined(TARGET_PPC) || defined(TARGET_X86_64) || defined(TARGET_MIPS) || defined(TARGET_SPARC) || defined(TARGET_HPPA) || defined (TARGET_MICROBLAZE)
+#if defined(TARGET_PPC) || defined(TARGET_X86_64) || defined(TARGET_MIPS) \
+    || defined(TARGET_SPARC) || defined(TARGET_HPPA) \
+    || defined(TARGET_MICROBLAZE) || defined(TARGET_TILEGX)
         int __pad;
 #endif
 	unsigned long long l_start;
@@ -2289,7 +2308,7 @@ struct target_f_owner_ex {
 };
 
 /* soundcard defines */
-/* XXX: convert them all to arch indepedent entries */
+/* XXX: convert them all to arch independent entries */
 #define TARGET_SNDCTL_COPR_HALT           TARGET_IOWR('C',  7, int);
 #define TARGET_SNDCTL_COPR_LOAD           0xcfb04301
 #define TARGET_SNDCTL_COPR_RCODE          0xc0144303
diff --git a/linux-user/tilegx/syscall.h b/linux-user/tilegx/syscall.h
new file mode 100644
index 000000000..a938d4e90
--- /dev/null
+++ b/linux-user/tilegx/syscall.h
@@ -0,0 +1,43 @@
+#ifndef TILEGX_SYSCALLS_H
+#define TILEGX_SYSCALLS_H
+
+#define UNAME_MACHINE "tilegx"
+#define UNAME_MINIMUM_RELEASE "3.19"
+
+#define MMAP_SHIFT TARGET_PAGE_BITS
+
+#define TILEGX_IS_ERRNO(ret) \
+                       ((ret) > 0xfffffffffffff000ULL) /* errno is 0 -- 4096 */
+
+typedef uint64_t tilegx_reg_t;
+
+struct target_pt_regs {
+
+    union {
+        /* Saved main processor registers; 56..63 are special. */
+        tilegx_reg_t regs[56];
+        struct {
+            tilegx_reg_t __regs[53];
+            tilegx_reg_t tp;    /* aliases regs[TREG_TP] */
+            tilegx_reg_t sp;    /* aliases regs[TREG_SP] */
+            tilegx_reg_t lr;    /* aliases regs[TREG_LR] */
+        };
+    };
+
+    /* Saved special registers. */
+    tilegx_reg_t pc;            /* stored in EX_CONTEXT_K_0 */
+    tilegx_reg_t ex1;           /* stored in EX_CONTEXT_K_1 (PL and ICS bit) */
+    tilegx_reg_t faultnum;      /* fault number (INT_SWINT_1 for syscall) */
+    tilegx_reg_t orig_r0;       /* r0 at syscall entry, else zero */
+    tilegx_reg_t flags;         /* flags (see below) */
+    tilegx_reg_t cmpexch;       /* value of CMPEXCH_VALUE SPR at interrupt */
+    tilegx_reg_t pad[2];
+};
+
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+/* For faultnum */
+#define TARGET_INT_SWINT_1            14
+
+#endif
diff --git a/linux-user/tilegx/syscall_nr.h b/linux-user/tilegx/syscall_nr.h
new file mode 100644
index 000000000..1dca34837
--- /dev/null
+++ b/linux-user/tilegx/syscall_nr.h
@@ -0,0 +1,324 @@
+#ifndef TILEGX_SYSCALL_NR
+#define TILEGX_SYSCALL_NR
+
+/*
+ * Copy from linux kernel asm-generic/unistd.h, which tilegx uses.
+ */
+#define TARGET_NR_io_setup                      0
+#define TARGET_NR_io_destroy                    1
+#define TARGET_NR_io_submit                     2
+#define TARGET_NR_io_cancel                     3
+#define TARGET_NR_io_getevents                  4
+#define TARGET_NR_setxattr                      5
+#define TARGET_NR_lsetxattr                     6
+#define TARGET_NR_fsetxattr                     7
+#define TARGET_NR_getxattr                      8
+#define TARGET_NR_lgetxattr                     9
+#define TARGET_NR_fgetxattr                     10
+#define TARGET_NR_listxattr                     11
+#define TARGET_NR_llistxattr                    12
+#define TARGET_NR_flistxattr                    13
+#define TARGET_NR_removexattr                   14
+#define TARGET_NR_lremovexattr                  15
+#define TARGET_NR_fremovexattr                  16
+#define TARGET_NR_getcwd                        17
+#define TARGET_NR_lookup_dcookie                18
+#define TARGET_NR_eventfd2                      19
+#define TARGET_NR_epoll_create1                 20
+#define TARGET_NR_epoll_ctl                     21
+#define TARGET_NR_epoll_pwait                   22
+#define TARGET_NR_dup                           23
+#define TARGET_NR_dup3                          24
+#define TARGET_NR_fcntl                         25
+#define TARGET_NR_inotify_init1                 26
+#define TARGET_NR_inotify_add_watch             27
+#define TARGET_NR_inotify_rm_watch              28
+#define TARGET_NR_ioctl                         29
+#define TARGET_NR_ioprio_set                    30
+#define TARGET_NR_ioprio_get                    31
+#define TARGET_NR_flock                         32
+#define TARGET_NR_mknodat                       33
+#define TARGET_NR_mkdirat                       34
+#define TARGET_NR_unlinkat                      35
+#define TARGET_NR_symlinkat                     36
+#define TARGET_NR_linkat                        37
+#define TARGET_NR_renameat                      38
+#define TARGET_NR_umount2                       39
+#define TARGET_NR_mount                         40
+#define TARGET_NR_pivot_root                    41
+#define TARGET_NR_nfsservctl                    42
+#define TARGET_NR_statfs                        43
+#define TARGET_NR_fstatfs                       44
+#define TARGET_NR_truncate                      45
+#define TARGET_NR_ftruncate                     46
+#define TARGET_NR_fallocate                     47
+#define TARGET_NR_faccessat                     48
+#define TARGET_NR_chdir                         49
+#define TARGET_NR_fchdir                        50
+#define TARGET_NR_chroot                        51
+#define TARGET_NR_fchmod                        52
+#define TARGET_NR_fchmodat                      53
+#define TARGET_NR_fchownat                      54
+#define TARGET_NR_fchown                        55
+#define TARGET_NR_openat                        56
+#define TARGET_NR_close                         57
+#define TARGET_NR_vhangup                       58
+#define TARGET_NR_pipe2                         59
+#define TARGET_NR_quotactl                      60
+#define TARGET_NR_getdents64                    61
+#define TARGET_NR_lseek                         62
+#define TARGET_NR_read                          63
+#define TARGET_NR_write                         64
+#define TARGET_NR_readv                         65
+#define TARGET_NR_writev                        66
+#define TARGET_NR_pread64                       67
+#define TARGET_NR_pwrite64                      68
+#define TARGET_NR_preadv                        69
+#define TARGET_NR_pwritev                       70
+#define TARGET_NR_sendfile                      71
+#define TARGET_NR_pselect6                      72
+#define TARGET_NR_ppoll                         73
+#define TARGET_NR_signalfd4                     74
+#define TARGET_NR_vmsplice                      75
+#define TARGET_NR_splice                        76
+#define TARGET_NR_tee                           77
+#define TARGET_NR_readlinkat                    78
+#define TARGET_NR_fstatat64                     79 /* let syscall.c known */
+#define TARGET_NR_fstat                         80
+#define TARGET_NR_sync                          81
+#define TARGET_NR_fsync                         82
+#define TARGET_NR_fdatasync                     83
+#define TARGET_NR_sync_file_range               84 /* For tilegx, no range2 */
+#define TARGET_NR_timerfd_create                85
+#define TARGET_NR_timerfd_settime               86
+#define TARGET_NR_timerfd_gettime               87
+#define TARGET_NR_utimensat                     88
+#define TARGET_NR_acct                          89
+#define TARGET_NR_capget                        90
+#define TARGET_NR_capset                        91
+#define TARGET_NR_personality                   92
+#define TARGET_NR_exit                          93
+#define TARGET_NR_exit_group                    94
+#define TARGET_NR_waitid                        95
+#define TARGET_NR_set_tid_address               96
+#define TARGET_NR_unshare                       97
+#define TARGET_NR_futex                         98
+#define TARGET_NR_set_robust_list               99
+#define TARGET_NR_get_robust_list               100
+#define TARGET_NR_nanosleep                     101
+#define TARGET_NR_getitimer                     102
+#define TARGET_NR_setitimer                     103
+#define TARGET_NR_kexec_load                    104
+#define TARGET_NR_init_module                   105
+#define TARGET_NR_delete_module                 106
+#define TARGET_NR_timer_create                  107
+#define TARGET_NR_timer_gettime                 108
+#define TARGET_NR_timer_getoverrun              109
+#define TARGET_NR_timer_settime                 110
+#define TARGET_NR_timer_delete                  111
+#define TARGET_NR_clock_settime                 112
+#define TARGET_NR_clock_gettime                 113
+#define TARGET_NR_clock_getres                  114
+#define TARGET_NR_clock_nanosleep               115
+#define TARGET_NR_syslog                        116
+#define TARGET_NR_ptrace                        117
+#define TARGET_NR_sched_setparam                118
+#define TARGET_NR_sched_setscheduler            119
+#define TARGET_NR_sched_getscheduler            120
+#define TARGET_NR_sched_getparam                121
+#define TARGET_NR_sched_setaffinity             122
+#define TARGET_NR_sched_getaffinity             123
+#define TARGET_NR_sched_yield                   124
+#define TARGET_NR_sched_get_priority_max        125
+#define TARGET_NR_sched_get_priority_min        126
+#define TARGET_NR_sched_rr_get_interval         127
+#define TARGET_NR_restart_syscall               128
+#define TARGET_NR_kill                          129
+#define TARGET_NR_tkill                         130
+#define TARGET_NR_tgkill                        131
+#define TARGET_NR_sigaltstack                   132
+#define TARGET_NR_rt_sigsuspend                 133
+#define TARGET_NR_rt_sigaction                  134
+#define TARGET_NR_rt_sigprocmask                135
+#define TARGET_NR_rt_sigpending                 136
+#define TARGET_NR_rt_sigtimedwait               137
+#define TARGET_NR_rt_sigqueueinfo               138
+#define TARGET_NR_rt_sigreturn                  139
+#define TARGET_NR_setpriority                   140
+#define TARGET_NR_getpriority                   141
+#define TARGET_NR_reboot                        142
+#define TARGET_NR_setregid                      143
+#define TARGET_NR_setgid                        144
+#define TARGET_NR_setreuid                      145
+#define TARGET_NR_setuid                        146
+#define TARGET_NR_setresuid                     147
+#define TARGET_NR_getresuid                     148
+#define TARGET_NR_setresgid                     149
+#define TARGET_NR_getresgid                     150
+#define TARGET_NR_setfsuid                      151
+#define TARGET_NR_setfsgid                      152
+#define TARGET_NR_times                         153
+#define TARGET_NR_setpgid                       154
+#define TARGET_NR_getpgid                       155
+#define TARGET_NR_getsid                        156
+#define TARGET_NR_setsid                        157
+#define TARGET_NR_getgroups                     158
+#define TARGET_NR_setgroups                     159
+#define TARGET_NR_uname                         160
+#define TARGET_NR_sethostname                   161
+#define TARGET_NR_setdomainname                 162
+#define TARGET_NR_getrlimit                     163
+#define TARGET_NR_setrlimit                     164
+#define TARGET_NR_getrusage                     165
+#define TARGET_NR_umask                         166
+#define TARGET_NR_prctl                         167
+#define TARGET_NR_getcpu                        168
+#define TARGET_NR_gettimeofday                  169
+#define TARGET_NR_settimeofday                  170
+#define TARGET_NR_adjtimex                      171
+#define TARGET_NR_getpid                        172
+#define TARGET_NR_getppid                       173
+#define TARGET_NR_getuid                        174
+#define TARGET_NR_geteuid                       175
+#define TARGET_NR_getgid                        176
+#define TARGET_NR_getegid                       177
+#define TARGET_NR_gettid                        178
+#define TARGET_NR_sysinfo                       179
+#define TARGET_NR_mq_open                       180
+#define TARGET_NR_mq_unlink                     181
+#define TARGET_NR_mq_timedsend                  182
+#define TARGET_NR_mq_timedreceive               183
+#define TARGET_NR_mq_notify                     184
+#define TARGET_NR_mq_getsetattr                 185
+#define TARGET_NR_msgget                        186
+#define TARGET_NR_msgctl                        187
+#define TARGET_NR_msgrcv                        188
+#define TARGET_NR_msgsnd                        189
+#define TARGET_NR_semget                        190
+#define TARGET_NR_semctl                        191
+#define TARGET_NR_semtimedop                    192
+#define TARGET_NR_semop                         193
+#define TARGET_NR_shmget                        194
+#define TARGET_NR_shmctl                        195
+#define TARGET_NR_shmat                         196
+#define TARGET_NR_shmdt                         197
+#define TARGET_NR_socket                        198
+#define TARGET_NR_socketpair                    199
+#define TARGET_NR_bind                          200
+#define TARGET_NR_listen                        201
+#define TARGET_NR_accept                        202
+#define TARGET_NR_connect                       203
+#define TARGET_NR_getsockname                   204
+#define TARGET_NR_getpeername                   205
+#define TARGET_NR_sendto                        206
+#define TARGET_NR_recvfrom                      207
+#define TARGET_NR_setsockopt                    208
+#define TARGET_NR_getsockopt                    209
+#define TARGET_NR_shutdown                      210
+#define TARGET_NR_sendmsg                       211
+#define TARGET_NR_recvmsg                       212
+#define TARGET_NR_readahead                     213
+#define TARGET_NR_brk                           214
+#define TARGET_NR_munmap                        215
+#define TARGET_NR_mremap                        216
+#define TARGET_NR_add_key                       217
+#define TARGET_NR_request_key                   218
+#define TARGET_NR_keyctl                        219
+#define TARGET_NR_clone                         220
+#define TARGET_NR_execve                        221
+#define TARGET_NR_mmap                          222
+#define TARGET_NR_fadvise64                     223
+#define TARGET_NR_swapon                        224
+#define TARGET_NR_swapoff                       225
+#define TARGET_NR_mprotect                      226
+#define TARGET_NR_msync                         227
+#define TARGET_NR_mlock                         228
+#define TARGET_NR_munlock                       229
+#define TARGET_NR_mlockall                      230
+#define TARGET_NR_munlockall                    231
+#define TARGET_NR_mincore                       232
+#define TARGET_NR_madvise                       233
+#define TARGET_NR_remap_file_pages              234
+#define TARGET_NR_mbind                         235
+#define TARGET_NR_get_mempolicy                 236
+#define TARGET_NR_set_mempolicy                 237
+#define TARGET_NR_migrate_pages                 238
+#define TARGET_NR_move_pages                    239
+#define TARGET_NR_rt_tgsigqueueinfo             240
+#define TARGET_NR_perf_event_open               241
+#define TARGET_NR_accept4                       242
+#define TARGET_NR_recvmmsg                      243
+
+#define TARGET_NR_arch_specific_syscall         244
+#define TARGET_NR_cacheflush                    245  /* tilegx own syscall */
+
+#define TARGET_NR_wait4                         260
+#define TARGET_NR_prlimit64                     261
+#define TARGET_NR_fanotify_init                 262
+#define TARGET_NR_fanotify_mark                 263
+#define TARGET_NR_name_to_handle_at             264
+#define TARGET_NR_open_by_handle_at             265
+#define TARGET_NR_clock_adjtime                 266
+#define TARGET_NR_syncfs                        267
+#define TARGET_NR_setns                         268
+#define TARGET_NR_sendmmsg                      269
+#define TARGET_NR_process_vm_readv              270
+#define TARGET_NR_process_vm_writev             271
+#define TARGET_NR_kcmp                          272
+#define TARGET_NR_finit_module                  273
+#define TARGET_NR_sched_setattr                 274
+#define TARGET_NR_sched_getattr                 275
+#define TARGET_NR_renameat2                     276
+#define TARGET_NR_seccomp                       277
+#define TARGET_NR_getrandom                     278
+#define TARGET_NR_memfd_create                  279
+#define TARGET_NR_bpf                           280
+#define TARGET_NR_execveat                      281
+
+#define TARGET_NR_open                          1024
+#define TARGET_NR_link                          1025
+#define TARGET_NR_unlink                        1026
+#define TARGET_NR_mknod                         1027
+#define TARGET_NR_chmod                         1028
+#define TARGET_NR_chown                         1029
+#define TARGET_NR_mkdir                         1030
+#define TARGET_NR_rmdir                         1031
+#define TARGET_NR_lchown                        1032
+#define TARGET_NR_access                        1033
+#define TARGET_NR_rename                        1034
+#define TARGET_NR_readlink                      1035
+#define TARGET_NR_symlink                       1036
+#define TARGET_NR_utimes                        1037
+#define TARGET_NR_stat64                        1038 /* let syscall.c known */
+#define TARGET_NR_lstat                         1039
+
+#define TARGET_NR_pipe                          1040
+#define TARGET_NR_dup2                          1041
+#define TARGET_NR_epoll_create                  1042
+#define TARGET_NR_inotify_init                  1043
+#define TARGET_NR_eventfd                       1044
+#define TARGET_NR_signalfd                      1045
+
+#define TARGET_NR_alarm                         1059
+#define TARGET_NR_getpgrp                       1060
+#define TARGET_NR_pause                         1061
+#define TARGET_NR_time                          1062
+#define TARGET_NR_utime                         1063
+#define TARGET_NR_creat                         1064
+#define TARGET_NR_getdents                      1065
+#define TARGET_NR_futimesat                     1066
+#define TARGET_NR_select                        1067
+#define TARGET_NR_poll                          1068
+#define TARGET_NR_epoll_wait                    1069
+#define TARGET_NR_ustat                         1070
+#define TARGET_NR_vfork                         1071
+#define TARGET_NR_oldwait4                      1072
+#define TARGET_NR_recv                          1073
+#define TARGET_NR_send                          1074
+#define TARGET_NR_bdflush                       1075
+#define TARGET_NR_umount                        1076
+#define TARGET_NR_uselib                        1077
+#define TARGET_NR__sysctl                       1078
+#define TARGET_NR_fork                          1079
+
+#endif
diff --git a/linux-user/tilegx/target_cpu.h b/linux-user/tilegx/target_cpu.h
new file mode 100644
index 000000000..c96e81d05
--- /dev/null
+++ b/linux-user/tilegx/target_cpu.h
@@ -0,0 +1,35 @@
+/*
+ * TILE-Gx specific CPU ABI and functions for linux-user
+ *
+ * Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TARGET_CPU_H
+#define TARGET_CPU_H
+
+static inline void cpu_clone_regs(CPUTLGState *env, target_ulong newsp)
+{
+    if (newsp) {
+        env->regs[TILEGX_R_SP] = newsp;
+    }
+    env->regs[TILEGX_R_RE] = 0;
+}
+
+static inline void cpu_set_tls(CPUTLGState *env, target_ulong newtls)
+{
+    env->regs[TILEGX_R_TP] = newtls;
+}
+
+#endif
diff --git a/linux-user/tilegx/target_signal.h b/linux-user/tilegx/target_signal.h
new file mode 100644
index 000000000..b595f985c
--- /dev/null
+++ b/linux-user/tilegx/target_signal.h
@@ -0,0 +1,28 @@
+#ifndef TARGET_SIGNAL_H
+#define TARGET_SIGNAL_H
+
+#include "cpu.h"
+
+/* this struct defines a stack used during syscall handling */
+
+typedef struct target_sigaltstack {
+    abi_ulong ss_sp;
+    abi_int ss_flags;
+    abi_ulong ss_size;
+} target_stack_t;
+
+/*
+ * sigaltstack controls
+ */
+#define TARGET_SS_ONSTACK     1
+#define TARGET_SS_DISABLE     2
+
+#define TARGET_MINSIGSTKSZ    2048
+#define TARGET_SIGSTKSZ       8192
+
+static inline abi_ulong get_sp_from_cpustate(CPUTLGState *state)
+{
+    return state->regs[TILEGX_R_SP];
+}
+
+#endif /* TARGET_SIGNAL_H */
diff --git a/linux-user/tilegx/target_structs.h b/linux-user/tilegx/target_structs.h
new file mode 100644
index 000000000..7d3ff782f
--- /dev/null
+++ b/linux-user/tilegx/target_structs.h
@@ -0,0 +1,46 @@
+/*
+ * TILE-Gx specific structures for linux-user
+ *
+ * Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TARGET_STRUCTS_H
+#define TARGET_STRUCTS_H
+
+struct target_ipc_perm {
+    abi_int __key;                      /* Key.  */
+    abi_uint uid;                       /* Owner's user ID.  */
+    abi_uint gid;                       /* Owner's group ID.  */
+    abi_uint cuid;                      /* Creator's user ID.  */
+    abi_uint cgid;                      /* Creator's group ID.  */
+    abi_uint mode;                      /* Read/write permission.  */
+    abi_ushort __seq;                   /* Sequence number.  */
+};
+
+struct target_shmid_ds {
+    struct target_ipc_perm shm_perm;    /* operation permission struct */
+    abi_long shm_segsz;                 /* size of segment in bytes */
+    abi_ulong shm_atime;                /* time of last shmat() */
+    abi_ulong shm_dtime;                /* time of last shmdt() */
+    abi_ulong shm_ctime;                /* time of last change by shmctl() */
+    abi_int shm_cpid;                   /* pid of creator */
+    abi_int shm_lpid;                   /* pid of last shmop */
+    abi_ushort shm_nattch;              /* number of current attaches */
+    abi_ushort shm_unused;              /* compatibility */
+    abi_ulong __unused4;
+    abi_ulong __unused5;
+};
+
+#endif
diff --git a/linux-user/tilegx/termbits.h b/linux-user/tilegx/termbits.h
new file mode 100644
index 000000000..91ec23654
--- /dev/null
+++ b/linux-user/tilegx/termbits.h
@@ -0,0 +1,274 @@
+#ifndef TILEGX_TERMBITS_H
+#define TILEGX_TERMBITS_H
+
+/* From asm-generic/termbits.h, which is used by tilegx */
+
+#define TARGET_NCCS 19
+struct target_termios {
+    unsigned int c_iflag;             /* input mode flags */
+    unsigned int c_oflag;             /* output mode flags */
+    unsigned int c_cflag;             /* control mode flags */
+    unsigned int c_lflag;             /* local mode flags */
+    unsigned char c_line;             /* line discipline */
+    unsigned char c_cc[TARGET_NCCS];  /* control characters */
+};
+
+struct target_termios2 {
+    unsigned int c_iflag;             /* input mode flags */
+    unsigned int c_oflag;             /* output mode flags */
+    unsigned int c_cflag;             /* control mode flags */
+    unsigned int c_lflag;             /* local mode flags */
+    unsigned char c_line;             /* line discipline */
+    unsigned char c_cc[TARGET_NCCS];  /* control characters */
+    unsigned int c_ispeed;            /* input speed */
+    unsigned int c_ospeed;            /* output speed */
+};
+
+/* c_cc characters */
+#define TARGET_VINTR     0
+#define TARGET_VQUIT     1
+#define TARGET_VERASE    2
+#define TARGET_VKILL     3
+#define TARGET_VEOF      4
+#define TARGET_VTIME     5
+#define TARGET_VMIN      6
+#define TARGET_VSWTC     7
+#define TARGET_VSTART    8
+#define TARGET_VSTOP     9
+#define TARGET_VSUSP     10
+#define TARGET_VEOL      11
+#define TARGET_VREPRINT  12
+#define TARGET_VDISCARD  13
+#define TARGET_VWERASE   14
+#define TARGET_VLNEXT    15
+#define TARGET_VEOL2     16
+
+/* c_iflag bits */
+#define TARGET_IGNBRK    0000001
+#define TARGET_BRKINT    0000002
+#define TARGET_IGNPAR    0000004
+#define TARGET_PARMRK    0000010
+#define TARGET_INPCK     0000020
+#define TARGET_ISTRIP    0000040
+#define TARGET_INLCR     0000100
+#define TARGET_IGNCR     0000200
+#define TARGET_ICRNL     0000400
+#define TARGET_IUCLC     0001000
+#define TARGET_IXON      0002000
+#define TARGET_IXANY     0004000
+#define TARGET_IXOFF     0010000
+#define TARGET_IMAXBEL   0020000
+#define TARGET_IUTF8     0040000
+
+/* c_oflag bits */
+#define TARGET_OPOST     0000001
+#define TARGET_OLCUC     0000002
+#define TARGET_ONLCR     0000004
+#define TARGET_OCRNL     0000010
+#define TARGET_ONOCR     0000020
+#define TARGET_ONLRET    0000040
+#define TARGET_OFILL     0000100
+#define TARGET_OFDEL     0000200
+#define TARGET_NLDLY     0000400
+#define   TARGET_NL0     0000000
+#define   TARGET_NL1     0000400
+#define TARGET_CRDLY     0003000
+#define   TARGET_CR0     0000000
+#define   TARGET_CR1     0001000
+#define   TARGET_CR2     0002000
+#define   TARGET_CR3     0003000
+#define TARGET_TABDLY    0014000
+#define   TARGET_TAB0    0000000
+#define   TARGET_TAB1    0004000
+#define   TARGET_TAB2    0010000
+#define   TARGET_TAB3    0014000
+#define   TARGET_XTABS   0014000
+#define TARGET_BSDLY     0020000
+#define   TARGET_BS0     0000000
+#define   TARGET_BS1     0020000
+#define TARGET_VTDLY     0040000
+#define   TARGET_VT0     0000000
+#define   TARGET_VT1     0040000
+#define TARGET_FFDLY     0100000
+#define   TARGET_FF0     0000000
+#define   TARGET_FF1     0100000
+
+/* c_cflag bit meaning */
+#define TARGET_CBAUD     0010017
+#define  TARGET_B0       0000000        /* hang up */
+#define  TARGET_B50      0000001
+#define  TARGET_B75      0000002
+#define  TARGET_B110     0000003
+#define  TARGET_B134     0000004
+#define  TARGET_B150     0000005
+#define  TARGET_B200     0000006
+#define  TARGET_B300     0000007
+#define  TARGET_B600     0000010
+#define  TARGET_B1200    0000011
+#define  TARGET_B1800    0000012
+#define  TARGET_B2400    0000013
+#define  TARGET_B4800    0000014
+#define  TARGET_B9600    0000015
+#define  TARGET_B19200   0000016
+#define  TARGET_B38400   0000017
+#define TARGET_EXTA      TARGET_B19200
+#define TARGET_EXTB      TARGET_B38400
+#define TARGET_CSIZE     0000060
+#define   TARGET_CS5     0000000
+#define   TARGET_CS6     0000020
+#define   TARGET_CS7     0000040
+#define   TARGET_CS8     0000060
+#define TARGET_CSTOPB    0000100
+#define TARGET_CREAD     0000200
+#define TARGET_PARENB    0000400
+#define TARGET_PARODD    0001000
+#define TARGET_HUPCL     0002000
+#define TARGET_CLOCAL    0004000
+#define TARGET_CBAUDEX   0010000
+#define    TARGET_BOTHER 0010000
+#define    TARGET_B57600 0010001
+#define   TARGET_B115200 0010002
+#define   TARGET_B230400 0010003
+#define   TARGET_B460800 0010004
+#define   TARGET_B500000 0010005
+#define   TARGET_B576000 0010006
+#define   TARGET_B921600 0010007
+#define  TARGET_B1000000 0010010
+#define  TARGET_B1152000 0010011
+#define  TARGET_B1500000 0010012
+#define  TARGET_B2000000 0010013
+#define  TARGET_B2500000 0010014
+#define  TARGET_B3000000 0010015
+#define  TARGET_B3500000 0010016
+#define  TARGET_B4000000 0010017
+#define TARGET_CIBAUD    002003600000    /* input baud rate */
+#define TARGET_CMSPAR    010000000000    /* mark or space (stick) parity */
+#define TARGET_CRTSCTS   020000000000    /* flow control */
+
+#define TARGET_IBSHIFT   16        /* Shift from CBAUD to CIBAUD */
+
+/* c_lflag bits */
+#define TARGET_ISIG      0000001
+#define TARGET_ICANON    0000002
+#define TARGET_XCASE     0000004
+#define TARGET_ECHO      0000010
+#define TARGET_ECHOE     0000020
+#define TARGET_ECHOK     0000040
+#define TARGET_ECHONL    0000100
+#define TARGET_NOFLSH    0000200
+#define TARGET_TOSTOP    0000400
+#define TARGET_ECHOCTL   0001000
+#define TARGET_ECHOPRT   0002000
+#define TARGET_ECHOKE    0004000
+#define TARGET_FLUSHO    0010000
+#define TARGET_PENDIN    0040000
+#define TARGET_IEXTEN    0100000
+#define TARGET_EXTPROC   0200000
+
+/* tcflow() and TCXONC use these */
+#define TARGET_TCOOFF    0
+#define TARGET_TCOON     1
+#define TARGET_TCIOFF    2
+#define TARGET_TCION     3
+
+/* tcflush() and TCFLSH use these */
+#define TARGET_TCIFLUSH  0
+#define TARGET_TCOFLUSH  1
+#define TARGET_TCIOFLUSH 2
+
+/* tcsetattr uses these */
+#define TARGET_TCSANOW   0
+#define TARGET_TCSADRAIN 1
+#define TARGET_TCSAFLUSH 2
+
+/* From asm-generic/ioctls.h, which is used by tilegx */
+
+#define TARGET_TCGETS                   0x5401
+#define TARGET_TCSETS                   0x5402
+#define TARGET_TCSETSW                  0x5403
+#define TARGET_TCSETSF                  0x5404
+#define TARGET_TCGETA                   0x5405
+#define TARGET_TCSETA                   0x5406
+#define TARGET_TCSETAW                  0x5407
+#define TARGET_TCSETAF                  0x5408
+#define TARGET_TCSBRK                   0x5409
+#define TARGET_TCXONC                   0x540A
+#define TARGET_TCFLSH                   0x540B
+#define TARGET_TIOCEXCL                 0x540C
+#define TARGET_TIOCNXCL                 0x540D
+#define TARGET_TIOCSCTTY                0x540E
+#define TARGET_TIOCGPGRP                0x540F
+#define TARGET_TIOCSPGRP                0x5410
+#define TARGET_TIOCOUTQ                 0x5411
+#define TARGET_TIOCSTI                  0x5412
+#define TARGET_TIOCGWINSZ               0x5413
+#define TARGET_TIOCSWINSZ               0x5414
+#define TARGET_TIOCMGET                 0x5415
+#define TARGET_TIOCMBIS                 0x5416
+#define TARGET_TIOCMBIC                 0x5417
+#define TARGET_TIOCMSET                 0x5418
+#define TARGET_TIOCGSOFTCAR             0x5419
+#define TARGET_TIOCSSOFTCAR             0x541A
+#define TARGET_FIONREAD                 0x541B
+#define TARGET_TIOCINQ                  TARGET_FIONREAD
+#define TARGET_TIOCLINUX                0x541C
+#define TARGET_TIOCCONS                 0x541D
+#define TARGET_TIOCGSERIAL              0x541E
+#define TARGET_TIOCSSERIAL              0x541F
+#define TARGET_TIOCPKT                  0x5420
+#define TARGET_FIONBIO                  0x5421
+#define TARGET_TIOCNOTTY                0x5422
+#define TARGET_TIOCSETD                 0x5423
+#define TARGET_TIOCGETD                 0x5424
+#define TARGET_TCSBRKP                  0x5425
+#define TARGET_TIOCSBRK                 0x5427
+#define TARGET_TIOCCBRK                 0x5428
+#define TARGET_TIOCGSID                 0x5429
+#define TARGET_TCGETS2                  TARGET_IOR('T', 0x2A, struct termios2)
+#define TARGET_TCSETS2                  TARGET_IOW('T', 0x2B, struct termios2)
+#define TARGET_TCSETSW2                 TARGET_IOW('T', 0x2C, struct termios2)
+#define TARGET_TCSETSF2                 TARGET_IOW('T', 0x2D, struct termios2)
+#define TARGET_TIOCGRS485               0x542E
+#define TARGET_TIOCSRS485               0x542F
+#define TARGET_TIOCGPTN                 TARGET_IOR('T', 0x30, unsigned int)
+#define TARGET_TIOCSPTLCK               TARGET_IOW('T', 0x31, int)
+#define TARGET_TIOCGDEV                 TARGET_IOR('T', 0x32, unsigned int)
+#define TARGET_TCGETX                   0x5432
+#define TARGET_TCSETX                   0x5433
+#define TARGET_TCSETXF                  0x5434
+#define TARGET_TCSETXW                  0x5435
+#define TARGET_TIOCSIG                  TARGET_IOW('T', 0x36, int)
+#define TARGET_TIOCVHANGUP              0x5437
+#define TARGET_TIOCGPKT                 TARGET_IOR('T', 0x38, int)
+#define TARGET_TIOCGPTLCK               TARGET_IOR('T', 0x39, int)
+#define TARGET_TIOCGEXCL                TARGET_IOR('T', 0x40, int)
+
+#define TARGET_FIONCLEX                 0x5450
+#define TARGET_FIOCLEX                  0x5451
+#define TARGET_FIOASYNC                 0x5452
+#define TARGET_TIOCSERCONFIG            0x5453
+#define TARGET_TIOCSERGWILD             0x5454
+#define TARGET_TIOCSERSWILD             0x5455
+#define TARGET_TIOCGLCKTRMIOS           0x5456
+#define TARGET_TIOCSLCKTRMIOS           0x5457
+#define TARGET_TIOCSERGSTRUCT           0x5458
+#define TARGET_TIOCSERGETLSR            0x5459
+#define TARGET_TIOCSERGETMULTI          0x545A
+#define TARGET_TIOCSERSETMULTI          0x545B
+
+#define TARGET_TIOCMIWAIT               0x545C
+#define TARGET_TIOCGICOUNT              0x545D
+#define TARGET_FIOQSIZE                 0x5460
+
+#define TARGET_TIOCPKT_DATA             0
+#define TARGET_TIOCPKT_FLUSHREAD        1
+#define TARGET_TIOCPKT_FLUSHWRITE       2
+#define TARGET_TIOCPKT_STOP             4
+#define TARGET_TIOCPKT_START            8
+#define TARGET_TIOCPKT_NOSTOP           16
+#define TARGET_TIOCPKT_DOSTOP           32
+#define TARGET_TIOCPKT_IOCTL            64
+
+#define TARGET_TIOCSER_TEMT             0x01
+
+#endif
diff --git a/main-loop.c b/main-loop.c
index 39970437f..587761538 100644
--- a/main-loop.c
+++ b/main-loop.c
@@ -161,6 +161,9 @@ int qemu_init_main_loop(Error **errp)
     src = aio_get_g_source(qemu_aio_context);
     g_source_attach(src, NULL);
     g_source_unref(src);
+    src = iohandler_get_g_source();
+    g_source_attach(src, NULL);
+    g_source_unref(src);
     return 0;
 }
 
@@ -227,7 +230,7 @@ static int os_host_main_loop_wait(int64_t timeout)
     if (!timeout && (spin_counter > MAX_MAIN_LOOP_SPIN)) {
         static bool notified;
 
-        if (!notified && !qtest_enabled()) {
+        if (!notified && !qtest_driver()) {
             fprintf(stderr,
                     "main-loop: WARNING: I/O thread spun for %d iterations\n",
                     MAX_MAIN_LOOP_SPIN);
@@ -487,7 +490,6 @@ int main_loop_wait(int nonblocking)
 #ifdef CONFIG_SLIRP
     slirp_pollfds_fill(gpollfds, &timeout);
 #endif
-    qemu_iohandler_fill(gpollfds);
 
     if (timeout == UINT32_MAX) {
         timeout_ns = -1;
@@ -500,11 +502,13 @@ int main_loop_wait(int nonblocking)
                                           &main_loop_tlg));
 
     ret = os_host_main_loop_wait(timeout_ns);
-    qemu_iohandler_poll(gpollfds, ret);
 #ifdef CONFIG_SLIRP
     slirp_pollfds_poll(gpollfds, (ret < 0));
 #endif
 
+    /* CPU thread can infinitely wait for event after
+       missing the warp */
+    qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
     qemu_clock_run_all_timers();
 
     return ret;
diff --git a/memory.c b/memory.c
index 10c1df536..e193658fc 100644
--- a/memory.c
+++ b/memory.c
@@ -1182,7 +1182,7 @@ void memory_region_init_io(MemoryRegion *mr,
                            uint64_t size)
 {
     memory_region_init(mr, owner, name, size);
-    mr->ops = ops;
+    mr->ops = ops ? ops : &unassigned_mem_ops;
     mr->opaque = opaque;
     mr->terminates = true;
 }
@@ -1251,7 +1251,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
 
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
-    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort);
+    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
 }
 
 void memory_region_set_skip_dump(MemoryRegion *mr)
@@ -1300,14 +1300,6 @@ void memory_region_init_iommu(MemoryRegion *mr,
     notifier_list_init(&mr->iommu_notify);
 }
 
-void memory_region_init_reservation(MemoryRegion *mr,
-                                    Object *owner,
-                                    const char *name,
-                                    uint64_t size)
-{
-    memory_region_init_io(mr, owner, &unassigned_mem_ops, mr, name, size);
-}
-
 static void memory_region_finalize(Object *obj)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
@@ -1426,6 +1418,26 @@ void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n)
     notifier_list_add(&mr->iommu_notify, n);
 }
 
+void memory_region_iommu_replay(MemoryRegion *mr, Notifier *n,
+                                hwaddr granularity, bool is_write)
+{
+    hwaddr addr;
+    IOMMUTLBEntry iotlb;
+
+    for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
+        iotlb = mr->iommu_ops->translate(mr, addr, is_write);
+        if (iotlb.perm != IOMMU_NONE) {
+            n->notify(n, &iotlb);
+        }
+
+        /* if (2^64 - MR size) < granularity, it's possible to get an
+         * infinite loop here.  This should catch such a wraparound */
+        if ((addr + granularity) < addr) {
+            break;
+        }
+    }
+}
+
 void memory_region_unregister_iommu_notifier(Notifier *n)
 {
     notifier_remove(n);
@@ -1676,7 +1688,9 @@ void memory_region_add_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
-    adjust_endianness(mr, &mrfd.data, size);
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
@@ -1709,7 +1723,9 @@ void memory_region_del_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
-    adjust_endianness(mr, &mrfd.data, size);
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
@@ -2024,6 +2040,9 @@ static void listener_add_address_space(MemoryListener *listener,
         return;
     }
 
+    if (listener->begin) {
+        listener->begin(listener);
+    }
     if (global_dirty_log) {
         if (listener->log_global_start) {
             listener->log_global_start(listener);
@@ -2040,10 +2059,16 @@ static void listener_add_address_space(MemoryListener *listener,
             .offset_within_address_space = int128_get64(fr->addr.start),
             .readonly = fr->readonly,
         };
+        if (fr->dirty_log_mask && listener->log_start) {
+            listener->log_start(listener, &section, 0, fr->dirty_log_mask);
+        }
         if (listener->region_add) {
             listener->region_add(listener, &section);
         }
     }
+    if (listener->commit) {
+        listener->commit(listener);
+    }
     flatview_unref(view);
 }
 
diff --git a/migration/Makefile.objs b/migration/Makefile.objs
index d929e969a..0cac6d707 100644
--- a/migration/Makefile.objs
+++ b/migration/Makefile.objs
@@ -1,7 +1,7 @@
 common-obj-y += migration.o tcp.o
 common-obj-y += vmstate.o
 common-obj-y += qemu-file.o qemu-file-buf.o qemu-file-unix.o qemu-file-stdio.o
-common-obj-y += xbzrle.o
+common-obj-y += xbzrle.o postcopy-ram.o
 
 common-obj-$(CONFIG_RDMA) += rdma.o
 common-obj-$(CONFIG_POSIX) += exec.o unix.o fd.o
diff --git a/migration/block.c b/migration/block.c
index ed865ed23..656f38f34 100644
--- a/migration/block.c
+++ b/migration/block.c
@@ -36,6 +36,8 @@
 
 #define MAX_IS_ALLOCATED_SEARCH 65536
 
+#define MAX_INFLIGHT_IO 512
+
 //#define DEBUG_BLK_MIGRATION
 
 #ifdef DEBUG_BLK_MIGRATION
@@ -591,7 +593,7 @@ static int64_t get_remaining_dirty(void)
 
 /* Called with iothread lock taken.  */
 
-static void blk_mig_cleanup(void)
+static void block_migration_cleanup(void *opaque)
 {
     BlkMigDevState *bmds;
     BlkMigBlock *blk;
@@ -618,11 +620,6 @@ static void blk_mig_cleanup(void)
     blk_mig_unlock();
 }
 
-static void block_migration_cancel(void *opaque)
-{
-    blk_mig_cleanup();
-}
-
 static int block_save_setup(QEMUFile *f, void *opaque)
 {
     int ret;
@@ -670,7 +667,10 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
     blk_mig_lock();
     while ((block_mig_state.submitted +
             block_mig_state.read_done) * BLOCK_SIZE <
-           qemu_file_get_rate_limit(f)) {
+           qemu_file_get_rate_limit(f) &&
+           (block_mig_state.submitted +
+            block_mig_state.read_done) <
+           MAX_INFLIGHT_IO) {
         blk_mig_unlock();
         if (block_mig_state.bulk_completed == 0) {
             /* first finish the bulk phase */
@@ -750,11 +750,12 @@ static int block_save_complete(QEMUFile *f, void *opaque)
 
     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 
-    blk_mig_cleanup();
     return 0;
 }
 
-static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                               uint64_t *non_postcopiable_pending,
+                               uint64_t *postcopiable_pending)
 {
     /* Estimate pending number of bytes to send */
     uint64_t pending;
@@ -773,7 +774,8 @@ static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
     qemu_mutex_unlock_iothread();
 
     DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
-    return pending;
+    /* We don't do postcopy */
+    *non_postcopiable_pending += pending;
 }
 
 static int block_load(QEMUFile *f, void *opaque, int version_id)
@@ -808,6 +810,11 @@ static int block_load(QEMUFile *f, void *opaque, int version_id)
                 return -EINVAL;
             }
             bs = blk_bs(blk);
+            if (!bs) {
+                fprintf(stderr, "Block device %s has no medium\n",
+                        device_name);
+                return -EINVAL;
+            }
 
             if (bs != bs_prev) {
                 bs_prev = bs;
@@ -877,10 +884,10 @@ static SaveVMHandlers savevm_block_handlers = {
     .set_params = block_set_params,
     .save_live_setup = block_save_setup,
     .save_live_iterate = block_save_iterate,
-    .save_live_complete = block_save_complete,
+    .save_live_complete_precopy = block_save_complete,
     .save_live_pending = block_save_pending,
     .load_state = block_load,
-    .cancel = block_migration_cancel,
+    .cleanup = block_migration_cleanup,
     .is_active = block_is_active,
 };
 
diff --git a/migration/migration.c b/migration/migration.c
index c4a7d0b70..adc6b6f1c 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -21,16 +21,20 @@
 #include "sysemu/sysemu.h"
 #include "block/block.h"
 #include "qapi/qmp/qerror.h"
+#include "qapi/util.h"
 #include "qemu/sockets.h"
 #include "qemu/rcu.h"
 #include "migration/block.h"
+#include "migration/postcopy-ram.h"
 #include "qemu/thread.h"
 #include "qmp-commands.h"
 #include "trace.h"
-#include "qapi/util.h"
 #include "qapi-event.h"
+#include "qom/cpu.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
-#define MAX_THROTTLE  (32 << 20)      /* Migration speed throttling */
+#define MAX_THROTTLE  (32 << 20)      /* Migration transfer speed throttling */
 
 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  * data. */
@@ -44,6 +48,9 @@
 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
+/* Define default autoconverge cpu throttle migration parameters */
+#define DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL 20
+#define DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT 10
 
 /* Migration XBZRLE default cache size */
 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
@@ -53,6 +60,13 @@ static NotifierList migration_state_notifiers =
 
 static bool deferred_incoming;
 
+/*
+ * Current state of incoming postcopy; note this is not part of
+ * MigrationIncomingState since it's state is used during cleanup
+ * at the end as MIS is being freed.
+ */
+static PostcopyState incoming_postcopy_state;
+
 /* When we add fault tolerance, we could have several
    migrations at once.  For now we don't need to add
    dynamic creation of migration */
@@ -60,6 +74,7 @@ static bool deferred_incoming;
 /* For outgoing */
 MigrationState *migrate_get_current(void)
 {
+    static bool once;
     static MigrationState current_migration = {
         .state = MIGRATION_STATUS_NONE,
         .bandwidth_limit = MAX_THROTTLE,
@@ -71,8 +86,16 @@ MigrationState *migrate_get_current(void)
                 DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT,
         .parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
                 DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT,
+        .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] =
+                DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL,
+        .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] =
+                DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT,
     };
 
+    if (!once) {
+        qemu_mutex_init(&current_migration.src_page_req_mutex);
+        once = true;
+    }
     return &current_migration;
 }
 
@@ -86,15 +109,18 @@ MigrationIncomingState *migration_incoming_get_current(void)
 
 MigrationIncomingState *migration_incoming_state_new(QEMUFile* f)
 {
-    mis_current = g_malloc0(sizeof(MigrationIncomingState));
-    mis_current->file = f;
+    mis_current = g_new0(MigrationIncomingState, 1);
+    mis_current->from_src_file = f;
     QLIST_INIT(&mis_current->loadvm_handlers);
+    qemu_mutex_init(&mis_current->rp_mutex);
+    qemu_event_init(&mis_current->main_thread_load_event, false);
 
     return mis_current;
 }
 
 void migration_incoming_state_destroy(void)
 {
+    qemu_event_destroy(&mis_current->main_thread_load_event);
     loadvm_free_handlers(mis_current);
     g_free(mis_current);
     mis_current = NULL;
@@ -240,6 +266,35 @@ static void deferred_incoming_migration(Error **errp)
     deferred_incoming = true;
 }
 
+/* Request a range of pages from the source VM at the given
+ * start address.
+ *   rbname: Name of the RAMBlock to request the page in, if NULL it's the same
+ *           as the last request (a name must have been given previously)
+ *   Start: Address offset within the RB
+ *   Len: Length in bytes required - must be a multiple of pagesize
+ */
+void migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
+                               ram_addr_t start, size_t len)
+{
+    uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname upto 256 */
+    size_t msglen = 12; /* start + len */
+
+    *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
+    *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
+
+    if (rbname) {
+        int rbname_len = strlen(rbname);
+        assert(rbname_len < 256);
+
+        bufc[msglen++] = rbname_len;
+        memcpy(bufc + msglen, rbname, rbname_len);
+        msglen += rbname_len;
+        migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES_ID, msglen, bufc);
+    } else {
+        migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES, msglen, bufc);
+    }
+}
+
 void qemu_start_incoming_migration(const char *uri, Error **errp)
 {
     const char *p;
@@ -270,12 +325,37 @@ static void process_incoming_migration_co(void *opaque)
 {
     QEMUFile *f = opaque;
     Error *local_err = NULL;
+    MigrationIncomingState *mis;
+    PostcopyState ps;
     int ret;
 
-    migration_incoming_state_new(f);
+    mis = migration_incoming_state_new(f);
+    postcopy_state_set(POSTCOPY_INCOMING_NONE);
     migrate_generate_event(MIGRATION_STATUS_ACTIVE);
+
     ret = qemu_loadvm_state(f);
 
+    ps = postcopy_state_get();
+    trace_process_incoming_migration_co_end(ret, ps);
+    if (ps != POSTCOPY_INCOMING_NONE) {
+        if (ps == POSTCOPY_INCOMING_ADVISE) {
+            /*
+             * Where a migration had postcopy enabled (and thus went to advise)
+             * but managed to complete within the precopy period, we can use
+             * the normal exit.
+             */
+            postcopy_ram_incoming_cleanup(mis);
+        } else if (ret >= 0) {
+            /*
+             * Postcopy was started, cleanup should happen at the end of the
+             * postcopy thread.
+             */
+            trace_process_incoming_migration_co_postcopy_end_main();
+            return;
+        }
+        /* Else if something went wrong then just fall out of the normal exit */
+    }
+
     qemu_fclose(f);
     free_xbzrle_decoded_buf();
     migration_incoming_state_destroy();
@@ -286,7 +366,6 @@ static void process_incoming_migration_co(void *opaque)
         migrate_decompress_threads_join();
         exit(EXIT_FAILURE);
     }
-    qemu_announce_self();
 
     /* Make sure all file formats flush their mutable metadata */
     bdrv_invalidate_cache_all(&local_err);
@@ -297,6 +376,12 @@ static void process_incoming_migration_co(void *opaque)
         exit(EXIT_FAILURE);
     }
 
+    /*
+     * This must happen after all error conditions are dealt with and
+     * we're sure the VM is going to be running on this host.
+     */
+    qemu_announce_self();
+
     /* If global state section was not received or we are in running
        state, we need to obey autostart. Any other state is set with
        runstate_set. */
@@ -331,6 +416,50 @@ void process_incoming_migration(QEMUFile *f)
     qemu_coroutine_enter(co, f);
 }
 
+/*
+ * Send a message on the return channel back to the source
+ * of the migration.
+ */
+void migrate_send_rp_message(MigrationIncomingState *mis,
+                             enum mig_rp_message_type message_type,
+                             uint16_t len, void *data)
+{
+    trace_migrate_send_rp_message((int)message_type, len);
+    qemu_mutex_lock(&mis->rp_mutex);
+    qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
+    qemu_put_be16(mis->to_src_file, len);
+    qemu_put_buffer(mis->to_src_file, data, len);
+    qemu_fflush(mis->to_src_file);
+    qemu_mutex_unlock(&mis->rp_mutex);
+}
+
+/*
+ * Send a 'SHUT' message on the return channel with the given value
+ * to indicate that we've finished with the RP.  Non-0 value indicates
+ * error.
+ */
+void migrate_send_rp_shut(MigrationIncomingState *mis,
+                          uint32_t value)
+{
+    uint32_t buf;
+
+    buf = cpu_to_be32(value);
+    migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
+}
+
+/*
+ * Send a 'PONG' message on the return channel with the given value
+ * (normally in response to a 'PING')
+ */
+void migrate_send_rp_pong(MigrationIncomingState *mis,
+                          uint32_t value)
+{
+    uint32_t buf;
+
+    buf = cpu_to_be32(value);
+    migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
+}
+
 /* amount of nanoseconds we are willing to wait for migration to be down.
  * the choice of nanoseconds is because it is the maximum resolution that
  * get_clock() can achieve. It is an internal measure. All user-visible
@@ -378,10 +507,32 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
             s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS];
     params->decompress_threads =
             s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS];
+    params->x_cpu_throttle_initial =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL];
+    params->x_cpu_throttle_increment =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT];
 
     return params;
 }
 
+/*
+ * Return true if we're already in the middle of a migration
+ * (i.e. any of the active or setup states)
+ */
+static bool migration_is_setup_or_active(int state)
+{
+    switch (state) {
+    case MIGRATION_STATUS_ACTIVE:
+    case MIGRATION_STATUS_POSTCOPY_ACTIVE:
+    case MIGRATION_STATUS_SETUP:
+        return true;
+
+    default:
+        return false;
+
+    }
+}
+
 static void get_xbzrle_cache_stats(MigrationInfo *info)
 {
     if (migrate_use_xbzrle()) {
@@ -441,6 +592,44 @@ MigrationInfo *qmp_query_migrate(Error **errp)
             info->disk->total = blk_mig_bytes_total();
         }
 
+        if (cpu_throttle_active()) {
+            info->has_x_cpu_throttle_percentage = true;
+            info->x_cpu_throttle_percentage = cpu_throttle_get_percentage();
+        }
+
+        get_xbzrle_cache_stats(info);
+        break;
+    case MIGRATION_STATUS_POSTCOPY_ACTIVE:
+        /* Mostly the same as active; TODO add some postcopy stats */
+        info->has_status = true;
+        info->has_total_time = true;
+        info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
+            - s->total_time;
+        info->has_expected_downtime = true;
+        info->expected_downtime = s->expected_downtime;
+        info->has_setup_time = true;
+        info->setup_time = s->setup_time;
+
+        info->has_ram = true;
+        info->ram = g_malloc0(sizeof(*info->ram));
+        info->ram->transferred = ram_bytes_transferred();
+        info->ram->remaining = ram_bytes_remaining();
+        info->ram->total = ram_bytes_total();
+        info->ram->duplicate = dup_mig_pages_transferred();
+        info->ram->skipped = skipped_mig_pages_transferred();
+        info->ram->normal = norm_mig_pages_transferred();
+        info->ram->normal_bytes = norm_mig_bytes_transferred();
+        info->ram->dirty_pages_rate = s->dirty_pages_rate;
+        info->ram->mbps = s->mbps;
+
+        if (blk_mig_active()) {
+            info->has_disk = true;
+            info->disk = g_malloc0(sizeof(*info->disk));
+            info->disk->transferred = blk_mig_bytes_transferred();
+            info->disk->remaining = blk_mig_bytes_remaining();
+            info->disk->total = blk_mig_bytes_total();
+        }
+
         get_xbzrle_cache_stats(info);
         break;
     case MIGRATION_STATUS_COMPLETED:
@@ -484,8 +673,7 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
     MigrationState *s = migrate_get_current();
     MigrationCapabilityStatusList *cap;
 
-    if (s->state == MIGRATION_STATUS_ACTIVE ||
-        s->state == MIGRATION_STATUS_SETUP) {
+    if (migration_is_setup_or_active(s->state)) {
         error_setg(errp, QERR_MIGRATION_ACTIVE);
         return;
     }
@@ -493,6 +681,20 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
     for (cap = params; cap; cap = cap->next) {
         s->enabled_capabilities[cap->value->capability] = cap->value->state;
     }
+
+    if (migrate_postcopy_ram()) {
+        if (migrate_use_compression()) {
+            /* The decompression threads asynchronously write into RAM
+             * rather than use the atomic copies needed to avoid
+             * userfaulting.  It should be possible to fix the decompression
+             * threads for compatibility in future.
+             */
+            error_report("Postcopy is not currently compatible with "
+                         "compression");
+            s->enabled_capabilities[MIGRATION_CAPABILITY_X_POSTCOPY_RAM] =
+                false;
+        }
+    }
 }
 
 void qmp_migrate_set_parameters(bool has_compress_level,
@@ -500,7 +702,11 @@ void qmp_migrate_set_parameters(bool has_compress_level,
                                 bool has_compress_threads,
                                 int64_t compress_threads,
                                 bool has_decompress_threads,
-                                int64_t decompress_threads, Error **errp)
+                                int64_t decompress_threads,
+                                bool has_x_cpu_throttle_initial,
+                                int64_t x_cpu_throttle_initial,
+                                bool has_x_cpu_throttle_increment,
+                                int64_t x_cpu_throttle_increment, Error **errp)
 {
     MigrationState *s = migrate_get_current();
 
@@ -523,6 +729,18 @@ void qmp_migrate_set_parameters(bool has_compress_level,
                    "is invalid, it should be in the range of 1 to 255");
         return;
     }
+    if (has_x_cpu_throttle_initial &&
+            (x_cpu_throttle_initial < 1 || x_cpu_throttle_initial > 99)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "x_cpu_throttle_initial",
+                   "an integer in the range of 1 to 99");
+    }
+    if (has_x_cpu_throttle_increment &&
+            (x_cpu_throttle_increment < 1 || x_cpu_throttle_increment > 99)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "x_cpu_throttle_increment",
+                   "an integer in the range of 1 to 99");
+    }
 
     if (has_compress_level) {
         s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level;
@@ -534,6 +752,37 @@ void qmp_migrate_set_parameters(bool has_compress_level,
         s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
                                                     decompress_threads;
     }
+    if (has_x_cpu_throttle_initial) {
+        s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] =
+                                                    x_cpu_throttle_initial;
+    }
+
+    if (has_x_cpu_throttle_increment) {
+        s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] =
+                                                    x_cpu_throttle_increment;
+    }
+}
+
+void qmp_migrate_start_postcopy(Error **errp)
+{
+    MigrationState *s = migrate_get_current();
+
+    if (!migrate_postcopy_ram()) {
+        error_setg(errp, "Enable postcopy with migrate_set_capability before"
+                         " the start of migration");
+        return;
+    }
+
+    if (s->state == MIGRATION_STATUS_NONE) {
+        error_setg(errp, "Postcopy must be started after migration has been"
+                         " started");
+        return;
+    }
+    /*
+     * we don't error if migration has finished since that would be racy
+     * with issuing this command.
+     */
+    atomic_set(&s->start_postcopy, true);
 }
 
 /* shared migration helpers */
@@ -553,10 +802,15 @@ static void migrate_fd_cleanup(void *opaque)
     qemu_bh_delete(s->cleanup_bh);
     s->cleanup_bh = NULL;
 
+    flush_page_queue(s);
+
     if (s->file) {
         trace_migrate_fd_cleanup();
         qemu_mutex_unlock_iothread();
-        qemu_thread_join(&s->thread);
+        if (s->migration_thread_running) {
+            qemu_thread_join(&s->thread);
+            s->migration_thread_running = false;
+        }
         qemu_mutex_lock_iothread();
 
         migrate_compress_threads_join();
@@ -564,14 +818,12 @@ static void migrate_fd_cleanup(void *opaque)
         s->file = NULL;
     }
 
-    assert(s->state != MIGRATION_STATUS_ACTIVE);
+    assert((s->state != MIGRATION_STATUS_ACTIVE) &&
+           (s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE));
 
-    if (s->state != MIGRATION_STATUS_COMPLETED) {
-        qemu_savevm_state_cancel();
-        if (s->state == MIGRATION_STATUS_CANCELLING) {
-            migrate_set_state(s, MIGRATION_STATUS_CANCELLING,
-                              MIGRATION_STATUS_CANCELLED);
-        }
+    if (s->state == MIGRATION_STATUS_CANCELLING) {
+        migrate_set_state(s, MIGRATION_STATUS_CANCELLING,
+                          MIGRATION_STATUS_CANCELLED);
     }
 
     notifier_list_notify(&migration_state_notifiers, s);
@@ -591,10 +843,14 @@ static void migrate_fd_cancel(MigrationState *s)
     QEMUFile *f = migrate_get_current()->file;
     trace_migrate_fd_cancel();
 
+    if (s->rp_state.from_dst_file) {
+        /* shutdown the rp socket, so causing the rp thread to shutdown */
+        qemu_file_shutdown(s->rp_state.from_dst_file);
+    }
+
     do {
         old_state = s->state;
-        if (old_state != MIGRATION_STATUS_SETUP &&
-            old_state != MIGRATION_STATUS_ACTIVE) {
+        if (!migration_is_setup_or_active(old_state)) {
             break;
         }
         migrate_set_state(s, old_state, MIGRATION_STATUS_CANCELLING);
@@ -638,35 +894,43 @@ bool migration_has_failed(MigrationState *s)
             s->state == MIGRATION_STATUS_FAILED);
 }
 
-static MigrationState *migrate_init(const MigrationParams *params)
+bool migration_in_postcopy(MigrationState *s)
 {
-    MigrationState *s = migrate_get_current();
-    int64_t bandwidth_limit = s->bandwidth_limit;
-    bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
-    int64_t xbzrle_cache_size = s->xbzrle_cache_size;
-    int compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL];
-    int compress_thread_count =
-            s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS];
-    int decompress_thread_count =
-            s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS];
+    return (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
+}
 
-    memcpy(enabled_capabilities, s->enabled_capabilities,
-           sizeof(enabled_capabilities));
+MigrationState *migrate_init(const MigrationParams *params)
+{
+    MigrationState *s = migrate_get_current();
 
-    memset(s, 0, sizeof(*s));
+    /*
+     * Reinitialise all migration state, except
+     * parameters/capabilities that the user set, and
+     * locks.
+     */
+    s->bytes_xfer = 0;
+    s->xfer_limit = 0;
+    s->cleanup_bh = 0;
+    s->file = NULL;
+    s->state = MIGRATION_STATUS_NONE;
     s->params = *params;
-    memcpy(s->enabled_capabilities, enabled_capabilities,
-           sizeof(enabled_capabilities));
-    s->xbzrle_cache_size = xbzrle_cache_size;
-
-    s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level;
-    s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] =
-               compress_thread_count;
-    s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
-               decompress_thread_count;
-    s->bandwidth_limit = bandwidth_limit;
+    s->rp_state.from_dst_file = NULL;
+    s->rp_state.error = false;
+    s->mbps = 0.0;
+    s->downtime = 0;
+    s->expected_downtime = 0;
+    s->dirty_pages_rate = 0;
+    s->dirty_bytes_rate = 0;
+    s->setup_time = 0;
+    s->dirty_sync_count = 0;
+    s->start_postcopy = false;
+    s->migration_thread_running = false;
+    s->last_req_rb = NULL;
+
     migrate_set_state(s, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
 
+    QSIMPLEQ_INIT(&s->src_page_requests);
+
     s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
     return s;
 }
@@ -718,8 +982,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
     params.blk = has_blk && blk;
     params.shared = has_inc && inc;
 
-    if (s->state == MIGRATION_STATUS_ACTIVE ||
-        s->state == MIGRATION_STATUS_SETUP ||
+    if (migration_is_setup_or_active(s->state) ||
         s->state == MIGRATION_STATUS_CANCELLING) {
         error_setg(errp, QERR_MIGRATION_ACTIVE);
         return;
@@ -838,6 +1101,15 @@ void qmp_migrate_set_downtime(double value, Error **errp)
     max_downtime = (uint64_t)value;
 }
 
+bool migrate_postcopy_ram(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_X_POSTCOPY_RAM];
+}
+
 bool migrate_auto_converge(void)
 {
     MigrationState *s;
@@ -920,76 +1192,489 @@ int64_t migrate_xbzrle_cache_size(void)
 }
 
 /* migration thread support */
+/*
+ * Something bad happened to the RP stream, mark an error
+ * The caller shall print or trace something to indicate why
+ */
+static void mark_source_rp_bad(MigrationState *s)
+{
+    s->rp_state.error = true;
+}
+
+static struct rp_cmd_args {
+    ssize_t     len; /* -1 = variable */
+    const char *name;
+} rp_cmd_args[] = {
+    [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
+    [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
+    [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
+    [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
+    [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
+    [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
+};
+
+/*
+ * Process a request for pages received on the return path,
+ * We're allowed to send more than requested (e.g. to round to our page size)
+ * and we don't need to send pages that have already been sent.
+ */
+static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
+                                       ram_addr_t start, size_t len)
+{
+    long our_host_ps = getpagesize();
+
+    trace_migrate_handle_rp_req_pages(rbname, start, len);
+
+    /*
+     * Since we currently insist on matching page sizes, just sanity check
+     * we're being asked for whole host pages.
+     */
+    if (start & (our_host_ps-1) ||
+       (len & (our_host_ps-1))) {
+        error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
+                     " len: %zd", __func__, start, len);
+        mark_source_rp_bad(ms);
+        return;
+    }
 
+    if (ram_save_queue_pages(ms, rbname, start, len)) {
+        mark_source_rp_bad(ms);
+    }
+}
+
+/*
+ * Handles messages sent on the return path towards the source VM
+ *
+ */
+static void *source_return_path_thread(void *opaque)
+{
+    MigrationState *ms = opaque;
+    QEMUFile *rp = ms->rp_state.from_dst_file;
+    uint16_t header_len, header_type;
+    const int max_len = 512;
+    uint8_t buf[max_len];
+    uint32_t tmp32, sibling_error;
+    ram_addr_t start = 0; /* =0 to silence warning */
+    size_t  len = 0, expected_len;
+    int res;
+
+    trace_source_return_path_thread_entry();
+    while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
+           migration_is_setup_or_active(ms->state)) {
+        trace_source_return_path_thread_loop_top();
+        header_type = qemu_get_be16(rp);
+        header_len = qemu_get_be16(rp);
+
+        if (header_type >= MIG_RP_MSG_MAX ||
+            header_type == MIG_RP_MSG_INVALID) {
+            error_report("RP: Received invalid message 0x%04x length 0x%04x",
+                    header_type, header_len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        if ((rp_cmd_args[header_type].len != -1 &&
+            header_len != rp_cmd_args[header_type].len) ||
+            header_len > max_len) {
+            error_report("RP: Received '%s' message (0x%04x) with"
+                    "incorrect length %d expecting %zu",
+                    rp_cmd_args[header_type].name, header_type, header_len,
+                    (size_t)rp_cmd_args[header_type].len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        /* We know we've got a valid header by this point */
+        res = qemu_get_buffer(rp, buf, header_len);
+        if (res != header_len) {
+            error_report("RP: Failed reading data for message 0x%04x"
+                         " read %d expected %d",
+                         header_type, res, header_len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        /* OK, we have the message and the data */
+        switch (header_type) {
+        case MIG_RP_MSG_SHUT:
+            sibling_error = be32_to_cpup((uint32_t *)buf);
+            trace_source_return_path_thread_shut(sibling_error);
+            if (sibling_error) {
+                error_report("RP: Sibling indicated error %d", sibling_error);
+                mark_source_rp_bad(ms);
+            }
+            /*
+             * We'll let the main thread deal with closing the RP
+             * we could do a shutdown(2) on it, but we're the only user
+             * anyway, so there's nothing gained.
+             */
+            goto out;
+
+        case MIG_RP_MSG_PONG:
+            tmp32 = be32_to_cpup((uint32_t *)buf);
+            trace_source_return_path_thread_pong(tmp32);
+            break;
+
+        case MIG_RP_MSG_REQ_PAGES:
+            start = be64_to_cpup((uint64_t *)buf);
+            len = be32_to_cpup((uint32_t *)(buf + 8));
+            migrate_handle_rp_req_pages(ms, NULL, start, len);
+            break;
+
+        case MIG_RP_MSG_REQ_PAGES_ID:
+            expected_len = 12 + 1; /* header + termination */
+
+            if (header_len >= expected_len) {
+                start = be64_to_cpup((uint64_t *)buf);
+                len = be32_to_cpup((uint32_t *)(buf + 8));
+                /* Now we expect an idstr */
+                tmp32 = buf[12]; /* Length of the following idstr */
+                buf[13 + tmp32] = '\0';
+                expected_len += tmp32;
+            }
+            if (header_len != expected_len) {
+                error_report("RP: Req_Page_id with length %d expecting %zd",
+                        header_len, expected_len);
+                mark_source_rp_bad(ms);
+                goto out;
+            }
+            migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
+            break;
+
+        default:
+            break;
+        }
+    }
+    if (qemu_file_get_error(rp)) {
+        trace_source_return_path_thread_bad_end();
+        mark_source_rp_bad(ms);
+    }
+
+    trace_source_return_path_thread_end();
+out:
+    ms->rp_state.from_dst_file = NULL;
+    qemu_fclose(rp);
+    return NULL;
+}
+
+static int open_return_path_on_source(MigrationState *ms)
+{
+
+    ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->file);
+    if (!ms->rp_state.from_dst_file) {
+        return -1;
+    }
+
+    trace_open_return_path_on_source();
+    qemu_thread_create(&ms->rp_state.rp_thread, "return path",
+                       source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
+
+    trace_open_return_path_on_source_continue();
+
+    return 0;
+}
+
+/* Returns 0 if the RP was ok, otherwise there was an error on the RP */
+static int await_return_path_close_on_source(MigrationState *ms)
+{
+    /*
+     * If this is a normal exit then the destination will send a SHUT and the
+     * rp_thread will exit, however if there's an error we need to cause
+     * it to exit.
+     */
+    if (qemu_file_get_error(ms->file) && ms->rp_state.from_dst_file) {
+        /*
+         * shutdown(2), if we have it, will cause it to unblock if it's stuck
+         * waiting for the destination.
+         */
+        qemu_file_shutdown(ms->rp_state.from_dst_file);
+        mark_source_rp_bad(ms);
+    }
+    trace_await_return_path_close_on_source_joining();
+    qemu_thread_join(&ms->rp_state.rp_thread);
+    trace_await_return_path_close_on_source_close();
+    return ms->rp_state.error;
+}
+
+/*
+ * Switch from normal iteration to postcopy
+ * Returns non-0 on error
+ */
+static int postcopy_start(MigrationState *ms, bool *old_vm_running)
+{
+    int ret;
+    const QEMUSizedBuffer *qsb;
+    int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    migrate_set_state(ms, MIGRATION_STATUS_ACTIVE,
+                      MIGRATION_STATUS_POSTCOPY_ACTIVE);
+
+    trace_postcopy_start();
+    qemu_mutex_lock_iothread();
+    trace_postcopy_start_set_run();
+
+    qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+    *old_vm_running = runstate_is_running();
+    global_state_store();
+    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /*
+     * Cause any non-postcopiable, but iterative devices to
+     * send out their final data.
+     */
+    qemu_savevm_state_complete_precopy(ms->file, true);
+
+    /*
+     * in Finish migrate and with the io-lock held everything should
+     * be quiet, but we've potentially still got dirty pages and we
+     * need to tell the destination to throw any pages it's already received
+     * that are dirty
+     */
+    if (ram_postcopy_send_discard_bitmap(ms)) {
+        error_report("postcopy send discard bitmap failed");
+        goto fail;
+    }
+
+    /*
+     * send rest of state - note things that are doing postcopy
+     * will notice we're in POSTCOPY_ACTIVE and not actually
+     * wrap their state up here
+     */
+    qemu_file_set_rate_limit(ms->file, INT64_MAX);
+    /* Ping just for debugging, helps line traces up */
+    qemu_savevm_send_ping(ms->file, 2);
+
+    /*
+     * While loading the device state we may trigger page transfer
+     * requests and the fd must be free to process those, and thus
+     * the destination must read the whole device state off the fd before
+     * it starts processing it.  Unfortunately the ad-hoc migration format
+     * doesn't allow the destination to know the size to read without fully
+     * parsing it through each devices load-state code (especially the open
+     * coded devices that use get/put).
+     * So we wrap the device state up in a package with a length at the start;
+     * to do this we use a qemu_buf to hold the whole of the device state.
+     */
+    QEMUFile *fb = qemu_bufopen("w", NULL);
+    if (!fb) {
+        error_report("Failed to create buffered file");
+        goto fail;
+    }
+
+    /*
+     * Make sure the receiver can get incoming pages before we send the rest
+     * of the state
+     */
+    qemu_savevm_send_postcopy_listen(fb);
+
+    qemu_savevm_state_complete_precopy(fb, false);
+    qemu_savevm_send_ping(fb, 3);
+
+    qemu_savevm_send_postcopy_run(fb);
+
+    /* <><> end of stuff going into the package */
+    qsb = qemu_buf_get(fb);
+
+    /* Now send that blob */
+    if (qemu_savevm_send_packaged(ms->file, qsb)) {
+        goto fail_closefb;
+    }
+    qemu_fclose(fb);
+    ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
+
+    qemu_mutex_unlock_iothread();
+
+    /*
+     * Although this ping is just for debug, it could potentially be
+     * used for getting a better measurement of downtime at the source.
+     */
+    qemu_savevm_send_ping(ms->file, 4);
+
+    ret = qemu_file_get_error(ms->file);
+    if (ret) {
+        error_report("postcopy_start: Migration stream errored");
+        migrate_set_state(ms, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                              MIGRATION_STATUS_FAILED);
+    }
+
+    return ret;
+
+fail_closefb:
+    qemu_fclose(fb);
+fail:
+    migrate_set_state(ms, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                          MIGRATION_STATUS_FAILED);
+    qemu_mutex_unlock_iothread();
+    return -1;
+}
+
+/**
+ * migration_completion: Used by migration_thread when there's not much left.
+ *   The caller 'breaks' the loop when this returns.
+ *
+ * @s: Current migration state
+ * @current_active_state: The migration state we expect to be in
+ * @*old_vm_running: Pointer to old_vm_running flag
+ * @*start_time: Pointer to time to update
+ */
+static void migration_completion(MigrationState *s, int current_active_state,
+                                 bool *old_vm_running,
+                                 int64_t *start_time)
+{
+    int ret;
+
+    if (s->state == MIGRATION_STATUS_ACTIVE) {
+        qemu_mutex_lock_iothread();
+        *start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+        qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+        *old_vm_running = runstate_is_running();
+        ret = global_state_store();
+
+        if (!ret) {
+            ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+            if (ret >= 0) {
+                qemu_file_set_rate_limit(s->file, INT64_MAX);
+                qemu_savevm_state_complete_precopy(s->file, false);
+            }
+        }
+        qemu_mutex_unlock_iothread();
+
+        if (ret < 0) {
+            goto fail;
+        }
+    } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
+        trace_migration_completion_postcopy_end();
+
+        qemu_savevm_state_complete_postcopy(s->file);
+        trace_migration_completion_postcopy_end_after_complete();
+    }
+
+    /*
+     * If rp was opened we must clean up the thread before
+     * cleaning everything else up (since if there are no failures
+     * it will wait for the destination to send it's status in
+     * a SHUT command).
+     * Postcopy opens rp if enabled (even if it's not avtivated)
+     */
+    if (migrate_postcopy_ram()) {
+        int rp_error;
+        trace_migration_completion_postcopy_end_before_rp();
+        rp_error = await_return_path_close_on_source(s);
+        trace_migration_completion_postcopy_end_after_rp(rp_error);
+        if (rp_error) {
+            goto fail;
+        }
+    }
+
+    if (qemu_file_get_error(s->file)) {
+        trace_migration_completion_file_err();
+        goto fail;
+    }
+
+    migrate_set_state(s, current_active_state, MIGRATION_STATUS_COMPLETED);
+    return;
+
+fail:
+    migrate_set_state(s, current_active_state, MIGRATION_STATUS_FAILED);
+}
+
+/*
+ * Master migration thread on the source VM.
+ * It drives the migration and pumps the data down the outgoing channel.
+ */
 static void *migration_thread(void *opaque)
 {
     MigrationState *s = opaque;
+    /* Used by the bandwidth calcs, updated later */
     int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
     int64_t initial_bytes = 0;
     int64_t max_size = 0;
     int64_t start_time = initial_time;
+    int64_t end_time;
     bool old_vm_running = false;
+    bool entered_postcopy = false;
+    /* The active state we expect to be in; ACTIVE or POSTCOPY_ACTIVE */
+    enum MigrationStatus current_active_state = MIGRATION_STATUS_ACTIVE;
 
     rcu_register_thread();
 
     qemu_savevm_state_header(s->file);
+
+    if (migrate_postcopy_ram()) {
+        /* Now tell the dest that it should open its end so it can reply */
+        qemu_savevm_send_open_return_path(s->file);
+
+        /* And do a ping that will make stuff easier to debug */
+        qemu_savevm_send_ping(s->file, 1);
+
+        /*
+         * Tell the destination that we *might* want to do postcopy later;
+         * if the other end can't do postcopy it should fail now, nice and
+         * early.
+         */
+        qemu_savevm_send_postcopy_advise(s->file);
+    }
+
     qemu_savevm_state_begin(s->file, &s->params);
 
     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
+    current_active_state = MIGRATION_STATUS_ACTIVE;
     migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE);
 
-    while (s->state == MIGRATION_STATUS_ACTIVE) {
+    trace_migration_thread_setup_complete();
+
+    while (s->state == MIGRATION_STATUS_ACTIVE ||
+           s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
         int64_t current_time;
         uint64_t pending_size;
 
         if (!qemu_file_rate_limit(s->file)) {
-            pending_size = qemu_savevm_state_pending(s->file, max_size);
-            trace_migrate_pending(pending_size, max_size);
+            uint64_t pend_post, pend_nonpost;
+
+            qemu_savevm_state_pending(s->file, max_size, &pend_nonpost,
+                                      &pend_post);
+            pending_size = pend_nonpost + pend_post;
+            trace_migrate_pending(pending_size, max_size,
+                                  pend_post, pend_nonpost);
             if (pending_size && pending_size >= max_size) {
-                qemu_savevm_state_iterate(s->file);
-            } else {
-                int ret;
-
-                qemu_mutex_lock_iothread();
-                start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-                qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-                old_vm_running = runstate_is_running();
-
-                ret = global_state_store();
-                if (!ret) {
-                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
-                    if (ret >= 0) {
-                        qemu_file_set_rate_limit(s->file, INT64_MAX);
-                        qemu_savevm_state_complete(s->file);
-                    }
-                }
-                qemu_mutex_unlock_iothread();
+                /* Still a significant amount to transfer */
 
-                if (ret < 0) {
-                    migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
-                                      MIGRATION_STATUS_FAILED);
-                    break;
-                }
+                if (migrate_postcopy_ram() &&
+                    s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE &&
+                    pend_nonpost <= max_size &&
+                    atomic_read(&s->start_postcopy)) {
+
+                    if (!postcopy_start(s, &old_vm_running)) {
+                        current_active_state = MIGRATION_STATUS_POSTCOPY_ACTIVE;
+                        entered_postcopy = true;
+                    }
 
-                if (!qemu_file_get_error(s->file)) {
-                    migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
-                                      MIGRATION_STATUS_COMPLETED);
-                    break;
+                    continue;
                 }
+                /* Just another iteration step */
+                qemu_savevm_state_iterate(s->file, entered_postcopy);
+            } else {
+                trace_migration_thread_low_pending(pending_size);
+                migration_completion(s, current_active_state,
+                                     &old_vm_running, &start_time);
+                break;
             }
         }
 
         if (qemu_file_get_error(s->file)) {
-            migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
-                              MIGRATION_STATUS_FAILED);
+            migrate_set_state(s, current_active_state, MIGRATION_STATUS_FAILED);
+            trace_migration_thread_file_err();
             break;
         }
         current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
         if (current_time >= initial_time + BUFFER_DELAY) {
             uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes;
             uint64_t time_spent = current_time - initial_time;
-            double bandwidth = transferred_bytes / time_spent;
+            double bandwidth = (double)transferred_bytes / time_spent;
             max_size = bandwidth * migrate_max_downtime() / 1000000;
 
             s->mbps = time_spent ? (((double) transferred_bytes * 8.0) /
@@ -1013,19 +1698,26 @@ static void *migration_thread(void *opaque)
         }
     }
 
+    trace_migration_thread_after_loop();
+    /* If we enabled cpu throttling for auto-converge, turn it off. */
+    cpu_throttle_stop();
+    end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+
     qemu_mutex_lock_iothread();
+    qemu_savevm_state_cleanup();
     if (s->state == MIGRATION_STATUS_COMPLETED) {
-        int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
         uint64_t transferred_bytes = qemu_ftell(s->file);
         s->total_time = end_time - s->total_time;
-        s->downtime = end_time - start_time;
+        if (!entered_postcopy) {
+            s->downtime = end_time - start_time;
+        }
         if (s->total_time) {
             s->mbps = (((double) transferred_bytes * 8.0) /
                        ((double) s->total_time)) / 1000;
         }
         runstate_set(RUN_STATE_POSTMIGRATE);
     } else {
-        if (old_vm_running) {
+        if (old_vm_running && !entered_postcopy) {
             vm_start();
         }
     }
@@ -1048,7 +1740,34 @@ void migrate_fd_connect(MigrationState *s)
     /* Notify before starting migration thread */
     notifier_list_notify(&migration_state_notifiers, s);
 
+    /*
+     * Open the return path; currently for postcopy but other things might
+     * also want it.
+     */
+    if (migrate_postcopy_ram()) {
+        if (open_return_path_on_source(s)) {
+            error_report("Unable to open return-path for postcopy");
+            migrate_set_state(s, MIGRATION_STATUS_SETUP,
+                              MIGRATION_STATUS_FAILED);
+            migrate_fd_cleanup(s);
+            return;
+        }
+    }
+
     migrate_compress_threads_create();
     qemu_thread_create(&s->thread, "migration", migration_thread, s,
                        QEMU_THREAD_JOINABLE);
+    s->migration_thread_running = true;
+}
+
+PostcopyState  postcopy_state_get(void)
+{
+    return atomic_mb_read(&incoming_postcopy_state);
 }
+
+/* Set the state and return the old state */
+PostcopyState postcopy_state_set(PostcopyState new_state)
+{
+    return atomic_xchg(&incoming_postcopy_state, new_state);
+}
+
diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
new file mode 100644
index 000000000..3946aa98a
--- /dev/null
+++ b/migration/postcopy-ram.c
@@ -0,0 +1,761 @@
+/*
+ * Postcopy migration for RAM
+ *
+ * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Dave Gilbert  <dgilbert@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+/*
+ * Postcopy is a migration technique where the execution flips from the
+ * source to the destination before all the data has been copied.
+ */
+
+#include <glib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include "qemu-common.h"
+#include "migration/migration.h"
+#include "migration/postcopy-ram.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/balloon.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+
+/* Arbitrary limit on size of each discard command,
+ * keeps them around ~200 bytes
+ */
+#define MAX_DISCARDS_PER_COMMAND 12
+
+struct PostcopyDiscardState {
+    const char *ramblock_name;
+    uint64_t offset; /* Bitmap entry for the 1st bit of this RAMBlock */
+    uint16_t cur_entry;
+    /*
+     * Start and length of a discard range (bytes)
+     */
+    uint64_t start_list[MAX_DISCARDS_PER_COMMAND];
+    uint64_t length_list[MAX_DISCARDS_PER_COMMAND];
+    unsigned int nsentwords;
+    unsigned int nsentcmds;
+};
+
+/* Postcopy needs to detect accesses to pages that haven't yet been copied
+ * across, and efficiently map new pages in, the techniques for doing this
+ * are target OS specific.
+ */
+#if defined(__linux__)
+
+#include <poll.h>
+#include <sys/eventfd.h>
+#include <sys/mman.h>
+#include <sys/ioctl.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <asm/types.h> /* for __u64 */
+#endif
+
+#if defined(__linux__) && defined(__NR_userfaultfd)
+#include <linux/userfaultfd.h>
+
+static bool ufd_version_check(int ufd)
+{
+    struct uffdio_api api_struct;
+    uint64_t ioctl_mask;
+
+    api_struct.api = UFFD_API;
+    api_struct.features = 0;
+    if (ioctl(ufd, UFFDIO_API, &api_struct)) {
+        error_report("postcopy_ram_supported_by_host: UFFDIO_API failed: %s",
+                     strerror(errno));
+        return false;
+    }
+
+    ioctl_mask = (__u64)1 << _UFFDIO_REGISTER |
+                 (__u64)1 << _UFFDIO_UNREGISTER;
+    if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) {
+        error_report("Missing userfault features: %" PRIx64,
+                     (uint64_t)(~api_struct.ioctls & ioctl_mask));
+        return false;
+    }
+
+    return true;
+}
+
+/*
+ * Note: This has the side effect of munlock'ing all of RAM, that's
+ * normally fine since if the postcopy succeeds it gets turned back on at the
+ * end.
+ */
+bool postcopy_ram_supported_by_host(void)
+{
+    long pagesize = getpagesize();
+    int ufd = -1;
+    bool ret = false; /* Error unless we change it */
+    void *testarea = NULL;
+    struct uffdio_register reg_struct;
+    struct uffdio_range range_struct;
+    uint64_t feature_mask;
+
+    if ((1ul << qemu_target_page_bits()) > pagesize) {
+        error_report("Target page size bigger than host page size");
+        goto out;
+    }
+
+    ufd = syscall(__NR_userfaultfd, O_CLOEXEC);
+    if (ufd == -1) {
+        error_report("%s: userfaultfd not available: %s", __func__,
+                     strerror(errno));
+        goto out;
+    }
+
+    /* Version and features check */
+    if (!ufd_version_check(ufd)) {
+        goto out;
+    }
+
+    /*
+     * userfault and mlock don't go together; we'll put it back later if
+     * it was enabled.
+     */
+    if (munlockall()) {
+        error_report("%s: munlockall: %s", __func__,  strerror(errno));
+        return -1;
+    }
+
+    /*
+     *  We need to check that the ops we need are supported on anon memory
+     *  To do that we need to register a chunk and see the flags that
+     *  are returned.
+     */
+    testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE |
+                                    MAP_ANONYMOUS, -1, 0);
+    if (testarea == MAP_FAILED) {
+        error_report("%s: Failed to map test area: %s", __func__,
+                     strerror(errno));
+        goto out;
+    }
+    g_assert(((size_t)testarea & (pagesize-1)) == 0);
+
+    reg_struct.range.start = (uintptr_t)testarea;
+    reg_struct.range.len = pagesize;
+    reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+    if (ioctl(ufd, UFFDIO_REGISTER, &reg_struct)) {
+        error_report("%s userfault register: %s", __func__, strerror(errno));
+        goto out;
+    }
+
+    range_struct.start = (uintptr_t)testarea;
+    range_struct.len = pagesize;
+    if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) {
+        error_report("%s userfault unregister: %s", __func__, strerror(errno));
+        goto out;
+    }
+
+    feature_mask = (__u64)1 << _UFFDIO_WAKE |
+                   (__u64)1 << _UFFDIO_COPY |
+                   (__u64)1 << _UFFDIO_ZEROPAGE;
+    if ((reg_struct.ioctls & feature_mask) != feature_mask) {
+        error_report("Missing userfault map features: %" PRIx64,
+                     (uint64_t)(~reg_struct.ioctls & feature_mask));
+        goto out;
+    }
+
+    /* Success! */
+    ret = true;
+out:
+    if (testarea) {
+        munmap(testarea, pagesize);
+    }
+    if (ufd != -1) {
+        close(ufd);
+    }
+    return ret;
+}
+
+/**
+ * postcopy_ram_discard_range: Discard a range of memory.
+ * We can assume that if we've been called postcopy_ram_hosttest returned true.
+ *
+ * @mis: Current incoming migration state.
+ * @start, @length: range of memory to discard.
+ *
+ * returns: 0 on success.
+ */
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length)
+{
+    trace_postcopy_ram_discard_range(start, length);
+    if (madvise(start, length, MADV_DONTNEED)) {
+        error_report("%s MADV_DONTNEED: %s", __func__, strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Setup an area of RAM so that it *can* be used for postcopy later; this
+ * must be done right at the start prior to pre-copy.
+ * opaque should be the MIS.
+ */
+static int init_range(const char *block_name, void *host_addr,
+                      ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+
+    trace_postcopy_init_range(block_name, host_addr, offset, length);
+
+    /*
+     * We need the whole of RAM to be truly empty for postcopy, so things
+     * like ROMs and any data tables built during init must be zero'd
+     * - we're going to get the copy from the source anyway.
+     * (Precopy will just overwrite this data, so doesn't need the discard)
+     */
+    if (postcopy_ram_discard_range(mis, host_addr, length)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * At the end of migration, undo the effects of init_range
+ * opaque should be the MIS.
+ */
+static int cleanup_range(const char *block_name, void *host_addr,
+                        ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffdio_range range_struct;
+    trace_postcopy_cleanup_range(block_name, host_addr, offset, length);
+
+    /*
+     * We turned off hugepage for the precopy stage with postcopy enabled
+     * we can turn it back on now.
+     */
+    qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE);
+
+    /*
+     * We can also turn off userfault now since we should have all the
+     * pages.   It can be useful to leave it on to debug postcopy
+     * if you're not sure it's always getting every page.
+     */
+    range_struct.start = (uintptr_t)host_addr;
+    range_struct.len = length;
+
+    if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) {
+        error_report("%s: userfault unregister %s", __func__, strerror(errno));
+
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Initialise postcopy-ram, setting the RAM to a state where we can go into
+ * postcopy later; must be called prior to any precopy.
+ * called from arch_init's similarly named ram_postcopy_incoming_init
+ */
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages)
+{
+    if (qemu_ram_foreach_block(init_range, mis)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * At the end of a migration where postcopy_ram_incoming_init was called.
+ */
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
+{
+    trace_postcopy_ram_incoming_cleanup_entry();
+
+    if (mis->have_fault_thread) {
+        uint64_t tmp64;
+
+        if (qemu_ram_foreach_block(cleanup_range, mis)) {
+            return -1;
+        }
+        /*
+         * Tell the fault_thread to exit, it's an eventfd that should
+         * currently be at 0, we're going to increment it to 1
+         */
+        tmp64 = 1;
+        if (write(mis->userfault_quit_fd, &tmp64, 8) == 8) {
+            trace_postcopy_ram_incoming_cleanup_join();
+            qemu_thread_join(&mis->fault_thread);
+        } else {
+            /* Not much we can do here, but may as well report it */
+            error_report("%s: incrementing userfault_quit_fd: %s", __func__,
+                         strerror(errno));
+        }
+        trace_postcopy_ram_incoming_cleanup_closeuf();
+        close(mis->userfault_fd);
+        close(mis->userfault_quit_fd);
+        mis->have_fault_thread = false;
+    }
+
+    qemu_balloon_inhibit(false);
+
+    if (enable_mlock) {
+        if (os_mlock() < 0) {
+            error_report("mlock: %s", strerror(errno));
+            /*
+             * It doesn't feel right to fail at this point, we have a valid
+             * VM state.
+             */
+        }
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_END);
+    migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
+
+    if (mis->postcopy_tmp_page) {
+        munmap(mis->postcopy_tmp_page, getpagesize());
+        mis->postcopy_tmp_page = NULL;
+    }
+    trace_postcopy_ram_incoming_cleanup_exit();
+    return 0;
+}
+
+/*
+ * Disable huge pages on an area
+ */
+static int nhp_range(const char *block_name, void *host_addr,
+                    ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    trace_postcopy_nhp_range(block_name, host_addr, offset, length);
+
+    /*
+     * Before we do discards we need to ensure those discards really
+     * do delete areas of the page, even if THP thinks a hugepage would
+     * be a good idea, so force hugepages off.
+     */
+    qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE);
+
+    return 0;
+}
+
+/*
+ * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard
+ * however leaving it until after precopy means that most of the precopy
+ * data is still THPd
+ */
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
+{
+    if (qemu_ram_foreach_block(nhp_range, mis)) {
+        return -1;
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_DISCARD);
+
+    return 0;
+}
+
+/*
+ * Mark the given area of RAM as requiring notification to unwritten areas
+ * Used as a  callback on qemu_ram_foreach_block.
+ *   host_addr: Base of area to mark
+ *   offset: Offset in the whole ram arena
+ *   length: Length of the section
+ *   opaque: MigrationIncomingState pointer
+ * Returns 0 on success
+ */
+static int ram_block_enable_notify(const char *block_name, void *host_addr,
+                                   ram_addr_t offset, ram_addr_t length,
+                                   void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffdio_register reg_struct;
+
+    reg_struct.range.start = (uintptr_t)host_addr;
+    reg_struct.range.len = length;
+    reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+    /* Now tell our userfault_fd that it's responsible for this area */
+    if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, &reg_struct)) {
+        error_report("%s userfault register: %s", __func__, strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Handle faults detected by the USERFAULT markings
+ */
+static void *postcopy_ram_fault_thread(void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffd_msg msg;
+    int ret;
+    size_t hostpagesize = getpagesize();
+    RAMBlock *rb = NULL;
+    RAMBlock *last_rb = NULL; /* last RAMBlock we sent part of */
+
+    trace_postcopy_ram_fault_thread_entry();
+    qemu_sem_post(&mis->fault_thread_sem);
+
+    while (true) {
+        ram_addr_t rb_offset;
+        ram_addr_t in_raspace;
+        struct pollfd pfd[2];
+
+        /*
+         * We're mainly waiting for the kernel to give us a faulting HVA,
+         * however we can be told to quit via userfault_quit_fd which is
+         * an eventfd
+         */
+        pfd[0].fd = mis->userfault_fd;
+        pfd[0].events = POLLIN;
+        pfd[0].revents = 0;
+        pfd[1].fd = mis->userfault_quit_fd;
+        pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */
+        pfd[1].revents = 0;
+
+        if (poll(pfd, 2, -1 /* Wait forever */) == -1) {
+            error_report("%s: userfault poll: %s", __func__, strerror(errno));
+            break;
+        }
+
+        if (pfd[1].revents) {
+            trace_postcopy_ram_fault_thread_quit();
+            break;
+        }
+
+        ret = read(mis->userfault_fd, &msg, sizeof(msg));
+        if (ret != sizeof(msg)) {
+            if (errno == EAGAIN) {
+                /*
+                 * if a wake up happens on the other thread just after
+                 * the poll, there is nothing to read.
+                 */
+                continue;
+            }
+            if (ret < 0) {
+                error_report("%s: Failed to read full userfault message: %s",
+                             __func__, strerror(errno));
+                break;
+            } else {
+                error_report("%s: Read %d bytes from userfaultfd expected %zd",
+                             __func__, ret, sizeof(msg));
+                break; /* Lost alignment, don't know what we'd read next */
+            }
+        }
+        if (msg.event != UFFD_EVENT_PAGEFAULT) {
+            error_report("%s: Read unexpected event %ud from userfaultfd",
+                         __func__, msg.event);
+            continue; /* It's not a page fault, shouldn't happen */
+        }
+
+        rb = qemu_ram_block_from_host(
+                 (void *)(uintptr_t)msg.arg.pagefault.address,
+                 true, &in_raspace, &rb_offset);
+        if (!rb) {
+            error_report("postcopy_ram_fault_thread: Fault outside guest: %"
+                         PRIx64, (uint64_t)msg.arg.pagefault.address);
+            break;
+        }
+
+        rb_offset &= ~(hostpagesize - 1);
+        trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address,
+                                                qemu_ram_get_idstr(rb),
+                                                rb_offset);
+
+        /*
+         * Send the request to the source - we want to request one
+         * of our host page sizes (which is >= TPS)
+         */
+        if (rb != last_rb) {
+            last_rb = rb;
+            migrate_send_rp_req_pages(mis, qemu_ram_get_idstr(rb),
+                                     rb_offset, hostpagesize);
+        } else {
+            /* Save some space */
+            migrate_send_rp_req_pages(mis, NULL,
+                                     rb_offset, hostpagesize);
+        }
+    }
+    trace_postcopy_ram_fault_thread_exit();
+    return NULL;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+    /* Open the fd for the kernel to give us userfaults */
+    mis->userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
+    if (mis->userfault_fd == -1) {
+        error_report("%s: Failed to open userfault fd: %s", __func__,
+                     strerror(errno));
+        return -1;
+    }
+
+    /*
+     * Although the host check already tested the API, we need to
+     * do the check again as an ABI handshake on the new fd.
+     */
+    if (!ufd_version_check(mis->userfault_fd)) {
+        return -1;
+    }
+
+    /* Now an eventfd we use to tell the fault-thread to quit */
+    mis->userfault_quit_fd = eventfd(0, EFD_CLOEXEC);
+    if (mis->userfault_quit_fd == -1) {
+        error_report("%s: Opening userfault_quit_fd: %s", __func__,
+                     strerror(errno));
+        close(mis->userfault_fd);
+        return -1;
+    }
+
+    qemu_sem_init(&mis->fault_thread_sem, 0);
+    qemu_thread_create(&mis->fault_thread, "postcopy/fault",
+                       postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE);
+    qemu_sem_wait(&mis->fault_thread_sem);
+    qemu_sem_destroy(&mis->fault_thread_sem);
+    mis->have_fault_thread = true;
+
+    /* Mark so that we get notified of accesses to unwritten areas */
+    if (qemu_ram_foreach_block(ram_block_enable_notify, mis)) {
+        return -1;
+    }
+
+    /*
+     * Ballooning can mark pages as absent while we're postcopying
+     * that would cause false userfaults.
+     */
+    qemu_balloon_inhibit(true);
+
+    trace_postcopy_ram_enable_notify();
+
+    return 0;
+}
+
+/*
+ * Place a host page (from) at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from)
+{
+    struct uffdio_copy copy_struct;
+
+    copy_struct.dst = (uint64_t)(uintptr_t)host;
+    copy_struct.src = (uint64_t)(uintptr_t)from;
+    copy_struct.len = getpagesize();
+    copy_struct.mode = 0;
+
+    /* copy also acks to the kernel waking the stalled thread up
+     * TODO: We can inhibit that ack and only do it if it was requested
+     * which would be slightly cheaper, but we'd have to be careful
+     * of the order of updating our page state.
+     */
+    if (ioctl(mis->userfault_fd, UFFDIO_COPY, &copy_struct)) {
+        int e = errno;
+        error_report("%s: %s copy host: %p from: %p",
+                     __func__, strerror(e), host, from);
+
+        return -e;
+    }
+
+    trace_postcopy_place_page(host);
+    return 0;
+}
+
+/*
+ * Place a zero page at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host)
+{
+    struct uffdio_zeropage zero_struct;
+
+    zero_struct.range.start = (uint64_t)(uintptr_t)host;
+    zero_struct.range.len = getpagesize();
+    zero_struct.mode = 0;
+
+    if (ioctl(mis->userfault_fd, UFFDIO_ZEROPAGE, &zero_struct)) {
+        int e = errno;
+        error_report("%s: %s zero host: %p",
+                     __func__, strerror(e), host);
+
+        return -e;
+    }
+
+    trace_postcopy_place_page_zero(host);
+    return 0;
+}
+
+/*
+ * Returns a target page of memory that can be mapped at a later point in time
+ * using postcopy_place_page
+ * The same address is used repeatedly, postcopy_place_page just takes the
+ * backing page away.
+ * Returns: Pointer to allocated page
+ *
+ */
+void *postcopy_get_tmp_page(MigrationIncomingState *mis)
+{
+    if (!mis->postcopy_tmp_page) {
+        mis->postcopy_tmp_page = mmap(NULL, getpagesize(),
+                             PROT_READ | PROT_WRITE, MAP_PRIVATE |
+                             MAP_ANONYMOUS, -1, 0);
+        if (!mis->postcopy_tmp_page) {
+            error_report("%s: %s", __func__, strerror(errno));
+            return NULL;
+        }
+    }
+
+    return mis->postcopy_tmp_page;
+}
+
+#else
+/* No target OS support, stubs just fail */
+bool postcopy_ram_supported_by_host(void)
+{
+    error_report("%s: No OS support", __func__);
+    return false;
+}
+
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages)
+{
+    error_report("postcopy_ram_incoming_init: No OS support");
+    return -1;
+}
+
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host)
+{
+    assert(0);
+    return -1;
+}
+
+void *postcopy_get_tmp_page(MigrationIncomingState *mis)
+{
+    assert(0);
+    return NULL;
+}
+
+#endif
+
+/* ------------------------------------------------------------------------- */
+
+/**
+ * postcopy_discard_send_init: Called at the start of each RAMBlock before
+ *   asking to discard individual ranges.
+ *
+ * @ms: The current migration state.
+ * @offset: the bitmap offset of the named RAMBlock in the migration
+ *   bitmap.
+ * @name: RAMBlock that discards will operate on.
+ *
+ * returns: a new PDS.
+ */
+PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms,
+                                                 unsigned long offset,
+                                                 const char *name)
+{
+    PostcopyDiscardState *res = g_malloc0(sizeof(PostcopyDiscardState));
+
+    if (res) {
+        res->ramblock_name = name;
+        res->offset = offset;
+    }
+
+    return res;
+}
+
+/**
+ * postcopy_discard_send_range: Called by the bitmap code for each chunk to
+ *   discard. May send a discard message, may just leave it queued to
+ *   be sent later.
+ *
+ * @ms: Current migration state.
+ * @pds: Structure initialised by postcopy_discard_send_init().
+ * @start,@length: a range of pages in the migration bitmap in the
+ *   RAM block passed to postcopy_discard_send_init() (length=1 is one page)
+ */
+void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds,
+                                unsigned long start, unsigned long length)
+{
+    size_t tp_bits = qemu_target_page_bits();
+    /* Convert to byte offsets within the RAM block */
+    pds->start_list[pds->cur_entry] = (start - pds->offset) << tp_bits;
+    pds->length_list[pds->cur_entry] = length << tp_bits;
+    trace_postcopy_discard_send_range(pds->ramblock_name, start, length);
+    pds->cur_entry++;
+    pds->nsentwords++;
+
+    if (pds->cur_entry == MAX_DISCARDS_PER_COMMAND) {
+        /* Full set, ship it! */
+        qemu_savevm_send_postcopy_ram_discard(ms->file, pds->ramblock_name,
+                                              pds->cur_entry,
+                                              pds->start_list,
+                                              pds->length_list);
+        pds->nsentcmds++;
+        pds->cur_entry = 0;
+    }
+}
+
+/**
+ * postcopy_discard_send_finish: Called at the end of each RAMBlock by the
+ * bitmap code. Sends any outstanding discard messages, frees the PDS
+ *
+ * @ms: Current migration state.
+ * @pds: Structure initialised by postcopy_discard_send_init().
+ */
+void postcopy_discard_send_finish(MigrationState *ms, PostcopyDiscardState *pds)
+{
+    /* Anything unsent? */
+    if (pds->cur_entry) {
+        qemu_savevm_send_postcopy_ram_discard(ms->file, pds->ramblock_name,
+                                              pds->cur_entry,
+                                              pds->start_list,
+                                              pds->length_list);
+        pds->nsentcmds++;
+    }
+
+    trace_postcopy_discard_send_finish(pds->ramblock_name, pds->nsentwords,
+                                       pds->nsentcmds);
+
+    g_free(pds);
+}
diff --git a/migration/qemu-file-buf.c b/migration/qemu-file-buf.c
index 2de9330ca..49516b864 100644
--- a/migration/qemu-file-buf.c
+++ b/migration/qemu-file-buf.c
@@ -29,7 +29,7 @@
 #include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
 #include "migration/qemu-file-internal.h"
@@ -372,7 +372,8 @@ typedef struct QEMUBuffer {
     bool qsb_allocated;
 } QEMUBuffer;
 
-static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                              size_t size)
 {
     QEMUBuffer *s = opaque;
     ssize_t len = qsb_get_length(s->qsb) - pos;
@@ -387,8 +388,8 @@ static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
     return qsb_get_buffer(s->qsb, pos, len, buf);
 }
 
-static int buf_put_buffer(void *opaque, const uint8_t *buf,
-                          int64_t pos, int size)
+static ssize_t buf_put_buffer(void *opaque, const uint8_t *buf,
+                              int64_t pos, size_t size)
 {
     QEMUBuffer *s = opaque;
 
@@ -439,7 +440,7 @@ QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUBuffer));
+    s = g_new0(QEMUBuffer, 1);
     s->qsb = input;
 
     if (s->qsb == NULL) {
diff --git a/migration/qemu-file-stdio.c b/migration/qemu-file-stdio.c
index 285068b30..9bde9db56 100644
--- a/migration/qemu-file-stdio.c
+++ b/migration/qemu-file-stdio.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/qemu-file.h"
 
 typedef struct QEMUFileStdio {
@@ -37,11 +37,11 @@ static int stdio_get_fd(void *opaque)
     return fileno(s->stdio_file);
 }
 
-static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
-                            int size)
+static ssize_t stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
+                                size_t size)
 {
     QEMUFileStdio *s = opaque;
-    int res;
+    size_t res;
 
     res = fwrite(buf, 1, size, s->stdio_file);
 
@@ -51,11 +51,12 @@ static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
     return res;
 }
 
-static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                                size_t size)
 {
     QEMUFileStdio *s = opaque;
     FILE *fp = s->stdio_file;
-    int bytes;
+    ssize_t bytes;
 
     for (;;) {
         clearerr(fp);
@@ -143,7 +144,7 @@ QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileStdio));
+    s = g_new0(QEMUFileStdio, 1);
 
     s->stdio_file = stdio_file;
 
@@ -175,7 +176,7 @@ QEMUFile *qemu_fopen(const char *filename, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileStdio));
+    s = g_new0(QEMUFileStdio, 1);
 
     s->stdio_file = fopen(filename, mode);
     if (!s->stdio_file) {
diff --git a/migration/qemu-file-unix.c b/migration/qemu-file-unix.c
index bfbc0861a..6ca53e7d6 100644
--- a/migration/qemu-file-unix.c
+++ b/migration/qemu-file-unix.c
@@ -22,9 +22,10 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/qemu-file.h"
 #include "migration/qemu-file-internal.h"
 
@@ -39,12 +40,44 @@ static ssize_t socket_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
     QEMUFileSocket *s = opaque;
     ssize_t len;
     ssize_t size = iov_size(iov, iovcnt);
+    ssize_t offset = 0;
+    int     err;
 
-    len = iov_send(s->fd, iov, iovcnt, 0, size);
-    if (len < size) {
-        len = -socket_error();
-    }
-    return len;
+    while (size > 0) {
+        len = iov_send(s->fd, iov, iovcnt, offset, size);
+
+        if (len > 0) {
+            size -= len;
+            offset += len;
+        }
+
+        if (size > 0) {
+            err = socket_error();
+
+            if (err != EAGAIN && err != EWOULDBLOCK) {
+                error_report("socket_writev_buffer: Got err=%d for (%zu/%zu)",
+                             err, (size_t)size, (size_t)len);
+                /*
+                 * If I've already sent some but only just got the error, I
+                 * could return the amount validly sent so far and wait for the
+                 * next call to report the error, but I'd rather flag the error
+                 * immediately.
+                 */
+                return -err;
+            }
+
+            /* Emulate blocking */
+            GPollFD pfd;
+
+            pfd.fd = s->fd;
+            pfd.events = G_IO_OUT | G_IO_ERR;
+            pfd.revents = 0;
+            TFR(err = g_poll(&pfd, 1, -1 /* no timeout */));
+            /* Errors other than EINTR intentionally ignored */
+        }
+     }
+
+    return offset;
 }
 
 static int socket_get_fd(void *opaque)
@@ -54,7 +87,8 @@ static int socket_get_fd(void *opaque)
     return s->fd;
 }
 
-static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                                 size_t size)
 {
     QEMUFileSocket *s = opaque;
     ssize_t len;
@@ -96,6 +130,56 @@ static int socket_shutdown(void *opaque, bool rd, bool wr)
     }
 }
 
+static int socket_return_close(void *opaque)
+{
+    QEMUFileSocket *s = opaque;
+    /*
+     * Note: We don't close the socket, that should be done by the forward
+     * path.
+     */
+    g_free(s);
+    return 0;
+}
+
+static const QEMUFileOps socket_return_read_ops = {
+    .get_fd          = socket_get_fd,
+    .get_buffer      = socket_get_buffer,
+    .close           = socket_return_close,
+    .shut_down       = socket_shutdown,
+};
+
+static const QEMUFileOps socket_return_write_ops = {
+    .get_fd          = socket_get_fd,
+    .writev_buffer   = socket_writev_buffer,
+    .close           = socket_return_close,
+    .shut_down       = socket_shutdown,
+};
+
+/*
+ * Give a QEMUFile* off the same socket but data in the opposite
+ * direction.
+ */
+static QEMUFile *socket_get_return_path(void *opaque)
+{
+    QEMUFileSocket *forward = opaque;
+    QEMUFileSocket *reverse;
+
+    if (qemu_file_get_error(forward->file)) {
+        /* If the forward file is in error, don't try and open a return */
+        return NULL;
+    }
+
+    reverse = g_malloc0(sizeof(QEMUFileSocket));
+    reverse->fd = forward->fd;
+    /* I don't think there's a better way to tell which direction 'this' is */
+    if (forward->file->ops->get_buffer != NULL) {
+        /* being called from the read side, so we need to be able to write */
+        return qemu_fopen_ops(reverse, &socket_return_write_ops);
+    } else {
+        return qemu_fopen_ops(reverse, &socket_return_read_ops);
+    }
+}
+
 static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
                                   int64_t pos)
 {
@@ -138,7 +222,8 @@ static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
     return total;
 }
 
-static int unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                              size_t size)
 {
     QEMUFileSocket *s = opaque;
     ssize_t len;
@@ -192,7 +277,7 @@ QEMUFile *qemu_fdopen(int fd, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileSocket));
+    s = g_new0(QEMUFileSocket, 1);
     s->fd = fd;
 
     if (mode[0] == 'r') {
@@ -204,18 +289,19 @@ QEMUFile *qemu_fdopen(int fd, const char *mode)
 }
 
 static const QEMUFileOps socket_read_ops = {
-    .get_fd     = socket_get_fd,
-    .get_buffer = socket_get_buffer,
-    .close      = socket_close,
-    .shut_down  = socket_shutdown
-
+    .get_fd          = socket_get_fd,
+    .get_buffer      = socket_get_buffer,
+    .close           = socket_close,
+    .shut_down       = socket_shutdown,
+    .get_return_path = socket_get_return_path
 };
 
 static const QEMUFileOps socket_write_ops = {
-    .get_fd        = socket_get_fd,
-    .writev_buffer = socket_writev_buffer,
-    .close         = socket_close,
-    .shut_down     = socket_shutdown
+    .get_fd          = socket_get_fd,
+    .writev_buffer   = socket_writev_buffer,
+    .close           = socket_close,
+    .shut_down       = socket_shutdown,
+    .get_return_path = socket_get_return_path
 };
 
 QEMUFile *qemu_fopen_socket(int fd, const char *mode)
@@ -226,7 +312,7 @@ QEMUFile *qemu_fopen_socket(int fd, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileSocket));
+    s = g_new0(QEMUFileSocket, 1);
     s->fd = fd;
     if (mode[0] == 'w') {
         qemu_set_block(s->fd);
diff --git a/migration/qemu-file.c b/migration/qemu-file.c
index 6bb3dc15c..0bbd2574a 100644
--- a/migration/qemu-file.c
+++ b/migration/qemu-file.c
@@ -26,7 +26,7 @@
 #include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
 #include "migration/qemu-file-internal.h"
@@ -44,6 +44,18 @@ int qemu_file_shutdown(QEMUFile *f)
     return f->ops->shut_down(f->opaque, true, true);
 }
 
+/*
+ * Result: QEMUFile* for a 'return path' for comms in the opposite direction
+ *         NULL if not available
+ */
+QEMUFile *qemu_file_get_return_path(QEMUFile *f)
+{
+    if (!f->ops->get_return_path) {
+        return NULL;
+    }
+    return f->ops->get_return_path(f->opaque);
+}
+
 bool qemu_file_mode_is_not_valid(const char *mode)
 {
     if (mode == NULL ||
@@ -60,7 +72,7 @@ QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
 {
     QEMUFile *f;
 
-    f = g_malloc0(sizeof(QEMUFile));
+    f = g_new0(QEMUFile, 1);
 
     f->opaque = opaque;
     f->ops = ops;
@@ -270,7 +282,7 @@ int qemu_fclose(QEMUFile *f)
     return ret;
 }
 
-static void add_to_iovec(QEMUFile *f, const uint8_t *buf, int size)
+static void add_to_iovec(QEMUFile *f, const uint8_t *buf, size_t size)
 {
     /* check for adjacent buffer and coalesce them */
     if (f->iovcnt > 0 && buf == f->iov[f->iovcnt - 1].iov_base +
@@ -286,7 +298,7 @@ static void add_to_iovec(QEMUFile *f, const uint8_t *buf, int size)
     }
 }
 
-void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size)
+void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size)
 {
     if (!f->ops->writev_buffer) {
         qemu_put_buffer(f, buf, size);
@@ -301,9 +313,9 @@ void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size)
     add_to_iovec(f, buf, size);
 }
 
-void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
+void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size)
 {
-    int l;
+    size_t l;
 
     if (f->last_error) {
         return;
@@ -363,10 +375,10 @@ void qemu_file_skip(QEMUFile *f, int size)
  * return as many as it managed to read (assuming blocking fd's which
  * all current QEMUFile are)
  */
-int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset)
+size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset)
 {
-    int pending;
-    int index;
+    ssize_t pending;
+    size_t index;
 
     assert(!qemu_file_is_writable(f));
     assert(offset < IO_BUF_SIZE);
@@ -411,13 +423,13 @@ int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset)
  * return as many as it managed to read (assuming blocking fd's which
  * all current QEMUFile are)
  */
-int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size)
+size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size)
 {
-    int pending = size;
-    int done = 0;
+    size_t pending = size;
+    size_t done = 0;
 
     while (pending > 0) {
-        int res;
+        size_t res;
         uint8_t *src;
 
         res = qemu_peek_buffer(f, &src, MIN(pending, IO_BUF_SIZE), 0);
@@ -434,6 +446,43 @@ int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size)
 }
 
 /*
+ * Read 'size' bytes of data from the file.
+ * 'size' can be larger than the internal buffer.
+ *
+ * The data:
+ *   may be held on an internal buffer (in which case *buf is updated
+ *     to point to it) that is valid until the next qemu_file operation.
+ * OR
+ *   will be copied to the *buf that was passed in.
+ *
+ * The code tries to avoid the copy if possible.
+ *
+ * It will return size bytes unless there was an error, in which case it will
+ * return as many as it managed to read (assuming blocking fd's which
+ * all current QEMUFile are)
+ *
+ * Note: Since **buf may get changed, the caller should take care to
+ *       keep a pointer to the original buffer if it needs to deallocate it.
+ */
+size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size)
+{
+    if (size < IO_BUF_SIZE) {
+        size_t res;
+        uint8_t *src;
+
+        res = qemu_peek_buffer(f, &src, size, 0);
+
+        if (res == size) {
+            qemu_file_skip(f, res);
+            *buf = src;
+            return res;
+        }
+    }
+
+    return qemu_get_buffer(f, *buf, size);
+}
+
+/*
  * Peeks a single byte from the buffer; this isn't guaranteed to work if
  * offset leaves a gap after the previous read/peeked data.
  */
@@ -611,3 +660,18 @@ size_t qemu_get_counted_string(QEMUFile *f, char buf[256])
 
     return res == len ? res : 0;
 }
+
+/*
+ * Set the blocking state of the QEMUFile.
+ * Note: On some transports the OS only keeps a single blocking state for
+ *       both directions, and thus changing the blocking on the main
+ *       QEMUFile can also affect the return path.
+ */
+void qemu_file_set_blocking(QEMUFile *f, bool block)
+{
+    if (block) {
+        qemu_set_block(qemu_get_fd(f));
+    } else {
+        qemu_set_nonblock(qemu_get_fd(f));
+    }
+}
diff --git a/migration/ram.c b/migration/ram.c
index 7f007e643..0490f005d 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -32,6 +32,7 @@
 #include "qemu/timer.h"
 #include "qemu/main-loop.h"
 #include "migration/migration.h"
+#include "migration/postcopy-ram.h"
 #include "exec/address-spaces.h"
 #include "migration/page_cache.h"
 #include "qemu/error-report.h"
@@ -47,9 +48,7 @@
     do { } while (0)
 #endif
 
-static bool mig_throttle_on;
 static int dirty_rate_high_cnt;
-static void check_guest_throttling(void);
 
 static uint64_t bitmap_sync_count;
 
@@ -221,12 +220,34 @@ static RAMBlock *last_seen_block;
 /* This is the last block from where we have sent data */
 static RAMBlock *last_sent_block;
 static ram_addr_t last_offset;
-static unsigned long *migration_bitmap;
 static QemuMutex migration_bitmap_mutex;
 static uint64_t migration_dirty_pages;
 static uint32_t last_version;
 static bool ram_bulk_stage;
 
+/* used by the search for pages to send */
+struct PageSearchStatus {
+    /* Current block being searched */
+    RAMBlock    *block;
+    /* Current offset to search from */
+    ram_addr_t   offset;
+    /* Set once we wrap around */
+    bool         complete_round;
+};
+typedef struct PageSearchStatus PageSearchStatus;
+
+static struct BitmapRcu {
+    struct rcu_head rcu;
+    /* Main migration bitmap */
+    unsigned long *bmap;
+    /* bitmap of pages that haven't been sent even once
+     * only maintained and used in postcopy at the moment
+     * where it's used to send the dirtymap at the start
+     * of the postcopy phase
+     */
+    unsigned long *unsentmap;
+} *migration_bitmap_rcu;
+
 struct CompressParam {
     bool start;
     bool done;
@@ -396,6 +417,29 @@ static size_t save_page_header(QEMUFile *f, RAMBlock *block, ram_addr_t offset)
     return size;
 }
 
+/* Reduce amount of guest cpu execution to hopefully slow down memory writes.
+ * If guest dirty memory rate is reduced below the rate at which we can
+ * transfer pages to the destination then we should be able to complete
+ * migration. Some workloads dirty memory way too fast and will not effectively
+ * converge, even with auto-converge.
+ */
+static void mig_throttle_guest_down(void)
+{
+    MigrationState *s = migrate_get_current();
+    uint64_t pct_initial =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL];
+    uint64_t pct_icrement =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT];
+
+    /* We have not started throttling yet. Let's start it. */
+    if (!cpu_throttle_active()) {
+        cpu_throttle_set(pct_initial);
+    } else {
+        /* Throttling already on, just increase the rate */
+        cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement);
+    }
+}
+
 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
  * The important thing is that a stale (not-yet-0'd) page be replaced
  * by the new data.
@@ -495,43 +539,60 @@ static int save_xbzrle_page(QEMUFile *f, uint8_t **current_data,
     return 1;
 }
 
-/* Called with rcu_read_lock() to protect migration_bitmap */
+/* Called with rcu_read_lock() to protect migration_bitmap
+ * rb: The RAMBlock  to search for dirty pages in
+ * start: Start address (typically so we can continue from previous page)
+ * ram_addr_abs: Pointer into which to store the address of the dirty page
+ *               within the global ram_addr space
+ *
+ * Returns: byte offset within memory region of the start of a dirty page
+ */
 static inline
-ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,
-                                                 ram_addr_t start)
+ram_addr_t migration_bitmap_find_dirty(RAMBlock *rb,
+                                       ram_addr_t start,
+                                       ram_addr_t *ram_addr_abs)
 {
-    unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS;
+    unsigned long base = rb->offset >> TARGET_PAGE_BITS;
     unsigned long nr = base + (start >> TARGET_PAGE_BITS);
-    uint64_t mr_size = TARGET_PAGE_ALIGN(memory_region_size(mr));
-    unsigned long size = base + (mr_size >> TARGET_PAGE_BITS);
+    uint64_t rb_size = rb->used_length;
+    unsigned long size = base + (rb_size >> TARGET_PAGE_BITS);
     unsigned long *bitmap;
 
     unsigned long next;
 
-    bitmap = atomic_rcu_read(&migration_bitmap);
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
     if (ram_bulk_stage && nr > base) {
         next = nr + 1;
     } else {
         next = find_next_bit(bitmap, size, nr);
     }
 
-    if (next < size) {
-        clear_bit(next, bitmap);
+    *ram_addr_abs = next << TARGET_PAGE_BITS;
+    return (next - base) << TARGET_PAGE_BITS;
+}
+
+static inline bool migration_bitmap_clear_dirty(ram_addr_t addr)
+{
+    bool ret;
+    int nr = addr >> TARGET_PAGE_BITS;
+    unsigned long *bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+
+    ret = test_and_clear_bit(nr, bitmap);
+
+    if (ret) {
         migration_dirty_pages--;
     }
-    return (next - base) << TARGET_PAGE_BITS;
+    return ret;
 }
 
-/* Called with rcu_read_lock() to protect migration_bitmap */
 static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
 {
     unsigned long *bitmap;
-    bitmap = atomic_rcu_read(&migration_bitmap);
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
     migration_dirty_pages +=
         cpu_physical_memory_sync_dirty_bitmap(bitmap, start, length);
 }
 
-
 /* Fix me: there are too many global variables used in migration process. */
 static int64_t start_time;
 static int64_t bytes_xfer_prev;
@@ -573,7 +634,7 @@ static void migration_bitmap_sync(void)
     qemu_mutex_lock(&migration_bitmap_mutex);
     rcu_read_lock();
     QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-        migration_bitmap_sync_range(block->mr->ram_addr, block->used_length);
+        migration_bitmap_sync_range(block->offset, block->used_length);
     }
     rcu_read_unlock();
     qemu_mutex_unlock(&migration_bitmap_mutex);
@@ -589,21 +650,21 @@ static void migration_bitmap_sync(void)
             /* The following detection logic can be refined later. For now:
                Check to see if the dirtied bytes is 50% more than the approx.
                amount of bytes that just got transferred since the last time we
-               were in this routine. If that happens >N times (for now N==4)
-               we turn on the throttle down logic */
+               were in this routine. If that happens twice, start or increase
+               throttling */
             bytes_xfer_now = ram_bytes_transferred();
+
             if (s->dirty_pages_rate &&
                (num_dirty_pages_period * TARGET_PAGE_SIZE >
                    (bytes_xfer_now - bytes_xfer_prev)/2) &&
-               (dirty_rate_high_cnt++ > 4)) {
+               (dirty_rate_high_cnt++ >= 2)) {
                     trace_migration_throttle();
-                    mig_throttle_on = true;
                     dirty_rate_high_cnt = 0;
+                    mig_throttle_guest_down();
              }
              bytes_xfer_prev = bytes_xfer_now;
-        } else {
-             mig_throttle_on = false;
         }
+
         if (migrate_use_xbzrle()) {
             if (iterations_prev != acct_info.iterations) {
                 acct_info.xbzrle_cache_miss_rate =
@@ -655,6 +716,9 @@ static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
  * ram_save_page: Send the given page to the stream
  *
  * Returns: Number of pages written.
+ *          < 0 - error
+ *          >=0 - Number of pages written - this might legally be 0
+ *                if xbzrle noticed the page was the same.
  *
  * @f: QEMUFile where to send the data
  * @block: block that contains the page we want to send
@@ -668,12 +732,11 @@ static int ram_save_page(QEMUFile *f, RAMBlock* block, ram_addr_t offset,
     int pages = -1;
     uint64_t bytes_xmit;
     ram_addr_t current_addr;
-    MemoryRegion *mr = block->mr;
     uint8_t *p;
     int ret;
     bool send_async = true;
 
-    p = memory_region_get_ram_ptr(mr) + offset;
+    p = block->host + offset;
 
     /* In doubt sent page as normal */
     bytes_xmit = 0;
@@ -744,7 +807,7 @@ static int do_compress_ram_page(CompressParam *param)
     RAMBlock *block = param->block;
     ram_addr_t offset = param->offset;
 
-    p = memory_region_get_ram_ptr(block->mr) + (offset & TARGET_PAGE_MASK);
+    p = block->host + (offset & TARGET_PAGE_MASK);
 
     bytes_sent = save_page_header(param->file, block, offset |
                                   RAM_SAVE_FLAG_COMPRESS_PAGE);
@@ -852,11 +915,10 @@ static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
 {
     int pages = -1;
     uint64_t bytes_xmit;
-    MemoryRegion *mr = block->mr;
     uint8_t *p;
     int ret;
 
-    p = memory_region_get_ram_ptr(mr) + offset;
+    p = block->host + offset;
 
     bytes_xmit = 0;
     ret = ram_control_save_page(f, block->offset,
@@ -909,6 +971,339 @@ static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
     return pages;
 }
 
+/*
+ * Find the next dirty page and update any state associated with
+ * the search process.
+ *
+ * Returns: True if a page is found
+ *
+ * @f: Current migration stream.
+ * @pss: Data about the state of the current dirty page scan.
+ * @*again: Set to false if the search has scanned the whole of RAM
+ * *ram_addr_abs: Pointer into which to store the address of the dirty page
+ *               within the global ram_addr space
+ */
+static bool find_dirty_block(QEMUFile *f, PageSearchStatus *pss,
+                             bool *again, ram_addr_t *ram_addr_abs)
+{
+    pss->offset = migration_bitmap_find_dirty(pss->block, pss->offset,
+                                              ram_addr_abs);
+    if (pss->complete_round && pss->block == last_seen_block &&
+        pss->offset >= last_offset) {
+        /*
+         * We've been once around the RAM and haven't found anything.
+         * Give up.
+         */
+        *again = false;
+        return false;
+    }
+    if (pss->offset >= pss->block->used_length) {
+        /* Didn't find anything in this RAM Block */
+        pss->offset = 0;
+        pss->block = QLIST_NEXT_RCU(pss->block, next);
+        if (!pss->block) {
+            /* Hit the end of the list */
+            pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
+            /* Flag that we've looped */
+            pss->complete_round = true;
+            ram_bulk_stage = false;
+            if (migrate_use_xbzrle()) {
+                /* If xbzrle is on, stop using the data compression at this
+                 * point. In theory, xbzrle can do better than compression.
+                 */
+                flush_compressed_data(f);
+                compression_switch = false;
+            }
+        }
+        /* Didn't find anything this time, but try again on the new block */
+        *again = true;
+        return false;
+    } else {
+        /* Can go around again, but... */
+        *again = true;
+        /* We've found something so probably don't need to */
+        return true;
+    }
+}
+
+/*
+ * Helper for 'get_queued_page' - gets a page off the queue
+ *      ms:      MigrationState in
+ * *offset:      Used to return the offset within the RAMBlock
+ * ram_addr_abs: global offset in the dirty/sent bitmaps
+ *
+ * Returns:      block (or NULL if none available)
+ */
+static RAMBlock *unqueue_page(MigrationState *ms, ram_addr_t *offset,
+                              ram_addr_t *ram_addr_abs)
+{
+    RAMBlock *block = NULL;
+
+    qemu_mutex_lock(&ms->src_page_req_mutex);
+    if (!QSIMPLEQ_EMPTY(&ms->src_page_requests)) {
+        struct MigrationSrcPageRequest *entry =
+                                QSIMPLEQ_FIRST(&ms->src_page_requests);
+        block = entry->rb;
+        *offset = entry->offset;
+        *ram_addr_abs = (entry->offset + entry->rb->offset) &
+                        TARGET_PAGE_MASK;
+
+        if (entry->len > TARGET_PAGE_SIZE) {
+            entry->len -= TARGET_PAGE_SIZE;
+            entry->offset += TARGET_PAGE_SIZE;
+        } else {
+            memory_region_unref(block->mr);
+            QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
+            g_free(entry);
+        }
+    }
+    qemu_mutex_unlock(&ms->src_page_req_mutex);
+
+    return block;
+}
+
+/*
+ * Unqueue a page from the queue fed by postcopy page requests; skips pages
+ * that are already sent (!dirty)
+ *
+ *      ms:      MigrationState in
+ *     pss:      PageSearchStatus structure updated with found block/offset
+ * ram_addr_abs: global offset in the dirty/sent bitmaps
+ *
+ * Returns:      true if a queued page is found
+ */
+static bool get_queued_page(MigrationState *ms, PageSearchStatus *pss,
+                            ram_addr_t *ram_addr_abs)
+{
+    RAMBlock  *block;
+    ram_addr_t offset;
+    bool dirty;
+
+    do {
+        block = unqueue_page(ms, &offset, ram_addr_abs);
+        /*
+         * We're sending this page, and since it's postcopy nothing else
+         * will dirty it, and we must make sure it doesn't get sent again
+         * even if this queue request was received after the background
+         * search already sent it.
+         */
+        if (block) {
+            unsigned long *bitmap;
+            bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+            dirty = test_bit(*ram_addr_abs >> TARGET_PAGE_BITS, bitmap);
+            if (!dirty) {
+                trace_get_queued_page_not_dirty(
+                    block->idstr, (uint64_t)offset,
+                    (uint64_t)*ram_addr_abs,
+                    test_bit(*ram_addr_abs >> TARGET_PAGE_BITS,
+                         atomic_rcu_read(&migration_bitmap_rcu)->unsentmap));
+            } else {
+                trace_get_queued_page(block->idstr,
+                                      (uint64_t)offset,
+                                      (uint64_t)*ram_addr_abs);
+            }
+        }
+
+    } while (block && !dirty);
+
+    if (block) {
+        /*
+         * As soon as we start servicing pages out of order, then we have
+         * to kill the bulk stage, since the bulk stage assumes
+         * in (migration_bitmap_find_and_reset_dirty) that every page is
+         * dirty, that's no longer true.
+         */
+        ram_bulk_stage = false;
+
+        /*
+         * We want the background search to continue from the queued page
+         * since the guest is likely to want other pages near to the page
+         * it just requested.
+         */
+        pss->block = block;
+        pss->offset = offset;
+    }
+
+    return !!block;
+}
+
+/**
+ * flush_page_queue: Flush any remaining pages in the ram request queue
+ *    it should be empty at the end anyway, but in error cases there may be
+ *    some left.
+ *
+ * ms: MigrationState
+ */
+void flush_page_queue(MigrationState *ms)
+{
+    struct MigrationSrcPageRequest *mspr, *next_mspr;
+    /* This queue generally should be empty - but in the case of a failed
+     * migration might have some droppings in.
+     */
+    rcu_read_lock();
+    QSIMPLEQ_FOREACH_SAFE(mspr, &ms->src_page_requests, next_req, next_mspr) {
+        memory_region_unref(mspr->rb->mr);
+        QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
+        g_free(mspr);
+    }
+    rcu_read_unlock();
+}
+
+/**
+ * Queue the pages for transmission, e.g. a request from postcopy destination
+ *   ms: MigrationStatus in which the queue is held
+ *   rbname: The RAMBlock the request is for - may be NULL (to mean reuse last)
+ *   start: Offset from the start of the RAMBlock
+ *   len: Length (in bytes) to send
+ *   Return: 0 on success
+ */
+int ram_save_queue_pages(MigrationState *ms, const char *rbname,
+                         ram_addr_t start, ram_addr_t len)
+{
+    RAMBlock *ramblock;
+
+    rcu_read_lock();
+    if (!rbname) {
+        /* Reuse last RAMBlock */
+        ramblock = ms->last_req_rb;
+
+        if (!ramblock) {
+            /*
+             * Shouldn't happen, we can't reuse the last RAMBlock if
+             * it's the 1st request.
+             */
+            error_report("ram_save_queue_pages no previous block");
+            goto err;
+        }
+    } else {
+        ramblock = qemu_ram_block_by_name(rbname);
+
+        if (!ramblock) {
+            /* We shouldn't be asked for a non-existent RAMBlock */
+            error_report("ram_save_queue_pages no block '%s'", rbname);
+            goto err;
+        }
+        ms->last_req_rb = ramblock;
+    }
+    trace_ram_save_queue_pages(ramblock->idstr, start, len);
+    if (start+len > ramblock->used_length) {
+        error_report("%s request overrun start=" RAM_ADDR_FMT " len="
+                     RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
+                     __func__, start, len, ramblock->used_length);
+        goto err;
+    }
+
+    struct MigrationSrcPageRequest *new_entry =
+        g_malloc0(sizeof(struct MigrationSrcPageRequest));
+    new_entry->rb = ramblock;
+    new_entry->offset = start;
+    new_entry->len = len;
+
+    memory_region_ref(ramblock->mr);
+    qemu_mutex_lock(&ms->src_page_req_mutex);
+    QSIMPLEQ_INSERT_TAIL(&ms->src_page_requests, new_entry, next_req);
+    qemu_mutex_unlock(&ms->src_page_req_mutex);
+    rcu_read_unlock();
+
+    return 0;
+
+err:
+    rcu_read_unlock();
+    return -1;
+}
+
+/**
+ * ram_save_target_page: Save one target page
+ *
+ *
+ * @f: QEMUFile where to send the data
+ * @block: pointer to block that contains the page we want to send
+ * @offset: offset inside the block for the page;
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
+ *
+ * Returns: Number of pages written.
+ */
+static int ram_save_target_page(MigrationState *ms, QEMUFile *f,
+                                RAMBlock *block, ram_addr_t offset,
+                                bool last_stage,
+                                uint64_t *bytes_transferred,
+                                ram_addr_t dirty_ram_abs)
+{
+    int res = 0;
+
+    /* Check the pages is dirty and if it is send it */
+    if (migration_bitmap_clear_dirty(dirty_ram_abs)) {
+        unsigned long *unsentmap;
+        if (compression_switch && migrate_use_compression()) {
+            res = ram_save_compressed_page(f, block, offset,
+                                           last_stage,
+                                           bytes_transferred);
+        } else {
+            res = ram_save_page(f, block, offset, last_stage,
+                                bytes_transferred);
+        }
+
+        if (res < 0) {
+            return res;
+        }
+        unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+        if (unsentmap) {
+            clear_bit(dirty_ram_abs >> TARGET_PAGE_BITS, unsentmap);
+        }
+        /* Only update last_sent_block if a block was actually sent; xbzrle
+         * might have decided the page was identical so didn't bother writing
+         * to the stream.
+         */
+        if (res > 0) {
+            last_sent_block = block;
+        }
+    }
+
+    return res;
+}
+
+/**
+ * ram_save_host_page: Starting at *offset send pages upto the end
+ *                     of the current host page.  It's valid for the initial
+ *                     offset to point into the middle of a host page
+ *                     in which case the remainder of the hostpage is sent.
+ *                     Only dirty target pages are sent.
+ *
+ * Returns: Number of pages written.
+ *
+ * @f: QEMUFile where to send the data
+ * @block: pointer to block that contains the page we want to send
+ * @offset: offset inside the block for the page; updated to last target page
+ *          sent
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
+ */
+static int ram_save_host_page(MigrationState *ms, QEMUFile *f, RAMBlock *block,
+                              ram_addr_t *offset, bool last_stage,
+                              uint64_t *bytes_transferred,
+                              ram_addr_t dirty_ram_abs)
+{
+    int tmppages, pages = 0;
+    do {
+        tmppages = ram_save_target_page(ms, f, block, *offset, last_stage,
+                                        bytes_transferred, dirty_ram_abs);
+        if (tmppages < 0) {
+            return tmppages;
+        }
+
+        pages += tmppages;
+        *offset += TARGET_PAGE_SIZE;
+        dirty_ram_abs += TARGET_PAGE_SIZE;
+    } while (*offset & (qemu_host_page_size - 1));
+
+    /* The offset we leave with is the last one we looked at */
+    *offset -= TARGET_PAGE_SIZE;
+    return pages;
+}
+
 /**
  * ram_find_and_save_block: Finds a dirty page and sends it to f
  *
@@ -920,61 +1315,47 @@ static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
  * @f: QEMUFile where to send the data
  * @last_stage: if we are at the completion stage
  * @bytes_transferred: increase it with the number of transferred bytes
+ *
+ * On systems where host-page-size > target-page-size it will send all the
+ * pages in a host page that are dirty.
  */
 
 static int ram_find_and_save_block(QEMUFile *f, bool last_stage,
                                    uint64_t *bytes_transferred)
 {
-    RAMBlock *block = last_seen_block;
-    ram_addr_t offset = last_offset;
-    bool complete_round = false;
+    PageSearchStatus pss;
+    MigrationState *ms = migrate_get_current();
     int pages = 0;
-    MemoryRegion *mr;
+    bool again, found;
+    ram_addr_t dirty_ram_abs; /* Address of the start of the dirty page in
+                                 ram_addr_t space */
 
-    if (!block)
-        block = QLIST_FIRST_RCU(&ram_list.blocks);
+    pss.block = last_seen_block;
+    pss.offset = last_offset;
+    pss.complete_round = false;
 
-    while (true) {
-        mr = block->mr;
-        offset = migration_bitmap_find_and_reset_dirty(mr, offset);
-        if (complete_round && block == last_seen_block &&
-            offset >= last_offset) {
-            break;
+    if (!pss.block) {
+        pss.block = QLIST_FIRST_RCU(&ram_list.blocks);
+    }
+
+    do {
+        again = true;
+        found = get_queued_page(ms, &pss, &dirty_ram_abs);
+
+        if (!found) {
+            /* priority queue empty, so just search for something dirty */
+            found = find_dirty_block(f, &pss, &again, &dirty_ram_abs);
         }
-        if (offset >= block->used_length) {
-            offset = 0;
-            block = QLIST_NEXT_RCU(block, next);
-            if (!block) {
-                block = QLIST_FIRST_RCU(&ram_list.blocks);
-                complete_round = true;
-                ram_bulk_stage = false;
-                if (migrate_use_xbzrle()) {
-                    /* If xbzrle is on, stop using the data compression at this
-                     * point. In theory, xbzrle can do better than compression.
-                     */
-                    flush_compressed_data(f);
-                    compression_switch = false;
-                }
-            }
-        } else {
-            if (compression_switch && migrate_use_compression()) {
-                pages = ram_save_compressed_page(f, block, offset, last_stage,
-                                                 bytes_transferred);
-            } else {
-                pages = ram_save_page(f, block, offset, last_stage,
-                                      bytes_transferred);
-            }
 
-            /* if page is unmodified, continue to the next */
-            if (pages > 0) {
-                last_sent_block = block;
-                break;
-            }
+        if (found) {
+            pages = ram_save_host_page(ms, f, pss.block, &pss.offset,
+                                       last_stage, bytes_transferred,
+                                       dirty_ram_abs);
         }
-    }
+    } while (!pages && again);
 
-    last_seen_block = block;
-    last_offset = offset;
+    last_seen_block = pss.block;
+    last_offset = pss.offset;
 
     return pages;
 }
@@ -1024,17 +1405,23 @@ void free_xbzrle_decoded_buf(void)
     xbzrle_decoded_buf = NULL;
 }
 
-static void migration_end(void)
+static void migration_bitmap_free(struct BitmapRcu *bmap)
+{
+    g_free(bmap->bmap);
+    g_free(bmap->unsentmap);
+    g_free(bmap);
+}
+
+static void ram_migration_cleanup(void *opaque)
 {
     /* caller have hold iothread lock or is in a bh, so there is
      * no writing race against this migration_bitmap
      */
-    unsigned long *bitmap = migration_bitmap;
-    atomic_rcu_set(&migration_bitmap, NULL);
+    struct BitmapRcu *bitmap = migration_bitmap_rcu;
+    atomic_rcu_set(&migration_bitmap_rcu, NULL);
     if (bitmap) {
         memory_global_dirty_log_stop();
-        synchronize_rcu();
-        g_free(bitmap);
+        call_rcu(bitmap, migration_bitmap_free, rcu);
     }
 
     XBZRLE_cache_lock();
@@ -1049,11 +1436,6 @@ static void migration_end(void)
     XBZRLE_cache_unlock();
 }
 
-static void ram_migration_cancel(void *opaque)
-{
-    migration_end();
-}
-
 static void reset_ram_globals(void)
 {
     last_seen_block = NULL;
@@ -1070,9 +1452,10 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t new)
     /* called in qemu main thread, so there is
      * no writing race against this migration_bitmap
      */
-    if (migration_bitmap) {
-        unsigned long *old_bitmap = migration_bitmap, *bitmap;
-        bitmap = bitmap_new(new);
+    if (migration_bitmap_rcu) {
+        struct BitmapRcu *old_bitmap = migration_bitmap_rcu, *bitmap;
+        bitmap = g_new(struct BitmapRcu, 1);
+        bitmap->bmap = bitmap_new(new);
 
         /* prevent migration_bitmap content from being set bit
          * by migration_bitmap_sync_range() at the same time.
@@ -1080,16 +1463,410 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t new)
          * at the same time.
          */
         qemu_mutex_lock(&migration_bitmap_mutex);
-        bitmap_copy(bitmap, old_bitmap, old);
-        bitmap_set(bitmap, old, new - old);
-        atomic_rcu_set(&migration_bitmap, bitmap);
+        bitmap_copy(bitmap->bmap, old_bitmap->bmap, old);
+        bitmap_set(bitmap->bmap, old, new - old);
+
+        /* We don't have a way to safely extend the sentmap
+         * with RCU; so mark it as missing, entry to postcopy
+         * will fail.
+         */
+        bitmap->unsentmap = NULL;
+
+        atomic_rcu_set(&migration_bitmap_rcu, bitmap);
         qemu_mutex_unlock(&migration_bitmap_mutex);
         migration_dirty_pages += new - old;
-        synchronize_rcu();
-        g_free(old_bitmap);
+        call_rcu(old_bitmap, migration_bitmap_free, rcu);
     }
 }
 
+/*
+ * 'expected' is the value you expect the bitmap mostly to be full
+ * of; it won't bother printing lines that are all this value.
+ * If 'todump' is null the migration bitmap is dumped.
+ */
+void ram_debug_dump_bitmap(unsigned long *todump, bool expected)
+{
+    int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    int64_t cur;
+    int64_t linelen = 128;
+    char linebuf[129];
+
+    if (!todump) {
+        todump = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    }
+
+    for (cur = 0; cur < ram_pages; cur += linelen) {
+        int64_t curb;
+        bool found = false;
+        /*
+         * Last line; catch the case where the line length
+         * is longer than remaining ram
+         */
+        if (cur + linelen > ram_pages) {
+            linelen = ram_pages - cur;
+        }
+        for (curb = 0; curb < linelen; curb++) {
+            bool thisbit = test_bit(cur + curb, todump);
+            linebuf[curb] = thisbit ? '1' : '.';
+            found = found || (thisbit != expected);
+        }
+        if (found) {
+            linebuf[curb] = '\0';
+            fprintf(stderr,  "0x%08" PRIx64 " : %s\n", cur, linebuf);
+        }
+    }
+}
+
+/* **** functions for postcopy ***** */
+
+/*
+ * Callback from postcopy_each_ram_send_discard for each RAMBlock
+ * Note: At this point the 'unsentmap' is the processed bitmap combined
+ *       with the dirtymap; so a '1' means it's either dirty or unsent.
+ * start,length: Indexes into the bitmap for the first bit
+ *            representing the named block and length in target-pages
+ */
+static int postcopy_send_discard_bm_ram(MigrationState *ms,
+                                        PostcopyDiscardState *pds,
+                                        unsigned long start,
+                                        unsigned long length)
+{
+    unsigned long end = start + length; /* one after the end */
+    unsigned long current;
+    unsigned long *unsentmap;
+
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+    for (current = start; current < end; ) {
+        unsigned long one = find_next_bit(unsentmap, end, current);
+
+        if (one <= end) {
+            unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1);
+            unsigned long discard_length;
+
+            if (zero >= end) {
+                discard_length = end - one;
+            } else {
+                discard_length = zero - one;
+            }
+            postcopy_discard_send_range(ms, pds, one, discard_length);
+            current = one + discard_length;
+        } else {
+            current = one;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Utility for the outgoing postcopy code.
+ *   Calls postcopy_send_discard_bm_ram for each RAMBlock
+ *   passing it bitmap indexes and name.
+ * Returns: 0 on success
+ * (qemu_ram_foreach_block ends up passing unscaled lengths
+ *  which would mean postcopy code would have to deal with target page)
+ */
+static int postcopy_each_ram_send_discard(MigrationState *ms)
+{
+    struct RAMBlock *block;
+    int ret;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        unsigned long first = block->offset >> TARGET_PAGE_BITS;
+        PostcopyDiscardState *pds = postcopy_discard_send_init(ms,
+                                                               first,
+                                                               block->idstr);
+
+        /*
+         * Postcopy sends chunks of bitmap over the wire, but it
+         * just needs indexes at this point, avoids it having
+         * target page specific code.
+         */
+        ret = postcopy_send_discard_bm_ram(ms, pds, first,
+                                    block->used_length >> TARGET_PAGE_BITS);
+        postcopy_discard_send_finish(ms, pds);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Helper for postcopy_chunk_hostpages; it's called twice to cleanup
+ *   the two bitmaps, that are similar, but one is inverted.
+ *
+ * We search for runs of target-pages that don't start or end on a
+ * host page boundary;
+ * unsent_pass=true: Cleans up partially unsent host pages by searching
+ *                 the unsentmap
+ * unsent_pass=false: Cleans up partially dirty host pages by searching
+ *                 the main migration bitmap
+ *
+ */
+static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass,
+                                          RAMBlock *block,
+                                          PostcopyDiscardState *pds)
+{
+    unsigned long *bitmap;
+    unsigned long *unsentmap;
+    unsigned int host_ratio = qemu_host_page_size / TARGET_PAGE_SIZE;
+    unsigned long first = block->offset >> TARGET_PAGE_BITS;
+    unsigned long len = block->used_length >> TARGET_PAGE_BITS;
+    unsigned long last = first + (len - 1);
+    unsigned long run_start;
+
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+
+    if (unsent_pass) {
+        /* Find a sent page */
+        run_start = find_next_zero_bit(unsentmap, last + 1, first);
+    } else {
+        /* Find a dirty page */
+        run_start = find_next_bit(bitmap, last + 1, first);
+    }
+
+    while (run_start <= last) {
+        bool do_fixup = false;
+        unsigned long fixup_start_addr;
+        unsigned long host_offset;
+
+        /*
+         * If the start of this run of pages is in the middle of a host
+         * page, then we need to fixup this host page.
+         */
+        host_offset = run_start % host_ratio;
+        if (host_offset) {
+            do_fixup = true;
+            run_start -= host_offset;
+            fixup_start_addr = run_start;
+            /* For the next pass */
+            run_start = run_start + host_ratio;
+        } else {
+            /* Find the end of this run */
+            unsigned long run_end;
+            if (unsent_pass) {
+                run_end = find_next_bit(unsentmap, last + 1, run_start + 1);
+            } else {
+                run_end = find_next_zero_bit(bitmap, last + 1, run_start + 1);
+            }
+            /*
+             * If the end isn't at the start of a host page, then the
+             * run doesn't finish at the end of a host page
+             * and we need to discard.
+             */
+            host_offset = run_end % host_ratio;
+            if (host_offset) {
+                do_fixup = true;
+                fixup_start_addr = run_end - host_offset;
+                /*
+                 * This host page has gone, the next loop iteration starts
+                 * from after the fixup
+                 */
+                run_start = fixup_start_addr + host_ratio;
+            } else {
+                /*
+                 * No discards on this iteration, next loop starts from
+                 * next sent/dirty page
+                 */
+                run_start = run_end + 1;
+            }
+        }
+
+        if (do_fixup) {
+            unsigned long page;
+
+            /* Tell the destination to discard this page */
+            if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) {
+                /* For the unsent_pass we:
+                 *     discard partially sent pages
+                 * For the !unsent_pass (dirty) we:
+                 *     discard partially dirty pages that were sent
+                 *     (any partially sent pages were already discarded
+                 *     by the previous unsent_pass)
+                 */
+                postcopy_discard_send_range(ms, pds, fixup_start_addr,
+                                            host_ratio);
+            }
+
+            /* Clean up the bitmap */
+            for (page = fixup_start_addr;
+                 page < fixup_start_addr + host_ratio; page++) {
+                /* All pages in this host page are now not sent */
+                set_bit(page, unsentmap);
+
+                /*
+                 * Remark them as dirty, updating the count for any pages
+                 * that weren't previously dirty.
+                 */
+                migration_dirty_pages += !test_and_set_bit(page, bitmap);
+            }
+        }
+
+        if (unsent_pass) {
+            /* Find the next sent page for the next iteration */
+            run_start = find_next_zero_bit(unsentmap, last + 1,
+                                           run_start);
+        } else {
+            /* Find the next dirty page for the next iteration */
+            run_start = find_next_bit(bitmap, last + 1, run_start);
+        }
+    }
+}
+
+/*
+ * Utility for the outgoing postcopy code.
+ *
+ * Discard any partially sent host-page size chunks, mark any partially
+ * dirty host-page size chunks as all dirty.
+ *
+ * Returns: 0 on success
+ */
+static int postcopy_chunk_hostpages(MigrationState *ms)
+{
+    struct RAMBlock *block;
+
+    if (qemu_host_page_size == TARGET_PAGE_SIZE) {
+        /* Easy case - TPS==HPS - nothing to be done */
+        return 0;
+    }
+
+    /* Easiest way to make sure we don't resume in the middle of a host-page */
+    last_seen_block = NULL;
+    last_sent_block = NULL;
+    last_offset     = 0;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        unsigned long first = block->offset >> TARGET_PAGE_BITS;
+
+        PostcopyDiscardState *pds =
+                         postcopy_discard_send_init(ms, first, block->idstr);
+
+        /* First pass: Discard all partially sent host pages */
+        postcopy_chunk_hostpages_pass(ms, true, block, pds);
+        /*
+         * Second pass: Ensure that all partially dirty host pages are made
+         * fully dirty.
+         */
+        postcopy_chunk_hostpages_pass(ms, false, block, pds);
+
+        postcopy_discard_send_finish(ms, pds);
+    } /* ram_list loop */
+
+    return 0;
+}
+
+/*
+ * Transmit the set of pages to be discarded after precopy to the target
+ * these are pages that:
+ *     a) Have been previously transmitted but are now dirty again
+ *     b) Pages that have never been transmitted, this ensures that
+ *        any pages on the destination that have been mapped by background
+ *        tasks get discarded (transparent huge pages is the specific concern)
+ * Hopefully this is pretty sparse
+ */
+int ram_postcopy_send_discard_bitmap(MigrationState *ms)
+{
+    int ret;
+    unsigned long *bitmap, *unsentmap;
+
+    rcu_read_lock();
+
+    /* This should be our last sync, the src is now paused */
+    migration_bitmap_sync();
+
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+    if (!unsentmap) {
+        /* We don't have a safe way to resize the sentmap, so
+         * if the bitmap was resized it will be NULL at this
+         * point.
+         */
+        error_report("migration ram resized during precopy phase");
+        rcu_read_unlock();
+        return -EINVAL;
+    }
+
+    /* Deal with TPS != HPS */
+    ret = postcopy_chunk_hostpages(ms);
+    if (ret) {
+        rcu_read_unlock();
+        return ret;
+    }
+
+    /*
+     * Update the unsentmap to be unsentmap = unsentmap | dirty
+     */
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    bitmap_or(unsentmap, unsentmap, bitmap,
+               last_ram_offset() >> TARGET_PAGE_BITS);
+
+
+    trace_ram_postcopy_send_discard_bitmap();
+#ifdef DEBUG_POSTCOPY
+    ram_debug_dump_bitmap(unsentmap, true);
+#endif
+
+    ret = postcopy_each_ram_send_discard(ms);
+    rcu_read_unlock();
+
+    return ret;
+}
+
+/*
+ * At the start of the postcopy phase of migration, any now-dirty
+ * precopied pages are discarded.
+ *
+ * start, length describe a byte address range within the RAMBlock
+ *
+ * Returns 0 on success.
+ */
+int ram_discard_range(MigrationIncomingState *mis,
+                      const char *block_name,
+                      uint64_t start, size_t length)
+{
+    int ret = -1;
+
+    rcu_read_lock();
+    RAMBlock *rb = qemu_ram_block_by_name(block_name);
+
+    if (!rb) {
+        error_report("ram_discard_range: Failed to find block '%s'",
+                     block_name);
+        goto err;
+    }
+
+    uint8_t *host_startaddr = rb->host + start;
+
+    if ((uintptr_t)host_startaddr & (qemu_host_page_size - 1)) {
+        error_report("ram_discard_range: Unaligned start address: %p",
+                     host_startaddr);
+        goto err;
+    }
+
+    if ((start + length) <= rb->used_length) {
+        uint8_t *host_endaddr = host_startaddr + length;
+        if ((uintptr_t)host_endaddr & (qemu_host_page_size - 1)) {
+            error_report("ram_discard_range: Unaligned end address: %p",
+                         host_endaddr);
+            goto err;
+        }
+        ret = postcopy_ram_discard_range(mis, host_startaddr, length);
+    } else {
+        error_report("ram_discard_range: Overrun block '%s' (%" PRIu64
+                     "/%zx/" RAM_ADDR_FMT")",
+                     block_name, start, length, rb->used_length);
+    }
+
+err:
+    rcu_read_unlock();
+
+    return ret;
+}
+
+
 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
  * long-running RCU critical section.  When rcu-reclaims in the code
  * start to become numerous it will be necessary to reduce the
@@ -1101,7 +1878,6 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     RAMBlock *block;
     int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
 
-    mig_throttle_on = false;
     dirty_rate_high_cnt = 0;
     bitmap_sync_count = 0;
     migration_bitmap_sync_init();
@@ -1145,8 +1921,14 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     reset_ram_globals();
 
     ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
-    migration_bitmap = bitmap_new(ram_bitmap_pages);
-    bitmap_set(migration_bitmap, 0, ram_bitmap_pages);
+    migration_bitmap_rcu = g_new0(struct BitmapRcu, 1);
+    migration_bitmap_rcu->bmap = bitmap_new(ram_bitmap_pages);
+    bitmap_set(migration_bitmap_rcu->bmap, 0, ram_bitmap_pages);
+
+    if (migrate_postcopy_ram()) {
+        migration_bitmap_rcu->unsentmap = bitmap_new(ram_bitmap_pages);
+        bitmap_set(migration_bitmap_rcu->unsentmap, 0, ram_bitmap_pages);
+    }
 
     /*
      * Count the total number of pages used by ram blocks not including any
@@ -1206,7 +1988,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
         }
         pages_sent += pages;
         acct_info.iterations++;
-        check_guest_throttling();
+
         /* we want to check in the 1st loop, just in case it was the 1st time
            and we had to sync the dirty bitmap.
            qemu_get_clock_ns() is a bit expensive, so we only check each some
@@ -1247,7 +2029,9 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 {
     rcu_read_lock();
 
-    migration_bitmap_sync();
+    if (!migration_in_postcopy(migrate_get_current())) {
+        migration_bitmap_sync();
+    }
 
     ram_control_before_iterate(f, RAM_CONTROL_FINISH);
 
@@ -1269,19 +2053,21 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 
     rcu_read_unlock();
 
-    migration_end();
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
 
     return 0;
 }
 
-static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                             uint64_t *non_postcopiable_pending,
+                             uint64_t *postcopiable_pending)
 {
     uint64_t remaining_size;
 
     remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
 
-    if (remaining_size < max_size) {
+    if (!migration_in_postcopy(migrate_get_current()) &&
+        remaining_size < max_size) {
         qemu_mutex_lock_iothread();
         rcu_read_lock();
         migration_bitmap_sync();
@@ -1289,7 +2075,9 @@ static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
         qemu_mutex_unlock_iothread();
         remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
     }
-    return remaining_size;
+
+    /* We can do postcopy, and all the data is postcopiable */
+    *postcopiable_pending += remaining_size;
 }
 
 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
@@ -1330,6 +2118,14 @@ static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
 /* Must be called from within a rcu critical section.
  * Returns a pointer from within the RCU-protected ram_list.
  */
+/*
+ * Read a RAMBlock ID from the stream f, find the host address of the
+ * start of that block and add on 'offset'
+ *
+ * f: Stream to read from
+ * offset: Offset within the block
+ * flags: Page flags (mostly to see if it's a continuation of previous block)
+ */
 static inline void *host_from_stream_offset(QEMUFile *f,
                                             ram_addr_t offset,
                                             int flags)
@@ -1344,21 +2140,19 @@ static inline void *host_from_stream_offset(QEMUFile *f,
             return NULL;
         }
 
-        return memory_region_get_ram_ptr(block->mr) + offset;
+        return block->host + offset;
     }
 
     len = qemu_get_byte(f);
     qemu_get_buffer(f, (uint8_t *)id, len);
     id[len] = 0;
 
-    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-        if (!strncmp(id, block->idstr, sizeof(id)) &&
-            block->max_length > offset) {
-            return memory_region_get_ram_ptr(block->mr) + offset;
-        }
+    block = qemu_ram_block_by_name(id);
+    if (block && block->max_length > offset) {
+        return block->host + offset;
     }
 
-    error_report("Can't find block %s!", id);
+    error_report("Can't find block %s", id);
     return NULL;
 }
 
@@ -1466,11 +2260,148 @@ static void decompress_data_with_multi_threads(uint8_t *compbuf,
     }
 }
 
+/*
+ * Allocate data structures etc needed by incoming migration with postcopy-ram
+ * postcopy-ram's similarly names postcopy_ram_incoming_init does the work
+ */
+int ram_postcopy_incoming_init(MigrationIncomingState *mis)
+{
+    size_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    return postcopy_ram_incoming_init(mis, ram_pages);
+}
+
+/*
+ * Called in postcopy mode by ram_load().
+ * rcu_read_lock is taken prior to this being called.
+ */
+static int ram_load_postcopy(QEMUFile *f)
+{
+    int flags = 0, ret = 0;
+    bool place_needed = false;
+    bool matching_page_sizes = qemu_host_page_size == TARGET_PAGE_SIZE;
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    /* Temporary page that is later 'placed' */
+    void *postcopy_host_page = postcopy_get_tmp_page(mis);
+    void *last_host = NULL;
+    bool all_zero = false;
+
+    while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+        ram_addr_t addr;
+        void *host = NULL;
+        void *page_buffer = NULL;
+        void *place_source = NULL;
+        uint8_t ch;
+
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        trace_ram_load_postcopy_loop((uint64_t)addr, flags);
+        place_needed = false;
+        if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE)) {
+            host = host_from_stream_offset(f, addr, flags);
+            if (!host) {
+                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
+                ret = -EINVAL;
+                break;
+            }
+            page_buffer = host;
+            /*
+             * Postcopy requires that we place whole host pages atomically.
+             * To make it atomic, the data is read into a temporary page
+             * that's moved into place later.
+             * The migration protocol uses,  possibly smaller, target-pages
+             * however the source ensures it always sends all the components
+             * of a host page in order.
+             */
+            page_buffer = postcopy_host_page +
+                          ((uintptr_t)host & ~qemu_host_page_mask);
+            /* If all TP are zero then we can optimise the place */
+            if (!((uintptr_t)host & ~qemu_host_page_mask)) {
+                all_zero = true;
+            } else {
+                /* not the 1st TP within the HP */
+                if (host != (last_host + TARGET_PAGE_SIZE)) {
+                    error_report("Non-sequential target page %p/%p\n",
+                                  host, last_host);
+                    ret = -EINVAL;
+                    break;
+                }
+            }
+
+
+            /*
+             * If it's the last part of a host page then we place the host
+             * page
+             */
+            place_needed = (((uintptr_t)host + TARGET_PAGE_SIZE) &
+                                     ~qemu_host_page_mask) == 0;
+            place_source = postcopy_host_page;
+        }
+        last_host = host;
+
+        switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
+        case RAM_SAVE_FLAG_COMPRESS:
+            ch = qemu_get_byte(f);
+            memset(page_buffer, ch, TARGET_PAGE_SIZE);
+            if (ch) {
+                all_zero = false;
+            }
+            break;
+
+        case RAM_SAVE_FLAG_PAGE:
+            all_zero = false;
+            if (!place_needed || !matching_page_sizes) {
+                qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
+            } else {
+                /* Avoids the qemu_file copy during postcopy, which is
+                 * going to do a copy later; can only do it when we
+                 * do this read in one go (matching page sizes)
+                 */
+                qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
+                                         TARGET_PAGE_SIZE);
+            }
+            break;
+        case RAM_SAVE_FLAG_EOS:
+            /* normal exit */
+            break;
+        default:
+            error_report("Unknown combination of migration flags: %#x"
+                         " (postcopy mode)", flags);
+            ret = -EINVAL;
+        }
+
+        if (place_needed) {
+            /* This gets called at the last target page in the host page */
+            if (all_zero) {
+                ret = postcopy_place_page_zero(mis,
+                                               host + TARGET_PAGE_SIZE -
+                                               qemu_host_page_size);
+            } else {
+                ret = postcopy_place_page(mis, host + TARGET_PAGE_SIZE -
+                                               qemu_host_page_size,
+                                               place_source);
+            }
+        }
+        if (!ret) {
+            ret = qemu_file_get_error(f);
+        }
+    }
+
+    return ret;
+}
+
 static int ram_load(QEMUFile *f, void *opaque, int version_id)
 {
     int flags = 0, ret = 0;
     static uint64_t seq_iter;
     int len = 0;
+    /*
+     * If system is running in postcopy mode, page inserts to host memory must
+     * be atomic
+     */
+    bool postcopy_running = postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING;
 
     seq_iter++;
 
@@ -1484,15 +2415,30 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
      * critical section.
      */
     rcu_read_lock();
-    while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+
+    if (postcopy_running) {
+        ret = ram_load_postcopy(f);
+    }
+
+    while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) {
         ram_addr_t addr, total_ram_bytes;
-        void *host;
+        void *host = NULL;
         uint8_t ch;
 
         addr = qemu_get_be64(f);
         flags = addr & ~TARGET_PAGE_MASK;
         addr &= TARGET_PAGE_MASK;
 
+        if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE |
+                     RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
+            host = host_from_stream_offset(f, addr, flags);
+            if (!host) {
+                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
+                ret = -EINVAL;
+                break;
+            }
+        }
+
         switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
         case RAM_SAVE_FLAG_MEM_SIZE:
             /* Synchronize RAM block list */
@@ -1507,23 +2453,20 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                 id[len] = 0;
                 length = qemu_get_be64(f);
 
-                QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-                    if (!strncmp(id, block->idstr, sizeof(id))) {
-                        if (length != block->used_length) {
-                            Error *local_err = NULL;
+                block = qemu_ram_block_by_name(id);
+                if (block) {
+                    if (length != block->used_length) {
+                        Error *local_err = NULL;
 
-                            ret = qemu_ram_resize(block->offset, length, &local_err);
-                            if (local_err) {
-                                error_report_err(local_err);
-                            }
+                        ret = qemu_ram_resize(block->offset, length,
+                                              &local_err);
+                        if (local_err) {
+                            error_report_err(local_err);
                         }
-                        ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
-                                              block->idstr);
-                        break;
                     }
-                }
-
-                if (!block) {
+                    ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
+                                          block->idstr);
+                } else {
                     error_report("Unknown ramblock \"%s\", cannot "
                                  "accept migration", id);
                     ret = -EINVAL;
@@ -1532,33 +2475,17 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                 total_ram_bytes -= length;
             }
             break;
+
         case RAM_SAVE_FLAG_COMPRESS:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
             ch = qemu_get_byte(f);
             ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
             break;
+
         case RAM_SAVE_FLAG_PAGE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
             qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
             break;
-        case RAM_SAVE_FLAG_COMPRESS_PAGE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Invalid RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
 
+        case RAM_SAVE_FLAG_COMPRESS_PAGE:
             len = qemu_get_be32(f);
             if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
                 error_report("Invalid compressed data length: %d", len);
@@ -1568,13 +2495,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
             qemu_get_buffer(f, compressed_data_buf, len);
             decompress_data_with_multi_threads(compressed_data_buf, host, len);
             break;
+
         case RAM_SAVE_FLAG_XBZRLE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
             if (load_xbzrle(f, addr, host) < 0) {
                 error_report("Failed to decompress XBZRLE page at "
                              RAM_ADDR_FMT, addr);
@@ -1608,10 +2530,11 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
 static SaveVMHandlers savevm_ram_handlers = {
     .save_live_setup = ram_save_setup,
     .save_live_iterate = ram_save_iterate,
-    .save_live_complete = ram_save_complete,
+    .save_live_complete_postcopy = ram_save_complete,
+    .save_live_complete_precopy = ram_save_complete,
     .save_live_pending = ram_save_pending,
     .load_state = ram_load,
-    .cancel = ram_migration_cancel,
+    .cleanup = ram_migration_cleanup,
 };
 
 void ram_mig_init(void)
@@ -1619,52 +2542,3 @@ void ram_mig_init(void)
     qemu_mutex_init(&XBZRLE.lock);
     register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);
 }
-/* Stub function that's gets run on the vcpu when its brought out of the
-   VM to run inside qemu via async_run_on_cpu()*/
-
-static void mig_sleep_cpu(void *opq)
-{
-    qemu_mutex_unlock_iothread();
-    g_usleep(30*1000);
-    qemu_mutex_lock_iothread();
-}
-
-/* To reduce the dirty rate explicitly disallow the VCPUs from spending
-   much time in the VM. The migration thread will try to catchup.
-   Workload will experience a performance drop.
-*/
-static void mig_throttle_guest_down(void)
-{
-    CPUState *cpu;
-
-    qemu_mutex_lock_iothread();
-    CPU_FOREACH(cpu) {
-        async_run_on_cpu(cpu, mig_sleep_cpu, NULL);
-    }
-    qemu_mutex_unlock_iothread();
-}
-
-static void check_guest_throttling(void)
-{
-    static int64_t t0;
-    int64_t        t1;
-
-    if (!mig_throttle_on) {
-        return;
-    }
-
-    if (!t0)  {
-        t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-        return;
-    }
-
-    t1 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-
-    /* If it has been more than 40 ms since the last time the guest
-     * was throttled then do it again.
-     */
-    if (40 < (t1-t0)/1000000) {
-        mig_throttle_guest_down();
-        t0 = t1;
-    }
-}
diff --git a/migration/rdma.c b/migration/rdma.c
index 74876fd7a..dcabb9100 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -19,7 +19,7 @@
 #include "qemu/main-loop.h"
 #include "qemu/sockets.h"
 #include "qemu/bitmap.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -541,7 +541,7 @@ static int rdma_add_block(RDMAContext *rdma, const char *block_name,
     RDMALocalBlock *block;
     RDMALocalBlock *old = local->block;
 
-    local->block = g_malloc0(sizeof(RDMALocalBlock) * (local->nb_blocks + 1));
+    local->block = g_new0(RDMALocalBlock, local->nb_blocks + 1);
 
     if (local->nb_blocks) {
         int x;
@@ -572,12 +572,12 @@ static int rdma_add_block(RDMAContext *rdma, const char *block_name,
     bitmap_clear(block->transit_bitmap, 0, block->nb_chunks);
     block->unregister_bitmap = bitmap_new(block->nb_chunks);
     bitmap_clear(block->unregister_bitmap, 0, block->nb_chunks);
-    block->remote_keys = g_malloc0(block->nb_chunks * sizeof(uint32_t));
+    block->remote_keys = g_new0(uint32_t, block->nb_chunks);
 
     block->is_ram_block = local->init ? false : true;
 
     if (rdma->blockmap) {
-        g_hash_table_insert(rdma->blockmap, (void *) block_offset, block);
+        g_hash_table_insert(rdma->blockmap, (void *)(uintptr_t)block_offset, block);
     }
 
     trace_rdma_add_block(block_name, local->nb_blocks,
@@ -617,8 +617,8 @@ static int qemu_rdma_init_ram_blocks(RDMAContext *rdma)
     memset(local, 0, sizeof *local);
     qemu_ram_foreach_block(qemu_rdma_init_one_block, rdma);
     trace_qemu_rdma_init_ram_blocks(local->nb_blocks);
-    rdma->dest_blocks = (RDMADestBlock *) g_malloc0(sizeof(RDMADestBlock) *
-                        rdma->local_ram_blocks.nb_blocks);
+    rdma->dest_blocks = g_new0(RDMADestBlock,
+                               rdma->local_ram_blocks.nb_blocks);
     local->init = true;
     return 0;
 }
@@ -677,8 +677,7 @@ static int rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block)
 
     if (local->nb_blocks > 1) {
 
-        local->block = g_malloc0(sizeof(RDMALocalBlock) *
-                                    (local->nb_blocks - 1));
+        local->block = g_new0(RDMALocalBlock, local->nb_blocks - 1);
 
         if (block->index) {
             memcpy(local->block, old, sizeof(RDMALocalBlock) * block->index);
@@ -778,7 +777,7 @@ static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id)
  *
  *  If the source VM connects with an IPv4 address without knowing that the
  *  destination has bound to '[::]' the migration will unconditionally fail
- *  unless the management software is explicitly listening on the the IPv4
+ *  unless the management software is explicitly listening on the IPv4
  *  address while using a RoCE-based device.
  *
  *  If the source VM connects with an IPv6 address, then we're OK because we can
@@ -1164,7 +1163,7 @@ static int qemu_rdma_register_and_get_keys(RDMAContext *rdma,
 
     /* allocate memory to store chunk MRs */
     if (!block->pmr) {
-        block->pmr = g_malloc0(block->nb_chunks * sizeof(struct ibv_mr *));
+        block->pmr = g_new0(struct ibv_mr *, block->nb_chunks);
     }
 
     /*
@@ -2494,7 +2493,7 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp)
     InetSocketAddress *addr;
 
     if (host_port) {
-        rdma = g_malloc0(sizeof(RDMAContext));
+        rdma = g_new0(RDMAContext, 1);
         rdma->current_index = -1;
         rdma->current_chunk = -1;
 
@@ -2519,8 +2518,8 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp)
  * SEND messages for control only.
  * VM's ram is handled with regular RDMA messages.
  */
-static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
-                                int64_t pos, int size)
+static ssize_t qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
+                                    int64_t pos, size_t size)
 {
     QEMUFileRDMA *r = opaque;
     QEMUFile *f = r->file;
@@ -2547,7 +2546,8 @@ static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
         r->len = MIN(remaining, RDMA_SEND_INCREMENT);
         remaining -= r->len;
 
-        head.len = r->len;
+        /* Guaranteed to fit due to RDMA_SEND_INCREMENT MIN above */
+        head.len = (uint32_t)r->len;
         head.type = RDMA_CONTROL_QEMU_FILE;
 
         ret = qemu_rdma_exchange_send(rdma, &head, data, NULL, NULL, NULL);
@@ -2564,7 +2564,7 @@ static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
 }
 
 static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf,
-                             int size, int idx)
+                             size_t size, int idx)
 {
     size_t len = 0;
 
@@ -2585,8 +2585,8 @@ static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf,
  * RDMA links don't use bytestreams, so we have to
  * return bytes to QEMUFile opportunistically.
  */
-static int qemu_rdma_get_buffer(void *opaque, uint8_t *buf,
-                                int64_t pos, int size)
+static ssize_t qemu_rdma_get_buffer(void *opaque, uint8_t *buf,
+                                    int64_t pos, size_t size)
 {
     QEMUFileRDMA *r = opaque;
     RDMAContext *rdma = r->rdma;
@@ -3399,7 +3399,7 @@ static void *qemu_fopen_rdma(RDMAContext *rdma, const char *mode)
         return NULL;
     }
 
-    r = g_malloc0(sizeof(QEMUFileRDMA));
+    r = g_new0(QEMUFileRDMA, 1);
     r->rdma = rdma;
 
     if (mode[0] == 'w') {
diff --git a/migration/savevm.c b/migration/savevm.c
index 60712153f..41be296f9 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -37,6 +37,7 @@
 #include "qemu/timer.h"
 #include "audio/audio.h"
 #include "migration/migration.h"
+#include "migration/postcopy-ram.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
@@ -45,6 +46,7 @@
 #include "exec/memory.h"
 #include "qmp-commands.h"
 #include "trace.h"
+#include "qemu/bitops.h"
 #include "qemu/iov.h"
 #include "block/snapshot.h"
 #include "block/qapi.h"
@@ -57,8 +59,26 @@
 #define ARP_PTYPE_IP 0x0800
 #define ARP_OP_REQUEST_REV 0x3
 
+const unsigned int postcopy_ram_discard_version = 0;
+
 static bool skip_section_footers;
 
+static struct mig_cmd_args {
+    ssize_t     len; /* -1 = variable */
+    const char *name;
+} mig_cmd_args[] = {
+    [MIG_CMD_INVALID]          = { .len = -1, .name = "INVALID" },
+    [MIG_CMD_OPEN_RETURN_PATH] = { .len =  0, .name = "OPEN_RETURN_PATH" },
+    [MIG_CMD_PING]             = { .len = sizeof(uint32_t), .name = "PING" },
+    [MIG_CMD_POSTCOPY_ADVISE]  = { .len = 16, .name = "POSTCOPY_ADVISE" },
+    [MIG_CMD_POSTCOPY_LISTEN]  = { .len =  0, .name = "POSTCOPY_LISTEN" },
+    [MIG_CMD_POSTCOPY_RUN]     = { .len =  0, .name = "POSTCOPY_RUN" },
+    [MIG_CMD_POSTCOPY_RAM_DISCARD] = {
+                                   .len = -1, .name = "POSTCOPY_RAM_DISCARD" },
+    [MIG_CMD_PACKAGED]         = { .len =  4, .name = "PACKAGED" },
+    [MIG_CMD_MAX]              = { .len = -1, .name = "MAX" },
+};
+
 static int announce_self_create(uint8_t *buf,
                                 uint8_t *mac_addr)
 {
@@ -138,14 +158,15 @@ static ssize_t block_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
     return qiov.size;
 }
 
-static int block_put_buffer(void *opaque, const uint8_t *buf,
-                           int64_t pos, int size)
+static ssize_t block_put_buffer(void *opaque, const uint8_t *buf,
+                                int64_t pos, size_t size)
 {
     bdrv_save_vmstate(opaque, buf, pos, size);
     return size;
 }
 
-static int block_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t block_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                                size_t size)
 {
     return bdrv_load_vmstate(opaque, buf, pos, size);
 }
@@ -480,7 +501,7 @@ int register_savevm_live(DeviceState *dev,
 {
     SaveStateEntry *se;
 
-    se = g_malloc0(sizeof(SaveStateEntry));
+    se = g_new0(SaveStateEntry, 1);
     se->version_id = version_id;
     se->section_id = savevm_state.global_section_id++;
     se->ops = ops;
@@ -498,7 +519,7 @@ int register_savevm_live(DeviceState *dev,
             pstrcat(se->idstr, sizeof(se->idstr), "/");
             g_free(id);
 
-            se->compat = g_malloc0(sizeof(CompatEntry));
+            se->compat = g_new0(CompatEntry, 1);
             pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), idstr);
             se->compat->instance_id = instance_id == -1 ?
                          calculate_compat_instance_id(idstr) : instance_id;
@@ -526,7 +547,7 @@ int register_savevm(DeviceState *dev,
                     LoadStateHandler *load_state,
                     void *opaque)
 {
-    SaveVMHandlers *ops = g_malloc0(sizeof(SaveVMHandlers));
+    SaveVMHandlers *ops = g_new0(SaveVMHandlers, 1);
     ops->save_state = save_state;
     ops->load_state = load_state;
     return register_savevm_live(dev, idstr, instance_id, version_id,
@@ -551,9 +572,7 @@ void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque)
     QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) {
         if (strcmp(se->idstr, id) == 0 && se->opaque == opaque) {
             QTAILQ_REMOVE(&savevm_state.handlers, se, entry);
-            if (se->compat) {
-                g_free(se->compat);
-            }
+            g_free(se->compat);
             g_free(se->ops);
             g_free(se);
         }
@@ -570,7 +589,7 @@ int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
     /* If this triggers, alias support can be dropped for the vmsd. */
     assert(alias_id == -1 || required_for_version >= vmsd->minimum_version_id);
 
-    se = g_malloc0(sizeof(SaveStateEntry));
+    se = g_new0(SaveStateEntry, 1);
     se->version_id = vmsd->version_id;
     se->section_id = savevm_state.global_section_id++;
     se->opaque = opaque;
@@ -584,7 +603,7 @@ int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
             pstrcat(se->idstr, sizeof(se->idstr), "/");
             g_free(id);
 
-            se->compat = g_malloc0(sizeof(CompatEntry));
+            se->compat = g_new0(CompatEntry, 1);
             pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), vmsd->name);
             se->compat->instance_id = instance_id == -1 ?
                          calculate_compat_instance_id(vmsd->name) : instance_id;
@@ -612,9 +631,7 @@ void vmstate_unregister(DeviceState *dev, const VMStateDescription *vmsd,
     QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) {
         if (se->vmsd == vmsd && se->opaque == opaque) {
             QTAILQ_REMOVE(&savevm_state.handlers, se, entry);
-            if (se->compat) {
-                g_free(se->compat);
-            }
+            g_free(se->compat);
             g_free(se);
         }
     }
@@ -697,6 +714,156 @@ static void save_section_footer(QEMUFile *f, SaveStateEntry *se)
     }
 }
 
+/**
+ * qemu_savevm_command_send: Send a 'QEMU_VM_COMMAND' type element with the
+ *                           command and associated data.
+ *
+ * @f: File to send command on
+ * @command: Command type to send
+ * @len: Length of associated data
+ * @data: Data associated with command.
+ */
+void qemu_savevm_command_send(QEMUFile *f,
+                              enum qemu_vm_cmd command,
+                              uint16_t len,
+                              uint8_t *data)
+{
+    trace_savevm_command_send(command, len);
+    qemu_put_byte(f, QEMU_VM_COMMAND);
+    qemu_put_be16(f, (uint16_t)command);
+    qemu_put_be16(f, len);
+    qemu_put_buffer(f, data, len);
+    qemu_fflush(f);
+}
+
+void qemu_savevm_send_ping(QEMUFile *f, uint32_t value)
+{
+    uint32_t buf;
+
+    trace_savevm_send_ping(value);
+    buf = cpu_to_be32(value);
+    qemu_savevm_command_send(f, MIG_CMD_PING, sizeof(value), (uint8_t *)&buf);
+}
+
+void qemu_savevm_send_open_return_path(QEMUFile *f)
+{
+    trace_savevm_send_open_return_path();
+    qemu_savevm_command_send(f, MIG_CMD_OPEN_RETURN_PATH, 0, NULL);
+}
+
+/* We have a buffer of data to send; we don't want that all to be loaded
+ * by the command itself, so the command contains just the length of the
+ * extra buffer that we then send straight after it.
+ * TODO: Must be a better way to organise that
+ *
+ * Returns:
+ *    0 on success
+ *    -ve on error
+ */
+int qemu_savevm_send_packaged(QEMUFile *f, const QEMUSizedBuffer *qsb)
+{
+    size_t cur_iov;
+    size_t len = qsb_get_length(qsb);
+    uint32_t tmp;
+
+    if (len > MAX_VM_CMD_PACKAGED_SIZE) {
+        error_report("%s: Unreasonably large packaged state: %zu",
+                     __func__, len);
+        return -1;
+    }
+
+    tmp = cpu_to_be32(len);
+
+    trace_qemu_savevm_send_packaged();
+    qemu_savevm_command_send(f, MIG_CMD_PACKAGED, 4, (uint8_t *)&tmp);
+
+    /* all the data follows (concatinating the iov's) */
+    for (cur_iov = 0; cur_iov < qsb->n_iov; cur_iov++) {
+        /* The iov entries are partially filled */
+        size_t towrite = MIN(qsb->iov[cur_iov].iov_len, len);
+        len -= towrite;
+
+        if (!towrite) {
+            break;
+        }
+
+        qemu_put_buffer(f, qsb->iov[cur_iov].iov_base, towrite);
+    }
+
+    return 0;
+}
+
+/* Send prior to any postcopy transfer */
+void qemu_savevm_send_postcopy_advise(QEMUFile *f)
+{
+    uint64_t tmp[2];
+    tmp[0] = cpu_to_be64(getpagesize());
+    tmp[1] = cpu_to_be64(1ul << qemu_target_page_bits());
+
+    trace_qemu_savevm_send_postcopy_advise();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_ADVISE, 16, (uint8_t *)tmp);
+}
+
+/* Sent prior to starting the destination running in postcopy, discard pages
+ * that have already been sent but redirtied on the source.
+ * CMD_POSTCOPY_RAM_DISCARD consist of:
+ *      byte   version (0)
+ *      byte   Length of name field (not including 0)
+ *  n x byte   RAM block name
+ *      byte   0 terminator (just for safety)
+ *  n x        Byte ranges within the named RAMBlock
+ *      be64   Start of the range
+ *      be64   Length
+ *
+ *  name:  RAMBlock name that these entries are part of
+ *  len: Number of page entries
+ *  start_list: 'len' addresses
+ *  length_list: 'len' addresses
+ *
+ */
+void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
+                                           uint16_t len,
+                                           uint64_t *start_list,
+                                           uint64_t *length_list)
+{
+    uint8_t *buf;
+    uint16_t tmplen;
+    uint16_t t;
+    size_t name_len = strlen(name);
+
+    trace_qemu_savevm_send_postcopy_ram_discard(name, len);
+    assert(name_len < 256);
+    buf = g_malloc0(1 + 1 + name_len + 1 + (8 + 8) * len);
+    buf[0] = postcopy_ram_discard_version;
+    buf[1] = name_len;
+    memcpy(buf + 2, name, name_len);
+    tmplen = 2 + name_len;
+    buf[tmplen++] = '\0';
+
+    for (t = 0; t < len; t++) {
+        cpu_to_be64w((uint64_t *)(buf + tmplen), start_list[t]);
+        tmplen += 8;
+        cpu_to_be64w((uint64_t *)(buf + tmplen), length_list[t]);
+        tmplen += 8;
+    }
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RAM_DISCARD, tmplen, buf);
+    g_free(buf);
+}
+
+/* Get the destination into a state where it can receive postcopy data. */
+void qemu_savevm_send_postcopy_listen(QEMUFile *f)
+{
+    trace_savevm_send_postcopy_listen();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_LISTEN, 0, NULL);
+}
+
+/* Kick the destination into running */
+void qemu_savevm_send_postcopy_run(QEMUFile *f)
+{
+    trace_savevm_send_postcopy_run();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RUN, 0, NULL);
+}
+
 bool qemu_savevm_state_blocked(Error **errp)
 {
     SaveStateEntry *se;
@@ -711,11 +878,23 @@ bool qemu_savevm_state_blocked(Error **errp)
     return false;
 }
 
+static bool enforce_config_section(void)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    return machine->enforce_config_section;
+}
+
 void qemu_savevm_state_header(QEMUFile *f)
 {
     trace_savevm_state_header();
     qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
     qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+
+    if (!savevm_state.skip_configuration || enforce_config_section()) {
+        qemu_put_byte(f, QEMU_VM_CONFIGURATION);
+        vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0);
+    }
+
 }
 
 void qemu_savevm_state_begin(QEMUFile *f,
@@ -732,11 +911,6 @@ void qemu_savevm_state_begin(QEMUFile *f,
         se->ops->set_params(params, se->opaque);
     }
 
-    if (!savevm_state.skip_configuration) {
-        qemu_put_byte(f, QEMU_VM_CONFIGURATION);
-        vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0);
-    }
-
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
         if (!se->ops || !se->ops->save_live_setup) {
             continue;
@@ -763,7 +937,7 @@ void qemu_savevm_state_begin(QEMUFile *f,
  *   0 : We haven't finished, caller have to go again
  *   1 : We have finished, we can go to complete phase
  */
-int qemu_savevm_state_iterate(QEMUFile *f)
+int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy)
 {
     SaveStateEntry *se;
     int ret = 1;
@@ -778,6 +952,15 @@ int qemu_savevm_state_iterate(QEMUFile *f)
                 continue;
             }
         }
+        /*
+         * In the postcopy phase, any device that doesn't know how to
+         * do postcopy should have saved it's state in the _complete
+         * call that's already run, it might get confused if we call
+         * iterate afterwards.
+         */
+        if (postcopy && !se->ops->save_live_complete_postcopy) {
+            continue;
+        }
         if (qemu_file_rate_limit(f)) {
             return 0;
         }
@@ -806,24 +989,69 @@ int qemu_savevm_state_iterate(QEMUFile *f)
 static bool should_send_vmdesc(void)
 {
     MachineState *machine = MACHINE(qdev_get_machine());
-    return !machine->suppress_vmdesc;
+    bool in_postcopy = migration_in_postcopy(migrate_get_current());
+    return !machine->suppress_vmdesc && !in_postcopy;
 }
 
-void qemu_savevm_state_complete(QEMUFile *f)
+/*
+ * Calls the save_live_complete_postcopy methods
+ * causing the last few pages to be sent immediately and doing any associated
+ * cleanup.
+ * Note postcopy also calls qemu_savevm_state_complete_precopy to complete
+ * all the other devices, but that happens at the point we switch to postcopy.
+ */
+void qemu_savevm_state_complete_postcopy(QEMUFile *f)
+{
+    SaveStateEntry *se;
+    int ret;
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops || !se->ops->save_live_complete_postcopy) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        trace_savevm_section_start(se->idstr, se->section_id);
+        /* Section type */
+        qemu_put_byte(f, QEMU_VM_SECTION_END);
+        qemu_put_be32(f, se->section_id);
+
+        ret = se->ops->save_live_complete_postcopy(f, se->opaque);
+        trace_savevm_section_end(se->idstr, se->section_id, ret);
+        save_section_footer(f, se);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+            return;
+        }
+    }
+
+    qemu_put_byte(f, QEMU_VM_EOF);
+    qemu_fflush(f);
+}
+
+void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only)
 {
     QJSON *vmdesc;
     int vmdesc_len;
     SaveStateEntry *se;
     int ret;
+    bool in_postcopy = migration_in_postcopy(migrate_get_current());
 
-    trace_savevm_state_complete();
+    trace_savevm_state_complete_precopy();
 
     cpu_synchronize_all_states();
 
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
-        if (!se->ops || !se->ops->save_live_complete) {
+        if (!se->ops ||
+            (in_postcopy && se->ops->save_live_complete_postcopy) ||
+            (in_postcopy && !iterable_only) ||
+            !se->ops->save_live_complete_precopy) {
             continue;
         }
+
         if (se->ops && se->ops->is_active) {
             if (!se->ops->is_active(se->opaque)) {
                 continue;
@@ -833,7 +1061,7 @@ void qemu_savevm_state_complete(QEMUFile *f)
 
         save_section_header(f, se, QEMU_VM_SECTION_END);
 
-        ret = se->ops->save_live_complete(f, se->opaque);
+        ret = se->ops->save_live_complete_precopy(f, se->opaque);
         trace_savevm_section_end(se->idstr, se->section_id, ret);
         save_section_footer(f, se);
         if (ret < 0) {
@@ -842,6 +1070,10 @@ void qemu_savevm_state_complete(QEMUFile *f)
         }
     }
 
+    if (iterable_only) {
+        return;
+    }
+
     vmdesc = qjson_new();
     json_prop_int(vmdesc, "page_size", TARGET_PAGE_SIZE);
     json_start_array(vmdesc, "devices");
@@ -870,7 +1102,10 @@ void qemu_savevm_state_complete(QEMUFile *f)
         save_section_footer(f, se);
     }
 
-    qemu_put_byte(f, QEMU_VM_EOF);
+    if (!in_postcopy) {
+        /* Postcopy stream will still be going */
+        qemu_put_byte(f, QEMU_VM_EOF);
+    }
 
     json_end_array(vmdesc);
     qjson_finish(vmdesc);
@@ -886,10 +1121,19 @@ void qemu_savevm_state_complete(QEMUFile *f)
     qemu_fflush(f);
 }
 
-uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
+/* Give an estimate of the amount left to be transferred,
+ * the result is split into the amount for units that can and
+ * for units that can't do postcopy.
+ */
+void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
+                               uint64_t *res_non_postcopiable,
+                               uint64_t *res_postcopiable)
 {
     SaveStateEntry *se;
-    uint64_t ret = 0;
+
+    *res_non_postcopiable = 0;
+    *res_postcopiable = 0;
+
 
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
         if (!se->ops || !se->ops->save_live_pending) {
@@ -900,19 +1144,19 @@ uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
                 continue;
             }
         }
-        ret += se->ops->save_live_pending(f, se->opaque, max_size);
+        se->ops->save_live_pending(f, se->opaque, max_size,
+                                   res_non_postcopiable, res_postcopiable);
     }
-    return ret;
 }
 
-void qemu_savevm_state_cancel(void)
+void qemu_savevm_state_cleanup(void)
 {
     SaveStateEntry *se;
 
-    trace_savevm_state_cancel();
+    trace_savevm_state_cleanup();
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
-        if (se->ops && se->ops->cancel) {
-            se->ops->cancel(se->opaque);
+        if (se->ops && se->ops->cleanup) {
+            se->ops->cleanup(se->opaque);
         }
     }
 }
@@ -924,6 +1168,8 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp)
         .blk = 0,
         .shared = 0
     };
+    MigrationState *ms = migrate_init(&params);
+    ms->file = f;
 
     if (qemu_savevm_state_blocked(errp)) {
         return -EINVAL;
@@ -935,18 +1181,18 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp)
     qemu_mutex_lock_iothread();
 
     while (qemu_file_get_error(f) == 0) {
-        if (qemu_savevm_state_iterate(f) > 0) {
+        if (qemu_savevm_state_iterate(f, false) > 0) {
             break;
         }
     }
 
     ret = qemu_file_get_error(f);
     if (ret == 0) {
-        qemu_savevm_state_complete(f);
+        qemu_savevm_state_complete_precopy(f, false);
         ret = qemu_file_get_error(f);
     }
+    qemu_savevm_state_cleanup();
     if (ret != 0) {
-        qemu_savevm_state_cancel();
         error_setg_errno(errp, -ret, "Error while writing VM state");
     }
     return ret;
@@ -1004,6 +1250,420 @@ static SaveStateEntry *find_se(const char *idstr, int instance_id)
     return NULL;
 }
 
+enum LoadVMExitCodes {
+    /* Allow a command to quit all layers of nested loadvm loops */
+    LOADVM_QUIT     =  1,
+};
+
+static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
+
+/* ------ incoming postcopy messages ------ */
+/* 'advise' arrives before any transfers just to tell us that a postcopy
+ * *might* happen - it might be skipped if precopy transferred everything
+ * quickly.
+ */
+static int loadvm_postcopy_handle_advise(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_ADVISE);
+    uint64_t remote_hps, remote_tps;
+
+    trace_loadvm_postcopy_handle_advise();
+    if (ps != POSTCOPY_INCOMING_NONE) {
+        error_report("CMD_POSTCOPY_ADVISE in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+
+    if (!postcopy_ram_supported_by_host()) {
+        return -1;
+    }
+
+    remote_hps = qemu_get_be64(mis->from_src_file);
+    if (remote_hps != getpagesize())  {
+        /*
+         * Some combinations of mismatch are probably possible but it gets
+         * a bit more complicated.  In particular we need to place whole
+         * host pages on the dest at once, and we need to ensure that we
+         * handle dirtying to make sure we never end up sending part of
+         * a hostpage on it's own.
+         */
+        error_report("Postcopy needs matching host page sizes (s=%d d=%d)",
+                     (int)remote_hps, getpagesize());
+        return -1;
+    }
+
+    remote_tps = qemu_get_be64(mis->from_src_file);
+    if (remote_tps != (1ul << qemu_target_page_bits())) {
+        /*
+         * Again, some differences could be dealt with, but for now keep it
+         * simple.
+         */
+        error_report("Postcopy needs matching target page sizes (s=%d d=%d)",
+                     (int)remote_tps, 1 << qemu_target_page_bits());
+        return -1;
+    }
+
+    if (ram_postcopy_incoming_init(mis)) {
+        return -1;
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_ADVISE);
+
+    return 0;
+}
+
+/* After postcopy we will be told to throw some pages away since they're
+ * dirty and will have to be demand fetched.  Must happen before CPU is
+ * started.
+ * There can be 0..many of these messages, each encoding multiple pages.
+ */
+static int loadvm_postcopy_ram_handle_discard(MigrationIncomingState *mis,
+                                              uint16_t len)
+{
+    int tmp;
+    char ramid[256];
+    PostcopyState ps = postcopy_state_get();
+
+    trace_loadvm_postcopy_ram_handle_discard();
+
+    switch (ps) {
+    case POSTCOPY_INCOMING_ADVISE:
+        /* 1st discard */
+        tmp = postcopy_ram_prepare_discard(mis);
+        if (tmp) {
+            return tmp;
+        }
+        break;
+
+    case POSTCOPY_INCOMING_DISCARD:
+        /* Expected state */
+        break;
+
+    default:
+        error_report("CMD_POSTCOPY_RAM_DISCARD in wrong postcopy state (%d)",
+                     ps);
+        return -1;
+    }
+    /* We're expecting a
+     *    Version (0)
+     *    a RAM ID string (length byte, name, 0 term)
+     *    then at least 1 16 byte chunk
+    */
+    if (len < (1 + 1 + 1 + 1 + 2 * 8)) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len);
+        return -1;
+    }
+
+    tmp = qemu_get_byte(mis->from_src_file);
+    if (tmp != postcopy_ram_discard_version) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid version (%d)", tmp);
+        return -1;
+    }
+
+    if (!qemu_get_counted_string(mis->from_src_file, ramid)) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD Failed to read RAMBlock ID");
+        return -1;
+    }
+    tmp = qemu_get_byte(mis->from_src_file);
+    if (tmp != 0) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD missing nil (%d)", tmp);
+        return -1;
+    }
+
+    len -= 3 + strlen(ramid);
+    if (len % 16) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len);
+        return -1;
+    }
+    trace_loadvm_postcopy_ram_handle_discard_header(ramid, len);
+    while (len) {
+        uint64_t start_addr, block_length;
+        start_addr = qemu_get_be64(mis->from_src_file);
+        block_length = qemu_get_be64(mis->from_src_file);
+
+        len -= 16;
+        int ret = ram_discard_range(mis, ramid, start_addr,
+                                    block_length);
+        if (ret) {
+            return ret;
+        }
+    }
+    trace_loadvm_postcopy_ram_handle_discard_end();
+
+    return 0;
+}
+
+/*
+ * Triggered by a postcopy_listen command; this thread takes over reading
+ * the input stream, leaving the main thread free to carry on loading the rest
+ * of the device state (from RAM).
+ * (TODO:This could do with being in a postcopy file - but there again it's
+ * just another input loop, not that postcopy specific)
+ */
+static void *postcopy_ram_listen_thread(void *opaque)
+{
+    QEMUFile *f = opaque;
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    int load_res;
+
+    qemu_sem_post(&mis->listen_thread_sem);
+    trace_postcopy_ram_listen_thread_start();
+
+    /*
+     * Because we're a thread and not a coroutine we can't yield
+     * in qemu_file, and thus we must be blocking now.
+     */
+    qemu_file_set_blocking(f, true);
+    load_res = qemu_loadvm_state_main(f, mis);
+    /* And non-blocking again so we don't block in any cleanup */
+    qemu_file_set_blocking(f, false);
+
+    trace_postcopy_ram_listen_thread_exit();
+    if (load_res < 0) {
+        error_report("%s: loadvm failed: %d", __func__, load_res);
+        qemu_file_set_error(f, load_res);
+    } else {
+        /*
+         * This looks good, but it's possible that the device loading in the
+         * main thread hasn't finished yet, and so we might not be in 'RUN'
+         * state yet; wait for the end of the main thread.
+         */
+        qemu_event_wait(&mis->main_thread_load_event);
+    }
+    postcopy_ram_incoming_cleanup(mis);
+    /*
+     * If everything has worked fine, then the main thread has waited
+     * for us to start, and we're the last use of the mis.
+     * (If something broke then qemu will have to exit anyway since it's
+     * got a bad migration state).
+     */
+    migration_incoming_state_destroy();
+
+    if (load_res < 0) {
+        /*
+         * If something went wrong then we have a bad state so exit;
+         * depending how far we got it might be possible at this point
+         * to leave the guest running and fire MCEs for pages that never
+         * arrived as a desperate recovery step.
+         */
+        exit(EXIT_FAILURE);
+    }
+
+    return NULL;
+}
+
+/* After this message we must be able to immediately receive postcopy data */
+static int loadvm_postcopy_handle_listen(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_LISTENING);
+    trace_loadvm_postcopy_handle_listen();
+    if (ps != POSTCOPY_INCOMING_ADVISE && ps != POSTCOPY_INCOMING_DISCARD) {
+        error_report("CMD_POSTCOPY_LISTEN in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+    if (ps == POSTCOPY_INCOMING_ADVISE) {
+        /*
+         * A rare case, we entered listen without having to do any discards,
+         * so do the setup that's normally done at the time of the 1st discard.
+         */
+        postcopy_ram_prepare_discard(mis);
+    }
+
+    /*
+     * Sensitise RAM - can now generate requests for blocks that don't exist
+     * However, at this point the CPU shouldn't be running, and the IO
+     * shouldn't be doing anything yet so don't actually expect requests
+     */
+    if (postcopy_ram_enable_notify(mis)) {
+        return -1;
+    }
+
+    if (mis->have_listen_thread) {
+        error_report("CMD_POSTCOPY_RAM_LISTEN already has a listen thread");
+        return -1;
+    }
+
+    mis->have_listen_thread = true;
+    /* Start up the listening thread and wait for it to signal ready */
+    qemu_sem_init(&mis->listen_thread_sem, 0);
+    qemu_thread_create(&mis->listen_thread, "postcopy/listen",
+                       postcopy_ram_listen_thread, mis->from_src_file,
+                       QEMU_THREAD_JOINABLE);
+    qemu_sem_wait(&mis->listen_thread_sem);
+    qemu_sem_destroy(&mis->listen_thread_sem);
+
+    return 0;
+}
+
+/* After all discards we can start running and asking for pages */
+static int loadvm_postcopy_handle_run(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_RUNNING);
+    Error *local_err = NULL;
+
+    trace_loadvm_postcopy_handle_run();
+    if (ps != POSTCOPY_INCOMING_LISTENING) {
+        error_report("CMD_POSTCOPY_RUN in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+
+    /* TODO we should move all of this lot into postcopy_ram.c or a shared code
+     * in migration.c
+     */
+    cpu_synchronize_all_post_init();
+
+    qemu_announce_self();
+
+    /* Make sure all file formats flush their mutable metadata */
+    bdrv_invalidate_cache_all(&local_err);
+    if (local_err) {
+        error_report_err(local_err);
+        return -1;
+    }
+
+    trace_loadvm_postcopy_handle_run_cpu_sync();
+    cpu_synchronize_all_post_init();
+
+    trace_loadvm_postcopy_handle_run_vmstart();
+
+    if (autostart) {
+        /* Hold onto your hats, starting the CPU */
+        vm_start();
+    } else {
+        /* leave it paused and let management decide when to start the CPU */
+        runstate_set(RUN_STATE_PAUSED);
+    }
+
+    /* We need to finish reading the stream from the package
+     * and also stop reading anything more from the stream that loaded the
+     * package (since it's now being read by the listener thread).
+     * LOADVM_QUIT will quit all the layers of nested loadvm loops.
+     */
+    return LOADVM_QUIT;
+}
+
+/**
+ * Immediately following this command is a blob of data containing an embedded
+ * chunk of migration stream; read it and load it.
+ *
+ * @mis: Incoming state
+ * @length: Length of packaged data to read
+ *
+ * Returns: Negative values on error
+ *
+ */
+static int loadvm_handle_cmd_packaged(MigrationIncomingState *mis)
+{
+    int ret;
+    uint8_t *buffer;
+    uint32_t length;
+    QEMUSizedBuffer *qsb;
+
+    length = qemu_get_be32(mis->from_src_file);
+    trace_loadvm_handle_cmd_packaged(length);
+
+    if (length > MAX_VM_CMD_PACKAGED_SIZE) {
+        error_report("Unreasonably large packaged state: %u", length);
+        return -1;
+    }
+    buffer = g_malloc0(length);
+    ret = qemu_get_buffer(mis->from_src_file, buffer, (int)length);
+    if (ret != length) {
+        g_free(buffer);
+        error_report("CMD_PACKAGED: Buffer receive fail ret=%d length=%d\n",
+                ret, length);
+        return (ret < 0) ? ret : -EAGAIN;
+    }
+    trace_loadvm_handle_cmd_packaged_received(ret);
+
+    /* Setup a dummy QEMUFile that actually reads from the buffer */
+    qsb = qsb_create(buffer, length);
+    g_free(buffer); /* Because qsb_create copies */
+    if (!qsb) {
+        error_report("Unable to create qsb");
+    }
+    QEMUFile *packf = qemu_bufopen("r", qsb);
+
+    ret = qemu_loadvm_state_main(packf, mis);
+    trace_loadvm_handle_cmd_packaged_main(ret);
+    qemu_fclose(packf);
+    qsb_free(qsb);
+
+    return ret;
+}
+
+/*
+ * Process an incoming 'QEMU_VM_COMMAND'
+ * 0           just a normal return
+ * LOADVM_QUIT All good, but exit the loop
+ * <0          Error
+ */
+static int loadvm_process_command(QEMUFile *f)
+{
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    uint16_t cmd;
+    uint16_t len;
+    uint32_t tmp32;
+
+    cmd = qemu_get_be16(f);
+    len = qemu_get_be16(f);
+
+    trace_loadvm_process_command(cmd, len);
+    if (cmd >= MIG_CMD_MAX || cmd == MIG_CMD_INVALID) {
+        error_report("MIG_CMD 0x%x unknown (len 0x%x)", cmd, len);
+        return -EINVAL;
+    }
+
+    if (mig_cmd_args[cmd].len != -1 && mig_cmd_args[cmd].len != len) {
+        error_report("%s received with bad length - expecting %zu, got %d",
+                     mig_cmd_args[cmd].name,
+                     (size_t)mig_cmd_args[cmd].len, len);
+        return -ERANGE;
+    }
+
+    switch (cmd) {
+    case MIG_CMD_OPEN_RETURN_PATH:
+        if (mis->to_src_file) {
+            error_report("CMD_OPEN_RETURN_PATH called when RP already open");
+            /* Not really a problem, so don't give up */
+            return 0;
+        }
+        mis->to_src_file = qemu_file_get_return_path(f);
+        if (!mis->to_src_file) {
+            error_report("CMD_OPEN_RETURN_PATH failed");
+            return -1;
+        }
+        break;
+
+    case MIG_CMD_PING:
+        tmp32 = qemu_get_be32(f);
+        trace_loadvm_process_command_ping(tmp32);
+        if (!mis->to_src_file) {
+            error_report("CMD_PING (0x%x) received with no return path",
+                         tmp32);
+            return -1;
+        }
+        migrate_send_rp_pong(mis, tmp32);
+        break;
+
+    case MIG_CMD_PACKAGED:
+        return loadvm_handle_cmd_packaged(mis);
+
+    case MIG_CMD_POSTCOPY_ADVISE:
+        return loadvm_postcopy_handle_advise(mis);
+
+    case MIG_CMD_POSTCOPY_LISTEN:
+        return loadvm_postcopy_handle_listen(mis);
+
+    case MIG_CMD_POSTCOPY_RUN:
+        return loadvm_postcopy_handle_run(mis);
+
+    case MIG_CMD_POSTCOPY_RAM_DISCARD:
+        return loadvm_postcopy_ram_handle_discard(mis, len);
+    }
+
+    return 0;
+}
+
 struct LoadStateEntry {
     QLIST_ENTRY(LoadStateEntry) entry;
     SaveStateEntry *se;
@@ -1056,47 +1716,10 @@ void loadvm_free_handlers(MigrationIncomingState *mis)
     }
 }
 
-int qemu_loadvm_state(QEMUFile *f)
+static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
 {
-    MigrationIncomingState *mis = migration_incoming_get_current();
-    Error *local_err = NULL;
     uint8_t section_type;
-    unsigned int v;
     int ret;
-    int file_error_after_eof = -1;
-
-    if (qemu_savevm_state_blocked(&local_err)) {
-        error_report_err(local_err);
-        return -EINVAL;
-    }
-
-    v = qemu_get_be32(f);
-    if (v != QEMU_VM_FILE_MAGIC) {
-        error_report("Not a migration stream");
-        return -EINVAL;
-    }
-
-    v = qemu_get_be32(f);
-    if (v == QEMU_VM_FILE_VERSION_COMPAT) {
-        error_report("SaveVM v2 format is obsolete and don't work anymore");
-        return -ENOTSUP;
-    }
-    if (v != QEMU_VM_FILE_VERSION) {
-        error_report("Unsupported migration stream version");
-        return -ENOTSUP;
-    }
-
-    if (!savevm_state.skip_configuration) {
-        if (qemu_get_byte(f) != QEMU_VM_CONFIGURATION) {
-            error_report("Configuration section missing");
-            return -EINVAL;
-        }
-        ret = vmstate_load_state(f, &vmstate_configuration, &savevm_state, 0);
-
-        if (ret) {
-            return ret;
-        }
-    }
 
     while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) {
         uint32_t instance_id, version_id, section_id;
@@ -1125,16 +1748,14 @@ int qemu_loadvm_state(QEMUFile *f)
             if (se == NULL) {
                 error_report("Unknown savevm section or instance '%s' %d",
                              idstr, instance_id);
-                ret = -EINVAL;
-                goto out;
+                return -EINVAL;
             }
 
             /* Validate version */
             if (version_id > se->version_id) {
                 error_report("savevm: unsupported version %d for '%s' v%d",
                              version_id, idstr, se->version_id);
-                ret = -EINVAL;
-                goto out;
+                return -EINVAL;
             }
 
             /* Add entry */
@@ -1149,11 +1770,10 @@ int qemu_loadvm_state(QEMUFile *f)
             if (ret < 0) {
                 error_report("error while loading state for instance 0x%x of"
                              " device '%s'", instance_id, idstr);
-                goto out;
+                return ret;
             }
             if (!check_section_footer(f, le)) {
-                ret = -EINVAL;
-                goto out;
+                return -EINVAL;
             }
             break;
         case QEMU_VM_SECTION_PART:
@@ -1168,29 +1788,88 @@ int qemu_loadvm_state(QEMUFile *f)
             }
             if (le == NULL) {
                 error_report("Unknown savevm section %d", section_id);
-                ret = -EINVAL;
-                goto out;
+                return -EINVAL;
             }
 
             ret = vmstate_load(f, le->se, le->version_id);
             if (ret < 0) {
                 error_report("error while loading state section id %d(%s)",
                              section_id, le->se->idstr);
-                goto out;
+                return ret;
             }
             if (!check_section_footer(f, le)) {
-                ret = -EINVAL;
-                goto out;
+                return -EINVAL;
+            }
+            break;
+        case QEMU_VM_COMMAND:
+            ret = loadvm_process_command(f);
+            trace_qemu_loadvm_state_section_command(ret);
+            if ((ret < 0) || (ret & LOADVM_QUIT)) {
+                return ret;
             }
             break;
         default:
             error_report("Unknown savevm section type %d", section_type);
-            ret = -EINVAL;
-            goto out;
+            return -EINVAL;
         }
     }
 
-    file_error_after_eof = qemu_file_get_error(f);
+    return 0;
+}
+
+int qemu_loadvm_state(QEMUFile *f)
+{
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    Error *local_err = NULL;
+    unsigned int v;
+    int ret;
+
+    if (qemu_savevm_state_blocked(&local_err)) {
+        error_report_err(local_err);
+        return -EINVAL;
+    }
+
+    v = qemu_get_be32(f);
+    if (v != QEMU_VM_FILE_MAGIC) {
+        error_report("Not a migration stream");
+        return -EINVAL;
+    }
+
+    v = qemu_get_be32(f);
+    if (v == QEMU_VM_FILE_VERSION_COMPAT) {
+        error_report("SaveVM v2 format is obsolete and don't work anymore");
+        return -ENOTSUP;
+    }
+    if (v != QEMU_VM_FILE_VERSION) {
+        error_report("Unsupported migration stream version");
+        return -ENOTSUP;
+    }
+
+    if (!savevm_state.skip_configuration || enforce_config_section()) {
+        if (qemu_get_byte(f) != QEMU_VM_CONFIGURATION) {
+            error_report("Configuration section missing");
+            return -EINVAL;
+        }
+        ret = vmstate_load_state(f, &vmstate_configuration, &savevm_state, 0);
+
+        if (ret) {
+            return ret;
+        }
+    }
+
+    ret = qemu_loadvm_state_main(f, mis);
+    qemu_event_set(&mis->main_thread_load_event);
+
+    trace_qemu_loadvm_state_post_main(ret);
+
+    if (mis->have_listen_thread) {
+        /* Listen thread still going, can't clean up yet */
+        return ret;
+    }
+
+    if (ret == 0) {
+        ret = qemu_file_get_error(f);
+    }
 
     /*
      * Try to read in the VMDESC section as well, so that dumping tools that
@@ -1202,10 +1881,10 @@ int qemu_loadvm_state(QEMUFile *f)
      * We also mustn't read data that isn't there; some transports (RDMA)
      * will stall waiting for that data when the source has already closed.
      */
-    if (should_send_vmdesc()) {
+    if (ret == 0 && should_send_vmdesc()) {
         uint8_t *buf;
         uint32_t size;
-        section_type = qemu_get_byte(f);
+        uint8_t  section_type = qemu_get_byte(f);
 
         if (section_type != QEMU_VM_VMDESCRIPTION) {
             error_report("Expected vmdescription section, but got %d",
@@ -1229,57 +1908,9 @@ int qemu_loadvm_state(QEMUFile *f)
 
     cpu_synchronize_all_post_init();
 
-    ret = 0;
-
-out:
-    if (ret == 0) {
-        /* We may not have a VMDESC section, so ignore relative errors */
-        ret = file_error_after_eof;
-    }
-
     return ret;
 }
 
-static BlockDriverState *find_vmstate_bs(void)
-{
-    BlockDriverState *bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            return bs;
-        }
-    }
-    return NULL;
-}
-
-/*
- * Deletes snapshots of a given name in all opened images.
- */
-static int del_existing_snapshots(Monitor *mon, const char *name)
-{
-    BlockDriverState *bs;
-    QEMUSnapshotInfo sn1, *snapshot = &sn1;
-    Error *err = NULL;
-
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs) &&
-            bdrv_snapshot_find(bs, snapshot, name) >= 0) {
-            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
-            if (err) {
-                monitor_printf(mon,
-                               "Error while deleting snapshot on device '%s':"
-                               " %s\n",
-                               bdrv_get_device_name(bs),
-                               error_get_pretty(err));
-                error_free(err);
-                return -1;
-            }
-        }
-    }
-
-    return 0;
-}
-
 void hmp_savevm(Monitor *mon, const QDict *qdict)
 {
     BlockDriverState *bs, *bs1;
@@ -1292,27 +1923,29 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
     struct tm tm;
     const char *name = qdict_get_try_str(qdict, "name");
     Error *local_err = NULL;
+    AioContext *aio_context;
 
-    /* Verify if there is a device that doesn't support snapshots and is writable */
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-
-        if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-            continue;
-        }
+    if (!bdrv_all_can_snapshot(&bs)) {
+        monitor_printf(mon, "Device '%s' is writable but does not "
+                       "support snapshots.\n", bdrv_get_device_name(bs));
+        return;
+    }
 
-        if (!bdrv_can_snapshot(bs)) {
-            monitor_printf(mon, "Device '%s' is writable but does not support snapshots.\n",
-                               bdrv_get_device_name(bs));
-            return;
-        }
+    /* Delete old snapshots of the same name */
+    if (name && bdrv_all_delete_snapshot(name, &bs1, &local_err) < 0) {
+        monitor_printf(mon,
+                       "Error while deleting snapshot on device '%s': %s\n",
+                       bdrv_get_device_name(bs1), error_get_pretty(local_err));
+        error_free(local_err);
+        return;
     }
 
-    bs = find_vmstate_bs();
-    if (!bs) {
+    bs = bdrv_all_find_vmstate_bs();
+    if (bs == NULL) {
         monitor_printf(mon, "No block device can accept snapshots\n");
         return;
     }
+    aio_context = bdrv_get_aio_context(bs);
 
     saved_vm_running = runstate_is_running();
 
@@ -1323,6 +1956,8 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
     }
     vm_stop(RUN_STATE_SAVE_VM);
 
+    aio_context_acquire(aio_context);
+
     memset(sn, 0, sizeof(*sn));
 
     /* fill auxiliary fields */
@@ -1345,11 +1980,6 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
         strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", &tm);
     }
 
-    /* Delete old snapshots of the same name */
-    if (name && del_existing_snapshots(mon, name) < 0) {
-        goto the_end;
-    }
-
     /* save the VM state */
     f = qemu_fopen_bdrv(bs, 1);
     if (!f) {
@@ -1365,22 +1995,14 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
         goto the_end;
     }
 
-    /* create the snapshots */
-
-    bs1 = NULL;
-    while ((bs1 = bdrv_next(bs1))) {
-        if (bdrv_can_snapshot(bs1)) {
-            /* Write VM state size only to the image that contains the state */
-            sn->vm_state_size = (bs == bs1 ? vm_state_size : 0);
-            ret = bdrv_snapshot_create(bs1, sn);
-            if (ret < 0) {
-                monitor_printf(mon, "Error while creating snapshot on '%s'\n",
-                               bdrv_get_device_name(bs1));
-            }
-        }
+    ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, &bs);
+    if (ret < 0) {
+        monitor_printf(mon, "Error while creating snapshot on '%s'\n",
+                       bdrv_get_device_name(bs));
     }
 
  the_end:
+    aio_context_release(aio_context);
     if (saved_vm_running) {
         vm_start();
     }
@@ -1419,15 +2041,31 @@ int load_vmstate(const char *name)
     QEMUSnapshotInfo sn;
     QEMUFile *f;
     int ret;
+    AioContext *aio_context;
 
-    bs_vm_state = find_vmstate_bs();
+    if (!bdrv_all_can_snapshot(&bs)) {
+        error_report("Device '%s' is writable but does not support snapshots.",
+                     bdrv_get_device_name(bs));
+        return -ENOTSUP;
+    }
+    ret = bdrv_all_find_snapshot(name, &bs);
+    if (ret < 0) {
+        error_report("Device '%s' does not have the requested snapshot '%s'",
+                     bdrv_get_device_name(bs), name);
+        return ret;
+    }
+
+    bs_vm_state = bdrv_all_find_vmstate_bs();
     if (!bs_vm_state) {
         error_report("No block device supports snapshots");
         return -ENOTSUP;
     }
+    aio_context = bdrv_get_aio_context(bs_vm_state);
 
     /* Don't even try to load empty VM states */
+    aio_context_acquire(aio_context);
     ret = bdrv_snapshot_find(bs_vm_state, &sn, name);
+    aio_context_release(aio_context);
     if (ret < 0) {
         return ret;
     } else if (sn.vm_state_size == 0) {
@@ -1436,42 +2074,14 @@ int load_vmstate(const char *name)
         return -EINVAL;
     }
 
-    /* Verify if there is any device that doesn't support snapshots and is
-    writable and check if the requested snapshot is available too. */
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-
-        if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-            continue;
-        }
-
-        if (!bdrv_can_snapshot(bs)) {
-            error_report("Device '%s' is writable but does not support snapshots.",
-                               bdrv_get_device_name(bs));
-            return -ENOTSUP;
-        }
-
-        ret = bdrv_snapshot_find(bs, &sn, name);
-        if (ret < 0) {
-            error_report("Device '%s' does not have the requested snapshot '%s'",
-                           bdrv_get_device_name(bs), name);
-            return ret;
-        }
-    }
-
     /* Flush all IO requests so they don't interfere with the new state.  */
     bdrv_drain_all();
 
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            ret = bdrv_snapshot_goto(bs, name);
-            if (ret < 0) {
-                error_report("Error %d while activating snapshot '%s' on '%s'",
-                             ret, name, bdrv_get_device_name(bs));
-                return ret;
-            }
-        }
+    ret = bdrv_all_goto_snapshot(name, &bs);
+    if (ret < 0) {
+        error_report("Error %d while activating snapshot '%s' on '%s'",
+                     ret, name, bdrv_get_device_name(bs));
+        return ret;
     }
 
     /* restore the VM state */
@@ -1483,9 +2093,12 @@ int load_vmstate(const char *name)
 
     qemu_system_reset(VMRESET_SILENT);
     migration_incoming_state_new(f);
-    ret = qemu_loadvm_state(f);
 
+    aio_context_acquire(aio_context);
+    ret = qemu_loadvm_state(f);
     qemu_fclose(f);
+    aio_context_release(aio_context);
+
     migration_incoming_state_destroy();
     if (ret < 0) {
         error_report("Error %d while loading VM state", ret);
@@ -1501,43 +2114,34 @@ void hmp_delvm(Monitor *mon, const QDict *qdict)
     Error *err;
     const char *name = qdict_get_str(qdict, "name");
 
-    if (!find_vmstate_bs()) {
-        monitor_printf(mon, "No block device supports snapshots\n");
-        return;
-    }
-
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            err = NULL;
-            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
-            if (err) {
-                monitor_printf(mon,
-                               "Error while deleting snapshot on device '%s':"
-                               " %s\n",
-                               bdrv_get_device_name(bs),
-                               error_get_pretty(err));
-                error_free(err);
-            }
-        }
+    if (bdrv_all_delete_snapshot(name, &bs, &err) < 0) {
+        monitor_printf(mon,
+                       "Error while deleting snapshot on device '%s': %s\n",
+                       bdrv_get_device_name(bs), error_get_pretty(err));
+        error_free(err);
     }
 }
 
 void hmp_info_snapshots(Monitor *mon, const QDict *qdict)
 {
     BlockDriverState *bs, *bs1;
-    QEMUSnapshotInfo *sn_tab, *sn, s, *sn_info = &s;
-    int nb_sns, i, ret, available;
+    QEMUSnapshotInfo *sn_tab, *sn;
+    int nb_sns, i;
     int total;
     int *available_snapshots;
+    AioContext *aio_context;
 
-    bs = find_vmstate_bs();
+    bs = bdrv_all_find_vmstate_bs();
     if (!bs) {
         monitor_printf(mon, "No available block device supports snapshots\n");
         return;
     }
+    aio_context = bdrv_get_aio_context(bs);
 
+    aio_context_acquire(aio_context);
     nb_sns = bdrv_snapshot_list(bs, &sn_tab);
+    aio_context_release(aio_context);
+
     if (nb_sns < 0) {
         monitor_printf(mon, "bdrv_snapshot_list: error %d\n", nb_sns);
         return;
@@ -1548,24 +2152,10 @@ void hmp_info_snapshots(Monitor *mon, const QDict *qdict)
         return;
     }
 
-    available_snapshots = g_malloc0(sizeof(int) * nb_sns);
+    available_snapshots = g_new0(int, nb_sns);
     total = 0;
     for (i = 0; i < nb_sns; i++) {
-        sn = &sn_tab[i];
-        available = 1;
-        bs1 = NULL;
-
-        while ((bs1 = bdrv_next(bs1))) {
-            if (bdrv_can_snapshot(bs1) && bs1 != bs) {
-                ret = bdrv_snapshot_find(bs1, sn_info, sn->id_str);
-                if (ret < 0) {
-                    available = 0;
-                    break;
-                }
-            }
-        }
-
-        if (available) {
+        if (bdrv_all_find_snapshot(sn_tab[i].id_str, &bs1) == 0) {
             available_snapshots[total] = i;
             total++;
         }
diff --git a/monitor.c b/monitor.c
index aeea2b5f9..9a35d7265 100644
--- a/monitor.c
+++ b/monitor.c
@@ -63,17 +63,18 @@
 #include "cpu.h"
 #include "trace.h"
 #include "trace/control.h"
+#include "monitor/hmp-target.h"
 #ifdef CONFIG_TRACE_SIMPLE
 #include "trace/simple.h"
 #endif
 #include "exec/memory.h"
-#include "exec/cpu_ldst.h"
 #include "qmp-commands.h"
 #include "hmp.h"
 #include "qemu/thread.h"
 #include "block/qapi.h"
 #include "qapi/qmp-event.h"
 #include "qapi-event.h"
+#include "qmp-introspect.h"
 #include "sysemu/block-backend.h"
 
 /* for hmp_info_irq/pic */
@@ -82,6 +83,10 @@
 #endif
 #include "hw/lm32/lm32_pic.h"
 
+#if defined(TARGET_S390X)
+#include "hw/s390x/storage-keys.h"
+#endif
+
 /*
  * Supported types:
  *
@@ -176,13 +181,16 @@ typedef struct {
  * instance.
  */
 typedef struct MonitorQAPIEventState {
-    QAPIEvent event;    /* Event being tracked */
-    int64_t rate;       /* Minimum time (in ns) between two events */
-    int64_t last;       /* QEMU_CLOCK_REALTIME value at last emission */
+    QAPIEvent event;    /* Throttling state for this event type and... */
+    QDict *data;        /* ... data, see qapi_event_throttle_equal() */
     QEMUTimer *timer;   /* Timer for handling delayed events */
-    QObject *data;      /* Event pending delayed dispatch */
+    QDict *qdict;       /* Delayed event (if any) */
 } MonitorQAPIEventState;
 
+typedef struct {
+    int64_t rate;       /* Minimum time (in ns) between two events */
+} MonitorQAPIEventConf;
+
 struct Monitor {
     CharDriverState *chr;
     int reset_seen;
@@ -367,8 +375,7 @@ void monitor_printf(Monitor *mon, const char *fmt, ...)
     va_end(ap);
 }
 
-static int GCC_FMT_ATTR(2, 3) monitor_fprintf(FILE *stream,
-                                              const char *fmt, ...)
+int monitor_fprintf(FILE *stream, const char *fmt, ...)
 {
     va_list ap;
     va_start(ap, fmt);
@@ -434,132 +441,161 @@ static void monitor_protocol_emitter(Monitor *mon, QObject *data,
 }
 
 
-static MonitorQAPIEventState monitor_qapi_event_state[QAPI_EVENT_MAX];
+static MonitorQAPIEventConf monitor_qapi_event_conf[QAPI_EVENT_MAX] = {
+    /* Limit guest-triggerable events to 1 per second */
+    [QAPI_EVENT_RTC_CHANGE]        = { 1000 * SCALE_MS },
+    [QAPI_EVENT_WATCHDOG]          = { 1000 * SCALE_MS },
+    [QAPI_EVENT_BALLOON_CHANGE]    = { 1000 * SCALE_MS },
+    [QAPI_EVENT_QUORUM_REPORT_BAD] = { 1000 * SCALE_MS },
+    [QAPI_EVENT_QUORUM_FAILURE]    = { 1000 * SCALE_MS },
+    [QAPI_EVENT_VSERPORT_CHANGE]   = { 1000 * SCALE_MS },
+};
+
+GHashTable *monitor_qapi_event_state;
 
 /*
  * Emits the event to every monitor instance, @event is only used for trace
  * Called with monitor_lock held.
  */
-static void monitor_qapi_event_emit(QAPIEvent event, QObject *data)
+static void monitor_qapi_event_emit(QAPIEvent event, QDict *qdict)
 {
     Monitor *mon;
 
-    trace_monitor_protocol_event_emit(event, data);
+    trace_monitor_protocol_event_emit(event, qdict);
     QLIST_FOREACH(mon, &mon_list, entry) {
         if (monitor_is_qmp(mon) && mon->qmp.in_command_mode) {
-            monitor_json_emitter(mon, data);
+            monitor_json_emitter(mon, QOBJECT(qdict));
         }
     }
 }
 
+static void monitor_qapi_event_handler(void *opaque);
+
 /*
  * Queue a new event for emission to Monitor instances,
  * applying any rate limiting if required.
  */
 static void
-monitor_qapi_event_queue(QAPIEvent event, QDict *data, Error **errp)
+monitor_qapi_event_queue(QAPIEvent event, QDict *qdict, Error **errp)
 {
+    MonitorQAPIEventConf *evconf;
     MonitorQAPIEventState *evstate;
-    assert(event < QAPI_EVENT_MAX);
-    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
 
-    evstate = &(monitor_qapi_event_state[event]);
-    trace_monitor_protocol_event_queue(event,
-                                       data,
-                                       evstate->rate,
-                                       evstate->last,
-                                       now);
+    assert(event < QAPI_EVENT_MAX);
+    evconf = &monitor_qapi_event_conf[event];
+    trace_monitor_protocol_event_queue(event, qdict, evconf->rate);
 
-    /* Rate limit of 0 indicates no throttling */
     qemu_mutex_lock(&monitor_lock);
-    if (!evstate->rate) {
-        monitor_qapi_event_emit(event, QOBJECT(data));
-        evstate->last = now;
+
+    if (!evconf->rate) {
+        /* Unthrottled event */
+        monitor_qapi_event_emit(event, qdict);
     } else {
-        int64_t delta = now - evstate->last;
-        if (evstate->data ||
-            delta < evstate->rate) {
-            /* If there's an existing event pending, replace
-             * it with the new event, otherwise schedule a
-             * timer for delayed emission
+        QDict *data = qobject_to_qdict(qdict_get(qdict, "data"));
+        MonitorQAPIEventState key = { .event = event, .data = data };
+
+        evstate = g_hash_table_lookup(monitor_qapi_event_state, &key);
+        assert(!evstate || timer_pending(evstate->timer));
+
+        if (evstate) {
+            /*
+             * Timer is pending for (at least) evconf->rate ns after
+             * last send.  Store event for sending when timer fires,
+             * replacing a prior stored event if any.
              */
-            if (evstate->data) {
-                qobject_decref(evstate->data);
-            } else {
-                int64_t then = evstate->last + evstate->rate;
-                timer_mod_ns(evstate->timer, then);
-            }
-            evstate->data = QOBJECT(data);
-            qobject_incref(evstate->data);
+            QDECREF(evstate->qdict);
+            evstate->qdict = qdict;
+            QINCREF(evstate->qdict);
         } else {
-            monitor_qapi_event_emit(event, QOBJECT(data));
-            evstate->last = now;
+            /*
+             * Last send was (at least) evconf->rate ns ago.
+             * Send immediately, and arm the timer to call
+             * monitor_qapi_event_handler() in evconf->rate ns.  Any
+             * events arriving before then will be delayed until then.
+             */
+            int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+
+            monitor_qapi_event_emit(event, qdict);
+
+            evstate = g_new(MonitorQAPIEventState, 1);
+            evstate->event = event;
+            evstate->data = data;
+            QINCREF(evstate->data);
+            evstate->qdict = NULL;
+            evstate->timer = timer_new_ns(QEMU_CLOCK_REALTIME,
+                                          monitor_qapi_event_handler,
+                                          evstate);
+            g_hash_table_add(monitor_qapi_event_state, evstate);
+            timer_mod_ns(evstate->timer, now + evconf->rate);
         }
     }
+
     qemu_mutex_unlock(&monitor_lock);
 }
 
 /*
- * The callback invoked by QemuTimer when a delayed
- * event is ready to be emitted
+ * This function runs evconf->rate ns after sending a throttled
+ * event.
+ * If another event has since been stored, send it.
  */
 static void monitor_qapi_event_handler(void *opaque)
 {
     MonitorQAPIEventState *evstate = opaque;
-    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    MonitorQAPIEventConf *evconf = &monitor_qapi_event_conf[evstate->event];
 
-    trace_monitor_protocol_event_handler(evstate->event,
-                                         evstate->data,
-                                         evstate->last,
-                                         now);
+    trace_monitor_protocol_event_handler(evstate->event, evstate->qdict);
     qemu_mutex_lock(&monitor_lock);
-    if (evstate->data) {
-        monitor_qapi_event_emit(evstate->event, evstate->data);
-        qobject_decref(evstate->data);
-        evstate->data = NULL;
+
+    if (evstate->qdict) {
+        int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+
+        monitor_qapi_event_emit(evstate->event, evstate->qdict);
+        QDECREF(evstate->qdict);
+        evstate->qdict = NULL;
+        timer_mod_ns(evstate->timer, now + evconf->rate);
+    } else {
+        g_hash_table_remove(monitor_qapi_event_state, evstate);
+        QDECREF(evstate->data);
+        timer_free(evstate->timer);
+        g_free(evstate);
     }
-    evstate->last = now;
+
     qemu_mutex_unlock(&monitor_lock);
 }
 
-/*
- * @event: the event ID to be limited
- * @rate: the rate limit in milliseconds
- *
- * Sets a rate limit on a particular event, so no
- * more than 1 event will be emitted within @rate
- * milliseconds
- */
-static void
-monitor_qapi_event_throttle(QAPIEvent event, int64_t rate)
+static unsigned int qapi_event_throttle_hash(const void *key)
 {
-    MonitorQAPIEventState *evstate;
-    assert(event < QAPI_EVENT_MAX);
+    const MonitorQAPIEventState *evstate = key;
+    unsigned int hash = evstate->event * 255;
 
-    evstate = &(monitor_qapi_event_state[event]);
+    if (evstate->event == QAPI_EVENT_VSERPORT_CHANGE) {
+        hash += g_str_hash(qdict_get_str(evstate->data, "id"));
+    }
 
-    trace_monitor_protocol_event_throttle(event, rate);
-    evstate->event = event;
-    assert(rate * SCALE_MS <= INT64_MAX);
-    evstate->rate = rate * SCALE_MS;
-    evstate->last = 0;
-    evstate->data = NULL;
-    evstate->timer = timer_new(QEMU_CLOCK_REALTIME,
-                               SCALE_MS,
-                               monitor_qapi_event_handler,
-                               evstate);
+    return hash;
 }
 
-static void monitor_qapi_event_init(void)
+static gboolean qapi_event_throttle_equal(const void *a, const void *b)
 {
-    /* Limit guest-triggerable events to 1 per second */
-    monitor_qapi_event_throttle(QAPI_EVENT_RTC_CHANGE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_WATCHDOG, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_BALLOON_CHANGE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_QUORUM_REPORT_BAD, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_QUORUM_FAILURE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_VSERPORT_CHANGE, 1000);
+    const MonitorQAPIEventState *eva = a;
+    const MonitorQAPIEventState *evb = b;
+
+    if (eva->event != evb->event) {
+        return FALSE;
+    }
+
+    if (eva->event == QAPI_EVENT_VSERPORT_CHANGE) {
+        return !strcmp(qdict_get_str(eva->data, "id"),
+                       qdict_get_str(evb->data, "id"));
+    }
+
+    return TRUE;
+}
 
+static void monitor_qapi_event_init(void)
+{
+    monitor_qapi_event_state = g_hash_table_new(qapi_event_throttle_hash,
+                                                qapi_event_throttle_equal);
     qmp_event_set_func_emit(monitor_qapi_event_queue);
 }
 
@@ -678,7 +714,7 @@ static int get_str(char *buf, int buf_size, const char **pp)
                 case '\"':
                     break;
                 default:
-                    qemu_printf("unsupported escape code: '\\%c'\n", c);
+                    printf("unsupported escape code: '\\%c'\n", c);
                     goto fail;
                 }
                 if ((q - buf) < buf_size - 1) {
@@ -692,7 +728,7 @@ static int get_str(char *buf, int buf_size, const char **pp)
             }
         }
         if (*p != '\"') {
-            qemu_printf("unterminated string\n");
+            printf("unterminated string\n");
             goto fail;
         }
         p++;
@@ -924,6 +960,21 @@ EventInfoList *qmp_query_events(Error **errp)
     return ev_list;
 }
 
+/*
+ * Minor hack: generated marshalling suppressed for this command
+ * ('gen': false in the schema) so we can parse the JSON string
+ * directly into QObject instead of first parsing it with
+ * visit_type_SchemaInfoList() into a SchemaInfoList, then marshal it
+ * to QObject with generated output marshallers, every time.  Instead,
+ * we do it in test-qmp-input-visitor.c, just to make sure
+ * qapi-introspect.py's output actually conforms to the schema.
+ */
+static void qmp_query_qmp_schema(QDict *qdict, QObject **ret_data,
+                                 Error **errp)
+{
+    *ret_data = qobject_from_json(qmp_schema_json);
+}
+
 /* set the current CPU defined by the user */
 int monitor_set_cpu(int cpu_index)
 {
@@ -937,7 +988,7 @@ int monitor_set_cpu(int cpu_index)
     return 0;
 }
 
-static CPUState *mon_get_cpu(void)
+CPUState *mon_get_cpu(void)
 {
     if (!cur_mon->mon_cpu) {
         monitor_set_cpu(0);
@@ -946,7 +997,7 @@ static CPUState *mon_get_cpu(void)
     return cur_mon->mon_cpu;
 }
 
-static CPUArchState *mon_get_cpu_env(void)
+CPUArchState *mon_get_cpu_env(void)
 {
     return mon_get_cpu()->env_ptr;
 }
@@ -1420,442 +1471,6 @@ static void hmp_boot_set(Monitor *mon, const QDict *qdict)
     }
 }
 
-#if defined(TARGET_I386)
-static void print_pte(Monitor *mon, hwaddr addr,
-                      hwaddr pte,
-                      hwaddr mask)
-{
-#ifdef TARGET_X86_64
-    if (addr & (1ULL << 47)) {
-        addr |= -1LL << 48;
-    }
-#endif
-    monitor_printf(mon, TARGET_FMT_plx ": " TARGET_FMT_plx
-                   " %c%c%c%c%c%c%c%c%c\n",
-                   addr,
-                   pte & mask,
-                   pte & PG_NX_MASK ? 'X' : '-',
-                   pte & PG_GLOBAL_MASK ? 'G' : '-',
-                   pte & PG_PSE_MASK ? 'P' : '-',
-                   pte & PG_DIRTY_MASK ? 'D' : '-',
-                   pte & PG_ACCESSED_MASK ? 'A' : '-',
-                   pte & PG_PCD_MASK ? 'C' : '-',
-                   pte & PG_PWT_MASK ? 'T' : '-',
-                   pte & PG_USER_MASK ? 'U' : '-',
-                   pte & PG_RW_MASK ? 'W' : '-');
-}
-
-static void tlb_info_32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2;
-    uint32_t pgd, pde, pte;
-
-    pgd = env->cr[3] & ~0xfff;
-    for(l1 = 0; l1 < 1024; l1++) {
-        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
-        pde = le32_to_cpu(pde);
-        if (pde & PG_PRESENT_MASK) {
-            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-                /* 4M pages */
-                print_pte(mon, (l1 << 22), pde, ~((1 << 21) - 1));
-            } else {
-                for(l2 = 0; l2 < 1024; l2++) {
-                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
-                    pte = le32_to_cpu(pte);
-                    if (pte & PG_PRESENT_MASK) {
-                        print_pte(mon, (l1 << 22) + (l2 << 12),
-                                  pte & ~PG_PSE_MASK,
-                                  ~0xfff);
-                    }
-                }
-            }
-        }
-    }
-}
-
-static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2, l3;
-    uint64_t pdpe, pde, pte;
-    uint64_t pdp_addr, pd_addr, pt_addr;
-
-    pdp_addr = env->cr[3] & ~0x1f;
-    for (l1 = 0; l1 < 4; l1++) {
-        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
-        pdpe = le64_to_cpu(pdpe);
-        if (pdpe & PG_PRESENT_MASK) {
-            pd_addr = pdpe & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
-                pde = le64_to_cpu(pde);
-                if (pde & PG_PRESENT_MASK) {
-                    if (pde & PG_PSE_MASK) {
-                        /* 2M pages with PAE, CR4.PSE is ignored */
-                        print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
-                                  ~((hwaddr)(1 << 20) - 1));
-                    } else {
-                        pt_addr = pde & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
-                            pte = le64_to_cpu(pte);
-                            if (pte & PG_PRESENT_MASK) {
-                                print_pte(mon, (l1 << 30 ) + (l2 << 21)
-                                          + (l3 << 12),
-                                          pte & ~PG_PSE_MASK,
-                                          ~(hwaddr)0xfff);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-#ifdef TARGET_X86_64
-static void tlb_info_64(Monitor *mon, CPUArchState *env)
-{
-    uint64_t l1, l2, l3, l4;
-    uint64_t pml4e, pdpe, pde, pte;
-    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr;
-
-    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
-    for (l1 = 0; l1 < 512; l1++) {
-        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
-        pml4e = le64_to_cpu(pml4e);
-        if (pml4e & PG_PRESENT_MASK) {
-            pdp_addr = pml4e & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
-                pdpe = le64_to_cpu(pdpe);
-                if (pdpe & PG_PRESENT_MASK) {
-                    if (pdpe & PG_PSE_MASK) {
-                        /* 1G pages, CR4.PSE is ignored */
-                        print_pte(mon, (l1 << 39) + (l2 << 30), pdpe,
-                                  0x3ffffc0000000ULL);
-                    } else {
-                        pd_addr = pdpe & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
-                            pde = le64_to_cpu(pde);
-                            if (pde & PG_PRESENT_MASK) {
-                                if (pde & PG_PSE_MASK) {
-                                    /* 2M pages, CR4.PSE is ignored */
-                                    print_pte(mon, (l1 << 39) + (l2 << 30) +
-                                              (l3 << 21), pde,
-                                              0x3ffffffe00000ULL);
-                                } else {
-                                    pt_addr = pde & 0x3fffffffff000ULL;
-                                    for (l4 = 0; l4 < 512; l4++) {
-                                        cpu_physical_memory_read(pt_addr
-                                                                 + l4 * 8,
-                                                                 &pte, 8);
-                                        pte = le64_to_cpu(pte);
-                                        if (pte & PG_PRESENT_MASK) {
-                                            print_pte(mon, (l1 << 39) +
-                                                      (l2 << 30) +
-                                                      (l3 << 21) + (l4 << 12),
-                                                      pte & ~PG_PSE_MASK,
-                                                      0x3fffffffff000ULL);
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-#endif
-
-static void hmp_info_tlb(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env;
-
-    env = mon_get_cpu_env();
-
-    if (!(env->cr[0] & CR0_PG_MASK)) {
-        monitor_printf(mon, "PG disabled\n");
-        return;
-    }
-    if (env->cr[4] & CR4_PAE_MASK) {
-#ifdef TARGET_X86_64
-        if (env->hflags & HF_LMA_MASK) {
-            tlb_info_64(mon, env);
-        } else
-#endif
-        {
-            tlb_info_pae32(mon, env);
-        }
-    } else {
-        tlb_info_32(mon, env);
-    }
-}
-
-static void mem_print(Monitor *mon, hwaddr *pstart,
-                      int *plast_prot,
-                      hwaddr end, int prot)
-{
-    int prot1;
-    prot1 = *plast_prot;
-    if (prot != prot1) {
-        if (*pstart != -1) {
-            monitor_printf(mon, TARGET_FMT_plx "-" TARGET_FMT_plx " "
-                           TARGET_FMT_plx " %c%c%c\n",
-                           *pstart, end, end - *pstart,
-                           prot1 & PG_USER_MASK ? 'u' : '-',
-                           'r',
-                           prot1 & PG_RW_MASK ? 'w' : '-');
-        }
-        if (prot != 0)
-            *pstart = end;
-        else
-            *pstart = -1;
-        *plast_prot = prot;
-    }
-}
-
-static void mem_info_32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2;
-    int prot, last_prot;
-    uint32_t pgd, pde, pte;
-    hwaddr start, end;
-
-    pgd = env->cr[3] & ~0xfff;
-    last_prot = 0;
-    start = -1;
-    for(l1 = 0; l1 < 1024; l1++) {
-        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
-        pde = le32_to_cpu(pde);
-        end = l1 << 22;
-        if (pde & PG_PRESENT_MASK) {
-            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-                prot = pde & (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
-                mem_print(mon, &start, &last_prot, end, prot);
-            } else {
-                for(l2 = 0; l2 < 1024; l2++) {
-                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
-                    pte = le32_to_cpu(pte);
-                    end = (l1 << 22) + (l2 << 12);
-                    if (pte & PG_PRESENT_MASK) {
-                        prot = pte & pde &
-                            (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
-                    } else {
-                        prot = 0;
-                    }
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
-}
-
-static void mem_info_pae32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2, l3;
-    int prot, last_prot;
-    uint64_t pdpe, pde, pte;
-    uint64_t pdp_addr, pd_addr, pt_addr;
-    hwaddr start, end;
-
-    pdp_addr = env->cr[3] & ~0x1f;
-    last_prot = 0;
-    start = -1;
-    for (l1 = 0; l1 < 4; l1++) {
-        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
-        pdpe = le64_to_cpu(pdpe);
-        end = l1 << 30;
-        if (pdpe & PG_PRESENT_MASK) {
-            pd_addr = pdpe & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
-                pde = le64_to_cpu(pde);
-                end = (l1 << 30) + (l2 << 21);
-                if (pde & PG_PRESENT_MASK) {
-                    if (pde & PG_PSE_MASK) {
-                        prot = pde & (PG_USER_MASK | PG_RW_MASK |
-                                      PG_PRESENT_MASK);
-                        mem_print(mon, &start, &last_prot, end, prot);
-                    } else {
-                        pt_addr = pde & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
-                            pte = le64_to_cpu(pte);
-                            end = (l1 << 30) + (l2 << 21) + (l3 << 12);
-                            if (pte & PG_PRESENT_MASK) {
-                                prot = pte & pde & (PG_USER_MASK | PG_RW_MASK |
-                                                    PG_PRESENT_MASK);
-                            } else {
-                                prot = 0;
-                            }
-                            mem_print(mon, &start, &last_prot, end, prot);
-                        }
-                    }
-                } else {
-                    prot = 0;
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
-}
-
-
-#ifdef TARGET_X86_64
-static void mem_info_64(Monitor *mon, CPUArchState *env)
-{
-    int prot, last_prot;
-    uint64_t l1, l2, l3, l4;
-    uint64_t pml4e, pdpe, pde, pte;
-    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr, start, end;
-
-    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
-    last_prot = 0;
-    start = -1;
-    for (l1 = 0; l1 < 512; l1++) {
-        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
-        pml4e = le64_to_cpu(pml4e);
-        end = l1 << 39;
-        if (pml4e & PG_PRESENT_MASK) {
-            pdp_addr = pml4e & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
-                pdpe = le64_to_cpu(pdpe);
-                end = (l1 << 39) + (l2 << 30);
-                if (pdpe & PG_PRESENT_MASK) {
-                    if (pdpe & PG_PSE_MASK) {
-                        prot = pdpe & (PG_USER_MASK | PG_RW_MASK |
-                                       PG_PRESENT_MASK);
-                        prot &= pml4e;
-                        mem_print(mon, &start, &last_prot, end, prot);
-                    } else {
-                        pd_addr = pdpe & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
-                            pde = le64_to_cpu(pde);
-                            end = (l1 << 39) + (l2 << 30) + (l3 << 21);
-                            if (pde & PG_PRESENT_MASK) {
-                                if (pde & PG_PSE_MASK) {
-                                    prot = pde & (PG_USER_MASK | PG_RW_MASK |
-                                                  PG_PRESENT_MASK);
-                                    prot &= pml4e & pdpe;
-                                    mem_print(mon, &start, &last_prot, end, prot);
-                                } else {
-                                    pt_addr = pde & 0x3fffffffff000ULL;
-                                    for (l4 = 0; l4 < 512; l4++) {
-                                        cpu_physical_memory_read(pt_addr
-                                                                 + l4 * 8,
-                                                                 &pte, 8);
-                                        pte = le64_to_cpu(pte);
-                                        end = (l1 << 39) + (l2 << 30) +
-                                            (l3 << 21) + (l4 << 12);
-                                        if (pte & PG_PRESENT_MASK) {
-                                            prot = pte & (PG_USER_MASK | PG_RW_MASK |
-                                                          PG_PRESENT_MASK);
-                                            prot &= pml4e & pdpe & pde;
-                                        } else {
-                                            prot = 0;
-                                        }
-                                        mem_print(mon, &start, &last_prot, end, prot);
-                                    }
-                                }
-                            } else {
-                                prot = 0;
-                                mem_print(mon, &start, &last_prot, end, prot);
-                            }
-                        }
-                    }
-                } else {
-                    prot = 0;
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
-}
-#endif
-
-static void hmp_info_mem(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env;
-
-    env = mon_get_cpu_env();
-
-    if (!(env->cr[0] & CR0_PG_MASK)) {
-        monitor_printf(mon, "PG disabled\n");
-        return;
-    }
-    if (env->cr[4] & CR4_PAE_MASK) {
-#ifdef TARGET_X86_64
-        if (env->hflags & HF_LMA_MASK) {
-            mem_info_64(mon, env);
-        } else
-#endif
-        {
-            mem_info_pae32(mon, env);
-        }
-    } else {
-        mem_info_32(mon, env);
-    }
-}
-#endif
-
-#if defined(TARGET_SH4)
-
-static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
-{
-    monitor_printf(mon, " tlb%i:\t"
-                   "asid=%hhu vpn=%x\tppn=%x\tsz=%hhu size=%u\t"
-                   "v=%hhu shared=%hhu cached=%hhu prot=%hhu "
-                   "dirty=%hhu writethrough=%hhu\n",
-                   idx,
-                   tlb->asid, tlb->vpn, tlb->ppn, tlb->sz, tlb->size,
-                   tlb->v, tlb->sh, tlb->c, tlb->pr,
-                   tlb->d, tlb->wt);
-}
-
-static void hmp_info_tlb(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    int i;
-
-    monitor_printf (mon, "ITLB:\n");
-    for (i = 0 ; i < ITLB_SIZE ; i++)
-        print_tlb (mon, i, &env->itlb[i]);
-    monitor_printf (mon, "UTLB:\n");
-    for (i = 0 ; i < UTLB_SIZE ; i++)
-        print_tlb (mon, i, &env->utlb[i]);
-}
-
-#endif
-
-#if defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_XTENSA)
-static void hmp_info_tlb(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env1 = mon_get_cpu_env();
-
-    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
-}
-#endif
-
 static void hmp_info_mtree(Monitor *mon, const QDict *qdict)
 {
     mtree_info((fprintf_function)monitor_printf, mon);
@@ -2066,31 +1681,6 @@ static void hmp_acl_remove(Monitor *mon, const QDict *qdict)
     }
 }
 
-#if defined(TARGET_I386)
-static void hmp_mce(Monitor *mon, const QDict *qdict)
-{
-    X86CPU *cpu;
-    CPUState *cs;
-    int cpu_index = qdict_get_int(qdict, "cpu_index");
-    int bank = qdict_get_int(qdict, "bank");
-    uint64_t status = qdict_get_int(qdict, "status");
-    uint64_t mcg_status = qdict_get_int(qdict, "mcg_status");
-    uint64_t addr = qdict_get_int(qdict, "addr");
-    uint64_t misc = qdict_get_int(qdict, "misc");
-    int flags = MCE_INJECT_UNCOND_AO;
-
-    if (qdict_get_try_bool(qdict, "broadcast", false)) {
-        flags |= MCE_INJECT_BROADCAST;
-    }
-    cs = qemu_get_cpu(cpu_index);
-    if (cs != NULL) {
-        cpu = X86_CPU(cs);
-        cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
-                           flags);
-    }
-}
-#endif
-
 void qmp_getfd(const char *fdname, Error **errp)
 {
     mon_fd_t *monfd;
@@ -2509,377 +2099,8 @@ int monitor_fd_param(Monitor *mon, const char *fdname, Error **errp)
 
 /* Please update hmp-commands.hx when adding or changing commands */
 static mon_cmd_t info_cmds[] = {
-    {
-        .name       = "version",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the version of QEMU",
-        .mhandler.cmd = hmp_info_version,
-    },
-    {
-        .name       = "network",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the network state",
-        .mhandler.cmd = hmp_info_network,
-    },
-    {
-        .name       = "chardev",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the character devices",
-        .mhandler.cmd = hmp_info_chardev,
-    },
-    {
-        .name       = "block",
-        .args_type  = "nodes:-n,verbose:-v,device:B?",
-        .params     = "[-n] [-v] [device]",
-        .help       = "show info of one block device or all block devices "
-                      "(-n: show named nodes; -v: show details)",
-        .mhandler.cmd = hmp_info_block,
-    },
-    {
-        .name       = "blockstats",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show block device statistics",
-        .mhandler.cmd = hmp_info_blockstats,
-    },
-    {
-        .name       = "block-jobs",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show progress of ongoing block device operations",
-        .mhandler.cmd = hmp_info_block_jobs,
-    },
-    {
-        .name       = "registers",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the cpu registers",
-        .mhandler.cmd = hmp_info_registers,
-    },
-    {
-        .name       = "cpus",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show infos for each CPU",
-        .mhandler.cmd = hmp_info_cpus,
-    },
-    {
-        .name       = "history",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the command line history",
-        .mhandler.cmd = hmp_info_history,
-    },
-#if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
-    defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
-    {
-        .name       = "irq",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the interrupts statistics (if available)",
-#ifdef TARGET_SPARC
-        .mhandler.cmd = sun4m_hmp_info_irq,
-#elif defined(TARGET_LM32)
-        .mhandler.cmd = lm32_hmp_info_irq,
-#else
-        .mhandler.cmd = hmp_info_irq,
-#endif
-    },
-    {
-        .name       = "pic",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show i8259 (PIC) state",
-#ifdef TARGET_SPARC
-        .mhandler.cmd = sun4m_hmp_info_pic,
-#elif defined(TARGET_LM32)
-        .mhandler.cmd = lm32_hmp_info_pic,
-#else
-        .mhandler.cmd = hmp_info_pic,
-#endif
-    },
-#endif
-    {
-        .name       = "pci",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show PCI info",
-        .mhandler.cmd = hmp_info_pci,
-    },
-#if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
-    defined(TARGET_PPC) || defined(TARGET_XTENSA)
-    {
-        .name       = "tlb",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show virtual to physical memory mappings",
-        .mhandler.cmd = hmp_info_tlb,
-    },
-#endif
-#if defined(TARGET_I386)
-    {
-        .name       = "mem",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the active virtual memory mappings",
-        .mhandler.cmd = hmp_info_mem,
-    },
-#endif
-    {
-        .name       = "mtree",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory tree",
-        .mhandler.cmd = hmp_info_mtree,
-    },
-    {
-        .name       = "jit",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show dynamic compiler info",
-        .mhandler.cmd = hmp_info_jit,
-    },
-    {
-        .name       = "opcount",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show dynamic compiler opcode counters",
-        .mhandler.cmd = hmp_info_opcount,
-    },
-    {
-        .name       = "kvm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show KVM information",
-        .mhandler.cmd = hmp_info_kvm,
-    },
-    {
-        .name       = "numa",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show NUMA information",
-        .mhandler.cmd = hmp_info_numa,
-    },
-    {
-        .name       = "usb",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show guest USB devices",
-        .mhandler.cmd = hmp_info_usb,
-    },
-    {
-        .name       = "usbhost",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show host USB devices",
-        .mhandler.cmd = hmp_info_usbhost,
-    },
-    {
-        .name       = "profile",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show profiling information",
-        .mhandler.cmd = hmp_info_profile,
-    },
-    {
-        .name       = "capture",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show capture information",
-        .mhandler.cmd = hmp_info_capture,
-    },
-    {
-        .name       = "snapshots",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the currently saved VM snapshots",
-        .mhandler.cmd = hmp_info_snapshots,
-    },
-    {
-        .name       = "status",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM status (running|paused)",
-        .mhandler.cmd = hmp_info_status,
-    },
-    {
-        .name       = "mice",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show which guest mouse is receiving events",
-        .mhandler.cmd = hmp_info_mice,
-    },
-    {
-        .name       = "vnc",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the vnc server status",
-        .mhandler.cmd = hmp_info_vnc,
-    },
-#if defined(CONFIG_SPICE)
-    {
-        .name       = "spice",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the spice server status",
-        .mhandler.cmd = hmp_info_spice,
-    },
-#endif
-    {
-        .name       = "name",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM name",
-        .mhandler.cmd = hmp_info_name,
-    },
-    {
-        .name       = "uuid",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM UUID",
-        .mhandler.cmd = hmp_info_uuid,
-    },
-    {
-        .name       = "cpustats",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show CPU statistics",
-        .mhandler.cmd = hmp_info_cpustats,
-    },
-#if defined(CONFIG_SLIRP)
-    {
-        .name       = "usernet",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show user network stack connection states",
-        .mhandler.cmd = hmp_info_usernet,
-    },
-#endif
-    {
-        .name       = "migrate",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show migration status",
-        .mhandler.cmd = hmp_info_migrate,
-    },
-    {
-        .name       = "migrate_capabilities",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration capabilities",
-        .mhandler.cmd = hmp_info_migrate_capabilities,
-    },
-    {
-        .name       = "migrate_parameters",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration parameters",
-        .mhandler.cmd = hmp_info_migrate_parameters,
-    },
-    {
-        .name       = "migrate_cache_size",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration xbzrle cache size",
-        .mhandler.cmd = hmp_info_migrate_cache_size,
-    },
-    {
-        .name       = "balloon",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show balloon information",
-        .mhandler.cmd = hmp_info_balloon,
-    },
-    {
-        .name       = "qtree",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show device tree",
-        .mhandler.cmd = hmp_info_qtree,
-    },
-    {
-        .name       = "qdm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show qdev device model list",
-        .mhandler.cmd = hmp_info_qdm,
-    },
-    {
-        .name       = "qom-tree",
-        .args_type  = "path:s?",
-        .params     = "[path]",
-        .help       = "show QOM composition tree",
-        .mhandler.cmd = hmp_info_qom_tree,
-    },
-    {
-        .name       = "roms",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show roms",
-        .mhandler.cmd = hmp_info_roms,
-    },
-    {
-        .name       = "trace-events",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show available trace-events & their state",
-        .mhandler.cmd = hmp_info_trace_events,
-    },
-    {
-        .name       = "tpm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the TPM device",
-        .mhandler.cmd = hmp_info_tpm,
-    },
-    {
-        .name       = "memdev",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory backends",
-        .mhandler.cmd = hmp_info_memdev,
-    },
-    {
-        .name       = "memory-devices",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory devices",
-        .mhandler.cmd = hmp_info_memory_devices,
-    },
-    {
-        .name       = "rocker",
-        .args_type  = "name:s",
-        .params     = "name",
-        .help       = "Show rocker switch",
-        .mhandler.cmd = hmp_rocker,
-    },
-    {
-        .name       = "rocker-ports",
-        .args_type  = "name:s",
-        .params     = "name",
-        .help       = "Show rocker ports",
-        .mhandler.cmd = hmp_rocker_ports,
-    },
-    {
-        .name       = "rocker-of-dpa-flows",
-        .args_type  = "name:s,tbl_id:i?",
-        .params     = "name [tbl_id]",
-        .help       = "Show rocker OF-DPA flow tables",
-        .mhandler.cmd = hmp_rocker_of_dpa_flows,
-    },
-    {
-        .name       = "rocker-of-dpa-groups",
-        .args_type  = "name:s,type:i?",
-        .params     = "name [type]",
-        .help       = "Show rocker OF-DPA groups",
-        .mhandler.cmd = hmp_rocker_of_dpa_groups,
-    },
-    {
-        .name       = NULL,
-    },
+#include "hmp-commands-info.h"
+    { NULL, NULL, },
 };
 
 /* mon_cmds and info_cmds would be sorted at runtime */
@@ -2898,394 +2119,6 @@ static const mon_cmd_t qmp_cmds[] = {
 static const char *pch;
 static sigjmp_buf expr_env;
 
-#define MD_TLONG 0
-#define MD_I32   1
-
-typedef struct MonitorDef {
-    const char *name;
-    int offset;
-    target_long (*get_value)(const struct MonitorDef *md, int val);
-    int type;
-} MonitorDef;
-
-#if defined(TARGET_I386)
-static target_long monitor_get_pc (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->eip + env->segs[R_CS].base;
-}
-#endif
-
-#if defined(TARGET_PPC)
-static target_long monitor_get_ccr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    unsigned int u;
-    int i;
-
-    u = 0;
-    for (i = 0; i < 8; i++)
-        u |= env->crf[i] << (32 - (4 * (i + 1)));
-
-    return u;
-}
-
-static target_long monitor_get_msr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->msr;
-}
-
-static target_long monitor_get_xer (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->xer;
-}
-
-static target_long monitor_get_decr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return cpu_ppc_load_decr(env);
-}
-
-static target_long monitor_get_tbu (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return cpu_ppc_load_tbu(env);
-}
-
-static target_long monitor_get_tbl (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return cpu_ppc_load_tbl(env);
-}
-#endif
-
-#if defined(TARGET_SPARC)
-#ifndef TARGET_SPARC64
-static target_long monitor_get_psr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-
-    return cpu_get_psr(env);
-}
-#endif
-
-static target_long monitor_get_reg(const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->regwptr[val];
-}
-#endif
-
-static const MonitorDef monitor_defs[] = {
-#ifdef TARGET_I386
-
-#define SEG(name, seg) \
-    { name, offsetof(CPUX86State, segs[seg].selector), NULL, MD_I32 },\
-    { name ".base", offsetof(CPUX86State, segs[seg].base) },\
-    { name ".limit", offsetof(CPUX86State, segs[seg].limit), NULL, MD_I32 },
-
-    { "eax", offsetof(CPUX86State, regs[0]) },
-    { "ecx", offsetof(CPUX86State, regs[1]) },
-    { "edx", offsetof(CPUX86State, regs[2]) },
-    { "ebx", offsetof(CPUX86State, regs[3]) },
-    { "esp|sp", offsetof(CPUX86State, regs[4]) },
-    { "ebp|fp", offsetof(CPUX86State, regs[5]) },
-    { "esi", offsetof(CPUX86State, regs[6]) },
-    { "edi", offsetof(CPUX86State, regs[7]) },
-#ifdef TARGET_X86_64
-    { "r8", offsetof(CPUX86State, regs[8]) },
-    { "r9", offsetof(CPUX86State, regs[9]) },
-    { "r10", offsetof(CPUX86State, regs[10]) },
-    { "r11", offsetof(CPUX86State, regs[11]) },
-    { "r12", offsetof(CPUX86State, regs[12]) },
-    { "r13", offsetof(CPUX86State, regs[13]) },
-    { "r14", offsetof(CPUX86State, regs[14]) },
-    { "r15", offsetof(CPUX86State, regs[15]) },
-#endif
-    { "eflags", offsetof(CPUX86State, eflags) },
-    { "eip", offsetof(CPUX86State, eip) },
-    SEG("cs", R_CS)
-    SEG("ds", R_DS)
-    SEG("es", R_ES)
-    SEG("ss", R_SS)
-    SEG("fs", R_FS)
-    SEG("gs", R_GS)
-    { "pc", 0, monitor_get_pc, },
-#elif defined(TARGET_PPC)
-    /* General purpose registers */
-    { "r0", offsetof(CPUPPCState, gpr[0]) },
-    { "r1", offsetof(CPUPPCState, gpr[1]) },
-    { "r2", offsetof(CPUPPCState, gpr[2]) },
-    { "r3", offsetof(CPUPPCState, gpr[3]) },
-    { "r4", offsetof(CPUPPCState, gpr[4]) },
-    { "r5", offsetof(CPUPPCState, gpr[5]) },
-    { "r6", offsetof(CPUPPCState, gpr[6]) },
-    { "r7", offsetof(CPUPPCState, gpr[7]) },
-    { "r8", offsetof(CPUPPCState, gpr[8]) },
-    { "r9", offsetof(CPUPPCState, gpr[9]) },
-    { "r10", offsetof(CPUPPCState, gpr[10]) },
-    { "r11", offsetof(CPUPPCState, gpr[11]) },
-    { "r12", offsetof(CPUPPCState, gpr[12]) },
-    { "r13", offsetof(CPUPPCState, gpr[13]) },
-    { "r14", offsetof(CPUPPCState, gpr[14]) },
-    { "r15", offsetof(CPUPPCState, gpr[15]) },
-    { "r16", offsetof(CPUPPCState, gpr[16]) },
-    { "r17", offsetof(CPUPPCState, gpr[17]) },
-    { "r18", offsetof(CPUPPCState, gpr[18]) },
-    { "r19", offsetof(CPUPPCState, gpr[19]) },
-    { "r20", offsetof(CPUPPCState, gpr[20]) },
-    { "r21", offsetof(CPUPPCState, gpr[21]) },
-    { "r22", offsetof(CPUPPCState, gpr[22]) },
-    { "r23", offsetof(CPUPPCState, gpr[23]) },
-    { "r24", offsetof(CPUPPCState, gpr[24]) },
-    { "r25", offsetof(CPUPPCState, gpr[25]) },
-    { "r26", offsetof(CPUPPCState, gpr[26]) },
-    { "r27", offsetof(CPUPPCState, gpr[27]) },
-    { "r28", offsetof(CPUPPCState, gpr[28]) },
-    { "r29", offsetof(CPUPPCState, gpr[29]) },
-    { "r30", offsetof(CPUPPCState, gpr[30]) },
-    { "r31", offsetof(CPUPPCState, gpr[31]) },
-    /* Floating point registers */
-    { "f0", offsetof(CPUPPCState, fpr[0]) },
-    { "f1", offsetof(CPUPPCState, fpr[1]) },
-    { "f2", offsetof(CPUPPCState, fpr[2]) },
-    { "f3", offsetof(CPUPPCState, fpr[3]) },
-    { "f4", offsetof(CPUPPCState, fpr[4]) },
-    { "f5", offsetof(CPUPPCState, fpr[5]) },
-    { "f6", offsetof(CPUPPCState, fpr[6]) },
-    { "f7", offsetof(CPUPPCState, fpr[7]) },
-    { "f8", offsetof(CPUPPCState, fpr[8]) },
-    { "f9", offsetof(CPUPPCState, fpr[9]) },
-    { "f10", offsetof(CPUPPCState, fpr[10]) },
-    { "f11", offsetof(CPUPPCState, fpr[11]) },
-    { "f12", offsetof(CPUPPCState, fpr[12]) },
-    { "f13", offsetof(CPUPPCState, fpr[13]) },
-    { "f14", offsetof(CPUPPCState, fpr[14]) },
-    { "f15", offsetof(CPUPPCState, fpr[15]) },
-    { "f16", offsetof(CPUPPCState, fpr[16]) },
-    { "f17", offsetof(CPUPPCState, fpr[17]) },
-    { "f18", offsetof(CPUPPCState, fpr[18]) },
-    { "f19", offsetof(CPUPPCState, fpr[19]) },
-    { "f20", offsetof(CPUPPCState, fpr[20]) },
-    { "f21", offsetof(CPUPPCState, fpr[21]) },
-    { "f22", offsetof(CPUPPCState, fpr[22]) },
-    { "f23", offsetof(CPUPPCState, fpr[23]) },
-    { "f24", offsetof(CPUPPCState, fpr[24]) },
-    { "f25", offsetof(CPUPPCState, fpr[25]) },
-    { "f26", offsetof(CPUPPCState, fpr[26]) },
-    { "f27", offsetof(CPUPPCState, fpr[27]) },
-    { "f28", offsetof(CPUPPCState, fpr[28]) },
-    { "f29", offsetof(CPUPPCState, fpr[29]) },
-    { "f30", offsetof(CPUPPCState, fpr[30]) },
-    { "f31", offsetof(CPUPPCState, fpr[31]) },
-    { "fpscr", offsetof(CPUPPCState, fpscr) },
-    /* Next instruction pointer */
-    { "nip|pc", offsetof(CPUPPCState, nip) },
-    { "lr", offsetof(CPUPPCState, lr) },
-    { "ctr", offsetof(CPUPPCState, ctr) },
-    { "decr", 0, &monitor_get_decr, },
-    { "ccr", 0, &monitor_get_ccr, },
-    /* Machine state register */
-    { "msr", 0, &monitor_get_msr, },
-    { "xer", 0, &monitor_get_xer, },
-    { "tbu", 0, &monitor_get_tbu, },
-    { "tbl", 0, &monitor_get_tbl, },
-    /* Segment registers */
-    { "sdr1", offsetof(CPUPPCState, spr[SPR_SDR1]) },
-    { "sr0", offsetof(CPUPPCState, sr[0]) },
-    { "sr1", offsetof(CPUPPCState, sr[1]) },
-    { "sr2", offsetof(CPUPPCState, sr[2]) },
-    { "sr3", offsetof(CPUPPCState, sr[3]) },
-    { "sr4", offsetof(CPUPPCState, sr[4]) },
-    { "sr5", offsetof(CPUPPCState, sr[5]) },
-    { "sr6", offsetof(CPUPPCState, sr[6]) },
-    { "sr7", offsetof(CPUPPCState, sr[7]) },
-    { "sr8", offsetof(CPUPPCState, sr[8]) },
-    { "sr9", offsetof(CPUPPCState, sr[9]) },
-    { "sr10", offsetof(CPUPPCState, sr[10]) },
-    { "sr11", offsetof(CPUPPCState, sr[11]) },
-    { "sr12", offsetof(CPUPPCState, sr[12]) },
-    { "sr13", offsetof(CPUPPCState, sr[13]) },
-    { "sr14", offsetof(CPUPPCState, sr[14]) },
-    { "sr15", offsetof(CPUPPCState, sr[15]) },
-    /* Too lazy to put BATs... */
-    { "pvr", offsetof(CPUPPCState, spr[SPR_PVR]) },
-
-    { "srr0", offsetof(CPUPPCState, spr[SPR_SRR0]) },
-    { "srr1", offsetof(CPUPPCState, spr[SPR_SRR1]) },
-    { "dar", offsetof(CPUPPCState, spr[SPR_DAR]) },
-    { "dsisr", offsetof(CPUPPCState, spr[SPR_DSISR]) },
-    { "cfar", offsetof(CPUPPCState, spr[SPR_CFAR]) },
-    { "sprg0", offsetof(CPUPPCState, spr[SPR_SPRG0]) },
-    { "sprg1", offsetof(CPUPPCState, spr[SPR_SPRG1]) },
-    { "sprg2", offsetof(CPUPPCState, spr[SPR_SPRG2]) },
-    { "sprg3", offsetof(CPUPPCState, spr[SPR_SPRG3]) },
-    { "sprg4", offsetof(CPUPPCState, spr[SPR_SPRG4]) },
-    { "sprg5", offsetof(CPUPPCState, spr[SPR_SPRG5]) },
-    { "sprg6", offsetof(CPUPPCState, spr[SPR_SPRG6]) },
-    { "sprg7", offsetof(CPUPPCState, spr[SPR_SPRG7]) },
-    { "pid", offsetof(CPUPPCState, spr[SPR_BOOKE_PID]) },
-    { "csrr0", offsetof(CPUPPCState, spr[SPR_BOOKE_CSRR0]) },
-    { "csrr1", offsetof(CPUPPCState, spr[SPR_BOOKE_CSRR1]) },
-    { "esr", offsetof(CPUPPCState, spr[SPR_BOOKE_ESR]) },
-    { "dear", offsetof(CPUPPCState, spr[SPR_BOOKE_DEAR]) },
-    { "mcsr", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSR]) },
-    { "tsr", offsetof(CPUPPCState, spr[SPR_BOOKE_TSR]) },
-    { "tcr", offsetof(CPUPPCState, spr[SPR_BOOKE_TCR]) },
-    { "vrsave", offsetof(CPUPPCState, spr[SPR_VRSAVE]) },
-    { "pir", offsetof(CPUPPCState, spr[SPR_BOOKE_PIR]) },
-    { "mcsrr0", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSRR0]) },
-    { "mcsrr1", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSRR1]) },
-    { "decar", offsetof(CPUPPCState, spr[SPR_BOOKE_DECAR]) },
-    { "ivpr", offsetof(CPUPPCState, spr[SPR_BOOKE_IVPR]) },
-    { "epcr", offsetof(CPUPPCState, spr[SPR_BOOKE_EPCR]) },
-    { "sprg8", offsetof(CPUPPCState, spr[SPR_BOOKE_SPRG8]) },
-    { "ivor0", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR0]) },
-    { "ivor1", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR1]) },
-    { "ivor2", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR2]) },
-    { "ivor3", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR3]) },
-    { "ivor4", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR4]) },
-    { "ivor5", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR5]) },
-    { "ivor6", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR6]) },
-    { "ivor7", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR7]) },
-    { "ivor8", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR8]) },
-    { "ivor9", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR9]) },
-    { "ivor10", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR10]) },
-    { "ivor11", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR11]) },
-    { "ivor12", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR12]) },
-    { "ivor13", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR13]) },
-    { "ivor14", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR14]) },
-    { "ivor15", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR15]) },
-    { "ivor32", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR32]) },
-    { "ivor33", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR33]) },
-    { "ivor34", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR34]) },
-    { "ivor35", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR35]) },
-    { "ivor36", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR36]) },
-    { "ivor37", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR37]) },
-    { "mas0", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS0]) },
-    { "mas1", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS1]) },
-    { "mas2", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS2]) },
-    { "mas3", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS3]) },
-    { "mas4", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS4]) },
-    { "mas6", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS6]) },
-    { "mas7", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS7]) },
-    { "mmucfg", offsetof(CPUPPCState, spr[SPR_MMUCFG]) },
-    { "tlb0cfg", offsetof(CPUPPCState, spr[SPR_BOOKE_TLB0CFG]) },
-    { "tlb1cfg", offsetof(CPUPPCState, spr[SPR_BOOKE_TLB1CFG]) },
-    { "epr", offsetof(CPUPPCState, spr[SPR_BOOKE_EPR]) },
-    { "eplc", offsetof(CPUPPCState, spr[SPR_BOOKE_EPLC]) },
-    { "epsc", offsetof(CPUPPCState, spr[SPR_BOOKE_EPSC]) },
-    { "svr", offsetof(CPUPPCState, spr[SPR_E500_SVR]) },
-    { "mcar", offsetof(CPUPPCState, spr[SPR_Exxx_MCAR]) },
-    { "pid1", offsetof(CPUPPCState, spr[SPR_BOOKE_PID1]) },
-    { "pid2", offsetof(CPUPPCState, spr[SPR_BOOKE_PID2]) },
-    { "hid0", offsetof(CPUPPCState, spr[SPR_HID0]) },
-
-#elif defined(TARGET_SPARC)
-    { "g0", offsetof(CPUSPARCState, gregs[0]) },
-    { "g1", offsetof(CPUSPARCState, gregs[1]) },
-    { "g2", offsetof(CPUSPARCState, gregs[2]) },
-    { "g3", offsetof(CPUSPARCState, gregs[3]) },
-    { "g4", offsetof(CPUSPARCState, gregs[4]) },
-    { "g5", offsetof(CPUSPARCState, gregs[5]) },
-    { "g6", offsetof(CPUSPARCState, gregs[6]) },
-    { "g7", offsetof(CPUSPARCState, gregs[7]) },
-    { "o0", 0, monitor_get_reg },
-    { "o1", 1, monitor_get_reg },
-    { "o2", 2, monitor_get_reg },
-    { "o3", 3, monitor_get_reg },
-    { "o4", 4, monitor_get_reg },
-    { "o5", 5, monitor_get_reg },
-    { "o6", 6, monitor_get_reg },
-    { "o7", 7, monitor_get_reg },
-    { "l0", 8, monitor_get_reg },
-    { "l1", 9, monitor_get_reg },
-    { "l2", 10, monitor_get_reg },
-    { "l3", 11, monitor_get_reg },
-    { "l4", 12, monitor_get_reg },
-    { "l5", 13, monitor_get_reg },
-    { "l6", 14, monitor_get_reg },
-    { "l7", 15, monitor_get_reg },
-    { "i0", 16, monitor_get_reg },
-    { "i1", 17, monitor_get_reg },
-    { "i2", 18, monitor_get_reg },
-    { "i3", 19, monitor_get_reg },
-    { "i4", 20, monitor_get_reg },
-    { "i5", 21, monitor_get_reg },
-    { "i6", 22, monitor_get_reg },
-    { "i7", 23, monitor_get_reg },
-    { "pc", offsetof(CPUSPARCState, pc) },
-    { "npc", offsetof(CPUSPARCState, npc) },
-    { "y", offsetof(CPUSPARCState, y) },
-#ifndef TARGET_SPARC64
-    { "psr", 0, &monitor_get_psr, },
-    { "wim", offsetof(CPUSPARCState, wim) },
-#endif
-    { "tbr", offsetof(CPUSPARCState, tbr) },
-    { "fsr", offsetof(CPUSPARCState, fsr) },
-    { "f0", offsetof(CPUSPARCState, fpr[0].l.upper) },
-    { "f1", offsetof(CPUSPARCState, fpr[0].l.lower) },
-    { "f2", offsetof(CPUSPARCState, fpr[1].l.upper) },
-    { "f3", offsetof(CPUSPARCState, fpr[1].l.lower) },
-    { "f4", offsetof(CPUSPARCState, fpr[2].l.upper) },
-    { "f5", offsetof(CPUSPARCState, fpr[2].l.lower) },
-    { "f6", offsetof(CPUSPARCState, fpr[3].l.upper) },
-    { "f7", offsetof(CPUSPARCState, fpr[3].l.lower) },
-    { "f8", offsetof(CPUSPARCState, fpr[4].l.upper) },
-    { "f9", offsetof(CPUSPARCState, fpr[4].l.lower) },
-    { "f10", offsetof(CPUSPARCState, fpr[5].l.upper) },
-    { "f11", offsetof(CPUSPARCState, fpr[5].l.lower) },
-    { "f12", offsetof(CPUSPARCState, fpr[6].l.upper) },
-    { "f13", offsetof(CPUSPARCState, fpr[6].l.lower) },
-    { "f14", offsetof(CPUSPARCState, fpr[7].l.upper) },
-    { "f15", offsetof(CPUSPARCState, fpr[7].l.lower) },
-    { "f16", offsetof(CPUSPARCState, fpr[8].l.upper) },
-    { "f17", offsetof(CPUSPARCState, fpr[8].l.lower) },
-    { "f18", offsetof(CPUSPARCState, fpr[9].l.upper) },
-    { "f19", offsetof(CPUSPARCState, fpr[9].l.lower) },
-    { "f20", offsetof(CPUSPARCState, fpr[10].l.upper) },
-    { "f21", offsetof(CPUSPARCState, fpr[10].l.lower) },
-    { "f22", offsetof(CPUSPARCState, fpr[11].l.upper) },
-    { "f23", offsetof(CPUSPARCState, fpr[11].l.lower) },
-    { "f24", offsetof(CPUSPARCState, fpr[12].l.upper) },
-    { "f25", offsetof(CPUSPARCState, fpr[12].l.lower) },
-    { "f26", offsetof(CPUSPARCState, fpr[13].l.upper) },
-    { "f27", offsetof(CPUSPARCState, fpr[13].l.lower) },
-    { "f28", offsetof(CPUSPARCState, fpr[14].l.upper) },
-    { "f29", offsetof(CPUSPARCState, fpr[14].l.lower) },
-    { "f30", offsetof(CPUSPARCState, fpr[15].l.upper) },
-    { "f31", offsetof(CPUSPARCState, fpr[15].l.lower) },
-#ifdef TARGET_SPARC64
-    { "f32", offsetof(CPUSPARCState, fpr[16]) },
-    { "f34", offsetof(CPUSPARCState, fpr[17]) },
-    { "f36", offsetof(CPUSPARCState, fpr[18]) },
-    { "f38", offsetof(CPUSPARCState, fpr[19]) },
-    { "f40", offsetof(CPUSPARCState, fpr[20]) },
-    { "f42", offsetof(CPUSPARCState, fpr[21]) },
-    { "f44", offsetof(CPUSPARCState, fpr[22]) },
-    { "f46", offsetof(CPUSPARCState, fpr[23]) },
-    { "f48", offsetof(CPUSPARCState, fpr[24]) },
-    { "f50", offsetof(CPUSPARCState, fpr[25]) },
-    { "f52", offsetof(CPUSPARCState, fpr[26]) },
-    { "f54", offsetof(CPUSPARCState, fpr[27]) },
-    { "f56", offsetof(CPUSPARCState, fpr[28]) },
-    { "f58", offsetof(CPUSPARCState, fpr[29]) },
-    { "f60", offsetof(CPUSPARCState, fpr[30]) },
-    { "f62", offsetof(CPUSPARCState, fpr[31]) },
-    { "asi", offsetof(CPUSPARCState, asi) },
-    { "pstate", offsetof(CPUSPARCState, pstate) },
-    { "cansave", offsetof(CPUSPARCState, cansave) },
-    { "canrestore", offsetof(CPUSPARCState, canrestore) },
-    { "otherwin", offsetof(CPUSPARCState, otherwin) },
-    { "wstate", offsetof(CPUSPARCState, wstate) },
-    { "cleanwin", offsetof(CPUSPARCState, cleanwin) },
-    { "fprs", offsetof(CPUSPARCState, fprs) },
-#endif
-#endif
-    { NULL },
-};
 
 static void GCC_FMT_ATTR(2, 3) QEMU_NORETURN
 expr_error(Monitor *mon, const char *fmt, ...)
@@ -3301,10 +2134,16 @@ expr_error(Monitor *mon, const char *fmt, ...)
 /* return 0 if OK, -1 if not found */
 static int get_monitor_def(target_long *pval, const char *name)
 {
-    const MonitorDef *md;
+    const MonitorDef *md = target_monitor_defs();
     void *ptr;
+    uint64_t tmp = 0;
+    int ret;
 
-    for(md = monitor_defs; md->name != NULL; md++) {
+    if (md == NULL) {
+        return -1;
+    }
+
+    for(; md->name != NULL; md++) {
         if (compare_cmd(name, md->name)) {
             if (md->get_value) {
                 *pval = md->get_value(md, md->offset);
@@ -3326,7 +2165,13 @@ static int get_monitor_def(target_long *pval, const char *name)
             return 0;
         }
     }
-    return -1;
+
+    ret = target_get_monitor_def(mon_get_cpu(), name, &tmp);
+    if (!ret) {
+        *pval = (target_long) tmp;
+    }
+
+    return ret;
 }
 
 static void next(void)
@@ -3891,7 +2736,7 @@ static QDict *monitor_parse_arguments(Monitor *mon,
                         break;
                     }
                 }
-                val = strtosz(p, &end);
+                val = qemu_strtosz(p, &end);
                 if (val < 0) {
                     monitor_printf(mon, "invalid size\n");
                     goto fail;
@@ -4429,6 +3274,26 @@ void netdev_del_completion(ReadLineState *rs, int nb_args, const char *str)
     }
 }
 
+void trace_event_completion(ReadLineState *rs, int nb_args, const char *str)
+{
+    size_t len;
+
+    len = strlen(str);
+    readline_set_completion_index(rs, len);
+    if (nb_args == 2) {
+        TraceEventID id;
+        for (id = 0; id < trace_event_count(); id++) {
+            const char *event_name = trace_event_get_name(trace_event_id(id));
+            if (!strncmp(str, event_name, len)) {
+                readline_add_completion(rs, event_name);
+            }
+        }
+    } else if (nb_args == 3) {
+        add_completion_option(rs, str, "on");
+        add_completion_option(rs, str, "off");
+    }
+}
+
 void watchdog_action_completion(ReadLineState *rs, int nb_args, const char *str)
 {
     int i;
@@ -4551,13 +3416,18 @@ static void vm_completion(ReadLineState *rs, const char *str)
     readline_set_completion_index(rs, len);
     while ((bs = bdrv_next(bs))) {
         SnapshotInfoList *snapshots, *snapshot;
+        AioContext *ctx = bdrv_get_aio_context(bs);
+        bool ok = false;
 
-        if (!bdrv_can_snapshot(bs)) {
-            continue;
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            ok = bdrv_query_snapshot_info_list(bs, &snapshots, NULL) == 0;
         }
-        if (bdrv_query_snapshot_info_list(bs, &snapshots, NULL)) {
+        aio_context_release(ctx);
+        if (!ok) {
             continue;
         }
+
         snapshot = snapshots;
         while (snapshot) {
             char *completion = snapshot->value->name;
@@ -4979,7 +3849,7 @@ static QDict *qmp_check_input_obj(QObject *input_obj, Error **errp)
     return input_dict;
 }
 
-static void handle_qmp_command(JSONMessageParser *parser, QList *tokens)
+static void handle_qmp_command(JSONMessageParser *parser, GQueue *tokens)
 {
     Error *local_err = NULL;
     QObject *obj, *data;
@@ -5037,6 +3907,7 @@ static void handle_qmp_command(JSONMessageParser *parser, QList *tokens)
 err_out:
     monitor_protocol_emitter(mon, data, local_err);
     qobject_decref(data);
+    error_free(local_err);
     QDECREF(input);
     QDECREF(args);
 }
@@ -5102,7 +3973,7 @@ static QObject *get_qmp_greeting(void)
 {
     QObject *ver = NULL;
 
-    qmp_marshal_input_query_version(NULL, &ver, NULL);
+    qmp_marshal_query_version(NULL, &ver, NULL);
     return qobject_from_jsonf("{'QMP':{'version': %p,'capabilities': []}}",ver);
 }
 
@@ -5320,6 +4191,10 @@ int monitor_read_block_device_key(Monitor *mon, const char *device,
         monitor_printf(mon, "Device not found %s\n", device);
         return -1;
     }
+    if (!blk_bs(blk)) {
+        monitor_printf(mon, "Device '%s' has no medium\n", device);
+        return -1;
+    }
 
     bdrv_add_key(blk_bs(blk), NULL, &err);
     if (err) {
@@ -5361,3 +4236,10 @@ void qmp_rtc_reset_reinjection(Error **errp)
     error_setg(errp, QERR_FEATURE_DISABLED, "rtc-reset-reinjection");
 }
 #endif
+
+#ifndef TARGET_S390X
+void qmp_dump_skeys(const char *filename, Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "dump-skeys");
+}
+#endif
diff --git a/nbd.c b/nbd.c
index 07240bd3e..b3d965449 100644
--- a/nbd.c
+++ b/nbd.c
@@ -19,7 +19,7 @@
 #include "block/nbd.h"
 #include "sysemu/block-backend.h"
 
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 #include <errno.h>
 #include <string.h>
@@ -1005,7 +1005,7 @@ static NBDRequest *nbd_request_get(NBDClient *client)
     client->nb_requests++;
     nbd_update_can_read(client);
 
-    req = g_slice_new0(NBDRequest);
+    req = g_new0(NBDRequest, 1);
     nbd_client_get(client);
     req->client = client;
     return req;
@@ -1018,7 +1018,7 @@ static void nbd_request_put(NBDRequest *req)
     if (req->data) {
         qemu_vfree(req->data);
     }
-    g_slice_free(NBDRequest, req);
+    g_free(req);
 
     client->nb_requests--;
     nbd_update_can_read(client);
@@ -1446,6 +1446,7 @@ static void nbd_set_handlers(NBDClient *client)
 {
     if (client->exp && client->exp->ctx) {
         aio_set_fd_handler(client->exp->ctx, client->sock,
+                           true,
                            client->can_read ? nbd_read : NULL,
                            client->send_coroutine ? nbd_restart_write : NULL,
                            client);
@@ -1455,7 +1456,8 @@ static void nbd_set_handlers(NBDClient *client)
 static void nbd_unset_handlers(NBDClient *client)
 {
     if (client->exp && client->exp->ctx) {
-        aio_set_fd_handler(client->exp->ctx, client->sock, NULL, NULL, NULL);
+        aio_set_fd_handler(client->exp->ctx, client->sock,
+                           true, NULL, NULL, NULL);
     }
 }
 
diff --git a/net/Makefile.objs b/net/Makefile.objs
index ec19cb31d..5fa2f9731 100644
--- a/net/Makefile.objs
+++ b/net/Makefile.objs
@@ -13,3 +13,5 @@ common-obj-$(CONFIG_HAIKU) += tap-haiku.o
 common-obj-$(CONFIG_SLIRP) += slirp.o
 common-obj-$(CONFIG_VDE) += vde.o
 common-obj-$(CONFIG_NETMAP) += netmap.o
+common-obj-y += filter.o
+common-obj-y += filter-buffer.o
diff --git a/net/checksum.c b/net/checksum.c
index 14c08550e..0942437c0 100644
--- a/net/checksum.c
+++ b/net/checksum.c
@@ -59,6 +59,11 @@ void net_checksum_calculate(uint8_t *data, int length)
     int hlen, plen, proto, csum_offset;
     uint16_t csum;
 
+    /* Ensure data has complete L2 & L3 headers. */
+    if (length < 14 + 20) {
+        return;
+    }
+
     if ((data[14] & 0xf0) != 0x40)
 	return; /* not IPv4 */
     hlen  = (data[14] & 0x0f) * 4;
@@ -76,8 +81,9 @@ void net_checksum_calculate(uint8_t *data, int length)
 	return;
     }
 
-    if (plen < csum_offset+2)
-	return;
+    if (plen < csum_offset + 2 || 14 + hlen + plen > length) {
+        return;
+    }
 
     data[14+hlen+csum_offset]   = 0;
     data[14+hlen+csum_offset+1] = 0;
diff --git a/net/dump.c b/net/dump.c
index 02c8064be..347b5ca33 100644
--- a/net/dump.c
+++ b/net/dump.c
@@ -25,12 +25,13 @@
 #include "clients.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
+#include "qemu/iov.h"
 #include "qemu/log.h"
 #include "qemu/timer.h"
-#include "hub.h"
+#include "qapi/visitor.h"
+#include "net/filter.h"
 
 typedef struct DumpState {
-    NetClientState nc;
     int64_t start_ts;
     int fd;
     int pcap_caplen;
@@ -57,27 +58,32 @@ struct pcap_sf_pkthdr {
     uint32_t len;
 };
 
-static ssize_t dump_receive(NetClientState *nc, const uint8_t *buf, size_t size)
+static ssize_t dump_receive_iov(DumpState *s, const struct iovec *iov, int cnt)
 {
-    DumpState *s = DO_UPCAST(DumpState, nc, nc);
     struct pcap_sf_pkthdr hdr;
     int64_t ts;
     int caplen;
+    size_t size = iov_size(iov, cnt);
+    struct iovec dumpiov[cnt + 1];
 
     /* Early return in case of previous error. */
     if (s->fd < 0) {
         return size;
     }
 
-    ts = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 1000000, get_ticks_per_sec());
+    ts = qemu_clock_get_us(QEMU_CLOCK_VIRTUAL);
     caplen = size > s->pcap_caplen ? s->pcap_caplen : size;
 
     hdr.ts.tv_sec = ts / 1000000 + s->start_ts;
     hdr.ts.tv_usec = ts % 1000000;
     hdr.caplen = caplen;
     hdr.len = size;
-    if (write(s->fd, &hdr, sizeof(hdr)) != sizeof(hdr) ||
-        write(s->fd, buf, caplen) != caplen) {
+
+    dumpiov[0].iov_base = &hdr;
+    dumpiov[0].iov_len = sizeof(hdr);
+    cnt = iov_copy(&dumpiov[1], cnt, iov, cnt, 0, caplen);
+
+    if (writev(s->fd, dumpiov, cnt + 1) != sizeof(hdr) + caplen) {
         qemu_log("-net dump write error - stop dump\n");
         close(s->fd);
         s->fd = -1;
@@ -86,27 +92,16 @@ static ssize_t dump_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     return size;
 }
 
-static void dump_cleanup(NetClientState *nc)
+static void dump_cleanup(DumpState *s)
 {
-    DumpState *s = DO_UPCAST(DumpState, nc, nc);
-
     close(s->fd);
+    s->fd = -1;
 }
 
-static NetClientInfo net_dump_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_DUMP,
-    .size = sizeof(DumpState),
-    .receive = dump_receive,
-    .cleanup = dump_cleanup,
-};
-
-static int net_dump_init(NetClientState *peer, const char *device,
-                         const char *name, const char *filename, int len,
-                         Error **errp)
+static int net_dump_state_init(DumpState *s, const char *filename,
+                               int len, Error **errp)
 {
     struct pcap_file_hdr hdr;
-    NetClientState *nc;
-    DumpState *s;
     struct tm tm;
     int fd;
 
@@ -130,13 +125,6 @@ static int net_dump_init(NetClientState *peer, const char *device,
         return -1;
     }
 
-    nc = qemu_new_net_client(&net_dump_info, peer, device, name);
-
-    snprintf(nc->info_str, sizeof(nc->info_str),
-             "dump to %s (len=%d)", filename, len);
-
-    s = DO_UPCAST(DumpState, nc, nc);
-
     s->fd = fd;
     s->pcap_caplen = len;
 
@@ -146,16 +134,61 @@ static int net_dump_init(NetClientState *peer, const char *device,
     return 0;
 }
 
+/* Dumping via VLAN netclient */
+
+struct DumpNetClient {
+    NetClientState nc;
+    DumpState ds;
+};
+typedef struct DumpNetClient DumpNetClient;
+
+static ssize_t dumpclient_receive(NetClientState *nc, const uint8_t *buf,
+                                  size_t size)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = size
+    };
+
+    return dump_receive_iov(&dc->ds, &iov, 1);
+}
+
+static ssize_t dumpclient_receive_iov(NetClientState *nc,
+                                      const struct iovec *iov, int cnt)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+
+    return dump_receive_iov(&dc->ds, iov, cnt);
+}
+
+static void dumpclient_cleanup(NetClientState *nc)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+
+    dump_cleanup(&dc->ds);
+}
+
+static NetClientInfo net_dump_info = {
+    .type = NET_CLIENT_OPTIONS_KIND_DUMP,
+    .size = sizeof(DumpNetClient),
+    .receive = dumpclient_receive,
+    .receive_iov = dumpclient_receive_iov,
+    .cleanup = dumpclient_cleanup,
+};
+
 int net_init_dump(const NetClientOptions *opts, const char *name,
                   NetClientState *peer, Error **errp)
 {
-    int len;
+    int len, rc;
     const char *file;
     char def_file[128];
     const NetdevDumpOptions *dump;
+    NetClientState *nc;
+    DumpNetClient *dnc;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_DUMP);
-    dump = opts->dump;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_DUMP);
+    dump = opts->u.dump;
 
     assert(peer);
 
@@ -182,5 +215,148 @@ int net_init_dump(const NetClientOptions *opts, const char *name,
         len = 65536;
     }
 
-    return net_dump_init(peer, "dump", name, file, len, errp);
+    nc = qemu_new_net_client(&net_dump_info, peer, "dump", name);
+    snprintf(nc->info_str, sizeof(nc->info_str),
+             "dump to %s (len=%d)", file, len);
+
+    dnc = DO_UPCAST(DumpNetClient, nc, nc);
+    rc = net_dump_state_init(&dnc->ds, file, len, errp);
+    if (rc) {
+        qemu_del_net_client(nc);
+    }
+    return rc;
+}
+
+/* Dumping via filter */
+
+#define TYPE_FILTER_DUMP "filter-dump"
+
+#define FILTER_DUMP(obj) \
+    OBJECT_CHECK(NetFilterDumpState, (obj), TYPE_FILTER_DUMP)
+
+struct NetFilterDumpState {
+    NetFilterState nfs;
+    DumpState ds;
+    char *filename;
+    uint32_t maxlen;
+};
+typedef struct NetFilterDumpState NetFilterDumpState;
+
+static ssize_t filter_dump_receive_iov(NetFilterState *nf, NetClientState *sndr,
+                                       unsigned flags, const struct iovec *iov,
+                                       int iovcnt, NetPacketSent *sent_cb)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    dump_receive_iov(&nfds->ds, iov, iovcnt);
+    return 0;
 }
+
+static void filter_dump_cleanup(NetFilterState *nf)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    dump_cleanup(&nfds->ds);
+}
+
+static void filter_dump_setup(NetFilterState *nf, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    if (!nfds->filename) {
+        error_setg(errp, "dump filter needs 'file' property set!");
+        return;
+    }
+
+    net_dump_state_init(&nfds->ds, nfds->filename, nfds->maxlen, errp);
+}
+
+static void filter_dump_get_maxlen(Object *obj, Visitor *v, void *opaque,
+                                   const char *name, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+    uint32_t value = nfds->maxlen;
+
+    visit_type_uint32(v, &value, name, errp);
+}
+
+static void filter_dump_set_maxlen(Object *obj, Visitor *v, void *opaque,
+                                   const char *name, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, &value, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (value == 0) {
+        error_setg(&local_err, "Property '%s.%s' doesn't take value '%u'",
+                   object_get_typename(obj), name, value);
+        goto out;
+    }
+    nfds->maxlen = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static char *file_dump_get_filename(Object *obj, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    return g_strdup(nfds->filename);
+}
+
+static void file_dump_set_filename(Object *obj, const char *value, Error **errp)
+{
+   NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    g_free(nfds->filename);
+    nfds->filename = g_strdup(value);
+}
+
+static void filter_dump_instance_init(Object *obj)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    nfds->maxlen = 65536;
+
+    object_property_add(obj, "maxlen", "int", filter_dump_get_maxlen,
+                        filter_dump_set_maxlen, NULL, NULL, NULL);
+    object_property_add_str(obj, "file", file_dump_get_filename,
+                            file_dump_set_filename, NULL);
+}
+
+static void filter_dump_instance_finalize(Object *obj)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    g_free(nfds->filename);
+}
+
+static void filter_dump_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_dump_setup;
+    nfc->cleanup = filter_dump_cleanup;
+    nfc->receive_iov = filter_dump_receive_iov;
+}
+
+static const TypeInfo filter_dump_info = {
+    .name = TYPE_FILTER_DUMP,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_dump_class_init,
+    .instance_init = filter_dump_instance_init,
+    .instance_finalize = filter_dump_instance_finalize,
+    .instance_size = sizeof(NetFilterDumpState),
+};
+
+static void filter_dump_register_types(void)
+{
+    type_register_static(&filter_dump_info);
+}
+
+type_init(filter_dump_register_types);
diff --git a/net/filter-buffer.c b/net/filter-buffer.c
new file mode 100644
index 000000000..57be14941
--- /dev/null
+++ b/net/filter-buffer.c
@@ -0,0 +1,186 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "net/filter.h"
+#include "net/queue.h"
+#include "qemu-common.h"
+#include "qemu/timer.h"
+#include "qemu/iov.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi-visit.h"
+#include "qom/object.h"
+
+#define TYPE_FILTER_BUFFER "filter-buffer"
+
+#define FILTER_BUFFER(obj) \
+    OBJECT_CHECK(FilterBufferState, (obj), TYPE_FILTER_BUFFER)
+
+typedef struct FilterBufferState {
+    NetFilterState parent_obj;
+
+    NetQueue *incoming_queue;
+    uint32_t interval;
+    QEMUTimer release_timer;
+} FilterBufferState;
+
+static void filter_buffer_flush(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (!qemu_net_queue_flush(s->incoming_queue)) {
+        /* Unable to empty the queue, purge remaining packets */
+        qemu_net_queue_purge(s->incoming_queue, nf->netdev);
+    }
+}
+
+static void filter_buffer_release_timer(void *opaque)
+{
+    NetFilterState *nf = opaque;
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * Note: filter_buffer_flush() drops packets that can't be sent
+     * TODO: We should leave them queued.  But currently there's no way
+     * for the next filter or receiver to notify us that it can receive
+     * more packets.
+     */
+    filter_buffer_flush(nf);
+    /* Timer rearmed to fire again in s->interval microseconds. */
+    timer_mod(&s->release_timer,
+              qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) + s->interval);
+}
+
+/* filter APIs */
+static ssize_t filter_buffer_receive_iov(NetFilterState *nf,
+                                         NetClientState *sender,
+                                         unsigned flags,
+                                         const struct iovec *iov,
+                                         int iovcnt,
+                                         NetPacketSent *sent_cb)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * We return size when buffer a packet, the sender will take it as
+     * a already sent packet, so sent_cb should not be called later.
+     *
+     * FIXME: Even if the guest can't receive packets for some reasons,
+     * the filter can still accept packets until its internal queue is full.
+     * For example:
+     *   For some reason, receiver could not receive more packets
+     * (.can_receive() returns zero). Without a filter, at most one packet
+     * will be queued in incoming queue and sender's poll will be disabled
+     * unit its sent_cb() was called. With a filter, it will keep receiving
+     * the packets without caring about the receiver. This is suboptimal.
+     * May need more thoughts (e.g keeping sent_cb).
+     */
+    qemu_net_queue_append_iov(s->incoming_queue, sender, flags,
+                              iov, iovcnt, NULL);
+    return iov_size(iov, iovcnt);
+}
+
+static void filter_buffer_cleanup(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (s->interval) {
+        timer_del(&s->release_timer);
+    }
+
+    /* flush packets */
+    if (s->incoming_queue) {
+        filter_buffer_flush(nf);
+        g_free(s->incoming_queue);
+    }
+}
+
+static void filter_buffer_setup(NetFilterState *nf, Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * We may want to accept zero interval when VM FT solutions like MC
+     * or COLO use this filter to release packets on demand.
+     */
+    if (!s->interval) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "interval",
+                   "a non-zero interval");
+        return;
+    }
+
+    s->incoming_queue = qemu_new_net_queue(qemu_netfilter_pass_to_next, nf);
+    if (s->interval) {
+        timer_init_us(&s->release_timer, QEMU_CLOCK_VIRTUAL,
+                      filter_buffer_release_timer, nf);
+        /* Timer armed to fire in s->interval microseconds. */
+        timer_mod(&s->release_timer,
+                  qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) + s->interval);
+    }
+}
+
+static void filter_buffer_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_buffer_setup;
+    nfc->cleanup = filter_buffer_cleanup;
+    nfc->receive_iov = filter_buffer_receive_iov;
+}
+
+static void filter_buffer_get_interval(Object *obj, Visitor *v, void *opaque,
+                                       const char *name, Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(obj);
+    uint32_t value = s->interval;
+
+    visit_type_uint32(v, &value, name, errp);
+}
+
+static void filter_buffer_set_interval(Object *obj, Visitor *v, void *opaque,
+                                       const char *name, Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, &value, name, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (!value) {
+        error_setg(&local_err, "Property '%s.%s' requires a positive value",
+                   object_get_typename(obj), name);
+        goto out;
+    }
+    s->interval = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void filter_buffer_init(Object *obj)
+{
+    object_property_add(obj, "interval", "int",
+                        filter_buffer_get_interval,
+                        filter_buffer_set_interval, NULL, NULL, NULL);
+}
+
+static const TypeInfo filter_buffer_info = {
+    .name = TYPE_FILTER_BUFFER,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_buffer_class_init,
+    .instance_init = filter_buffer_init,
+    .instance_size = sizeof(FilterBufferState),
+};
+
+static void register_types(void)
+{
+    type_register_static(&filter_buffer_info);
+}
+
+type_init(register_types);
diff --git a/net/filter.c b/net/filter.c
new file mode 100644
index 000000000..f777ba289
--- /dev/null
+++ b/net/filter.c
@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
+
+#include "net/filter.h"
+#include "net/net.h"
+#include "net/vhost_net.h"
+#include "qom/object_interfaces.h"
+#include "qemu/iov.h"
+#include "qapi/string-output-visitor.h"
+
+ssize_t qemu_netfilter_receive(NetFilterState *nf,
+                               NetFilterDirection direction,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb)
+{
+    if (nf->direction == direction ||
+        nf->direction == NET_FILTER_DIRECTION_ALL) {
+        return NETFILTER_GET_CLASS(OBJECT(nf))->receive_iov(
+                                   nf, sender, flags, iov, iovcnt, sent_cb);
+    }
+
+    return 0;
+}
+
+ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
+                                    unsigned flags,
+                                    const struct iovec *iov,
+                                    int iovcnt,
+                                    void *opaque)
+{
+    int ret = 0;
+    int direction;
+    NetFilterState *nf = opaque;
+    NetFilterState *next = QTAILQ_NEXT(nf, next);
+
+    if (!sender || !sender->peer) {
+        /* no receiver, or sender been deleted, no need to pass it further */
+        goto out;
+    }
+
+    if (nf->direction == NET_FILTER_DIRECTION_ALL) {
+        if (sender == nf->netdev) {
+            /* This packet is sent by netdev itself */
+            direction = NET_FILTER_DIRECTION_TX;
+        } else {
+            direction = NET_FILTER_DIRECTION_RX;
+        }
+    } else {
+        direction = nf->direction;
+    }
+
+    while (next) {
+        /*
+         * if qemu_netfilter_pass_to_next been called, means that
+         * the packet has been hold by filter and has already retured size
+         * to the sender, so sent_cb shouldn't be called later, just
+         * pass NULL to next.
+         */
+        ret = qemu_netfilter_receive(next, direction, sender, flags, iov,
+                                     iovcnt, NULL);
+        if (ret) {
+            return ret;
+        }
+        next = QTAILQ_NEXT(next, next);
+    }
+
+    /*
+     * We have gone through all filters, pass it to receiver.
+     * Do the valid check again incase sender or receiver been
+     * deleted while we go through filters.
+     */
+    if (sender && sender->peer) {
+        qemu_net_queue_send_iov(sender->peer->incoming_queue,
+                                sender, flags, iov, iovcnt, NULL);
+    }
+
+out:
+    /* no receiver, or sender been deleted */
+    return iov_size(iov, iovcnt);
+}
+
+static char *netfilter_get_netdev_id(Object *obj, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    return g_strdup(nf->netdev_id);
+}
+
+static void netfilter_set_netdev_id(Object *obj, const char *str, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    nf->netdev_id = g_strdup(str);
+}
+
+static int netfilter_get_direction(Object *obj, Error **errp G_GNUC_UNUSED)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    return nf->direction;
+}
+
+static void netfilter_set_direction(Object *obj, int direction, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    nf->direction = direction;
+}
+
+static void netfilter_init(Object *obj)
+{
+    object_property_add_str(obj, "netdev",
+                            netfilter_get_netdev_id, netfilter_set_netdev_id,
+                            NULL);
+    object_property_add_enum(obj, "queue", "NetFilterDirection",
+                             NetFilterDirection_lookup,
+                             netfilter_get_direction, netfilter_set_direction,
+                             NULL);
+}
+
+static void netfilter_complete(UserCreatable *uc, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(uc);
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(uc);
+    int queues;
+    Error *local_err = NULL;
+    char *str, *info;
+    ObjectProperty *prop;
+    ObjectPropertyIterator *iter;
+    StringOutputVisitor *ov;
+
+    if (!nf->netdev_id) {
+        error_setg(errp, "Parameter 'netdev' is required");
+        return;
+    }
+
+    queues = qemu_find_net_clients_except(nf->netdev_id, ncs,
+                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          MAX_QUEUE_NUM);
+    if (queues < 1) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "netdev",
+                   "a network backend id");
+        return;
+    } else if (queues > 1) {
+        error_setg(errp, "multiqueue is not supported");
+        return;
+    }
+
+    if (get_vhost_net(ncs[0])) {
+        error_setg(errp, "Vhost is not supported");
+        return;
+    }
+
+    nf->netdev = ncs[0];
+
+    if (nfc->setup) {
+        nfc->setup(nf, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+    QTAILQ_INSERT_TAIL(&nf->netdev->filters, nf, next);
+
+    /* generate info str */
+    iter = object_property_iter_init(OBJECT(nf));
+    while ((prop = object_property_iter_next(iter))) {
+        if (!strcmp(prop->name, "type")) {
+            continue;
+        }
+        ov = string_output_visitor_new(false);
+        object_property_get(OBJECT(nf), string_output_get_visitor(ov),
+                            prop->name, errp);
+        str = string_output_get_string(ov);
+        string_output_visitor_cleanup(ov);
+        info = g_strdup_printf(",%s=%s", prop->name, str);
+        g_strlcat(nf->info_str, info, sizeof(nf->info_str));
+        g_free(str);
+        g_free(info);
+    }
+    object_property_iter_free(iter);
+}
+
+static void netfilter_finalize(Object *obj)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(obj);
+
+    if (nfc->cleanup) {
+        nfc->cleanup(nf);
+    }
+
+    if (nf->netdev && !QTAILQ_EMPTY(&nf->netdev->filters)) {
+        QTAILQ_REMOVE(&nf->netdev->filters, nf, next);
+    }
+    g_free(nf->netdev_id);
+}
+
+static void netfilter_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = netfilter_complete;
+}
+
+static const TypeInfo netfilter_info = {
+    .name = TYPE_NETFILTER,
+    .parent = TYPE_OBJECT,
+    .abstract = true,
+    .class_size = sizeof(NetFilterClass),
+    .class_init = netfilter_class_init,
+    .instance_size = sizeof(NetFilterState),
+    .instance_init = netfilter_init,
+    .instance_finalize = netfilter_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+static void register_types(void)
+{
+    type_register_static(&netfilter_info);
+}
+
+type_init(register_types);
diff --git a/net/hub.c b/net/hub.c
index 3047f1276..9ae9f012c 100644
--- a/net/hub.c
+++ b/net/hub.c
@@ -285,9 +285,9 @@ int net_init_hubport(const NetClientOptions *opts, const char *name,
 {
     const NetdevHubPortOptions *hubport;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_HUBPORT);
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_HUBPORT);
     assert(!peer);
-    hubport = opts->hubport;
+    hubport = opts->u.hubport;
 
     net_hub_add_port(hubport->hubid, name);
     return 0;
diff --git a/net/l2tpv3.c b/net/l2tpv3.c
index 4f9bceecc..8e68e540e 100644
--- a/net/l2tpv3.c
+++ b/net/l2tpv3.c
@@ -545,8 +545,8 @@ int net_init_l2tpv3(const NetClientOptions *opts,
     s->queue_tail = 0;
     s->header_mismatch = false;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_L2TPV3);
-    l2tpv3 = opts->l2tpv3;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_L2TPV3);
+    l2tpv3 = opts->u.l2tpv3;
 
     if (l2tpv3->has_ipv6 && l2tpv3->ipv6) {
         s->ipv6 = l2tpv3->ipv6;
diff --git a/net/net.c b/net/net.c
index 8e96011e6..ade605184 100644
--- a/net/net.c
+++ b/net/net.c
@@ -44,6 +44,7 @@
 #include "qapi/opts-visitor.h"
 #include "qapi/dealloc-visitor.h"
 #include "sysemu/sysemu.h"
+#include "net/filter.h"
 
 /* Net bridge is currently not supported for W32. */
 #if !defined(_WIN32)
@@ -285,8 +286,9 @@ static void qemu_net_client_setup(NetClientState *nc,
     }
     QTAILQ_INSERT_TAIL(&net_clients, nc, next);
 
-    nc->incoming_queue = qemu_new_net_queue(nc);
+    nc->incoming_queue = qemu_new_net_queue(qemu_deliver_packet_iov, nc);
     nc->destructor = destructor;
+    QTAILQ_INIT(&nc->filters);
 }
 
 NetClientState *qemu_new_net_client(NetClientInfo *info,
@@ -384,6 +386,7 @@ void qemu_del_net_client(NetClientState *nc)
 {
     NetClientState *ncs[MAX_QUEUE_NUM];
     int queues, i;
+    NetFilterState *nf, *next;
 
     assert(nc->info->type != NET_CLIENT_OPTIONS_KIND_NIC);
 
@@ -395,6 +398,10 @@ void qemu_del_net_client(NetClientState *nc)
                                           MAX_QUEUE_NUM);
     assert(queues != 0);
 
+    QTAILQ_FOREACH_SAFE(nf, &nc->filters, next, next) {
+        object_unparent(OBJECT(nf));
+    }
+
     /* If there is a peer NIC, delete and cleanup client, but do not free. */
     if (nc->peer && nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
         NICState *nic = qemu_get_nic(nc->peer);
@@ -562,34 +569,42 @@ int qemu_can_send_packet(NetClientState *sender)
     return 1;
 }
 
-ssize_t qemu_deliver_packet(NetClientState *sender,
-                            unsigned flags,
-                            const uint8_t *data,
-                            size_t size,
-                            void *opaque)
+static ssize_t filter_receive_iov(NetClientState *nc,
+                                  NetFilterDirection direction,
+                                  NetClientState *sender,
+                                  unsigned flags,
+                                  const struct iovec *iov,
+                                  int iovcnt,
+                                  NetPacketSent *sent_cb)
 {
-    NetClientState *nc = opaque;
-    ssize_t ret;
-
-    if (nc->link_down) {
-        return size;
-    }
+    ssize_t ret = 0;
+    NetFilterState *nf = NULL;
 
-    if (nc->receive_disabled) {
-        return 0;
+    QTAILQ_FOREACH(nf, &nc->filters, next) {
+        ret = qemu_netfilter_receive(nf, direction, sender, flags, iov,
+                                     iovcnt, sent_cb);
+        if (ret) {
+            return ret;
+        }
     }
 
-    if (flags & QEMU_NET_PACKET_FLAG_RAW && nc->info->receive_raw) {
-        ret = nc->info->receive_raw(nc, data, size);
-    } else {
-        ret = nc->info->receive(nc, data, size);
-    }
+    return ret;
+}
 
-    if (ret == 0) {
-        nc->receive_disabled = 1;
-    }
+static ssize_t filter_receive(NetClientState *nc,
+                              NetFilterDirection direction,
+                              NetClientState *sender,
+                              unsigned flags,
+                              const uint8_t *data,
+                              size_t size,
+                              NetPacketSent *sent_cb)
+{
+    struct iovec iov = {
+        .iov_base = (void *)data,
+        .iov_len = size
+    };
 
-    return ret;
+    return filter_receive_iov(nc, direction, sender, flags, &iov, 1, sent_cb);
 }
 
 void qemu_purge_queued_packets(NetClientState *nc)
@@ -633,6 +648,7 @@ static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender,
                                                  NetPacketSent *sent_cb)
 {
     NetQueue *queue;
+    int ret;
 
 #ifdef DEBUG_NET
     printf("qemu_send_packet_async:\n");
@@ -643,6 +659,19 @@ static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender,
         return size;
     }
 
+    /* Let filters handle the packet first */
+    ret = filter_receive(sender, NET_FILTER_DIRECTION_TX,
+                         sender, flags, buf, size, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
+    ret = filter_receive(sender->peer, NET_FILTER_DIRECTION_RX,
+                         sender, flags, buf, size, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
     queue = sender->peer->incoming_queue;
 
     return qemu_net_queue_send(queue, sender, flags, buf, size, sent_cb);
@@ -668,14 +697,25 @@ ssize_t qemu_send_packet_raw(NetClientState *nc, const uint8_t *buf, int size)
 }
 
 static ssize_t nc_sendv_compat(NetClientState *nc, const struct iovec *iov,
-                               int iovcnt)
+                               int iovcnt, unsigned flags)
 {
-    uint8_t buffer[NET_BUFSIZE];
+    uint8_t buf[NET_BUFSIZE];
+    uint8_t *buffer;
     size_t offset;
 
-    offset = iov_to_buf(iov, iovcnt, 0, buffer, sizeof(buffer));
+    if (iovcnt == 1) {
+        buffer = iov[0].iov_base;
+        offset = iov[0].iov_len;
+    } else {
+        buffer = buf;
+        offset = iov_to_buf(iov, iovcnt, 0, buf, sizeof(buf));
+    }
 
-    return nc->info->receive(nc, buffer, offset);
+    if (flags & QEMU_NET_PACKET_FLAG_RAW && nc->info->receive_raw) {
+        return nc->info->receive_raw(nc, buffer, offset);
+    } else {
+        return nc->info->receive(nc, buffer, offset);
+    }
 }
 
 ssize_t qemu_deliver_packet_iov(NetClientState *sender,
@@ -698,7 +738,7 @@ ssize_t qemu_deliver_packet_iov(NetClientState *sender,
     if (nc->info->receive_iov) {
         ret = nc->info->receive_iov(nc, iov, iovcnt);
     } else {
-        ret = nc_sendv_compat(nc, iov, iovcnt);
+        ret = nc_sendv_compat(nc, iov, iovcnt, flags);
     }
 
     if (ret == 0) {
@@ -713,11 +753,25 @@ ssize_t qemu_sendv_packet_async(NetClientState *sender,
                                 NetPacketSent *sent_cb)
 {
     NetQueue *queue;
+    int ret;
 
     if (sender->link_down || !sender->peer) {
         return iov_size(iov, iovcnt);
     }
 
+    /* Let filters handle the packet first */
+    ret = filter_receive_iov(sender, NET_FILTER_DIRECTION_TX, sender,
+                             QEMU_NET_PACKET_FLAG_NONE, iov, iovcnt, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
+    ret = filter_receive_iov(sender->peer, NET_FILTER_DIRECTION_RX, sender,
+                             QEMU_NET_PACKET_FLAG_NONE, iov, iovcnt, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
     queue = sender->peer->incoming_queue;
 
     return qemu_net_queue_send_iov(queue, sender,
@@ -828,8 +882,8 @@ static int net_init_nic(const NetClientOptions *opts, const char *name,
     NICInfo *nd;
     const NetLegacyNicOptions *nic;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_NIC);
-    nic = opts->nic;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_NIC);
+    nic = opts->u.nic;
 
     idx = nic_get_free_idx();
     if (idx == -1 || nb_nics >= MAX_NICS) {
@@ -930,9 +984,9 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp)
         opts = netdev->opts;
         name = netdev->id;
 
-        if (opts->kind == NET_CLIENT_OPTIONS_KIND_DUMP ||
-            opts->kind == NET_CLIENT_OPTIONS_KIND_NIC ||
-            !net_client_init_fun[opts->kind]) {
+        if (opts->type == NET_CLIENT_OPTIONS_KIND_DUMP ||
+            opts->type == NET_CLIENT_OPTIONS_KIND_NIC ||
+            !net_client_init_fun[opts->type]) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
                        "a netdev backend type");
             return -1;
@@ -943,16 +997,16 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp)
         /* missing optional values have been initialized to "all bits zero" */
         name = net->has_id ? net->id : net->name;
 
-        if (opts->kind == NET_CLIENT_OPTIONS_KIND_NONE) {
+        if (opts->type == NET_CLIENT_OPTIONS_KIND_NONE) {
             return 0; /* nothing to do */
         }
-        if (opts->kind == NET_CLIENT_OPTIONS_KIND_HUBPORT) {
+        if (opts->type == NET_CLIENT_OPTIONS_KIND_HUBPORT) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
                        "a net type");
             return -1;
         }
 
-        if (!net_client_init_fun[opts->kind]) {
+        if (!net_client_init_fun[opts->type]) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
                        "a net backend type (maybe it is not compiled "
                        "into this binary)");
@@ -960,17 +1014,17 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp)
         }
 
         /* Do not add to a vlan if it's a nic with a netdev= parameter. */
-        if (opts->kind != NET_CLIENT_OPTIONS_KIND_NIC ||
-            !opts->nic->has_netdev) {
+        if (opts->type != NET_CLIENT_OPTIONS_KIND_NIC ||
+            !opts->u.nic->has_netdev) {
             peer = net_hub_add_port(net->has_vlan ? net->vlan : 0, NULL);
         }
     }
 
-    if (net_client_init_fun[opts->kind](opts, name, peer, errp) < 0) {
+    if (net_client_init_fun[opts->type](opts, name, peer, errp) < 0) {
         /* FIXME drop when all init functions store an Error */
         if (errp && !*errp) {
             error_setg(errp, QERR_DEVICE_INIT_FAILED,
-                       NetClientOptionsKind_lookup[opts->kind]);
+                       NetClientOptionsKind_lookup[opts->type]);
         }
         return -1;
     }
@@ -1133,10 +1187,22 @@ void qmp_netdev_del(const char *id, Error **errp)
 
 void print_net_client(Monitor *mon, NetClientState *nc)
 {
+    NetFilterState *nf;
+
     monitor_printf(mon, "%s: index=%d,type=%s,%s\n", nc->name,
                    nc->queue_index,
                    NetClientOptionsKind_lookup[nc->info->type],
                    nc->info_str);
+    if (!QTAILQ_EMPTY(&nc->filters)) {
+        monitor_printf(mon, "filters:\n");
+    }
+    QTAILQ_FOREACH(nf, &nc->filters, next) {
+        char *path = object_get_canonical_path_component(OBJECT(nf));
+        monitor_printf(mon, "  - %s: type=%s%s\n", path,
+                       object_get_typename(OBJECT(nf)),
+                       nf->info_str);
+        g_free(path);
+    }
 }
 
 RxFilterInfoList *qmp_query_rx_filter(bool has_name, const char *name,
@@ -1162,6 +1228,12 @@ RxFilterInfoList *qmp_query_rx_filter(bool has_name, const char *name,
             continue;
         }
 
+        /* only query information on queue 0 since the info is per nic,
+         * not per queue
+         */
+        if (nc->queue_index != 0)
+            continue;
+
         if (nc->info->query_rx_filter) {
             info = nc->info->query_rx_filter(nc);
             entry = g_malloc0(sizeof(*entry));
diff --git a/net/netmap.c b/net/netmap.c
index 508b82947..555836829 100644
--- a/net/netmap.c
+++ b/net/netmap.c
@@ -90,7 +90,7 @@ pkt_copy(const void *_src, void *_dst, int l)
  * Open a netmap device. We assume there is only one queue
  * (which is the case for the VALE bridge).
  */
-static int netmap_open(NetmapPriv *me)
+static void netmap_open(NetmapPriv *me, Error **errp)
 {
     int fd;
     int err;
@@ -99,9 +99,8 @@ static int netmap_open(NetmapPriv *me)
 
     me->fd = fd = open(me->fdname, O_RDWR);
     if (fd < 0) {
-        error_report("Unable to open netmap device '%s' (%s)",
-                        me->fdname, strerror(errno));
-        return -1;
+        error_setg_file_open(errp, errno, me->fdname);
+        return;
     }
     memset(&req, 0, sizeof(req));
     pstrcpy(req.nr_name, sizeof(req.nr_name), me->ifname);
@@ -109,15 +108,14 @@ static int netmap_open(NetmapPriv *me)
     req.nr_version = NETMAP_API;
     err = ioctl(fd, NIOCREGIF, &req);
     if (err) {
-        error_report("Unable to register %s: %s", me->ifname, strerror(errno));
+        error_setg_errno(errp, errno, "Unable to register %s", me->ifname);
         goto error;
     }
     l = me->memsize = req.nr_memsize;
 
     me->mem = mmap(0, l, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
     if (me->mem == MAP_FAILED) {
-        error_report("Unable to mmap netmap shared memory: %s",
-                        strerror(errno));
+        error_setg_errno(errp, errno, "Unable to mmap netmap shared memory");
         me->mem = NULL;
         goto error;
     }
@@ -125,11 +123,11 @@ static int netmap_open(NetmapPriv *me)
     me->nifp = NETMAP_IF(me->mem, req.nr_offset);
     me->tx = NETMAP_TXRING(me->nifp, 0);
     me->rx = NETMAP_RXRING(me->nifp, 0);
-    return 0;
+
+    return;
 
 error:
     close(me->fd);
-    return -1;
 }
 
 static void netmap_send(void *opaque);
@@ -438,9 +436,9 @@ static NetClientInfo net_netmap_info = {
 int net_init_netmap(const NetClientOptions *opts,
                     const char *name, NetClientState *peer, Error **errp)
 {
-    /* FIXME error_setg(errp, ...) on failure */
-    const NetdevNetmapOptions *netmap_opts = opts->netmap;
+    const NetdevNetmapOptions *netmap_opts = opts->u.netmap;
     NetClientState *nc;
+    Error *err = NULL;
     NetmapPriv me;
     NetmapState *s;
 
@@ -448,7 +446,9 @@ int net_init_netmap(const NetClientOptions *opts,
         netmap_opts->has_devname ? netmap_opts->devname : "/dev/netmap");
     /* Set default name for the port if not supplied. */
     pstrcpy(me.ifname, sizeof(me.ifname), netmap_opts->ifname);
-    if (netmap_open(&me)) {
+    netmap_open(&me, &err);
+    if (err) {
+        error_propagate(errp, err);
         return -1;
     }
     /* Create the object. */
diff --git a/net/queue.c b/net/queue.c
index ebbe2bb93..de8b9d31c 100644
--- a/net/queue.c
+++ b/net/queue.c
@@ -52,13 +52,14 @@ struct NetQueue {
     void *opaque;
     uint32_t nq_maxlen;
     uint32_t nq_count;
+    NetQueueDeliverFunc *deliver;
 
     QTAILQ_HEAD(packets, NetPacket) packets;
 
     unsigned delivering : 1;
 };
 
-NetQueue *qemu_new_net_queue(void *opaque)
+NetQueue *qemu_new_net_queue(NetQueueDeliverFunc *deliver, void *opaque)
 {
     NetQueue *queue;
 
@@ -67,6 +68,7 @@ NetQueue *qemu_new_net_queue(void *opaque)
     queue->opaque = opaque;
     queue->nq_maxlen = 10000;
     queue->nq_count = 0;
+    queue->deliver = deliver;
 
     QTAILQ_INIT(&queue->packets);
 
@@ -110,12 +112,12 @@ static void qemu_net_queue_append(NetQueue *queue,
     QTAILQ_INSERT_TAIL(&queue->packets, packet, entry);
 }
 
-static void qemu_net_queue_append_iov(NetQueue *queue,
-                                      NetClientState *sender,
-                                      unsigned flags,
-                                      const struct iovec *iov,
-                                      int iovcnt,
-                                      NetPacketSent *sent_cb)
+void qemu_net_queue_append_iov(NetQueue *queue,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb)
 {
     NetPacket *packet;
     size_t max_len = 0;
@@ -152,9 +154,13 @@ static ssize_t qemu_net_queue_deliver(NetQueue *queue,
                                       size_t size)
 {
     ssize_t ret = -1;
+    struct iovec iov = {
+        .iov_base = (void *)data,
+        .iov_len = size
+    };
 
     queue->delivering = 1;
-    ret = qemu_deliver_packet(sender, flags, data, size, queue->opaque);
+    ret = queue->deliver(sender, flags, &iov, 1, queue->opaque);
     queue->delivering = 0;
 
     return ret;
@@ -169,7 +175,7 @@ static ssize_t qemu_net_queue_deliver_iov(NetQueue *queue,
     ssize_t ret = -1;
 
     queue->delivering = 1;
-    ret = qemu_deliver_packet_iov(sender, flags, iov, iovcnt, queue->opaque);
+    ret = queue->deliver(sender, flags, iov, iovcnt, queue->opaque);
     queue->delivering = 0;
 
     return ret;
diff --git a/net/slirp.c b/net/slirp.c
index 7657b38fd..f505570ad 100644
--- a/net/slirp.c
+++ b/net/slirp.c
@@ -746,8 +746,8 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     const NetdevUserOptions *user;
     const char **dnssearch;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_USER);
-    user = opts->user;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_USER);
+    user = opts->u.user;
 
     vnet = user->has_net ? g_strdup(user->net) :
            user->has_ip  ? g_strdup_printf("%s/24", user->ip) :
diff --git a/net/socket.c b/net/socket.c
index b1e3b1c8d..e8605d4de 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -706,8 +706,8 @@ int net_init_socket(const NetClientOptions *opts, const char *name,
     Error *err = NULL;
     const NetdevSocketOptions *sock;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_SOCKET);
-    sock = opts->socket;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_SOCKET);
+    sock = opts->u.socket;
 
     if (sock->has_fd + sock->has_listen + sock->has_connect + sock->has_mcast +
         sock->has_udp != 1) {
diff --git a/net/tap-bsd.c b/net/tap-bsd.c
index 7028d9be9..0103a97bf 100644
--- a/net/tap-bsd.c
+++ b/net/tap-bsd.c
@@ -109,8 +109,7 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
 
 #define PATH_NET_TAP "/dev/tap"
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required, Error **errp)
+static int tap_open_clone(char *ifname, int ifname_size, Error **errp)
 {
     int fd, s, ret;
     struct ifreq ifr;
@@ -126,7 +125,8 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
     ret = ioctl(fd, TAPGIFNAME, (void *)&ifr);
     if (ret < 0) {
         error_setg_errno(errp, errno, "could not get tap interface name");
-        goto error;
+        close(fd);
+        return -1;
     }
 
     if (ifname[0] != '\0') {
@@ -135,19 +135,47 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
         if (s < 0) {
             error_setg_errno(errp, errno,
                              "could not open socket to set interface name");
-            goto error;
+            close(fd);
+            return -1;
         }
         ifr.ifr_data = ifname;
         ret = ioctl(s, SIOCSIFNAME, (void *)&ifr);
         close(s);
         if (ret < 0) {
             error_setg(errp, "could not set tap interface name");
-            goto error;
+            close(fd);
+            return -1;
         }
     } else {
         pstrcpy(ifname, ifname_size, ifr.ifr_name);
     }
 
+    return fd;
+}
+
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required, Error **errp)
+{
+    int fd = -1;
+
+    /* If the specified tap device already exists just use it. */
+    if (ifname[0] != '\0') {
+        char dname[100];
+        snprintf(dname, sizeof dname, "/dev/%s", ifname);
+        TFR(fd = open(dname, O_RDWR));
+        if (fd < 0 && errno != ENOENT) {
+            error_setg_errno(errp, errno, "could not open %s", dname);
+            return -1;
+        }
+    }
+
+    if (fd < 0) {
+        /* Tap device not specified or does not exist. */
+        if ((fd = tap_open_clone(ifname, ifname_size, errp)) < 0) {
+            return -1;
+        }
+    }
+
     if (*vnet_hdr) {
         /* BSD doesn't have IFF_VNET_HDR */
         *vnet_hdr = 0;
diff --git a/net/tap-linux.c b/net/tap-linux.c
index 394f2a646..5bd9d2189 100644
--- a/net/tap-linux.c
+++ b/net/tap-linux.c
@@ -211,7 +211,7 @@ int tap_fd_set_vnet_le(int fd, int is_le)
         return -errno;
     }
 
-    error_report("TUNSETVNETLE ioctl() failed: %s.\n", strerror(errno));
+    error_report("TUNSETVNETLE ioctl() failed: %s.", strerror(errno));
     abort();
 }
 
@@ -228,7 +228,7 @@ int tap_fd_set_vnet_be(int fd, int is_be)
         return -errno;
     }
 
-    error_report("TUNSETVNETBE ioctl() failed: %s.\n", strerror(errno));
+    error_report("TUNSETVNETBE ioctl() failed: %s.", strerror(errno));
     abort();
 }
 
diff --git a/net/tap-win32.c b/net/tap-win32.c
index 625d53c64..7fddb20b5 100644
--- a/net/tap-win32.c
+++ b/net/tap-win32.c
@@ -77,7 +77,12 @@
 
 //#define DEBUG_TAP_WIN32
 
-#define TUN_ASYNCHRONOUS_WRITES 1
+/* FIXME: The asynch write path appears to be broken at
+ * present. WriteFile() ignores the lpNumberOfBytesWritten parameter
+ * for overlapped writes, with the result we return zero bytes sent,
+ * and after handling a single packet, receive is disabled for this
+ * interface. */
+/* #define TUN_ASYNCHRONOUS_WRITES 1 */
 
 #define TUN_BUFFER_SIZE 1560
 #define TUN_MAX_BUFFER_COUNT 32
@@ -356,7 +361,8 @@ static int get_device_guid(
                 &len);
 
             if (status != ERROR_SUCCESS || name_type != REG_SZ) {
-                    return -1;
+                ++i;
+                continue;
             }
             else {
                 if (is_tap_win32_dev(enum_name)) {
@@ -460,27 +466,48 @@ static int tap_win32_write(tap_win32_overlapped_t *overlapped,
     BOOL result;
     DWORD error;
 
+#ifdef TUN_ASYNCHRONOUS_WRITES
     result = GetOverlappedResult( overlapped->handle, &overlapped->write_overlapped,
                                   &write_size, FALSE);
 
     if (!result && GetLastError() == ERROR_IO_INCOMPLETE)
         WaitForSingleObject(overlapped->write_event, INFINITE);
+#endif
 
     result = WriteFile(overlapped->handle, buffer, size,
                        &write_size, &overlapped->write_overlapped);
 
+#ifdef TUN_ASYNCHRONOUS_WRITES
+    /* FIXME: we can't sensibly set write_size here, without waiting
+     * for the IO to complete! Moreover, we can't return zero,
+     * because that will disable receive on this interface, and we
+     * also can't assume it will succeed and return the full size,
+     * because that will result in the buffer being reclaimed while
+     * the IO is in progress. */
+#error Async writes are broken. Please disable TUN_ASYNCHRONOUS_WRITES.
+#else /* !TUN_ASYNCHRONOUS_WRITES */
     if (!result) {
-        switch (error = GetLastError())
-        {
-        case ERROR_IO_PENDING:
-#ifndef TUN_ASYNCHRONOUS_WRITES
-            WaitForSingleObject(overlapped->write_event, INFINITE);
-#endif
-            break;
-        default:
-            return -1;
+        error = GetLastError();
+        if (error == ERROR_IO_PENDING) {
+            result = GetOverlappedResult(overlapped->handle,
+                                         &overlapped->write_overlapped,
+                                         &write_size, TRUE);
         }
     }
+#endif
+
+    if (!result) {
+#ifdef DEBUG_TAP_WIN32
+        LPTSTR msgbuf;
+        error = GetLastError();
+        FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_FROM_SYSTEM,
+                      NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                      &msgbuf, 0, NULL);
+        fprintf(stderr, "Tap-Win32: Error WriteFile %d - %s\n", error, msgbuf);
+        LocalFree(msgbuf);
+#endif
+        return 0;
+    }
 
     return write_size;
 }
@@ -767,8 +794,8 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
     /* FIXME error_setg(errp, ...) on failure */
     const NetdevTapOptions *tap;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_TAP);
-    tap = opts->tap;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    tap = opts->u.tap;
 
     if (!tap->has_ifname) {
         error_report("tap: no interface name");
diff --git a/net/tap.c b/net/tap.c
index bd01590e8..85c4142d1 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -565,8 +565,8 @@ int net_init_bridge(const NetClientOptions *opts, const char *name,
     TAPState *s;
     int fd, vnet_hdr;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_BRIDGE);
-    bridge = opts->bridge;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_BRIDGE);
+    bridge = opts->u.bridge;
 
     helper = bridge->has_helper ? bridge->helper : DEFAULT_BRIDGE_HELPER;
     br     = bridge->has_br     ? bridge->br     : DEFAULT_BRIDGE_INTERFACE;
@@ -728,8 +728,8 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
     const char *vhostfdname;
     char ifname[128];
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_TAP);
-    tap = opts->tap;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    tap = opts->u.tap;
     queues = tap->has_queues ? tap->queues : 1;
     vhostfdname = tap->has_vhostfd ? tap->vhostfd : NULL;
 
diff --git a/net/vde.c b/net/vde.c
index dacaa64b4..4475d929e 100644
--- a/net/vde.c
+++ b/net/vde.c
@@ -115,8 +115,8 @@ int net_init_vde(const NetClientOptions *opts, const char *name,
     /* FIXME error_setg(errp, ...) on failure */
     const NetdevVdeOptions *vde;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_VDE);
-    vde = opts->vde;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_VDE);
+    vde = opts->u.vde;
 
     /* missing optional values have been initialized to "all bits zero" */
     if (net_vde_init(peer, "vde", name, vde->sock, vde->port, vde->group,
diff --git a/net/vhost-user.c b/net/vhost-user.c
index 93dcecd66..b368a9021 100644
--- a/net/vhost-user.c
+++ b/net/vhost-user.c
@@ -14,6 +14,8 @@
 #include "sysemu/char.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
+#include "qmp-commands.h"
+#include "trace.h"
 
 typedef struct VhostUserState {
     NetClientState nc;
@@ -39,37 +41,106 @@ static int vhost_user_running(VhostUserState *s)
     return (s->vhost_net) ? 1 : 0;
 }
 
-static int vhost_user_start(VhostUserState *s)
+static void vhost_user_stop(int queues, NetClientState *ncs[])
 {
-    VhostNetOptions options;
+    VhostUserState *s;
+    int i;
 
-    if (vhost_user_running(s)) {
-        return 0;
+    for (i = 0; i < queues; i++) {
+        assert (ncs[i]->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+
+        s = DO_UPCAST(VhostUserState, nc, ncs[i]);
+        if (!vhost_user_running(s)) {
+            continue;
+        }
+
+        if (s->vhost_net) {
+            vhost_net_cleanup(s->vhost_net);
+            s->vhost_net = NULL;
+        }
     }
+}
+
+static int vhost_user_start(int queues, NetClientState *ncs[])
+{
+    VhostNetOptions options;
+    VhostUserState *s;
+    int max_queues;
+    int i;
 
     options.backend_type = VHOST_BACKEND_TYPE_USER;
-    options.net_backend = &s->nc;
-    options.opaque = s->chr;
 
-    s->vhost_net = vhost_net_init(&options);
+    for (i = 0; i < queues; i++) {
+        assert (ncs[i]->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+
+        s = DO_UPCAST(VhostUserState, nc, ncs[i]);
+        if (vhost_user_running(s)) {
+            continue;
+        }
+
+        options.net_backend = ncs[i];
+        options.opaque      = s->chr;
+        s->vhost_net = vhost_net_init(&options);
+        if (!s->vhost_net) {
+            error_report("failed to init vhost_net for queue %d\n", i);
+            goto err;
+        }
+
+        if (i == 0) {
+            max_queues = vhost_net_get_max_queues(s->vhost_net);
+            if (queues > max_queues) {
+                error_report("you are asking more queues than "
+                             "supported: %d\n", max_queues);
+                goto err;
+            }
+        }
+    }
 
-    return vhost_user_running(s) ? 0 : -1;
+    return 0;
+
+err:
+    vhost_user_stop(i + 1, ncs);
+    return -1;
 }
 
-static void vhost_user_stop(VhostUserState *s)
+static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
+                                  size_t size)
 {
-    if (vhost_user_running(s)) {
-        vhost_net_cleanup(s->vhost_net);
+    /* In case of RARP (message size is 60) notify backup to send a fake RARP.
+       This fake RARP will be sent by backend only for guest
+       without GUEST_ANNOUNCE capability.
+     */
+    if (size == 60) {
+        VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
+        int r;
+        static int display_rarp_failure = 1;
+        char mac_addr[6];
+
+        /* extract guest mac address from the RARP message */
+        memcpy(mac_addr, &buf[6], 6);
+
+        r = vhost_net_notify_migration_done(s->vhost_net, mac_addr);
+
+        if ((r != 0) && (display_rarp_failure)) {
+            fprintf(stderr,
+                    "Vhost user backend fails to broadcast fake RARP\n");
+            fflush(stderr);
+            display_rarp_failure = 0;
+        }
     }
 
-    s->vhost_net = 0;
+    return size;
 }
 
 static void vhost_user_cleanup(NetClientState *nc)
 {
     VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
 
-    vhost_user_stop(s);
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
     qemu_purge_queued_packets(nc);
 }
 
@@ -90,64 +161,64 @@ static bool vhost_user_has_ufo(NetClientState *nc)
 static NetClientInfo net_vhost_user_info = {
         .type = NET_CLIENT_OPTIONS_KIND_VHOST_USER,
         .size = sizeof(VhostUserState),
+        .receive = vhost_user_receive,
         .cleanup = vhost_user_cleanup,
         .has_vnet_hdr = vhost_user_has_vnet_hdr,
         .has_ufo = vhost_user_has_ufo,
 };
 
-static void net_vhost_link_down(VhostUserState *s, bool link_down)
-{
-    s->nc.link_down = link_down;
-
-    if (s->nc.peer) {
-        s->nc.peer->link_down = link_down;
-    }
-
-    if (s->nc.info->link_status_changed) {
-        s->nc.info->link_status_changed(&s->nc);
-    }
-
-    if (s->nc.peer && s->nc.peer->info->link_status_changed) {
-        s->nc.peer->info->link_status_changed(s->nc.peer);
-    }
-}
-
 static void net_vhost_user_event(void *opaque, int event)
 {
-    VhostUserState *s = opaque;
-
+    const char *name = opaque;
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    VhostUserState *s;
+    Error *err = NULL;
+    int queues;
+
+    queues = qemu_find_net_clients_except(name, ncs,
+                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          MAX_QUEUE_NUM);
+    s = DO_UPCAST(VhostUserState, nc, ncs[0]);
+    trace_vhost_user_event(s->chr->label, event);
     switch (event) {
     case CHR_EVENT_OPENED:
-        vhost_user_start(s);
-        net_vhost_link_down(s, false);
-        error_report("chardev \"%s\" went up", s->chr->label);
+        if (vhost_user_start(queues, ncs) < 0) {
+            exit(1);
+        }
+        qmp_set_link(name, true, &err);
         break;
     case CHR_EVENT_CLOSED:
-        net_vhost_link_down(s, true);
-        vhost_user_stop(s);
-        error_report("chardev \"%s\" went down", s->chr->label);
+        qmp_set_link(name, false, &err);
+        vhost_user_stop(queues, ncs);
         break;
     }
+
+    if (err) {
+        error_report_err(err);
+    }
 }
 
 static int net_vhost_user_init(NetClientState *peer, const char *device,
-                               const char *name, CharDriverState *chr)
+                               const char *name, CharDriverState *chr,
+                               int queues)
 {
     NetClientState *nc;
     VhostUserState *s;
+    int i;
 
-    nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name);
+    for (i = 0; i < queues; i++) {
+        nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name);
 
-    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user to %s",
-             chr->label);
+        snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user%d to %s",
+                 i, chr->label);
 
-    s = DO_UPCAST(VhostUserState, nc, nc);
+        nc->queue_index = i;
 
-    /* We don't provide a receive callback */
-    s->nc.receive_disabled = 1;
-    s->chr = chr;
+        s = DO_UPCAST(VhostUserState, nc, nc);
+        s->chr = chr;
+    }
 
-    qemu_chr_add_handlers(s->chr, NULL, NULL, net_vhost_user_event, s);
+    qemu_chr_add_handlers(chr, NULL, NULL, net_vhost_user_event, (void*)name);
 
     return 0;
 }
@@ -226,11 +297,12 @@ static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
 int net_init_vhost_user(const NetClientOptions *opts, const char *name,
                         NetClientState *peer, Error **errp)
 {
+    int queues;
     const NetdevVhostUserOptions *vhost_user_opts;
     CharDriverState *chr;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
-    vhost_user_opts = opts->vhost_user;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+    vhost_user_opts = opts->u.vhost_user;
 
     chr = net_vhost_parse_chardev(vhost_user_opts, errp);
     if (!chr) {
@@ -243,6 +315,12 @@ int net_init_vhost_user(const NetClientOptions *opts, const char *name,
         return -1;
     }
 
+    queues = vhost_user_opts->has_queues ? vhost_user_opts->queues : 1;
+    if (queues < 1) {
+        error_setg(errp,
+                   "vhost-user number of queues must be bigger than zero");
+        return -1;
+    }
 
-    return net_vhost_user_init(peer, "vhost_user", name, chr);
+    return net_vhost_user_init(peer, "vhost_user", name, chr, queues);
 }
diff --git a/numa.c b/numa.c
index 402804bdf..fdfe2949d 100644
--- a/numa.c
+++ b/numa.c
@@ -226,9 +226,9 @@ static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
         goto error;
     }
 
-    switch (object->kind) {
+    switch (object->type) {
     case NUMA_OPTIONS_KIND_NODE:
-        numa_node_parse(object->node, opts, &err);
+        numa_node_parse(object->u.node, opts, &err);
         if (err) {
             goto error;
         }
@@ -280,7 +280,7 @@ static void validate_numa_cpus(void)
             bitmap_and(seen_cpus, seen_cpus,
                        numa_info[i].node_cpu, MAX_CPUMASK_BITS);
             error_report("CPU(s) present in multiple NUMA nodes: %s",
-                         enumerate_cpus(seen_cpus, max_cpus));;
+                         enumerate_cpus(seen_cpus, max_cpus));
             exit(EXIT_FAILURE);
         }
         bitmap_or(seen_cpus, seen_cpus,
@@ -424,14 +424,14 @@ static void allocate_system_memory_nonnuma(MemoryRegion *mr, Object *owner,
          */
         if (err) {
             error_report_err(err);
-            memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
+            memory_region_init_ram(mr, owner, name, ram_size, &error_fatal);
         }
 #else
         fprintf(stderr, "-mem-path not supported on this host\n");
         exit(1);
 #endif
     } else {
-        memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
+        memory_region_init_ram(mr, owner, name, ram_size, &error_fatal);
     }
     vmstate_register_ram_global(mr);
 }
@@ -487,9 +487,9 @@ static void numa_stat_memory_devices(uint64_t node_mem[])
         MemoryDeviceInfo *value = info->value;
 
         if (value) {
-            switch (value->kind) {
+            switch (value->type) {
             case MEMORY_DEVICE_INFO_KIND_DIMM:
-                node_mem[value->dimm->node] += value->dimm->size;
+                node_mem[value->u.dimm->node] += value->u.dimm->size;
                 break;
             default:
                 break;
diff --git a/os-win32.c b/os-win32.c
index c0daf8e18..cc09196d8 100644
--- a/os-win32.c
+++ b/os-win32.c
@@ -26,7 +26,6 @@
 #include <mmsystem.h>
 #include <unistd.h>
 #include <fcntl.h>
-#include <signal.h>
 #include <time.h>
 #include <errno.h>
 #include <sys/time.h>
diff --git a/page_cache.c b/page_cache.c
index cf8878d1d..a9eb0769b 100644
--- a/page_cache.c
+++ b/page_cache.c
@@ -15,7 +15,6 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <strings.h>
 #include <string.h>
 #include <sys/time.h>
 #include <sys/types.h>
diff --git a/pc-bios/README b/pc-bios/README
index 05cf0421b..d260c1bbb 100644
--- a/pc-bios/README
+++ b/pc-bios/README
@@ -17,7 +17,7 @@
 - SLOF (Slimline Open Firmware) is a free IEEE 1275 Open Firmware
   implementation for certain IBM POWER hardware.  The sources are at
   https://github.com/aik/SLOF, and the image currently in qemu is
-  built from git tag qemu-slof-20150429.
+  built from git tag qemu-slof-20151103.
 
 - sgabios (the Serial Graphics Adapter option ROM) provides a means for
   legacy x86 software to communicate with an attached serial console as
diff --git a/pc-bios/efi-e1000.rom b/pc-bios/efi-e1000.rom
index 4e29d9d1a..4bc89a3d2 100644
--- a/pc-bios/efi-e1000.rom
+++ b/pc-bios/efi-e1000.rom
diff --git a/pc-bios/efi-eepro100.rom b/pc-bios/efi-eepro100.rom
index 2a92d6f32..85b7f9bc7 100644
--- a/pc-bios/efi-eepro100.rom
+++ b/pc-bios/efi-eepro100.rom
diff --git a/pc-bios/efi-ne2k_pci.rom b/pc-bios/efi-ne2k_pci.rom
index 636601700..ebafd8452 100644
--- a/pc-bios/efi-ne2k_pci.rom
+++ b/pc-bios/efi-ne2k_pci.rom
diff --git a/pc-bios/efi-pcnet.rom b/pc-bios/efi-pcnet.rom
index a61f586ee..6f1972375 100644
--- a/pc-bios/efi-pcnet.rom
+++ b/pc-bios/efi-pcnet.rom
diff --git a/pc-bios/efi-rtl8139.rom b/pc-bios/efi-rtl8139.rom
index c9c77ea7f..086551b76 100644
--- a/pc-bios/efi-rtl8139.rom
+++ b/pc-bios/efi-rtl8139.rom
diff --git a/pc-bios/efi-virtio.rom b/pc-bios/efi-virtio.rom
index eec279099..140c6806d 100644
--- a/pc-bios/efi-virtio.rom
+++ b/pc-bios/efi-virtio.rom
diff --git a/pc-bios/openbios-ppc b/pc-bios/openbios-ppc
index 540e45a74..a419bfd03 100644
--- a/pc-bios/openbios-ppc
+++ b/pc-bios/openbios-ppc
diff --git a/pc-bios/openbios-sparc32 b/pc-bios/openbios-sparc32
index 0da11883b..cf5278703 100644
--- a/pc-bios/openbios-sparc32
+++ b/pc-bios/openbios-sparc32
diff --git a/pc-bios/openbios-sparc64 b/pc-bios/openbios-sparc64
index 9bf3ce545..08eecdf22 100644
--- a/pc-bios/openbios-sparc64
+++ b/pc-bios/openbios-sparc64
diff --git a/pc-bios/s390-ccw.img b/pc-bios/s390-ccw.img
index f64380a97..bd8f21050 100644
--- a/pc-bios/s390-ccw.img
+++ b/pc-bios/s390-ccw.img
diff --git a/pc-bios/s390-ccw/Makefile b/pc-bios/s390-ccw/Makefile
index 746603a31..11c5dd479 100644
--- a/pc-bios/s390-ccw/Makefile
+++ b/pc-bios/s390-ccw/Makefile
@@ -10,7 +10,8 @@ $(call set-vpath, $(SRC_PATH)/pc-bios/s390-ccw)
 .PHONY : all clean build-all
 
 OBJECTS = start.o main.o bootmap.o sclp-ascii.o virtio.o
-CFLAGS += -fPIE -fno-stack-protector -ffreestanding -fno-delete-null-pointer-checks
+CFLAGS += -fPIE -fno-stack-protector -ffreestanding -march=z900
+CFLAGS += -fno-delete-null-pointer-checks -msoft-float
 LDFLAGS += -Wl,-pie -nostdlib
 
 build-all: s390-ccw.img
diff --git a/pc-bios/s390-ccw/bootmap.c b/pc-bios/s390-ccw/bootmap.c
index b678d5ebb..415508b27 100644
--- a/pc-bios/s390-ccw/bootmap.c
+++ b/pc-bios/s390-ccw/bootmap.c
@@ -445,6 +445,206 @@ static void ipl_scsi(void)
 }
 
 /***********************************************************************
+ * IPL El Torito ISO9660 image or DVD
+ */
+
+static bool is_iso_bc_entry_compatible(IsoBcSection *s)
+{
+    uint8_t *magic_sec = (uint8_t *)(sec + ISO_SECTOR_SIZE);
+
+    if (s->unused || !s->sector_count) {
+        return false;
+    }
+    read_iso_sector(bswap32(s->load_rba), magic_sec,
+                    "Failed to read image sector 0");
+
+    /* Checking bytes 8 - 32 for S390 Linux magic */
+    return !_memcmp(magic_sec + 8, linux_s390_magic, 24);
+}
+
+/* Location of the current sector of the directory */
+static uint32_t sec_loc[ISO9660_MAX_DIR_DEPTH];
+/* Offset in the current sector of the directory */
+static uint32_t sec_offset[ISO9660_MAX_DIR_DEPTH];
+/* Remained directory space in bytes */
+static uint32_t dir_rem[ISO9660_MAX_DIR_DEPTH];
+
+static inline uint32_t iso_get_file_size(uint32_t load_rba)
+{
+    IsoVolDesc *vd = (IsoVolDesc *)sec;
+    IsoDirHdr *cur_record = &vd->vd.primary.rootdir;
+    uint8_t *temp = sec + ISO_SECTOR_SIZE;
+    int level = 0;
+
+    read_iso_sector(ISO_PRIMARY_VD_SECTOR, sec,
+                    "Failed to read ISO primary descriptor");
+    sec_loc[0] = iso_733_to_u32(cur_record->ext_loc);
+    dir_rem[0] = 0;
+    sec_offset[0] = 0;
+
+    while (level >= 0) {
+        IPL_assert(sec_offset[level] <= ISO_SECTOR_SIZE,
+                   "Directory tree structure violation");
+
+        cur_record = (IsoDirHdr *)(temp + sec_offset[level]);
+
+        if (sec_offset[level] == 0) {
+            read_iso_sector(sec_loc[level], temp,
+                            "Failed to read ISO directory");
+            if (dir_rem[level] == 0) {
+                /* Skip self and parent records */
+                dir_rem[level] = iso_733_to_u32(cur_record->data_len) -
+                                 cur_record->dr_len;
+                sec_offset[level] += cur_record->dr_len;
+
+                cur_record = (IsoDirHdr *)(temp + sec_offset[level]);
+                dir_rem[level] -= cur_record->dr_len;
+                sec_offset[level] += cur_record->dr_len;
+                continue;
+            }
+        }
+
+        if (!cur_record->dr_len || sec_offset[level] == ISO_SECTOR_SIZE) {
+            /* Zero-padding and/or the end of current sector */
+            dir_rem[level] -= ISO_SECTOR_SIZE - sec_offset[level];
+            sec_offset[level] = 0;
+            sec_loc[level]++;
+        } else {
+            /* The directory record is valid */
+            if (load_rba == iso_733_to_u32(cur_record->ext_loc)) {
+                return iso_733_to_u32(cur_record->data_len);
+            }
+
+            dir_rem[level] -= cur_record->dr_len;
+            sec_offset[level] += cur_record->dr_len;
+
+            if (cur_record->file_flags & 0x2) {
+                /* Subdirectory */
+                if (level == ISO9660_MAX_DIR_DEPTH - 1) {
+                    sclp_print("ISO-9660 directory depth limit exceeded\n");
+                } else {
+                    level++;
+                    sec_loc[level] = iso_733_to_u32(cur_record->ext_loc);
+                    sec_offset[level] = 0;
+                    dir_rem[level] = 0;
+                    continue;
+                }
+            }
+        }
+
+        if (dir_rem[level] == 0) {
+            /* Nothing remaining */
+            level--;
+            read_iso_sector(sec_loc[level], temp,
+                            "Failed to read ISO directory");
+        }
+    }
+
+    return 0;
+}
+
+static void load_iso_bc_entry(IsoBcSection *load)
+{
+    IsoBcSection s = *load;
+    /*
+     * According to spec, extent for each file
+     * is padded and ISO_SECTOR_SIZE bytes aligned
+     */
+    uint32_t blks_to_load = bswap16(s.sector_count) >> ET_SECTOR_SHIFT;
+    uint32_t real_size = iso_get_file_size(bswap32(s.load_rba));
+
+    if (real_size) {
+        /* Round up blocks to load */
+        blks_to_load = (real_size + ISO_SECTOR_SIZE - 1) / ISO_SECTOR_SIZE;
+        sclp_print("ISO boot image size verified\n");
+    } else {
+        sclp_print("ISO boot image size could not be verified\n");
+    }
+
+    read_iso_boot_image(bswap32(s.load_rba),
+                        (void *)((uint64_t)bswap16(s.load_segment)),
+                        blks_to_load);
+
+    /* Trying to get PSW at zero address */
+    if (*((uint64_t *)0) & IPL_PSW_MASK) {
+        jump_to_IPL_code((*((uint64_t *)0)) & 0x7fffffff);
+    }
+
+    /* Try default linux start address */
+    jump_to_IPL_code(KERN_IMAGE_START);
+}
+
+static uint32_t find_iso_bc(void)
+{
+    IsoVolDesc *vd = (IsoVolDesc *)sec;
+    uint32_t block_num = ISO_PRIMARY_VD_SECTOR;
+
+    if (virtio_read_many(block_num++, sec, 1)) {
+        /* If primary vd cannot be read, there is no boot catalog */
+        return 0;
+    }
+
+    while (is_iso_vd_valid(vd) && vd->type != VOL_DESC_TERMINATOR) {
+        if (vd->type == VOL_DESC_TYPE_BOOT) {
+            IsoVdElTorito *et = &vd->vd.boot;
+
+            if (!_memcmp(&et->el_torito[0], el_torito_magic, 32)) {
+                return bswap32(et->bc_offset);
+            }
+        }
+        read_iso_sector(block_num++, sec,
+                        "Failed to read ISO volume descriptor");
+    }
+
+    return 0;
+}
+
+static IsoBcSection *find_iso_bc_entry(void)
+{
+    IsoBcEntry *e = (IsoBcEntry *)sec;
+    uint32_t offset = find_iso_bc();
+    int i;
+
+    if (!offset) {
+        return NULL;
+    }
+
+    read_iso_sector(offset, sec, "Failed to read El Torito boot catalog");
+
+    if (!is_iso_bc_valid(e)) {
+        /* The validation entry is mandatory */
+        virtio_panic("No valid boot catalog found!\n");
+        return NULL;
+    }
+
+    /*
+     * Each entry has 32 bytes size, so one sector cannot contain > 64 entries.
+     * We consider only boot catalogs with no more than 64 entries.
+     */
+    for (i = 1; i < ISO_BC_ENTRY_PER_SECTOR; i++) {
+        if (e[i].id == ISO_BC_BOOTABLE_SECTION) {
+            if (is_iso_bc_entry_compatible(&e[i].body.sect)) {
+                return &e[i].body.sect;
+            }
+        }
+    }
+
+    virtio_panic("No suitable boot entry found on ISO-9660 media!\n");
+
+    return NULL;
+}
+
+static void ipl_iso_el_torito(void)
+{
+    IsoBcSection *s = find_iso_bc_entry();
+
+    if (s) {
+        load_iso_bc_entry(s);
+        /* no return */
+    }
+}
+
+/***********************************************************************
  * IPL starts here
  */
 
@@ -463,6 +663,12 @@ void zipl_load(void)
         ipl_scsi(); /* no return */
     }
 
+    /* Check if we can boot as ISO media */
+    if (virtio_guessed_disk_nature()) {
+        virtio_assume_iso9660();
+    }
+    ipl_iso_el_torito();
+
     /* We have failed to follow the SCSI scheme, so */
     if (virtio_guessed_disk_nature()) {
         sclp_print("Using guessed DASD geometry.\n");
diff --git a/pc-bios/s390-ccw/bootmap.h b/pc-bios/s390-ccw/bootmap.h
index ab132e357..f98765b84 100644
--- a/pc-bios/s390-ccw/bootmap.h
+++ b/pc-bios/s390-ccw/bootmap.h
@@ -341,4 +341,210 @@ static inline bool magic_match(const void *data, const void *magic)
     return *((uint32_t *)data) == *((uint32_t *)magic);
 }
 
+static inline int _memcmp(const void *s1, const void *s2, size_t n)
+{
+    int i;
+    const uint8_t *p1 = s1, *p2 = s2;
+
+    for (i = 0; i < n; i++) {
+        if (p1[i] != p2[i]) {
+            return p1[i] > p2[i] ? 1 : -1;
+        }
+    }
+
+    return 0;
+}
+
+/* from include/qemu/bswap.h */
+
+/* El Torito is always little-endian */
+static inline uint16_t bswap16(uint16_t x)
+{
+    return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8);
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return ((x & 0x000000ffU) << 24) | ((x & 0x0000ff00U) <<  8) |
+           ((x & 0x00ff0000U) >>  8) | ((x & 0xff000000U) >> 24);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return ((x & 0x00000000000000ffULL) << 56) |
+           ((x & 0x000000000000ff00ULL) << 40) |
+           ((x & 0x0000000000ff0000ULL) << 24) |
+           ((x & 0x00000000ff000000ULL) <<  8) |
+           ((x & 0x000000ff00000000ULL) >>  8) |
+           ((x & 0x0000ff0000000000ULL) >> 24) |
+           ((x & 0x00ff000000000000ULL) >> 40) |
+           ((x & 0xff00000000000000ULL) >> 56);
+}
+
+static inline uint32_t iso_733_to_u32(uint64_t x)
+{
+    return (uint32_t)x;
+}
+
+#define ISO_SECTOR_SIZE 2048
+/* El Torito specifies boot image size in 512 byte blocks */
+#define ET_SECTOR_SHIFT 2
+#define KERN_IMAGE_START 0x010000UL
+#define PSW_MASK_64 0x0000000100000000ULL
+#define PSW_MASK_32 0x0000000080000000ULL
+#define IPL_PSW_MASK (PSW_MASK_32 | PSW_MASK_64)
+
+#define ISO_PRIMARY_VD_SECTOR 16
+
+static inline void read_iso_sector(uint32_t block_offset, void *buf,
+                                   const char *errmsg)
+{
+    IPL_assert(virtio_read_many(block_offset, buf, 1) == 0, errmsg);
+}
+
+static inline void read_iso_boot_image(uint32_t block_offset, void *load_addr,
+                                       uint32_t blks_to_load)
+{
+    IPL_assert(virtio_read_many(block_offset, load_addr, blks_to_load) == 0,
+               "Failed to read boot image!");
+}
+
+const uint8_t el_torito_magic[] = "EL TORITO SPECIFICATION"
+                                  "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
+
+#define ISO9660_MAX_DIR_DEPTH 8
+
+typedef struct IsoDirHdr {
+    uint8_t dr_len;
+    uint8_t ear_len;
+    uint64_t ext_loc;
+    uint64_t data_len;
+    uint8_t recording_datetime[7];
+    uint8_t file_flags;
+    uint8_t file_unit_size;
+    uint8_t gap_size;
+    uint32_t vol_seqnum;
+    uint8_t fileid_len;
+} __attribute__((packed)) IsoDirHdr;
+
+typedef struct IsoVdElTorito {
+    uint8_t el_torito[32]; /* must contain el_torito_magic value */
+    uint8_t unused0[32];
+    uint32_t bc_offset;
+    uint8_t unused1[1974];
+} __attribute__((packed)) IsoVdElTorito;
+
+typedef struct IsoVdPrimary {
+    uint8_t unused1;
+    uint8_t sys_id[32];
+    uint8_t vol_id[32];
+    uint8_t unused2[8];
+    uint64_t vol_space_size;
+    uint8_t unused3[32];
+    uint32_t vol_set_size;
+    uint32_t vol_seqnum;
+    uint32_t log_block_size;
+    uint64_t path_table_size;
+    uint32_t l_path_table;
+    uint32_t opt_l_path_table;
+    uint32_t m_path_table;
+    uint32_t opt_m_path_table;
+    IsoDirHdr rootdir;
+    uint8_t root_null;
+    uint8_t reserved2[1858];
+} __attribute__((packed)) IsoVdPrimary;
+
+typedef struct IsoVolDesc {
+    uint8_t type;
+    uint8_t ident[5];
+    uint8_t version;
+    union {
+        IsoVdElTorito boot;
+        IsoVdPrimary primary;
+    } vd;
+} __attribute__((packed)) IsoVolDesc;
+
+const uint8_t vol_desc_magic[] = "CD001";
+#define VOL_DESC_TYPE_BOOT 0
+#define VOL_DESC_TYPE_PRIMARY 1
+#define VOL_DESC_TYPE_SUPPLEMENT 2
+#define VOL_DESC_TYPE_PARTITION 3
+#define VOL_DESC_TERMINATOR 255
+
+static inline bool is_iso_vd_valid(IsoVolDesc *vd)
+{
+    return !_memcmp(&vd->ident[0], vol_desc_magic, 5) &&
+           vd->version == 0x1 &&
+           vd->type <= VOL_DESC_TYPE_PARTITION;
+}
+
+typedef struct IsoBcValid {
+    uint8_t platform_id;
+    uint16_t reserved;
+    uint8_t id[24];
+    uint16_t checksum;
+    uint8_t key[2];
+} __attribute__((packed)) IsoBcValid;
+
+typedef struct IsoBcSection {
+    uint8_t boot_type;
+    uint16_t load_segment;
+    uint8_t sys_type;
+    uint8_t unused;
+    uint16_t sector_count;
+    uint32_t load_rba;
+    uint8_t selection[20];
+} __attribute__((packed)) IsoBcSection;
+
+typedef struct IsoBcHdr {
+    uint8_t platform_id;
+    uint16_t sect_num;
+    uint8_t id[28];
+} __attribute__((packed)) IsoBcHdr;
+
+/*
+ * Match two CCWs located after PSW and eight filler bytes.
+ * From libmagic and arch/s390/kernel/head.S.
+ */
+const uint8_t linux_s390_magic[] = "\x02\x00\x00\x18\x60\x00\x00\x50\x02\x00"
+                                   "\x00\x68\x60\x00\x00\x50\x40\x40\x40\x40"
+                                   "\x40\x40\x40\x40";
+
+typedef struct IsoBcEntry {
+    uint8_t id;
+    union {
+        IsoBcValid valid; /* id == 0x01 */
+        IsoBcSection sect; /* id == 0x88 || id == 0x0 */
+        IsoBcHdr hdr; /* id == 0x90 || id == 0x91 */
+    } body;
+} __attribute__((packed)) IsoBcEntry;
+
+#define ISO_BC_ENTRY_PER_SECTOR (ISO_SECTOR_SIZE / sizeof(IsoBcEntry))
+#define ISO_BC_HDR_VALIDATION 0x01
+#define ISO_BC_BOOTABLE_SECTION 0x88
+#define ISO_BC_MAGIC_55 0x55
+#define ISO_BC_MAGIC_AA 0xaa
+#define ISO_BC_PLATFORM_X86 0x0
+#define ISO_BC_PLATFORM_PPC 0x1
+#define ISO_BC_PLATFORM_MAC 0x2
+
+static inline bool is_iso_bc_valid(IsoBcEntry *e)
+{
+    IsoBcValid *v = &e->body.valid;
+
+    if (e->id != ISO_BC_HDR_VALIDATION) {
+        return false;
+    }
+
+    if (v->platform_id != ISO_BC_PLATFORM_X86 &&
+        v->platform_id != ISO_BC_PLATFORM_PPC &&
+        v->platform_id != ISO_BC_PLATFORM_MAC) {
+        return false;
+    }
+
+    return v->key[0] == ISO_BC_MAGIC_55 &&
+           v->key[1] == ISO_BC_MAGIC_AA &&
+           v->reserved == 0x0;
+}
+
 #endif /* _PC_BIOS_S390_CCW_BOOTMAP_H */
diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c
index 584d4a276..d5fe4cea1 100644
--- a/pc-bios/s390-ccw/main.c
+++ b/pc-bios/s390-ccw/main.c
@@ -38,40 +38,56 @@ void virtio_panic(const char *string)
     while (1) { }
 }
 
+static bool find_dev(struct schib *schib, int dev_no)
+{
+    int i, r;
+
+    for (i = 0; i < 0x10000; i++) {
+        blk_schid.sch_no = i;
+        r = stsch_err(blk_schid, schib);
+        if ((r == 3) || (r == -EIO)) {
+            break;
+        }
+        if (!schib->pmcw.dnv) {
+            continue;
+        }
+        if (!virtio_is_blk(blk_schid)) {
+            continue;
+        }
+        if ((dev_no < 0) || (schib->pmcw.dev == dev_no)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 static void virtio_setup(uint64_t dev_info)
 {
     struct schib schib;
-    int i;
-    int r;
+    int ssid;
     bool found = false;
-    bool check_devno = false;
-    uint16_t dev_no = -1;
+    uint16_t dev_no;
+
+    /*
+     * We unconditionally enable mss support. In every sane configuration,
+     * this will succeed; and even if it doesn't, stsch_err() can deal
+     * with the consequences.
+     */
+    enable_mss_facility();
 
     if (dev_info != -1) {
-        check_devno = true;
         dev_no = dev_info & 0xffff;
         debug_print_int("device no. ", dev_no);
         blk_schid.ssid = (dev_info >> 16) & 0x3;
-        if (blk_schid.ssid != 0) {
-            debug_print_int("ssid ", blk_schid.ssid);
-            if (enable_mss_facility() != 0) {
-                virtio_panic("Failed to enable mss facility\n");
-            }
-        }
-    }
-
-    for (i = 0; i < 0x10000; i++) {
-        blk_schid.sch_no = i;
-        r = stsch_err(blk_schid, &schib);
-        if (r == 3) {
-            break;
-        }
-        if (schib.pmcw.dnv) {
-            if (!check_devno || (schib.pmcw.dev == dev_no)) {
-                if (virtio_is_blk(blk_schid)) {
-                    found = true;
-                    break;
-                }
+        debug_print_int("ssid ", blk_schid.ssid);
+        found = find_dev(&schib, dev_no);
+    } else {
+        for (ssid = 0; ssid < 0x3; ssid++) {
+            blk_schid.ssid = ssid;
+            found = find_dev(&schib, -1);
+            if (found) {
+                break;
             }
         }
     }
diff --git a/pc-bios/s390-ccw/virtio.c b/pc-bios/s390-ccw/virtio.c
index 57ff1b07e..87aed38a9 100644
--- a/pc-bios/s390-ccw/virtio.c
+++ b/pc-bios/s390-ccw/virtio.c
@@ -278,6 +278,13 @@ void virtio_assume_scsi(void)
     blk_cfg.physical_block_exp = 0;
 }
 
+void virtio_assume_iso9660(void)
+{
+    guessed_disk_nature = true;
+    blk_cfg.blk_size = 2048;
+    blk_cfg.physical_block_exp = 0;
+}
+
 void virtio_assume_eckd(void)
 {
     guessed_disk_nature = true;
diff --git a/pc-bios/s390-ccw/virtio.h b/pc-bios/s390-ccw/virtio.h
index c23466b8d..afa01a885 100644
--- a/pc-bios/s390-ccw/virtio.h
+++ b/pc-bios/s390-ccw/virtio.h
@@ -187,6 +187,7 @@ typedef struct VirtioBlkConfig {
 bool virtio_guessed_disk_nature(void);
 void virtio_assume_scsi(void);
 void virtio_assume_eckd(void);
+void virtio_assume_iso9660(void);
 
 extern bool virtio_disk_is_scsi(void);
 extern bool virtio_disk_is_eckd(void);
@@ -199,14 +200,9 @@ extern int virtio_read_many(ulong sector, void *load_addr, int sec_num);
 
 #define VIRTIO_SECTOR_SIZE 512
 
-static inline ulong virtio_eckd_sector_adjust(ulong sector)
-{
-     return sector * (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
-}
-
 static inline ulong virtio_sector_adjust(ulong sector)
 {
-    return virtio_disk_is_eckd() ? virtio_eckd_sector_adjust(sector) : sector;
+    return sector * (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
 }
 
 #endif /* VIRTIO_H */
diff --git a/pc-bios/slof.bin b/pc-bios/slof.bin
index 0398ac67b..90f30996f 100644
--- a/pc-bios/slof.bin
+++ b/pc-bios/slof.bin
diff --git a/po/de_DE.po b/po/de_DE.po
index dcdcf229d..cbba20308 100644
--- a/po/de_DE.po
+++ b/po/de_DE.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-17 20:39+0200\n"
+"POT-Creation-Date: 2015-12-10 13:29+0000\n"
 "PO-Revision-Date: 2012-02-28 16:00+0100\n"
 "Last-Translator: Kevin Wolf <kwolf@redhat.com>\n"
 "Language-Team: Deutsch <de@li.org>\n"
@@ -16,70 +16,70 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n!=1);\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:271
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Strg+Alt+G drücken, um Eingabegeräte freizugeben"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:275
 msgid " [Paused]"
 msgstr " [Angehalten]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1783
 msgid "_Pause"
 msgstr "_Angehalten"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1789
 msgid "_Reset"
 msgstr "_Reset"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1792
 msgid "Power _Down"
 msgstr "_Herunterfahren"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1798
 msgid "_Quit"
 msgstr "_Beenden"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1890
 msgid "_Fullscreen"
 msgstr "_Vollbild"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1904
 msgid "Zoom _In"
 msgstr "_Heranzoomen"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1911
 msgid "Zoom _Out"
 msgstr "_Wegzoomen"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1918
 msgid "Best _Fit"
 msgstr "_Einpassen"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1925
 msgid "Zoom To _Fit"
 msgstr "Auf _Fenstergröße skalieren"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1931
 msgid "Grab On _Hover"
 msgstr "Tastatur _automatisch einfangen"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1934
 msgid "_Grab Input"
 msgstr "_Eingabegeräte einfangen"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:1963
 msgid "Show _Tabs"
 msgstr "Reiter anzeigen"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:1966
 msgid "Detach Tab"
 msgstr "Reiter abtrennen"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:1978
 msgid "_Machine"
 msgstr "_Maschine"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:1983
 msgid "_View"
 msgstr "_Ansicht"
diff --git a/po/fr_FR.po b/po/fr_FR.po
index bbb5ef87b..8acbb6b06 100644
--- a/po/fr_FR.po
+++ b/po/fr_FR.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-28 23:14+0200\n"
+"POT-Creation-Date: 2015-12-10 13:29+0000\n"
 "PO-Revision-Date: 2014-07-28 23:25+0200\n"
 "Last-Translator: Aurelien Jarno <aurelien@aurel32.net>\n"
 "Language-Team: French <FR@li.org>\n"
@@ -17,70 +17,70 @@ msgstr ""
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 "X-Generator: Lokalize 1.4\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:271
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr "- Appuyer sur Ctrl+Alt+G pour arrêter la capture"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:275
 msgid " [Paused]"
 msgstr " [En pause]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1783
 msgid "_Pause"
 msgstr "_Pause"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1789
 msgid "_Reset"
 msgstr "_Réinitialiser"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1792
 msgid "Power _Down"
 msgstr "_Éteindre"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1798
 msgid "_Quit"
 msgstr "_Quitter"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1890
 msgid "_Fullscreen"
 msgstr "Mode _plein écran"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1904
 msgid "Zoom _In"
 msgstr "Zoom _avant"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1911
 msgid "Zoom _Out"
 msgstr "_Zoom arrière"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1918
 msgid "Best _Fit"
 msgstr "Zoom _idéal"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1925
 msgid "Zoom To _Fit"
 msgstr "Zoomer pour a_juster"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1931
 msgid "Grab On _Hover"
 msgstr "Capturer en _survolant"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1934
 msgid "_Grab Input"
 msgstr "_Capturer les entrées"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:1963
 msgid "Show _Tabs"
 msgstr "Montrer les _onglets"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:1966
 msgid "Detach Tab"
 msgstr "_Détacher l'onglet"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:1978
 msgid "_Machine"
 msgstr "_Machine"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:1983
 msgid "_View"
 msgstr "_Vue"
diff --git a/po/hu.po b/po/hu.po
index 401ed211a..ee1540bf6 100644
--- a/po/hu.po
+++ b/po/hu.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2015-12-10 13:29+0000\n"
 "PO-Revision-Date: 2013-05-06 20:42+0200\n"
 "Last-Translator: Ákos Kovács <akoskovacs@gmx.com>\n"
 "Language-Team: Hungarian <hu@li.org>\n"
@@ -15,47 +15,73 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:271
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Nyomj Ctrl+Alt+G-t a bemeneti eszközök elengedéséhez"
 
-#: ui/gtk.c:218
+#: ui/gtk.c:275
 msgid " [Paused]"
 msgstr " [Megállítva]"
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1783
 msgid "_Pause"
 msgstr "_Megállítás"
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1789
 msgid "_Reset"
 msgstr "Új_raindítás"
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1792
 msgid "Power _Down"
 msgstr "_Leállítás"
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1798
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1890
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1904
+#, fuzzy
+msgid "Zoom _In"
+msgstr "Ablakmérethez _igazítás"
+
+#: ui/gtk.c:1911
+#, fuzzy
+msgid "Zoom _Out"
+msgstr "Ablakmérethez _igazítás"
+
+#: ui/gtk.c:1918
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1925
 msgid "Zoom To _Fit"
 msgstr "Ablakmérethez _igazítás"
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1931
 msgid "Grab On _Hover"
 msgstr "Automatikus _elfogás"
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1934
 msgid "_Grab Input"
 msgstr "_Bemeneti eszközök megragadása"
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:1963
 msgid "Show _Tabs"
 msgstr "_Fülek megjelenítése"
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:1966
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:1978
 msgid "_Machine"
 msgstr "_Gép"
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:1983
 msgid "_View"
 msgstr "_Nézet"
 
diff --git a/po/it.po b/po/it.po
index e46fb3a8b..c141e2ff9 100644
--- a/po/it.po
+++ b/po/it.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-29 08:14+0200\n"
+"POT-Creation-Date: 2015-12-10 13:29+0000\n"
 "PO-Revision-Date: 2014-07-29 08:25+0200\n"
 "Last-Translator: Paolo Bonzini <pbonzini@redhat.com>\n"
 "Language-Team: Italian <it@li.org>\n"
@@ -16,66 +16,70 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:271
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Premere Ctrl+Alt+G per rilasciare l'input"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:275
 msgid " [Paused]"
 msgstr " [Pausa]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1783
 msgid "_Pause"
 msgstr "_Pausa"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1789
 msgid "_Reset"
 msgstr "_Reset"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1792
 msgid "Power _Down"
 msgstr "_Spegni"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1798
 msgid "_Quit"
 msgstr "_Esci"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1890
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1904
 msgid "Zoom _In"
 msgstr "_Aumenta zoom"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1911
 msgid "Zoom _Out"
 msgstr "_Riduci zoom"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1918
 msgid "Best _Fit"
 msgstr "A_nnulla zoom"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1925
 msgid "Zoom To _Fit"
 msgstr "Adatta alla _finestra"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1931
 msgid "Grab On _Hover"
 msgstr "Cattura _automatica input"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1934
 msgid "_Grab Input"
 msgstr "_Cattura input"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:1963
 msgid "Show _Tabs"
 msgstr "Mostra _tab"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:1966
 msgid "Detach Tab"
 msgstr "_Sposta in una nuova finestra"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:1978
 msgid "_Machine"
 msgstr "_Macchina virtuale"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:1983
 msgid "_View"
 msgstr "_Visualizza"
diff --git a/po/messages.po b/po/messages.po
index 26c76bce8..00f53f13a 100644
--- a/po/messages.po
+++ b/po/messages.po
@@ -5,9 +5,9 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: QEMU 1.5.50\n"
+"Project-Id-Version: QEMU 2.4.93\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2015-12-10 13:29+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -16,46 +16,70 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:271
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr ""
 
-#: ui/gtk.c:218
+#: ui/gtk.c:275
 msgid " [Paused]"
 msgstr ""
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1783
 msgid "_Pause"
 msgstr ""
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1789
 msgid "_Reset"
 msgstr ""
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1792
 msgid "Power _Down"
 msgstr ""
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1798
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1890
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1904
+msgid "Zoom _In"
+msgstr ""
+
+#: ui/gtk.c:1911
+msgid "Zoom _Out"
+msgstr ""
+
+#: ui/gtk.c:1918
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1925
 msgid "Zoom To _Fit"
 msgstr ""
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1931
 msgid "Grab On _Hover"
 msgstr ""
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1934
 msgid "_Grab Input"
 msgstr ""
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:1963
 msgid "Show _Tabs"
 msgstr ""
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:1966
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:1978
 msgid "_Machine"
 msgstr ""
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:1983
 msgid "_View"
 msgstr ""
diff --git a/po/tr.po b/po/tr.po
index d712ced9c..837776e89 100644
--- a/po/tr.po
+++ b/po/tr.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2015-12-10 13:29+0000\n"
 "PO-Revision-Date: 2013-04-22 18:35+0300\n"
 "Last-Translator: Ozan Çağlayan <ozancag@gmail.com>\n"
 "Language-Team: Türkçe <>\n"
@@ -17,46 +17,72 @@ msgstr ""
 "Plural-Forms: nplurals=1; plural=0;\n"
 "X-Generator: Gtranslator 2.91.6\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:271
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Yakalamayı durdurmak için Ctrl+Alt+G tuşlarına basın"
 
-#: ui/gtk.c:218
+#: ui/gtk.c:275
 msgid " [Paused]"
 msgstr " [Duraklatıldı]"
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1783
 msgid "_Pause"
 msgstr "_Duraklat"
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1789
 msgid "_Reset"
 msgstr "_Sıfırla"
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1792
 msgid "Power _Down"
 msgstr "_Kapat"
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1798
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1890
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1904
+#, fuzzy
+msgid "Zoom _In"
+msgstr "Yakınlaş ve Sığ_dır"
+
+#: ui/gtk.c:1911
+#, fuzzy
+msgid "Zoom _Out"
+msgstr "Yakınlaş ve Sığ_dır"
+
+#: ui/gtk.c:1918
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1925
 msgid "Zoom To _Fit"
 msgstr "Yakınlaş ve Sığ_dır"
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1931
 msgid "Grab On _Hover"
 msgstr "Ü_zerindeyken Yakala"
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1934
 msgid "_Grab Input"
 msgstr "Girdiyi _Yakala"
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:1963
 msgid "Show _Tabs"
 msgstr "Se_kmeleri Göster"
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:1966
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:1978
 msgid "_Machine"
 msgstr "_Makine"
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:1983
 msgid "_View"
 msgstr "_Görüntüle"
diff --git a/po/zh_CN.po b/po/zh_CN.po
index 2b1d42e97..f50c1fcf4 100644
--- a/po/zh_CN.po
+++ b/po/zh_CN.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 2.2\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-31 10:03+0800\n"
+"POT-Creation-Date: 2015-12-10 13:29+0000\n"
 "PO-Revision-Date: 2014-07-31 10:00+0800\n"
 "Last-Translator: Fam Zheng <famz@redhat.com>\n"
 "Language-Team: Chinese <zh@li.org>\n"
@@ -17,70 +17,70 @@ msgstr ""
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 "X-Generator: Lokalize 1.4\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:271
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - 按下 Ctrl+Alt+G 取消捕获"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:275
 msgid " [Paused]"
 msgstr " [已暂停]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1783
 msgid "_Pause"
 msgstr "暂停(_P)"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1789
 msgid "_Reset"
 msgstr "重置(_R)"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1792
 msgid "Power _Down"
 msgstr "关闭电源(_D)"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1798
 msgid "_Quit"
 msgstr "退出(_Q)"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1890
 msgid "_Fullscreen"
 msgstr "全屏(_F)"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1904
 msgid "Zoom _In"
 msgstr "放大(_I)"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1911
 msgid "Zoom _Out"
 msgstr "缩小(_O)"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1918
 msgid "Best _Fit"
 msgstr "最合适大小(_F)"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1925
 msgid "Zoom To _Fit"
 msgstr "缩放以适应大小(_F)"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1931
 msgid "Grab On _Hover"
 msgstr "鼠标经过时捕获(_H)"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1934
 msgid "_Grab Input"
 msgstr "捕获输入(_G)"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:1963
 msgid "Show _Tabs"
 msgstr "显示标签页(_T)"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:1966
 msgid "Detach Tab"
 msgstr "分离标签页"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:1978
 msgid "_Machine"
 msgstr "虚拟机(_M)"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:1983
 msgid "_View"
 msgstr "视图(_V)"
diff --git a/qapi-schema.json b/qapi-schema.json
index 4342a08d3..8b1a423fa 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -5,6 +5,9 @@
 # QAPI common definitions
 { 'include': 'qapi/common.json' }
 
+# QAPI crypto definitions
+{ 'include': 'qapi/crypto.json' }
+
 # QAPI block definitions
 { 'include': 'qapi/block.json' }
 
@@ -14,6 +17,9 @@
 # Tracing commands
 { 'include': 'qapi/trace.json' }
 
+# QAPI introspection
+{ 'include': 'qapi/introspect.json' }
+
 ##
 # @LostTickPolicy:
 #
@@ -424,6 +430,8 @@
 #
 # @active: in the process of doing migration.
 #
+# @postcopy-active: like active, but now in postcopy mode. (since 2.5)
+#
 # @completed: migration is finished.
 #
 # @failed: some error occurred during migration process.
@@ -433,7 +441,7 @@
 ##
 { 'enum': 'MigrationStatus',
   'data': [ 'none', 'setup', 'cancelling', 'cancelled',
-            'active', 'completed', 'failed' ] }
+            'active', 'postcopy-active', 'completed', 'failed' ] }
 
 ##
 # @MigrationInfo
@@ -474,6 +482,10 @@
 #        may be expensive, but do not actually occur during the iterative
 #        migration rounds themselves. (since 1.6)
 #
+# @x-cpu-throttle-percentage: #optional percentage of time guest cpus are being
+#       throttled during auto-converge. This is only present when auto-converge
+#       has started throttling guest cpus. (Since 2.5)
+#
 # Since: 0.14.0
 ##
 { 'struct': 'MigrationInfo',
@@ -483,7 +495,8 @@
            '*total-time': 'int',
            '*expected-downtime': 'int',
            '*downtime': 'int',
-           '*setup-time': 'int'} }
+           '*setup-time': 'int',
+           '*x-cpu-throttle-percentage': 'int'} }
 
 ##
 # @query-migrate
@@ -529,11 +542,15 @@
 # @auto-converge: If enabled, QEMU will automatically throttle down the guest
 #          to speed up convergence of RAM migration. (since 1.6)
 #
+# @x-postcopy-ram: Start executing on the migration target before all of RAM has
+#          been migrated, pulling the remaining pages along as needed. NOTE: If
+#          the migration fails during postcopy the VM will fail.  (since 2.5)
+#
 # Since: 1.2
 ##
 { 'enum': 'MigrationCapability',
   'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks',
-           'compress', 'events'] }
+           'compress', 'events', 'x-postcopy-ram'] }
 
 ##
 # @MigrationCapabilityStatus
@@ -590,10 +607,18 @@
 #          compression, so set the decompress-threads to the number about 1/4
 #          of compress-threads is adequate.
 #
+# @x-cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                          when migration auto-converge is activated. The
+#                          default value is 20. (Since 2.5)
+#
+# @x-cpu-throttle-increment: throttle percentage increase each time
+#                            auto-converge detects that migration is not making
+#                            progress. The default value is 10. (Since 2.5)
 # Since: 2.4
 ##
 { 'enum': 'MigrationParameter',
-  'data': ['compress-level', 'compress-threads', 'decompress-threads'] }
+  'data': ['compress-level', 'compress-threads', 'decompress-threads',
+           'x-cpu-throttle-initial', 'x-cpu-throttle-increment'] }
 
 #
 # @migrate-set-parameters
@@ -606,12 +631,21 @@
 #
 # @decompress-threads: decompression thread count
 #
+# @x-cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                          when migration auto-converge is activated. The
+#                          default value is 20. (Since 2.5)
+#
+# @x-cpu-throttle-increment: throttle percentage increase each time
+#                            auto-converge detects that migration is not making
+#                            progress. The default value is 10. (Since 2.5)
 # Since: 2.4
 ##
 { 'command': 'migrate-set-parameters',
   'data': { '*compress-level': 'int',
             '*compress-threads': 'int',
-            '*decompress-threads': 'int'} }
+            '*decompress-threads': 'int',
+            '*x-cpu-throttle-initial': 'int',
+            '*x-cpu-throttle-increment': 'int'} }
 
 #
 # @MigrationParameters
@@ -622,12 +656,22 @@
 #
 # @decompress-threads: decompression thread count
 #
+# @x-cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                          when migration auto-converge is activated. The
+#                          default value is 20. (Since 2.5)
+#
+# @x-cpu-throttle-increment: throttle percentage increase each time
+#                            auto-converge detects that migration is not making
+#                            progress. The default value is 10. (Since 2.5)
+#
 # Since: 2.4
 ##
 { 'struct': 'MigrationParameters',
   'data': { 'compress-level': 'int',
             'compress-threads': 'int',
-            'decompress-threads': 'int'} }
+            'decompress-threads': 'int',
+            'x-cpu-throttle-initial': 'int',
+            'x-cpu-throttle-increment': 'int'} }
 ##
 # @query-migrate-parameters
 #
@@ -660,6 +704,16 @@
             '*tls-port': 'int', '*cert-subject': 'str' } }
 
 ##
+# @migrate-start-postcopy
+#
+# Followup to a migration command to switch the migration to postcopy mode.
+# The x-postcopy-ram capability must be set before the original migration
+# command.
+#
+# Since: 2.5
+{ 'command': 'migrate-start-postcopy' }
+
+##
 # @MouseInfo:
 #
 # Information about a mouse device.
@@ -1482,6 +1536,26 @@
   'data': { } }
 
 ##
+# @ActionCompletionMode
+#
+# An enumeration of Transactional completion modes.
+#
+# @individual: Do not attempt to cancel any other Actions if any Actions fail
+#              after the Transaction request succeeds. All Actions that
+#              can complete successfully will do so without waiting on others.
+#              This is the default.
+#
+# @grouped: If any Action fails after the Transaction succeeds, cancel all
+#           Actions. Actions do not complete until all Actions are ready to
+#           complete. May be rejected by Actions that do not support this
+#           completion mode.
+#
+# Since: 2.5
+##
+{ 'enum': 'ActionCompletionMode',
+  'data': [ 'individual', 'grouped' ] }
+
+##
 # @TransactionAction
 #
 # A discriminated record of operations that can be performed with
@@ -1493,25 +1567,52 @@
 # abort since 1.6
 # blockdev-snapshot-internal-sync since 1.7
 # blockdev-backup since 2.3
+# blockdev-snapshot since 2.5
+# block-dirty-bitmap-add since 2.5
+# block-dirty-bitmap-clear since 2.5
 ##
 { 'union': 'TransactionAction',
   'data': {
-       'blockdev-snapshot-sync': 'BlockdevSnapshot',
+       'blockdev-snapshot': 'BlockdevSnapshot',
+       'blockdev-snapshot-sync': 'BlockdevSnapshotSync',
        'drive-backup': 'DriveBackup',
        'blockdev-backup': 'BlockdevBackup',
        'abort': 'Abort',
-       'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal'
+       'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal',
+       'block-dirty-bitmap-add': 'BlockDirtyBitmapAdd',
+       'block-dirty-bitmap-clear': 'BlockDirtyBitmap'
    } }
 
 ##
+# @TransactionProperties
+#
+# Optional arguments to modify the behavior of a Transaction.
+#
+# @completion-mode: #optional Controls how jobs launched asynchronously by
+#                   Actions will complete or fail as a group.
+#                   See @ActionCompletionMode for details.
+#
+# Since: 2.5
+##
+{ 'struct': 'TransactionProperties',
+  'data': {
+       '*completion-mode': 'ActionCompletionMode'
+  }
+}
+
+##
 # @transaction
 #
 # Executes a number of transactionable QMP commands atomically. If any
 # operation fails, then the entire set of actions will be abandoned and the
 # appropriate error returned.
 #
-#  List of:
-#  @TransactionAction: information needed for the respective operation
+# @actions: List of @TransactionAction;
+#           information needed for the respective operations.
+#
+# @properties: #optional structure of additional options to control the
+#              execution of the transaction. See @TransactionProperties
+#              for additional detail.
 #
 # Returns: nothing on success
 #          Errors depend on the operations of the transaction
@@ -1523,7 +1624,10 @@
 # Since 1.1
 ##
 { 'command': 'transaction',
-  'data': { 'actions': [ 'TransactionAction' ] } }
+  'data': { 'actions': [ 'TransactionAction' ],
+            '*properties': 'TransactionProperties'
+          }
+}
 
 ##
 # @human-monitor-command:
@@ -1632,13 +1736,10 @@
 #        1) A primitive type such as 'u8', 'u16', 'bool', 'str', or 'double'.
 #           These types are mapped to the appropriate JSON type.
 #
-#        2) A legacy type in the form 'legacy<subtype>' where subtype is the
-#           legacy qdev typename.  These types are always treated as strings.
-#
-#        3) A child type in the form 'child<subtype>' where subtype is a qdev
+#        2) A child type in the form 'child<subtype>' where subtype is a qdev
 #           device type name.  Child properties create the composition tree.
 #
-#        4) A link type in the form 'link<subtype>' where subtype is a qdev
+#        3) A link type in the form 'link<subtype>' where subtype is a qdev
 #           device type name.  Link properties form the device model graph.
 #
 # Since: 1.2
@@ -1689,17 +1790,16 @@
 #
 # @property: The property name to read
 #
-# Returns: The property value.  The type depends on the property type.  legacy<>
-#          properties are returned as #str.  child<> and link<> properties are
-#          returns as #str pathnames.  All integer property types (u8, u16, etc)
-#          are returned as #int.
+# Returns: The property value.  The type depends on the property
+#          type. child<> and link<> properties are returned as #str
+#          pathnames.  All integer property types (u8, u16, etc) are
+#          returned as #int.
 #
 # Since: 1.2
 ##
 { 'command': 'qom-get',
   'data': { 'path': 'str', 'property': 'str' },
-  'returns': '**',
-  'gen': false }
+  'returns': 'any' }
 
 ##
 # @qom-set:
@@ -1716,8 +1816,7 @@
 # Since: 1.2
 ##
 { 'command': 'qom-set',
-  'data': { 'path': 'str', 'property': 'str', 'value': '**' },
-  'gen': false }
+  'data': { 'path': 'str', 'property': 'str', 'value': 'any' } }
 
 ##
 # @set_password:
@@ -1809,8 +1908,10 @@
 #          device's password.  The behavior of reads and writes to the block
 #          device between when these calls are executed is undefined.
 #
-# Notes:  It is strongly recommended that this interface is not used especially
-#         for changing block devices.
+# Notes:  This interface is deprecated, and it is strongly recommended that you
+#         avoid using it.  For changing block devices, use
+#         blockdev-change-medium; for changing VNC parameters, use
+#         change-vnc-password.
 #
 # Since: 0.14.0
 ##
@@ -1950,7 +2051,7 @@
 #
 # Remove a device from a guest
 #
-# @id: the name of the device
+# @id: the name or QOM path of the device
 #
 # Returns: Nothing on success
 #          If @id is not a valid device, DeviceNotFound
@@ -2058,6 +2159,20 @@
   'returns': 'DumpGuestMemoryCapability' }
 
 ##
+# @dump-skeys
+#
+# Dump guest's storage keys
+#
+# @filename: the path to the file to dump to
+#
+# This command is only supported on s390 architecture.
+#
+# Since: 2.5
+##
+{ 'command': 'dump-skeys',
+  'data': { 'filename': 'str' } }
+
+##
 # @netdev_add:
 #
 # Add a network backend.
@@ -2067,11 +2182,12 @@
 #
 # @id: the name of the new network backend
 #
-# @props: #optional a list of properties to be passed to the backend in
-#         the format 'name=value', like 'ifname=tap0,script=no'
+# Additional arguments depend on the type.
 #
-# Notes: The semantics of @props is not well defined.  Future commands will be
-#        introduced that provide stronger typing for backend creation.
+# TODO This command effectively bypasses QAPI completely due to its
+# "additional arguments" business.  It shouldn't have been added to
+# the schema in this form.  It should be qapified properly, or
+# replaced by a properly qapified command.
 #
 # Since: 0.14.0
 #
@@ -2079,8 +2195,8 @@
 #          If @type is not a valid network backend, DeviceNotFound
 ##
 { 'command': 'netdev_add',
-  'data': {'type': 'str', 'id': 'str', '*props': '**'},
-  'gen': false }
+  'data': {'type': 'str', 'id': 'str'},
+  'gen': false }                # so we can get the additional arguments
 
 ##
 # @netdev_del:
@@ -2113,8 +2229,7 @@
 # Since: 2.0
 ##
 { 'command': 'object-add',
-  'data': {'qom-type': 'str', 'id': 'str', '*props': '**'},
-  'gen': false }
+  'data': {'qom-type': 'str', 'id': 'str', '*props': 'any'} }
 
 ##
 # @object-del:
@@ -2466,12 +2581,16 @@
 #
 # @vhostforce: #optional vhost on for non-MSIX virtio guests (default: false).
 #
+# @queues: #optional number of queues to be created for multiqueue vhost-user
+#          (default: 1) (Since 2.5)
+#
 # Since 2.1
 ##
 { 'struct': 'NetdevVhostUserOptions',
   'data': {
     'chardev':        'str',
-    '*vhostforce':    'bool' } }
+    '*vhostforce':    'bool',
+    '*queues':        'int' } }
 
 ##
 # @NetClientOptions
@@ -2537,6 +2656,26 @@
     'opts': 'NetClientOptions' } }
 
 ##
+# @NetFilterDirection
+#
+# Indicates whether a netfilter is attached to a netdev's transmit queue or
+# receive queue or both.
+#
+# @all: the filter is attached both to the receive and the transmit
+#       queue of the netdev (default).
+#
+# @rx: the filter is attached to the receive queue of the netdev,
+#      where it will receive packets sent to the netdev.
+#
+# @tx: the filter is attached to the transmit queue of the netdev,
+#      where it will receive packets sent by the netdev.
+#
+# Since 2.5
+##
+{ 'enum': 'NetFilterDirection',
+  'data': [ 'all', 'rx', 'tx' ] }
+
+##
 # @InetSocketAddress
 #
 # Captures a socket address or address range in the Internet namespace.
@@ -3345,6 +3484,17 @@
             'features': 'int' } }
 
 ##
+# @DummyForceArrays
+#
+# Not used by QMP; hack to let us use X86CPUFeatureWordInfoList internally
+#
+# Since 2.5
+##
+{ 'struct': 'DummyForceArrays',
+  'data': { 'unused': ['X86CPUFeatureWordInfo'] } }
+
+
+##
 # @RxState:
 #
 # Packets receiving state
@@ -3429,16 +3579,22 @@
 # Button of a pointer input device (mouse, tablet).
 #
 # Since: 2.0
+#
+# Note that the spelling of these values may change when the
+# x-input-send-event is promoted out of experimental status.
 ##
 { 'enum'  : 'InputButton',
   'data'  : [ 'Left', 'Middle', 'Right', 'WheelUp', 'WheelDown' ] }
 
 ##
-# @InputButton
+# @InputAxis
 #
 # Position axis of a pointer input device (mouse, tablet).
 #
 # Since: 2.0
+#
+# Note that the spelling of these values may change when the
+# x-input-send-event is promoted out of experimental status.
 ##
 { 'enum'  : 'InputAxis',
   'data'  : [ 'X', 'Y' ] }
@@ -3529,7 +3685,10 @@
 #
 # Since: 2.2
 #
-# Note: this command is experimental, and not a stable API.
+# Note: this command is experimental, and not a stable API.  Things that
+# might change before it becomes stable include the spelling of enum
+# values for InputButton and InputAxis, and the notion of how to designate
+# which console will receive the event.
 #
 ##
 { 'command': 'x-input-send-event',
@@ -3794,3 +3953,21 @@
 
 # Rocker ethernet network switch
 { 'include': 'qapi/rocker.json' }
+
+##
+# ReplayMode:
+#
+# Mode of the replay subsystem.
+#
+# @none: normal execution mode. Replay or record are not enabled.
+#
+# @record: record mode. All non-deterministic data is written into the
+#          replay log.
+#
+# @play: replay mode. Non-deterministic data required for system execution
+#        is read from the log.
+#
+# Since: 2.5
+##
+{ 'enum': 'ReplayMode',
+  'data': [ 'none', 'record', 'play' ] }
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 7b2efb867..419f1124e 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -215,10 +215,11 @@
 # @drv: the name of the block format used to open the backing device. As of
 #       0.14.0 this can be: 'blkdebug', 'bochs', 'cloop', 'cow', 'dmg',
 #       'file', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
-#       'host_floppy', 'http', 'https', 'nbd', 'parallels', 'qcow',
+#       'http', 'https', 'nbd', 'parallels', 'qcow',
 #       'qcow2', 'raw', 'tftp', 'vdi', 'vmdk', 'vpc', 'vvfat'
 #       2.2: 'archipelago' added, 'cow' dropped
 #       2.3: 'host_floppy' deprecated
+#       2.5: 'host_floppy' dropped
 #
 # @backing_file: #optional the name of the backing file (for copy-on-write)
 #
@@ -413,6 +414,59 @@
 ##
 { 'command': 'query-block', 'returns': ['BlockInfo'] }
 
+
+##
+# @BlockDeviceTimedStats:
+#
+# Statistics of a block device during a given interval of time.
+#
+# @interval_length: Interval used for calculating the statistics,
+#                   in seconds.
+#
+# @min_rd_latency_ns: Minimum latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @min_wr_latency_ns: Minimum latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @min_flush_latency_ns: Minimum latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @max_rd_latency_ns: Maximum latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @max_wr_latency_ns: Maximum latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @max_flush_latency_ns: Maximum latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @avg_rd_latency_ns: Average latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @avg_wr_latency_ns: Average latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @avg_flush_latency_ns: Average latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @avg_rd_queue_depth: Average number of pending read operations
+#                      in the defined interval.
+#
+# @avg_wr_queue_depth: Average number of pending write operations
+#                      in the defined interval.
+#
+# Since: 2.5
+##
+
+{ 'struct': 'BlockDeviceTimedStats',
+  'data': { 'interval_length': 'int', 'min_rd_latency_ns': 'int',
+            'max_rd_latency_ns': 'int', 'avg_rd_latency_ns': 'int',
+            'min_wr_latency_ns': 'int', 'max_wr_latency_ns': 'int',
+            'avg_wr_latency_ns': 'int', 'min_flush_latency_ns': 'int',
+            'max_flush_latency_ns': 'int', 'avg_flush_latency_ns': 'int',
+            'avg_rd_queue_depth': 'number', 'avg_wr_queue_depth': 'number' } }
+
 ##
 # @BlockDeviceStats:
 #
@@ -447,6 +501,37 @@
 # @wr_merged: Number of write requests that have been merged into another
 #             request (Since 2.3).
 #
+# @idle_time_ns: #optional Time since the last I/O operation, in
+#                nanoseconds. If the field is absent it means that
+#                there haven't been any operations yet (Since 2.5).
+#
+# @failed_rd_operations: The number of failed read operations
+#                        performed by the device (Since 2.5)
+#
+# @failed_wr_operations: The number of failed write operations
+#                        performed by the device (Since 2.5)
+#
+# @failed_flush_operations: The number of failed flush operations
+#                           performed by the device (Since 2.5)
+#
+# @invalid_rd_operations: The number of invalid read operations
+#                          performed by the device (Since 2.5)
+#
+# @invalid_wr_operations: The number of invalid write operations
+#                         performed by the device (Since 2.5)
+#
+# @invalid_flush_operations: The number of invalid flush operations
+#                            performed by the device (Since 2.5)
+#
+# @account_invalid: Whether invalid operations are included in the
+#                   last access statistics (Since 2.5)
+#
+# @account_failed: Whether failed operations are included in the
+#                  latency and last access statistics (Since 2.5)
+#
+# @timed_stats: Statistics specific to the set of previously defined
+#               intervals of time (Since 2.5)
+#
 # Since: 0.14.0
 ##
 { 'struct': 'BlockDeviceStats',
@@ -454,7 +539,12 @@
            'wr_operations': 'int', 'flush_operations': 'int',
            'flush_total_time_ns': 'int', 'wr_total_time_ns': 'int',
            'rd_total_time_ns': 'int', 'wr_highest_offset': 'int',
-           'rd_merged': 'int', 'wr_merged': 'int' } }
+           'rd_merged': 'int', 'wr_merged': 'int', '*idle_time_ns': 'int',
+           'failed_rd_operations': 'int', 'failed_wr_operations': 'int',
+           'failed_flush_operations': 'int', 'invalid_rd_operations': 'int',
+           'invalid_wr_operations': 'int', 'invalid_flush_operations': 'int',
+           'account_invalid': 'bool', 'account_failed': 'bool',
+           'timed_stats': ['BlockDeviceTimedStats'] } }
 
 ##
 # @BlockStats:
@@ -681,7 +771,7 @@
   'data': [ 'existing', 'absolute-paths' ] }
 
 ##
-# @BlockdevSnapshot
+# @BlockdevSnapshotSync
 #
 # Either @device or @node-name must be set but not both.
 #
@@ -698,12 +788,27 @@
 # @mode: #optional whether and how QEMU should create a new image, default is
 #        'absolute-paths'.
 ##
-{ 'struct': 'BlockdevSnapshot',
+{ 'struct': 'BlockdevSnapshotSync',
   'data': { '*device': 'str', '*node-name': 'str',
             'snapshot-file': 'str', '*snapshot-node-name': 'str',
             '*format': 'str', '*mode': 'NewImageMode' } }
 
 ##
+# @BlockdevSnapshot
+#
+# @node: device or node name that will have a snapshot created.
+#
+# @overlay: reference to the existing block device that will become
+#           the overlay of @node, as part of creating the snapshot.
+#           It must not have a current backing file (this can be
+#           achieved by passing "backing": "" to blockdev-add).
+#
+# Since 2.5
+##
+{ 'struct': 'BlockdevSnapshot',
+  'data': { 'node': 'str', 'overlay': 'str' } }
+
+##
 # @DriveBackup
 #
 # @device: the name of the device which should be copied.
@@ -789,7 +894,7 @@
 #
 # Generates a synchronous snapshot of a block device.
 #
-# For the arguments, see the documentation of BlockdevSnapshot.
+# For the arguments, see the documentation of BlockdevSnapshotSync.
 #
 # Returns: nothing on success
 #          If @device is not a valid block device, DeviceNotFound
@@ -797,6 +902,19 @@
 # Since 0.14.0
 ##
 { 'command': 'blockdev-snapshot-sync',
+  'data': 'BlockdevSnapshotSync' }
+
+
+##
+# @blockdev-snapshot
+#
+# Generates a snapshot of a block device.
+#
+# For the arguments, see the documentation of BlockdevSnapshot.
+#
+# Since 2.5
+##
+{ 'command': 'blockdev-snapshot',
   'data': 'BlockdevSnapshot' }
 
 ##
@@ -1373,15 +1491,14 @@
 #
 # Drivers that are supported in block device operations.
 #
-# @host_device, @host_cdrom, @host_floppy: Since 2.1
-# @host_floppy: deprecated since 2.3
+# @host_device, @host_cdrom: Since 2.1
 #
 # Since: 2.0
 ##
 { 'enum': 'BlockdevDriver',
   'data': [ 'archipelago', 'blkdebug', 'blkverify', 'bochs', 'cloop',
             'dmg', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
-            'host_floppy', 'http', 'https', 'null-aio', 'null-co', 'parallels',
+            'http', 'https', 'null-aio', 'null-co', 'parallels',
             'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'tftp', 'vdi', 'vhdx',
             'vmdk', 'vpc', 'vvfat' ] }
 
@@ -1393,9 +1510,12 @@
 #
 # @driver:        block driver name
 # @id:            #optional id by which the new block device can be referred to.
-#                 This is a required option on the top level of blockdev-add, and
-#                 currently not allowed on any other level.
-# @node-name:     #optional the name of a block driver state node (Since 2.0)
+#                 This option is only allowed on the top level of blockdev-add.
+#                 A BlockBackend will be created by blockdev-add if and only if
+#                 this option is given.
+# @node-name:     #optional the name of a block driver state node (Since 2.0).
+#                 This option is required on the top level of blockdev-add if
+#                 the @id option is not given there.
 # @discard:       #optional discard-related options (default: ignore)
 # @cache:         #optional cache-related options
 # @aio:           #optional AIO backend (default: threads)
@@ -1405,6 +1525,14 @@
 #                 (default: enospc)
 # @read-only:     #optional whether the block device should be read-only
 #                 (default: false)
+# @stats-account-invalid: #optional whether to include invalid
+#                         operations when computing last access statistics
+#                         (default: true) (Since 2.5)
+# @stats-account-failed: #optional whether to include failed
+#                         operations when computing latency and last
+#                         access statistics (default: true) (Since 2.5)
+# @stats-intervals: #optional list of intervals for collecting I/O
+#                   statistics, in seconds (default: none) (Since 2.5)
 # @detect-zeroes: #optional detect and optimize zero writes (Since 2.1)
 #                 (default: off)
 #
@@ -1420,6 +1548,9 @@
             '*rerror': 'BlockdevOnError',
             '*werror': 'BlockdevOnError',
             '*read-only': 'bool',
+            '*stats-account-invalid': 'bool',
+            '*stats-account-failed': 'bool',
+            '*stats-intervals': ['int'],
             '*detect-zeroes': 'BlockdevDetectZeroesOptions' } }
 
 ##
@@ -1592,6 +1723,10 @@
 # @refcount-cache-size:   #optional the maximum size of the refcount block cache
 #                         in bytes (since 2.2)
 #
+# @cache-clean-interval:  #optional clean unused entries in the L2 and refcount
+#                         caches. The interval is in seconds. The default value
+#                         is 0 and it disables this feature (since 2.5)
+#
 # Since: 1.7
 ##
 { 'struct': 'BlockdevOptionsQcow2',
@@ -1603,7 +1738,8 @@
             '*overlap-check': 'Qcow2OverlapChecks',
             '*cache-size': 'int',
             '*l2-cache-size': 'int',
-            '*refcount-cache-size': 'int' } }
+            '*refcount-cache-size': 'int',
+            '*cache-clean-interval': 'int' } }
 
 
 ##
@@ -1811,7 +1947,6 @@
 # TODO gluster: Wait for structured options
       'host_cdrom': 'BlockdevOptionsFile',
       'host_device':'BlockdevOptionsFile',
-      'host_floppy':'BlockdevOptionsFile',
       'http':       'BlockdevOptionsFile',
       'https':      'BlockdevOptionsFile',
 # TODO iscsi: Wait for structured options
@@ -1855,11 +1990,13 @@
 ##
 # @blockdev-add:
 #
-# Creates a new block device.
+# Creates a new block device. If the @id option is given at the top level, a
+# BlockBackend will be created; otherwise, @node-name is mandatory at the top
+# level and no BlockBackend will be created.
 #
 # This command is still a work in progress.  It doesn't support all
-# block drivers, it lacks a matching blockdev-del, and more.  Stay
-# away from it unless you want to help with its development.
+# block drivers among other things.  Stay away from it unless you want
+# to help with its development.
 #
 # @options: block device options for the new device
 #
@@ -1867,6 +2004,165 @@
 ##
 { 'command': 'blockdev-add', 'data': { 'options': 'BlockdevOptions' } }
 
+##
+# @x-blockdev-del:
+#
+# Deletes a block device that has been added using blockdev-add.
+# The selected device can be either a block backend or a graph node.
+#
+# In the former case the backend will be destroyed, along with its
+# inserted medium if there's any. The command will fail if the backend
+# or its medium are in use.
+#
+# In the latter case the node will be destroyed. The command will fail
+# if the node is attached to a block backend or is otherwise being
+# used.
+#
+# One of @id or @node-name must be specified, but not both.
+#
+# This command is still a work in progress and is considered
+# experimental. Stay away from it unless you want to help with its
+# development.
+#
+# @id: #optional Name of the block backend device to delete.
+#
+# @node-name: #optional Name of the graph node to delete.
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-del', 'data': { '*id': 'str', '*node-name': 'str' } }
+
+##
+# @blockdev-open-tray:
+#
+# Opens a block device's tray. If there is a block driver state tree inserted as
+# a medium, it will become inaccessible to the guest (but it will remain
+# associated to the block device, so closing the tray will make it accessible
+# again).
+#
+# If the tray was already open before, this will be a no-op.
+#
+# Once the tray opens, a DEVICE_TRAY_MOVED event is emitted. There are cases in
+# which no such event will be generated, these include:
+# - if the guest has locked the tray, @force is false and the guest does not
+#   respond to the eject request
+# - if the BlockBackend denoted by @device does not have a guest device attached
+#   to it
+# - if the guest device does not have an actual tray
+#
+# @device: block device name
+#
+# @force:  #optional if false (the default), an eject request will be sent to
+#          the guest if it has locked the tray (and the tray will not be opened
+#          immediately); if true, the tray will be opened regardless of whether
+#          it is locked
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-open-tray',
+  'data': { 'device': 'str',
+            '*force': 'bool' } }
+
+##
+# @blockdev-close-tray:
+#
+# Closes a block device's tray. If there is a block driver state tree associated
+# with the block device (which is currently ejected), that tree will be loaded
+# as the medium.
+#
+# If the tray was already closed before, this will be a no-op.
+#
+# @device: block device name
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-close-tray',
+  'data': { 'device': 'str' } }
+
+##
+# @x-blockdev-remove-medium:
+#
+# Removes a medium (a block driver state tree) from a block device. That block
+# device's tray must currently be open (unless there is no attached guest
+# device).
+#
+# If the tray is open and there is no medium inserted, this will be a no-op.
+#
+# This command is still a work in progress and is considered experimental.
+# Stay away from it unless you want to help with its development.
+#
+# @device: block device name
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-remove-medium',
+  'data': { 'device': 'str' } }
+
+##
+# @x-blockdev-insert-medium:
+#
+# Inserts a medium (a block driver state tree) into a block device. That block
+# device's tray must currently be open (unless there is no attached guest
+# device) and there must be no medium inserted already.
+#
+# This command is still a work in progress and is considered experimental.
+# Stay away from it unless you want to help with its development.
+#
+# @device:    block device name
+#
+# @node-name: name of a node in the block driver state graph
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-insert-medium',
+  'data': { 'device': 'str',
+            'node-name': 'str'} }
+
+
+##
+# @BlockdevChangeReadOnlyMode:
+#
+# Specifies the new read-only mode of a block device subject to the
+# @blockdev-change-medium command.
+#
+# @retain:      Retains the current read-only mode
+#
+# @read-only:   Makes the device read-only
+#
+# @read-write:  Makes the device writable
+#
+# Since: 2.3
+##
+{ 'enum': 'BlockdevChangeReadOnlyMode',
+  'data': ['retain', 'read-only', 'read-write'] }
+
+
+##
+# @blockdev-change-medium:
+#
+# Changes the medium inserted into a block device by ejecting the current medium
+# and loading a new image file which is inserted as the new medium (this command
+# combines blockdev-open-tray, x-blockdev-remove-medium,
+# x-blockdev-insert-medium and blockdev-close-tray).
+#
+# @device:          block device name
+#
+# @filename:        filename of the new image to be loaded
+#
+# @format:          #optional, format to open the new image with (defaults to
+#                   the probed format)
+#
+# @read-only-mode:  #optional, change the read-only mode of the device; defaults
+#                   to 'retain'
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-change-medium',
+  'data': { 'device': 'str',
+            'filename': 'str',
+            '*format': 'str',
+            '*read-only-mode': 'BlockdevChangeReadOnlyMode' } }
+
 
 ##
 # @BlockErrorAction
diff --git a/qapi/block.json b/qapi/block.json
index aad645c4a..84022f12b 100644
--- a/qapi/block.json
+++ b/qapi/block.json
@@ -6,7 +6,7 @@
 { 'include': 'block-core.json' }
 
 ##
-# BiosAtaTranslation:
+# @BiosAtaTranslation:
 #
 # Policy that BIOS should use to interpret cylinder/head/sector
 # addresses.  Note that Bochs BIOS and SeaBIOS will not actually
diff --git a/qapi/crypto.json b/qapi/crypto.json
new file mode 100644
index 000000000..b058b14d7
--- /dev/null
+++ b/qapi/crypto.json
@@ -0,0 +1,21 @@
+# -*- Mode: Python -*-
+#
+# QAPI crypto definitions
+
+##
+# QCryptoTLSCredsEndpoint:
+#
+# The type of network endpoint that will be using the credentials.
+# Most types of credential require different setup / structures
+# depending on whether they will be used in a server versus a
+# client.
+#
+# @client: the network endpoint is acting as the client
+#
+# @server: the network endpoint is acting as the server
+#
+# Since: 2.5
+##
+{ 'enum': 'QCryptoTLSCredsEndpoint',
+  'prefix': 'QCRYPTO_TLS_CREDS_ENDPOINT',
+  'data': ['client', 'server']}
diff --git a/qapi/introspect.json b/qapi/introspect.json
new file mode 100644
index 000000000..9e9369e16
--- /dev/null
+++ b/qapi/introspect.json
@@ -0,0 +1,289 @@
+# -*- Mode: Python -*-
+#
+# QAPI/QMP introspection
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+##
+# @query-qmp-schema
+#
+# Command query-qmp-schema exposes the QMP wire ABI as an array of
+# SchemaInfo.  This lets QMP clients figure out what commands and
+# events are available in this QEMU, and their parameters and results.
+#
+# However, the SchemaInfo can't reflect all the rules and restrictions
+# that apply to QMP.  It's interface introspection (figuring out
+# what's there), not interface specification.  The specification is in
+# the QAPI schema.
+#
+# Furthermore, while we strive to keep the QMP wire format
+# backwards-compatible across qemu versions, the introspection output
+# is not guaranteed to have the same stability.  For example, one
+# version of qemu may list an object member as an optional
+# non-variant, while another lists the same member only through the
+# object's variants; or the type of a member may change from a generic
+# string into a specific enum or from one specific type into an
+# alternate that includes the original type alongside something else.
+#
+# Returns: array of @SchemaInfo, where each element describes an
+# entity in the ABI: command, event, type, ...
+#
+# The order of the various SchemaInfo is unspecified; however, all
+# names are guaranteed to be unique (no name will be duplicated with
+# different meta-types).
+#
+# Note: the QAPI schema is also used to help define *internal*
+# interfaces, by defining QAPI types.  These are not part of the QMP
+# wire ABI, and therefore not returned by this command.
+#
+# Since: 2.5
+##
+{ 'command': 'query-qmp-schema',
+  'returns': [ 'SchemaInfo' ],
+  'gen': false }                # just to simplify qmp_query_json()
+
+##
+# @SchemaMetaType
+#
+# This is a @SchemaInfo's meta type, i.e. the kind of entity it
+# describes.
+#
+# @builtin: a predefined type such as 'int' or 'bool'.
+#
+# @enum: an enumeration type
+#
+# @array: an array type
+#
+# @object: an object type (struct or union)
+#
+# @alternate: an alternate type
+#
+# @command: a QMP command
+#
+# @event: a QMP event
+#
+# Since: 2.5
+##
+{ 'enum': 'SchemaMetaType',
+  'data': [ 'builtin', 'enum', 'array', 'object', 'alternate',
+            'command', 'event' ] }
+
+##
+# @SchemaInfoBase
+#
+# Members common to any @SchemaInfo.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoBase',
+  'data': { 'name': 'str', 'meta-type': 'SchemaMetaType' } }
+
+##
+# @SchemaInfo
+#
+# @name: the entity's name, inherited from @base.
+#        Commands and events have the name defined in the QAPI schema.
+#        Unlike command and event names, type names are not part of
+#        the wire ABI.  Consequently, type names are meaningless
+#        strings here, although they are still guaranteed unique
+#        regardless of @meta-type.
+#
+# All references to other SchemaInfo are by name.
+#
+# @meta-type: the entity's meta type, inherited from @base.
+#
+# Additional members depend on the value of @meta-type.
+#
+# Since: 2.5
+##
+{ 'union': 'SchemaInfo',
+  'base': 'SchemaInfoBase',
+  'discriminator': 'meta-type',
+  'data': {
+      'builtin': 'SchemaInfoBuiltin',
+      'enum': 'SchemaInfoEnum',
+      'array': 'SchemaInfoArray',
+      'object': 'SchemaInfoObject',
+      'alternate': 'SchemaInfoAlternate',
+      'command': 'SchemaInfoCommand',
+      'event': 'SchemaInfoEvent' } }
+
+##
+# @SchemaInfoBuiltin
+#
+# Additional SchemaInfo members for meta-type 'builtin'.
+#
+# @json-type: the JSON type used for this type on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoBuiltin',
+  'data': { 'json-type': 'JSONType' } }
+
+##
+# @JSONType
+#
+# The four primitive and two structured types according to RFC 7159
+# section 1, plus 'int' (split off 'number'), plus the obvious top
+# type 'value'.
+#
+# Since: 2.5
+##
+{ 'enum': 'JSONType',
+  'data': [ 'string', 'number', 'int', 'boolean', 'null',
+            'object', 'array', 'value' ] }
+
+##
+# @SchemaInfoEnum
+#
+# Additional SchemaInfo members for meta-type 'enum'.
+#
+# @values: the enumeration type's values, in no particular order.
+#
+# Values of this type are JSON string on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoEnum',
+  'data': { 'values': ['str'] } }
+
+##
+# @SchemaInfoArray
+#
+# Additional SchemaInfo members for meta-type 'array'.
+#
+# @element-type: the array type's element type.
+#
+# Values of this type are JSON array on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoArray',
+  'data': { 'element-type': 'str' } }
+
+##
+# @SchemaInfoObject
+#
+# Additional SchemaInfo members for meta-type 'object'.
+#
+# @members: the object type's (non-variant) members, in no particular order.
+#
+# @tag: #optional the name of the member serving as type tag.
+#       An element of @members with this name must exist.
+#
+# @variants: #optional variant members, i.e. additional members that
+#            depend on the type tag's value.  Present exactly when
+#            @tag is present.  The variants are in no particular order,
+#            and may even differ from the order of the values of the
+#            enum type of the @tag.
+#
+# Values of this type are JSON object on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObject',
+  'data': { 'members': [ 'SchemaInfoObjectMember' ],
+            '*tag': 'str',
+            '*variants': [ 'SchemaInfoObjectVariant' ] } }
+
+##
+# @SchemaInfoObjectMember
+#
+# An object member.
+#
+# @name: the member's name, as defined in the QAPI schema.
+#
+# @type: the name of the member's type.
+#
+# @default: #optional default when used as command parameter.
+#           If absent, the parameter is mandatory.
+#           If present, the value must be null.  The parameter is
+#           optional, and behavior when it's missing is not specified
+#           here.
+#           Future extension: if present and non-null, the parameter
+#           is optional, and defaults to this value.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObjectMember',
+  'data': { 'name': 'str', 'type': 'str', '*default': 'any' } }
+# @default's type must be null or match @type
+
+##
+# @SchemaInfoObjectVariant
+#
+# The variant members for a value of the type tag.
+#
+# @case: a value of the type tag.
+#
+# @type: the name of the object type that provides the variant members
+#        when the type tag has value @case.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObjectVariant',
+  'data': { 'case': 'str', 'type': 'str' } }
+
+##
+# @SchemaInfoAlternate
+#
+# Additional SchemaInfo members for meta-type 'alternate'.
+#
+# @members: the alternate type's members, in no particular order.
+#           The members' wire encoding is distinct, see
+#           docs/qapi-code-gen.txt section Alternate types.
+#
+# On the wire, this can be any of the members.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoAlternate',
+  'data': { 'members': [ 'SchemaInfoAlternateMember' ] } }
+
+##
+# @SchemaInfoAlternateMember
+#
+# An alternate member.
+#
+# @type: the name of the member's type.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoAlternateMember',
+  'data': { 'type': 'str' } }
+
+##
+# @SchemaInfoCommand
+#
+# Additional SchemaInfo members for meta-type 'command'.
+#
+# @arg-type: the name of the object type that provides the command's
+#            parameters.
+#
+# @ret-type: the name of the command's result type.
+#
+# TODO @success-response (currently irrelevant, because it's QGA, not QMP)
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoCommand',
+  'data': { 'arg-type': 'str', 'ret-type': 'str' } }
+
+##
+# @SchemaInfoEvent
+#
+# Additional SchemaInfo members for meta-type 'event'.
+#
+# @arg-type: the name of the object type that provides the event's
+#            parameters.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoEvent',
+  'data': { 'arg-type': 'str' } }
diff --git a/qapi/opts-visitor.c b/qapi/opts-visitor.c
index 7ae33b311..cd10392f1 100644
--- a/qapi/opts-visitor.c
+++ b/qapi/opts-visitor.c
@@ -474,8 +474,8 @@ opts_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp)
         return;
     }
 
-    val = strtosz_suffix(opt->str ? opt->str : "", &endptr,
-                         STRTOSZ_DEFSUFFIX_B);
+    val = qemu_strtosz_suffix(opt->str ? opt->str : "", &endptr,
+                         QEMU_STRTOSZ_DEFSUFFIX_B);
     if (val < 0 || *endptr) {
         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
                    "a size value representible as a non-negative int64");
diff --git a/qapi/qapi-dealloc-visitor.c b/qapi/qapi-dealloc-visitor.c
index d7f92c5d6..737deab9e 100644
--- a/qapi/qapi-dealloc-visitor.c
+++ b/qapi/qapi-dealloc-visitor.c
@@ -151,6 +151,14 @@ static void qapi_dealloc_type_number(Visitor *v, double *obj, const char *name,
 {
 }
 
+static void qapi_dealloc_type_anything(Visitor *v, QObject **obj,
+                                       const char *name, Error **errp)
+{
+    if (obj) {
+        qobject_decref(*obj);
+    }
+}
+
 static void qapi_dealloc_type_size(Visitor *v, uint64_t *obj, const char *name,
                                    Error **errp)
 {
@@ -216,6 +224,7 @@ QapiDeallocVisitor *qapi_dealloc_visitor_new(void)
     v->visitor.type_bool = qapi_dealloc_type_bool;
     v->visitor.type_str = qapi_dealloc_type_str;
     v->visitor.type_number = qapi_dealloc_type_number;
+    v->visitor.type_any = qapi_dealloc_type_anything;
     v->visitor.type_size = qapi_dealloc_type_size;
     v->visitor.start_union = qapi_dealloc_start_union;
 
diff --git a/qapi/qapi-visit-core.c b/qapi/qapi-visit-core.c
index 5a7c90050..59ed5067f 100644
--- a/qapi/qapi-visit-core.c
+++ b/qapi/qapi-visit-core.c
@@ -260,6 +260,12 @@ void visit_type_number(Visitor *v, double *obj, const char *name, Error **errp)
     v->type_number(v, obj, name, errp);
 }
 
+void visit_type_any(Visitor *v, QObject **obj, const char *name,
+                    Error **errp)
+{
+    v->type_any(v, obj, name, errp);
+}
+
 void output_type_enum(Visitor *v, int *obj, const char * const strings[],
                       const char *kind, const char *name,
                       Error **errp)
diff --git a/qapi/qmp-event.c b/qapi/qmp-event.c
index 0d1ce0bd1..c0e435f99 100644
--- a/qapi/qmp-event.c
+++ b/qapi/qmp-event.c
@@ -18,14 +18,6 @@
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qjson.h"
 
-#ifdef _WIN32
-#include "sysemu/os-win32.h"
-#endif
-
-#ifdef CONFIG_POSIX
-#include "sysemu/os-posix.h"
-#endif
-
 static QMPEventFuncEmit qmp_emit;
 
 void qmp_event_set_func_emit(QMPEventFuncEmit emit)
diff --git a/qapi/qmp-input-visitor.c b/qapi/qmp-input-visitor.c
index e97b8a428..eb6e11030 100644
--- a/qapi/qmp-input-visitor.c
+++ b/qapi/qmp-input-visitor.c
@@ -225,45 +225,45 @@ static void qmp_input_type_int(Visitor *v, int64_t *obj, const char *name,
                                Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QInt *qint = qobject_to_qint(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QINT) {
+    if (!qint) {
         error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
                    "integer");
         return;
     }
 
-    *obj = qint_get_int(qobject_to_qint(qobj));
+    *obj = qint_get_int(qint);
 }
 
 static void qmp_input_type_bool(Visitor *v, bool *obj, const char *name,
                                 Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QBool *qbool = qobject_to_qbool(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QBOOL) {
+    if (!qbool) {
         error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
                    "boolean");
         return;
     }
 
-    *obj = qbool_get_bool(qobject_to_qbool(qobj));
+    *obj = qbool_get_bool(qbool);
 }
 
 static void qmp_input_type_str(Visitor *v, char **obj, const char *name,
                                Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QString *qstr = qobject_to_qstring(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QSTRING) {
+    if (!qstr) {
         error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
                    "string");
         return;
     }
 
-    *obj = g_strdup(qstring_get_str(qobject_to_qstring(qobj)));
+    *obj = g_strdup(qstring_get_str(qstr));
 }
 
 static void qmp_input_type_number(Visitor *v, double *obj, const char *name,
@@ -271,19 +271,33 @@ static void qmp_input_type_number(Visitor *v, double *obj, const char *name,
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QInt *qint;
+    QFloat *qfloat;
 
-    if (!qobj || (qobject_type(qobj) != QTYPE_QFLOAT &&
-        qobject_type(qobj) != QTYPE_QINT)) {
-        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                   "number");
+    qint = qobject_to_qint(qobj);
+    if (qint) {
+        *obj = qint_get_int(qobject_to_qint(qobj));
         return;
     }
 
-    if (qobject_type(qobj) == QTYPE_QINT) {
-        *obj = qint_get_int(qobject_to_qint(qobj));
-    } else {
+    qfloat = qobject_to_qfloat(qobj);
+    if (qfloat) {
         *obj = qfloat_get_double(qobject_to_qfloat(qobj));
+        return;
     }
+
+    error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+               "number");
+}
+
+static void qmp_input_type_any(Visitor *v, QObject **obj, const char *name,
+                               Error **errp)
+{
+    QmpInputVisitor *qiv = to_qiv(v);
+    QObject *qobj = qmp_input_get_object(qiv, name, true);
+
+    qobject_incref(qobj);
+    *obj = qobj;
 }
 
 static void qmp_input_optional(Visitor *v, bool *present, const char *name,
@@ -329,6 +343,7 @@ QmpInputVisitor *qmp_input_visitor_new(QObject *obj)
     v->visitor.type_bool = qmp_input_type_bool;
     v->visitor.type_str = qmp_input_type_str;
     v->visitor.type_number = qmp_input_type_number;
+    v->visitor.type_any = qmp_input_type_any;
     v->visitor.optional = qmp_input_optional;
     v->visitor.get_next_type = qmp_input_get_next_type;
 
diff --git a/qapi/qmp-output-visitor.c b/qapi/qmp-output-visitor.c
index efc19d584..43cfebfe4 100644
--- a/qapi/qmp-output-visitor.c
+++ b/qapi/qmp-output-visitor.c
@@ -29,6 +29,15 @@ typedef QTAILQ_HEAD(QStack, QStackEntry) QStack;
 struct QmpOutputVisitor
 {
     Visitor visitor;
+    /* FIXME: we are abusing stack to hold two separate pieces of
+     * information: the current root object in slot 0, and the stack
+     * of N objects still being built in slots 1 through N (for N+1
+     * slots in use).  Worse, our behavior is inconsistent:
+     * qmp_output_add_obj() visiting two top-level scalars in a row
+     * discards the first in favor of the second, but visiting two
+     * top-level objects in a row tries to append the second object
+     * into the first (since the first object was placed in the stack
+     * in both slot 0 and 1, but only popped from slot 1).  */
     QStack stack;
 };
 
@@ -41,10 +50,12 @@ static QmpOutputVisitor *to_qov(Visitor *v)
     return container_of(v, QmpOutputVisitor, visitor);
 }
 
+/* Push @value onto the stack of current QObjects being built */
 static void qmp_output_push_obj(QmpOutputVisitor *qov, QObject *value)
 {
     QStackEntry *e = g_malloc0(sizeof(*e));
 
+    assert(value);
     e->value = value;
     if (qobject_type(e->value) == QTYPE_QLIST) {
         e->is_list_head = true;
@@ -52,40 +63,53 @@ static void qmp_output_push_obj(QmpOutputVisitor *qov, QObject *value)
     QTAILQ_INSERT_HEAD(&qov->stack, e, node);
 }
 
+/* Pop a value off the stack of QObjects being built, and return it. */
 static QObject *qmp_output_pop(QmpOutputVisitor *qov)
 {
     QStackEntry *e = QTAILQ_FIRST(&qov->stack);
     QObject *value;
+
+    assert(e);
     QTAILQ_REMOVE(&qov->stack, e, node);
     value = e->value;
+    assert(value);
     g_free(e);
     return value;
 }
 
+/* Grab the root QObject, if any */
 static QObject *qmp_output_first(QmpOutputVisitor *qov)
 {
     QStackEntry *e = QTAILQ_LAST(&qov->stack, QStack);
 
-    /* FIXME - find a better way to deal with NULL values */
     if (!e) {
+        /* No root */
         return NULL;
     }
-
+    assert(e->value);
     return e->value;
 }
 
+/* Peek at the top of the stack of QObjects being built.
+ * The stack must not be empty. */
 static QObject *qmp_output_last(QmpOutputVisitor *qov)
 {
     QStackEntry *e = QTAILQ_FIRST(&qov->stack);
+
+    assert(e && e->value);
     return e->value;
 }
 
+/* Add @value to the current QObject being built.
+ * If the stack is visiting a dictionary or list, @value is now owned
+ * by that container. Otherwise, @value is now the root.  */
 static void qmp_output_add_obj(QmpOutputVisitor *qov, const char *name,
                                QObject *value)
 {
     QObject *cur;
 
     if (QTAILQ_EMPTY(&qov->stack)) {
+        /* Stack was empty, track this object as root */
         qmp_output_push_obj(qov, value);
         return;
     }
@@ -94,13 +118,17 @@ static void qmp_output_add_obj(QmpOutputVisitor *qov, const char *name,
 
     switch (qobject_type(cur)) {
     case QTYPE_QDICT:
+        assert(name);
         qdict_put_obj(qobject_to_qdict(cur), name, value);
         break;
     case QTYPE_QLIST:
         qlist_append_obj(qobject_to_qlist(cur), value);
         break;
     default:
+        /* The previous root was a scalar, replace it with a new root */
+        /* FIXME this is abusing the stack; see comment above */
         qobject_decref(qmp_output_pop(qov));
+        assert(QTAILQ_EMPTY(&qov->stack));
         qmp_output_push_obj(qov, value);
         break;
     }
@@ -186,11 +214,22 @@ static void qmp_output_type_number(Visitor *v, double *obj, const char *name,
     qmp_output_add(qov, name, qfloat_from_double(*obj));
 }
 
+static void qmp_output_type_any(Visitor *v, QObject **obj, const char *name,
+                                Error **errp)
+{
+    QmpOutputVisitor *qov = to_qov(v);
+    qobject_incref(*obj);
+    qmp_output_add_obj(qov, name, *obj);
+}
+
+/* Finish building, and return the root object. Will not be NULL. */
 QObject *qmp_output_get_qobject(QmpOutputVisitor *qov)
 {
     QObject *obj = qmp_output_first(qov);
     if (obj) {
         qobject_incref(obj);
+    } else {
+        obj = qnull();
     }
     return obj;
 }
@@ -233,6 +272,7 @@ QmpOutputVisitor *qmp_output_visitor_new(void)
     v->visitor.type_bool = qmp_output_type_bool;
     v->visitor.type_str = qmp_output_type_str;
     v->visitor.type_number = qmp_output_type_number;
+    v->visitor.type_any = qmp_output_type_any;
 
     QTAILQ_INIT(&v->stack);
 
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 0e147478c..a35098f71 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -50,6 +50,7 @@ static const QDevAlias qdev_alias_table[] = {
     { "lsi53c895a", "lsi" },
     { "ich9-ahci", "ahci" },
     { "kvm-pci-assign", "pci-assign" },
+    { "e1000", "e1000-82540em" },
     { }
 };
 
@@ -292,37 +293,35 @@ static Object *qdev_get_peripheral_anon(void)
     return dev;
 }
 
-#if 0 /* conversion from qerror_report() to error_set() broke their use */
-static void qbus_list_bus(DeviceState *dev)
+static void qbus_list_bus(DeviceState *dev, Error **errp)
 {
     BusState *child;
     const char *sep = " ";
 
-    error_printf("child buses at \"%s\":",
-                 dev->id ? dev->id : object_get_typename(OBJECT(dev)));
+    error_append_hint(errp, "child buses at \"%s\":",
+                      dev->id ? dev->id : object_get_typename(OBJECT(dev)));
     QLIST_FOREACH(child, &dev->child_bus, sibling) {
-        error_printf("%s\"%s\"", sep, child->name);
+        error_append_hint(errp, "%s\"%s\"", sep, child->name);
         sep = ", ";
     }
-    error_printf("\n");
 }
 
-static void qbus_list_dev(BusState *bus)
+static void qbus_list_dev(BusState *bus, Error **errp)
 {
     BusChild *kid;
     const char *sep = " ";
 
-    error_printf("devices at \"%s\":", bus->name);
+    error_append_hint(errp, "devices at \"%s\":", bus->name);
     QTAILQ_FOREACH(kid, &bus->children, sibling) {
         DeviceState *dev = kid->child;
-        error_printf("%s\"%s\"", sep, object_get_typename(OBJECT(dev)));
-        if (dev->id)
-            error_printf("/\"%s\"", dev->id);
+        error_append_hint(errp, "%s\"%s\"", sep,
+                          object_get_typename(OBJECT(dev)));
+        if (dev->id) {
+            error_append_hint(errp, "/\"%s\"", dev->id);
+        }
         sep = ", ";
     }
-    error_printf("\n");
 }
-#endif
 
 static BusState *qbus_find_bus(DeviceState *dev, char *elem)
 {
@@ -464,11 +463,7 @@ static BusState *qbus_find(const char *path, Error **errp)
         if (!dev) {
             error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                       "Device '%s' not found", elem);
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            if (!monitor_cur_is_qmp()) {
-                qbus_list_dev(bus);
-            }
-#endif
+            qbus_list_dev(bus, errp);
             return NULL;
         }
 
@@ -486,11 +481,7 @@ static BusState *qbus_find(const char *path, Error **errp)
             if (dev->num_child_bus) {
                 error_setg(errp, "Device '%s' has multiple child buses",
                            elem);
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-                if (!monitor_cur_is_qmp()) {
-                    qbus_list_bus(dev);
-                }
-#endif
+                qbus_list_bus(dev, errp);
             } else {
                 error_setg(errp, "Device '%s' has no child bus", elem);
             }
@@ -506,11 +497,7 @@ static BusState *qbus_find(const char *path, Error **errp)
         bus = qbus_find_bus(dev, elem);
         if (!bus) {
             error_setg(errp, "Bus '%s' not found", elem);
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            if (!monitor_cur_is_qmp()) {
-                qbus_list_bus(dev);
-            }
-#endif
+            qbus_list_bus(dev, errp);
             return NULL;
         }
     }
@@ -788,12 +775,17 @@ void qmp_device_add(QDict *qdict, QObject **ret_data, Error **errp)
 void qmp_device_del(const char *id, Error **errp)
 {
     Object *obj;
-    char *root_path = object_get_canonical_path(qdev_get_peripheral());
-    char *path = g_strdup_printf("%s/%s", root_path, id);
 
-    g_free(root_path);
-    obj = object_resolve_path_type(path, TYPE_DEVICE, NULL);
-    g_free(path);
+    if (id[0] == '/') {
+        obj = object_resolve_path(id, NULL);
+    } else {
+        char *root_path = object_get_canonical_path(qdev_get_peripheral());
+        char *path = g_strdup_printf("%s/%s", root_path, id);
+
+        g_free(root_path);
+        obj = object_resolve_path_type(path, TYPE_DEVICE, NULL);
+        g_free(path);
+    }
 
     if (!obj) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
@@ -801,6 +793,11 @@ void qmp_device_del(const char *id, Error **errp)
         return;
     }
 
+    if (!object_dynamic_cast(obj, TYPE_DEVICE)) {
+        error_setg(errp, "%s is not a hotpluggable device", id);
+        return;
+    }
+
     qdev_unplug(DEVICE(obj), errp);
 }
 
diff --git a/qemu-char.c b/qemu-char.c
index d956f8db6..2969c44e8 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -51,7 +51,6 @@
 #include <netinet/in.h>
 #include <net/if.h>
 #include <arpa/inet.h>
-#include <dirent.h>
 #include <netdb.h>
 #include <sys/select.h>
 #ifdef CONFIG_BSD
@@ -93,48 +92,23 @@
 
 /***********************************************************/
 /* Socket address helpers */
-static void qapi_copy_SocketAddress(SocketAddress **p_dest,
-                                    SocketAddress *src)
-{
-    QmpOutputVisitor *qov;
-    QmpInputVisitor *qiv;
-    Visitor *ov, *iv;
-    QObject *obj;
-
-    *p_dest = NULL;
-
-    qov = qmp_output_visitor_new();
-    ov = qmp_output_get_visitor(qov);
-    visit_type_SocketAddress(ov, &src, NULL, &error_abort);
-    obj = qmp_output_get_qobject(qov);
-    qmp_output_visitor_cleanup(qov);
-    if (!obj) {
-        return;
-    }
-
-    qiv = qmp_input_visitor_new(obj);
-    iv = qmp_input_get_visitor(qiv);
-    visit_type_SocketAddress(iv, p_dest, NULL, &error_abort);
-    qmp_input_visitor_cleanup(qiv);
-    qobject_decref(obj);
-}
 
 static int SocketAddress_to_str(char *dest, int max_len,
                                 const char *prefix, SocketAddress *addr,
                                 bool is_listen, bool is_telnet)
 {
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
         return snprintf(dest, max_len, "%s%s:%s:%s%s", prefix,
-                        is_telnet ? "telnet" : "tcp", addr->inet->host,
-                        addr->inet->port, is_listen ? ",server" : "");
+                        is_telnet ? "telnet" : "tcp", addr->u.inet->host,
+                        addr->u.inet->port, is_listen ? ",server" : "");
         break;
     case SOCKET_ADDRESS_KIND_UNIX:
         return snprintf(dest, max_len, "%sunix:%s%s", prefix,
-                        addr->q_unix->path, is_listen ? ",server" : "");
+                        addr->u.q_unix->path, is_listen ? ",server" : "");
         break;
     case SOCKET_ADDRESS_KIND_FD:
-        return snprintf(dest, max_len, "%sfd:%s%s", prefix, addr->fd->str,
+        return snprintf(dest, max_len, "%sfd:%s%s", prefix, addr->u.fd->str,
                         is_listen ? ",server" : "");
         break;
     default:
@@ -384,7 +358,10 @@ static int null_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
     return len;
 }
 
-static CharDriverState *qemu_chr_open_null(void)
+static CharDriverState *qemu_chr_open_null(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
     CharDriverState *chr;
 
@@ -680,13 +657,22 @@ static GSource *mux_chr_add_watch(CharDriverState *s, GIOCondition cond)
     return d->drv->chr_add_watch(d->drv, cond);
 }
 
-static CharDriverState *qemu_chr_open_mux(CharDriverState *drv)
+static CharDriverState *qemu_chr_open_mux(const char *id,
+                                          ChardevBackend *backend,
+                                          ChardevReturn *ret, Error **errp)
 {
-    CharDriverState *chr;
+    ChardevMux *mux = backend->u.mux;
+    CharDriverState *chr, *drv;
     MuxDriver *d;
 
+    drv = qemu_chr_find(mux->chardev);
+    if (drv == NULL) {
+        error_setg(errp, "mux: base chardev %s not found", mux->chardev);
+        return NULL;
+    }
+
     chr = qemu_chr_alloc();
-    d = g_malloc0(sizeof(MuxDriver));
+    d = g_new0(MuxDriver, 1);
 
     chr->opaque = d;
     d->drv = drv;
@@ -1065,7 +1051,7 @@ static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out)
     FDCharDriver *s;
 
     chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(FDCharDriver));
+    s = g_new0(FDCharDriver, 1);
     s->fd_in = io_channel_from_fd(fd_in);
     s->fd_out = io_channel_from_fd(fd_out);
     qemu_set_nonblock(fd_out);
@@ -1079,18 +1065,17 @@ static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out)
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
+static CharDriverState *qemu_chr_open_pipe(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
+    ChardevHostdev *opts = backend->u.pipe;
     int fd_in, fd_out;
     char filename_in[CHR_MAX_FILENAME_SIZE];
     char filename_out[CHR_MAX_FILENAME_SIZE];
     const char *filename = opts->device;
 
-    if (filename == NULL) {
-        fprintf(stderr, "chardev: pipe: no filename given\n");
-        return NULL;
-    }
-
     snprintf(filename_in, CHR_MAX_FILENAME_SIZE, "%s.in", filename);
     snprintf(filename_out, CHR_MAX_FILENAME_SIZE, "%s.out", filename);
     TFR(fd_in = qemu_open(filename_in, O_RDWR | O_BINARY));
@@ -1102,6 +1087,7 @@ static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
 	    close(fd_out);
         TFR(fd_in = fd_out = qemu_open(filename, O_RDWR | O_BINARY));
         if (fd_in < 0) {
+            error_setg_file_open(errp, errno, filename);
             return NULL;
         }
     }
@@ -1157,19 +1143,23 @@ static void qemu_chr_close_stdio(struct CharDriverState *chr)
     fd_chr_close(chr);
 }
 
-static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
+static CharDriverState *qemu_chr_open_stdio(const char *id,
+                                            ChardevBackend *backend,
+                                            ChardevReturn *ret,
+                                            Error **errp)
 {
+    ChardevStdio *opts = backend->u.stdio;
     CharDriverState *chr;
     struct sigaction act;
 
     if (is_daemonized()) {
-        error_report("cannot use stdio with -daemonize");
+        error_setg(errp, "cannot use stdio with -daemonize");
         return NULL;
     }
 
     if (stdio_in_use) {
-        error_report("cannot use stdio by multiple character devices");
-        exit(1);
+        error_setg(errp, "cannot use stdio by multiple character devices");
+        return NULL;
     }
 
     stdio_in_use = true;
@@ -1197,7 +1187,8 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) \
     || defined(__GLIBC__)
 
-#define HAVE_CHARDEV_TTY 1
+#define HAVE_CHARDEV_SERIAL 1
+#define HAVE_CHARDEV_PTY 1
 
 typedef struct {
     GIOChannel *fd;
@@ -1250,11 +1241,16 @@ static void pty_chr_update_read_handler_locked(CharDriverState *chr)
 {
     PtyCharDriver *s = chr->opaque;
     GPollFD pfd;
+    int rc;
 
     pfd.fd = g_io_channel_unix_get_fd(s->fd);
     pfd.events = G_IO_OUT;
     pfd.revents = 0;
-    g_poll(&pfd, 1, 0);
+    do {
+        rc = g_poll(&pfd, 1, 0);
+    } while (rc == -1 && errno == EINTR);
+    assert(rc >= 0);
+
     if (pfd.revents & G_IO_HUP) {
         pty_chr_state(chr, 0);
     } else {
@@ -1390,7 +1386,9 @@ static void pty_chr_close(struct CharDriverState *chr)
 }
 
 static CharDriverState *qemu_chr_open_pty(const char *id,
-                                          ChardevReturn *ret)
+                                          ChardevBackend *backend,
+                                          ChardevReturn *ret,
+                                          Error **errp)
 {
     CharDriverState *chr;
     PtyCharDriver *s;
@@ -1399,6 +1397,7 @@ static CharDriverState *qemu_chr_open_pty(const char *id,
 
     master_fd = qemu_openpty_raw(&slave_fd, pty_name);
     if (master_fd < 0) {
+        error_setg_errno(errp, errno, "Failed to create PTY");
         return NULL;
     }
 
@@ -1414,7 +1413,7 @@ static CharDriverState *qemu_chr_open_pty(const char *id,
     fprintf(stderr, "char device redirected to %s (label %s)\n",
             pty_name, id);
 
-    s = g_malloc0(sizeof(PtyCharDriver));
+    s = g_new0(PtyCharDriver, 1);
     chr->opaque = s;
     chr->chr_write = pty_chr_write;
     chr->chr_update_read_handler = pty_chr_update_read_handler;
@@ -1753,17 +1752,18 @@ static void pp_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_pp_fd(int fd)
+static CharDriverState *qemu_chr_open_pp_fd(int fd, Error **errp)
 {
     CharDriverState *chr;
     ParallelCharDriver *drv;
 
     if (ioctl(fd, PPCLAIM) < 0) {
+        error_setg_errno(errp, errno, "not a parallel port");
         close(fd);
         return NULL;
     }
 
-    drv = g_malloc0(sizeof(ParallelCharDriver));
+    drv = g_new0(ParallelCharDriver, 1);
     drv->fd = fd;
     drv->mode = IEEE1284_MODE_COMPAT;
 
@@ -1818,7 +1818,7 @@ static int pp_ioctl(CharDriverState *chr, int cmd, void *arg)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_pp_fd(int fd)
+static CharDriverState *qemu_chr_open_pp_fd(int fd, Error **errp)
 {
     CharDriverState *chr;
 
@@ -1833,6 +1833,8 @@ static CharDriverState *qemu_chr_open_pp_fd(int fd)
 
 #else /* _WIN32 */
 
+#define HAVE_CHARDEV_SERIAL 1
+
 typedef struct {
     int max_size;
     HANDLE hcom, hrecv, hsend;
@@ -1884,7 +1886,7 @@ static void win_chr_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static int win_chr_init(CharDriverState *chr, const char *filename)
+static int win_chr_init(CharDriverState *chr, const char *filename, Error **errp)
 {
     WinCharState *s = chr->opaque;
     COMMCONFIG comcfg;
@@ -1895,25 +1897,25 @@ static int win_chr_init(CharDriverState *chr, const char *filename)
 
     s->hsend = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hsend) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
     s->hrecv = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hrecv) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
 
     s->hcom = CreateFile(filename, GENERIC_READ|GENERIC_WRITE, 0, NULL,
                       OPEN_EXISTING, FILE_FLAG_OVERLAPPED, 0);
     if (s->hcom == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "Failed CreateFile (%lu)\n", GetLastError());
+        error_setg(errp, "Failed CreateFile (%lu)", GetLastError());
         s->hcom = NULL;
         goto fail;
     }
 
     if (!SetupComm(s->hcom, NRECVBUF, NSENDBUF)) {
-        fprintf(stderr, "Failed SetupComm\n");
+        error_setg(errp, "Failed SetupComm");
         goto fail;
     }
 
@@ -1924,23 +1926,23 @@ static int win_chr_init(CharDriverState *chr, const char *filename)
     CommConfigDialog(filename, NULL, &comcfg);
 
     if (!SetCommState(s->hcom, &comcfg.dcb)) {
-        fprintf(stderr, "Failed SetCommState\n");
+        error_setg(errp, "Failed SetCommState");
         goto fail;
     }
 
     if (!SetCommMask(s->hcom, EV_ERR)) {
-        fprintf(stderr, "Failed SetCommMask\n");
+        error_setg(errp, "Failed SetCommMask");
         goto fail;
     }
 
     cto.ReadIntervalTimeout = MAXDWORD;
     if (!SetCommTimeouts(s->hcom, &cto)) {
-        fprintf(stderr, "Failed SetCommTimeouts\n");
+        error_setg(errp, "Failed SetCommTimeouts");
         goto fail;
     }
 
     if (!ClearCommError(s->hcom, &err, &comstat)) {
-        fprintf(stderr, "Failed ClearCommError\n");
+        error_setg(errp, "Failed ClearCommError");
         goto fail;
     }
     qemu_add_polling_cb(win_chr_poll, chr);
@@ -2045,18 +2047,19 @@ static int win_chr_poll(void *opaque)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_win_path(const char *filename)
+static CharDriverState *qemu_chr_open_win_path(const char *filename,
+                                               Error **errp)
 {
     CharDriverState *chr;
     WinCharState *s;
 
     chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    s = g_new0(WinCharState, 1);
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     chr->chr_close = win_chr_close;
 
-    if (win_chr_init(chr, filename) < 0) {
+    if (win_chr_init(chr, filename, errp) < 0) {
         g_free(s);
         g_free(chr);
         return NULL;
@@ -2080,7 +2083,8 @@ static int win_chr_pipe_poll(void *opaque)
     return 0;
 }
 
-static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
+static int win_chr_pipe_init(CharDriverState *chr, const char *filename,
+                             Error **errp)
 {
     WinCharState *s = chr->opaque;
     OVERLAPPED ov;
@@ -2092,12 +2096,12 @@ static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
 
     s->hsend = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hsend) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
     s->hrecv = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hrecv) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
 
@@ -2107,7 +2111,7 @@ static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
                               PIPE_WAIT,
                               MAXCONNECT, NSENDBUF, NRECVBUF, NTIMEOUT, NULL);
     if (s->hcom == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "Failed CreateNamedPipe (%lu)\n", GetLastError());
+        error_setg(errp, "Failed CreateNamedPipe (%lu)", GetLastError());
         s->hcom = NULL;
         goto fail;
     }
@@ -2116,13 +2120,13 @@ static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
     ov.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
     ret = ConnectNamedPipe(s->hcom, &ov);
     if (ret) {
-        fprintf(stderr, "Failed ConnectNamedPipe\n");
+        error_setg(errp, "Failed ConnectNamedPipe");
         goto fail;
     }
 
     ret = GetOverlappedResult(s->hcom, &ov, &size, TRUE);
     if (!ret) {
-        fprintf(stderr, "Failed GetOverlappedResult\n");
+        error_setg(errp, "Failed GetOverlappedResult");
         if (ov.hEvent) {
             CloseHandle(ov.hEvent);
             ov.hEvent = NULL;
@@ -2143,19 +2147,23 @@ static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
 }
 
 
-static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
+static CharDriverState *qemu_chr_open_pipe(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
+    ChardevHostdev *opts = backend->u.pipe;
     const char *filename = opts->device;
     CharDriverState *chr;
     WinCharState *s;
 
     chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    s = g_new0(WinCharState, 1);
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     chr->chr_close = win_chr_close;
 
-    if (win_chr_pipe_init(chr, filename) < 0) {
+    if (win_chr_pipe_init(chr, filename, errp) < 0) {
         g_free(s);
         g_free(chr);
         return NULL;
@@ -2169,14 +2177,17 @@ static CharDriverState *qemu_chr_open_win_file(HANDLE fd_out)
     WinCharState *s;
 
     chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    s = g_new0(WinCharState, 1);
     s->hcom = fd_out;
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_win_con(void)
+static CharDriverState *qemu_chr_open_win_con(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
     return qemu_chr_open_win_file(GetStdHandle(STD_OUTPUT_HANDLE));
 }
@@ -2317,7 +2328,10 @@ static void win_stdio_close(CharDriverState *chr)
     g_free(chr);
 }
 
-static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
+static CharDriverState *qemu_chr_open_stdio(const char *id,
+                                            ChardevBackend *backend,
+                                            ChardevReturn *ret,
+                                            Error **errp)
 {
     CharDriverState   *chr;
     WinStdioCharState *stdio;
@@ -2325,12 +2339,12 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     int                is_console = 0;
 
     chr   = qemu_chr_alloc();
-    stdio = g_malloc0(sizeof(WinStdioCharState));
+    stdio = g_new0(WinStdioCharState, 1);
 
     stdio->hStdIn = GetStdHandle(STD_INPUT_HANDLE);
     if (stdio->hStdIn == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "cannot open stdio: invalid handle\n");
-        exit(1);
+        error_setg(errp, "cannot open stdio: invalid handle");
+        return NULL;
     }
 
     is_console = GetConsoleMode(stdio->hStdIn, &dwMode) != 0;
@@ -2342,25 +2356,30 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     if (is_console) {
         if (qemu_add_wait_object(stdio->hStdIn,
                                  win_stdio_wait_func, chr)) {
-            fprintf(stderr, "qemu_add_wait_object: failed\n");
+            error_setg(errp, "qemu_add_wait_object: failed");
+            goto err1;
         }
     } else {
         DWORD   dwId;
             
         stdio->hInputReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
         stdio->hInputDoneEvent  = CreateEvent(NULL, FALSE, FALSE, NULL);
-        stdio->hInputThread     = CreateThread(NULL, 0, win_stdio_thread,
-                                               chr, 0, &dwId);
-
-        if (stdio->hInputThread == INVALID_HANDLE_VALUE
-            || stdio->hInputReadyEvent == INVALID_HANDLE_VALUE
+        if (stdio->hInputReadyEvent == INVALID_HANDLE_VALUE
             || stdio->hInputDoneEvent == INVALID_HANDLE_VALUE) {
-            fprintf(stderr, "cannot create stdio thread or event\n");
-            exit(1);
+            error_setg(errp, "cannot create event");
+            goto err2;
         }
         if (qemu_add_wait_object(stdio->hInputReadyEvent,
                                  win_stdio_thread_wait_func, chr)) {
-            fprintf(stderr, "qemu_add_wait_object: failed\n");
+            error_setg(errp, "qemu_add_wait_object: failed");
+            goto err2;
+        }
+        stdio->hInputThread     = CreateThread(NULL, 0, win_stdio_thread,
+                                               chr, 0, &dwId);
+
+        if (stdio->hInputThread == INVALID_HANDLE_VALUE) {
+            error_setg(errp, "cannot create stdio thread");
+            goto err3;
         }
     }
 
@@ -2378,6 +2397,15 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     qemu_chr_fe_set_echo(chr, false);
 
     return chr;
+
+err3:
+    qemu_del_wait_object(stdio->hInputReadyEvent, NULL, NULL);
+err2:
+    CloseHandle(stdio->hInputReadyEvent);
+    CloseHandle(stdio->hInputDoneEvent);
+err1:
+    qemu_del_wait_object(stdio->hStdIn, NULL, NULL);
+    return NULL;
 }
 #endif /* !_WIN32 */
 
@@ -2488,7 +2516,7 @@ static CharDriverState *qemu_chr_open_udp_fd(int fd)
     NetCharDriver *s = NULL;
 
     chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(NetCharDriver));
+    s = g_new0(NetCharDriver, 1);
 
     s->fd = fd;
     s->chan = io_channel_from_socket(s->fd);
@@ -2711,12 +2739,10 @@ static int tcp_set_msgfds(CharDriverState *chr, int *fds, int num)
     TCPCharDriver *s = chr->opaque;
 
     /* clear old pending fd array */
-    if (s->write_msgfds) {
-        g_free(s->write_msgfds);
-    }
+    g_free(s->write_msgfds);
 
     if (num) {
-        s->write_msgfds = g_malloc(num * sizeof(int));
+        s->write_msgfds = g_new(int, num);
         memcpy(s->write_msgfds, fds, num * sizeof(int));
     }
 
@@ -3176,9 +3202,12 @@ static void ringbuf_chr_close(struct CharDriverState *chr)
     chr->opaque = NULL;
 }
 
-static CharDriverState *qemu_chr_open_ringbuf(ChardevRingbuf *opts,
+static CharDriverState *qemu_chr_open_ringbuf(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
                                               Error **errp)
 {
+    ChardevRingbuf *opts = backend->u.ringbuf;
     CharDriverState *chr;
     RingBufCharDriver *d;
 
@@ -3453,18 +3482,19 @@ static void qemu_chr_parse_file_out(QemuOpts *opts, ChardevBackend *backend,
         error_setg(errp, "chardev: file: no filename given");
         return;
     }
-    backend->file = g_new0(ChardevFile, 1);
-    backend->file->out = g_strdup(path);
+    backend->u.file = g_new0(ChardevFile, 1);
+    backend->u.file->out = g_strdup(path);
 }
 
 static void qemu_chr_parse_stdio(QemuOpts *opts, ChardevBackend *backend,
                                  Error **errp)
 {
-    backend->stdio = g_new0(ChardevStdio, 1);
-    backend->stdio->has_signal = true;
-    backend->stdio->signal = qemu_opt_get_bool(opts, "signal", true);
+    backend->u.stdio = g_new0(ChardevStdio, 1);
+    backend->u.stdio->has_signal = true;
+    backend->u.stdio->signal = qemu_opt_get_bool(opts, "signal", true);
 }
 
+#ifdef HAVE_CHARDEV_SERIAL
 static void qemu_chr_parse_serial(QemuOpts *opts, ChardevBackend *backend,
                                   Error **errp)
 {
@@ -3474,10 +3504,12 @@ static void qemu_chr_parse_serial(QemuOpts *opts, ChardevBackend *backend,
         error_setg(errp, "chardev: serial/tty: no device path given");
         return;
     }
-    backend->serial = g_new0(ChardevHostdev, 1);
-    backend->serial->device = g_strdup(device);
+    backend->u.serial = g_new0(ChardevHostdev, 1);
+    backend->u.serial->device = g_strdup(device);
 }
+#endif
 
+#ifdef HAVE_CHARDEV_PARPORT
 static void qemu_chr_parse_parallel(QemuOpts *opts, ChardevBackend *backend,
                                     Error **errp)
 {
@@ -3487,9 +3519,10 @@ static void qemu_chr_parse_parallel(QemuOpts *opts, ChardevBackend *backend,
         error_setg(errp, "chardev: parallel: no device path given");
         return;
     }
-    backend->parallel = g_new0(ChardevHostdev, 1);
-    backend->parallel->device = g_strdup(device);
+    backend->u.parallel = g_new0(ChardevHostdev, 1);
+    backend->u.parallel->device = g_strdup(device);
 }
+#endif
 
 static void qemu_chr_parse_pipe(QemuOpts *opts, ChardevBackend *backend,
                                 Error **errp)
@@ -3500,8 +3533,8 @@ static void qemu_chr_parse_pipe(QemuOpts *opts, ChardevBackend *backend,
         error_setg(errp, "chardev: pipe: no device path given");
         return;
     }
-    backend->pipe = g_new0(ChardevHostdev, 1);
-    backend->pipe->device = g_strdup(device);
+    backend->u.pipe = g_new0(ChardevHostdev, 1);
+    backend->u.pipe->device = g_strdup(device);
 }
 
 static void qemu_chr_parse_ringbuf(QemuOpts *opts, ChardevBackend *backend,
@@ -3509,12 +3542,12 @@ static void qemu_chr_parse_ringbuf(QemuOpts *opts, ChardevBackend *backend,
 {
     int val;
 
-    backend->ringbuf = g_new0(ChardevRingbuf, 1);
+    backend->u.ringbuf = g_new0(ChardevRingbuf, 1);
 
     val = qemu_opt_get_size(opts, "size", 0);
     if (val != 0) {
-        backend->ringbuf->has_size = true;
-        backend->ringbuf->size = val;
+        backend->u.ringbuf->has_size = true;
+        backend->u.ringbuf->size = val;
     }
 }
 
@@ -3527,8 +3560,8 @@ static void qemu_chr_parse_mux(QemuOpts *opts, ChardevBackend *backend,
         error_setg(errp, "chardev: mux: no chardev given");
         return;
     }
-    backend->mux = g_new0(ChardevMux, 1);
-    backend->mux->chardev = g_strdup(chardev);
+    backend->u.mux = g_new0(ChardevMux, 1);
+    backend->u.mux->chardev = g_strdup(chardev);
 }
 
 static void qemu_chr_parse_socket(QemuOpts *opts, ChardevBackend *backend,
@@ -3555,37 +3588,37 @@ static void qemu_chr_parse_socket(QemuOpts *opts, ChardevBackend *backend,
         }
     }
 
-    backend->socket = g_new0(ChardevSocket, 1);
+    backend->u.socket = g_new0(ChardevSocket, 1);
 
-    backend->socket->has_nodelay = true;
-    backend->socket->nodelay = do_nodelay;
-    backend->socket->has_server = true;
-    backend->socket->server = is_listen;
-    backend->socket->has_telnet = true;
-    backend->socket->telnet = is_telnet;
-    backend->socket->has_wait = true;
-    backend->socket->wait = is_waitconnect;
-    backend->socket->has_reconnect = true;
-    backend->socket->reconnect = reconnect;
+    backend->u.socket->has_nodelay = true;
+    backend->u.socket->nodelay = do_nodelay;
+    backend->u.socket->has_server = true;
+    backend->u.socket->server = is_listen;
+    backend->u.socket->has_telnet = true;
+    backend->u.socket->telnet = is_telnet;
+    backend->u.socket->has_wait = true;
+    backend->u.socket->wait = is_waitconnect;
+    backend->u.socket->has_reconnect = true;
+    backend->u.socket->reconnect = reconnect;
 
     addr = g_new0(SocketAddress, 1);
     if (path) {
-        addr->kind = SOCKET_ADDRESS_KIND_UNIX;
-        addr->q_unix = g_new0(UnixSocketAddress, 1);
-        addr->q_unix->path = g_strdup(path);
+        addr->type = SOCKET_ADDRESS_KIND_UNIX;
+        addr->u.q_unix = g_new0(UnixSocketAddress, 1);
+        addr->u.q_unix->path = g_strdup(path);
     } else {
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = g_new0(InetSocketAddress, 1);
-        addr->inet->host = g_strdup(host);
-        addr->inet->port = g_strdup(port);
-        addr->inet->has_to = qemu_opt_get(opts, "to");
-        addr->inet->to = qemu_opt_get_number(opts, "to", 0);
-        addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
-        addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
-        addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
-        addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet = g_new0(InetSocketAddress, 1);
+        addr->u.inet->host = g_strdup(host);
+        addr->u.inet->port = g_strdup(port);
+        addr->u.inet->has_to = qemu_opt_get(opts, "to");
+        addr->u.inet->to = qemu_opt_get_number(opts, "to", 0);
+        addr->u.inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
+        addr->u.inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
+        addr->u.inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
+        addr->u.inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
     }
-    backend->socket->addr = addr;
+    backend->u.socket->addr = addr;
 }
 
 static void qemu_chr_parse_udp(QemuOpts *opts, ChardevBackend *backend,
@@ -3616,27 +3649,27 @@ static void qemu_chr_parse_udp(QemuOpts *opts, ChardevBackend *backend,
         has_local = true;
     }
 
-    backend->udp = g_new0(ChardevUdp, 1);
+    backend->u.udp = g_new0(ChardevUdp, 1);
 
     addr = g_new0(SocketAddress, 1);
-    addr->kind = SOCKET_ADDRESS_KIND_INET;
-    addr->inet = g_new0(InetSocketAddress, 1);
-    addr->inet->host = g_strdup(host);
-    addr->inet->port = g_strdup(port);
-    addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
-    addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
-    addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
-    addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
-    backend->udp->remote = addr;
+    addr->type = SOCKET_ADDRESS_KIND_INET;
+    addr->u.inet = g_new0(InetSocketAddress, 1);
+    addr->u.inet->host = g_strdup(host);
+    addr->u.inet->port = g_strdup(port);
+    addr->u.inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
+    addr->u.inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
+    addr->u.inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
+    addr->u.inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
+    backend->u.udp->remote = addr;
 
     if (has_local) {
-        backend->udp->has_local = true;
+        backend->u.udp->has_local = true;
         addr = g_new0(SocketAddress, 1);
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = g_new0(InetSocketAddress, 1);
-        addr->inet->host = g_strdup(localaddr);
-        addr->inet->port = g_strdup(localport);
-        backend->udp->local = addr;
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet = g_new0(InetSocketAddress, 1);
+        addr->u.inet->host = g_strdup(localaddr);
+        addr->u.inet->port = g_strdup(localport);
+        backend->u.udp->local = addr;
     }
 }
 
@@ -3644,12 +3677,16 @@ typedef struct CharDriver {
     const char *name;
     ChardevBackendKind kind;
     void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp);
+    CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                               ChardevReturn *ret, Error **errp);
 } CharDriver;
 
 static GSList *backends;
 
 void register_char_driver(const char *name, ChardevBackendKind kind,
-        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp))
+        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp),
+        CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                                   ChardevReturn *ret, Error **errp))
 {
     CharDriver *s;
 
@@ -3657,6 +3694,7 @@ void register_char_driver(const char *name, ChardevBackendKind kind,
     s->name = g_strdup(name);
     s->kind = kind;
     s->parse = parse;
+    s->create = create;
 
     backends = g_slist_append(backends, s);
 }
@@ -3704,7 +3742,7 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
     }
 
     chr = NULL;
-    backend->kind = cd->kind;
+    backend->type = cd->kind;
     if (cd->parse) {
         cd->parse(opts, backend, &local_err);
         if (local_err) {
@@ -3721,9 +3759,9 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
         qapi_free_ChardevBackend(backend);
         qapi_free_ChardevReturn(ret);
         backend = g_new0(ChardevBackend, 1);
-        backend->mux = g_new0(ChardevMux, 1);
-        backend->kind = CHARDEV_BACKEND_KIND_MUX;
-        backend->mux->chardev = g_strdup(bid);
+        backend->u.mux = g_new0(ChardevMux, 1);
+        backend->type = CHARDEV_BACKEND_KIND_MUX;
+        backend->u.mux->chardev = g_strdup(bid);
         ret = qmp_chardev_add(id, backend, errp);
         if (!ret) {
             chr = qemu_chr_find(bid);
@@ -3843,9 +3881,8 @@ void qemu_chr_fe_release(CharDriverState *s)
     s->avail_connections++;
 }
 
-void qemu_chr_delete(CharDriverState *chr)
+void qemu_chr_free(CharDriverState *chr)
 {
-    QTAILQ_REMOVE(&chardevs, chr, next);
     if (chr->chr_close) {
         chr->chr_close(chr);
     }
@@ -3855,6 +3892,12 @@ void qemu_chr_delete(CharDriverState *chr)
     g_free(chr);
 }
 
+void qemu_chr_delete(CharDriverState *chr)
+{
+    QTAILQ_REMOVE(&chardevs, chr, next);
+    qemu_chr_free(chr);
+}
+
 ChardevInfoList *qmp_query_chardev(Error **errp)
 {
     ChardevInfoList *chr_list = NULL;
@@ -4005,8 +4048,12 @@ QemuOptsList qemu_chardev_opts = {
 
 #ifdef _WIN32
 
-static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+static CharDriverState *qmp_chardev_open_file(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevFile *file = backend->u.file;
     HANDLE out;
 
     if (file->has_in) {
@@ -4023,17 +4070,13 @@ static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
     return qemu_chr_open_win_file(out);
 }
 
-static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+static CharDriverState *qmp_chardev_open_serial(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
-    return qemu_chr_open_win_path(serial->device);
-}
-
-static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
-                                                  Error **errp)
-{
-    error_setg(errp, "character device backend type 'parallel' not supported");
-    return NULL;
+    ChardevHostdev *serial = backend->u.serial;
+    return qemu_chr_open_win_path(serial->device, errp);
 }
 
 #else /* WIN32 */
@@ -4050,8 +4093,12 @@ static int qmp_chardev_open_file_source(char *src, int flags,
     return fd;
 }
 
-static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+static CharDriverState *qmp_chardev_open_file(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevFile *file = backend->u.file;
     int flags, in = -1, out;
 
     flags = O_WRONLY | O_TRUNC | O_CREAT | O_BINARY;
@@ -4072,10 +4119,13 @@ static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
     return qemu_chr_open_fd(in, out);
 }
 
-static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+#ifdef HAVE_CHARDEV_SERIAL
+static CharDriverState *qmp_chardev_open_serial(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
-#ifdef HAVE_CHARDEV_TTY
+    ChardevHostdev *serial = backend->u.serial;
     int fd;
 
     fd = qmp_chardev_open_file_source(serial->device, O_RDWR, errp);
@@ -4084,28 +4134,25 @@ static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
     }
     qemu_set_nonblock(fd);
     return qemu_chr_open_tty_fd(fd);
-#else
-    error_setg(errp, "character device backend type 'serial' not supported");
-    return NULL;
-#endif
 }
+#endif
 
-static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
+#ifdef HAVE_CHARDEV_PARPORT
+static CharDriverState *qmp_chardev_open_parallel(const char *id,
+                                                  ChardevBackend *backend,
+                                                  ChardevReturn *ret,
                                                   Error **errp)
 {
-#ifdef HAVE_CHARDEV_PARPORT
+    ChardevHostdev *parallel = backend->u.parallel;
     int fd;
 
     fd = qmp_chardev_open_file_source(parallel->device, O_RDWR, errp);
     if (fd < 0) {
         return NULL;
     }
-    return qemu_chr_open_pp_fd(fd);
-#else
-    error_setg(errp, "character device backend type 'parallel' not supported");
-    return NULL;
-#endif
+    return qemu_chr_open_pp_fd(fd, errp);
 }
+#endif
 
 #endif /* WIN32 */
 
@@ -4134,11 +4181,14 @@ static gboolean socket_reconnect_timeout(gpointer opaque)
     return false;
 }
 
-static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
+static CharDriverState *qmp_chardev_open_socket(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
     CharDriverState *chr;
     TCPCharDriver *s;
+    ChardevSocket *sock = backend->u.socket;
     SocketAddress *addr = sock->addr;
     bool do_nodelay     = sock->has_nodelay ? sock->nodelay : false;
     bool is_listen      = sock->has_server  ? sock->server  : true;
@@ -4147,11 +4197,11 @@ static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
     int64_t reconnect   = sock->has_reconnect ? sock->reconnect : 0;
 
     chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(TCPCharDriver));
+    s = g_new0(TCPCharDriver, 1);
 
     s->fd = -1;
     s->listen_fd = -1;
-    s->is_unix = addr->kind == SOCKET_ADDRESS_KIND_UNIX;
+    s->is_unix = addr->type == SOCKET_ADDRESS_KIND_UNIX;
     s->is_listen = is_listen;
     s->is_telnet = is_telnet;
     s->do_nodelay = do_nodelay;
@@ -4200,9 +4250,12 @@ static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
     return chr;
 }
 
-static CharDriverState *qmp_chardev_open_udp(ChardevUdp *udp,
+static CharDriverState *qmp_chardev_open_udp(const char *id,
+                                             ChardevBackend *backend,
+                                             ChardevReturn *ret,
                                              Error **errp)
 {
+    ChardevUdp *udp = backend->u.udp;
     int fd;
 
     fd = socket_dgram(udp->remote, udp->local, errp);
@@ -4216,7 +4269,10 @@ ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
                                Error **errp)
 {
     ChardevReturn *ret = g_new0(ChardevReturn, 1);
-    CharDriverState *base, *chr = NULL;
+    CharDriverState *chr = NULL;
+    Error *local_err = NULL;
+    GSList *i;
+    CharDriver *cd;
 
     chr = qemu_chr_find(id);
     if (chr) {
@@ -4225,106 +4281,40 @@ ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
         return NULL;
     }
 
-    switch (backend->kind) {
-    case CHARDEV_BACKEND_KIND_FILE:
-        chr = qmp_chardev_open_file(backend->file, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_SERIAL:
-        chr = qmp_chardev_open_serial(backend->serial, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_PARALLEL:
-        chr = qmp_chardev_open_parallel(backend->parallel, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_PIPE:
-        chr = qemu_chr_open_pipe(backend->pipe);
-        break;
-    case CHARDEV_BACKEND_KIND_SOCKET:
-        chr = qmp_chardev_open_socket(backend->socket, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_UDP:
-        chr = qmp_chardev_open_udp(backend->udp, errp);
-        break;
-#ifdef HAVE_CHARDEV_TTY
-    case CHARDEV_BACKEND_KIND_PTY:
-        chr = qemu_chr_open_pty(id, ret);
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_NULL:
-        chr = qemu_chr_open_null();
-        break;
-    case CHARDEV_BACKEND_KIND_MUX:
-        base = qemu_chr_find(backend->mux->chardev);
-        if (base == NULL) {
-            error_setg(errp, "mux: base chardev %s not found",
-                       backend->mux->chardev);
+    for (i = backends; i; i = i->next) {
+        cd = i->data;
+
+        if (cd->kind == backend->type) {
+            chr = cd->create(id, backend, ret, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                goto out_error;
+            }
             break;
         }
-        chr = qemu_chr_open_mux(base);
-        break;
-    case CHARDEV_BACKEND_KIND_MSMOUSE:
-        chr = qemu_chr_open_msmouse();
-        break;
-#ifdef CONFIG_BRLAPI
-    case CHARDEV_BACKEND_KIND_BRAILLE:
-        chr = chr_baum_init();
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_TESTDEV:
-        chr = chr_testdev_init();
-        break;
-    case CHARDEV_BACKEND_KIND_STDIO:
-        chr = qemu_chr_open_stdio(backend->stdio);
-        break;
-#ifdef _WIN32
-    case CHARDEV_BACKEND_KIND_CONSOLE:
-        chr = qemu_chr_open_win_con();
-        break;
-#endif
-#ifdef CONFIG_SPICE
-    case CHARDEV_BACKEND_KIND_SPICEVMC:
-        chr = qemu_chr_open_spice_vmc(backend->spicevmc->type);
-        break;
-    case CHARDEV_BACKEND_KIND_SPICEPORT:
-        chr = qemu_chr_open_spice_port(backend->spiceport->fqdn);
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_VC:
-        chr = vc_init(backend->vc);
-        break;
-    case CHARDEV_BACKEND_KIND_RINGBUF:
-    case CHARDEV_BACKEND_KIND_MEMORY:
-        chr = qemu_chr_open_ringbuf(backend->ringbuf, errp);
-        break;
-    default:
-        error_setg(errp, "unknown chardev backend (%d)", backend->kind);
-        break;
     }
 
-    /*
-     * Character backend open hasn't been fully converted to the Error
-     * API.  Some opens fail without setting an error.  Set a generic
-     * error then.
-     * TODO full conversion to Error API
-     */
-    if (chr == NULL && errp && !*errp) {
-        error_setg(errp, "Failed to create chardev");
+    if (chr == NULL) {
+        assert(!i);
+        error_setg(errp, "chardev backend not available");
+        goto out_error;
     }
-    if (chr) {
-        chr->label = g_strdup(id);
-        chr->avail_connections =
-            (backend->kind == CHARDEV_BACKEND_KIND_MUX) ? MAX_MUX : 1;
-        if (!chr->filename) {
-            chr->filename = g_strdup(ChardevBackendKind_lookup[backend->kind]);
-        }
-        if (!chr->explicit_be_open) {
-            qemu_chr_be_event(chr, CHR_EVENT_OPENED);
-        }
-        QTAILQ_INSERT_TAIL(&chardevs, chr, next);
-        return ret;
-    } else {
-        g_free(ret);
-        return NULL;
+
+    chr->label = g_strdup(id);
+    chr->avail_connections =
+        (backend->type == CHARDEV_BACKEND_KIND_MUX) ? MAX_MUX : 1;
+    if (!chr->filename) {
+        chr->filename = g_strdup(ChardevBackendKind_lookup[backend->type]);
+    }
+    if (!chr->explicit_be_open) {
+        qemu_chr_be_event(chr, CHR_EVENT_OPENED);
     }
+    QTAILQ_INSERT_TAIL(&chardevs, chr, next);
+    return ret;
+
+out_error:
+    g_free(ret);
+    return NULL;
 }
 
 void qmp_chardev_remove(const char *id, Error **errp)
@@ -4346,32 +4336,45 @@ void qmp_chardev_remove(const char *id, Error **errp)
 
 static void register_types(void)
 {
-    register_char_driver("null", CHARDEV_BACKEND_KIND_NULL, NULL);
+    register_char_driver("null", CHARDEV_BACKEND_KIND_NULL, NULL,
+                         qemu_chr_open_null);
     register_char_driver("socket", CHARDEV_BACKEND_KIND_SOCKET,
-                         qemu_chr_parse_socket);
-    register_char_driver("udp", CHARDEV_BACKEND_KIND_UDP, qemu_chr_parse_udp);
+                         qemu_chr_parse_socket, qmp_chardev_open_socket);
+    register_char_driver("udp", CHARDEV_BACKEND_KIND_UDP, qemu_chr_parse_udp,
+                         qmp_chardev_open_udp);
     register_char_driver("ringbuf", CHARDEV_BACKEND_KIND_RINGBUF,
-                         qemu_chr_parse_ringbuf);
+                         qemu_chr_parse_ringbuf, qemu_chr_open_ringbuf);
     register_char_driver("file", CHARDEV_BACKEND_KIND_FILE,
-                         qemu_chr_parse_file_out);
+                         qemu_chr_parse_file_out, qmp_chardev_open_file);
     register_char_driver("stdio", CHARDEV_BACKEND_KIND_STDIO,
-                         qemu_chr_parse_stdio);
+                         qemu_chr_parse_stdio, qemu_chr_open_stdio);
+#if defined HAVE_CHARDEV_SERIAL
     register_char_driver("serial", CHARDEV_BACKEND_KIND_SERIAL,
-                         qemu_chr_parse_serial);
+                         qemu_chr_parse_serial, qmp_chardev_open_serial);
     register_char_driver("tty", CHARDEV_BACKEND_KIND_SERIAL,
-                         qemu_chr_parse_serial);
+                         qemu_chr_parse_serial, qmp_chardev_open_serial);
+#endif
+#ifdef HAVE_CHARDEV_PARPORT
     register_char_driver("parallel", CHARDEV_BACKEND_KIND_PARALLEL,
-                         qemu_chr_parse_parallel);
+                         qemu_chr_parse_parallel, qmp_chardev_open_parallel);
     register_char_driver("parport", CHARDEV_BACKEND_KIND_PARALLEL,
-                         qemu_chr_parse_parallel);
-    register_char_driver("pty", CHARDEV_BACKEND_KIND_PTY, NULL);
-    register_char_driver("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL);
+                         qemu_chr_parse_parallel, qmp_chardev_open_parallel);
+#endif
+#ifdef HAVE_CHARDEV_PTY
+    register_char_driver("pty", CHARDEV_BACKEND_KIND_PTY, NULL,
+                         qemu_chr_open_pty);
+#endif
+#ifdef _WIN32
+    register_char_driver("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL,
+                         qemu_chr_open_win_con);
+#endif
     register_char_driver("pipe", CHARDEV_BACKEND_KIND_PIPE,
-                         qemu_chr_parse_pipe);
-    register_char_driver("mux", CHARDEV_BACKEND_KIND_MUX, qemu_chr_parse_mux);
+                         qemu_chr_parse_pipe, qemu_chr_open_pipe);
+    register_char_driver("mux", CHARDEV_BACKEND_KIND_MUX, qemu_chr_parse_mux,
+                         qemu_chr_open_mux);
     /* Bug-compatibility: */
     register_char_driver("memory", CHARDEV_BACKEND_KIND_MEMORY,
-                         qemu_chr_parse_ringbuf);
+                         qemu_chr_parse_ringbuf, qemu_chr_open_ringbuf);
     /* this must be done after machine init, since we register FEs with muxes
      * as part of realize functions like serial_isa_realizefn when -nographic
      * is specified
diff --git a/qemu-doc.texi b/qemu-doc.texi
index 94af8c0f3..ffc3e50ab 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -391,6 +391,8 @@ The following commands are available:
 
 @include qemu-monitor.texi
 
+@include qemu-monitor-info.texi
+
 @subsection Integer expressions
 
 The monitor understands integers expressions for every integer
@@ -412,6 +414,7 @@ snapshots.
 * vm_snapshots::              VM snapshots
 * qemu_img_invocation::       qemu-img Invocation
 * qemu_nbd_invocation::       qemu-nbd Invocation
+* qemu_ga_invocation::        qemu-ga Invocation
 * disk_images_formats::       Disk image file formats
 * host_drives::               Using host drives
 * disk_images_fat_images::    Virtual FAT disk images
@@ -505,6 +508,11 @@ state is not saved or restored properly (in particular USB).
 
 @include qemu-nbd.texi
 
+@node qemu_ga_invocation
+@subsection @code{qemu-ga} Invocation
+
+@include qemu-ga.texi
+
 @node disk_images_formats
 @subsection Disk image file formats
 
@@ -1248,7 +1256,7 @@ zero-copy communication to the application level of the guests.  The basic
 syntax is:
 
 @example
-qemu-system-i386 -device ivshmem,size=<size in format accepted by -m>[,shm=<shm name>]
+qemu-system-i386 -device ivshmem,size=@var{size},shm=@var{shm-name}
 @end example
 
 If desired, interrupts can be sent between guest VMs accessing the same shared
@@ -1258,9 +1266,13 @@ is qemu.git/contrib/ivshmem-server.  An example syntax when using the shared
 memory server is:
 
 @example
-qemu-system-i386 -device ivshmem,size=<size in format accepted by -m>[,chardev=<id>]
-                 [,msi=on][,ioeventfd=on][,vectors=n][,role=peer|master]
-qemu-system-i386 -chardev socket,path=<path>,id=<id>
+# First start the ivshmem server once and for all
+ivshmem-server -p @var{pidfile} -S @var{path} -m @var{shm-name} -l @var{shm-size} -n @var{vectors}
+
+# Then start your qemu instances with matching arguments
+qemu-system-i386 -device ivshmem,size=@var{shm-size},vectors=@var{vectors},chardev=@var{id}
+                 [,msi=on][,ioeventfd=on][,role=peer|master]
+                 -chardev socket,path=@var{path},id=@var{id}
 @end example
 
 When using the server, the guest will be assigned a VM ID (>=0) that allows guests
@@ -1281,6 +1293,19 @@ copy the shared memory on migration to the destination host.  With
 With the @option{peer} case, the device should be detached and then reattached
 after migration using the PCI hotplug support.
 
+@subsubsection ivshmem and hugepages
+
+Instead of specifying the <shm size> using POSIX shm, you may specify
+a memory backend that has hugepage support:
+
+@example
+qemu-system-i386 -object memory-backend-file,size=1G,mem-path=/mnt/hugepages/my-shmem-file,id=mb1
+                 -device ivshmem,x-memdev=mb1
+@end example
+
+ivshmem-server also supports hugepages mount points with the
+@option{-m} memory path argument.
+
 @node direct_linux_boot
 @section Direct Linux Boot
 
@@ -1766,7 +1791,7 @@ Use @code{set architecture i8086} to dump 16 bit code. Then use
 
 Advanced debugging options:
 
-The default single stepping behavior is step with the IRQs and timer service routines off.  It is set this way because when gdb executes a single step it expects to advance beyond the current instruction.  With the IRQs and and timer service routines on, a single step might jump into the one of the interrupt or exception vectors instead of executing the current instruction. This means you may hit the same breakpoint a number of times before executing the instruction gdb wants to have executed.  Because there are rare circumstances where you want to single step into an interrupt vector the behavior can be controlled from GDB.  There are three commands you can query and set the single step behavior:
+The default single stepping behavior is step with the IRQs and timer service routines off.  It is set this way because when gdb executes a single step it expects to advance beyond the current instruction.  With the IRQs and timer service routines on, a single step might jump into the one of the interrupt or exception vectors instead of executing the current instruction. This means you may hit the same breakpoint a number of times before executing the instruction gdb wants to have executed.  Because there are rare circumstances where you want to single step into an interrupt vector the behavior can be controlled from GDB.  There are three commands you can query and set the single step behavior:
 @table @code
 @item maintenance packet qqemu.sstepbits
 
@@ -3021,9 +3046,63 @@ and all other qemu-system-@var{target}.exe compiled for Win32.
 @node Mac OS X
 @section Mac OS X
 
-The Mac OS X patches are not fully merged in QEMU, so you should look
-at the QEMU mailing list archive to have all the necessary
-information.
+System Requirements:
+@itemize
+@item Mac OS 10.5 or higher
+@item The clang compiler shipped with Xcode 4.2 or higher,
+or GCC 4.3 or higher
+@end itemize
+
+Additional Requirements (install in order):
+@enumerate
+@item libffi: @uref{https://sourceware.org/libffi/}
+@item gettext: @uref{http://www.gnu.org/software/gettext/}
+@item glib: @uref{http://ftp.gnome.org/pub/GNOME/sources/glib/}
+@item pkg-config: @uref{http://www.freedesktop.org/wiki/Software/pkg-config/}
+@item autoconf: @uref{http://www.gnu.org/software/autoconf/autoconf.html}
+@item automake: @uref{http://www.gnu.org/software/automake/}
+@item libtool: @uref{http://www.gnu.org/software/libtool/}
+@item pixman: @uref{http://www.pixman.org/}
+@end enumerate
+
+* You may find it easiest to get these from a third-party packager
+such as Homebrew, Macports, or Fink.
+
+After downloading the QEMU source code, double-click it to expand it.
+
+Then configure and make QEMU:
+@example
+./configure
+make
+@end example
+
+If you have a recent version of Mac OS X (OSX 10.7 or better
+with Xcode 4.2 or better) we recommend building QEMU with the
+default compiler provided by Apple, for your version of Mac OS X
+(which will be 'clang'). The configure script will
+automatically pick this.
+
+Note: If after the configure step you see a message like this:
+@example
+ERROR: Your compiler does not support the __thread specifier for
+       Thread-Local Storage (TLS). Please upgrade to a version that does.
+@end example
+you may have to build your own version of gcc from source. Expect that to take
+several hours. More information can be found here:
+@uref{https://gcc.gnu.org/install/} @*
+
+These are some of the third party binaries of gcc available for download:
+@itemize
+@item Homebrew: @uref{http://brew.sh/}
+@item @uref{https://www.litebeam.net/gcc/gcc_472.pkg}
+@item @uref{http://www.macports.org/ports.php?by=name&substr=gcc}
+@end itemize
+
+You can have several versions of GCC on your system. To specify a certain version,
+use the --cc and --cxx options.
+@example
+./configure --cxx=<path of your c++ compiler> --cc=<path of your c compiler> <other options>
+@end example
 
 @node Make targets
 @section Make targets
diff --git a/qemu-ga.texi b/qemu-ga.texi
new file mode 100644
index 000000000..536a9b524
--- /dev/null
+++ b/qemu-ga.texi
@@ -0,0 +1,137 @@
+@example
+@c man begin SYNOPSIS
+usage: qemu-ga [OPTIONS]
+@c man end
+@end example
+
+@c man begin DESCRIPTION
+
+The QEMU Guest Agent is a daemon intended to be run within virtual
+machines. It allows the hypervisor host to perform various operations
+in the guest, such as:
+
+@itemize
+@item
+get information from the guest
+@item
+set the guest's system time
+@item
+read/write a file
+@item
+sync and freeze the filesystems
+@item
+suspend the guest
+@item
+reconfigure guest local processors
+@item
+set user's password
+@item
+...
+@end itemize
+
+qemu-ga will read a system configuration file on startup (located at
+q@file{/etc/qemu/qemu-ga.conf} by default), then parse remaining
+configuration options on the command line. For the same key, the last
+option wins, but the lists accumulate (see below for configuration
+file format).
+
+@c man end
+
+@c man begin OPTIONS
+@table @option
+@item -m, --method=@var{method}
+  Transport method: one of @samp{unix-listen}, @samp{virtio-serial}, or
+  @samp{isa-serial} (@samp{virtio-serial} is the default).
+
+@item -p, --path=@var{path}
+  Device/socket path (the default for virtio-serial is
+  @samp{/dev/virtio-ports/org.qemu.guest_agent.0},
+  the default for isa-serial is @samp{/dev/ttyS0})
+
+@item -l, --logfile=@var{path}
+  Set log file path (default is stderr).
+
+@item -f, --pidfile=@var{path}
+  Specify pid file (default is @samp{/var/run/qemu-ga.pid}).
+
+@item -F, --fsfreeze-hook=@var{path}
+  Enable fsfreeze hook. Accepts an optional argument that specifies
+  script to run on freeze/thaw. Script will be called with
+  'freeze'/'thaw' arguments accordingly (default is
+  @samp{/etc/qemu/fsfreeze-hook}). If using -F with an argument, do
+  not follow -F with a space (for example:
+  @samp{-F/var/run/fsfreezehook.sh}).
+
+@item -t, --statedir=@var{path}
+  Specify the directory to store state information (absolute paths only,
+  default is @samp{/var/run}).
+
+@item -v, --verbose
+  Log extra debugging information.
+
+@item -V, --version
+  Print version information and exit.
+
+@item -d, --daemon
+  Daemonize after startup (detach from terminal).
+
+@item -b, --blacklist=@var{list}
+  Comma-separated list of RPCs to disable (no spaces, @samp{?} to list
+  available RPCs).
+
+@item -D, --dump-conf
+  Dump the configuration in a format compatible with @file{qemu-ga.conf}
+  and exit.
+
+@item -h, --help
+  Display this help and exit.
+@end table
+
+@c man end
+
+@c man begin FILES
+
+The syntax of the @file{qemu-ga.conf} configuration file follows the
+Desktop Entry Specification, here is a quick summary: it consists of
+groups of key-value pairs, interspersed with comments.
+
+@example
+# qemu-ga configuration sample
+[general]
+daemonize = 0
+pidfile = /var/run/qemu-ga.pid
+verbose = 0
+method = virtio-serial
+path = /dev/virtio-ports/org.qemu.guest_agent.0
+statedir = /var/run
+@end example
+
+The list of keys follows the command line options:
+@table @option
+@item daemon= boolean
+@item method= string
+@item path= string
+@item logfile= string
+@item pidfile= string
+@item fsfreeze-hook= string
+@item statedir= string
+@item verbose= boolean
+@item blacklist= string list
+@end table
+
+@c man end
+
+@ignore
+
+@setfilename qemu-ga
+@settitle QEMU Guest Agent
+
+@c man begin AUTHOR
+Michael Roth <mdroth@linux.vnet.ibm.com>
+@c man end
+
+@c man begin SEEALSO
+qemu(1)
+@c man end
+
+@end ignore
diff --git a/qemu-img.c b/qemu-img.c
index 6ff4e852b..033011c4e 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -338,7 +338,8 @@ static int img_create(int argc, char **argv)
     if (optind < argc) {
         int64_t sval;
         char *end;
-        sval = strtosz_suffix(argv[optind++], &end, STRTOSZ_DEFSUFFIX_B);
+        sval = qemu_strtosz_suffix(argv[optind++], &end,
+                                   QEMU_STRTOSZ_DEFSUFFIX_B);
         if (sval < 0 || *end) {
             if (sval == -ERANGE) {
                 error_report("Image size must be less than 8 EiB!");
@@ -644,9 +645,6 @@ static void common_block_job_cb(void *opaque, int ret)
     if (ret < 0) {
         error_setg_errno(cbi->errp, -ret, "Block job failed");
     }
-
-    /* Drop this block job's reference */
-    bdrv_unref(cbi->bs);
 }
 
 static void run_block_job(BlockJob *job, Error **errp)
@@ -655,7 +653,8 @@ static void run_block_job(BlockJob *job, Error **errp)
 
     do {
         aio_poll(aio_context, true);
-        qemu_progress_print((float)job->offset / job->len * 100.f, 0);
+        qemu_progress_print(job->len ?
+                            ((float)job->offset / job->len * 100.f) : 0.0f, 0);
     } while (!job->ready);
 
     block_job_complete_sync(job, errp);
@@ -747,7 +746,7 @@ static int img_commit(int argc, char **argv)
         /* This is different from QMP, which by default uses the deepest file in
          * the backing chain (i.e., the very base); however, the traditional
          * behavior of qemu-img commit is using the immediate backing file. */
-        base_bs = bs->backing_hd;
+        base_bs = backing_bs(bs);
         if (!base_bs) {
             error_setg(&local_err, "Image does not have a backing file");
             goto done;
@@ -765,12 +764,12 @@ static int img_commit(int argc, char **argv)
         goto done;
     }
 
-    /* The block job will swap base_bs and bs (which is not what we really want
-     * here, but okay) and unref base_bs (after the swap, i.e., the old top
-     * image). In order to still be able to empty that top image afterwards,
-     * increment the reference counter here preemptively. */
+    /* When the block job completes, the BlockBackend reference will point to
+     * the old backing file. In order to avoid that the top image is already
+     * deleted, so we can still empty it afterwards, increment the reference
+     * counter here preemptively. */
     if (!drop) {
-        bdrv_ref(base_bs);
+        bdrv_ref(bs);
     }
 
     run_block_job(bs->job, &local_err);
@@ -778,8 +777,8 @@ static int img_commit(int argc, char **argv)
         goto unref_backing;
     }
 
-    if (!drop && base_bs->drv->bdrv_make_empty) {
-        ret = base_bs->drv->bdrv_make_empty(base_bs);
+    if (!drop && bs->drv->bdrv_make_empty) {
+        ret = bs->drv->bdrv_make_empty(bs);
         if (ret) {
             error_setg_errno(&local_err, -ret, "Could not empty %s",
                              filename);
@@ -789,7 +788,7 @@ static int img_commit(int argc, char **argv)
 
 unref_backing:
     if (!drop) {
-        bdrv_unref(base_bs);
+        bdrv_unref(bs);
     }
 
 done:
@@ -1607,7 +1606,7 @@ static int img_convert(int argc, char **argv)
         {
             int64_t sval;
             char *end;
-            sval = strtosz_suffix(optarg, &end, STRTOSZ_DEFSUFFIX_B);
+            sval = qemu_strtosz_suffix(optarg, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
             if (sval < 0 || *end) {
                 error_report("Invalid minimum zero buffer size for sparse output specified");
                 ret = -1;
@@ -2206,7 +2205,7 @@ static int get_block_status(BlockDriverState *bs, int64_t sector_num,
         if (ret & (BDRV_BLOCK_ZERO|BDRV_BLOCK_DATA)) {
             break;
         }
-        bs = bs->backing_hd;
+        bs = backing_bs(bs);
         if (bs == NULL) {
             ret = 0;
             break;
diff --git a/qemu-img.texi b/qemu-img.texi
index 0a1ab3598..55c6be391 100644
--- a/qemu-img.texi
+++ b/qemu-img.texi
@@ -103,7 +103,7 @@ First image format
 @item -F
 Second image format
 @item -s
-Strict mode - fail on on different image size or sector allocation
+Strict mode - fail on different image size or sector allocation
 @end table
 
 Parameters to convert subcommand:
diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index 53477e1e1..18fc2bdc1 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -136,7 +136,29 @@ static char **breakline(char *input, int *count)
 static int64_t cvtnum(const char *s)
 {
     char *end;
-    return strtosz_suffix(s, &end, STRTOSZ_DEFSUFFIX_B);
+    int64_t ret;
+
+    ret = qemu_strtosz_suffix(s, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
+    if (*end != '\0') {
+        /* Detritus at the end of the string */
+        return -EINVAL;
+    }
+    return ret;
+}
+
+static void print_cvtnum_err(int64_t rc, const char *arg)
+{
+    switch (rc) {
+    case -EINVAL:
+        printf("Parsing error: non-numeric argument,"
+               " or extraneous/unrecognized suffix -- %s\n", arg);
+        break;
+    case -ERANGE:
+        printf("Parsing error: argument too large -- %s\n", arg);
+        break;
+    default:
+        printf("Parsing error: %s\n", arg);
+    }
 }
 
 #define EXABYTES(x)     ((long long)(x) << 60)
@@ -294,9 +316,10 @@ static void qemu_io_free(void *p)
     qemu_vfree(p);
 }
 
-static void dump_buffer(const void *buffer, int64_t offset, int len)
+static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
 {
-    int i, j;
+    uint64_t i;
+    int j;
     const uint8_t *p;
 
     for (i = 0, p = buffer; i < len; i += 16) {
@@ -319,7 +342,7 @@ static void dump_buffer(const void *buffer, int64_t offset, int len)
 }
 
 static void print_report(const char *op, struct timeval *t, int64_t offset,
-                         int count, int total, int cnt, int Cflag)
+                         int64_t count, int64_t total, int cnt, int Cflag)
 {
     char s1[64], s2[64], ts[64];
 
@@ -327,12 +350,12 @@ static void print_report(const char *op, struct timeval *t, int64_t offset,
     if (!Cflag) {
         cvtstr((double)total, s1, sizeof(s1));
         cvtstr(tdiv((double)total, *t), s2, sizeof(s2));
-        printf("%s %d/%d bytes at offset %" PRId64 "\n",
+        printf("%s %"PRId64"/%"PRId64" bytes at offset %" PRId64 "\n",
                op, total, count, offset);
         printf("%s, %d ops; %s (%s/sec and %.4f ops/sec)\n",
                s1, cnt, ts, s2, tdiv((double)cnt, *t));
     } else {/* bytes,ops,time,bytes/sec,ops/sec */
-        printf("%d,%d,%s,%.3f,%.3f\n",
+        printf("%"PRId64",%d,%s,%.3f,%.3f\n",
             total, cnt, ts,
             tdiv((double)total, *t),
             tdiv((double)cnt, *t));
@@ -359,13 +382,13 @@ create_iovec(BlockBackend *blk, QEMUIOVector *qiov, char **argv, int nr_iov,
 
         len = cvtnum(arg);
         if (len < 0) {
-            printf("non-numeric length argument -- %s\n", arg);
+            print_cvtnum_err(len, arg);
             goto fail;
         }
 
         /* should be SIZE_T_MAX, but that doesn't exist */
         if (len > INT_MAX) {
-            printf("too large length argument -- %s\n", arg);
+            printf("Argument '%s' exceeds maximum size %d\n", arg, INT_MAX);
             goto fail;
         }
 
@@ -393,11 +416,15 @@ fail:
     return buf;
 }
 
-static int do_read(BlockBackend *blk, char *buf, int64_t offset, int count,
-                   int *total)
+static int do_read(BlockBackend *blk, char *buf, int64_t offset, int64_t count,
+                   int64_t *total)
 {
     int ret;
 
+    if (count >> 9 > INT_MAX) {
+        return -ERANGE;
+    }
+
     ret = blk_read(blk, offset >> 9, (uint8_t *)buf, count >> 9);
     if (ret < 0) {
         return ret;
@@ -406,11 +433,15 @@ static int do_read(BlockBackend *blk, char *buf, int64_t offset, int count,
     return 1;
 }
 
-static int do_write(BlockBackend *blk, char *buf, int64_t offset, int count,
-                    int *total)
+static int do_write(BlockBackend *blk, char *buf, int64_t offset, int64_t count,
+                    int64_t *total)
 {
     int ret;
 
+    if (count >> 9 > INT_MAX) {
+        return -ERANGE;
+    }
+
     ret = blk_write(blk, offset >> 9, (uint8_t *)buf, count >> 9);
     if (ret < 0) {
         return ret;
@@ -419,9 +450,13 @@ static int do_write(BlockBackend *blk, char *buf, int64_t offset, int count,
     return 1;
 }
 
-static int do_pread(BlockBackend *blk, char *buf, int64_t offset, int count,
-                    int *total)
+static int do_pread(BlockBackend *blk, char *buf, int64_t offset,
+                    int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_pread(blk, offset, (uint8_t *)buf, count);
     if (*total < 0) {
         return *total;
@@ -429,9 +464,13 @@ static int do_pread(BlockBackend *blk, char *buf, int64_t offset, int count,
     return 1;
 }
 
-static int do_pwrite(BlockBackend *blk, char *buf, int64_t offset, int count,
-                     int *total)
+static int do_pwrite(BlockBackend *blk, char *buf, int64_t offset,
+                     int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_pwrite(blk, offset, (uint8_t *)buf, count);
     if (*total < 0) {
         return *total;
@@ -442,8 +481,8 @@ static int do_pwrite(BlockBackend *blk, char *buf, int64_t offset, int count,
 typedef struct {
     BlockBackend *blk;
     int64_t offset;
-    int count;
-    int *total;
+    int64_t count;
+    int64_t *total;
     int ret;
     bool done;
 } CoWriteZeroes;
@@ -463,8 +502,8 @@ static void coroutine_fn co_write_zeroes_entry(void *opaque)
     *data->total = data->count;
 }
 
-static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int count,
-                              int *total)
+static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int64_t count,
+                              int64_t *total)
 {
     Coroutine *co;
     CoWriteZeroes data = {
@@ -475,6 +514,10 @@ static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int count,
         .done   = false,
     };
 
+    if (count >> BDRV_SECTOR_BITS > INT_MAX) {
+        return -ERANGE;
+    }
+
     co = qemu_coroutine_create(co_write_zeroes_entry);
     qemu_coroutine_enter(co, &data);
     while (!data.done) {
@@ -488,10 +531,14 @@ static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int count,
 }
 
 static int do_write_compressed(BlockBackend *blk, char *buf, int64_t offset,
-                               int count, int *total)
+                               int64_t count, int64_t *total)
 {
     int ret;
 
+    if (count >> 9 > INT_MAX) {
+        return -ERANGE;
+    }
+
     ret = blk_write_compressed(blk, offset >> 9, (uint8_t *)buf, count >> 9);
     if (ret < 0) {
         return ret;
@@ -501,8 +548,12 @@ static int do_write_compressed(BlockBackend *blk, char *buf, int64_t offset,
 }
 
 static int do_load_vmstate(BlockBackend *blk, char *buf, int64_t offset,
-                           int count, int *total)
+                           int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_load_vmstate(blk, (uint8_t *)buf, offset, count);
     if (*total < 0) {
         return *total;
@@ -511,8 +562,12 @@ static int do_load_vmstate(BlockBackend *blk, char *buf, int64_t offset,
 }
 
 static int do_save_vmstate(BlockBackend *blk, char *buf, int64_t offset,
-                           int count, int *total)
+                           int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_save_vmstate(blk, (uint8_t *)buf, offset, count);
     if (*total < 0) {
         return *total;
@@ -642,10 +697,11 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
     int c, cnt;
     char *buf;
     int64_t offset;
-    int count;
+    int64_t count;
     /* Some compilers get confused and warn if this is not initialized.  */
-    int total = 0;
-    int pattern = 0, pattern_offset = 0, pattern_count = 0;
+    int64_t total = 0;
+    int pattern = 0;
+    int64_t pattern_offset = 0, pattern_count = 0;
 
     while ((c = getopt(argc, argv, "bCl:pP:qs:v")) != -1) {
         switch (c) {
@@ -659,7 +715,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
             lflag = 1;
             pattern_count = cvtnum(optarg);
             if (pattern_count < 0) {
-                printf("non-numeric length argument -- %s\n", optarg);
+                print_cvtnum_err(pattern_count, optarg);
                 return 0;
             }
             break;
@@ -680,7 +736,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
             sflag = 1;
             pattern_offset = cvtnum(optarg);
             if (pattern_offset < 0) {
-                printf("non-numeric length argument -- %s\n", optarg);
+                print_cvtnum_err(pattern_offset, optarg);
                 return 0;
             }
             break;
@@ -703,14 +759,18 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count > SIZE_MAX) {
+        printf("length cannot exceed %" PRIu64 ", given %s\n",
+               (uint64_t) SIZE_MAX, argv[optind]);
         return 0;
     }
 
@@ -734,7 +794,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
             return 0;
         }
         if (count & 0x1ff) {
-            printf("count %d is not sector aligned\n",
+            printf("count %"PRId64" is not sector aligned\n",
                    count);
             return 0;
         }
@@ -762,7 +822,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
         memset(cmp_buf, pattern, pattern_count);
         if (memcmp(buf + pattern_offset, cmp_buf, pattern_count)) {
             printf("Pattern verification failed at offset %"
-                   PRId64 ", %d bytes\n",
+                   PRId64 ", %"PRId64" bytes\n",
                    offset + pattern_offset, pattern_count);
         }
         g_free(cmp_buf);
@@ -861,7 +921,7 @@ static int readv_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
     optind++;
@@ -957,9 +1017,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
     int c, cnt;
     char *buf = NULL;
     int64_t offset;
-    int count;
+    int64_t count;
     /* Some compilers get confused and warn if this is not initialized.  */
-    int total = 0;
+    int64_t total = 0;
     int pattern = 0xcd;
 
     while ((c = getopt(argc, argv, "bcCpP:qz")) != -1) {
@@ -1010,14 +1070,18 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count > SIZE_MAX) {
+        printf("length cannot exceed %" PRIu64 ", given %s\n",
+               (uint64_t) SIZE_MAX, argv[optind]);
         return 0;
     }
 
@@ -1029,7 +1093,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
         }
 
         if (count & 0x1ff) {
-            printf("count %d is not sector aligned\n",
+            printf("count %"PRId64" is not sector aligned\n",
                    count);
             return 0;
         }
@@ -1142,7 +1206,7 @@ static int writev_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
     optind++;
@@ -1269,7 +1333,7 @@ static int multiwrite_f(BlockBackend *blk, int argc, char **argv)
         /* Read the offset of the request */
         offset = cvtnum(argv[optind]);
         if (offset < 0) {
-            printf("non-numeric offset argument -- %s\n", argv[optind]);
+            print_cvtnum_err(offset, argv[optind]);
             goto out;
         }
         optind++;
@@ -1364,6 +1428,7 @@ static void aio_write_done(void *opaque, int ret)
 
     if (ret < 0) {
         printf("aio_write failed: %s\n", strerror(-ret));
+        block_acct_failed(blk_get_stats(ctx->blk), &ctx->acct);
         goto out;
     }
 
@@ -1392,6 +1457,7 @@ static void aio_read_done(void *opaque, int ret)
 
     if (ret < 0) {
         printf("readv failed: %s\n", strerror(-ret));
+        block_acct_failed(blk_get_stats(ctx->blk), &ctx->acct);
         goto out;
     }
 
@@ -1496,7 +1562,7 @@ static int aio_read_f(BlockBackend *blk, int argc, char **argv)
 
     ctx->offset = cvtnum(argv[optind]);
     if (ctx->offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(ctx->offset, argv[optind]);
         g_free(ctx);
         return 0;
     }
@@ -1505,6 +1571,7 @@ static int aio_read_f(BlockBackend *blk, int argc, char **argv)
     if (ctx->offset & 0x1ff) {
         printf("offset %" PRId64 " is not sector aligned\n",
                ctx->offset);
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
         g_free(ctx);
         return 0;
     }
@@ -1512,6 +1579,7 @@ static int aio_read_f(BlockBackend *blk, int argc, char **argv)
     nr_iov = argc - optind;
     ctx->buf = create_iovec(blk, &ctx->qiov, &argv[optind], nr_iov, 0xab);
     if (ctx->buf == NULL) {
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
         g_free(ctx);
         return 0;
     }
@@ -1591,7 +1659,7 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
 
     ctx->offset = cvtnum(argv[optind]);
     if (ctx->offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(ctx->offset, argv[optind]);
         g_free(ctx);
         return 0;
     }
@@ -1600,6 +1668,7 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
     if (ctx->offset & 0x1ff) {
         printf("offset %" PRId64 " is not sector aligned\n",
                ctx->offset);
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
         g_free(ctx);
         return 0;
     }
@@ -1607,6 +1676,7 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
     nr_iov = argc - optind;
     ctx->buf = create_iovec(blk, &ctx->qiov, &argv[optind], nr_iov, pattern);
     if (ctx->buf == NULL) {
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
         g_free(ctx);
         return 0;
     }
@@ -1621,7 +1691,10 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
 
 static int aio_flush_f(BlockBackend *blk, int argc, char **argv)
 {
+    BlockAcctCookie cookie;
+    block_acct_start(blk_get_stats(blk), &cookie, 0, BLOCK_ACCT_FLUSH);
     blk_drain_all();
+    block_acct_done(blk_get_stats(blk), &cookie);
     return 0;
 }
 
@@ -1651,7 +1724,7 @@ static int truncate_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[1]);
     if (offset < 0) {
-        printf("non-numeric truncate argument -- %s\n", argv[1]);
+        print_cvtnum_err(offset, argv[1]);
         return 0;
     }
 
@@ -1777,8 +1850,7 @@ static int discard_f(BlockBackend *blk, int argc, char **argv)
     struct timeval t1, t2;
     int Cflag = 0, qflag = 0;
     int c, ret;
-    int64_t offset;
-    int count;
+    int64_t offset, count;
 
     while ((c = getopt(argc, argv, "Cq")) != -1) {
         switch (c) {
@@ -1799,14 +1871,19 @@ static int discard_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count >> BDRV_SECTOR_BITS > INT_MAX) {
+        printf("length cannot exceed %"PRIu64", given %s\n",
+               (uint64_t)INT_MAX << BDRV_SECTOR_BITS,
+               argv[optind]);
         return 0;
     }
 
@@ -1833,15 +1910,14 @@ out:
 static int alloc_f(BlockBackend *blk, int argc, char **argv)
 {
     BlockDriverState *bs = blk_bs(blk);
-    int64_t offset, sector_num;
-    int nb_sectors, remaining;
+    int64_t offset, sector_num, nb_sectors, remaining;
     char s1[64];
-    int num, sum_alloc;
-    int ret;
+    int num, ret;
+    int64_t sum_alloc;
 
     offset = cvtnum(argv[1]);
     if (offset < 0) {
-        printf("non-numeric offset argument -- %s\n", argv[1]);
+        print_cvtnum_err(offset, argv[1]);
         return 0;
     } else if (offset & 0x1ff) {
         printf("offset %" PRId64 " is not sector aligned\n",
@@ -1852,7 +1928,11 @@ static int alloc_f(BlockBackend *blk, int argc, char **argv)
     if (argc == 3) {
         nb_sectors = cvtnum(argv[2]);
         if (nb_sectors < 0) {
-            printf("non-numeric length argument -- %s\n", argv[2]);
+            print_cvtnum_err(nb_sectors, argv[2]);
+            return 0;
+        } else if (nb_sectors > INT_MAX) {
+            printf("length argument cannot exceed %d, given %s\n",
+                   INT_MAX, argv[2]);
             return 0;
         }
     } else {
@@ -1881,7 +1961,7 @@ static int alloc_f(BlockBackend *blk, int argc, char **argv)
 
     cvtstr(offset, s1, sizeof(s1));
 
-    printf("%d/%d sectors allocated at offset %s\n",
+    printf("%"PRId64"/%"PRId64" sectors allocated at offset %s\n",
            sum_alloc, nb_sectors, s1);
     return 0;
 }
@@ -1979,6 +2059,95 @@ static const cmdinfo_t map_cmd = {
        .oneline        = "prints the allocated areas of a file",
 };
 
+static void reopen_help(void)
+{
+    printf(
+"\n"
+" Changes the open options of an already opened image\n"
+"\n"
+" Example:\n"
+" 'reopen -o lazy-refcounts=on' - activates lazy refcount writeback on a qcow2 image\n"
+"\n"
+" -r, -- Reopen the image read-only\n"
+" -c, -- Change the cache mode to the given value\n"
+" -o, -- Changes block driver options (cf. 'open' command)\n"
+"\n");
+}
+
+static int reopen_f(BlockBackend *blk, int argc, char **argv);
+
+static QemuOptsList reopen_opts = {
+    .name = "reopen",
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(reopen_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
+static const cmdinfo_t reopen_cmd = {
+       .name           = "reopen",
+       .argmin         = 0,
+       .argmax         = -1,
+       .cfunc          = reopen_f,
+       .args           = "[-r] [-c cache] [-o options]",
+       .oneline        = "reopens an image with new options",
+       .help           = reopen_help,
+};
+
+static int reopen_f(BlockBackend *blk, int argc, char **argv)
+{
+    BlockDriverState *bs = blk_bs(blk);
+    QemuOpts *qopts;
+    QDict *opts;
+    int c;
+    int flags = bs->open_flags;
+
+    BlockReopenQueue *brq;
+    Error *local_err = NULL;
+
+    while ((c = getopt(argc, argv, "c:o:r")) != -1) {
+        switch (c) {
+        case 'c':
+            if (bdrv_parse_cache_flags(optarg, &flags) < 0) {
+                error_report("Invalid cache option: %s", optarg);
+                return 0;
+            }
+            break;
+        case 'o':
+            if (!qemu_opts_parse_noisily(&reopen_opts, optarg, 0)) {
+                qemu_opts_reset(&reopen_opts);
+                return 0;
+            }
+            break;
+        case 'r':
+            flags &= ~BDRV_O_RDWR;
+            break;
+        default:
+            qemu_opts_reset(&reopen_opts);
+            return qemuio_command_usage(&reopen_cmd);
+        }
+    }
+
+    if (optind != argc) {
+        qemu_opts_reset(&reopen_opts);
+        return qemuio_command_usage(&reopen_cmd);
+    }
+
+    qopts = qemu_opts_find(&reopen_opts, NULL);
+    opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
+    qemu_opts_reset(&reopen_opts);
+
+    brq = bdrv_reopen_queue(NULL, bs, opts, flags);
+    bdrv_reopen_multiple(brq, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    }
+
+    return 0;
+}
+
 static int break_f(BlockBackend *blk, int argc, char **argv)
 {
     int ret;
@@ -2102,9 +2271,13 @@ static const cmdinfo_t sigraise_cmd = {
 
 static int sigraise_f(BlockBackend *blk, int argc, char **argv)
 {
-    int sig = cvtnum(argv[1]);
+    int64_t sig = cvtnum(argv[1]);
     if (sig < 0) {
-        printf("non-numeric signal number argument -- %s\n", argv[1]);
+        print_cvtnum_err(sig, argv[1]);
+        return 0;
+    } else if (sig > NSIG) {
+        printf("signal argument '%s' is too large to be a valid signal\n",
+               argv[1]);
         return 0;
     }
 
@@ -2266,6 +2439,7 @@ static void __attribute((constructor)) init_qemuio_commands(void)
     qemuio_add_command(&discard_cmd);
     qemuio_add_command(&alloc_cmd);
     qemuio_add_command(&map_cmd);
+    qemuio_add_command(&reopen_cmd);
     qemuio_add_command(&break_cmd);
     qemuio_add_command(&remove_break_cmd);
     qemuio_add_command(&resume_cmd);
diff --git a/qemu-io.c b/qemu-io.c
index f1e3a6713..269f17cc2 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -156,7 +156,6 @@ static int open_f(BlockBackend *blk, int argc, char **argv)
             break;
         case 'o':
             if (!qemu_opts_parse_noisily(&empty_opts, optarg, false)) {
-                printf("could not parse option list -- %s\n", optarg);
                 qemu_opts_reset(&empty_opts);
                 return 0;
             }
diff --git a/qemu-log.c b/qemu-log.c
index 13f3813f6..7cb01a802 100644
--- a/qemu-log.c
+++ b/qemu-log.c
@@ -112,13 +112,14 @@ const QEMULogItem qemu_log_items[] = {
       "x86 only: show protected mode far calls/returns/exceptions" },
     { CPU_LOG_RESET, "cpu_reset",
       "show CPU state before CPU resets" },
-    { CPU_LOG_IOPORT, "ioport",
-      "show all i/o ports accesses" },
     { LOG_UNIMP, "unimp",
       "log unimplemented functionality" },
     { LOG_GUEST_ERROR, "guest_errors",
       "log when the guest OS does something invalid (eg accessing a\n"
       "non-existent register)" },
+    { CPU_LOG_TB_NOCHAIN, "nochain",
+      "do not chain compiled TBs so that \"exec\" and \"cpu\" show\n"
+      "complete traces" },
     { 0, NULL, NULL },
 };
 
diff --git a/qemu-nbd.c b/qemu-nbd.c
index 5106b802e..3afec7650 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -49,7 +49,7 @@
 static NBDExport *exp;
 static int verbose;
 static char *srcpath;
-static char *sockpath;
+static SocketAddress *saddr;
 static int persistent = 0;
 static enum { RUNNING, TERMINATE, TERMINATING, TERMINATED } state;
 static int shared = 1;
@@ -98,11 +98,9 @@ static void usage(const char *name)
 "                            '[ID_OR_NAME]'\n"
 "  -n, --nocache             disable host cache\n"
 "      --cache=MODE          set cache mode (none, writeback, ...)\n"
-#ifdef CONFIG_LINUX_AIO
 "      --aio=MODE            set AIO mode (native or threads)\n"
-#endif
 "      --discard=MODE        set discard mode (ignore, unmap)\n"
-"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, discard)\n"
+"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, unmap)\n"
 "\n"
 "Report bugs to <qemu-devel@nongnu.org>\n"
     , name, NBD_DEFAULT_PORT, "DEVICE");
@@ -213,52 +211,6 @@ static void termsig_handler(int signum)
     qemu_notify_event();
 }
 
-static void combine_addr(char *buf, size_t len, const char* address,
-                         uint16_t port)
-{
-    /* If the address-part contains a colon, it's an IPv6 IP so needs [] */
-    if (strstr(address, ":")) {
-        snprintf(buf, len, "[%s]:%u", address, port);
-    } else {
-        snprintf(buf, len, "%s:%u", address, port);
-    }
-}
-
-static int tcp_socket_incoming(const char *address, uint16_t port)
-{
-    char address_and_port[128];
-    Error *local_err = NULL;
-
-    combine_addr(address_and_port, 128, address, port);
-    int fd = inet_listen(address_and_port, NULL, 0, SOCK_STREAM, 0, &local_err);
-
-    if (local_err != NULL) {
-        error_report_err(local_err);
-    }
-    return fd;
-}
-
-static int unix_socket_incoming(const char *path)
-{
-    Error *local_err = NULL;
-    int fd = unix_listen(path, NULL, 0, &local_err);
-
-    if (local_err != NULL) {
-        error_report_err(local_err);
-    }
-    return fd;
-}
-
-static int unix_socket_outgoing(const char *path)
-{
-    Error *local_err = NULL;
-    int fd = unix_connect(path, &local_err);
-
-    if (local_err != NULL) {
-        error_report_err(local_err);
-    }
-    return fd;
-}
 
 static void *show_parts(void *arg)
 {
@@ -287,8 +239,10 @@ static void *nbd_client_thread(void *arg)
     pthread_t show_parts_thread;
     Error *local_error = NULL;
 
-    sock = unix_socket_outgoing(sockpath);
+
+    sock = socket_connect(saddr, &local_error, NULL, NULL);
     if (sock < 0) {
+        error_report_err(local_error);
         goto out;
     }
 
@@ -362,7 +316,6 @@ static void nbd_client_closed(NBDClient *client)
         state = TERMINATE;
     }
     nbd_update_server_fd_handler(server_fd);
-    qemu_notify_event();
     nbd_client_put(client);
 }
 
@@ -400,6 +353,33 @@ static void nbd_update_server_fd_handler(int fd)
     }
 }
 
+
+static SocketAddress *nbd_build_socket_address(const char *sockpath,
+                                               const char *bindto,
+                                               const char *port)
+{
+    SocketAddress *saddr;
+
+    saddr = g_new0(SocketAddress, 1);
+    if (sockpath) {
+        saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+        saddr->u.q_unix = g_new0(UnixSocketAddress, 1);
+        saddr->u.q_unix->path = g_strdup(sockpath);
+    } else {
+        saddr->type = SOCKET_ADDRESS_KIND_INET;
+        saddr->u.inet = g_new0(InetSocketAddress, 1);
+        saddr->u.inet->host = g_strdup(bindto);
+        if (port) {
+            saddr->u.inet->port = g_strdup(port);
+        } else  {
+            saddr->u.inet->port = g_strdup_printf("%d", NBD_DEFAULT_PORT);
+        }
+    }
+
+    return saddr;
+}
+
+
 int main(int argc, char **argv)
 {
     BlockBackend *blk;
@@ -408,8 +388,9 @@ int main(int argc, char **argv)
     uint32_t nbdflags = 0;
     bool disconnect = false;
     const char *bindto = "0.0.0.0";
+    const char *port = NULL;
+    char *sockpath = NULL;
     char *device = NULL;
-    int port = NBD_DEFAULT_PORT;
     off_t fd_size;
     QemuOpts *sn_opts = NULL;
     const char *sn_id_or_name = NULL;
@@ -429,9 +410,7 @@ int main(int argc, char **argv)
         { "load-snapshot", 1, NULL, 'l' },
         { "nocache", 0, NULL, 'n' },
         { "cache", 1, NULL, QEMU_NBD_OPT_CACHE },
-#ifdef CONFIG_LINUX_AIO
         { "aio", 1, NULL, QEMU_NBD_OPT_AIO },
-#endif
         { "discard", 1, NULL, QEMU_NBD_OPT_DISCARD },
         { "detect-zeroes", 1, NULL, QEMU_NBD_OPT_DETECT_ZEROES },
         { "shared", 1, NULL, 'e' },
@@ -442,7 +421,6 @@ int main(int argc, char **argv)
     };
     int ch;
     int opt_ind = 0;
-    int li;
     char *end;
     int flags = BDRV_O_RDWR;
     int partition = -1;
@@ -450,9 +428,7 @@ int main(int argc, char **argv)
     int fd;
     bool seen_cache = false;
     bool seen_discard = false;
-#ifdef CONFIG_LINUX_AIO
     bool seen_aio = false;
-#endif
     pthread_t client_thread;
     const char *fmt = NULL;
     Error *local_err = NULL;
@@ -485,7 +461,6 @@ int main(int argc, char **argv)
                 errx(EXIT_FAILURE, "Invalid cache mode `%s'", optarg);
             }
             break;
-#ifdef CONFIG_LINUX_AIO
         case QEMU_NBD_OPT_AIO:
             if (seen_aio) {
                 errx(EXIT_FAILURE, "--aio can only be specified once");
@@ -499,7 +474,6 @@ int main(int argc, char **argv)
                errx(EXIT_FAILURE, "invalid aio mode `%s'", optarg);
             }
             break;
-#endif
         case QEMU_NBD_OPT_DISCARD:
             if (seen_discard) {
                 errx(EXIT_FAILURE, "--discard can only be specified once");
@@ -530,14 +504,7 @@ int main(int argc, char **argv)
             bindto = optarg;
             break;
         case 'p':
-            li = strtol(optarg, &end, 0);
-            if (*end) {
-                errx(EXIT_FAILURE, "Invalid port `%s'", optarg);
-            }
-            if (li < 1 || li > 65535) {
-                errx(EXIT_FAILURE, "Port out of range `%s'", optarg);
-            }
-            port = (uint16_t)li;
+            port = optarg;
             break;
         case 'o':
                 dev_offset = strtoll (optarg, &end, 0);
@@ -696,6 +663,8 @@ int main(int argc, char **argv)
         snprintf(sockpath, 128, SOCKET_PATH, basename(device));
     }
 
+    saddr = nbd_build_socket_address(sockpath, bindto, port);
+
     if (qemu_init_main_loop(&local_err)) {
         error_report_err(local_err);
         exit(EXIT_FAILURE);
@@ -753,13 +722,9 @@ int main(int argc, char **argv)
         errx(EXIT_FAILURE, "%s", error_get_pretty(local_err));
     }
 
-    if (sockpath) {
-        fd = unix_socket_incoming(sockpath);
-    } else {
-        fd = tcp_socket_incoming(bindto, port);
-    }
-
+    fd = socket_listen(saddr, &local_err);
     if (fd < 0) {
+        error_report_err(local_err);
         return 1;
     }
 
diff --git a/qemu-options.hx b/qemu-options.hx
index 77f5853d5..85a6cb8d4 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -38,9 +38,11 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
     "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
     "                mem-merge=on|off controls memory merge support (default: on)\n"
     "                iommu=on|off controls emulated Intel IOMMU (VT-d) support (default=off)\n"
+    "                igd-passthru=on|off controls IGD GFX passthrough support (default=off)\n"
     "                aes-key-wrap=on|off controls support for AES key wrapping (default=on)\n"
     "                dea-key-wrap=on|off controls support for DEA key wrapping (default=on)\n"
-    "                suppress-vmdesc=on|off disables self-describing migration (default=off)\n",
+    "                suppress-vmdesc=on|off disables self-describing migration (default=off)\n"
+    "                enforce-config-section=on|off enforce configuration section migration (default=off)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -machine [type=]@var{name}[,prop=@var{value}[,...]]
@@ -55,6 +57,8 @@ than one accelerator specified, the next one is used if the previous one fails
 to initialize.
 @item kernel_irqchip=on|off
 Enables in-kernel irqchip support for the chosen accelerator when available.
+@item gfx_passthru=on|off
+Enables IGD GFX passthrough support for the chosen machine when available.
 @item vmport=on|off|auto
 Enables emulation of VMWare IO port, for vmmouse etc. auto says to select the
 value based on accel. For accel=xen the default is off otherwise the default
@@ -1214,8 +1218,9 @@ By definition the Websocket port is 5700+@var{display}. If @var{host} is
 specified connections will only be allowed from this host.
 As an alternative the Websocket port could be specified by using
 @code{websocket}=@var{port}.
-TLS encryption for the Websocket connection is supported if the required
-certificates are specified with the VNC option @option{x509}.
+If no TLS credentials are provided, the websocket connection runs in
+unencrypted mode. If TLS credentials are provided, the websocket connection
+requires encrypted client connections.
 
 @item password
 
@@ -1236,6 +1241,20 @@ date and time).
 You can also use keywords "now" or "never" for the expiration time to
 allow <protocol> password to expire immediately or never expire.
 
+@item tls-creds=@var{ID}
+
+Provides the ID of a set of TLS credentials to use to secure the
+VNC server. They will apply to both the normal VNC server socket
+and the websocket socket (if enabled). Setting TLS credentials
+will cause the VNC server socket to enable the VeNCrypt auth
+mechanism.  The credentials should have been previously created
+using the @option{-object tls-creds} argument.
+
+The @option{tls-creds} parameter obsoletes the @option{tls},
+@option{x509}, and @option{x509verify} options, and as such
+it is not permitted to set both new and old type options at
+the same time.
+
 @item tls
 
 Require that client use TLS when communicating with the VNC server. This
@@ -1243,6 +1262,9 @@ uses anonymous TLS credentials so is susceptible to a man-in-the-middle
 attack. It is recommended that this option be combined with either the
 @option{x509} or @option{x509verify} options.
 
+This option is now deprecated in favor of using the @option{tls-creds}
+argument.
+
 @item x509=@var{/path/to/certificate/dir}
 
 Valid if @option{tls} is specified. Require that x509 credentials are used
@@ -1252,6 +1274,9 @@ to provide authentication of the client when this is used. The path following
 this option specifies where the x509 certificates are to be loaded from.
 See the @ref{vnc_security} section for details on generating certificates.
 
+This option is now deprecated in favour of using the @option{tls-creds}
+argument.
+
 @item x509verify=@var{/path/to/certificate/dir}
 
 Valid if @option{tls} is specified. Require that x509 credentials are used
@@ -1265,6 +1290,9 @@ path following this option specifies where the x509 certificates are to
 be loaded from. See the @ref{vnc_security} section for details on generating
 certificates.
 
+This option is now deprecated in favour of using the @option{tls-creds}
+argument.
+
 @item sasl
 
 Require that the client use SASL to authenticate with the VNC server.
@@ -1412,7 +1440,7 @@ DEF("smbios", HAS_ARG, QEMU_OPTION_smbios,
     "-smbios type=17[,loc_pfx=str][,bank=str][,manufacturer=str][,serial=str]\n"
     "               [,asset=str][,part=str][,speed=%d]\n"
     "                specify SMBIOS type 17 fields\n",
-    QEMU_ARCH_I386)
+    QEMU_ARCH_I386 | QEMU_ARCH_ARM)
 STEXI
 @item -smbios file=@var{binary}
 @findex -smbios
@@ -1963,13 +1991,14 @@ The hubport netdev lets you connect a NIC to a QEMU "vlan" instead of a single
 netdev.  @code{-net} and @code{-device} with parameter @option{vlan} create the
 required hub automatically.
 
-@item -netdev vhost-user,chardev=@var{id}[,vhostforce=on|off]
+@item -netdev vhost-user,chardev=@var{id}[,vhostforce=on|off][,queues=n]
 
 Establish a vhost-user netdev, backed by a chardev @var{id}. The chardev should
 be a unix domain socket backed one. The vhost-user uses a specifically defined
 protocol to pass vhost ioctl replacement messages to an application on the other
 end of the socket. On non-MSIX guests, the feature can be forced with
-@var{vhostforce}.
+@var{vhostforce}. Use 'queues=@var{n}' to specify the number of queues to
+be created for multiqueue vhost-user.
 
 Example:
 @example
@@ -1984,6 +2013,7 @@ qemu -m 512 -object memory-backend-file,id=mem,size=512M,mem-path=/hugetlbfs,sha
 Dump network traffic on VLAN @var{n} to file @var{file} (@file{qemu-vlan0.pcap} by default).
 At most @var{len} bytes (64k by default) per packet are stored. The file format is
 libpcap, so it can be analyzed with tools such as tcpdump or Wireshark.
+Note: For devices created with '-netdev', use '-object filter-dump,...' instead.
 
 @item -net none
 Indicate that no network devices should be configured. It is used to
@@ -2696,13 +2726,18 @@ ETEXI
 
 DEF("fw_cfg", HAS_ARG, QEMU_OPTION_fwcfg,
     "-fw_cfg [name=]<name>,file=<file>\n"
-    "                add named fw_cfg entry from file\n",
+    "                add named fw_cfg entry from file\n"
+    "-fw_cfg [name=]<name>,string=<str>\n"
+    "                add named fw_cfg entry from string\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -fw_cfg [name=]@var{name},file=@var{file}
 @findex -fw_cfg
 Add named fw_cfg entry from file. @var{name} determines the name of
 the entry in the fw_cfg file directory exposed to the guest.
+
+@item -fw_cfg [name=]@var{name},string=@var{str}
+Add named fw_cfg entry from string.
 ETEXI
 
 DEF("serial", HAS_ARG, QEMU_OPTION_serial, \
@@ -3123,12 +3158,12 @@ re-inject them.
 ETEXI
 
 DEF("icount", HAS_ARG, QEMU_OPTION_icount, \
-    "-icount [shift=N|auto][,align=on|off][,sleep=no]\n" \
+    "-icount [shift=N|auto][,align=on|off][,sleep=no,rr=record|replay,rrfile=<filename>]\n" \
     "                enable virtual instruction counter with 2^N clock ticks per\n" \
     "                instruction, enable aligning the host and virtual clocks\n" \
     "                or disable real time cpu sleeping\n", QEMU_ARCH_ALL)
 STEXI
-@item -icount [shift=@var{N}|auto]
+@item -icount [shift=@var{N}|auto][,rr=record|replay,rrfile=@var{filename}]
 @findex -icount
 Enable virtual instruction counter.  The virtual cpu will execute one
 instruction every 2^@var{N} ns of virtual time.  If @code{auto} is specified
@@ -3147,7 +3182,7 @@ provide cycle accurate emulation.  Modern CPUs contain superscalar out of
 order cores with complex cache hierarchies.  The number of instructions
 executed often has little or no correlation with actual performance.
 
-@option{align=on} will activate the delay algorithm which will try to
+@option{align=on} will activate the delay algorithm which will try
 to synchronise the host clock and the virtual clock. The goal is to
 have a guest running at the real frequency imposed by the shift option.
 Whenever the guest clock is behind the host clock and if
@@ -3157,6 +3192,10 @@ Currently this option does not work when @option{shift} is @code{auto}.
 Note: The sync algorithm will work for those shift values for which
 the guest clock runs ahead of the host clock. Typically this happens
 when the shift value is high (how high depends on the host machine).
+
+When @option{rr} option is specified deterministic record/replay is enabled.
+Replay log is written into @var{filename} file in record mode and
+read from this file in replay mode.
 ETEXI
 
 DEF("watchdog", HAS_ARG, QEMU_OPTION_watchdog, \
@@ -3510,7 +3549,7 @@ DEF("dump-vmstate", HAS_ARG, QEMU_OPTION_dump_vmstate,
     "                Output vmstate information in JSON format to file.\n"
     "                Use the scripts/vmstate-static-checker.py file to\n"
     "                check for possible regressions in migration code\n"
-    "                by comparing two such vmstate dumps.",
+    "                by comparing two such vmstate dumps.\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -dump-vmstate @var{file}
@@ -3568,6 +3607,77 @@ the @option{virtio-rng} device. The @option{chardev} parameter is
 the unique ID of a character device backend that provides the connection
 to the RNG daemon.
 
+@item -object tls-creds-anon,id=@var{id},endpoint=@var{endpoint},dir=@var{/path/to/cred/dir},verify-peer=@var{on|off}
+
+Creates a TLS anonymous credentials object, which can be used to provide
+TLS support on network backends. The @option{id} parameter is a unique
+ID which network backends will use to access the credentials. The
+@option{endpoint} is either @option{server} or @option{client} depending
+on whether the QEMU network backend that uses the credentials will be
+acting as a client or as a server. If @option{verify-peer} is enabled
+(the default) then once the handshake is completed, the peer credentials
+will be verified, though this is a no-op for anonymous credentials.
+
+The @var{dir} parameter tells QEMU where to find the credential
+files. For server endpoints, this directory may contain a file
+@var{dh-params.pem} providing diffie-hellman parameters to use
+for the TLS server. If the file is missing, QEMU will generate
+a set of DH parameters at startup. This is a computationally
+expensive operation that consumes random pool entropy, so it is
+recommended that a persistent set of parameters be generated
+upfront and saved.
+
+@item -object tls-creds-x509,id=@var{id},endpoint=@var{endpoint},dir=@var{/path/to/cred/dir},verify-peer=@var{on|off}
+
+Creates a TLS anonymous credentials object, which can be used to provide
+TLS support on network backends. The @option{id} parameter is a unique
+ID which network backends will use to access the credentials. The
+@option{endpoint} is either @option{server} or @option{client} depending
+on whether the QEMU network backend that uses the credentials will be
+acting as a client or as a server. If @option{verify-peer} is enabled
+(the default) then once the handshake is completed, the peer credentials
+will be verified. With x509 certificates, this implies that the clients
+must be provided with valid client certificates too.
+
+The @var{dir} parameter tells QEMU where to find the credential
+files. For server endpoints, this directory may contain a file
+@var{dh-params.pem} providing diffie-hellman parameters to use
+for the TLS server. If the file is missing, QEMU will generate
+a set of DH parameters at startup. This is a computationally
+expensive operation that consumes random pool entropy, so it is
+recommended that a persistent set of parameters be generated
+upfront and saved.
+
+For x509 certificate credentials the directory will contain further files
+providing the x509 certificates. The certificates must be stored
+in PEM format, in filenames @var{ca-cert.pem}, @var{ca-crl.pem} (optional),
+@var{server-cert.pem} (only servers), @var{server-key.pem} (only servers),
+@var{client-cert.pem} (only clients), and @var{client-key.pem} (only clients).
+
+@item -object filter-buffer,id=@var{id},netdev=@var{netdevid},interval=@var{t}[,queue=@var{all|rx|tx}]
+
+Interval @var{t} can't be 0, this filter batches the packet delivery: all
+packets arriving in a given interval on netdev @var{netdevid} are delayed
+until the end of the interval. Interval is in microseconds.
+
+queue @var{all|rx|tx} is an option that can be applied to any netfilter.
+
+@option{all}: the filter is attached both to the receive and the transmit
+              queue of the netdev (default).
+
+@option{rx}: the filter is attached to the receive queue of the netdev,
+             where it will receive packets sent to the netdev.
+
+@option{tx}: the filter is attached to the transmit queue of the netdev,
+             where it will receive packets sent by the netdev.
+
+@item -object filter-dump,id=@var{id},netdev=@var{dev},file=@var{filename}][,maxlen=@var{len}]
+
+Dump the network traffic on netdev @var{dev} to the file specified by
+@var{filename}. At most @var{len} bytes (64k by default) per packet are stored.
+The file format is libpcap, so it can be analyzed with tools such as tcpdump
+or Wireshark.
+
 @end table
 
 ETEXI
diff --git a/qemu-seccomp.c b/qemu-seccomp.c
index f9de0d339..c831fe83a 100644
--- a/qemu-seccomp.c
+++ b/qemu-seccomp.c
@@ -16,6 +16,14 @@
 #include <seccomp.h>
 #include "sysemu/seccomp.h"
 
+#if SCMP_VER_MAJOR >= 3
+  #define HAVE_CACHEFLUSH
+#elif SCMP_VER_MAJOR == 2 && SCMP_VER_MINOR >= 3
+  #define HAVE_CACHEFLUSH
+#elif SCMP_VER_MAJOR == 2 && SCMP_VER_MINOR == 2 && SCMP_VER_MICRO >= 3
+  #define HAVE_CACHEFLUSH
+#endif
+
 struct QemuSeccompSyscall {
     int32_t num;
     uint8_t priority;
@@ -237,7 +245,11 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(fadvise64), 240 },
     { SCMP_SYS(inotify_init1), 240 },
     { SCMP_SYS(inotify_add_watch), 240 },
-    { SCMP_SYS(mbind), 240 }
+    { SCMP_SYS(mbind), 240 },
+    { SCMP_SYS(memfd_create), 240 },
+#ifdef HAVE_CACHEFLUSH
+    { SCMP_SYS(cacheflush), 240 },
+#endif
 };
 
 int seccomp_start(void)
diff --git a/qemu-tech.texi b/qemu-tech.texi
index b6fcb2d61..022017d55 100644
--- a/qemu-tech.texi
+++ b/qemu-tech.texi
@@ -320,7 +320,7 @@ SH4
 @node QEMU compared to other emulators
 @section QEMU compared to other emulators
 
-Like bochs [3], QEMU emulates an x86 CPU. But QEMU is much faster than
+Like bochs [1], QEMU emulates an x86 CPU. But QEMU is much faster than
 bochs as it uses dynamic compilation. Bochs is closely tied to x86 PC
 emulation while QEMU can emulate several processors.
 
@@ -333,25 +333,25 @@ than QEMU (in particular it does register allocation) but it is closely
 tied to an x86 host and target and has no support for precise exceptions
 and system emulation.
 
-EM86 [4] is the closest project to user space QEMU (and QEMU still uses
+EM86 [3] is the closest project to user space QEMU (and QEMU still uses
 some of its code, in particular the ELF file loader). EM86 was limited
 to an alpha host and used a proprietary and slow interpreter (the
-interpreter part of the FX!32 Digital Win32 code translator [5]).
+interpreter part of the FX!32 Digital Win32 code translator [4]).
 
-TWIN [6] is a Windows API emulator like Wine. It is less accurate than
-Wine but includes a protected mode x86 interpreter to launch x86 Windows
-executables. Such an approach has greater potential because most of the
-Windows API is executed natively but it is far more difficult to develop
-because all the data structures and function parameters exchanged
+TWIN from Willows Software was a Windows API emulator like Wine. It is less
+accurate than Wine but includes a protected mode x86 interpreter to launch
+x86 Windows executables. Such an approach has greater potential because most
+of the Windows API is executed natively but it is far more difficult to
+develop because all the data structures and function parameters exchanged
 between the API and the x86 code must be converted.
 
-User mode Linux [7] was the only solution before QEMU to launch a
+User mode Linux [5] was the only solution before QEMU to launch a
 Linux kernel as a process while not needing any host kernel
 patches. However, user mode Linux requires heavy kernel patches while
 QEMU accepts unpatched Linux kernels. The price to pay is that QEMU is
 slower.
 
-The Plex86 [8] PC virtualizer is done in the same spirit as the now
+The Plex86 [6] PC virtualizer is done in the same spirit as the now
 obsolete qemu-fast system emulator. It requires a patched Linux kernel
 to work (you cannot launch the same kernel on your PC), but the
 patches are really small. As it is a PC virtualizer (no emulation is
@@ -359,13 +359,13 @@ done except for some privileged instructions), it has the potential of
 being faster than QEMU. The downside is that a complicated (and
 potentially unsafe) host kernel patch is needed.
 
-The commercial PC Virtualizers (VMWare [9], VirtualPC [10], TwoOStwo
-[11]) are faster than QEMU, but they all need specific, proprietary
+The commercial PC Virtualizers (VMWare [7], VirtualPC [8]) are faster
+than QEMU (without virtualization), but they all need specific, proprietary
 and potentially unsafe host drivers. Moreover, they are unable to
 provide cycle exact simulation as an emulator can.
 
-VirtualBox [12], Xen [13] and KVM [14] are based on QEMU. QEMU-SystemC
-[15] uses QEMU to simulate a system where some hardware devices are
+VirtualBox [9], Xen [10] and KVM [11] are based on QEMU. QEMU-SystemC
+[12] uses QEMU to simulate a system where some hardware devices are
 developed in SystemC.
 
 @node Portable dynamic translation
@@ -608,64 +608,51 @@ way, it can be relocated at load time.
 @table @asis
 
 @item [1]
-@url{http://citeseer.nj.nec.com/piumarta98optimizing.html}, Optimizing
-direct threaded code by selective inlining (1998) by Ian Piumarta, Fabio
-Riccardi.
+@url{http://bochs.sourceforge.net/}, the Bochs IA-32 Emulator Project,
+by Kevin Lawton et al.
 
 @item [2]
-@url{http://developer.kde.org/~sewardj/}, Valgrind, an open-source
-memory debugger for x86-GNU/Linux, by Julian Seward.
+@url{http://www.valgrind.org/}, Valgrind, an open-source memory debugger
+for GNU/Linux.
 
 @item [3]
-@url{http://bochs.sourceforge.net/}, the Bochs IA-32 Emulator Project,
-by Kevin Lawton et al.
+@url{http://ftp.dreamtime.org/pub/linux/Linux-Alpha/em86/v0.2/docs/em86.html},
+the EM86 x86 emulator on Alpha-Linux.
 
 @item [4]
-@url{http://www.cs.rose-hulman.edu/~donaldlf/em86/index.html}, the EM86
-x86 emulator on Alpha-Linux.
-
-@item [5]
 @url{http://www.usenix.org/publications/library/proceedings/usenix-nt97/@/full_papers/chernoff/chernoff.pdf},
 DIGITAL FX!32: Running 32-Bit x86 Applications on Alpha NT, by Anton
 Chernoff and Ray Hookway.
 
-@item [6]
-@url{http://www.willows.com/}, Windows API library emulation from
-Willows Software.
-
-@item [7]
+@item [5]
 @url{http://user-mode-linux.sourceforge.net/},
 The User-mode Linux Kernel.
 
-@item [8]
+@item [6]
 @url{http://www.plex86.org/},
 The new Plex86 project.
 
-@item [9]
+@item [7]
 @url{http://www.vmware.com/},
 The VMWare PC virtualizer.
 
-@item [10]
-@url{http://www.microsoft.com/windowsxp/virtualpc/},
+@item [8]
+@url{https://www.microsoft.com/download/details.aspx?id=3702},
 The VirtualPC PC virtualizer.
 
-@item [11]
-@url{http://www.twoostwo.org/},
-The TwoOStwo PC virtualizer.
-
-@item [12]
+@item [9]
 @url{http://virtualbox.org/},
 The VirtualBox PC virtualizer.
 
-@item [13]
+@item [10]
 @url{http://www.xen.org/},
 The Xen hypervisor.
 
-@item [14]
-@url{http://kvm.qumranet.com/kvmwiki/Front_Page},
+@item [11]
+@url{http://www.linux-kvm.org/},
 Kernel Based Virtual Machine (KVM).
 
-@item [15]
+@item [12]
 @url{http://www.greensocs.com/projects/QEMUSystemC},
 QEMU-SystemC, a hardware co-simulator.
 
diff --git a/qemu-timer.c b/qemu-timer.c
index 2463fe6f6..f16e42283 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -24,6 +24,8 @@
 
 #include "qemu/main-loop.h"
 #include "qemu/timer.h"
+#include "sysemu/replay.h"
+#include "sysemu/sysemu.h"
 
 #ifdef CONFIG_POSIX
 #include <pthread.h>
@@ -477,10 +479,31 @@ bool timerlist_run_timers(QEMUTimerList *timer_list)
     void *opaque;
 
     qemu_event_reset(&timer_list->timers_done_ev);
-    if (!timer_list->clock->enabled) {
+    if (!timer_list->clock->enabled || !timer_list->active_timers) {
         goto out;
     }
 
+    switch (timer_list->clock->type) {
+    case QEMU_CLOCK_REALTIME:
+        break;
+    default:
+    case QEMU_CLOCK_VIRTUAL:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_HOST:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_HOST)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_VIRTUAL_RT:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL_RT)) {
+            goto out;
+        }
+        break;
+    }
+
     current_time = qemu_clock_get_ns(timer_list->clock->type);
     for(;;) {
         qemu_mutex_lock(&timer_list->active_timers_lock);
@@ -544,11 +567,17 @@ int64_t timerlistgroup_deadline_ns(QEMUTimerListGroup *tlg)
 {
     int64_t deadline = -1;
     QEMUClockType type;
+    bool play = replay_mode == REPLAY_MODE_PLAY;
     for (type = 0; type < QEMU_CLOCK_MAX; type++) {
-        if (qemu_clock_use_for_deadline(tlg->tl[type]->clock->type)) {
-            deadline = qemu_soonest_timeout(deadline,
-                                            timerlist_deadline_ns(
-                                                tlg->tl[type]));
+        if (qemu_clock_use_for_deadline(type)) {
+            if (!play || type == QEMU_CLOCK_REALTIME) {
+                deadline = qemu_soonest_timeout(deadline,
+                                                timerlist_deadline_ns(tlg->tl[type]));
+            } else {
+                /* Read clock from the replay file and
+                   do not calculate the deadline, based on virtual clock. */
+                qemu_clock_get_ns(type);
+            }
         }
     }
     return deadline;
@@ -570,7 +599,7 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
             return cpu_get_clock();
         }
     case QEMU_CLOCK_HOST:
-        now = get_clock_realtime();
+        now = REPLAY_CLOCK(REPLAY_CLOCK_HOST, get_clock_realtime());
         last = clock->last;
         clock->last = now;
         if (now < last || now > (last + get_max_clock_jump())) {
@@ -578,7 +607,7 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
         }
         return now;
     case QEMU_CLOCK_VIRTUAL_RT:
-        return cpu_get_clock();
+        return REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT, cpu_get_clock());
     }
 }
 
diff --git a/qemu.nsi b/qemu.nsi
index cc5fafd57..a20f6ef35 100644
--- a/qemu.nsi
+++ b/qemu.nsi
@@ -139,6 +139,9 @@ Section "${PRODUCT} (required)"
 
     ; Write the uninstall keys for Windows
     WriteRegStr HKLM "${UNINST_KEY}" "DisplayName" "QEMU"
+!ifdef DISPLAYVERSION
+    WriteRegStr HKLM "${UNINST_KEY}" "DisplayVersion" "${DISPLAYVERSION}"
+!endif
     WriteRegStr HKLM "${UNINST_KEY}" "UninstallString" '"${UNINST_EXE}"'
     WriteRegDWORD HKLM "${UNINST_KEY}" "NoModify" 1
     WriteRegDWORD HKLM "${UNINST_KEY}" "NoRepair" 1
diff --git a/qga/channel-posix.c b/qga/channel-posix.c
index 8aad4fee9..50d9dd374 100644
--- a/qga/channel-posix.c
+++ b/qga/channel-posix.c
@@ -217,25 +217,24 @@ GIOStatus ga_channel_write_all(GAChannel *c, const gchar *buf, gsize size)
     GIOStatus status = G_IO_STATUS_NORMAL;
 
     while (size) {
+        g_debug("sending data, count: %d", (int)size);
         status = g_io_channel_write_chars(c->client_channel, buf, size,
                                           &written, &err);
-        g_debug("sending data, count: %d", (int)size);
-        if (err != NULL) {
+        if (status == G_IO_STATUS_NORMAL) {
+            size -= written;
+            buf += written;
+        } else if (status != G_IO_STATUS_AGAIN) {
             g_warning("error writing to channel: %s", err->message);
-            return G_IO_STATUS_ERROR;
-        }
-        if (status != G_IO_STATUS_NORMAL) {
-            break;
+            return status;
         }
-        size -= written;
     }
 
-    if (status == G_IO_STATUS_NORMAL) {
+    do {
         status = g_io_channel_flush(c->client_channel, &err);
-        if (err != NULL) {
-            g_warning("error flushing channel: %s", err->message);
-            return G_IO_STATUS_ERROR;
-        }
+    } while (status == G_IO_STATUS_AGAIN);
+
+    if (status != G_IO_STATUS_NORMAL) {
+        g_warning("error flushing channel: %s", err->message);
     }
 
     return status;
@@ -249,7 +248,7 @@ GIOStatus ga_channel_read(GAChannel *c, gchar *buf, gsize size, gsize *count)
 GAChannel *ga_channel_new(GAChannelMethod method, const gchar *path,
                           GAChannelCallback cb, gpointer opaque)
 {
-    GAChannel *c = g_malloc0(sizeof(GAChannel));
+    GAChannel *c = g_new0(GAChannel, 1);
     c->event_cb = cb;
     c->user_data = opaque;
 
diff --git a/qga/channel-win32.c b/qga/channel-win32.c
index 04fa5e4d1..0452b9f75 100644
--- a/qga/channel-win32.c
+++ b/qga/channel-win32.c
@@ -269,7 +269,7 @@ static GIOStatus ga_channel_write(GAChannel *c, const char *buf, size_t size,
 GIOStatus ga_channel_write_all(GAChannel *c, const char *buf, size_t size)
 {
     GIOStatus status = G_IO_STATUS_NORMAL;
-    size_t count;
+    size_t count = 0;
 
     while (size) {
         status = ga_channel_write(c, buf, size, &count);
@@ -322,7 +322,7 @@ static gboolean ga_channel_open(GAChannel *c, GAChannelMethod method,
 GAChannel *ga_channel_new(GAChannelMethod method, const gchar *path,
                           GAChannelCallback cb, gpointer opaque)
 {
-    GAChannel *c = g_malloc0(sizeof(GAChannel));
+    GAChannel *c = g_new0(GAChannel, 1);
     SECURITY_ATTRIBUTES sec_attrs;
 
     if (!ga_channel_open(c, method, path)) {
diff --git a/qga/commands-posix.c b/qga/commands-posix.c
index 675f4b4c6..c2ff97021 100644
--- a/qga/commands-posix.c
+++ b/qga/commands-posix.c
@@ -28,6 +28,7 @@
 #include "qapi/qmp/qerror.h"
 #include "qemu/queue.h"
 #include "qemu/host-utils.h"
+#include "qemu/sockets.h"
 
 #ifndef CONFIG_HAS_ENVIRON
 #ifdef __APPLE__
@@ -215,15 +216,24 @@ void qmp_guest_set_time(bool has_time, int64_t time_ns, Error **errp)
     }
 }
 
+typedef enum {
+    RW_STATE_NEW,
+    RW_STATE_READING,
+    RW_STATE_WRITING,
+} RwState;
+
 typedef struct GuestFileHandle {
     uint64_t id;
     FILE *fh;
+    RwState state;
     QTAILQ_ENTRY(GuestFileHandle) next;
 } GuestFileHandle;
 
 static struct {
     QTAILQ_HEAD(, GuestFileHandle) filehandles;
-} guest_file_state;
+} guest_file_state = {
+    .filehandles = QTAILQ_HEAD_INITIALIZER(guest_file_state.filehandles),
+};
 
 static int64_t guest_file_handle_add(FILE *fh, Error **errp)
 {
@@ -235,7 +245,7 @@ static int64_t guest_file_handle_add(FILE *fh, Error **errp)
         return -1;
     }
 
-    gfh = g_malloc0(sizeof(GuestFileHandle));
+    gfh = g_new0(GuestFileHandle, 1);
     gfh->id = handle;
     gfh->fh = fh;
     QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
@@ -383,27 +393,6 @@ safe_open_or_create(const char *path, const char *mode, Error **errp)
     return NULL;
 }
 
-static int guest_file_toggle_flags(int fd, int flags, bool set, Error **err)
-{
-    int ret, old_flags;
-
-    old_flags = fcntl(fd, F_GETFL);
-    if (old_flags == -1) {
-        error_setg_errno(err, errno, QERR_QGA_COMMAND_FAILED,
-                         "failed to fetch filehandle flags");
-        return -1;
-    }
-
-    ret = fcntl(fd, F_SETFL, set ? (old_flags | flags) : (old_flags & ~flags));
-    if (ret == -1) {
-        error_setg_errno(err, errno, QERR_QGA_COMMAND_FAILED,
-                         "failed to set filehandle flags");
-        return -1;
-    }
-
-    return ret;
-}
-
 int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode,
                             Error **errp)
 {
@@ -424,10 +413,7 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode,
     /* set fd non-blocking to avoid common use cases (like reading from a
      * named pipe) from hanging the agent
      */
-    if (guest_file_toggle_flags(fileno(fh), O_NONBLOCK, true, errp) < 0) {
-        fclose(fh);
-        return -1;
-    }
+    qemu_set_nonblock(fileno(fh));
 
     handle = guest_file_handle_add(fh, errp);
     if (handle < 0) {
@@ -481,6 +467,17 @@ struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
     }
 
     fh = gfh->fh;
+
+    /* explicitly flush when switching from writing to reading */
+    if (gfh->state == RW_STATE_WRITING) {
+        int ret = fflush(fh);
+        if (ret == EOF) {
+            error_setg_errno(errp, errno, "failed to flush file");
+            return NULL;
+        }
+        gfh->state = RW_STATE_NEW;
+    }
+
     buf = g_malloc0(count+1);
     read_count = fread(buf, 1, count, fh);
     if (ferror(fh)) {
@@ -488,12 +485,13 @@ struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
         slog("guest-file-read failed, handle: %" PRId64, handle);
     } else {
         buf[read_count] = 0;
-        read_data = g_malloc0(sizeof(GuestFileRead));
+        read_data = g_new0(GuestFileRead, 1);
         read_data->count = read_count;
         read_data->eof = feof(fh);
         if (read_count) {
             read_data->buf_b64 = g_base64_encode(buf, read_count);
         }
+        gfh->state = RW_STATE_READING;
     }
     g_free(buf);
     clearerr(fh);
@@ -517,6 +515,16 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
     }
 
     fh = gfh->fh;
+
+    if (gfh->state == RW_STATE_READING) {
+        int ret = fseek(fh, 0, SEEK_CUR);
+        if (ret == -1) {
+            error_setg_errno(errp, errno, "failed to seek file");
+            return NULL;
+        }
+        gfh->state = RW_STATE_NEW;
+    }
+
     buf = g_base64_decode(buf_b64, &buf_len);
 
     if (!has_count) {
@@ -533,9 +541,10 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
         error_setg_errno(errp, errno, "failed to write to file");
         slog("guest-file-write failed, handle: %" PRId64, handle);
     } else {
-        write_data = g_malloc0(sizeof(GuestFileWrite));
+        write_data = g_new0(GuestFileWrite, 1);
         write_data->count = write_count;
         write_data->eof = feof(fh);
+        gfh->state = RW_STATE_WRITING;
     }
     g_free(buf);
     clearerr(fh);
@@ -544,25 +553,47 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
 }
 
 struct GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
-                                          int64_t whence, Error **errp)
+                                          int64_t whence_code, Error **errp)
 {
     GuestFileHandle *gfh = guest_file_handle_find(handle, errp);
     GuestFileSeek *seek_data = NULL;
     FILE *fh;
     int ret;
+    int whence;
 
     if (!gfh) {
         return NULL;
     }
 
+    /* We stupidly exposed 'whence':'int' in our qapi */
+    switch (whence_code) {
+    case QGA_SEEK_SET:
+        whence = SEEK_SET;
+        break;
+    case QGA_SEEK_CUR:
+        whence = SEEK_CUR;
+        break;
+    case QGA_SEEK_END:
+        whence = SEEK_END;
+        break;
+    default:
+        error_setg(errp, "invalid whence code %"PRId64, whence_code);
+        return NULL;
+    }
+
     fh = gfh->fh;
     ret = fseek(fh, offset, whence);
     if (ret == -1) {
         error_setg_errno(errp, errno, "failed to seek file");
+        if (errno == ESPIPE) {
+            /* file is non-seekable, stdio shouldn't be buffering anyways */
+            gfh->state = RW_STATE_NEW;
+        }
     } else {
         seek_data = g_new0(GuestFileSeek, 1);
         seek_data->position = ftell(fh);
         seek_data->eof = feof(fh);
+        gfh->state = RW_STATE_NEW;
     }
     clearerr(fh);
 
@@ -583,14 +614,11 @@ void qmp_guest_file_flush(int64_t handle, Error **errp)
     ret = fflush(fh);
     if (ret == EOF) {
         error_setg_errno(errp, errno, "failed to flush file");
+    } else {
+        gfh->state = RW_STATE_NEW;
     }
 }
 
-static void guest_file_init(void)
-{
-    QTAILQ_INIT(&guest_file_state.filehandles);
-}
-
 /* linux-specific implementations. avoid this if at all possible. */
 #if defined(__linux__)
 
@@ -678,7 +706,7 @@ static void build_fs_mount_list_from_mtab(FsMountList *mounts, Error **errp)
             continue;
         }
 
-        mount = g_malloc0(sizeof(FsMount));
+        mount = g_new0(FsMount, 1);
         mount->dirname = g_strdup(ment->mnt_dir);
         mount->devtype = g_strdup(ment->mnt_type);
         mount->devmajor = devmajor;
@@ -757,7 +785,7 @@ static void build_fs_mount_list(FsMountList *mounts, Error **errp)
             }
         }
 
-        mount = g_malloc0(sizeof(FsMount));
+        mount = g_new0(FsMount, 1);
         mount->dirname = g_strdup(line + dir_s);
         mount->devtype = g_strdup(dash + type_s);
         mount->devmajor = devmajor;
@@ -2158,7 +2186,7 @@ static void transfer_memory_block(GuestMemoryBlock *mem_blk, bool sys2memblk,
 
         ga_read_sysfs_file(dirfd, "removable", &removable, 1, &local_err);
         if (local_err) {
-            /* if no 'removable' file, it does't support offline mem blk */
+            /* if no 'removable' file, it doesn't support offline mem blk */
             if (errno == ENOENT) {
                 error_free(local_err);
                 mem_blk->can_offline = false;
@@ -2213,8 +2241,14 @@ GuestMemoryBlockList *qmp_guest_get_memory_blocks(Error **errp)
 
     dp = opendir("/sys/devices/system/memory/");
     if (!dp) {
-        error_setg_errno(errp, errno, "Can't open directory"
-                         "\"/sys/devices/system/memory/\"\n");
+        /* it's ok if this happens to be a system that doesn't expose
+         * memory blocks via sysfs, but otherwise we should report
+         * an error
+         */
+        if (errno != ENOENT) {
+            error_setg_errno(errp, errno, "Can't open directory"
+                             "\"/sys/devices/system/memory/\"\n");
+        }
         return NULL;
     }
 
@@ -2454,7 +2488,7 @@ GList *ga_command_blacklist_init(GList *blacklist)
         char **p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 #endif
@@ -2468,13 +2502,13 @@ GList *ga_command_blacklist_init(GList *blacklist)
         char **p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 #endif
 
 #if !defined(CONFIG_FSTRIM)
-    blacklist = g_list_append(blacklist, (char *)"guest-fstrim");
+    blacklist = g_list_append(blacklist, g_strdup("guest-fstrim"));
 #endif
 
     return blacklist;
@@ -2486,5 +2520,4 @@ void ga_command_state_init(GAState *s, GACommandState *cs)
 #if defined(CONFIG_FSFREEZE)
     ga_command_state_add(cs, NULL, guest_fsfreeze_cleanup);
 #endif
-    ga_command_state_add(cs, guest_file_init, NULL);
 }
diff --git a/qga/commands-win32.c b/qga/commands-win32.c
index a7822d5ff..0654fe4fe 100644
--- a/qga/commands-win32.c
+++ b/qga/commands-win32.c
@@ -26,6 +26,8 @@
 #include <setupapi.h>
 #include <initguid.h>
 #endif
+#include <lm.h>
+
 #include "qga/guest-agent-core.h"
 #include "qga/vss-win32.h"
 #include "qga-qmp-commands.h"
@@ -53,8 +55,11 @@ typedef struct GuestFileHandle {
 
 static struct {
     QTAILQ_HEAD(, GuestFileHandle) filehandles;
-} guest_file_state;
+} guest_file_state = {
+    .filehandles = QTAILQ_HEAD_INITIALIZER(guest_file_state.filehandles),
+};
 
+#define FILE_GENERIC_APPEND (FILE_GENERIC_WRITE & ~FILE_WRITE_DATA)
 
 typedef struct OpenFlags {
     const char *forms;
@@ -62,20 +67,20 @@ typedef struct OpenFlags {
     DWORD creation_disposition;
 } OpenFlags;
 static OpenFlags guest_file_open_modes[] = {
-    {"r",   GENERIC_READ,               OPEN_EXISTING},
-    {"rb",  GENERIC_READ,               OPEN_EXISTING},
-    {"w",   GENERIC_WRITE,              CREATE_ALWAYS},
-    {"wb",  GENERIC_WRITE,              CREATE_ALWAYS},
-    {"a",   GENERIC_WRITE,              OPEN_ALWAYS  },
-    {"r+",  GENERIC_WRITE|GENERIC_READ, OPEN_EXISTING},
-    {"rb+", GENERIC_WRITE|GENERIC_READ, OPEN_EXISTING},
-    {"r+b", GENERIC_WRITE|GENERIC_READ, OPEN_EXISTING},
-    {"w+",  GENERIC_WRITE|GENERIC_READ, CREATE_ALWAYS},
-    {"wb+", GENERIC_WRITE|GENERIC_READ, CREATE_ALWAYS},
-    {"w+b", GENERIC_WRITE|GENERIC_READ, CREATE_ALWAYS},
-    {"a+",  GENERIC_WRITE|GENERIC_READ, OPEN_ALWAYS  },
-    {"ab+", GENERIC_WRITE|GENERIC_READ, OPEN_ALWAYS  },
-    {"a+b", GENERIC_WRITE|GENERIC_READ, OPEN_ALWAYS  }
+    {"r",   GENERIC_READ,                     OPEN_EXISTING},
+    {"rb",  GENERIC_READ,                     OPEN_EXISTING},
+    {"w",   GENERIC_WRITE,                    CREATE_ALWAYS},
+    {"wb",  GENERIC_WRITE,                    CREATE_ALWAYS},
+    {"a",   FILE_GENERIC_APPEND,              OPEN_ALWAYS  },
+    {"r+",  GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"rb+", GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"r+b", GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"w+",  GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"wb+", GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"w+b", GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"a+",  FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  },
+    {"ab+", FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  },
+    {"a+b", FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  }
 };
 
 static OpenFlags *find_open_flag(const char *mode_str)
@@ -104,7 +109,7 @@ static int64_t guest_file_handle_add(HANDLE fh, Error **errp)
     if (handle < 0) {
         return -1;
     }
-    gfh = g_malloc0(sizeof(GuestFileHandle));
+    gfh = g_new0(GuestFileHandle, 1);
     gfh->id = handle;
     gfh->fh = fh;
     QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
@@ -124,6 +129,28 @@ static GuestFileHandle *guest_file_handle_find(int64_t id, Error **errp)
     return NULL;
 }
 
+static void handle_set_nonblocking(HANDLE fh)
+{
+    DWORD file_type, pipe_state;
+    file_type = GetFileType(fh);
+    if (file_type != FILE_TYPE_PIPE) {
+        return;
+    }
+    /* If file_type == FILE_TYPE_PIPE, according to MSDN
+     * the specified file is socket or named pipe */
+    if (!GetNamedPipeHandleState(fh, &pipe_state, NULL,
+                                 NULL, NULL, NULL, 0)) {
+        return;
+    }
+    /* The fd is named pipe fd */
+    if (pipe_state & PIPE_NOWAIT) {
+        return;
+    }
+
+    pipe_state |= PIPE_NOWAIT;
+    SetNamedPipeHandleState(fh, &pipe_state, NULL, NULL);
+}
+
 int64_t qmp_guest_file_open(const char *path, bool has_mode,
                             const char *mode, Error **errp)
 {
@@ -154,9 +181,14 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode,
         return -1;
     }
 
+    /* set fd non-blocking to avoid common use cases (like reading from a
+     * named pipe) from hanging the agent
+     */
+    handle_set_nonblocking(fh);
+
     fd = guest_file_handle_add(fh, errp);
     if (fd < 0) {
-        CloseHandle(&fh);
+        CloseHandle(fh);
         error_setg(errp, "failed to add handle to qmp handle table");
         return -1;
     }
@@ -296,7 +328,7 @@ GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
         slog("guest-file-read failed, handle %" PRId64, handle);
     } else {
         buf[read_count] = 0;
-        read_data = g_malloc0(sizeof(GuestFileRead));
+        read_data = g_new0(GuestFileRead, 1);
         read_data->count = (size_t)read_count;
         read_data->eof = read_count == 0;
 
@@ -340,7 +372,7 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
         error_setg_win32(errp, GetLastError(), "failed to write to file");
         slog("guest-file-write-failed, handle: %" PRId64, handle);
     } else {
-        write_data = g_malloc0(sizeof(GuestFileWrite));
+        write_data = g_new0(GuestFileWrite, 1);
         write_data->count = (size_t) write_count;
     }
 
@@ -350,7 +382,7 @@ done:
 }
 
 GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
-                                   int64_t whence, Error **errp)
+                                   int64_t whence_code, Error **errp)
 {
     GuestFileHandle *gfh;
     GuestFileSeek *seek_data;
@@ -358,11 +390,29 @@ GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
     LARGE_INTEGER new_pos, off_pos;
     off_pos.QuadPart = offset;
     BOOL res;
+    int whence;
+
     gfh = guest_file_handle_find(handle, errp);
     if (!gfh) {
         return NULL;
     }
 
+    /* We stupidly exposed 'whence':'int' in our qapi */
+    switch (whence_code) {
+    case QGA_SEEK_SET:
+        whence = SEEK_SET;
+        break;
+    case QGA_SEEK_CUR:
+        whence = SEEK_CUR;
+        break;
+    case QGA_SEEK_END:
+        whence = SEEK_END;
+        break;
+    default:
+        error_setg(errp, "invalid whence code %"PRId64, whence_code);
+        return NULL;
+    }
+
     fh = gfh->fh;
     res = SetFilePointerEx(fh, off_pos, &new_pos, whence);
     if (!res) {
@@ -388,11 +438,6 @@ void qmp_guest_file_flush(int64_t handle, Error **errp)
     }
 }
 
-static void guest_file_init(void)
-{
-    QTAILQ_INIT(&guest_file_state.filehandles);
-}
-
 #ifdef CONFIG_QGA_NTDDSCSI
 
 static STORAGE_BUS_TYPE win2qemu[] = {
@@ -657,7 +702,7 @@ static GuestFilesystemInfo *build_guest_fsinfo(char *guid, Error **errp)
         fs->mountpoint = g_strndup(mnt_point, len);
     }
     fs->type = g_strdup(fs_name);
-    fs->disk = build_guest_disk_info(guid, errp);;
+    fs->disk = build_guest_disk_info(guid, errp);
 free:
     g_free(mnt_point);
     return fs;
@@ -863,7 +908,7 @@ static DWORD WINAPI do_suspend(LPVOID opaque)
 void qmp_guest_suspend_disk(Error **errp)
 {
     Error *local_err = NULL;
-    GuestSuspendMode *mode = g_malloc(sizeof(GuestSuspendMode));
+    GuestSuspendMode *mode = g_new(GuestSuspendMode, 1);
 
     *mode = GUEST_SUSPEND_MODE_DISK;
     check_suspend_mode(*mode, &local_err);
@@ -879,7 +924,7 @@ void qmp_guest_suspend_disk(Error **errp)
 void qmp_guest_suspend_ram(Error **errp)
 {
     Error *local_err = NULL;
-    GuestSuspendMode *mode = g_malloc(sizeof(GuestSuspendMode));
+    GuestSuspendMode *mode = g_new(GuestSuspendMode, 1);
 
     *mode = GUEST_SUSPEND_MODE_RAM;
     check_suspend_mode(*mode, &local_err);
@@ -1192,12 +1237,84 @@ int64_t qmp_guest_set_vcpus(GuestLogicalProcessorList *vcpus, Error **errp)
     return -1;
 }
 
+static gchar *
+get_net_error_message(gint error)
+{
+    HMODULE module = NULL;
+    gchar *retval = NULL;
+    wchar_t *msg = NULL;
+    int flags, nchars;
+
+    flags = FORMAT_MESSAGE_ALLOCATE_BUFFER
+        |FORMAT_MESSAGE_IGNORE_INSERTS
+        |FORMAT_MESSAGE_FROM_SYSTEM;
+
+    if (error >= NERR_BASE && error <= MAX_NERR) {
+        module = LoadLibraryExW(L"netmsg.dll", NULL, LOAD_LIBRARY_AS_DATAFILE);
+
+        if (module != NULL) {
+            flags |= FORMAT_MESSAGE_FROM_HMODULE;
+        }
+    }
+
+    FormatMessageW(flags, module, error, 0, (LPWSTR)&msg, 0, NULL);
+
+    if (msg != NULL) {
+        nchars = wcslen(msg);
+
+        if (nchars > 2 && msg[nchars-1] == '\n' && msg[nchars-2] == '\r') {
+            msg[nchars-2] = '\0';
+        }
+
+        retval = g_utf16_to_utf8(msg, -1, NULL, NULL, NULL);
+
+        LocalFree(msg);
+    }
+
+    if (module != NULL) {
+        FreeLibrary(module);
+    }
+
+    return retval;
+}
+
 void qmp_guest_set_user_password(const char *username,
                                  const char *password,
                                  bool crypted,
                                  Error **errp)
 {
-    error_setg(errp, QERR_UNSUPPORTED);
+    NET_API_STATUS nas;
+    char *rawpasswddata = NULL;
+    size_t rawpasswdlen;
+    wchar_t *user, *wpass;
+    USER_INFO_1003 pi1003 = { 0, };
+
+    if (crypted) {
+        error_setg(errp, QERR_UNSUPPORTED);
+        return;
+    }
+
+    rawpasswddata = (char *)g_base64_decode(password, &rawpasswdlen);
+    rawpasswddata = g_renew(char, rawpasswddata, rawpasswdlen + 1);
+    rawpasswddata[rawpasswdlen] = '\0';
+
+    user = g_utf8_to_utf16(username, -1, NULL, NULL, NULL);
+    wpass = g_utf8_to_utf16(rawpasswddata, -1, NULL, NULL, NULL);
+
+    pi1003.usri1003_password = wpass;
+    nas = NetUserSetInfo(NULL, user,
+                         1003, (LPBYTE)&pi1003,
+                         NULL);
+
+    if (nas != NERR_Success) {
+        gchar *msg = get_net_error_message(nas);
+        error_setg(errp, "failed to set password: %s", msg);
+        g_free(msg);
+    }
+
+    g_free(user);
+    g_free(wpass);
+    g_free(rawpasswddata);
 }
 
 GuestMemoryBlockList *qmp_guest_get_memory_blocks(Error **errp)
@@ -1225,7 +1342,6 @@ GList *ga_command_blacklist_init(GList *blacklist)
     const char *list_unsupported[] = {
         "guest-suspend-hybrid",
         "guest-get-vcpus", "guest-set-vcpus",
-        "guest-set-user-password",
         "guest-get-memory-blocks", "guest-set-memory-blocks",
         "guest-get-memory-block-size",
         "guest-fsfreeze-freeze-list",
@@ -1233,7 +1349,7 @@ GList *ga_command_blacklist_init(GList *blacklist)
     char **p = (char **)list_unsupported;
 
     while (*p) {
-        blacklist = g_list_append(blacklist, *p++);
+        blacklist = g_list_append(blacklist, g_strdup(*p++));
     }
 
     if (!vss_init(true)) {
@@ -1244,7 +1360,7 @@ GList *ga_command_blacklist_init(GList *blacklist)
         p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 
@@ -1257,5 +1373,4 @@ void ga_command_state_init(GAState *s, GACommandState *cs)
     if (!vss_initialized()) {
         ga_command_state_add(cs, NULL, guest_fsfreeze_cleanup);
     }
-    ga_command_state_add(cs, guest_file_init, NULL);
 }
diff --git a/qga/commands.c b/qga/commands.c
index 783496791..bb73e7dfb 100644
--- a/qga/commands.c
+++ b/qga/commands.c
@@ -15,6 +15,11 @@
 #include "qga-qmp-commands.h"
 #include "qapi/qmp/qerror.h"
 
+/* Maximum captured guest-exec out_data/err_data - 16MB */
+#define GUEST_EXEC_MAX_OUTPUT (16*1024*1024)
+/* Allocation and I/O buffer for reading guest-exec out_data/err_data - 4KB */
+#define GUEST_EXEC_IO_SIZE (4*1024)
+
 /* Note: in some situations, like with the fsfreeze, logging may be
  * temporarilly disabled. if it is necessary that a command be able
  * to log for accounting purposes, check ga_logging_enabled() beforehand,
@@ -51,12 +56,12 @@ static void qmp_command_info(QmpCommand *cmd, void *opaque)
     GuestAgentCommandInfo *cmd_info;
     GuestAgentCommandInfoList *cmd_info_list;
 
-    cmd_info = g_malloc0(sizeof(GuestAgentCommandInfo));
+    cmd_info = g_new0(GuestAgentCommandInfo, 1);
     cmd_info->name = g_strdup(qmp_command_name(cmd));
     cmd_info->enabled = qmp_command_is_enabled(cmd);
     cmd_info->success_response = qmp_has_success_response(cmd);
 
-    cmd_info_list = g_malloc0(sizeof(GuestAgentCommandInfoList));
+    cmd_info_list = g_new0(GuestAgentCommandInfoList, 1);
     cmd_info_list->value = cmd_info;
     cmd_info_list->next = info->supported_commands;
     info->supported_commands = cmd_info_list;
@@ -64,9 +69,395 @@ static void qmp_command_info(QmpCommand *cmd, void *opaque)
 
 struct GuestAgentInfo *qmp_guest_info(Error **errp)
 {
-    GuestAgentInfo *info = g_malloc0(sizeof(GuestAgentInfo));
+    GuestAgentInfo *info = g_new0(GuestAgentInfo, 1);
 
     info->version = g_strdup(QEMU_VERSION);
     qmp_for_each_command(qmp_command_info, info);
     return info;
 }
+
+struct GuestExecIOData {
+    guchar *data;
+    gsize size;
+    gsize length;
+    gint closed;
+    bool truncated;
+    const char *name;
+};
+typedef struct GuestExecIOData GuestExecIOData;
+
+struct GuestExecInfo {
+    GPid pid;
+    int64_t pid_numeric;
+    gint status;
+    bool has_output;
+    gint finished;
+    GuestExecIOData in;
+    GuestExecIOData out;
+    GuestExecIOData err;
+    QTAILQ_ENTRY(GuestExecInfo) next;
+};
+typedef struct GuestExecInfo GuestExecInfo;
+
+static struct {
+    QTAILQ_HEAD(, GuestExecInfo) processes;
+} guest_exec_state = {
+    .processes = QTAILQ_HEAD_INITIALIZER(guest_exec_state.processes),
+};
+
+static int64_t gpid_to_int64(GPid pid)
+{
+#ifdef G_OS_WIN32
+    return GetProcessId(pid);
+#else
+    return (int64_t)pid;
+#endif
+}
+
+static GuestExecInfo *guest_exec_info_add(GPid pid)
+{
+    GuestExecInfo *gei;
+
+    gei = g_new0(GuestExecInfo, 1);
+    gei->pid = pid;
+    gei->pid_numeric = gpid_to_int64(pid);
+    QTAILQ_INSERT_TAIL(&guest_exec_state.processes, gei, next);
+
+    return gei;
+}
+
+static GuestExecInfo *guest_exec_info_find(int64_t pid_numeric)
+{
+    GuestExecInfo *gei;
+
+    QTAILQ_FOREACH(gei, &guest_exec_state.processes, next) {
+        if (gei->pid_numeric == pid_numeric) {
+            return gei;
+        }
+    }
+
+    return NULL;
+}
+
+GuestExecStatus *qmp_guest_exec_status(int64_t pid, Error **err)
+{
+    GuestExecInfo *gei;
+    GuestExecStatus *ges;
+
+    slog("guest-exec-status called, pid: %u", (uint32_t)pid);
+
+    gei = guest_exec_info_find(pid);
+    if (gei == NULL) {
+        error_setg(err, QERR_INVALID_PARAMETER, "pid");
+        return NULL;
+    }
+
+    ges = g_new0(GuestExecStatus, 1);
+
+    bool finished = g_atomic_int_get(&gei->finished);
+
+    /* need to wait till output channels are closed
+     * to be sure we captured all output at this point */
+    if (gei->has_output) {
+        finished = finished && g_atomic_int_get(&gei->out.closed);
+        finished = finished && g_atomic_int_get(&gei->err.closed);
+    }
+
+    ges->exited = finished;
+    if (finished) {
+        /* Glib has no portable way to parse exit status.
+         * On UNIX, we can get either exit code from normal termination
+         * or signal number.
+         * On Windows, it is either the same exit code or the exception
+         * value for an unhandled exception that caused the process
+         * to terminate.
+         * See MSDN for GetExitCodeProcess() and ntstatus.h for possible
+         * well-known codes, e.g. C0000005 ACCESS_DENIED - analog of SIGSEGV
+         * References:
+         *   https://msdn.microsoft.com/en-us/library/windows/desktop/ms683189(v=vs.85).aspx
+         *   https://msdn.microsoft.com/en-us/library/aa260331(v=vs.60).aspx
+         */
+#ifdef G_OS_WIN32
+        /* Additionally WIN32 does not provide any additional information
+         * on whetherthe child exited or terminated via signal.
+         * We use this simple range check to distingish application exit code
+         * (usually value less then 256) and unhandled exception code with
+         * ntstatus (always value greater then 0xC0000005). */
+        if ((uint32_t)gei->status < 0xC0000000U) {
+            ges->has_exitcode = true;
+            ges->exitcode = gei->status;
+        } else {
+            ges->has_signal = true;
+            ges->signal = gei->status;
+        }
+#else
+        if (WIFEXITED(gei->status)) {
+            ges->has_exitcode = true;
+            ges->exitcode = WEXITSTATUS(gei->status);
+        } else if (WIFSIGNALED(gei->status)) {
+            ges->has_signal = true;
+            ges->signal = WTERMSIG(gei->status);
+        }
+#endif
+        if (gei->out.length > 0) {
+            ges->has_out_data = true;
+            ges->out_data = g_base64_encode(gei->out.data, gei->out.length);
+            g_free(gei->out.data);
+            ges->has_out_truncated = gei->out.truncated;
+        }
+
+        if (gei->err.length > 0) {
+            ges->has_err_data = true;
+            ges->err_data = g_base64_encode(gei->err.data, gei->err.length);
+            g_free(gei->err.data);
+            ges->has_err_truncated = gei->err.truncated;
+        }
+
+        QTAILQ_REMOVE(&guest_exec_state.processes, gei, next);
+        g_free(gei);
+    }
+
+    return ges;
+}
+
+/* Get environment variables or arguments array for execve(). */
+static char **guest_exec_get_args(const strList *entry, bool log)
+{
+    const strList *it;
+    int count = 1, i = 0;  /* reserve for NULL terminator */
+    char **args;
+    char *str; /* for logging array of arguments */
+    size_t str_size = 1;
+
+    for (it = entry; it != NULL; it = it->next) {
+        count++;
+        str_size += 1 + strlen(it->value);
+    }
+
+    str = g_malloc(str_size);
+    *str = 0;
+    args = g_malloc(count * sizeof(char *));
+    for (it = entry; it != NULL; it = it->next) {
+        args[i++] = it->value;
+        pstrcat(str, str_size, it->value);
+        if (it->next) {
+            pstrcat(str, str_size, " ");
+        }
+    }
+    args[i] = NULL;
+
+    if (log) {
+        slog("guest-exec called: \"%s\"", str);
+    }
+    g_free(str);
+
+    return args;
+}
+
+static void guest_exec_child_watch(GPid pid, gint status, gpointer data)
+{
+    GuestExecInfo *gei = (GuestExecInfo *)data;
+
+    g_debug("guest_exec_child_watch called, pid: %d, status: %u",
+            (int32_t)gpid_to_int64(pid), (uint32_t)status);
+
+    gei->status = status;
+    gei->finished = true;
+
+    g_spawn_close_pid(pid);
+}
+
+/** Reset ignored signals back to default. */
+static void guest_exec_task_setup(gpointer data)
+{
+#if !defined(G_OS_WIN32)
+    struct sigaction sigact;
+
+    memset(&sigact, 0, sizeof(struct sigaction));
+    sigact.sa_handler = SIG_DFL;
+
+    if (sigaction(SIGPIPE, &sigact, NULL) != 0) {
+        slog("sigaction() failed to reset child process's SIGPIPE: %s",
+             strerror(errno));
+    }
+#endif
+}
+
+static gboolean guest_exec_input_watch(GIOChannel *ch,
+        GIOCondition cond, gpointer p_)
+{
+    GuestExecIOData *p = (GuestExecIOData *)p_;
+    gsize bytes_written = 0;
+    GIOStatus status;
+    GError *gerr = NULL;
+
+    /* nothing left to write */
+    if (p->size == p->length) {
+        goto done;
+    }
+
+    status = g_io_channel_write_chars(ch, (gchar *)p->data + p->length,
+            p->size - p->length, &bytes_written, &gerr);
+
+    /* can be not 0 even if not G_IO_STATUS_NORMAL */
+    if (bytes_written != 0) {
+        p->length += bytes_written;
+    }
+
+    /* continue write, our callback will be called again */
+    if (status == G_IO_STATUS_NORMAL || status == G_IO_STATUS_AGAIN) {
+        return true;
+    }
+
+    if (gerr) {
+        g_warning("qga: i/o error writing to input_data channel: %s",
+                gerr->message);
+        g_error_free(gerr);
+    }
+
+done:
+    g_io_channel_shutdown(ch, true, NULL);
+    g_io_channel_unref(ch);
+    g_atomic_int_set(&p->closed, 1);
+    g_free(p->data);
+
+    return false;
+}
+
+static gboolean guest_exec_output_watch(GIOChannel *ch,
+        GIOCondition cond, gpointer p_)
+{
+    GuestExecIOData *p = (GuestExecIOData *)p_;
+    gsize bytes_read;
+    GIOStatus gstatus;
+
+    if (cond == G_IO_HUP || cond == G_IO_ERR) {
+        goto close;
+    }
+
+    if (p->size == p->length) {
+        gpointer t = NULL;
+        if (!p->truncated && p->size < GUEST_EXEC_MAX_OUTPUT) {
+            t = g_try_realloc(p->data, p->size + GUEST_EXEC_IO_SIZE);
+        }
+        if (t == NULL) {
+            /* ignore truncated output */
+            gchar buf[GUEST_EXEC_IO_SIZE];
+
+            p->truncated = true;
+            gstatus = g_io_channel_read_chars(ch, buf, sizeof(buf),
+                                              &bytes_read, NULL);
+            if (gstatus == G_IO_STATUS_EOF || gstatus == G_IO_STATUS_ERROR) {
+                goto close;
+            }
+
+            return true;
+        }
+        p->size += GUEST_EXEC_IO_SIZE;
+        p->data = t;
+    }
+
+    /* Calling read API once.
+     * On next available data our callback will be called again */
+    gstatus = g_io_channel_read_chars(ch, (gchar *)p->data + p->length,
+            p->size - p->length, &bytes_read, NULL);
+    if (gstatus == G_IO_STATUS_EOF || gstatus == G_IO_STATUS_ERROR) {
+        goto close;
+    }
+
+    p->length += bytes_read;
+
+    return true;
+
+close:
+    g_io_channel_unref(ch);
+    g_atomic_int_set(&p->closed, 1);
+    return false;
+}
+
+GuestExec *qmp_guest_exec(const char *path,
+                       bool has_arg, strList *arg,
+                       bool has_env, strList *env,
+                       bool has_input_data, const char *input_data,
+                       bool has_capture_output, bool capture_output,
+                       Error **err)
+{
+    GPid pid;
+    GuestExec *ge = NULL;
+    GuestExecInfo *gei;
+    char **argv, **envp;
+    strList arglist;
+    gboolean ret;
+    GError *gerr = NULL;
+    gint in_fd, out_fd, err_fd;
+    GIOChannel *in_ch, *out_ch, *err_ch;
+    GSpawnFlags flags;
+    bool has_output = (has_capture_output && capture_output);
+
+    arglist.value = (char *)path;
+    arglist.next = has_arg ? arg : NULL;
+
+    argv = guest_exec_get_args(&arglist, true);
+    envp = has_env ? guest_exec_get_args(env, false) : NULL;
+
+    flags = G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD;
+#if GLIB_CHECK_VERSION(2, 33, 2)
+    flags |= G_SPAWN_SEARCH_PATH_FROM_ENVP;
+#endif
+    if (!has_output) {
+        flags |= G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL;
+    }
+
+    ret = g_spawn_async_with_pipes(NULL, argv, envp, flags,
+            guest_exec_task_setup, NULL, &pid, has_input_data ? &in_fd : NULL,
+            has_output ? &out_fd : NULL, has_output ? &err_fd : NULL, &gerr);
+    if (!ret) {
+        error_setg(err, QERR_QGA_COMMAND_FAILED, gerr->message);
+        g_error_free(gerr);
+        goto done;
+    }
+
+    ge = g_new0(GuestExec, 1);
+    ge->pid = gpid_to_int64(pid);
+
+    gei = guest_exec_info_add(pid);
+    gei->has_output = has_output;
+    g_child_watch_add(pid, guest_exec_child_watch, gei);
+
+    if (has_input_data) {
+        gei->in.data = g_base64_decode(input_data, &gei->in.size);
+#ifdef G_OS_WIN32
+        in_ch = g_io_channel_win32_new_fd(in_fd);
+#else
+        in_ch = g_io_channel_unix_new(in_fd);
+#endif
+        g_io_channel_set_encoding(in_ch, NULL, NULL);
+        g_io_channel_set_buffered(in_ch, false);
+        g_io_channel_set_flags(in_ch, G_IO_FLAG_NONBLOCK, NULL);
+        g_io_add_watch(in_ch, G_IO_OUT, guest_exec_input_watch, &gei->in);
+    }
+
+    if (has_output) {
+#ifdef G_OS_WIN32
+        out_ch = g_io_channel_win32_new_fd(out_fd);
+        err_ch = g_io_channel_win32_new_fd(err_fd);
+#else
+        out_ch = g_io_channel_unix_new(out_fd);
+        err_ch = g_io_channel_unix_new(err_fd);
+#endif
+        g_io_channel_set_encoding(out_ch, NULL, NULL);
+        g_io_channel_set_encoding(err_ch, NULL, NULL);
+        g_io_channel_set_buffered(out_ch, false);
+        g_io_channel_set_buffered(err_ch, false);
+        g_io_add_watch(out_ch, G_IO_IN | G_IO_HUP,
+                guest_exec_output_watch, &gei->out);
+        g_io_add_watch(err_ch, G_IO_IN | G_IO_HUP,
+                guest_exec_output_watch, &gei->err);
+    }
+
+done:
+    g_free(argv);
+    g_free(envp);
+
+    return ge;
+}
diff --git a/qga/guest-agent-command-state.c b/qga/guest-agent-command-state.c
index 969da2328..128c549ed 100644
--- a/qga/guest-agent-command-state.c
+++ b/qga/guest-agent-command-state.c
@@ -27,7 +27,7 @@ void ga_command_state_add(GACommandState *cs,
                           void (*init)(void),
                           void (*cleanup)(void))
 {
-    GACommandGroup *cg = g_malloc0(sizeof(GACommandGroup));
+    GACommandGroup *cg = g_new0(GACommandGroup, 1);
     cg->init = init;
     cg->cleanup = cleanup;
     cs->groups = g_slist_append(cs->groups, cg);
@@ -67,7 +67,7 @@ void ga_command_state_cleanup_all(GACommandState *cs)
 
 GACommandState *ga_command_state_new(void)
 {
-    GACommandState *cs = g_malloc0(sizeof(GACommandState));
+    GACommandState *cs = g_new0(GACommandState, 1);
     cs->groups = NULL;
     return cs;
 }
diff --git a/qga/guest-agent-core.h b/qga/guest-agent-core.h
index e92c6abaf..238dc6b08 100644
--- a/qga/guest-agent-core.h
+++ b/qga/guest-agent-core.h
@@ -15,6 +15,13 @@
 
 #define QGA_READ_COUNT_DEFAULT 4096
 
+/* Mapping of whence codes used by guest-file-seek. */
+enum {
+    QGA_SEEK_SET = 0,
+    QGA_SEEK_CUR = 1,
+    QGA_SEEK_END = 2,
+};
+
 typedef struct GAState GAState;
 typedef struct GACommandState GACommandState;
 extern GAState *ga_state;
diff --git a/qga/installer/qemu-ga.wxs b/qga/installer/qemu-ga.wxs
index 2c43f1b5a..947387572 100644
--- a/qga/installer/qemu-ga.wxs
+++ b/qga/installer/qemu-ga.wxs
@@ -42,7 +42,7 @@
   <Product
     Name="QEMU guest agent"
     Id="*"
-    UpgradeCode="{EB6B8302-C06E-4bec-ADAC-932C68A3A98D}"
+    UpgradeCode="{EB6B8302-C06E-4BEC-ADAC-932C68A3A98D}"
     Manufacturer="$(env.QEMU_GA_MANUFACTURER)"
     Version="$(env.QEMU_GA_VERSION)"
     Language="1033">
@@ -58,29 +58,15 @@
       />
     <Media Id="1" Cabinet="qemu_ga.$(env.QEMU_GA_VERSION).cab" EmbedCab="yes" />
     <Property Id="WHSLogo">1</Property>
-    <Property Id="PREVIOUSVERSIONSINSTALLED" />
-    <Upgrade Id="{EB6B8302-C06E-4bec-ADAC-932C68A3A98D}">
-      <UpgradeVersion
-        Minimum="1.0.0.0" Maximum="$(env.QEMU_GA_VERSION)"
-        Property="PREVIOUSVERSIONSINSTALLED"
-        IncludeMinimum="yes" IncludeMaximum="no" />
-    </Upgrade>
+    <MajorUpgrade
+      DowngradeErrorMessage="Error: A newer version of QEMU guest agent is already installed."
+      />
 
     <Directory Id="TARGETDIR" Name="SourceDir">
       <Directory Id="$(var.GaProgramFilesFolder)" Name="QEMU Guest Agent">
         <Directory Id="qemu_ga_directory" Name="Qemu-ga">
-          <Component Id="qemu_ga" Guid="{908B7199-DE2A-4dc6-A8D0-27A5AE444FEA}">
-            <File Id="qemu_ga.exe" Name="qemu-ga.exe" Source="../../qemu-ga.exe" KeyPath="yes" DiskId="1"/>
-            <?ifdef var.InstallVss ?>
-            <File Id="qga_vss.dll" Name="qga-vss.dll" Source="../vss-win32/qga-vss.dll" KeyPath="no" DiskId="1"/>
-            <File Id="qga_vss.tlb" Name="qga-vss.tlb" Source="../vss-win32/qga-vss.tlb" KeyPath="no" DiskId="1"/>
-            <?endif?>
-            <File Id="iconv.dll" Name="iconv.dll" Source="$(var.Mingw_bin)/iconv.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libgcc_arch_lib" Name="$(var.ArchLib)" Source="$(var.Mingw_bin)/$(var.ArchLib)" KeyPath="no" DiskId="1"/>
-            <File Id="libglib_2.0_0.dll" Name="libglib-2.0-0.dll" Source="$(var.Mingw_bin)/libglib-2.0-0.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libintl_8.dll" Name="libintl-8.dll" Source="$(var.Mingw_bin)/libintl-8.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libssp_0.dll" Name="libssp-0.dll" Source="$(var.Mingw_bin)/libssp-0.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libwinpthread_1.dll" Name="libwinpthread-1.dll" Source="$(var.Mingw_bin)/libwinpthread-1.dll" KeyPath="no" DiskId="1"/>
+          <Component Id="qemu_ga" Guid="{908B7199-DE2A-4DC6-A8D0-27A5AE444FEA}">
+            <File Id="qemu_ga.exe" Name="qemu-ga.exe" Source="$(env.BUILD_DIR)/qemu-ga.exe" KeyPath="yes" DiskId="1"/>
             <ServiceInstall
               Id="ServiceInstaller"
               Type="ownProcess"
@@ -97,8 +83,49 @@
             </ServiceInstall>
             <ServiceControl Id="StartService" Start="install" Stop="both" Remove="uninstall" Name="QEMU-GA" Wait="no" />
           </Component>
-
-          <Component Id="registry_entries" Guid="d075d109-51ca-11e3-9f8b-000c29858960">
+          <?ifdef var.InstallVss?>
+          <Component Id="qga_vss_dll" Guid="{CB19C453-FABB-4BB1-ABAB-6B74F687BFBB}">
+            <File Id="qga_vss.dll" Name="qga-vss.dll" Source="$(env.BUILD_DIR)/qga/vss-win32/qga-vss.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="qga_vss_tlb" Guid="{D8D584B1-59C2-4FB7-A91F-636FF7BFA66E}">
+            <File Id="qga_vss.tlb" Name="qga-vss.tlb" Source="$(env.BUILD_DIR)/qga/vss-win32/qga-vss.tlb" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <?if $(var.Arch) = "32"?>
+          <Component Id="gspawn-helper-console" Guid="{446185B3-87BE-43D2-96B8-0FEFD9E8696D}">
+            <File Id="gspawn-win32-helper-console.exe" Name="gspawn-win32-helper-console.exe" Source="$(var.Mingw_bin)/gspawn-win32-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="gspawn-helper" Guid="{CD67A5A3-2DB1-4DA1-A67A-8D71E797B466}">
+            <File Id="gspawn-win32-helper.exe" Name="gspawn-win32-helper.exe" Source="$(var.Mingw_bin)/gspawn-win32-helper.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <?if $(var.Arch) = "64"?>
+          <Component Id="gspawn-helper-console" Guid="{9E615A9F-349A-4992-A5C2-C10BAD173660}">
+            <File Id="gspawn-win64-helper-console.exe" Name="gspawn-win64-helper-console.exe" Source="$(var.Mingw_bin)/gspawn-win64-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="gspawn-helper" Guid="{D201AD22-1846-4E4F-B6E1-C7A908ED2457}">
+            <File Id="gspawn-win64-helper.exe" Name="gspawn-win64-helper.exe" Source="$(var.Mingw_bin)/gspawn-win64-helper.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <Component Id="iconv" Guid="{35EE3558-D34B-4F0A-B8BD-430FF0775246}">
+            <File Id="iconv.dll" Name="iconv.dll" Source="$(var.Mingw_bin)/iconv.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libgcc_arch_lib" Guid="{ADD4D07D-4515-4AB6-AF3E-C904961B4BB0}">
+            <File Id="libgcc_arch_lib" Name="$(var.ArchLib)" Source="$(var.Mingw_bin)/$(var.ArchLib)" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libglib" Guid="{D31BFD83-2773-4B65-B45A-E0D2ADA58679}">
+            <File Id="libglib_2.0_0.dll" Name="libglib-2.0-0.dll" Source="$(var.Mingw_bin)/libglib-2.0-0.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libintl" Guid="{A641BC2D-A907-4A94-9149-F30ED430878F}">
+            <File Id="libintl_8.dll" Name="libintl-8.dll" Source="$(var.Mingw_bin)/libintl-8.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libssp" Guid="{7880087B-02B4-4EF6-A5D3-D18F8E3D90E1}">
+            <File Id="libssp_0.dll" Name="libssp-0.dll" Source="$(var.Mingw_bin)/libssp-0.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libwinpthread" Guid="{6C117C78-0F47-4B07-8F34-6BEE11643829}">
+            <File Id="libwinpthread_1.dll" Name="libwinpthread-1.dll" Source="$(var.Mingw_bin)/libwinpthread-1.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="registry_entries" Guid="{D075D109-51CA-11E3-9F8B-000C29858960}">
             <RegistryKey Root="HKLM"
                          Key="Software\$(env.QEMU_GA_MANUFACTURER)\$(env.QEMU_GA_DISTRO)\Tools\QemuGA">
               <RegistryValue Type="string" Name="ProductID" Value="fb0a0d66-c7fb-4e2e-a16b-c4a3bfe8d13b" />
@@ -110,10 +137,11 @@
     </Directory>
 
     <Property Id="cmd" Value="cmd.exe"/>
+    <Property Id="REINSTALLMODE" Value="amus"/>
 
-    <?ifdef var.InstallVss ?>
+    <?ifdef var.InstallVss?>
     <CustomAction Id="RegisterCom"
-             ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-install'
+              ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-install'
               Execute="deferred"
               Property="cmd"
               Impersonate="no"
@@ -126,19 +154,31 @@
               Property="cmd"
               Impersonate="no"
               Return="check"
-              ></CustomAction>
+              >
+    </CustomAction>
     <?endif?>
 
     <Feature Id="QEMUFeature" Title="QEMU Guest Agent" Level="1">
       <ComponentRef Id="qemu_ga" />
+      <?ifdef var.InstallVss?>
+      <ComponentRef Id="qga_vss_dll" />
+      <ComponentRef Id="qga_vss_tlb" />
+      <?endif?>
+      <ComponentRef Id="gspawn-helper-console" />
+      <ComponentRef Id="gspawn-helper" />
+      <ComponentRef Id="iconv" />
+      <ComponentRef Id="libgcc_arch_lib" />
+      <ComponentRef Id="libglib" />
+      <ComponentRef Id="libintl" />
+      <ComponentRef Id="libssp" />
+      <ComponentRef Id="libwinpthread" />
       <ComponentRef Id="registry_entries" />
     </Feature>
 
     <InstallExecuteSequence>
-      <RemoveExistingProducts Before="InstallInitialize" />
-      <?ifdef var.InstallVss ?>
-      <Custom Action="RegisterCom" After="InstallServices">NOT Installed</Custom>
+      <?ifdef var.InstallVss?>
       <Custom Action="UnRegisterCom" After="StopServices">Installed</Custom>
+      <Custom Action="RegisterCom" After="InstallServices">NOT REMOVE</Custom>
       <?endif?>
     </InstallExecuteSequence>
   </Product>
diff --git a/qga/main.c b/qga/main.c
index 791982ef0..f83a97d24 100644
--- a/qga/main.c
+++ b/qga/main.c
@@ -56,6 +56,7 @@
 #define QGA_FSFREEZE_HOOK_DEFAULT CONFIG_QEMU_CONFDIR "/fsfreeze-hook"
 #endif
 #define QGA_SENTINEL_BYTE 0xFF
+#define QGA_CONF_DEFAULT CONFIG_QEMU_CONFDIR G_DIR_SEPARATOR_S "qemu-ga.conf"
 
 static struct {
     const char *state_dir;
@@ -82,7 +83,7 @@ struct GAState {
     bool delimit_response;
     bool frozen;
     GList *blacklist;
-    const char *state_filepath_isfrozen;
+    char *state_filepath_isfrozen;
     struct {
         const char *log_filepath;
         const char *pid_filepath;
@@ -90,7 +91,7 @@ struct GAState {
 #ifdef CONFIG_FSFREEZE
     const char *fsfreeze_hook;
 #endif
-    const gchar *pstate_filepath;
+    gchar *pstate_filepath;
     GAPersistentState pstate;
 };
 
@@ -160,6 +161,12 @@ static gboolean register_signal_handlers(void)
         g_error("error configuring signal handler: %s", strerror(errno));
     }
 
+    sigact.sa_handler = SIG_IGN;
+    if (sigaction(SIGPIPE, &sigact, NULL) != 0) {
+        g_error("error configuring SIGPIPE signal handler: %s",
+                strerror(errno));
+    }
+
     return true;
 }
 
@@ -215,6 +222,8 @@ static void usage(const char *cmd)
 #endif
 "  -b, --blacklist   comma-separated list of RPCs to disable (no spaces, \"?\"\n"
 "                    to list available RPCs)\n"
+"  -D, --dump-conf   dump a qemu-ga config file based on current config\n"
+"                    options / command-line parameters to stdout\n"
 "  -h, --help        display this help and exit\n"
 "\n"
 "Report bugs to <mdroth@linux.vnet.ibm.com>\n"
@@ -561,10 +570,9 @@ static void process_command(GAState *s, QDict *req)
 }
 
 /* handle requests/control events coming in over the channel */
-static void process_event(JSONMessageParser *parser, QList *tokens)
+static void process_event(JSONMessageParser *parser, GQueue *tokens)
 {
     GAState *s = container_of(parser, GAState, parser);
-    QObject *obj;
     QDict *qdict;
     Error *err = NULL;
     int ret;
@@ -572,9 +580,9 @@ static void process_event(JSONMessageParser *parser, QList *tokens)
     g_assert(s && parser);
 
     g_debug("process_event: called");
-    obj = json_parser_parse_err(tokens, NULL, &err);
-    if (err || !obj || qobject_type(obj) != QTYPE_QDICT) {
-        qobject_decref(obj);
+    qdict = qobject_to_qdict(json_parser_parse_err(tokens, NULL, &err));
+    if (err || !qdict) {
+        QDECREF(qdict);
         qdict = qdict_new();
         if (!err) {
             g_warning("failed to parse event: unknown error");
@@ -584,12 +592,8 @@ static void process_event(JSONMessageParser *parser, QList *tokens)
         }
         qdict_put_obj(qdict, "error", qmp_build_error_object(err));
         error_free(err);
-    } else {
-        qdict = qobject_to_qdict(obj);
     }
 
-    g_assert(qdict);
-
     /* handle host->guest commands */
     if (qdict_haskey(qdict, "execute")) {
         process_command(s, qdict);
@@ -658,23 +662,6 @@ static gboolean channel_init(GAState *s, const gchar *method, const gchar *path)
 {
     GAChannelMethod channel_method;
 
-    if (method == NULL) {
-        method = "virtio-serial";
-    }
-
-    if (path == NULL) {
-        if (strcmp(method, "virtio-serial") == 0 ) {
-            /* try the default path for the virtio-serial port */
-            path = QGA_VIRTIO_PATH_DEFAULT;
-        } else if (strcmp(method, "isa-serial") == 0){
-            /* try the default path for the serial port - COM1 */
-            path = QGA_SERIAL_PATH_DEFAULT;
-        } else {
-            g_critical("must specify a path for this channel");
-            return false;
-        }
-    }
-
     if (strcmp(method, "virtio-serial") == 0) {
         s->virtio = true; /* virtio requires special handling in some cases */
         channel_method = GA_CHANNEL_VIRTIO_SERIAL;
@@ -921,22 +908,165 @@ static void ga_print_cmd(QmpCommand *cmd, void *opaque)
     printf("%s\n", qmp_command_name(cmd));
 }
 
-int main(int argc, char **argv)
+static GList *split_list(const gchar *str, const gchar *delim)
 {
-    const char *sopt = "hVvdm:p:l:f:F::b:s:t:";
-    const char *method = NULL, *path = NULL;
-    const char *log_filepath = NULL;
-    const char *pid_filepath;
+    GList *list = NULL;
+    int i;
+    gchar **strv;
+
+    strv = g_strsplit(str, delim, -1);
+    for (i = 0; strv[i]; i++) {
+        list = g_list_prepend(list, strv[i]);
+    }
+    g_free(strv);
+
+    return list;
+}
+
+typedef struct GAConfig {
+    char *channel_path;
+    char *method;
+    char *log_filepath;
+    char *pid_filepath;
 #ifdef CONFIG_FSFREEZE
-    const char *fsfreeze_hook = NULL;
+    char *fsfreeze_hook;
 #endif
-    const char *state_dir;
+    char *state_dir;
 #ifdef _WIN32
-    const char *service = NULL;
+    const char *service;
+#endif
+    gchar *bliststr; /* blacklist may point to this string */
+    GList *blacklist;
+    int daemonize;
+    GLogLevelFlags log_level;
+    int dumpconf;
+} GAConfig;
+
+static void config_load(GAConfig *config)
+{
+    GError *gerr = NULL;
+    GKeyFile *keyfile;
+    const char *conf = g_getenv("QGA_CONF") ?: QGA_CONF_DEFAULT;
+
+    /* read system config */
+    keyfile = g_key_file_new();
+    if (!g_key_file_load_from_file(keyfile, conf, 0, &gerr)) {
+        goto end;
+    }
+    if (g_key_file_has_key(keyfile, "general", "daemon", NULL)) {
+        config->daemonize =
+            g_key_file_get_boolean(keyfile, "general", "daemon", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "method", NULL)) {
+        config->method =
+            g_key_file_get_string(keyfile, "general", "method", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "path", NULL)) {
+        config->channel_path =
+            g_key_file_get_string(keyfile, "general", "path", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "logfile", NULL)) {
+        config->log_filepath =
+            g_key_file_get_string(keyfile, "general", "logfile", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "pidfile", NULL)) {
+        config->pid_filepath =
+            g_key_file_get_string(keyfile, "general", "pidfile", &gerr);
+    }
+#ifdef CONFIG_FSFREEZE
+    if (g_key_file_has_key(keyfile, "general", "fsfreeze-hook", NULL)) {
+        config->fsfreeze_hook =
+            g_key_file_get_string(keyfile,
+                                  "general", "fsfreeze-hook", &gerr);
+    }
 #endif
+    if (g_key_file_has_key(keyfile, "general", "statedir", NULL)) {
+        config->state_dir =
+            g_key_file_get_string(keyfile, "general", "statedir", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "verbose", NULL) &&
+        g_key_file_get_boolean(keyfile, "general", "verbose", &gerr)) {
+        /* enable all log levels */
+        config->log_level = G_LOG_LEVEL_MASK;
+    }
+    if (g_key_file_has_key(keyfile, "general", "blacklist", NULL)) {
+        config->bliststr =
+            g_key_file_get_string(keyfile, "general", "blacklist", &gerr);
+        config->blacklist = g_list_concat(config->blacklist,
+                                          split_list(config->bliststr, ","));
+    }
+
+end:
+    g_key_file_free(keyfile);
+    if (gerr &&
+        !(gerr->domain == G_FILE_ERROR && gerr->code == G_FILE_ERROR_NOENT)) {
+        g_critical("error loading configuration from path: %s, %s",
+                   QGA_CONF_DEFAULT, gerr->message);
+        exit(EXIT_FAILURE);
+    }
+    g_clear_error(&gerr);
+}
+
+static gchar *list_join(GList *list, const gchar separator)
+{
+    GString *str = g_string_new("");
+
+    while (list) {
+        str = g_string_append(str, (gchar *)list->data);
+        list = g_list_next(list);
+        if (list) {
+            str = g_string_append_c(str, separator);
+        }
+    }
+
+    return g_string_free(str, FALSE);
+}
+
+static void config_dump(GAConfig *config)
+{
+    GError *error = NULL;
+    GKeyFile *keyfile;
+    gchar *tmp;
+
+    keyfile = g_key_file_new();
+    g_assert(keyfile);
+
+    g_key_file_set_boolean(keyfile, "general", "daemon", config->daemonize);
+    g_key_file_set_string(keyfile, "general", "method", config->method);
+    g_key_file_set_string(keyfile, "general", "path", config->channel_path);
+    if (config->log_filepath) {
+        g_key_file_set_string(keyfile, "general", "logfile",
+                              config->log_filepath);
+    }
+    g_key_file_set_string(keyfile, "general", "pidfile", config->pid_filepath);
+#ifdef CONFIG_FSFREEZE
+    if (config->fsfreeze_hook) {
+        g_key_file_set_string(keyfile, "general", "fsfreeze-hook",
+                              config->fsfreeze_hook);
+    }
+#endif
+    g_key_file_set_string(keyfile, "general", "statedir", config->state_dir);
+    g_key_file_set_boolean(keyfile, "general", "verbose",
+                           config->log_level == G_LOG_LEVEL_MASK);
+    tmp = list_join(config->blacklist, ',');
+    g_key_file_set_string(keyfile, "general", "blacklist", tmp);
+    g_free(tmp);
+
+    tmp = g_key_file_to_data(keyfile, NULL, &error);
+    printf("%s", tmp);
+
+    g_free(tmp);
+    g_key_file_free(keyfile);
+}
+
+static void config_parse(GAConfig *config, int argc, char **argv)
+{
+    const char *sopt = "hVvdm:p:l:f:F::b:s:t:D";
+    int opt_ind = 0, ch;
     const struct option lopt[] = {
         { "help", 0, NULL, 'h' },
         { "version", 0, NULL, 'V' },
+        { "dump-conf", 0, NULL, 'D' },
         { "logfile", 1, NULL, 'l' },
         { "pidfile", 1, NULL, 'f' },
 #ifdef CONFIG_FSFREEZE
@@ -953,141 +1083,113 @@ int main(int argc, char **argv)
         { "statedir", 1, NULL, 't' },
         { NULL, 0, NULL, 0 }
     };
-    int opt_ind = 0, ch, daemonize = 0, i, j, len;
-    GLogLevelFlags log_level = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL;
-    GList *blacklist = NULL;
-    GAState *s;
-
-    module_call_init(MODULE_INIT_QAPI);
-
-    init_dfl_pathnames();
-    pid_filepath = dfl_pathnames.pidfile;
-    state_dir = dfl_pathnames.state_dir;
 
     while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
         switch (ch) {
         case 'm':
-            method = optarg;
+            g_free(config->method);
+            config->method = g_strdup(optarg);
             break;
         case 'p':
-            path = optarg;
+            g_free(config->channel_path);
+            config->channel_path = g_strdup(optarg);
             break;
         case 'l':
-            log_filepath = optarg;
+            g_free(config->log_filepath);
+            config->log_filepath = g_strdup(optarg);
             break;
         case 'f':
-            pid_filepath = optarg;
+            g_free(config->pid_filepath);
+            config->pid_filepath = g_strdup(optarg);
             break;
 #ifdef CONFIG_FSFREEZE
         case 'F':
-            fsfreeze_hook = optarg ? optarg : QGA_FSFREEZE_HOOK_DEFAULT;
+            g_free(config->fsfreeze_hook);
+            config->fsfreeze_hook = g_strdup(optarg ?: QGA_FSFREEZE_HOOK_DEFAULT);
             break;
 #endif
         case 't':
-             state_dir = optarg;
-             break;
+            g_free(config->state_dir);
+            config->state_dir = g_strdup(optarg);
+            break;
         case 'v':
             /* enable all log levels */
-            log_level = G_LOG_LEVEL_MASK;
+            config->log_level = G_LOG_LEVEL_MASK;
             break;
         case 'V':
             printf("QEMU Guest Agent %s\n", QEMU_VERSION);
-            return 0;
+            exit(EXIT_SUCCESS);
         case 'd':
-            daemonize = 1;
+            config->daemonize = 1;
+            break;
+        case 'D':
+            config->dumpconf = 1;
             break;
         case 'b': {
             if (is_help_option(optarg)) {
                 qmp_for_each_command(ga_print_cmd, NULL);
-                return 0;
-            }
-            for (j = 0, i = 0, len = strlen(optarg); i < len; i++) {
-                if (optarg[i] == ',') {
-                    optarg[i] = 0;
-                    blacklist = g_list_append(blacklist, &optarg[j]);
-                    j = i + 1;
-                }
-            }
-            if (j < i) {
-                blacklist = g_list_append(blacklist, &optarg[j]);
+                exit(EXIT_SUCCESS);
             }
+            config->blacklist = g_list_concat(config->blacklist,
+                                             split_list(optarg, ","));
             break;
         }
 #ifdef _WIN32
         case 's':
-            service = optarg;
-            if (strcmp(service, "install") == 0) {
-                const char *fixed_state_dir;
-
-                /* If the user passed the "-t" option, we save that state dir
-                 * in the service. Otherwise we let the service fetch the state
-                 * dir from the environment when it starts.
-                 */
-                fixed_state_dir = (state_dir == dfl_pathnames.state_dir) ?
-                                  NULL :
-                                  state_dir;
+            config->service = optarg;
+            if (strcmp(config->service, "install") == 0) {
                 if (ga_install_vss_provider()) {
-                    return EXIT_FAILURE;
+                    exit(EXIT_FAILURE);
                 }
-                if (ga_install_service(path, log_filepath, fixed_state_dir)) {
-                    return EXIT_FAILURE;
+                if (ga_install_service(config->channel_path,
+                                       config->log_filepath, config->state_dir)) {
+                    exit(EXIT_FAILURE);
                 }
-                return 0;
-            } else if (strcmp(service, "uninstall") == 0) {
+                exit(EXIT_SUCCESS);
+            } else if (strcmp(config->service, "uninstall") == 0) {
                 ga_uninstall_vss_provider();
-                return ga_uninstall_service();
-            } else if (strcmp(service, "vss-install") == 0) {
+                exit(ga_uninstall_service());
+            } else if (strcmp(config->service, "vss-install") == 0) {
                 if (ga_install_vss_provider()) {
-                    return EXIT_FAILURE;
+                    exit(EXIT_FAILURE);
                 }
-                return EXIT_SUCCESS;
-            } else if (strcmp(service, "vss-uninstall") == 0) {
+                exit(EXIT_SUCCESS);
+            } else if (strcmp(config->service, "vss-uninstall") == 0) {
                 ga_uninstall_vss_provider();
-                return EXIT_SUCCESS;
+                exit(EXIT_SUCCESS);
             } else {
                 printf("Unknown service command.\n");
-                return EXIT_FAILURE;
+                exit(EXIT_FAILURE);
             }
             break;
 #endif
         case 'h':
             usage(argv[0]);
-            return 0;
+            exit(EXIT_SUCCESS);
         case '?':
             g_print("Unknown option, try '%s --help' for more information.\n",
                     argv[0]);
-            return EXIT_FAILURE;
+            exit(EXIT_FAILURE);
         }
     }
+}
 
-#ifdef _WIN32
-    /* On win32 the state directory is application specific (be it the default
-     * or a user override). We got past the command line parsing; let's create
-     * the directory (with any intermediate directories). If we run into an
-     * error later on, we won't try to clean up the directory, it is considered
-     * persistent.
-     */
-    if (g_mkdir_with_parents(state_dir, S_IRWXU) == -1) {
-        g_critical("unable to create (an ancestor of) the state directory"
-                   " '%s': %s", state_dir, strerror(errno));
-        return EXIT_FAILURE;
-    }
-#endif
-
-    s = g_malloc0(sizeof(GAState));
-    s->log_level = log_level;
-    s->log_file = stderr;
+static void config_free(GAConfig *config)
+{
+    g_free(config->method);
+    g_free(config->log_filepath);
+    g_free(config->pid_filepath);
+    g_free(config->state_dir);
+    g_free(config->channel_path);
+    g_free(config->bliststr);
 #ifdef CONFIG_FSFREEZE
-    s->fsfreeze_hook = fsfreeze_hook;
+    g_free(config->fsfreeze_hook);
 #endif
-    g_log_set_default_handler(ga_log, s);
-    g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
-    ga_enable_logging(s);
-    s->state_filepath_isfrozen = g_strdup_printf("%s/qga.state.isfrozen",
-                                                 state_dir);
-    s->pstate_filepath = g_strdup_printf("%s/qga.state", state_dir);
-    s->frozen = false;
+    g_free(config);
+}
 
+static bool check_is_frozen(GAState *s)
+{
 #ifndef _WIN32
     /* check if a previous instance of qemu-ga exited with filesystems' state
      * marked as frozen. this could be a stale value (a non-qemu-ga process
@@ -1113,32 +1215,56 @@ int main(int argc, char **argv)
                   " guest-fsfreeze-thaw is issued, or filesystems are"
                   " manually unfrozen and the file %s is removed",
                   s->state_filepath_isfrozen);
-        s->frozen = true;
+        return true;
+    }
+#endif
+    return false;
+}
+
+static int run_agent(GAState *s, GAConfig *config)
+{
+    ga_state = s;
+
+    g_log_set_default_handler(ga_log, s);
+    g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
+    ga_enable_logging(s);
+
+#ifdef _WIN32
+    /* On win32 the state directory is application specific (be it the default
+     * or a user override). We got past the command line parsing; let's create
+     * the directory (with any intermediate directories). If we run into an
+     * error later on, we won't try to clean up the directory, it is considered
+     * persistent.
+     */
+    if (g_mkdir_with_parents(config->state_dir, S_IRWXU) == -1) {
+        g_critical("unable to create (an ancestor of) the state directory"
+                   " '%s': %s", config->state_dir, strerror(errno));
+        return EXIT_FAILURE;
     }
 #endif
 
     if (ga_is_frozen(s)) {
-        if (daemonize) {
+        if (config->daemonize) {
             /* delay opening/locking of pidfile till filesystems are unfrozen */
-            s->deferred_options.pid_filepath = pid_filepath;
+            s->deferred_options.pid_filepath = config->pid_filepath;
             become_daemon(NULL);
         }
-        if (log_filepath) {
+        if (config->log_filepath) {
             /* delay opening the log file till filesystems are unfrozen */
-            s->deferred_options.log_filepath = log_filepath;
+            s->deferred_options.log_filepath = config->log_filepath;
         }
         ga_disable_logging(s);
         qmp_for_each_command(ga_disable_non_whitelisted, NULL);
     } else {
-        if (daemonize) {
-            become_daemon(pid_filepath);
+        if (config->daemonize) {
+            become_daemon(config->pid_filepath);
         }
-        if (log_filepath) {
-            FILE *log_file = ga_open_logfile(log_filepath);
+        if (config->log_filepath) {
+            FILE *log_file = ga_open_logfile(config->log_filepath);
             if (!log_file) {
                 g_critical("unable to open specified log file: %s",
                            strerror(errno));
-                goto out_bad;
+                return EXIT_FAILURE;
             }
             s->log_file = log_file;
         }
@@ -1149,17 +1275,18 @@ int main(int argc, char **argv)
                                s->pstate_filepath,
                                ga_is_frozen(s))) {
         g_critical("failed to load persistent state");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 
-    blacklist = ga_command_blacklist_init(blacklist);
-    if (blacklist) {
-        s->blacklist = blacklist;
+    config->blacklist = ga_command_blacklist_init(config->blacklist);
+    if (config->blacklist) {
+        GList *l = config->blacklist;
+        s->blacklist = config->blacklist;
         do {
-            g_debug("disabling command: %s", (char *)blacklist->data);
-            qmp_disable_command(blacklist->data);
-            blacklist = g_list_next(blacklist);
-        } while (blacklist);
+            g_debug("disabling command: %s", (char *)l->data);
+            qmp_disable_command(l->data);
+            l = g_list_next(l);
+        } while (l);
     }
     s->command_state = ga_command_state_new();
     ga_command_state_init(s, s->command_state);
@@ -1169,19 +1296,19 @@ int main(int argc, char **argv)
 #ifndef _WIN32
     if (!register_signal_handlers()) {
         g_critical("failed to register signal handlers");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 #endif
 
     s->main_loop = g_main_loop_new(NULL, false);
-    if (!channel_init(ga_state, method, path)) {
+    if (!channel_init(ga_state, config->method, config->channel_path)) {
         g_critical("failed to initialize guest agent channel");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 #ifndef _WIN32
     g_main_loop_run(ga_state->main_loop);
 #else
-    if (daemonize) {
+    if (config->daemonize) {
         SERVICE_TABLE_ENTRY service_table[] = {
             { (char *)QGA_SERVICE_NAME, service_main }, { NULL, NULL } };
         StartServiceCtrlDispatcher(service_table);
@@ -1190,17 +1317,87 @@ int main(int argc, char **argv)
     }
 #endif
 
-    ga_command_state_cleanup_all(ga_state->command_state);
-    ga_channel_free(ga_state->channel);
+    return EXIT_SUCCESS;
+}
+
+static void free_blacklist_entry(gpointer entry, gpointer unused)
+{
+    g_free(entry);
+}
+
+int main(int argc, char **argv)
+{
+    int ret = EXIT_SUCCESS;
+    GAState *s = g_new0(GAState, 1);
+    GAConfig *config = g_new0(GAConfig, 1);
+
+    config->log_level = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL;
+
+    module_call_init(MODULE_INIT_QAPI);
+
+    init_dfl_pathnames();
+    config_load(config);
+    config_parse(config, argc, argv);
 
-    if (daemonize) {
-        unlink(pid_filepath);
+    if (config->pid_filepath == NULL) {
+        config->pid_filepath = g_strdup(dfl_pathnames.pidfile);
+    }
+
+    if (config->state_dir == NULL) {
+        config->state_dir = g_strdup(dfl_pathnames.state_dir);
     }
-    return 0;
 
-out_bad:
-    if (daemonize) {
-        unlink(pid_filepath);
+    if (config->method == NULL) {
+        config->method = g_strdup("virtio-serial");
     }
-    return EXIT_FAILURE;
+
+    if (config->channel_path == NULL) {
+        if (strcmp(config->method, "virtio-serial") == 0) {
+            /* try the default path for the virtio-serial port */
+            config->channel_path = g_strdup(QGA_VIRTIO_PATH_DEFAULT);
+        } else if (strcmp(config->method, "isa-serial") == 0) {
+            /* try the default path for the serial port - COM1 */
+            config->channel_path = g_strdup(QGA_SERIAL_PATH_DEFAULT);
+        } else {
+            g_critical("must specify a path for this channel");
+            ret = EXIT_FAILURE;
+            goto end;
+        }
+    }
+
+    s->log_level = config->log_level;
+    s->log_file = stderr;
+#ifdef CONFIG_FSFREEZE
+    s->fsfreeze_hook = config->fsfreeze_hook;
+#endif
+    s->pstate_filepath = g_strdup_printf("%s/qga.state", config->state_dir);
+    s->state_filepath_isfrozen = g_strdup_printf("%s/qga.state.isfrozen",
+                                                 config->state_dir);
+    s->frozen = check_is_frozen(s);
+
+    if (config->dumpconf) {
+        config_dump(config);
+        goto end;
+    }
+
+    ret = run_agent(s, config);
+
+end:
+    if (s->command_state) {
+        ga_command_state_cleanup_all(s->command_state);
+    }
+    if (s->channel) {
+        ga_channel_free(s->channel);
+    }
+    g_list_foreach(config->blacklist, free_blacklist_entry, NULL);
+    g_free(s->pstate_filepath);
+    g_free(s->state_filepath_isfrozen);
+
+    if (config->daemonize) {
+        unlink(config->pid_filepath);
+    }
+
+    config_free(config);
+
+    return ret;
 }
diff --git a/qga/qapi-schema.json b/qga/qapi-schema.json
index 18e3cc37d..01c9ee48d 100644
--- a/qga/qapi-schema.json
+++ b/qga/qapi-schema.json
@@ -12,6 +12,7 @@
 ##
 
 ##
+# @guest-sync-delimited:
 #
 # Echo back a unique integer value, and prepend to response a
 # leading sentinel byte (0xFF) the client can check scan for.
@@ -41,7 +42,7 @@
 # Returns: The unique integer id passed in by the client
 #
 # Since: 1.1
-# ##
+##
 { 'command': 'guest-sync-delimited',
   'data':    { 'id': 'int' },
   'returns': 'int' }
@@ -317,13 +318,13 @@
 #
 # Seek to a position in the file, as with fseek(), and return the
 # current file position afterward. Also encapsulates ftell()'s
-# functionality, just Set offset=0, whence=SEEK_CUR.
+# functionality, with offset=0 and whence=1.
 #
 # @handle: filehandle returned by guest-file-open
 #
 # @offset: bytes to skip over in the file stream
 #
-# @whence: SEEK_SET, SEEK_CUR, or SEEK_END, as with fseek()
+# @whence: 0 for SEEK_SET, 1 for SEEK_CUR, or 2 for SEEK_END
 #
 # Returns: @GuestFileSeek on success.
 #
@@ -793,7 +794,7 @@
 # scheme. Refer to the documentation of the guest operating system
 # in question to determine what is supported.
 #
-# Note all guest operating systems will support use of the
+# Not all guest operating systems will support use of the
 # @crypted flag, as they may require the clear-text password
 #
 # The @password parameter must always be base64 encoded before
@@ -929,3 +930,70 @@
 ##
 { 'command': 'guest-get-memory-block-info',
   'returns': 'GuestMemoryBlockInfo' }
+
+# @GuestExecStatus:
+#
+# @exited: true if process has already terminated.
+# @exitcode: #optional process exit code if it was normally terminated.
+# @signal: #optional signal number (linux) or unhandled exception code
+#       (windows) if the process was abnormally terminated.
+# @out-data: #optional base64-encoded stdout of the process
+# @err-data: #optional base64-encoded stderr of the process
+#       Note: @out-data and @err-data are present only
+#       if 'capture-output' was specified for 'guest-exec'
+# @out-truncated: #optional true if stdout was not fully captured
+#       due to size limitation.
+# @err-truncated: #optional true if stderr was not fully captured
+#       due to size limitation.
+#
+# Since: 2.5
+##
+{ 'struct': 'GuestExecStatus',
+  'data': { 'exited': 'bool', '*exitcode': 'int', '*signal': 'int',
+            '*out-data': 'str', '*err-data': 'str',
+            '*out-truncated': 'bool', '*err-truncated': 'bool' }}
+##
+# @guest-exec-status
+#
+# Check status of process associated with PID retrieved via guest-exec.
+# Reap the process and associated metadata if it has exited.
+#
+# @pid: pid returned from guest-exec
+#
+# Returns: GuestExecStatus on success.
+#
+# Since 2.5
+##
+{ 'command': 'guest-exec-status',
+  'data':    { 'pid': 'int' },
+  'returns': 'GuestExecStatus' }
+
+##
+# @GuestExec:
+# @pid: pid of child process in guest OS
+#
+#Since: 2.5
+##
+{ 'struct': 'GuestExec',
+  'data': { 'pid': 'int'} }
+
+##
+# @guest-exec:
+#
+# Execute a command in the guest
+#
+# @path: path or executable name to execute
+# @arg: #optional argument list to pass to executable
+# @env: #optional environment variables to pass to executable
+# @input-data: #optional data to be passed to process stdin (base64 encoded)
+# @capture-output: #optional bool flag to enable capture of
+#                  stdout/stderr of running process. defaults to false.
+#
+# Returns: PID on success.
+#
+# Since: 2.5
+##
+{ 'command': 'guest-exec',
+  'data':    { 'path': 'str', '*arg': ['str'], '*env': ['str'],
+               '*input-data': 'str', '*capture-output': 'bool' },
+  'returns': 'GuestExec' }
diff --git a/qga/vss-win32.c b/qga/vss-win32.c
index 0e4095736..2142b4964 100644
--- a/qga/vss-win32.c
+++ b/qga/vss-win32.c
@@ -150,11 +150,11 @@ void qga_vss_fsfreeze(int *nr_volume, Error **errp, bool freeze)
     const char *func_name = freeze ? "requester_freeze" : "requester_thaw";
     QGAVSSRequesterFunc func;
     ErrorSet errset = {
-        .error_set = (ErrorSetFunc)error_set_win32,
-        .errp = (void **)errp,
-        .err_class = ERROR_CLASS_GENERIC_ERROR
+        .error_setg_win32 = error_setg_win32_internal,
+        .errp = errp,
     };
 
+    g_assert(errp);             /* requester.cpp requires it */
     func = (QGAVSSRequesterFunc)GetProcAddress(provider_lib, func_name);
     if (!func) {
         error_setg_win32(errp, GetLastError(), "failed to load %s from %s",
diff --git a/qga/vss-win32/requester.cpp b/qga/vss-win32/requester.cpp
index 922e74ddf..9b3e31097 100644
--- a/qga/vss-win32/requester.cpp
+++ b/qga/vss-win32/requester.cpp
@@ -23,10 +23,12 @@
 /* Call QueryStatus every 10 ms while waiting for frozen event */
 #define VSS_TIMEOUT_EVENT_MSEC 10
 
-#define err_set(e, err, fmt, ...) \
-    ((e)->error_set((e)->errp, err, (e)->err_class, fmt, ## __VA_ARGS__))
+#define err_set(e, err, fmt, ...)                                       \
+    ((e)->error_setg_win32((e)->errp, __FILE__, __LINE__, __func__,     \
+                           err, fmt, ## __VA_ARGS__))
+/* Bad idea, works only when (e)->errp != NULL: */
 #define err_is_set(e) ((e)->errp && *(e)->errp)
-
+/* To lift this restriction, error_propagate(), like we do in QEMU code */
 
 /* Handle to VSSAPI.DLL */
 static HMODULE hLib;
diff --git a/qga/vss-win32/requester.h b/qga/vss-win32/requester.h
index 374f9b8d1..c3093cf4b 100644
--- a/qga/vss-win32/requester.h
+++ b/qga/vss-win32/requester.h
@@ -20,13 +20,16 @@
 extern "C" {
 #endif
 
+struct Error;
+
 /* Callback to set Error; used to avoid linking glib to the DLL */
-typedef void (*ErrorSetFunc)(void **errp, int win32_err, int err_class,
-                             const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+typedef void (*ErrorSetFunc)(struct Error **errp,
+                             const char *src, int line, const char *func,
+                             int win32_err, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
 typedef struct ErrorSet {
-    ErrorSetFunc error_set;
-    void **errp;
-    int err_class;
+    ErrorSetFunc error_setg_win32;
+    struct Error **errp;        /* restriction: must not be null */
 } ErrorSet;
 
 STDAPI requester_init(void);
diff --git a/qmp-commands.hx b/qmp-commands.hx
index ba630b1e7..20a92f998 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -63,7 +63,7 @@ EQMP
     {
         .name       = "quit",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_quit,
+        .mhandler.cmd_new = qmp_marshal_quit,
     },
 
 SQMP
@@ -84,7 +84,7 @@ EQMP
     {
         .name       = "eject",
         .args_type  = "force:-f,device:B",
-        .mhandler.cmd_new = qmp_marshal_input_eject,
+        .mhandler.cmd_new = qmp_marshal_eject,
     },
 
 SQMP
@@ -110,7 +110,7 @@ EQMP
     {
         .name       = "change",
         .args_type  = "device:B,target:F,arg:s?",
-        .mhandler.cmd_new = qmp_marshal_input_change,
+        .mhandler.cmd_new = qmp_marshal_change,
     },
 
 SQMP
@@ -146,7 +146,7 @@ EQMP
     {
         .name       = "screendump",
         .args_type  = "filename:F",
-        .mhandler.cmd_new = qmp_marshal_input_screendump,
+        .mhandler.cmd_new = qmp_marshal_screendump,
     },
 
 SQMP
@@ -169,7 +169,7 @@ EQMP
     {
         .name       = "stop",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_stop,
+        .mhandler.cmd_new = qmp_marshal_stop,
     },
 
 SQMP
@@ -190,7 +190,7 @@ EQMP
     {
         .name       = "cont",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_cont,
+        .mhandler.cmd_new = qmp_marshal_cont,
     },
 
 SQMP
@@ -211,7 +211,7 @@ EQMP
     {
         .name       = "system_wakeup",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_wakeup,
+        .mhandler.cmd_new = qmp_marshal_system_wakeup,
     },
 
 SQMP
@@ -232,7 +232,7 @@ EQMP
     {
         .name       = "system_reset",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_reset,
+        .mhandler.cmd_new = qmp_marshal_system_reset,
     },
 
 SQMP
@@ -253,7 +253,7 @@ EQMP
     {
         .name       = "system_powerdown",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_powerdown,
+        .mhandler.cmd_new = qmp_marshal_system_powerdown,
     },
 
 SQMP
@@ -310,7 +310,7 @@ EQMP
     {
         .name       = "device_del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_device_del,
+        .mhandler.cmd_new = qmp_marshal_device_del,
     },
 
 SQMP
@@ -321,19 +321,24 @@ Remove a device.
 
 Arguments:
 
-- "id": the device's ID (json-string)
+- "id": the device's ID or QOM path (json-string)
 
 Example:
 
 -> { "execute": "device_del", "arguments": { "id": "net1" } }
 <- { "return": {} }
 
+Example:
+
+-> { "execute": "device_del", "arguments": { "id": "/machine/peripheral-anon/device[0]" } }
+<- { "return": {} }
+
 EQMP
 
     {
         .name       = "send-key",
         .args_type  = "keys:q,hold-time:i?",
-        .mhandler.cmd_new = qmp_marshal_input_send_key,
+        .mhandler.cmd_new = qmp_marshal_send_key,
     },
 
 SQMP
@@ -364,7 +369,7 @@ EQMP
     {
         .name       = "cpu",
         .args_type  = "index:i",
-        .mhandler.cmd_new = qmp_marshal_input_cpu,
+        .mhandler.cmd_new = qmp_marshal_cpu,
     },
 
 SQMP
@@ -389,7 +394,7 @@ EQMP
     {
         .name       = "cpu-add",
         .args_type  = "id:i",
-        .mhandler.cmd_new = qmp_marshal_input_cpu_add,
+        .mhandler.cmd_new = qmp_marshal_cpu_add,
     },
 
 SQMP
@@ -412,7 +417,7 @@ EQMP
     {
         .name       = "memsave",
         .args_type  = "val:l,size:i,filename:s,cpu:i?",
-        .mhandler.cmd_new = qmp_marshal_input_memsave,
+        .mhandler.cmd_new = qmp_marshal_memsave,
     },
 
 SQMP
@@ -441,7 +446,7 @@ EQMP
     {
         .name       = "pmemsave",
         .args_type  = "val:l,size:i,filename:s",
-        .mhandler.cmd_new = qmp_marshal_input_pmemsave,
+        .mhandler.cmd_new = qmp_marshal_pmemsave,
     },
 
 SQMP
@@ -469,7 +474,7 @@ EQMP
     {
         .name       = "inject-nmi",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_inject_nmi,
+        .mhandler.cmd_new = qmp_marshal_inject_nmi,
     },
 
 SQMP
@@ -492,7 +497,7 @@ EQMP
     {
         .name       = "ringbuf-write",
         .args_type  = "device:s,data:s,format:s?",
-        .mhandler.cmd_new = qmp_marshal_input_ringbuf_write,
+        .mhandler.cmd_new = qmp_marshal_ringbuf_write,
     },
 
 SQMP
@@ -523,7 +528,7 @@ EQMP
     {
         .name       = "ringbuf-read",
         .args_type  = "device:s,size:i,format:s?",
-        .mhandler.cmd_new = qmp_marshal_input_ringbuf_read,
+        .mhandler.cmd_new = qmp_marshal_ringbuf_read,
     },
 
 SQMP
@@ -559,7 +564,7 @@ EQMP
     {
         .name       = "xen-save-devices-state",
         .args_type  = "filename:F",
-    .mhandler.cmd_new = qmp_marshal_input_xen_save_devices_state,
+    .mhandler.cmd_new = qmp_marshal_xen_save_devices_state,
     },
 
 SQMP
@@ -586,7 +591,7 @@ EQMP
     {
         .name       = "xen-set-global-dirty-log",
         .args_type  = "enable:b",
-        .mhandler.cmd_new = qmp_marshal_input_xen_set_global_dirty_log,
+        .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
     },
 
 SQMP
@@ -610,7 +615,7 @@ EQMP
     {
         .name       = "migrate",
         .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
-        .mhandler.cmd_new = qmp_marshal_input_migrate,
+        .mhandler.cmd_new = qmp_marshal_migrate,
     },
 
 SQMP
@@ -643,7 +648,7 @@ EQMP
     {
         .name       = "migrate_cancel",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_cancel,
+        .mhandler.cmd_new = qmp_marshal_migrate_cancel,
     },
 
 SQMP
@@ -664,7 +669,7 @@ EQMP
     {
         .name       = "migrate-incoming",
         .args_type  = "uri:s",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_incoming,
+        .mhandler.cmd_new = qmp_marshal_migrate_incoming,
     },
 
 SQMP
@@ -692,7 +697,7 @@ EQMP
     {
         .name       = "migrate-set-cache-size",
         .args_type  = "value:o",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_cache_size,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_cache_size,
     },
 
 SQMP
@@ -713,9 +718,28 @@ Example:
 
 EQMP
     {
+        .name       = "migrate-start-postcopy",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_migrate_start_postcopy,
+    },
+
+SQMP
+migrate-start-postcopy
+----------------------
+
+Switch an in-progress migration to postcopy mode. Ignored after the end of
+migration (or once already in postcopy).
+
+Example:
+-> { "execute": "migrate-start-postcopy" }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-migrate-cache-size",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_cache_size,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_cache_size,
     },
 
 SQMP
@@ -737,7 +761,7 @@ EQMP
     {
         .name       = "migrate_set_speed",
         .args_type  = "value:o",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_speed,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_speed,
     },
 
 SQMP
@@ -760,7 +784,7 @@ EQMP
     {
         .name       = "migrate_set_downtime",
         .args_type  = "value:T",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_downtime,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_downtime,
     },
 
 SQMP
@@ -785,7 +809,7 @@ EQMP
         .args_type  = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
         .params     = "protocol hostname port tls-port cert-subject",
         .help       = "set migration information for remote display",
-        .mhandler.cmd_new = qmp_marshal_input_client_migrate_info,
+        .mhandler.cmd_new = qmp_marshal_client_migrate_info,
     },
 
 SQMP
@@ -819,7 +843,7 @@ EQMP
         .args_type  = "paging:b,protocol:s,begin:i?,end:i?,format:s?",
         .params     = "-p protocol [begin] [length] [format]",
         .help       = "dump guest memory to file",
-        .mhandler.cmd_new = qmp_marshal_input_dump_guest_memory,
+        .mhandler.cmd_new = qmp_marshal_dump_guest_memory,
     },
 
 SQMP
@@ -855,7 +879,7 @@ EQMP
     {
         .name       = "query-dump-guest-memory-capability",
         .args_type  = "",
-    .mhandler.cmd_new = qmp_marshal_input_query_dump_guest_memory_capability,
+    .mhandler.cmd_new = qmp_marshal_query_dump_guest_memory_capability,
     },
 
 SQMP
@@ -872,6 +896,31 @@ Example:
 
 EQMP
 
+#if defined TARGET_S390X
+    {
+        .name       = "dump-skeys",
+        .args_type  = "filename:F",
+        .mhandler.cmd_new = qmp_marshal_dump_skeys,
+    },
+#endif
+
+SQMP
+dump-skeys
+----------
+
+Save guest storage keys to file.
+
+Arguments:
+
+- "filename": file path (json-string)
+
+Example:
+
+-> { "execute": "dump-skeys", "arguments": { "filename": "/tmp/skeys" } }
+<- { "return": {} }
+
+EQMP
+
     {
         .name       = "netdev_add",
         .args_type  = "netdev:O",
@@ -892,7 +941,9 @@ Arguments:
 
 Example:
 
--> { "execute": "netdev_add", "arguments": { "type": "user", "id": "netdev1" } }
+-> { "execute": "netdev_add",
+     "arguments": { "type": "user", "id": "netdev1",
+                    "dnssearch": "example.org" } }
 <- { "return": {} }
 
 Note: The supported device options are the same ones supported by the '-netdev'
@@ -904,7 +955,7 @@ EQMP
     {
         .name       = "netdev_del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_netdev_del,
+        .mhandler.cmd_new = qmp_marshal_netdev_del,
     },
 
 SQMP
@@ -928,7 +979,7 @@ EQMP
     {
         .name       = "object-add",
         .args_type  = "qom-type:s,id:s,props:q?",
-        .mhandler.cmd_new = qmp_object_add,
+        .mhandler.cmd_new = qmp_marshal_object_add,
     },
 
 SQMP
@@ -954,7 +1005,7 @@ EQMP
     {
         .name       = "object-del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_object_del,
+        .mhandler.cmd_new = qmp_marshal_object_del,
     },
 
 SQMP
@@ -979,7 +1030,7 @@ EQMP
     {
         .name       = "block_resize",
         .args_type  = "device:s?,node-name:s?,size:o",
-        .mhandler.cmd_new = qmp_marshal_input_block_resize,
+        .mhandler.cmd_new = qmp_marshal_block_resize,
     },
 
 SQMP
@@ -1004,7 +1055,7 @@ EQMP
     {
         .name       = "block-stream",
         .args_type  = "device:B,base:s?,speed:o?,backing-file:s?,on-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_block_stream,
+        .mhandler.cmd_new = qmp_marshal_block_stream,
     },
 
 SQMP
@@ -1047,7 +1098,7 @@ EQMP
     {
         .name       = "block-commit",
         .args_type  = "device:B,base:s?,top:s?,backing-file:s?,speed:o?",
-        .mhandler.cmd_new = qmp_marshal_input_block_commit,
+        .mhandler.cmd_new = qmp_marshal_block_commit,
     },
 
 SQMP
@@ -1111,7 +1162,7 @@ EQMP
         .name       = "drive-backup",
         .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
                       "bitmap:s?,on-source-error:s?,on-target-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_drive_backup,
+        .mhandler.cmd_new = qmp_marshal_drive_backup,
     },
 
 SQMP
@@ -1165,7 +1216,7 @@ EQMP
         .name       = "blockdev-backup",
         .args_type  = "sync:s,device:B,target:B,speed:i?,"
                       "on-source-error:s?,on-target-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_backup,
+        .mhandler.cmd_new = qmp_marshal_blockdev_backup,
     },
 
 SQMP
@@ -1205,44 +1256,55 @@ EQMP
     {
         .name       = "block-job-set-speed",
         .args_type  = "device:B,speed:o",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_set_speed,
+        .mhandler.cmd_new = qmp_marshal_block_job_set_speed,
     },
 
     {
         .name       = "block-job-cancel",
         .args_type  = "device:B,force:b?",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_cancel,
+        .mhandler.cmd_new = qmp_marshal_block_job_cancel,
     },
     {
         .name       = "block-job-pause",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_pause,
+        .mhandler.cmd_new = qmp_marshal_block_job_pause,
     },
     {
         .name       = "block-job-resume",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_resume,
+        .mhandler.cmd_new = qmp_marshal_block_job_resume,
     },
     {
         .name       = "block-job-complete",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_complete,
+        .mhandler.cmd_new = qmp_marshal_block_job_complete,
     },
     {
         .name       = "transaction",
-        .args_type  = "actions:q",
-        .mhandler.cmd_new = qmp_marshal_input_transaction,
+        .args_type  = "actions:q,properties:q?",
+        .mhandler.cmd_new = qmp_marshal_transaction,
     },
 
 SQMP
 transaction
 -----------
 
-Atomically operate on one or more block devices.  The only supported operations
-for now are drive-backup, internal and external snapshotting.  A list of
-dictionaries is accepted, that contains the actions to be performed.
-If there is any failure performing any of the operations, all operations
-for the group are abandoned.
+Atomically operate on one or more block devices.  Operations that are
+currently supported:
+
+    - drive-backup
+    - blockdev-backup
+    - blockdev-snapshot-sync
+    - blockdev-snapshot-internal-sync
+    - abort
+    - block-dirty-bitmap-add
+    - block-dirty-bitmap-clear
+
+Refer to the qemu/qapi-schema.json file for minimum required QEMU
+versions for these operations.  A list of dictionaries is accepted,
+that contains the actions to be performed.  If there is any failure
+performing any of the operations, all operations for the group are
+abandoned.
 
 For external snapshots, the dictionary contains the device, the file to use for
 the new snapshot, and the format.  The default format, if not specified, is
@@ -1269,8 +1331,12 @@ it later with qemu-img or other command.
 Arguments:
 
 actions array:
-    - "type": the operation to perform.  The only supported
-      value is "blockdev-snapshot-sync". (json-string)
+    - "type": the operation to perform (json-string).  Possible
+              values: "drive-backup", "blockdev-backup",
+                      "blockdev-snapshot-sync",
+                      "blockdev-snapshot-internal-sync",
+                      "abort", "block-dirty-bitmap-add",
+                      "block-dirty-bitmap-clear"
     - "data": a dictionary.  The contents depend on the value
       of "type".  When "type" is "blockdev-snapshot-sync":
       - "device": device name to snapshot (json-string)
@@ -1310,7 +1376,7 @@ EQMP
     {
         .name       = "block-dirty-bitmap-add",
         .args_type  = "node:B,name:s,granularity:i?",
-        .mhandler.cmd_new = qmp_marshal_input_block_dirty_bitmap_add,
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_add,
     },
 
 SQMP
@@ -1338,7 +1404,7 @@ EQMP
     {
         .name       = "block-dirty-bitmap-remove",
         .args_type  = "node:B,name:s",
-        .mhandler.cmd_new = qmp_marshal_input_block_dirty_bitmap_remove,
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_remove,
     },
 
 SQMP
@@ -1366,7 +1432,7 @@ EQMP
     {
         .name       = "block-dirty-bitmap-clear",
         .args_type  = "node:B,name:s",
-        .mhandler.cmd_new = qmp_marshal_input_block_dirty_bitmap_clear,
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_clear,
     },
 
 SQMP
@@ -1395,7 +1461,7 @@ EQMP
     {
         .name       = "blockdev-snapshot-sync",
         .args_type  = "device:s?,node-name:s?,snapshot-file:s,snapshot-node-name:s?,format:s?,mode:s?",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_snapshot_sync,
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot_sync,
     },
 
 SQMP
@@ -1429,9 +1495,47 @@ Example:
 EQMP
 
     {
+        .name       = "blockdev-snapshot",
+        .args_type  = "node:s,overlay:s",
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot,
+    },
+
+SQMP
+blockdev-snapshot
+-----------------
+Since 2.5
+
+Create a snapshot, by installing 'node' as the backing image of
+'overlay'. Additionally, if 'node' is associated with a block
+device, the block device changes to using 'overlay' as its new active
+image.
+
+Arguments:
+
+- "node": device that will have a snapshot created (json-string)
+- "overlay": device that will have 'node' as its backing image (json-string)
+
+Example:
+
+-> { "execute": "blockdev-add",
+                "arguments": { "options": { "driver": "qcow2",
+                                            "node-name": "node1534",
+                                            "file": { "driver": "file",
+                                                      "filename": "hd1.qcow2" },
+                                            "backing": "" } } }
+
+<- { "return": {} }
+
+-> { "execute": "blockdev-snapshot", "arguments": { "node": "ide-hd0",
+                                                    "overlay": "node1534" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "blockdev-snapshot-internal-sync",
         .args_type  = "device:B,name:s",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_snapshot_internal_sync,
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot_internal_sync,
     },
 
 SQMP
@@ -1461,7 +1565,7 @@ EQMP
         .name       = "blockdev-snapshot-delete-internal-sync",
         .args_type  = "device:B,id:s?,name:s?",
         .mhandler.cmd_new =
-                      qmp_marshal_input_blockdev_snapshot_delete_internal_sync,
+                      qmp_marshal_blockdev_snapshot_delete_internal_sync,
     },
 
 SQMP
@@ -1505,7 +1609,7 @@ EQMP
                       "on-source-error:s?,on-target-error:s?,"
                       "unmap:b?,"
                       "granularity:i?,buf-size:i?",
-        .mhandler.cmd_new = qmp_marshal_input_drive_mirror,
+        .mhandler.cmd_new = qmp_marshal_drive_mirror,
     },
 
 SQMP
@@ -1565,7 +1669,7 @@ EQMP
     {
         .name       = "change-backing-file",
         .args_type  = "device:s,image-node-name:s,backing-file:s",
-        .mhandler.cmd_new = qmp_marshal_input_change_backing_file,
+        .mhandler.cmd_new = qmp_marshal_change_backing_file,
     },
 
 SQMP
@@ -1604,7 +1708,7 @@ EQMP
     {
         .name       = "balloon",
         .args_type  = "value:M",
-        .mhandler.cmd_new = qmp_marshal_input_balloon,
+        .mhandler.cmd_new = qmp_marshal_balloon,
     },
 
 SQMP
@@ -1627,7 +1731,7 @@ EQMP
     {
         .name       = "set_link",
         .args_type  = "name:s,up:b",
-        .mhandler.cmd_new = qmp_marshal_input_set_link,
+        .mhandler.cmd_new = qmp_marshal_set_link,
     },
 
 SQMP
@@ -1653,7 +1757,7 @@ EQMP
         .args_type  = "fdname:s",
         .params     = "getfd name",
         .help       = "receive a file descriptor via SCM rights and assign it a name",
-        .mhandler.cmd_new = qmp_marshal_input_getfd,
+        .mhandler.cmd_new = qmp_marshal_getfd,
     },
 
 SQMP
@@ -1686,7 +1790,7 @@ EQMP
         .args_type  = "fdname:s",
         .params     = "closefd name",
         .help       = "close a file descriptor previously passed via SCM rights",
-        .mhandler.cmd_new = qmp_marshal_input_closefd,
+        .mhandler.cmd_new = qmp_marshal_closefd,
     },
 
 SQMP
@@ -1711,7 +1815,7 @@ EQMP
         .args_type  = "fdset-id:i?,opaque:s?",
         .params     = "add-fd fdset-id opaque",
         .help       = "Add a file descriptor, that was passed via SCM rights, to an fd set",
-        .mhandler.cmd_new = qmp_marshal_input_add_fd,
+        .mhandler.cmd_new = qmp_marshal_add_fd,
     },
 
 SQMP
@@ -1750,7 +1854,7 @@ EQMP
         .args_type  = "fdset-id:i,fd:i?",
         .params     = "remove-fd fdset-id fd",
         .help       = "Remove a file descriptor from an fd set",
-        .mhandler.cmd_new = qmp_marshal_input_remove_fd,
+        .mhandler.cmd_new = qmp_marshal_remove_fd,
     },
 
 SQMP
@@ -1782,7 +1886,7 @@ EQMP
         .name       = "query-fdsets",
         .args_type  = "",
         .help       = "Return information describing all fd sets",
-        .mhandler.cmd_new = qmp_marshal_input_query_fdsets,
+        .mhandler.cmd_new = qmp_marshal_query_fdsets,
     },
 
 SQMP
@@ -1831,7 +1935,7 @@ EQMP
     {
         .name       = "block_passwd",
         .args_type  = "device:s?,node-name:s?,password:s",
-        .mhandler.cmd_new = qmp_marshal_input_block_passwd,
+        .mhandler.cmd_new = qmp_marshal_block_passwd,
     },
 
 SQMP
@@ -1857,7 +1961,7 @@ EQMP
     {
         .name       = "block_set_io_throttle",
         .args_type  = "device:B,bps:l,bps_rd:l,bps_wr:l,iops:l,iops_rd:l,iops_wr:l,bps_max:l?,bps_rd_max:l?,bps_wr_max:l?,iops_max:l?,iops_rd_max:l?,iops_wr_max:l?,iops_size:l?,group:s?",
-        .mhandler.cmd_new = qmp_marshal_input_block_set_io_throttle,
+        .mhandler.cmd_new = qmp_marshal_block_set_io_throttle,
     },
 
 SQMP
@@ -1907,7 +2011,7 @@ EQMP
     {
         .name       = "set_password",
         .args_type  = "protocol:s,password:s,connected:s?",
-        .mhandler.cmd_new = qmp_marshal_input_set_password,
+        .mhandler.cmd_new = qmp_marshal_set_password,
     },
 
 SQMP
@@ -1933,7 +2037,7 @@ EQMP
     {
         .name       = "expire_password",
         .args_type  = "protocol:s,time:s",
-        .mhandler.cmd_new = qmp_marshal_input_expire_password,
+        .mhandler.cmd_new = qmp_marshal_expire_password,
     },
 
 SQMP
@@ -1958,7 +2062,7 @@ EQMP
     {
         .name       = "add_client",
         .args_type  = "protocol:s,fdname:s,skipauth:b?,tls:b?",
-        .mhandler.cmd_new = qmp_marshal_input_add_client,
+        .mhandler.cmd_new = qmp_marshal_add_client,
     },
 
 SQMP
@@ -2009,7 +2113,7 @@ EQMP
     {
         .name       = "human-monitor-command",
         .args_type  = "command-line:s,cpu-index:i?",
-        .mhandler.cmd_new = qmp_marshal_input_human_monitor_command,
+        .mhandler.cmd_new = qmp_marshal_human_monitor_command,
     },
 
 SQMP
@@ -2088,7 +2192,7 @@ EQMP
     {
         .name       = "query-version",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_version,
+        .mhandler.cmd_new = qmp_marshal_query_version,
     },
 
 SQMP
@@ -2125,7 +2229,7 @@ EQMP
     {
         .name       = "query-commands",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_commands,
+        .mhandler.cmd_new = qmp_marshal_query_commands,
     },
 
 SQMP
@@ -2162,7 +2266,24 @@ EQMP
     {
         .name       = "query-events",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_events,
+        .mhandler.cmd_new = qmp_marshal_query_events,
+    },
+
+SQMP
+query-qmp-schema
+----------------
+
+Return the QMP wire schema.  The returned value is a json-array of
+named schema entities.  Entities are commands, events and various
+types.  See docs/qapi-code-gen.txt for information on their structure
+and intended use.
+
+EQMP
+
+    {
+        .name       = "query-qmp-schema",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_query_qmp_schema,
     },
 
 SQMP
@@ -2207,7 +2328,7 @@ EQMP
     {
         .name       = "query-chardev",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_chardev,
+        .mhandler.cmd_new = qmp_marshal_query_chardev,
     },
 
 SQMP
@@ -2248,7 +2369,7 @@ EQMP
     {
         .name       = "query-chardev-backends",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_chardev_backends,
+        .mhandler.cmd_new = qmp_marshal_query_chardev_backends,
     },
 
 SQMP
@@ -2432,7 +2553,7 @@ EQMP
     {
         .name       = "query-block",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_block,
+        .mhandler.cmd_new = qmp_marshal_query_block,
     },
 
 SQMP
@@ -2456,12 +2577,70 @@ Each json-object contain the following:
     - "wr_total_time_ns": total time spend on writes in nano-seconds (json-int)
     - "rd_total_time_ns": total time spend on reads in nano-seconds (json-int)
     - "flush_total_time_ns": total time spend on cache flushes in nano-seconds (json-int)
-    - "wr_highest_offset": Highest offset of a sector written since the
-                           BlockDriverState has been opened (json-int)
+    - "wr_highest_offset": The offset after the greatest byte written to the
+                           BlockDriverState since it has been opened (json-int)
     - "rd_merged": number of read requests that have been merged into
                    another request (json-int)
     - "wr_merged": number of write requests that have been merged into
                    another request (json-int)
+    - "idle_time_ns": time since the last I/O operation, in
+                      nanoseconds. If the field is absent it means
+                      that there haven't been any operations yet
+                      (json-int, optional)
+    - "failed_rd_operations": number of failed read operations
+                              (json-int)
+    - "failed_wr_operations": number of failed write operations
+                              (json-int)
+    - "failed_flush_operations": number of failed flush operations
+                               (json-int)
+    - "invalid_rd_operations": number of invalid read operations
+                               (json-int)
+    - "invalid_wr_operations": number of invalid write operations
+                               (json-int)
+    - "invalid_flush_operations": number of invalid flush operations
+                                  (json-int)
+    - "account_invalid": whether invalid operations are included in
+                         the last access statistics (json-bool)
+    - "account_failed": whether failed operations are included in the
+                         latency and last access statistics
+                         (json-bool)
+    - "timed_stats": A json-array containing statistics collected in
+                     specific intervals, with the following members:
+        - "interval_length": interval used for calculating the
+                             statistics, in seconds (json-int)
+        - "min_rd_latency_ns": minimum latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "min_wr_latency_ns": minimum latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "min_flush_latency_ns": minimum latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "max_rd_latency_ns": maximum latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "max_wr_latency_ns": maximum latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "max_flush_latency_ns": maximum latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "avg_rd_latency_ns": average latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "avg_wr_latency_ns": average latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "avg_flush_latency_ns": average latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "avg_rd_queue_depth": average number of pending read
+                                operations in the defined interval
+                                (json-number)
+        - "avg_wr_queue_depth": average number of pending write
+                                operations in the defined interval
+                                (json-number).
 - "parent": Contains recursively the statistics of the underlying
             protocol (e.g. the host file for a qcow2 image). If there is
             no underlying protocol, this field is omitted
@@ -2486,7 +2665,10 @@ Example:
                   "flush_total_times_ns":49653
                   "flush_operations":61,
                   "rd_merged":0,
-                  "wr_merged":0
+                  "wr_merged":0,
+                  "idle_time_ns":2953431879,
+                  "account_invalid":true,
+                  "account_failed":false
                }
             },
             "stats":{
@@ -2500,7 +2682,10 @@ Example:
                "rd_total_times_ns":3465673657
                "flush_total_times_ns":49653,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "idle_time_ns":2953431879,
+               "account_invalid":true,
+               "account_failed":false
             }
          },
          {
@@ -2516,7 +2701,9 @@ Example:
                "rd_total_times_ns":0
                "flush_total_times_ns":0,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          },
          {
@@ -2532,7 +2719,9 @@ Example:
                "rd_total_times_ns":0
                "flush_total_times_ns":0,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          },
          {
@@ -2548,7 +2737,9 @@ Example:
                "rd_total_times_ns":0
                "flush_total_times_ns":0,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          }
       ]
@@ -2559,7 +2750,7 @@ EQMP
     {
         .name       = "query-blockstats",
         .args_type  = "query-nodes:b?",
-        .mhandler.cmd_new = qmp_marshal_input_query_blockstats,
+        .mhandler.cmd_new = qmp_marshal_query_blockstats,
     },
 
 SQMP
@@ -2610,7 +2801,7 @@ EQMP
     {
         .name       = "query-cpus",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_cpus,
+        .mhandler.cmd_new = qmp_marshal_query_cpus,
     },
 
 SQMP
@@ -2649,7 +2840,7 @@ EQMP
     {
         .name       = "query-iothreads",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_iothreads,
+        .mhandler.cmd_new = qmp_marshal_query_iothreads,
     },
 
 SQMP
@@ -2866,7 +3057,7 @@ EQMP
     {
         .name       = "query-pci",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_pci,
+        .mhandler.cmd_new = qmp_marshal_query_pci,
     },
 
 SQMP
@@ -2890,7 +3081,7 @@ EQMP
     {
         .name       = "query-kvm",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_kvm,
+        .mhandler.cmd_new = qmp_marshal_query_kvm,
     },
 
 SQMP
@@ -2930,7 +3121,7 @@ EQMP
     {
         .name       = "query-status",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_status,
+        .mhandler.cmd_new = qmp_marshal_query_status,
     },
 
 SQMP
@@ -2974,7 +3165,7 @@ EQMP
     {
         .name       = "query-mice",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_mice,
+        .mhandler.cmd_new = qmp_marshal_query_mice,
     },
 
 SQMP
@@ -3037,12 +3228,12 @@ EQMP
     {
         .name       = "query-vnc",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_vnc,
+        .mhandler.cmd_new = qmp_marshal_query_vnc,
     },
     {
         .name       = "query-vnc-servers",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_vnc_servers,
+        .mhandler.cmd_new = qmp_marshal_query_vnc_servers,
     },
 
 SQMP
@@ -3119,7 +3310,7 @@ EQMP
     {
         .name       = "query-spice",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_spice,
+        .mhandler.cmd_new = qmp_marshal_query_spice,
     },
 #endif
 
@@ -3143,7 +3334,7 @@ EQMP
     {
         .name       = "query-name",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_name,
+        .mhandler.cmd_new = qmp_marshal_query_name,
     },
 
 SQMP
@@ -3166,7 +3357,7 @@ EQMP
     {
         .name       = "query-uuid",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_uuid,
+        .mhandler.cmd_new = qmp_marshal_query_uuid,
     },
 
 SQMP
@@ -3215,7 +3406,7 @@ EQMP
     {
         .name       = "query-command-line-options",
         .args_type  = "option:s?",
-        .mhandler.cmd_new = qmp_marshal_input_query_command_line_options,
+        .mhandler.cmd_new = qmp_marshal_query_command_line_options,
     },
 
 SQMP
@@ -3393,7 +3584,7 @@ EQMP
     {
         .name       = "query-migrate",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate,
+        .mhandler.cmd_new = qmp_marshal_query_migrate,
     },
 
 SQMP
@@ -3421,7 +3612,7 @@ EQMP
         .name       = "migrate-set-capabilities",
         .args_type  = "capabilities:q",
         .params     = "capability:s,state:b",
-	.mhandler.cmd_new = qmp_marshal_input_migrate_set_capabilities,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_capabilities,
     },
 SQMP
 query-migrate-capabilities
@@ -3447,7 +3638,7 @@ EQMP
     {
         .name       = "query-migrate-capabilities",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_capabilities,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_capabilities,
     },
 
 SQMP
@@ -3473,7 +3664,7 @@ EQMP
         .name       = "migrate-set-parameters",
         .args_type  =
             "compress-level:i?,compress-threads:i?,decompress-threads:i?",
-	.mhandler.cmd_new = qmp_marshal_input_migrate_set_parameters,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_parameters,
     },
 SQMP
 query-migrate-parameters
@@ -3504,7 +3695,7 @@ EQMP
     {
         .name       = "query-migrate-parameters",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_parameters,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_parameters,
     },
 
 SQMP
@@ -3532,88 +3723,88 @@ EQMP
     {
         .name       = "query-balloon",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_balloon,
+        .mhandler.cmd_new = qmp_marshal_query_balloon,
     },
 
     {
         .name       = "query-block-jobs",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_block_jobs,
+        .mhandler.cmd_new = qmp_marshal_query_block_jobs,
     },
 
     {
         .name       = "qom-list",
         .args_type  = "path:s",
-        .mhandler.cmd_new = qmp_marshal_input_qom_list,
+        .mhandler.cmd_new = qmp_marshal_qom_list,
     },
 
     {
         .name       = "qom-set",
 	.args_type  = "path:s,property:s,value:q",
-	.mhandler.cmd_new = qmp_qom_set,
+        .mhandler.cmd_new = qmp_marshal_qom_set,
     },
 
     {
         .name       = "qom-get",
 	.args_type  = "path:s,property:s",
-	.mhandler.cmd_new = qmp_qom_get,
+        .mhandler.cmd_new = qmp_marshal_qom_get,
     },
 
     {
         .name       = "nbd-server-start",
         .args_type  = "addr:q",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_start,
+        .mhandler.cmd_new = qmp_marshal_nbd_server_start,
     },
     {
         .name       = "nbd-server-add",
         .args_type  = "device:B,writable:b?",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_add,
+        .mhandler.cmd_new = qmp_marshal_nbd_server_add,
     },
     {
         .name       = "nbd-server-stop",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_stop,
+        .mhandler.cmd_new = qmp_marshal_nbd_server_stop,
     },
 
     {
         .name       = "change-vnc-password",
         .args_type  = "password:s",
-        .mhandler.cmd_new = qmp_marshal_input_change_vnc_password,
+        .mhandler.cmd_new = qmp_marshal_change_vnc_password,
     },
     {
         .name       = "qom-list-types",
         .args_type  = "implements:s?,abstract:b?",
-        .mhandler.cmd_new = qmp_marshal_input_qom_list_types,
+        .mhandler.cmd_new = qmp_marshal_qom_list_types,
     },
 
     {
         .name       = "device-list-properties",
         .args_type  = "typename:s",
-        .mhandler.cmd_new = qmp_marshal_input_device_list_properties,
+        .mhandler.cmd_new = qmp_marshal_device_list_properties,
     },
 
     {
         .name       = "query-machines",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_machines,
+        .mhandler.cmd_new = qmp_marshal_query_machines,
     },
 
     {
         .name       = "query-cpu-definitions",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_cpu_definitions,
+        .mhandler.cmd_new = qmp_marshal_query_cpu_definitions,
     },
 
     {
         .name       = "query-target",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_target,
+        .mhandler.cmd_new = qmp_marshal_query_target,
     },
 
     {
         .name       = "query-tpm",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm,
+        .mhandler.cmd_new = qmp_marshal_query_tpm,
     },
 
 SQMP
@@ -3647,7 +3838,7 @@ EQMP
     {
         .name       = "query-tpm-models",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm_models,
+        .mhandler.cmd_new = qmp_marshal_query_tpm_models,
     },
 
 SQMP
@@ -3668,7 +3859,7 @@ EQMP
     {
         .name       = "query-tpm-types",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm_types,
+        .mhandler.cmd_new = qmp_marshal_query_tpm_types,
     },
 
 SQMP
@@ -3689,7 +3880,7 @@ EQMP
     {
         .name       = "chardev-add",
         .args_type  = "id:s,backend:q",
-        .mhandler.cmd_new = qmp_marshal_input_chardev_add,
+        .mhandler.cmd_new = qmp_marshal_chardev_add,
     },
 
 SQMP
@@ -3726,7 +3917,7 @@ EQMP
     {
         .name       = "chardev-remove",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_chardev_remove,
+        .mhandler.cmd_new = qmp_marshal_chardev_remove,
     },
 
 
@@ -3749,7 +3940,7 @@ EQMP
     {
         .name       = "query-rx-filter",
         .args_type  = "name:s?",
-        .mhandler.cmd_new = qmp_marshal_input_query_rx_filter,
+        .mhandler.cmd_new = qmp_marshal_query_rx_filter,
     },
 
 SQMP
@@ -3815,7 +4006,7 @@ EQMP
     {
         .name       = "blockdev-add",
         .args_type  = "options:q",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_add,
+        .mhandler.cmd_new = qmp_marshal_blockdev_add,
     },
 
 SQMP
@@ -3825,8 +4016,8 @@ blockdev-add
 Add a block device.
 
 This command is still a work in progress.  It doesn't support all
-block drivers, it lacks a matching blockdev-del, and more.  Stay away
-from it unless you want to help with its development.
+block drivers among other things.  Stay away from it unless you want
+to help with its development.
 
 Arguments:
 
@@ -3872,9 +4063,237 @@ Example (2):
 EQMP
 
     {
+        .name       = "x-blockdev-del",
+        .args_type  = "id:s?,node-name:s?",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_del,
+    },
+
+SQMP
+x-blockdev-del
+------------
+Since 2.5
+
+Deletes a block device thas has been added using blockdev-add.
+The selected device can be either a block backend or a graph node.
+
+In the former case the backend will be destroyed, along with its
+inserted medium if there's any. The command will fail if the backend
+or its medium are in use.
+
+In the latter case the node will be destroyed. The command will fail
+if the node is attached to a block backend or is otherwise being
+used.
+
+One of "id" or "node-name" must be specified, but not both.
+
+This command is still a work in progress and is considered
+experimental. Stay away from it unless you want to help with its
+development.
+
+Arguments:
+
+- "id": Name of the block backend device to delete (json-string, optional)
+- "node-name": Name of the graph node to delete (json-string, optional)
+
+Example:
+
+-> { "execute": "blockdev-add",
+     "arguments": {
+         "options": {
+             "driver": "qcow2",
+             "id": "drive0",
+             "file": {
+                 "driver": "file",
+                 "filename": "test.qcow2"
+             }
+         }
+     }
+   }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-del",
+     "arguments": { "id": "drive0" }
+   }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-open-tray",
+        .args_type  = "device:s,force:b?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_open_tray,
+    },
+
+SQMP
+blockdev-open-tray
+------------------
+
+Opens a block device's tray. If there is a block driver state tree inserted as a
+medium, it will become inaccessible to the guest (but it will remain associated
+to the block device, so closing the tray will make it accessible again).
+
+If the tray was already open before, this will be a no-op.
+
+Once the tray opens, a DEVICE_TRAY_MOVED event is emitted. There are cases in
+which no such event will be generated, these include:
+- if the guest has locked the tray, @force is false and the guest does not
+  respond to the eject request
+- if the BlockBackend denoted by @device does not have a guest device attached
+  to it
+- if the guest device does not have an actual tray and is empty, for instance
+  for floppy disk drives
+
+Arguments:
+
+- "device": block device name (json-string)
+- "force": if false (the default), an eject request will be sent to the guest if
+           it has locked the tray (and the tray will not be opened immediately);
+           if true, the tray will be opened regardless of whether it is locked
+           (json-bool, optional)
+
+Example:
+
+-> { "execute": "blockdev-open-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751016,
+                    "microseconds": 716996 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": true } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-close-tray",
+        .args_type  = "device:s",
+        .mhandler.cmd_new = qmp_marshal_blockdev_close_tray,
+    },
+
+SQMP
+blockdev-close-tray
+-------------------
+
+Closes a block device's tray. If there is a block driver state tree associated
+with the block device (which is currently ejected), that tree will be loaded as
+the medium.
+
+If the tray was already closed before, this will be a no-op.
+
+Arguments:
+
+- "device": block device name (json-string)
+
+Example:
+
+-> { "execute": "blockdev-close-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751345,
+                    "microseconds": 272147 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": false } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "x-blockdev-remove-medium",
+        .args_type  = "device:s",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_remove_medium,
+    },
+
+SQMP
+x-blockdev-remove-medium
+------------------------
+
+Removes a medium (a block driver state tree) from a block device. That block
+device's tray must currently be open (unless there is no attached guest device).
+
+If the tray is open and there is no medium inserted, this will be a no-op.
+
+This command is still a work in progress and is considered experimental.
+Stay away from it unless you want to help with its development.
+
+Arguments:
+
+- "device": block device name (json-string)
+
+Example:
+
+-> { "execute": "x-blockdev-remove-medium",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "error": { "class": "GenericError",
+                "desc": "Tray of device 'ide1-cd0' is not open" } }
+
+-> { "execute": "blockdev-open-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751627,
+                    "microseconds": 549958 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": true } }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-remove-medium",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "x-blockdev-insert-medium",
+        .args_type  = "device:s,node-name:s",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_insert_medium,
+    },
+
+SQMP
+x-blockdev-insert-medium
+------------------------
+
+Inserts a medium (a block driver state tree) into a block device. That block
+device's tray must currently be open (unless there is no attached guest device)
+and there must be no medium inserted already.
+
+This command is still a work in progress and is considered experimental.
+Stay away from it unless you want to help with its development.
+
+Arguments:
+
+- "device": block device name (json-string)
+- "node-name": root node of the BDS tree to insert into the block device
+
+Example:
+
+-> { "execute": "blockdev-add",
+     "arguments": { "options": { "node-name": "node0",
+                                 "driver": "raw",
+                                 "file": { "driver": "file",
+                                           "filename": "fedora.iso" } } } }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-insert-medium",
+     "arguments": { "device": "ide1-cd0",
+                    "node-name": "node0" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-named-block-nodes",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_named_block_nodes,
+        .mhandler.cmd_new = qmp_marshal_query_named_block_nodes,
     },
 
 SQMP
@@ -3934,9 +4353,62 @@ Example:
 EQMP
 
     {
+        .name       = "blockdev-change-medium",
+        .args_type  = "device:B,filename:F,format:s?,read-only-mode:s?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_change_medium,
+    },
+
+SQMP
+blockdev-change-medium
+----------------------
+
+Changes the medium inserted into a block device by ejecting the current medium
+and loading a new image file which is inserted as the new medium.
+
+Arguments:
+
+- "device": device name (json-string)
+- "filename": filename of the new image (json-string)
+- "format": format of the new image (json-string, optional)
+- "read-only-mode": new read-only mode (json-string, optional)
+          - Possible values: "retain" (default), "read-only", "read-write"
+
+Examples:
+
+1. Change a removable medium
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "ide1-cd0",
+                            "filename": "/srv/images/Fedora-12-x86_64-DVD.iso",
+                            "format": "raw" } }
+<- { "return": {} }
+
+2. Load a read-only medium into a writable drive
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "isa-fd0",
+                            "filename": "/srv/images/ro.img",
+                            "format": "raw",
+                            "read-only-mode": "retain" } }
+
+<- { "error":
+     { "class": "GenericError",
+       "desc": "Could not open '/srv/images/ro.img': Permission denied" } }
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "isa-fd0",
+                            "filename": "/srv/images/ro.img",
+                            "format": "raw",
+                            "read-only-mode": "read-only" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-memdev",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_memdev,
+        .mhandler.cmd_new = qmp_marshal_query_memdev,
     },
 
 SQMP
@@ -3974,7 +4446,7 @@ EQMP
     {
         .name       = "query-memory-devices",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_memory_devices,
+        .mhandler.cmd_new = qmp_marshal_query_memory_devices,
     },
 
 SQMP
@@ -4001,7 +4473,7 @@ EQMP
     {
         .name       = "query-acpi-ospm-status",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_acpi_ospm_status,
+        .mhandler.cmd_new = qmp_marshal_query_acpi_ospm_status,
     },
 
 SQMP
@@ -4024,7 +4496,7 @@ EQMP
     {
         .name       = "rtc-reset-reinjection",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_rtc_reset_reinjection,
+        .mhandler.cmd_new = qmp_marshal_rtc_reset_reinjection,
     },
 #endif
 
@@ -4045,7 +4517,7 @@ EQMP
     {
         .name       = "trace-event-get-state",
         .args_type  = "name:s",
-        .mhandler.cmd_new = qmp_marshal_input_trace_event_get_state,
+        .mhandler.cmd_new = qmp_marshal_trace_event_get_state,
     },
 
 SQMP
@@ -4063,7 +4535,7 @@ EQMP
     {
         .name       = "trace-event-set-state",
         .args_type  = "name:s,enable:b,ignore-unavailable:b?",
-        .mhandler.cmd_new = qmp_marshal_input_trace_event_set_state,
+        .mhandler.cmd_new = qmp_marshal_trace_event_set_state,
     },
 
 SQMP
@@ -4081,7 +4553,7 @@ EQMP
     {
         .name       = "x-input-send-event",
         .args_type  = "console:i?,events:q",
-        .mhandler.cmd_new = qmp_marshal_input_x_input_send_event,
+        .mhandler.cmd_new = qmp_marshal_x_input_send_event,
     },
 
 SQMP
@@ -4146,7 +4618,7 @@ EQMP
     {
         .name       = "block-set-write-threshold",
         .args_type  = "node-name:s,write-threshold:l",
-        .mhandler.cmd_new = qmp_marshal_input_block_set_write_threshold,
+        .mhandler.cmd_new = qmp_marshal_block_set_write_threshold,
     },
 
 SQMP
@@ -4174,7 +4646,7 @@ EQMP
     {
         .name       = "query-rocker",
         .args_type  = "name:s",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker,
+        .mhandler.cmd_new = qmp_marshal_query_rocker,
     },
 
 SQMP
@@ -4195,7 +4667,7 @@ EQMP
     {
         .name       = "query-rocker-ports",
         .args_type  = "name:s",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker_ports,
+        .mhandler.cmd_new = qmp_marshal_query_rocker_ports,
     },
 
 SQMP
@@ -4220,7 +4692,7 @@ EQMP
     {
         .name       = "query-rocker-of-dpa-flows",
         .args_type  = "name:s,tbl-id:i?",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker_of_dpa_flows,
+        .mhandler.cmd_new = qmp_marshal_query_rocker_of_dpa_flows,
     },
 
 SQMP
@@ -4249,7 +4721,7 @@ EQMP
     {
         .name       = "query-rocker-of-dpa-groups",
         .args_type  = "name:s,type:i?",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker_of_dpa_groups,
+        .mhandler.cmd_new = qmp_marshal_query_rocker_of_dpa_groups,
     },
 
 SQMP
diff --git a/qmp.c b/qmp.c
index 6998ca2fc..0a1fa1992 100644
--- a/qmp.c
+++ b/qmp.c
@@ -24,6 +24,7 @@
 #include "sysemu/arch_init.h"
 #include "hw/qdev.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "qom/qom-qobject.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qobject.h"
@@ -49,14 +50,20 @@ VersionInfo *qmp_query_version(Error **errp)
 {
     VersionInfo *info = g_new0(VersionInfo, 1);
     const char *version = QEMU_VERSION;
-    char *tmp;
+    const char *tmp;
+    int err;
 
     info->qemu = g_new0(VersionTriple, 1);
-    info->qemu->major = strtol(version, &tmp, 10);
+    err = qemu_strtoll(version, &tmp, 10, &info->qemu->major);
+    assert(err == 0);
     tmp++;
-    info->qemu->minor = strtol(tmp, &tmp, 10);
+
+    err = qemu_strtoll(tmp, &tmp, 10, &info->qemu->minor);
+    assert(err == 0);
     tmp++;
-    info->qemu->micro = strtol(tmp, &tmp, 10);
+
+    err = qemu_strtoll(tmp, &tmp, 10, &info->qemu->micro);
+    assert(err == 0);
     info->package = g_strdup(QEMU_PKGVERSION);
 
     return info;
@@ -151,9 +158,9 @@ VncInfo2List *qmp_query_vnc_servers(Error **errp)
  * #ifdef CONFIG_SPICE.  Necessary for an accurate query-commands
  * result.  However, the QAPI schema is blissfully unaware of that,
  * and the QAPI code generator happily generates a dead
- * qmp_marshal_input_query_spice() that calls qmp_query_spice().
- * Provide it one, or else linking fails.
- * FIXME Educate the QAPI schema on CONFIG_SPICE.
+ * qmp_marshal_query_spice() that calls qmp_query_spice().  Provide it
+ * one, or else linking fails.  FIXME Educate the QAPI schema on
+ * CONFIG_SPICE.
  */
 SpiceInfo *qmp_query_spice(Error **errp)
 {
@@ -164,6 +171,7 @@ SpiceInfo *qmp_query_spice(Error **errp)
 void qmp_cont(Error **errp)
 {
     Error *local_err = NULL;
+    BlockBackend *blk;
     BlockDriverState *bs;
 
     if (runstate_needs_reset()) {
@@ -173,8 +181,8 @@ void qmp_cont(Error **errp)
         return;
     }
 
-    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-        bdrv_iostatus_reset(bs);
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+        blk_iostatus_reset(blk);
     }
     for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
         bdrv_add_key(bs, NULL, &local_err);
@@ -202,6 +210,7 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp)
     bool ambiguous = false;
     ObjectPropertyInfoList *props = NULL;
     ObjectProperty *prop;
+    ObjectPropertyIterator *iter;
 
     obj = object_resolve_path(path, &ambiguous);
     if (obj == NULL) {
@@ -214,7 +223,8 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp)
         return NULL;
     }
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    iter = object_property_iter_init(obj);
+    while ((prop = object_property_iter_next(iter))) {
         ObjectPropertyInfoList *entry = g_malloc0(sizeof(*entry));
 
         entry->value = g_malloc0(sizeof(ObjectPropertyInfo));
@@ -224,16 +234,14 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp)
         entry->value->name = g_strdup(prop->name);
         entry->value->type = g_strdup(prop->type);
     }
+    object_property_iter_free(iter);
 
     return props;
 }
 
-/* FIXME: teach qapi about how to pass through Visitors */
-void qmp_qom_set(QDict *qdict, QObject **ret, Error **errp)
+void qmp_qom_set(const char *path, const char *property, QObject *value,
+                 Error **errp)
 {
-    const char *path = qdict_get_str(qdict, "path");
-    const char *property = qdict_get_str(qdict, "property");
-    QObject *value = qdict_get(qdict, "value");
     Object *obj;
 
     obj = object_resolve_path(path, NULL);
@@ -246,20 +254,18 @@ void qmp_qom_set(QDict *qdict, QObject **ret, Error **errp)
     object_property_set_qobject(obj, value, property, errp);
 }
 
-void qmp_qom_get(QDict *qdict, QObject **ret, Error **errp)
+QObject *qmp_qom_get(const char *path, const char *property, Error **errp)
 {
-    const char *path = qdict_get_str(qdict, "path");
-    const char *property = qdict_get_str(qdict, "property");
     Object *obj;
 
     obj = object_resolve_path(path, NULL);
     if (!obj) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                   "Device '%s' not found", path);
-        return;
+        return NULL;
     }
 
-    *ret = object_property_get_qobject(obj, property, errp);
+    return object_property_get_qobject(obj, property, errp);
 }
 
 void qmp_set_password(const char *protocol, const char *password,
@@ -411,7 +417,8 @@ void qmp_change(const char *device, const char *target,
     if (strcmp(device, "vnc") == 0) {
         qmp_change_vnc(target, has_arg, arg, errp);
     } else {
-        qmp_change_blockdev(device, target, arg, errp);
+        qmp_blockdev_change_medium(device, target, has_arg, arg, false, 0,
+                                   errp);
     }
 }
 
@@ -499,6 +506,7 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
     ObjectClass *klass;
     Object *obj;
     ObjectProperty *prop;
+    ObjectPropertyIterator *iter;
     DevicePropertyInfoList *prop_list = NULL;
 
     klass = object_class_by_name(typename);
@@ -527,7 +535,8 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
 
     obj = object_new(typename);
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    iter = object_property_iter_init(obj);
+    while ((prop = object_property_iter_next(iter))) {
         DevicePropertyInfo *info;
         DevicePropertyInfoList *entry;
 
@@ -558,6 +567,7 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
         entry->next = prop_list;
         prop_list = entry;
     }
+    object_property_iter_free(iter);
 
     object_unref(obj);
 
@@ -666,11 +676,9 @@ out:
     object_unref(obj);
 }
 
-void qmp_object_add(QDict *qdict, QObject **ret, Error **errp)
+void qmp_object_add(const char *type, const char *id,
+                    bool has_props, QObject *props, Error **errp)
 {
-    const char *type = qdict_get_str(qdict, "qom-type");
-    const char *id = qdict_get_str(qdict, "id");
-    QObject *props = qdict_get(qdict, "props");
     const QDict *pdict = NULL;
     QmpInputVisitor *qiv;
 
diff --git a/qobject/json-lexer.c b/qobject/json-lexer.c
index b19623e22..92798ae3a 100644
--- a/qobject/json-lexer.c
+++ b/qobject/json-lexer.c
@@ -11,12 +11,9 @@
  *
  */
 
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qdict.h"
-#include "qapi/qmp/qint.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
+#include <stdint.h>
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 
@@ -30,7 +27,7 @@
  */
 
 enum json_lexer_state {
-    IN_ERROR = 0,
+    IN_ERROR = 0,               /* must really be 0, see json_lexer[] */
     IN_DQ_UCODE3,
     IN_DQ_UCODE2,
     IN_DQ_UCODE1,
@@ -62,6 +59,8 @@ enum json_lexer_state {
     IN_START,
 };
 
+QEMU_BUILD_BUG_ON((int)JSON_MIN <= (int)IN_START);
+
 #define TERMINAL(state) [0 ... 0x7F] = (state)
 
 /* Return whether TERMINAL is a terminal state and the transition to it
@@ -71,6 +70,8 @@ enum json_lexer_state {
             (json_lexer[(old_state)][0] == (terminal))
 
 static const uint8_t json_lexer[][256] =  {
+    /* Relies on default initialization to IN_ERROR! */
+
     /* double quote string */
     [IN_DQ_UCODE3] = {
         ['0' ... '9'] = IN_DQ_STRING,
@@ -253,12 +254,12 @@ static const uint8_t json_lexer[][256] =  {
         ['0'] = IN_ZERO,
         ['1' ... '9'] = IN_NONZERO_NUMBER,
         ['-'] = IN_NEG_NONZERO_NUMBER,
-        ['{'] = JSON_OPERATOR,
-        ['}'] = JSON_OPERATOR,
-        ['['] = JSON_OPERATOR,
-        [']'] = JSON_OPERATOR,
-        [','] = JSON_OPERATOR,
-        [':'] = JSON_OPERATOR,
+        ['{'] = JSON_LCURLY,
+        ['}'] = JSON_RCURLY,
+        ['['] = JSON_LSQUARE,
+        [']'] = JSON_RSQUARE,
+        [','] = JSON_COMMA,
+        [':'] = JSON_COLON,
         ['a' ... 'z'] = IN_KEYWORD,
         ['%'] = IN_ESCAPE,
         [' '] = IN_WHITESPACE,
@@ -272,7 +273,7 @@ void json_lexer_init(JSONLexer *lexer, JSONLexerEmitter func)
 {
     lexer->emit = func;
     lexer->state = IN_START;
-    lexer->token = qstring_new();
+    lexer->token = g_string_sized_new(3);
     lexer->x = lexer->y = 0;
 }
 
@@ -287,14 +288,20 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
     }
 
     do {
+        assert(lexer->state <= ARRAY_SIZE(json_lexer));
         new_state = json_lexer[lexer->state][(uint8_t)ch];
         char_consumed = !TERMINAL_NEEDED_LOOKAHEAD(lexer->state, new_state);
         if (char_consumed) {
-            qstring_append_chr(lexer->token, ch);
+            g_string_append_c(lexer->token, ch);
         }
 
         switch (new_state) {
-        case JSON_OPERATOR:
+        case JSON_LCURLY:
+        case JSON_RCURLY:
+        case JSON_LSQUARE:
+        case JSON_RSQUARE:
+        case JSON_COLON:
+        case JSON_COMMA:
         case JSON_ESCAPE:
         case JSON_INTEGER:
         case JSON_FLOAT:
@@ -303,8 +310,7 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
             lexer->emit(lexer, lexer->token, new_state, lexer->x, lexer->y);
             /* fall through */
         case JSON_SKIP:
-            QDECREF(lexer->token);
-            lexer->token = qstring_new();
+            g_string_truncate(lexer->token, 0);
             new_state = IN_START;
             break;
         case IN_ERROR:
@@ -322,8 +328,7 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
              * induce an error/flush state.
              */
             lexer->emit(lexer, lexer->token, JSON_ERROR, lexer->x, lexer->y);
-            QDECREF(lexer->token);
-            lexer->token = qstring_new();
+            g_string_truncate(lexer->token, 0);
             new_state = IN_START;
             lexer->state = new_state;
             return 0;
@@ -336,10 +341,9 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
     /* Do not let a single token grow to an arbitrarily large size,
      * this is a security consideration.
      */
-    if (lexer->token->length > MAX_TOKEN_SIZE) {
+    if (lexer->token->len > MAX_TOKEN_SIZE) {
         lexer->emit(lexer, lexer->token, lexer->state, lexer->x, lexer->y);
-        QDECREF(lexer->token);
-        lexer->token = qstring_new();
+        g_string_truncate(lexer->token, 0);
         lexer->state = IN_START;
     }
 
@@ -369,5 +373,5 @@ int json_lexer_flush(JSONLexer *lexer)
 
 void json_lexer_destroy(JSONLexer *lexer)
 {
-    QDECREF(lexer->token);
+    g_string_free(lexer->token, true);
 }
diff --git a/qobject/json-parser.c b/qobject/json-parser.c
index ac991ba3c..3c5d35d7b 100644
--- a/qobject/json-parser.c
+++ b/qobject/json-parser.c
@@ -22,15 +22,13 @@
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-streamer.h"
 
 typedef struct JSONParserContext
 {
     Error *err;
-    struct {
-        QObject **buf;
-        size_t pos;
-        size_t count;
-    } tokens;
+    JSONToken *current;
+    GQueue *buf;
 } JSONParserContext;
 
 #define BUG_ON(cond) assert(!(cond))
@@ -47,58 +45,10 @@ typedef struct JSONParserContext
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap);
 
 /**
- * Token manipulators
- *
- * tokens are dictionaries that contain a type, a string value, and geometry information
- * about a token identified by the lexer.  These are routines that make working with
- * these objects a bit easier.
- */
-static const char *token_get_value(QObject *obj)
-{
-    return qdict_get_str(qobject_to_qdict(obj), "token");
-}
-
-static JSONTokenType token_get_type(QObject *obj)
-{
-    return qdict_get_int(qobject_to_qdict(obj), "type");
-}
-
-static int token_is_operator(QObject *obj, char op)
-{
-    const char *val;
-
-    if (token_get_type(obj) != JSON_OPERATOR) {
-        return 0;
-    }
-
-    val = token_get_value(obj);
-
-    return (val[0] == op) && (val[1] == 0);
-}
-
-static int token_is_keyword(QObject *obj, const char *value)
-{
-    if (token_get_type(obj) != JSON_KEYWORD) {
-        return 0;
-    }
-
-    return strcmp(token_get_value(obj), value) == 0;
-}
-
-static int token_is_escape(QObject *obj, const char *value)
-{
-    if (token_get_type(obj) != JSON_ESCAPE) {
-        return 0;
-    }
-
-    return (strcmp(token_get_value(obj), value) == 0);
-}
-
-/**
  * Error handler
  */
 static void GCC_FMT_ATTR(3, 4) parse_error(JSONParserContext *ctxt,
-                                           QObject *token, const char *msg, ...)
+                                           JSONToken *token, const char *msg, ...)
 {
     va_list ap;
     char message[1024];
@@ -176,9 +126,10 @@ static int hex2decimal(char ch)
  *      \t
  *      \u four-hex-digits 
  */
-static QString *qstring_from_escaped_str(JSONParserContext *ctxt, QObject *token)
+static QString *qstring_from_escaped_str(JSONParserContext *ctxt,
+                                         JSONToken *token)
 {
-    const char *ptr = token_get_value(token);
+    const char *ptr = token->str;
     QString *str;
     int double_quote = 1;
 
@@ -274,73 +225,34 @@ out:
     return NULL;
 }
 
-static QObject *parser_context_pop_token(JSONParserContext *ctxt)
-{
-    QObject *token;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    token = ctxt->tokens.buf[ctxt->tokens.pos];
-    ctxt->tokens.pos++;
-    return token;
-}
-
-/* Note: parser_context_{peek|pop}_token do not increment the
- * token object's refcount. In both cases the references will continue
- * to be tracked and cleaned up in parser_context_free(), so do not
- * attempt to free the token object.
+/* Note: the token object returned by parser_context_peek_token or
+ * parser_context_pop_token is deleted as soon as parser_context_pop_token
+ * is called again.
  */
-static QObject *parser_context_peek_token(JSONParserContext *ctxt)
-{
-    QObject *token;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    token = ctxt->tokens.buf[ctxt->tokens.pos];
-    return token;
-}
-
-static JSONParserContext parser_context_save(JSONParserContext *ctxt)
-{
-    JSONParserContext saved_ctxt = {0};
-    saved_ctxt.tokens.pos = ctxt->tokens.pos;
-    saved_ctxt.tokens.count = ctxt->tokens.count;
-    saved_ctxt.tokens.buf = ctxt->tokens.buf;
-    return saved_ctxt;
-}
-
-static void parser_context_restore(JSONParserContext *ctxt,
-                                   JSONParserContext saved_ctxt)
+static JSONToken *parser_context_pop_token(JSONParserContext *ctxt)
 {
-    ctxt->tokens.pos = saved_ctxt.tokens.pos;
-    ctxt->tokens.count = saved_ctxt.tokens.count;
-    ctxt->tokens.buf = saved_ctxt.tokens.buf;
+    g_free(ctxt->current);
+    assert(!g_queue_is_empty(ctxt->buf));
+    ctxt->current = g_queue_pop_head(ctxt->buf);
+    return ctxt->current;
 }
 
-static void tokens_append_from_iter(QObject *obj, void *opaque)
+static JSONToken *parser_context_peek_token(JSONParserContext *ctxt)
 {
-    JSONParserContext *ctxt = opaque;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    ctxt->tokens.buf[ctxt->tokens.pos++] = obj;
-    qobject_incref(obj);
+    assert(!g_queue_is_empty(ctxt->buf));
+    return g_queue_peek_head(ctxt->buf);
 }
 
-static JSONParserContext *parser_context_new(QList *tokens)
+static JSONParserContext *parser_context_new(GQueue *tokens)
 {
     JSONParserContext *ctxt;
-    size_t count;
 
     if (!tokens) {
         return NULL;
     }
 
-    count = qlist_size(tokens);
-    if (count == 0) {
-        return NULL;
-    }
-
     ctxt = g_malloc0(sizeof(JSONParserContext));
-    ctxt->tokens.pos = 0;
-    ctxt->tokens.count = count;
-    ctxt->tokens.buf = g_malloc(count * sizeof(QObject *));
-    qlist_iter(tokens, tokens_append_from_iter, ctxt);
-    ctxt->tokens.pos = 0;
+    ctxt->buf = tokens;
 
     return ctxt;
 }
@@ -348,12 +260,12 @@ static JSONParserContext *parser_context_new(QList *tokens)
 /* to support error propagation, ctxt->err must be freed separately */
 static void parser_context_free(JSONParserContext *ctxt)
 {
-    int i;
     if (ctxt) {
-        for (i = 0; i < ctxt->tokens.count; i++) {
-            qobject_decref(ctxt->tokens.buf[i]);
+        while (!g_queue_is_empty(ctxt->buf)) {
+            parser_context_pop_token(ctxt);
         }
-        g_free(ctxt->tokens.buf);
+        g_free(ctxt->current);
+        g_queue_free(ctxt->buf);
         g_free(ctxt);
     }
 }
@@ -363,8 +275,8 @@ static void parser_context_free(JSONParserContext *ctxt)
  */
 static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
 {
-    QObject *key = NULL, *token = NULL, *value, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    QObject *key = NULL, *value;
+    JSONToken *peek, *token;
 
     peek = parser_context_peek_token(ctxt);
     if (peek == NULL) {
@@ -384,7 +296,7 @@ static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
         goto out;
     }
 
-    if (!token_is_operator(token, ':')) {
+    if (token->type != JSON_COLON) {
         parse_error(ctxt, token, "missing : in object pair");
         goto out;
     }
@@ -402,7 +314,6 @@ static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
     return 0;
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     qobject_decref(key);
 
     return -1;
@@ -411,17 +322,10 @@ out:
 static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
 {
     QDict *dict = NULL;
-    QObject *token, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token, *peek;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (!token_is_operator(token, '{')) {
-        goto out;
-    }
+    assert(token && token->type == JSON_LCURLY);
 
     dict = qdict_new();
 
@@ -431,7 +335,7 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
         goto out;
     }
 
-    if (!token_is_operator(peek, '}')) {
+    if (peek->type != JSON_RCURLY) {
         if (parse_pair(ctxt, dict, ap) == -1) {
             goto out;
         }
@@ -442,8 +346,8 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
             goto out;
         }
 
-        while (!token_is_operator(token, '}')) {
-            if (!token_is_operator(token, ',')) {
+        while (token->type != JSON_RCURLY) {
+            if (token->type != JSON_COMMA) {
                 parse_error(ctxt, token, "expected separator in dict");
                 goto out;
             }
@@ -465,7 +369,6 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
     return QOBJECT(dict);
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     QDECREF(dict);
     return NULL;
 }
@@ -473,17 +376,10 @@ out:
 static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
 {
     QList *list = NULL;
-    QObject *token, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token, *peek;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (!token_is_operator(token, '[')) {
-        goto out;
-    }
+    assert(token && token->type == JSON_LSQUARE);
 
     list = qlist_new();
 
@@ -493,7 +389,7 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
         goto out;
     }
 
-    if (!token_is_operator(peek, ']')) {
+    if (peek->type != JSON_RSQUARE) {
         QObject *obj;
 
         obj = parse_value(ctxt, ap);
@@ -510,8 +406,8 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
             goto out;
         }
 
-        while (!token_is_operator(token, ']')) {
-            if (!token_is_operator(token, ',')) {
+        while (token->type != JSON_RSQUARE) {
+            if (token->type != JSON_COMMA) {
                 parse_error(ctxt, token, "expected separator in list");
                 goto out;
             }
@@ -537,99 +433,68 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
     return QOBJECT(list);
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     QDECREF(list);
     return NULL;
 }
 
 static QObject *parse_keyword(JSONParserContext *ctxt)
 {
-    QObject *token, *ret;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (token_get_type(token) != JSON_KEYWORD) {
-        goto out;
-    }
+    assert(token && token->type == JSON_KEYWORD);
 
-    if (token_is_keyword(token, "true")) {
-        ret = QOBJECT(qbool_from_bool(true));
-    } else if (token_is_keyword(token, "false")) {
-        ret = QOBJECT(qbool_from_bool(false));
-    } else if (token_is_keyword(token, "null")) {
-        ret = qnull();
-    } else {
-        parse_error(ctxt, token, "invalid keyword `%s'", token_get_value(token));
-        goto out;
+    if (!strcmp(token->str, "true")) {
+        return QOBJECT(qbool_from_bool(true));
+    } else if (!strcmp(token->str, "false")) {
+        return QOBJECT(qbool_from_bool(false));
+    } else if (!strcmp(token->str, "null")) {
+        return qnull();
     }
-
-    return ret;
-
-out: 
-    parser_context_restore(ctxt, saved_ctxt);
-
+    parse_error(ctxt, token, "invalid keyword '%s'", token->str);
     return NULL;
 }
 
 static QObject *parse_escape(JSONParserContext *ctxt, va_list *ap)
 {
-    QObject *token = NULL, *obj;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     if (ap == NULL) {
-        goto out;
+        return NULL;
     }
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (token_is_escape(token, "%p")) {
-        obj = va_arg(*ap, QObject *);
-    } else if (token_is_escape(token, "%i")) {
-        obj = QOBJECT(qbool_from_bool(va_arg(*ap, int)));
-    } else if (token_is_escape(token, "%d")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, int)));
-    } else if (token_is_escape(token, "%ld")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, long)));
-    } else if (token_is_escape(token, "%lld") ||
-               token_is_escape(token, "%I64d")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, long long)));
-    } else if (token_is_escape(token, "%s")) {
-        obj = QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
-    } else if (token_is_escape(token, "%f")) {
-        obj = QOBJECT(qfloat_from_double(va_arg(*ap, double)));
-    } else {
-        goto out;
+    assert(token && token->type == JSON_ESCAPE);
+
+    if (!strcmp(token->str, "%p")) {
+        return va_arg(*ap, QObject *);
+    } else if (!strcmp(token->str, "%i")) {
+        return QOBJECT(qbool_from_bool(va_arg(*ap, int)));
+    } else if (!strcmp(token->str, "%d")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, int)));
+    } else if (!strcmp(token->str, "%ld")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, long)));
+    } else if (!strcmp(token->str, "%lld") ||
+               !strcmp(token->str, "%I64d")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, long long)));
+    } else if (!strcmp(token->str, "%s")) {
+        return QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
+    } else if (!strcmp(token->str, "%f")) {
+        return QOBJECT(qfloat_from_double(va_arg(*ap, double)));
     }
-
-    return obj;
-
-out:
-    parser_context_restore(ctxt, saved_ctxt);
-
     return NULL;
 }
 
 static QObject *parse_literal(JSONParserContext *ctxt)
 {
-    QObject *token, *obj;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
+    assert(token);
 
-    switch (token_get_type(token)) {
+    switch (token->type) {
     case JSON_STRING:
-        obj = QOBJECT(qstring_from_escaped_str(ctxt, token));
-        break;
+        return QOBJECT(qstring_from_escaped_str(ctxt, token));
     case JSON_INTEGER: {
         /* A possibility exists that this is a whole-valued float where the
          * fractional part was left out due to being 0 (.0). It's not a big
@@ -646,56 +511,55 @@ static QObject *parse_literal(JSONParserContext *ctxt)
         int64_t value;
 
         errno = 0; /* strtoll doesn't set errno on success */
-        value = strtoll(token_get_value(token), NULL, 10);
+        value = strtoll(token->str, NULL, 10);
         if (errno != ERANGE) {
-            obj = QOBJECT(qint_from_int(value));
-            break;
+            return QOBJECT(qint_from_int(value));
         }
         /* fall through to JSON_FLOAT */
     }
     case JSON_FLOAT:
         /* FIXME dependent on locale */
-        obj = QOBJECT(qfloat_from_double(strtod(token_get_value(token), NULL)));
-        break;
+        return QOBJECT(qfloat_from_double(strtod(token->str, NULL)));
     default:
-        goto out;
+        abort();
     }
-
-    return obj;
-
-out:
-    parser_context_restore(ctxt, saved_ctxt);
-
-    return NULL;
 }
 
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
 {
-    QObject *obj;
+    JSONToken *token;
 
-    obj = parse_object(ctxt, ap);
-    if (obj == NULL) {
-        obj = parse_array(ctxt, ap);
-    }
-    if (obj == NULL) {
-        obj = parse_escape(ctxt, ap);
-    }
-    if (obj == NULL) {
-        obj = parse_keyword(ctxt);
-    } 
-    if (obj == NULL) {
-        obj = parse_literal(ctxt);
+    token = parser_context_peek_token(ctxt);
+    if (token == NULL) {
+        parse_error(ctxt, NULL, "premature EOI");
+        return NULL;
     }
 
-    return obj;
+    switch (token->type) {
+    case JSON_LCURLY:
+        return parse_object(ctxt, ap);
+    case JSON_LSQUARE:
+        return parse_array(ctxt, ap);
+    case JSON_ESCAPE:
+        return parse_escape(ctxt, ap);
+    case JSON_INTEGER:
+    case JSON_FLOAT:
+    case JSON_STRING:
+        return parse_literal(ctxt);
+    case JSON_KEYWORD:
+        return parse_keyword(ctxt);
+    default:
+        parse_error(ctxt, token, "expecting value");
+        return NULL;
+    }
 }
 
-QObject *json_parser_parse(QList *tokens, va_list *ap)
+QObject *json_parser_parse(GQueue *tokens, va_list *ap)
 {
     return json_parser_parse_err(tokens, ap, NULL);
 }
 
-QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp)
+QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp)
 {
     JSONParserContext *ctxt = parser_context_new(tokens);
     QObject *result;
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
index 1b2f9b1d1..a4db4b832 100644
--- a/qobject/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -11,50 +11,55 @@
  *
  */
 
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qint.h"
-#include "qapi/qmp/qdict.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
 #include "qapi/qmp/json-streamer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
+#define MAX_TOKEN_COUNT (2ULL << 20)
 #define MAX_NESTING (1ULL << 10)
 
-static void json_message_process_token(JSONLexer *lexer, QString *token, JSONTokenType type, int x, int y)
+static void json_message_free_tokens(JSONMessageParser *parser)
+{
+    if (parser->tokens) {
+        g_queue_free(parser->tokens);
+        parser->tokens = NULL;
+    }
+}
+
+static void json_message_process_token(JSONLexer *lexer, GString *input,
+                                       JSONTokenType type, int x, int y)
 {
     JSONMessageParser *parser = container_of(lexer, JSONMessageParser, lexer);
-    QDict *dict;
-
-    if (type == JSON_OPERATOR) {
-        switch (qstring_get_str(token)[0]) {
-        case '{':
-            parser->brace_count++;
-            break;
-        case '}':
-            parser->brace_count--;
-            break;
-        case '[':
-            parser->bracket_count++;
-            break;
-        case ']':
-            parser->bracket_count--;
-            break;
-        default:
-            break;
-        }
+    JSONToken *token;
+
+    switch (type) {
+    case JSON_LCURLY:
+        parser->brace_count++;
+        break;
+    case JSON_RCURLY:
+        parser->brace_count--;
+        break;
+    case JSON_LSQUARE:
+        parser->bracket_count++;
+        break;
+    case JSON_RSQUARE:
+        parser->bracket_count--;
+        break;
+    default:
+        break;
     }
 
-    dict = qdict_new();
-    qdict_put(dict, "type", qint_from_int(type));
-    QINCREF(token);
-    qdict_put(dict, "token", token);
-    qdict_put(dict, "x", qint_from_int(x));
-    qdict_put(dict, "y", qint_from_int(y));
+    token = g_malloc(sizeof(JSONToken) + input->len + 1);
+    token->type = type;
+    memcpy(token->str, input->str, input->len);
+    token->str[input->len] = 0;
+    token->x = x;
+    token->y = y;
 
-    parser->token_size += token->length;
+    parser->token_size += input->len;
 
-    qlist_append(parser->tokens, dict);
+    g_queue_push_tail(parser->tokens, token);
 
     if (type == JSON_ERROR) {
         goto out_emit_bad;
@@ -64,41 +69,39 @@ static void json_message_process_token(JSONLexer *lexer, QString *token, JSONTok
          parser->bracket_count == 0)) {
         goto out_emit;
     } else if (parser->token_size > MAX_TOKEN_SIZE ||
-               parser->bracket_count > MAX_NESTING ||
-               parser->brace_count > MAX_NESTING) {
+               g_queue_get_length(parser->tokens) > MAX_TOKEN_COUNT ||
+               parser->bracket_count + parser->brace_count > MAX_NESTING) {
         /* Security consideration, we limit total memory allocated per object
          * and the maximum recursion depth that a message can force.
          */
-        goto out_emit;
+        goto out_emit_bad;
     }
 
     return;
 
 out_emit_bad:
-    /* clear out token list and tell the parser to emit and error
+    /*
+     * Clear out token list and tell the parser to emit an error
      * indication by passing it a NULL list
      */
-    QDECREF(parser->tokens);
-    parser->tokens = NULL;
+    json_message_free_tokens(parser);
 out_emit:
     /* send current list of tokens to parser and reset tokenizer */
     parser->brace_count = 0;
     parser->bracket_count = 0;
+    /* parser->emit takes ownership of parser->tokens.  */
     parser->emit(parser, parser->tokens);
-    if (parser->tokens) {
-        QDECREF(parser->tokens);
-    }
-    parser->tokens = qlist_new();
+    parser->tokens = g_queue_new();
     parser->token_size = 0;
 }
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, QList *))
+                              void (*func)(JSONMessageParser *, GQueue *))
 {
     parser->emit = func;
     parser->brace_count = 0;
     parser->bracket_count = 0;
-    parser->tokens = qlist_new();
+    parser->tokens = g_queue_new();
     parser->token_size = 0;
 
     json_lexer_init(&parser->lexer, json_message_process_token);
@@ -118,5 +121,5 @@ int json_message_parser_flush(JSONMessageParser *parser)
 void json_message_parser_destroy(JSONMessageParser *parser)
 {
     json_lexer_destroy(&parser->lexer);
-    QDECREF(parser->tokens);
+    json_message_free_tokens(parser);
 }
diff --git a/qobject/qbool.c b/qobject/qbool.c
index 5ff69f0b2..bc6535fa4 100644
--- a/qobject/qbool.c
+++ b/qobject/qbool.c
@@ -51,9 +51,9 @@ bool qbool_get_bool(const QBool *qb)
  */
 QBool *qobject_to_qbool(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QBOOL)
+    if (!obj || qobject_type(obj) != QTYPE_QBOOL) {
         return NULL;
-
+    }
     return container_of(obj, QBool, base);
 }
 
diff --git a/qobject/qdict.c b/qobject/qdict.c
index 67b1a58ab..2d67bf157 100644
--- a/qobject/qdict.c
+++ b/qobject/qdict.c
@@ -46,9 +46,9 @@ QDict *qdict_new(void)
  */
 QDict *qobject_to_qdict(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QDICT)
+    if (!obj || qobject_type(obj) != QTYPE_QDICT) {
         return NULL;
-
+    }
     return container_of(obj, QDict, base);
 }
 
@@ -229,8 +229,7 @@ double qdict_get_double(const QDict *qdict, const char *key)
  */
 int64_t qdict_get_int(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QINT);
-    return qint_get_int(qobject_to_qint(obj));
+    return qint_get_int(qobject_to_qint(qdict_get(qdict, key)));
 }
 
 /**
@@ -243,8 +242,7 @@ int64_t qdict_get_int(const QDict *qdict, const char *key)
  */
 bool qdict_get_bool(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QBOOL);
-    return qbool_get_bool(qobject_to_qbool(obj));
+    return qbool_get_bool(qobject_to_qbool(qdict_get(qdict, key)));
 }
 
 /**
@@ -270,7 +268,7 @@ QList *qdict_get_qlist(const QDict *qdict, const char *key)
  */
 QDict *qdict_get_qdict(const QDict *qdict, const char *key)
 {
-    return qobject_to_qdict(qdict_get_obj(qdict, key, QTYPE_QDICT));
+    return qobject_to_qdict(qdict_get(qdict, key));
 }
 
 /**
@@ -284,8 +282,7 @@ QDict *qdict_get_qdict(const QDict *qdict, const char *key)
  */
 const char *qdict_get_str(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QSTRING);
-    return qstring_get_str(qobject_to_qstring(obj));
+    return qstring_get_str(qobject_to_qstring(qdict_get(qdict, key)));
 }
 
 /**
@@ -298,13 +295,9 @@ const char *qdict_get_str(const QDict *qdict, const char *key)
 int64_t qdict_get_try_int(const QDict *qdict, const char *key,
                           int64_t def_value)
 {
-    QObject *obj;
+    QInt *qint = qobject_to_qint(qdict_get(qdict, key));
 
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QINT)
-        return def_value;
-
-    return qint_get_int(qobject_to_qint(obj));
+    return qint ? qint_get_int(qint) : def_value;
 }
 
 /**
@@ -316,13 +309,9 @@ int64_t qdict_get_try_int(const QDict *qdict, const char *key,
  */
 bool qdict_get_try_bool(const QDict *qdict, const char *key, bool def_value)
 {
-    QObject *obj;
+    QBool *qbool = qobject_to_qbool(qdict_get(qdict, key));
 
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QBOOL)
-        return def_value;
-
-    return qbool_get_bool(qobject_to_qbool(obj));
+    return qbool ? qbool_get_bool(qbool) : def_value;
 }
 
 /**
@@ -335,13 +324,9 @@ bool qdict_get_try_bool(const QDict *qdict, const char *key, bool def_value)
  */
 const char *qdict_get_try_str(const QDict *qdict, const char *key)
 {
-    QObject *obj;
-
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QSTRING)
-        return NULL;
+    QString *qstr = qobject_to_qstring(qdict_get(qdict, key));
 
-    return qstring_get_str(qobject_to_qstring(obj));
+    return qstr ? qstring_get_str(qstr) : NULL;
 }
 
 /**
diff --git a/qobject/qfloat.c b/qobject/qfloat.c
index 7de0992db..c86516327 100644
--- a/qobject/qfloat.c
+++ b/qobject/qfloat.c
@@ -51,9 +51,9 @@ double qfloat_get_double(const QFloat *qf)
  */
 QFloat *qobject_to_qfloat(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QFLOAT)
+    if (!obj || qobject_type(obj) != QTYPE_QFLOAT) {
         return NULL;
-
+    }
     return container_of(obj, QFloat, base);
 }
 
diff --git a/qobject/qint.c b/qobject/qint.c
index 86b9b04f0..999688e9c 100644
--- a/qobject/qint.c
+++ b/qobject/qint.c
@@ -50,9 +50,9 @@ int64_t qint_get_int(const QInt *qi)
  */
 QInt *qobject_to_qint(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QINT)
+    if (!obj || qobject_type(obj) != QTYPE_QINT) {
         return NULL;
-
+    }
     return container_of(obj, QInt, base);
 }
 
diff --git a/qobject/qjson.c b/qobject/qjson.c
index 33f8ef530..a3e6a7ca4 100644
--- a/qobject/qjson.c
+++ b/qobject/qjson.c
@@ -28,7 +28,7 @@ typedef struct JSONParsingState
     QObject *result;
 } JSONParsingState;
 
-static void parse_json(JSONMessageParser *parser, QList *tokens)
+static void parse_json(JSONMessageParser *parser, GQueue *tokens)
 {
     JSONParsingState *s = container_of(parser, JSONParsingState, parser);
     s->result = json_parser_parse(tokens, s->ap);
diff --git a/qobject/qlist.c b/qobject/qlist.c
index 1ced0de58..298003aaf 100644
--- a/qobject/qlist.c
+++ b/qobject/qlist.c
@@ -142,10 +142,9 @@ size_t qlist_size(const QList *qlist)
  */
 QList *qobject_to_qlist(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QLIST) {
+    if (!obj || qobject_type(obj) != QTYPE_QLIST) {
         return NULL;
     }
-
     return container_of(obj, QList, base);
 }
 
diff --git a/qobject/qstring.c b/qobject/qstring.c
index 607b7a142..cb72dfbfc 100644
--- a/qobject/qstring.c
+++ b/qobject/qstring.c
@@ -117,9 +117,9 @@ void qstring_append_chr(QString *qstring, int c)
  */
 QString *qobject_to_qstring(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QSTRING)
+    if (!obj || qobject_type(obj) != QTYPE_QSTRING) {
         return NULL;
-
+    }
     return container_of(obj, QString, base);
 }
 
diff --git a/qom/Makefile.objs b/qom/Makefile.objs
index 985003bd0..516349eec 100644
--- a/qom/Makefile.objs
+++ b/qom/Makefile.objs
@@ -1,3 +1,4 @@
-common-obj-y = object.o container.o qom-qobject.o
-common-obj-y += cpu.o
-common-obj-y += object_interfaces.o
+qom-obj-y = object.o container.o qom-qobject.o
+qom-obj-y += object_interfaces.o
+
+common-obj-y = cpu.o
diff --git a/qom/cpu.c b/qom/cpu.c
index eb9cfeca1..fb80d13a3 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -114,6 +114,8 @@ void cpu_reset_interrupt(CPUState *cpu, int mask)
 void cpu_exit(CPUState *cpu)
 {
     cpu->exit_request = 1;
+    /* Ensure cpu_exec will see the exit request after TCG has exited.  */
+    smp_wmb();
     cpu->tcg_exit_req = 1;
 }
 
@@ -247,8 +249,9 @@ static void cpu_common_reset(CPUState *cpu)
     cpu->mem_io_vaddr = 0;
     cpu->icount_extra = 0;
     cpu->icount_decr.u32 = 0;
-    cpu->can_do_io = 0;
+    cpu->can_do_io = 1;
     cpu->exception_index = -1;
+    cpu->crash_occurred = false;
     memset(cpu->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
 }
 
@@ -314,6 +317,7 @@ static void cpu_common_initfn(Object *obj)
 
     cpu->cpu_index = -1;
     cpu->gdb_num_regs = cpu->gdb_num_g_regs = cc->gdb_num_core_regs;
+    qemu_mutex_init(&cpu->work_mutex);
     QTAILQ_INIT(&cpu->breakpoints);
     QTAILQ_INIT(&cpu->watchpoints);
 }
diff --git a/qom/object.c b/qom/object.c
index e14b9d866..d7515697a 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -67,6 +67,10 @@ struct TypeImpl
     InterfaceImpl interfaces[MAX_INTERFACES];
 };
 
+struct ObjectPropertyIterator {
+    GHashTableIter iter;
+};
+
 static Type type_interface;
 
 static GHashTable *type_table_get(void)
@@ -204,7 +208,7 @@ static bool type_is_ancestor(TypeImpl *type, TypeImpl *target_type)
 {
     assert(target_type);
 
-    /* Check if typename is a direct ancestor of type */
+    /* Check if target_type is a direct ancestor of type */
     while (type) {
         if (type == target_type) {
             return true;
@@ -261,7 +265,7 @@ static void type_initialize(TypeImpl *ti)
         GSList *e;
         int i;
 
-        g_assert(parent->class_size <= ti->class_size);
+        g_assert_cmpint(parent->class_size, <=, ti->class_size);
         memcpy(ti->class, parent->class, parent->class_size);
         ti->class->interfaces = NULL;
 
@@ -326,6 +330,16 @@ static void object_post_init_with_type(Object *obj, TypeImpl *ti)
     }
 }
 
+static void object_property_free(gpointer data)
+{
+    ObjectProperty *prop = data;
+
+    g_free(prop->name);
+    g_free(prop->type);
+    g_free(prop->description);
+    g_free(prop);
+}
+
 void object_initialize_with_type(void *data, size_t size, TypeImpl *type)
 {
     Object *obj = data;
@@ -333,14 +347,15 @@ void object_initialize_with_type(void *data, size_t size, TypeImpl *type)
     g_assert(type != NULL);
     type_initialize(type);
 
-    g_assert(type->instance_size >= sizeof(Object));
+    g_assert_cmpint(type->instance_size, >=, sizeof(Object));
     g_assert(type->abstract == false);
-    g_assert(size >= type->instance_size);
+    g_assert_cmpint(size, >=, type->instance_size);
 
     memset(obj, 0, type->instance_size);
     obj->class = type->class;
     object_ref(obj);
-    QTAILQ_INIT(&obj->properties);
+    obj->properties = g_hash_table_new_full(g_str_hash, g_str_equal,
+                                            NULL, object_property_free);
     object_init_with_type(obj, type);
     object_post_init_with_type(obj, type);
 }
@@ -359,29 +374,51 @@ static inline bool object_property_is_child(ObjectProperty *prop)
 
 static void object_property_del_all(Object *obj)
 {
-    while (!QTAILQ_EMPTY(&obj->properties)) {
-        ObjectProperty *prop = QTAILQ_FIRST(&obj->properties);
-
-        QTAILQ_REMOVE(&obj->properties, prop, node);
-
-        if (prop->release) {
-            prop->release(obj, prop->name, prop->opaque);
+    ObjectProperty *prop;
+    GHashTableIter iter;
+    gpointer key, value;
+    bool released;
+
+    do {
+        released = false;
+        g_hash_table_iter_init(&iter, obj->properties);
+        while (g_hash_table_iter_next(&iter, &key, &value)) {
+            prop = value;
+            if (prop->release) {
+                prop->release(obj, prop->name, prop->opaque);
+                prop->release = NULL;
+                released = true;
+                break;
+            }
+            g_hash_table_iter_remove(&iter);
         }
+    } while (released);
 
-        g_free(prop->name);
-        g_free(prop->type);
-        g_free(prop->description);
-        g_free(prop);
-    }
+    g_hash_table_unref(obj->properties);
 }
 
 static void object_property_del_child(Object *obj, Object *child, Error **errp)
 {
     ObjectProperty *prop;
+    GHashTableIter iter;
+    gpointer key, value;
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        prop = value;
         if (object_property_is_child(prop) && prop->opaque == child) {
-            object_property_del(obj, prop->name, errp);
+            if (prop->release) {
+                prop->release(obj, prop->name, prop->opaque);
+                prop->release = NULL;
+            }
+            break;
+        }
+    }
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        prop = value;
+        if (object_property_is_child(prop) && prop->opaque == child) {
+            g_hash_table_iter_remove(&iter);
             break;
         }
     }
@@ -413,7 +450,7 @@ static void object_finalize(void *data)
     object_property_del_all(obj);
     object_deinit(obj, ti);
 
-    g_assert(obj->ref == 0);
+    g_assert_cmpint(obj->ref, ==, 0);
     if (obj->free) {
         obj->free(obj);
     }
@@ -775,23 +812,44 @@ void object_class_foreach(void (*fn)(ObjectClass *klass, void *opaque),
     enumerating_types = false;
 }
 
-int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
-                         void *opaque)
+static int do_object_child_foreach(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque, bool recurse)
 {
-    ObjectProperty *prop, *next;
+    GHashTableIter iter;
+    ObjectProperty *prop;
     int ret = 0;
 
-    QTAILQ_FOREACH_SAFE(prop, &obj->properties, node, next) {
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         if (object_property_is_child(prop)) {
-            ret = fn(prop->opaque, opaque);
+            Object *child = prop->opaque;
+
+            ret = fn(child, opaque);
             if (ret != 0) {
                 break;
             }
+            if (recurse) {
+                do_object_child_foreach(child, fn, opaque, true);
+            }
         }
     }
     return ret;
 }
 
+int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
+                         void *opaque)
+{
+    return do_object_child_foreach(obj, fn, opaque, false);
+}
+
+int object_child_foreach_recursive(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque)
+{
+    return do_object_child_foreach(obj, fn, opaque, true);
+}
+
 static void object_class_get_list_tramp(ObjectClass *klass, void *opaque)
 {
     GSList **list = opaque;
@@ -814,7 +872,7 @@ void object_ref(Object *obj)
     if (!obj) {
         return;
     }
-     atomic_inc(&obj->ref);
+    atomic_inc(&obj->ref);
 }
 
 void object_unref(Object *obj)
@@ -822,7 +880,7 @@ void object_unref(Object *obj)
     if (!obj) {
         return;
     }
-    g_assert(obj->ref > 0);
+    g_assert_cmpint(obj->ref, >, 0);
 
     /* parent always holds a reference to its children */
     if (atomic_fetch_dec(&obj->ref) == 1) {
@@ -860,13 +918,11 @@ object_property_add(Object *obj, const char *name, const char *type,
         return ret;
     }
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
-        if (strcmp(prop->name, name) == 0) {
-            error_setg(errp, "attempt to add duplicate property '%s'"
+    if (g_hash_table_lookup(obj->properties, name) != NULL) {
+        error_setg(errp, "attempt to add duplicate property '%s'"
                        " to object (type '%s')", name,
                        object_get_typename(obj));
-            return NULL;
-        }
+        return NULL;
     }
 
     prop = g_malloc0(sizeof(*prop));
@@ -879,7 +935,7 @@ object_property_add(Object *obj, const char *name, const char *type,
     prop->release = release;
     prop->opaque = opaque;
 
-    QTAILQ_INSERT_TAIL(&obj->properties, prop, node);
+    g_hash_table_insert(obj->properties, prop->name, prop);
     return prop;
 }
 
@@ -888,33 +944,52 @@ ObjectProperty *object_property_find(Object *obj, const char *name,
 {
     ObjectProperty *prop;
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
-        if (strcmp(prop->name, name) == 0) {
-            return prop;
-        }
+    prop = g_hash_table_lookup(obj->properties, name);
+    if (prop) {
+        return prop;
     }
 
     error_setg(errp, "Property '.%s' not found", name);
     return NULL;
 }
 
+ObjectPropertyIterator *object_property_iter_init(Object *obj)
+{
+    ObjectPropertyIterator *ret = g_new0(ObjectPropertyIterator, 1);
+    g_hash_table_iter_init(&ret->iter, obj->properties);
+    return ret;
+}
+
+void object_property_iter_free(ObjectPropertyIterator *iter)
+{
+    if (!iter) {
+        return;
+    }
+    g_free(iter);
+}
+
+ObjectProperty *object_property_iter_next(ObjectPropertyIterator *iter)
+{
+    gpointer key, val;
+    if (!g_hash_table_iter_next(&iter->iter, &key, &val)) {
+        return NULL;
+    }
+    return val;
+}
+
 void object_property_del(Object *obj, const char *name, Error **errp)
 {
-    ObjectProperty *prop = object_property_find(obj, name, errp);
-    if (prop == NULL) {
+    ObjectProperty *prop = g_hash_table_lookup(obj->properties, name);
+
+    if (!prop) {
+        error_setg(errp, "Property '.%s' not found", name);
         return;
     }
 
     if (prop->release) {
         prop->release(obj, name, prop->opaque);
     }
-
-    QTAILQ_REMOVE(&obj->properties, prop, node);
-
-    g_free(prop->name);
-    g_free(prop->type);
-    g_free(prop->description);
-    g_free(prop);
+    g_hash_table_remove(obj->properties, name);
 }
 
 void object_property_get(Object *obj, Visitor *v, const char *name,
@@ -1148,31 +1223,31 @@ out:
 void object_property_parse(Object *obj, const char *string,
                            const char *name, Error **errp)
 {
-    StringInputVisitor *mi;
-    mi = string_input_visitor_new(string);
-    object_property_set(obj, string_input_get_visitor(mi), name, errp);
+    StringInputVisitor *siv;
+    siv = string_input_visitor_new(string);
+    object_property_set(obj, string_input_get_visitor(siv), name, errp);
 
-    string_input_visitor_cleanup(mi);
+    string_input_visitor_cleanup(siv);
 }
 
 char *object_property_print(Object *obj, const char *name, bool human,
                             Error **errp)
 {
-    StringOutputVisitor *mo;
+    StringOutputVisitor *sov;
     char *string = NULL;
     Error *local_err = NULL;
 
-    mo = string_output_visitor_new(human);
-    object_property_get(obj, string_output_get_visitor(mo), name, &local_err);
+    sov = string_output_visitor_new(human);
+    object_property_get(obj, string_output_get_visitor(sov), name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto out;
     }
 
-    string = string_output_get_string(mo);
+    string = string_output_get_string(sov);
 
 out:
-    string_output_visitor_cleanup(mo);
+    string_output_visitor_cleanup(sov);
     return string;
 }
 
@@ -1311,8 +1386,8 @@ static Object *object_resolve_link(Object *obj, const char *name,
     target = object_resolve_path_type(path, target_type, &ambiguous);
 
     if (ambiguous) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "Path '%s' does not uniquely identify an object", path);
+        error_setg(errp, "Path '%s' does not uniquely identify an object",
+                   path);
     } else if (!target) {
         target = object_resolve_path(path, &ambiguous);
         if (target || ambiguous) {
@@ -1434,11 +1509,13 @@ void object_property_add_const_link(Object *obj, const char *name,
 gchar *object_get_canonical_path_component(Object *obj)
 {
     ObjectProperty *prop = NULL;
+    GHashTableIter iter;
 
     g_assert(obj);
     g_assert(obj->parent != NULL);
 
-    QTAILQ_FOREACH(prop, &obj->parent->properties, node) {
+    g_hash_table_iter_init(&iter, obj->parent->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         if (!object_property_is_child(prop)) {
             continue;
         }
@@ -1522,11 +1599,13 @@ static Object *object_resolve_partial_path(Object *parent,
                                               bool *ambiguous)
 {
     Object *obj;
+    GHashTableIter iter;
     ObjectProperty *prop;
 
     obj = object_resolve_abs_path(parent, parts, typename, 0);
 
-    QTAILQ_FOREACH(prop, &parent->properties, node) {
+    g_hash_table_iter_init(&iter, parent->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         Object *found;
 
         if (!object_property_is_child(prop)) {
diff --git a/qom/qom-qobject.c b/qom/qom-qobject.c
index 6384b8e98..964989065 100644
--- a/qom/qom-qobject.c
+++ b/qom/qom-qobject.c
@@ -19,11 +19,11 @@
 void object_property_set_qobject(Object *obj, QObject *value,
                                  const char *name, Error **errp)
 {
-    QmpInputVisitor *mi;
-    mi = qmp_input_visitor_new(value);
-    object_property_set(obj, qmp_input_get_visitor(mi), name, errp);
+    QmpInputVisitor *qiv;
+    qiv = qmp_input_visitor_new(value);
+    object_property_set(obj, qmp_input_get_visitor(qiv), name, errp);
 
-    qmp_input_visitor_cleanup(mi);
+    qmp_input_visitor_cleanup(qiv);
 }
 
 QObject *object_property_get_qobject(Object *obj, const char *name,
@@ -31,14 +31,14 @@ QObject *object_property_get_qobject(Object *obj, const char *name,
 {
     QObject *ret = NULL;
     Error *local_err = NULL;
-    QmpOutputVisitor *mo;
+    QmpOutputVisitor *qov;
 
-    mo = qmp_output_visitor_new();
-    object_property_get(obj, qmp_output_get_visitor(mo), name, &local_err);
+    qov = qmp_output_visitor_new();
+    object_property_get(obj, qmp_output_get_visitor(qov), name, &local_err);
     if (!local_err) {
-        ret = qmp_output_get_qobject(mo);
+        ret = qmp_output_get_qobject(qov);
     }
     error_propagate(errp, local_err);
-    qmp_output_visitor_cleanup(mo);
+    qmp_output_visitor_cleanup(qov);
     return ret;
 }
diff --git a/replay/Makefile.objs b/replay/Makefile.objs
new file mode 100644
index 000000000..232193a24
--- /dev/null
+++ b/replay/Makefile.objs
@@ -0,0 +1,5 @@
+common-obj-y += replay.o
+common-obj-y += replay-internal.o
+common-obj-y += replay-events.o
+common-obj-y += replay-time.o
+common-obj-y += replay-input.o
diff --git a/replay/replay-events.c b/replay/replay-events.c
new file mode 100644
index 000000000..402f64406
--- /dev/null
+++ b/replay/replay-events.c
@@ -0,0 +1,279 @@
+/*
+ * replay-events.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "block/aio.h"
+#include "ui/input.h"
+
+typedef struct Event {
+    ReplayAsyncEventKind event_kind;
+    void *opaque;
+    void *opaque2;
+    uint64_t id;
+
+    QTAILQ_ENTRY(Event) events;
+} Event;
+
+static QTAILQ_HEAD(, Event) events_list = QTAILQ_HEAD_INITIALIZER(events_list);
+static unsigned int read_event_kind = -1;
+static uint64_t read_id = -1;
+static int read_checkpoint = -1;
+
+static bool events_enabled;
+
+/* Functions */
+
+static void replay_run_event(Event *event)
+{
+    switch (event->event_kind) {
+    case REPLAY_ASYNC_EVENT_BH:
+        aio_bh_call(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT:
+        qemu_input_event_send_impl(NULL, (InputEvent *)event->opaque);
+        qapi_free_InputEvent((InputEvent *)event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+        qemu_input_event_sync_impl();
+        break;
+    default:
+        error_report("Replay: invalid async event ID (%d) in the queue",
+                    event->event_kind);
+        exit(1);
+        break;
+    }
+}
+
+void replay_enable_events(void)
+{
+    events_enabled = true;
+}
+
+bool replay_has_events(void)
+{
+    return !QTAILQ_EMPTY(&events_list);
+}
+
+void replay_flush_events(void)
+{
+    replay_mutex_lock();
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+        QTAILQ_REMOVE(&events_list, event, events);
+        g_free(event);
+    }
+    replay_mutex_unlock();
+}
+
+void replay_disable_events(void)
+{
+    if (replay_mode != REPLAY_MODE_NONE) {
+        events_enabled = false;
+        /* Flush events queue before waiting of completion */
+        replay_flush_events();
+    }
+}
+
+void replay_clear_events(void)
+{
+    replay_mutex_lock();
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        QTAILQ_REMOVE(&events_list, event, events);
+
+        g_free(event);
+    }
+    replay_mutex_unlock();
+}
+
+/*! Adds specified async event to the queue */
+static void replay_add_event(ReplayAsyncEventKind event_kind,
+                             void *opaque,
+                             void *opaque2, uint64_t id)
+{
+    assert(event_kind < REPLAY_ASYNC_COUNT);
+
+    if (!replay_file || replay_mode == REPLAY_MODE_NONE
+        || !events_enabled) {
+        Event e;
+        e.event_kind = event_kind;
+        e.opaque = opaque;
+        e.opaque2 = opaque2;
+        e.id = id;
+        replay_run_event(&e);
+        return;
+    }
+
+    Event *event = g_malloc0(sizeof(Event));
+    event->event_kind = event_kind;
+    event->opaque = opaque;
+    event->opaque2 = opaque2;
+    event->id = id;
+
+    replay_mutex_lock();
+    QTAILQ_INSERT_TAIL(&events_list, event, events);
+    replay_mutex_unlock();
+}
+
+void replay_bh_schedule_event(QEMUBH *bh)
+{
+    if (replay_mode != REPLAY_MODE_NONE) {
+        uint64_t id = replay_get_current_step();
+        replay_add_event(REPLAY_ASYNC_EVENT_BH, bh, NULL, id);
+    } else {
+        qemu_bh_schedule(bh);
+    }
+}
+
+void replay_add_input_event(struct InputEvent *event)
+{
+    replay_add_event(REPLAY_ASYNC_EVENT_INPUT, event, NULL, 0);
+}
+
+void replay_add_input_sync_event(void)
+{
+    replay_add_event(REPLAY_ASYNC_EVENT_INPUT_SYNC, NULL, NULL, 0);
+}
+
+static void replay_save_event(Event *event, int checkpoint)
+{
+    if (replay_mode != REPLAY_MODE_PLAY) {
+        /* put the event into the file */
+        replay_put_event(EVENT_ASYNC);
+        replay_put_byte(checkpoint);
+        replay_put_byte(event->event_kind);
+
+        /* save event-specific data */
+        switch (event->event_kind) {
+        case REPLAY_ASYNC_EVENT_BH:
+            replay_put_qword(event->id);
+            break;
+        case REPLAY_ASYNC_EVENT_INPUT:
+            replay_save_input_event(event->opaque);
+            break;
+        case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+            break;
+        default:
+            error_report("Unknown ID %d of replay event", read_event_kind);
+            exit(1);
+        }
+    }
+}
+
+/* Called with replay mutex locked */
+void replay_save_events(int checkpoint)
+{
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        replay_save_event(event, checkpoint);
+
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+        QTAILQ_REMOVE(&events_list, event, events);
+        g_free(event);
+    }
+}
+
+static Event *replay_read_event(int checkpoint)
+{
+    Event *event;
+    if (read_event_kind == -1) {
+        read_checkpoint = replay_get_byte();
+        read_event_kind = replay_get_byte();
+        read_id = -1;
+        replay_check_error();
+    }
+
+    if (checkpoint != read_checkpoint) {
+        return NULL;
+    }
+
+    /* Events that has not to be in the queue */
+    switch (read_event_kind) {
+    case REPLAY_ASYNC_EVENT_BH:
+        if (read_id == -1) {
+            read_id = replay_get_qword();
+        }
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = replay_read_input_event();
+        return event;
+    case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = 0;
+        return event;
+    default:
+        error_report("Unknown ID %d of replay event", read_event_kind);
+        exit(1);
+        break;
+    }
+
+    QTAILQ_FOREACH(event, &events_list, events) {
+        if (event->event_kind == read_event_kind
+            && (read_id == -1 || read_id == event->id)) {
+            break;
+        }
+    }
+
+    if (event) {
+        QTAILQ_REMOVE(&events_list, event, events);
+    } else {
+        return NULL;
+    }
+
+    /* Read event-specific data */
+
+    return event;
+}
+
+/* Called with replay mutex locked */
+void replay_read_events(int checkpoint)
+{
+    while (replay_data_kind == EVENT_ASYNC) {
+        Event *event = replay_read_event(checkpoint);
+        if (!event) {
+            break;
+        }
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+
+        g_free(event);
+        replay_finish_event();
+        read_event_kind = -1;
+    }
+}
+
+void replay_init_events(void)
+{
+    read_event_kind = -1;
+}
+
+void replay_finish_events(void)
+{
+    events_enabled = false;
+    replay_clear_events();
+}
+
+bool replay_events_enabled(void)
+{
+    return events_enabled;
+}
diff --git a/replay/replay-input.c b/replay/replay-input.c
new file mode 100644
index 000000000..98798955a
--- /dev/null
+++ b/replay/replay-input.c
@@ -0,0 +1,160 @@
+/*
+ * replay-input.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/notify.h"
+#include "ui/input.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi-visit.h"
+
+static InputEvent *qapi_clone_InputEvent(InputEvent *src)
+{
+    QmpOutputVisitor *qov;
+    QmpInputVisitor *qiv;
+    Visitor *ov, *iv;
+    QObject *obj;
+    InputEvent *dst = NULL;
+
+    qov = qmp_output_visitor_new();
+    ov = qmp_output_get_visitor(qov);
+    visit_type_InputEvent(ov, &src, NULL, &error_abort);
+    obj = qmp_output_get_qobject(qov);
+    qmp_output_visitor_cleanup(qov);
+    if (!obj) {
+        return NULL;
+    }
+
+    qiv = qmp_input_visitor_new(obj);
+    iv = qmp_input_get_visitor(qiv);
+    visit_type_InputEvent(iv, &dst, NULL, &error_abort);
+    qmp_input_visitor_cleanup(qiv);
+    qobject_decref(obj);
+
+    return dst;
+}
+
+void replay_save_input_event(InputEvent *evt)
+{
+    replay_put_dword(evt->type);
+
+    switch (evt->type) {
+    case INPUT_EVENT_KIND_KEY:
+        replay_put_dword(evt->u.key->key->type);
+
+        switch (evt->u.key->key->type) {
+        case KEY_VALUE_KIND_NUMBER:
+            replay_put_qword(evt->u.key->key->u.number);
+            replay_put_byte(evt->u.key->down);
+            break;
+        case KEY_VALUE_KIND_QCODE:
+            replay_put_dword(evt->u.key->key->u.qcode);
+            replay_put_byte(evt->u.key->down);
+            break;
+        case KEY_VALUE_KIND_MAX:
+            /* keep gcc happy */
+            break;
+        }
+        break;
+    case INPUT_EVENT_KIND_BTN:
+        replay_put_dword(evt->u.btn->button);
+        replay_put_byte(evt->u.btn->down);
+        break;
+    case INPUT_EVENT_KIND_REL:
+        replay_put_dword(evt->u.rel->axis);
+        replay_put_qword(evt->u.rel->value);
+        break;
+    case INPUT_EVENT_KIND_ABS:
+        replay_put_dword(evt->u.abs->axis);
+        replay_put_qword(evt->u.abs->value);
+        break;
+    case INPUT_EVENT_KIND_MAX:
+        /* keep gcc happy */
+        break;
+    }
+}
+
+InputEvent *replay_read_input_event(void)
+{
+    InputEvent evt;
+    KeyValue keyValue;
+    InputKeyEvent key;
+    key.key = &keyValue;
+    InputBtnEvent btn;
+    InputMoveEvent rel;
+    InputMoveEvent abs;
+
+    evt.type = replay_get_dword();
+    switch (evt.type) {
+    case INPUT_EVENT_KIND_KEY:
+        evt.u.key = &key;
+        evt.u.key->key->type = replay_get_dword();
+
+        switch (evt.u.key->key->type) {
+        case KEY_VALUE_KIND_NUMBER:
+            evt.u.key->key->u.number = replay_get_qword();
+            evt.u.key->down = replay_get_byte();
+            break;
+        case KEY_VALUE_KIND_QCODE:
+            evt.u.key->key->u.qcode = (QKeyCode)replay_get_dword();
+            evt.u.key->down = replay_get_byte();
+            break;
+        case KEY_VALUE_KIND_MAX:
+            /* keep gcc happy */
+            break;
+        }
+        break;
+    case INPUT_EVENT_KIND_BTN:
+        evt.u.btn = &btn;
+        evt.u.btn->button = (InputButton)replay_get_dword();
+        evt.u.btn->down = replay_get_byte();
+        break;
+    case INPUT_EVENT_KIND_REL:
+        evt.u.rel = &rel;
+        evt.u.rel->axis = (InputAxis)replay_get_dword();
+        evt.u.rel->value = replay_get_qword();
+        break;
+    case INPUT_EVENT_KIND_ABS:
+        evt.u.abs = &abs;
+        evt.u.abs->axis = (InputAxis)replay_get_dword();
+        evt.u.abs->value = replay_get_qword();
+        break;
+    case INPUT_EVENT_KIND_MAX:
+        /* keep gcc happy */
+        break;
+    }
+
+    return qapi_clone_InputEvent(&evt);
+}
+
+void replay_input_event(QemuConsole *src, InputEvent *evt)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        /* Nothing */
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_add_input_event(qapi_clone_InputEvent(evt));
+    } else {
+        qemu_input_event_send_impl(src, evt);
+    }
+}
+
+void replay_input_sync_event(void)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        /* Nothing */
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_add_input_sync_event();
+    } else {
+        qemu_input_event_sync_impl();
+    }
+}
diff --git a/replay/replay-internal.c b/replay/replay-internal.c
new file mode 100644
index 000000000..35cff44a3
--- /dev/null
+++ b/replay/replay-internal.c
@@ -0,0 +1,206 @@
+/*
+ * replay-internal.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+
+unsigned int replay_data_kind = -1;
+static unsigned int replay_has_unread_data;
+
+/* Mutex to protect reading and writing events to the log.
+   replay_data_kind and replay_has_unread_data are also protected
+   by this mutex.
+   It also protects replay events queue which stores events to be
+   written or read to the log. */
+static QemuMutex lock;
+
+/* File for replay writing */
+FILE *replay_file;
+
+void replay_put_byte(uint8_t byte)
+{
+    if (replay_file) {
+        putc(byte, replay_file);
+    }
+}
+
+void replay_put_event(uint8_t event)
+{
+    assert(event < EVENT_COUNT);
+    replay_put_byte(event);
+}
+
+
+void replay_put_word(uint16_t word)
+{
+    replay_put_byte(word >> 8);
+    replay_put_byte(word);
+}
+
+void replay_put_dword(uint32_t dword)
+{
+    replay_put_word(dword >> 16);
+    replay_put_word(dword);
+}
+
+void replay_put_qword(int64_t qword)
+{
+    replay_put_dword(qword >> 32);
+    replay_put_dword(qword);
+}
+
+void replay_put_array(const uint8_t *buf, size_t size)
+{
+    if (replay_file) {
+        replay_put_dword(size);
+        fwrite(buf, 1, size, replay_file);
+    }
+}
+
+uint8_t replay_get_byte(void)
+{
+    uint8_t byte = 0;
+    if (replay_file) {
+        byte = getc(replay_file);
+    }
+    return byte;
+}
+
+uint16_t replay_get_word(void)
+{
+    uint16_t word = 0;
+    if (replay_file) {
+        word = replay_get_byte();
+        word = (word << 8) + replay_get_byte();
+    }
+
+    return word;
+}
+
+uint32_t replay_get_dword(void)
+{
+    uint32_t dword = 0;
+    if (replay_file) {
+        dword = replay_get_word();
+        dword = (dword << 16) + replay_get_word();
+    }
+
+    return dword;
+}
+
+int64_t replay_get_qword(void)
+{
+    int64_t qword = 0;
+    if (replay_file) {
+        qword = replay_get_dword();
+        qword = (qword << 32) + replay_get_dword();
+    }
+
+    return qword;
+}
+
+void replay_get_array(uint8_t *buf, size_t *size)
+{
+    if (replay_file) {
+        *size = replay_get_dword();
+        if (fread(buf, 1, *size, replay_file) != *size) {
+            error_report("replay read error");
+        }
+    }
+}
+
+void replay_get_array_alloc(uint8_t **buf, size_t *size)
+{
+    if (replay_file) {
+        *size = replay_get_dword();
+        *buf = g_malloc(*size);
+        if (fread(*buf, 1, *size, replay_file) != *size) {
+            error_report("replay read error");
+        }
+    }
+}
+
+void replay_check_error(void)
+{
+    if (replay_file) {
+        if (feof(replay_file)) {
+            error_report("replay file is over");
+            qemu_system_vmstop_request_prepare();
+            qemu_system_vmstop_request(RUN_STATE_PAUSED);
+        } else if (ferror(replay_file)) {
+            error_report("replay file is over or something goes wrong");
+            qemu_system_vmstop_request_prepare();
+            qemu_system_vmstop_request(RUN_STATE_INTERNAL_ERROR);
+        }
+    }
+}
+
+void replay_fetch_data_kind(void)
+{
+    if (replay_file) {
+        if (!replay_has_unread_data) {
+            replay_data_kind = replay_get_byte();
+            if (replay_data_kind == EVENT_INSTRUCTION) {
+                replay_state.instructions_count = replay_get_dword();
+            }
+            replay_check_error();
+            replay_has_unread_data = 1;
+            if (replay_data_kind >= EVENT_COUNT) {
+                error_report("Replay: unknown event kind %d", replay_data_kind);
+                exit(1);
+            }
+        }
+    }
+}
+
+void replay_finish_event(void)
+{
+    replay_has_unread_data = 0;
+    replay_fetch_data_kind();
+}
+
+void replay_mutex_init(void)
+{
+    qemu_mutex_init(&lock);
+}
+
+void replay_mutex_destroy(void)
+{
+    qemu_mutex_destroy(&lock);
+}
+
+void replay_mutex_lock(void)
+{
+    qemu_mutex_lock(&lock);
+}
+
+void replay_mutex_unlock(void)
+{
+    qemu_mutex_unlock(&lock);
+}
+
+/*! Saves cached instructions. */
+void replay_save_instructions(void)
+{
+    if (replay_file && replay_mode == REPLAY_MODE_RECORD) {
+        replay_mutex_lock();
+        int diff = (int)(replay_get_current_step() - replay_state.current_step);
+        if (diff > 0) {
+            replay_put_event(EVENT_INSTRUCTION);
+            replay_put_dword(diff);
+            replay_state.current_step += diff;
+        }
+        replay_mutex_unlock();
+    }
+}
diff --git a/replay/replay-internal.h b/replay/replay-internal.h
new file mode 100644
index 000000000..77e0d292c
--- /dev/null
+++ b/replay/replay-internal.h
@@ -0,0 +1,140 @@
+#ifndef REPLAY_INTERNAL_H
+#define REPLAY_INTERNAL_H
+
+/*
+ * replay-internal.h
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <stdio.h>
+
+enum ReplayEvents {
+    /* for instruction event */
+    EVENT_INSTRUCTION,
+    /* for software interrupt */
+    EVENT_INTERRUPT,
+    /* for emulated exceptions */
+    EVENT_EXCEPTION,
+    /* for async events */
+    EVENT_ASYNC,
+    /* for shutdown request */
+    EVENT_SHUTDOWN,
+    /* for clock read/writes */
+    /* some of greater codes are reserved for clocks */
+    EVENT_CLOCK,
+    EVENT_CLOCK_LAST = EVENT_CLOCK + REPLAY_CLOCK_COUNT - 1,
+    /* for checkpoint event */
+    /* some of greater codes are reserved for checkpoints */
+    EVENT_CHECKPOINT,
+    EVENT_CHECKPOINT_LAST = EVENT_CHECKPOINT + CHECKPOINT_COUNT - 1,
+    /* end of log event */
+    EVENT_END,
+    EVENT_COUNT
+};
+
+/* Asynchronous events IDs */
+
+enum ReplayAsyncEventKind {
+    REPLAY_ASYNC_EVENT_BH,
+    REPLAY_ASYNC_EVENT_INPUT,
+    REPLAY_ASYNC_EVENT_INPUT_SYNC,
+    REPLAY_ASYNC_COUNT
+};
+
+typedef enum ReplayAsyncEventKind ReplayAsyncEventKind;
+
+typedef struct ReplayState {
+    /*! Cached clock values. */
+    int64_t cached_clock[REPLAY_CLOCK_COUNT];
+    /*! Current step - number of processed instructions and timer events. */
+    uint64_t current_step;
+    /*! Number of instructions to be executed before other events happen. */
+    int instructions_count;
+} ReplayState;
+extern ReplayState replay_state;
+
+extern unsigned int replay_data_kind;
+
+/* File for replay writing */
+extern FILE *replay_file;
+
+void replay_put_byte(uint8_t byte);
+void replay_put_event(uint8_t event);
+void replay_put_word(uint16_t word);
+void replay_put_dword(uint32_t dword);
+void replay_put_qword(int64_t qword);
+void replay_put_array(const uint8_t *buf, size_t size);
+
+uint8_t replay_get_byte(void);
+uint16_t replay_get_word(void);
+uint32_t replay_get_dword(void);
+int64_t replay_get_qword(void);
+void replay_get_array(uint8_t *buf, size_t *size);
+void replay_get_array_alloc(uint8_t **buf, size_t *size);
+
+/* Mutex functions for protecting replay log file */
+
+void replay_mutex_init(void);
+void replay_mutex_destroy(void);
+void replay_mutex_lock(void);
+void replay_mutex_unlock(void);
+
+/*! Checks error status of the file. */
+void replay_check_error(void);
+
+/*! Finishes processing of the replayed event and fetches
+    the next event from the log. */
+void replay_finish_event(void);
+/*! Reads data type from the file and stores it in the
+    replay_data_kind variable. */
+void replay_fetch_data_kind(void);
+
+/*! Saves queued events (like instructions and sound). */
+void replay_save_instructions(void);
+
+/*! Skips async events until some sync event will be found.
+    \return true, if event was found */
+bool replay_next_event_is(int event);
+
+/*! Reads next clock value from the file.
+    If clock kind read from the file is different from the parameter,
+    the value is not used. */
+void replay_read_next_clock(unsigned int kind);
+
+/* Asynchronous events queue */
+
+/*! Initializes events' processing internals */
+void replay_init_events(void);
+/*! Clears internal data structures for events handling */
+void replay_finish_events(void);
+/*! Enables storing events in the queue */
+void replay_enable_events(void);
+/*! Flushes events queue */
+void replay_flush_events(void);
+/*! Clears events list before loading new VM state */
+void replay_clear_events(void);
+/*! Returns true if there are any unsaved events in the queue */
+bool replay_has_events(void);
+/*! Saves events from queue into the file */
+void replay_save_events(int checkpoint);
+/*! Read events from the file into the input queue */
+void replay_read_events(int checkpoint);
+
+/* Input events */
+
+/*! Saves input event to the log */
+void replay_save_input_event(InputEvent *evt);
+/*! Reads input event from the log */
+InputEvent *replay_read_input_event(void);
+/*! Adds input event to the queue */
+void replay_add_input_event(struct InputEvent *event);
+/*! Adds input sync event to the queue */
+void replay_add_input_sync_event(void);
+
+#endif
diff --git a/replay/replay-time.c b/replay/replay-time.c
new file mode 100644
index 000000000..6d06951f5
--- /dev/null
+++ b/replay/replay-time.c
@@ -0,0 +1,64 @@
+/*
+ * replay-time.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/error-report.h"
+
+int64_t replay_save_clock(ReplayClockKind kind, int64_t clock)
+{
+    replay_save_instructions();
+
+    if (replay_file) {
+        replay_mutex_lock();
+        replay_put_event(EVENT_CLOCK + kind);
+        replay_put_qword(clock);
+        replay_mutex_unlock();
+    }
+
+    return clock;
+}
+
+void replay_read_next_clock(ReplayClockKind kind)
+{
+    unsigned int read_kind = replay_data_kind - EVENT_CLOCK;
+
+    assert(read_kind == kind);
+
+    int64_t clock = replay_get_qword();
+
+    replay_check_error();
+    replay_finish_event();
+
+    replay_state.cached_clock[read_kind] = clock;
+}
+
+/*! Reads next clock event from the input. */
+int64_t replay_read_clock(ReplayClockKind kind)
+{
+    replay_account_executed_instructions();
+
+    if (replay_file) {
+        int64_t ret;
+        replay_mutex_lock();
+        if (replay_next_event_is(EVENT_CLOCK + kind)) {
+            replay_read_next_clock(kind);
+        }
+        ret = replay_state.cached_clock[kind];
+        replay_mutex_unlock();
+
+        return ret;
+    }
+
+    error_report("REPLAY INTERNAL ERROR %d", __LINE__);
+    exit(1);
+}
diff --git a/replay/replay.c b/replay/replay.c
new file mode 100644
index 000000000..0d33e82c9
--- /dev/null
+++ b/replay/replay.c
@@ -0,0 +1,342 @@
+/*
+ * replay.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/timer.h"
+#include "qemu/main-loop.h"
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
+
+/* Current version of the replay mechanism.
+   Increase it when file format changes. */
+#define REPLAY_VERSION              0xe02002
+/* Size of replay log header */
+#define HEADER_SIZE                 (sizeof(uint32_t) + sizeof(uint64_t))
+
+ReplayMode replay_mode = REPLAY_MODE_NONE;
+
+/* Name of replay file  */
+static char *replay_filename;
+ReplayState replay_state;
+static GSList *replay_blockers;
+
+bool replay_next_event_is(int event)
+{
+    bool res = false;
+
+    /* nothing to skip - not all instructions used */
+    if (replay_state.instructions_count != 0) {
+        assert(replay_data_kind == EVENT_INSTRUCTION);
+        return event == EVENT_INSTRUCTION;
+    }
+
+    while (true) {
+        if (event == replay_data_kind) {
+            res = true;
+        }
+        switch (replay_data_kind) {
+        case EVENT_SHUTDOWN:
+            replay_finish_event();
+            qemu_system_shutdown_request();
+            break;
+        default:
+            /* clock, time_t, checkpoint and other events */
+            return res;
+        }
+    }
+    return res;
+}
+
+uint64_t replay_get_current_step(void)
+{
+    return cpu_get_icount_raw();
+}
+
+int replay_get_instructions(void)
+{
+    int res = 0;
+    replay_mutex_lock();
+    if (replay_next_event_is(EVENT_INSTRUCTION)) {
+        res = replay_state.instructions_count;
+    }
+    replay_mutex_unlock();
+    return res;
+}
+
+void replay_account_executed_instructions(void)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_mutex_lock();
+        if (replay_state.instructions_count > 0) {
+            int count = (int)(replay_get_current_step()
+                              - replay_state.current_step);
+            replay_state.instructions_count -= count;
+            replay_state.current_step += count;
+            if (replay_state.instructions_count == 0) {
+                assert(replay_data_kind == EVENT_INSTRUCTION);
+                replay_finish_event();
+                /* Wake up iothread. This is required because
+                   timers will not expire until clock counters
+                   will be read from the log. */
+                qemu_notify_event();
+            }
+        }
+        replay_mutex_unlock();
+    }
+}
+
+bool replay_exception(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_save_instructions();
+        replay_mutex_lock();
+        replay_put_event(EVENT_EXCEPTION);
+        replay_mutex_unlock();
+        return true;
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        bool res = replay_has_exception();
+        if (res) {
+            replay_mutex_lock();
+            replay_finish_event();
+            replay_mutex_unlock();
+        }
+        return res;
+    }
+
+    return true;
+}
+
+bool replay_has_exception(void)
+{
+    bool res = false;
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_account_executed_instructions();
+        replay_mutex_lock();
+        res = replay_next_event_is(EVENT_EXCEPTION);
+        replay_mutex_unlock();
+    }
+
+    return res;
+}
+
+bool replay_interrupt(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_save_instructions();
+        replay_mutex_lock();
+        replay_put_event(EVENT_INTERRUPT);
+        replay_mutex_unlock();
+        return true;
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        bool res = replay_has_interrupt();
+        if (res) {
+            replay_mutex_lock();
+            replay_finish_event();
+            replay_mutex_unlock();
+        }
+        return res;
+    }
+
+    return true;
+}
+
+bool replay_has_interrupt(void)
+{
+    bool res = false;
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_account_executed_instructions();
+        replay_mutex_lock();
+        res = replay_next_event_is(EVENT_INTERRUPT);
+        replay_mutex_unlock();
+    }
+    return res;
+}
+
+void replay_shutdown_request(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_mutex_lock();
+        replay_put_event(EVENT_SHUTDOWN);
+        replay_mutex_unlock();
+    }
+}
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    bool res = false;
+    assert(EVENT_CHECKPOINT + checkpoint <= EVENT_CHECKPOINT_LAST);
+    replay_save_instructions();
+
+    if (!replay_file) {
+        return true;
+    }
+
+    replay_mutex_lock();
+
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        if (replay_next_event_is(EVENT_CHECKPOINT + checkpoint)) {
+            replay_finish_event();
+        } else if (replay_data_kind != EVENT_ASYNC) {
+            res = false;
+            goto out;
+        }
+        replay_read_events(checkpoint);
+        /* replay_read_events may leave some unread events.
+           Return false if not all of the events associated with
+           checkpoint were processed */
+        res = replay_data_kind != EVENT_ASYNC;
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_put_event(EVENT_CHECKPOINT + checkpoint);
+        replay_save_events(checkpoint);
+        res = true;
+    }
+out:
+    replay_mutex_unlock();
+    return res;
+}
+
+static void replay_enable(const char *fname, int mode)
+{
+    const char *fmode = NULL;
+    assert(!replay_file);
+
+    switch (mode) {
+    case REPLAY_MODE_RECORD:
+        fmode = "wb";
+        break;
+    case REPLAY_MODE_PLAY:
+        fmode = "rb";
+        break;
+    default:
+        fprintf(stderr, "Replay: internal error: invalid replay mode\n");
+        exit(1);
+    }
+
+    atexit(replay_finish);
+
+    replay_mutex_init();
+
+    replay_file = fopen(fname, fmode);
+    if (replay_file == NULL) {
+        fprintf(stderr, "Replay: open %s: %s\n", fname, strerror(errno));
+        exit(1);
+    }
+
+    replay_filename = g_strdup(fname);
+
+    replay_mode = mode;
+    replay_data_kind = -1;
+    replay_state.instructions_count = 0;
+    replay_state.current_step = 0;
+
+    /* skip file header for RECORD and check it for PLAY */
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        fseek(replay_file, HEADER_SIZE, SEEK_SET);
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        unsigned int version = replay_get_dword();
+        if (version != REPLAY_VERSION) {
+            fprintf(stderr, "Replay: invalid input log file version\n");
+            exit(1);
+        }
+        /* go to the beginning */
+        fseek(replay_file, HEADER_SIZE, SEEK_SET);
+        replay_fetch_data_kind();
+    }
+
+    replay_init_events();
+}
+
+void replay_configure(QemuOpts *opts)
+{
+    const char *fname;
+    const char *rr;
+    ReplayMode mode = REPLAY_MODE_NONE;
+
+    rr = qemu_opt_get(opts, "rr");
+    if (!rr) {
+        /* Just enabling icount */
+        return;
+    } else if (!strcmp(rr, "record")) {
+        mode = REPLAY_MODE_RECORD;
+    } else if (!strcmp(rr, "replay")) {
+        mode = REPLAY_MODE_PLAY;
+    } else {
+        error_report("Invalid icount rr option: %s", rr);
+        exit(1);
+    }
+
+    fname = qemu_opt_get(opts, "rrfile");
+    if (!fname) {
+        error_report("File name not specified for replay");
+        exit(1);
+    }
+
+    replay_enable(fname, mode);
+}
+
+void replay_start(void)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+
+    if (replay_blockers) {
+        error_report("Record/replay: %s",
+                     error_get_pretty(replay_blockers->data));
+        exit(1);
+    }
+    if (!use_icount) {
+        error_report("Please enable icount to use record/replay");
+        exit(1);
+    }
+
+    /* Timer for snapshotting will be set up here. */
+
+    replay_enable_events();
+}
+
+void replay_finish(void)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+
+    replay_save_instructions();
+
+    /* finalize the file */
+    if (replay_file) {
+        if (replay_mode == REPLAY_MODE_RECORD) {
+            /* write end event */
+            replay_put_event(EVENT_END);
+
+            /* write header */
+            fseek(replay_file, 0, SEEK_SET);
+            replay_put_dword(REPLAY_VERSION);
+        }
+
+        fclose(replay_file);
+        replay_file = NULL;
+    }
+    if (replay_filename) {
+        g_free(replay_filename);
+        replay_filename = NULL;
+    }
+
+    replay_finish_events();
+    replay_mutex_destroy();
+}
+
+void replay_add_blocker(Error *reason)
+{
+    replay_blockers = g_slist_prepend(replay_blockers, reason);
+}
diff --git a/roms/Makefile b/roms/Makefile
index 7b3f15632..09e33b59e 100644
--- a/roms/Makefile
+++ b/roms/Makefile
@@ -120,20 +120,17 @@ efi-rom-%: build-pxe-roms build-efi-roms
 		-ec ipxe/src/bin-x86_64-efi/$(VID)$(DID).efidrv \
 		-o ../pc-bios/efi-$*.rom
 
-build-pxe-roms: ipxe/src/config/local/general.h
-	$(MAKE) -C ipxe/src GITVERSION="" \
+build-pxe-roms:
+	$(MAKE) -C ipxe/src CONFIG=qemu \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		$(patsubst %,bin/%.rom,$(pxerom_targets))
 
-build-efi-roms: build-pxe-roms ipxe/src/config/local/general.h
-	$(MAKE) -C ipxe/src GITVERSION="" \
+build-efi-roms: build-pxe-roms
+	$(MAKE) -C ipxe/src CONFIG=qemu \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		$(patsubst %,bin-i386-efi/%.efidrv,$(pxerom_targets)) \
 		$(patsubst %,bin-x86_64-efi/%.efidrv,$(pxerom_targets))
 
-ipxe/src/config/local/%: config.ipxe.%
-	cp $< $@
-
 
 slof:
 	$(MAKE) -C SLOF CROSS=$(powerpc64_cross_prefix) qemu
diff --git a/roms/SLOF/README b/roms/SLOF/README
index 58e929427..789504568 100644
--- a/roms/SLOF/README
+++ b/roms/SLOF/README
@@ -11,6 +11,7 @@ Index
 2.2 Overview of the source code
 2.4 Extending the Forth engine
 3.0 Limitations
+4.0 Submitting patches
 
 
 1.0 Introduction to Slimline Open Firmware
@@ -236,6 +237,16 @@ To add primitives:
  On a JS21 all memory configurations should work.
 
 
+4.0 Submitting patches
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Patches for SLOF should be made against https://github.com/aik/SLOF,
+the master branch and posted to slof@lists.ozlabs.org.
+The patches must be signed using "Signed-off-by" tag with a real name to
+confirm that you certify the Developer Certificate of Origin  Version 1.1,
+see [3] for details.
+
+
 Documentation
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -244,3 +255,6 @@ Documentation
 
 [2] PAPR Standard, Power.org(TM) Standard for Power Architecture(R) Platform
     Requirements (Workstation, Server), Version 2.4, December 7, 2009
+
+[3] Developer Certificate of Origin Version 1.1
+    http://developercertificate.org/
diff --git a/roms/SLOF/VERSION b/roms/SLOF/VERSION
index 20bdb2eb7..c99837873 100644
--- a/roms/SLOF/VERSION
+++ b/roms/SLOF/VERSION
@@ -1 +1 @@
-20150429
+20151103
diff --git a/roms/SLOF/board-js2x/llfw/stage2.lds b/roms/SLOF/board-js2x/llfw/stage2.lds
index f91f0658a..e6315c3c8 100644
--- a/roms/SLOF/board-js2x/llfw/stage2.lds
+++ b/roms/SLOF/board-js2x/llfw/stage2.lds
@@ -45,7 +45,8 @@ SECTIONS {
 	__bss_end = .;
 	__bss_size = (__bss_end - __bss_start);
 
-	__toc_start = .;
+	. = ALIGN(256);
+	__toc_start = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
 	.got       :
 	{
 		 *(.toc .got)
diff --git a/roms/SLOF/board-js2x/llfw/stage2_head.S b/roms/SLOF/board-js2x/llfw/stage2_head.S
index 5460bfebb..f3f5e0c8c 100644
--- a/roms/SLOF/board-js2x/llfw/stage2_head.S
+++ b/roms/SLOF/board-js2x/llfw/stage2_head.S
@@ -79,8 +79,6 @@ bsscdone:
 	/* ------------------------------------ */
 ASM_ENTRY(toc_init)	
 	LOAD64(r2, __toc_start)
-	addi	r2,r2,0x4000
-	addi	r2,r2,0x4000
 	blr
 
 	/* ------------------------------------ */
diff --git a/roms/SLOF/board-js2x/slof/Makefile b/roms/SLOF/board-js2x/slof/Makefile
index ab3e683a4..fdc716fc6 100644
--- a/roms/SLOF/board-js2x/slof/Makefile
+++ b/roms/SLOF/board-js2x/slof/Makefile
@@ -82,6 +82,7 @@ OF_FFS_FILES = \
 	$(SLOFCMNDIR)/fs/scsi-host-helpers.fs \
 	$(SLOFCMNDIR)/fs/scsi-probe-helpers.fs \
 	$(SLOFCMNDIR)/fs/scsi-support.fs \
+	$(SLOFCMNDIR)/fs/dma-function.fs \
 	$(SLOFCMNDIR)/fs/pci-device.fs \
 	$(SLOFCMNDIR)/fs/pci-bridge.fs \
 	$(SLOFCMNDIR)/fs/pci-properties.fs \
diff --git a/roms/SLOF/board-js2x/slof/helper.fs b/roms/SLOF/board-js2x/slof/helper.fs
index 34d60da1f..1e2b03063 100644
--- a/roms/SLOF/board-js2x/slof/helper.fs
+++ b/roms/SLOF/board-js2x/slof/helper.fs
@@ -26,3 +26,14 @@
    s" , " $cat
    bdate2human $cat encode-string THEN
 ;
+
+: invert-region ( addr len -- )
+   2dup or 7 and CASE
+      0 OF 3 rshift 0 ?DO dup dup rx@ -1 xor swap rx! xa1+ LOOP ENDOF
+      4 OF 2 rshift 0 ?DO dup dup rl@ -1 xor swap rl! la1+ LOOP ENDOF
+      3 and
+      2 OF 1 rshift 0 ?DO dup dup rw@ -1 xor swap rw! wa1+ LOOP ENDOF
+      dup OF 0 ?DO dup dup rb@ -1 xor swap rb! 1+ LOOP ENDOF
+   ENDCASE
+   drop
+;
diff --git a/roms/SLOF/board-qemu/llfw/stage2.lds b/roms/SLOF/board-qemu/llfw/stage2.lds
index e060dd189..28c9dca93 100644
--- a/roms/SLOF/board-qemu/llfw/stage2.lds
+++ b/roms/SLOF/board-qemu/llfw/stage2.lds
@@ -49,7 +49,8 @@ SECTIONS {
 	__bss_end = .;
 	__bss_size = (__bss_end - __bss_start);
 
-	__toc_start = .;
+	. = ALIGN(256);
+	__toc_start = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
 	.got :
 	{
 		 *(.toc .got)
diff --git a/roms/SLOF/board-qemu/llfw/stage2_head.S b/roms/SLOF/board-qemu/llfw/stage2_head.S
index c56b117ce..adf75547b 100644
--- a/roms/SLOF/board-qemu/llfw/stage2_head.S
+++ b/roms/SLOF/board-qemu/llfw/stage2_head.S
@@ -77,8 +77,6 @@ bsscdone:
 	/* ------------------------------------ */
 ASM_ENTRY(toc_init)
 	LOAD64(r2, __toc_start)
-	addi	r2,r2,0x4000
-	addi	r2,r2,0x4000
 	blr
 
 	/* ------------------------------------ */
diff --git a/roms/SLOF/board-qemu/slof/Makefile b/roms/SLOF/board-qemu/slof/Makefile
index 283f77d32..a3fe6abd0 100644
--- a/roms/SLOF/board-qemu/slof/Makefile
+++ b/roms/SLOF/board-qemu/slof/Makefile
@@ -69,6 +69,7 @@ VIO_FFS_FILES = \
 	$(SLOFBRDDIR)/pci-device_1af4_1001.fs \
 	$(SLOFBRDDIR)/pci-device_1af4_1004.fs \
 	$(SLOFBRDDIR)/pci-device_1af4_1009.fs \
+	$(SLOFBRDDIR)/pci-device_1af4_1050.fs \
 	$(SLOFBRDDIR)/vio-hvterm.fs \
 	$(SLOFBRDDIR)/vio-vscsi.fs \
 	$(SLOFBRDDIR)/vio-veth.fs \
@@ -103,6 +104,7 @@ OF_FFS_FILES = \
 	$(SLOFBRDDIR)/pci-device_1013_00b8.fs \
 	$(SLOFBRDDIR)/pci-device_8086_100e.fs \
 	$(SLOFBRDDIR)/e1k.fs \
+	$(SLOFBRDDIR)/qemu-vga.fs \
 	$(FCODE_FFS_FILES)
 
 # Uncomment the following line to enable the USB code:
diff --git a/roms/SLOF/board-qemu/slof/helper.fs b/roms/SLOF/board-qemu/slof/helper.fs
index 96da49894..40d4abc3a 100644
--- a/roms/SLOF/board-qemu/slof/helper.fs
+++ b/roms/SLOF/board-qemu/slof/helper.fs
@@ -33,3 +33,16 @@
   swap -
 ;
 
+: invert-region-cs ( addr len cellsize -- )
+   >r over swap r@ rshift r> swap 1 hv-logical-memop drop
+;
+
+: invert-region ( addr len -- )
+   2dup or 7 and CASE
+      0 OF 3 invert-region-cs ENDOF
+      4 OF 2 invert-region-cs ENDOF
+      3 and
+      2 OF 1 invert-region-cs ENDOF
+      dup OF 0 invert-region-cs ENDOF
+   ENDCASE
+;
diff --git a/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs b/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs
index a5c3584f9..22ea45d5c 100644
--- a/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs
+++ b/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs
@@ -10,233 +10,6 @@
 \ *     IBM Corporation - initial implementation
 \ ****************************************************************************/
 
-my-space pci-device-generic-setup
-
-\ Defaults, overriden from qemu
-d# 800 VALUE disp-width
-d# 600 VALUE disp-height
-d#   8 VALUE disp-depth
-
-\ Determine base address
-10 config-l@ translate-my-address f not AND VALUE fb-base
-
-\ Fixed up later
--1 VALUE io-base
-
-\ We support only one instance
-false VALUE is-installed?
-
-: vga-io-xlate ( port -- addr )
-  io-base -1 = IF
-    dup translate-my-address fff not and to io-base
-  THEN
-  io-base +
-;
-
-: vga-w! ( value port -- )
-  vga-io-xlate rw!-le
-;
-
-: vga-w@ ( port -- value )
-  vga-io-xlate rw@-le
-;
-
-: vga-b! ( value port -- )
-  vga-io-xlate rb!
-;
-
-: vga-b@ ( port -- value )
-  vga-io-xlate rb@
-;
-
-: vbe!	( value index -- )
-  1ce vga-w! 1d0 vga-w!
-;
-
-: vbe@	( index -- value )
-  1ce vga-w! 1d0 vga-w@
-;
-
-: color! ( r g b number -- ) 
-   3c8 vga-b!
-   rot 3c9 vga-b!
-   swap 3c9 vga-b!
-   3c9 vga-b!
-;
-
-: color@ ( number -- r g b ) 
-   3c8 vga-b!
-   3c9 vga-b@
-   3c9 vga-b@
-   3c9 vga-b@
-;
-
-: set-colors ( adr number #numbers -- )
-   over 3c8 vga-b!
-   swap DO
-     rb@ 3c9 vga-b!
-     rb@ 3c9 vga-b!
-     rb@ 3c9 vga-b!
-   LOOP
-   3drop
-;
-
-: get-colors ( adr number #numbers -- )
-   3drop
-;
-
-include graphics.fs
-
-\ qemu fake VBE IO registers
-0 CONSTANT VBE_DISPI_INDEX_ID
-1 CONSTANT VBE_DISPI_INDEX_XRES
-2 CONSTANT VBE_DISPI_INDEX_YRES
-3 CONSTANT VBE_DISPI_INDEX_BPP
-4 CONSTANT VBE_DISPI_INDEX_ENABLE
-5 CONSTANT VBE_DISPI_INDEX_BANK
-6 CONSTANT VBE_DISPI_INDEX_VIRT_WIDTH
-7 CONSTANT VBE_DISPI_INDEX_VIRT_HEIGHT
-8 CONSTANT VBE_DISPI_INDEX_X_OFFSET
-9 CONSTANT VBE_DISPI_INDEX_Y_OFFSET
-a CONSTANT VBE_DISPI_INDEX_NB
-
-\ ENABLE register
-00 CONSTANT VBE_DISPI_DISABLED
-01 CONSTANT VBE_DISPI_ENABLED
-02 CONSTANT VBE_DISPI_GETCAPS
-20 CONSTANT VBE_DISPI_8BIT_DAC
-40 CONSTANT VBE_DISPI_LFB_ENABLED
-80 CONSTANT VBE_DISPI_NOCLEARMEM
-
-: init-mode
-  0 3c0 vga-b!
-  VBE_DISPI_DISABLED VBE_DISPI_INDEX_ENABLE vbe!
-  0 VBE_DISPI_INDEX_X_OFFSET vbe!
-  0 VBE_DISPI_INDEX_Y_OFFSET vbe!
-  disp-width VBE_DISPI_INDEX_XRES vbe!
-  disp-height VBE_DISPI_INDEX_YRES vbe!
-  disp-depth VBE_DISPI_INDEX_BPP vbe!
-  VBE_DISPI_ENABLED VBE_DISPI_8BIT_DAC or VBE_DISPI_INDEX_ENABLE vbe!
-  0 3c0 vga-b!
-  20 3c0 vga-b!
-;
-
-: clear-screen
-  fb-base disp-width disp-height disp-depth 7 + 8 / * * 0 rfill
-;
-
-: read-settings
-  s" qemu,graphic-width" get-chosen IF
-     decode-int to disp-width 2drop
-  THEN
-  s" qemu,graphic-height" get-chosen IF
-     decode-int to disp-height 2drop
-  THEN
-  s" qemu,graphic-depth" get-chosen IF
-     decode-int nip nip     
-       dup 8 =
-       over f = or
-       over 10 = or
-       over 20 = or IF
-         to disp-depth
-       ELSE
-         ." Unsupported bit depth, using 8bpp " drop cr
-       THEN
-  THEN
-;
-
-: add-legacy-reg
-  \ add legacy I/O Ports / Memory regions to assigned-addresses
-  \ see PCI Bus Binding Revision 2.1 Section 7.
-  s" reg" get-node get-property IF
-    \ "reg" does not exist, create new
-    encode-start
-  ELSE
-    \ "reg" does exist, copy it 
-    encode-bytes
-  THEN
-  \ I/O Range 0x1ce-0x1d2
-  my-space a1000000 or encode-int+ \ non-relocatable, aliased I/O space
-  1ce encode-64+ 4 encode-64+ \ addr size
-  \ I/O Range 0x3B0-0x3BB
-  my-space a1000000 or encode-int+ \ non-relocatable, aliased I/O space
-  3b0 encode-64+ c encode-64+ \ addr size
-  \ I/O Range 0x3C0-0x3DF
-  my-space a1000000 or encode-int+ \ non-relocatable, aliased I/O space
-  3c0 encode-64+ 20 encode-64+ \ addr size
-  \ Memory Range 0xA0000-0xBFFFF
-  my-space a2000000 or encode-int+ \ non-relocatable, <1MB Memory space
-  a0000 encode-64+ 20000 encode-64+ \ addr size
-  s" reg" property \ store "reg" property
-;
-
-: setup-properties
-   \ Shouldn't this be done from open ?
-   disp-width encode-int s" width" property
-   disp-height encode-int s" height" property
-   disp-width disp-depth 7 + 8 / * encode-int s" linebytes" property
-   disp-depth encode-int s" depth" property
-   s" ISO8859-1" encode-string s" character-set" property \ i hope this is ok...
-   \ add "device_type" property
-   s" display" device-type
-   s" qemu,std-vga" encode-string s" compatible" property
-   \ XXX We don't create an "address" property because Linux doesn't know what
-   \ to do with it for >32-bit
-;
-
-\ words for installation/removal, needed by is-install/is-remove, see display.fs
-: display-remove ( -- ) 
-;
-
-: hcall-invert-screen ( -- )
-    frame-buffer-adr frame-buffer-adr 3
-    screen-height screen-width * screen-depth * /x /
-    1 hv-logical-memop
-    drop
-;
-
-: hcall-blink-screen ( -- )
-    \ 32 msec delay for visually noticing the blink
-    hcall-invert-screen 20 ms hcall-invert-screen
-;
-
-: display-install ( -- )
-    is-installed? NOT IF
-        ." Installing QEMU fb" cr
-        fb-base to frame-buffer-adr
-        clear-screen
-        default-font 
-        set-font
-        disp-width disp-height
-        disp-width char-width / disp-height char-height /
-        disp-depth 7 + 8 /                      ( width height #lines #cols depth )
-        fb-install
-	['] hcall-invert-screen to invert-screen
-	['] hcall-blink-screen to blink-screen
-         true to is-installed?
-    THEN
-;
-
-: set-alias
-    s" screen" find-alias 0= IF
-      \ no previous screen alias defined, define it...
-      s" screen" get-node node>path set-alias
-    ELSE
-       drop
-    THEN 
-;
-
-
 ." qemu vga" cr
 
-pci-master-enable
-pci-mem-enable
-pci-io-enable
-add-legacy-reg
-read-settings
-init-mode
-init-default-palette
-setup-properties
-' display-install is-install
-' display-remove is-remove
-set-alias
+s" qemu-vga.fs" included
diff --git a/roms/SLOF/board-qemu/slof/pci-device_1af4_1050.fs b/roms/SLOF/board-qemu/slof/pci-device_1af4_1050.fs
new file mode 100644
index 000000000..516056aad
--- /dev/null
+++ b/roms/SLOF/board-qemu/slof/pci-device_1af4_1050.fs
@@ -0,0 +1,15 @@
+\ *****************************************************************************
+\ * Copyright (c) 2015 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+s" virtio [ vga ]" type cr
+
+s" qemu-vga.fs" included
diff --git a/roms/SLOF/board-qemu/slof/qemu-vga.fs b/roms/SLOF/board-qemu/slof/qemu-vga.fs
new file mode 100644
index 000000000..3f4c237fc
--- /dev/null
+++ b/roms/SLOF/board-qemu/slof/qemu-vga.fs
@@ -0,0 +1,198 @@
+\ *****************************************************************************
+\ * Copyright (c) 2015 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+my-space pci-device-generic-setup
+
+\ Defaults, overriden from qemu
+d# 800 VALUE disp-width
+d# 600 VALUE disp-height
+d#   8 VALUE disp-depth
+
+: map-in  " map-in" my-phandle parent $call-static ;
+: map-out  " map-out" my-phandle parent $call-static ;
+
+\ Determine base address
+0 0  my-space h# 02000010 + 1  map-in VALUE fb-base
+0 0  my-space h# 02000018 + 1 map-in VALUE reg-base
+
+\ We support only one instance
+false VALUE is-installed?
+
+: vga-w! ( value port -- )
+  3c0 - reg-base 400 + + rw!-le
+;
+
+: vga-w@ ( port -- value )
+  3c0 - reg-base 400 + + rw@-le
+;
+
+: vga-b! ( value port -- )
+  3c0 - reg-base 400 + + rb!
+;
+
+: vga-b@ ( port -- value )
+  3c0 - reg-base 400 + + rb@
+;
+
+: vbe!	( value index -- )
+  1 << reg-base 500 + + rw!-le
+;
+
+: vbe@	( index -- value )
+  1 << reg-base 500 + + rw@-le
+;
+
+: color! ( r g b number -- )
+   3c8 vga-b!
+   rot 3c9 vga-b!
+   swap 3c9 vga-b!
+   3c9 vga-b!
+;
+
+: color@ ( number -- r g b )
+   3c8 vga-b!
+   3c9 vga-b@
+   3c9 vga-b@
+   3c9 vga-b@
+;
+
+: set-colors ( adr number #numbers -- )
+   over 3c8 vga-b!
+   swap DO
+     rb@ 3c9 vga-b!
+     rb@ 3c9 vga-b!
+     rb@ 3c9 vga-b!
+   LOOP
+   3drop
+;
+
+: get-colors ( adr number #numbers -- )
+   3drop
+;
+
+include graphics.fs
+
+\ qemu fake VBE IO registers
+0 CONSTANT VBE_DISPI_INDEX_ID
+1 CONSTANT VBE_DISPI_INDEX_XRES
+2 CONSTANT VBE_DISPI_INDEX_YRES
+3 CONSTANT VBE_DISPI_INDEX_BPP
+4 CONSTANT VBE_DISPI_INDEX_ENABLE
+5 CONSTANT VBE_DISPI_INDEX_BANK
+6 CONSTANT VBE_DISPI_INDEX_VIRT_WIDTH
+7 CONSTANT VBE_DISPI_INDEX_VIRT_HEIGHT
+8 CONSTANT VBE_DISPI_INDEX_X_OFFSET
+9 CONSTANT VBE_DISPI_INDEX_Y_OFFSET
+a CONSTANT VBE_DISPI_INDEX_NB
+
+\ ENABLE register
+00 CONSTANT VBE_DISPI_DISABLED
+01 CONSTANT VBE_DISPI_ENABLED
+02 CONSTANT VBE_DISPI_GETCAPS
+20 CONSTANT VBE_DISPI_8BIT_DAC
+40 CONSTANT VBE_DISPI_LFB_ENABLED
+80 CONSTANT VBE_DISPI_NOCLEARMEM
+
+: init-mode
+  0 3c0 vga-b!
+  VBE_DISPI_DISABLED VBE_DISPI_INDEX_ENABLE vbe!
+  0 VBE_DISPI_INDEX_X_OFFSET vbe!
+  0 VBE_DISPI_INDEX_Y_OFFSET vbe!
+  disp-width VBE_DISPI_INDEX_XRES vbe!
+  disp-height VBE_DISPI_INDEX_YRES vbe!
+  disp-depth VBE_DISPI_INDEX_BPP vbe!
+  VBE_DISPI_ENABLED VBE_DISPI_8BIT_DAC or VBE_DISPI_INDEX_ENABLE vbe!
+  0 3c0 vga-b!
+  20 3c0 vga-b!
+;
+
+: clear-screen
+  fb-base disp-width disp-height disp-depth 7 + 8 / * * 0 rfill
+;
+
+: read-settings
+  s" qemu,graphic-width" get-chosen IF
+     decode-int to disp-width 2drop
+  THEN
+  s" qemu,graphic-height" get-chosen IF
+     decode-int to disp-height 2drop
+  THEN
+  s" qemu,graphic-depth" get-chosen IF
+     decode-int nip nip
+       dup 8 =
+       over f = or
+       over 10 = or
+       over 20 = or IF
+         to disp-depth
+       ELSE
+         ." Unsupported bit depth, using 8bpp " drop cr
+       THEN
+  THEN
+;
+
+: setup-properties
+   \ Shouldn't this be done from open ?
+   disp-width encode-int s" width" property
+   disp-height encode-int s" height" property
+   disp-width disp-depth 7 + 8 / * encode-int s" linebytes" property
+   disp-depth encode-int s" depth" property
+   s" ISO8859-1" encode-string s" character-set" property \ i hope this is ok...
+   \ add "device_type" property
+   s" display" device-type
+   s" qemu,std-vga" encode-string s" compatible" property
+   \ XXX We don't create an "address" property because Linux doesn't know what
+   \ to do with it for >32-bit
+;
+
+\ words for installation/removal, needed by is-install/is-remove, see display.fs
+: display-remove ( -- )
+;
+
+: slow-blink-screen ( -- )
+    \ 32 msec delay for visually noticing the blink
+    invert-screen 20 ms invert-screen
+;
+
+: display-install ( -- )
+    is-installed? NOT IF
+        ." Installing QEMU fb" cr
+        fb-base to frame-buffer-adr
+        clear-screen
+        default-font
+        set-font
+        disp-width disp-height
+        disp-width char-width / disp-height char-height /
+        disp-depth 7 + 8 /                      ( width height #lines #cols depth )
+        fb-install
+	['] slow-blink-screen to blink-screen
+	true to is-installed?
+    THEN
+;
+
+: set-alias
+    s" screen" find-alias 0= IF
+      \ no previous screen alias defined, define it...
+      s" screen" get-node node>path set-alias
+    ELSE
+       drop
+    THEN
+;
+
+pci-master-enable
+pci-mem-enable
+read-settings
+init-mode
+init-default-palette
+setup-properties
+' display-install is-install
+' display-remove is-remove
+set-alias
diff --git a/roms/SLOF/clients/net-snk/client.lds b/roms/SLOF/clients/net-snk/client.lds
index 39d04594e..c2086445b 100644
--- a/roms/SLOF/clients/net-snk/client.lds
+++ b/roms/SLOF/clients/net-snk/client.lds
@@ -44,10 +44,10 @@ SECTIONS {
 	  *(.opd)
 	}
 
-	. = ALIGN(0x10);
+	. = ALIGN(256);
 	.got :
 	{
-	  _got = .;
+	  _got = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
           *(.got)
 	  *(.toc)
 	  _got_end = .;
diff --git a/roms/SLOF/clients/net-snk/kernel/entry.S b/roms/SLOF/clients/net-snk/kernel/entry.S
index 8849fb9d1..bf10542bd 100644
--- a/roms/SLOF/clients/net-snk/kernel/entry.S
+++ b/roms/SLOF/clients/net-snk/kernel/entry.S
@@ -44,7 +44,7 @@ C_ENTRY(_entry)
 	bcl	20,31,over	# branch after pointer table
 base:	
 	.align  3
-.LCgot:		.quad   _got-base+0x8000
+.LCgot:		.quad   _got-base
 .LCstack:	.quad   _stack+STACKSIZE-0x80-base
 over:	
 	mflr	r8		# gpr 8 is the base
diff --git a/roms/SLOF/clients/takeover/client.lds b/roms/SLOF/clients/takeover/client.lds
index 2701d8e1e..0ab428a01 100644
--- a/roms/SLOF/clients/takeover/client.lds
+++ b/roms/SLOF/clients/takeover/client.lds
@@ -43,8 +43,8 @@ SECTIONS {
 
 	.got :
 	{
-	  . = ALIGN(8);
-	  _got = .;
+	  . = ALIGN(256);
+	  _got = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
           *(.got .toc)
 	  _got_end = .;
 	}  	
diff --git a/roms/SLOF/clients/takeover/entry.S b/roms/SLOF/clients/takeover/entry.S
index a1030eb40..ff482732d 100644
--- a/roms/SLOF/clients/takeover/entry.S
+++ b/roms/SLOF/clients/takeover/entry.S
@@ -21,7 +21,7 @@ _wrapclient:
 	bcl	20,31,over	# branch after pointer table
 base:
 	.align  3
-.LCgot:		.quad   _got-base+0x8000
+.LCgot:		.quad   _got-base
 over:
 	mflr	r8		# gpr 8 is the base
 	ld	r2, .LCgot-base(r8)	# load got pointer
diff --git a/roms/SLOF/clients/takeover/main.c b/roms/SLOF/clients/takeover/main.c
index 360d8eaed..1e1b02614 100644
--- a/roms/SLOF/clients/takeover/main.c
+++ b/roms/SLOF/clients/takeover/main.c
@@ -16,7 +16,7 @@
 #include <of.h>
 #include <pci.h>
 #include <cpu.h>
-#include <ioctl.h>
+#include <unistd.h>
 #include <takeover.h>
 
 extern void call_client_interface(of_arg_t *);
diff --git a/roms/SLOF/include/ppc970/cache.h b/roms/SLOF/include/ppc970/cache.h
index b74868986..500182ea6 100644
--- a/roms/SLOF/include/ppc970/cache.h
+++ b/roms/SLOF/include/ppc970/cache.h
@@ -55,8 +55,8 @@ cache_inhibited_access(uint64_t, 64)
 #define _FASTMOVE(s, d, size) \
 	switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) { \
 		case 0:			_MOVE(s, d, size, type_u); break; \
-		case sizeof(type_l):	_MOVE(s, d, size, type_l); break; \
-		case sizeof(type_w):	_MOVE(s, d, size, type_w); break; \
+		case 4:			_MOVE(s, d, size, type_l); break; \
+		case 2: case 6:		_MOVE(s, d, size, type_w); break; \
 		default:		_MOVE(s, d, size, type_c); break; \
 	}
 
@@ -78,9 +78,51 @@ cache_inhibited_access(uint64_t, 64)
 #define _FASTRMOVE(s, d, size) \
 	switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) { \
 		case 0:			_RMOVE(s, d, size, type_u); break; \
-		case sizeof(type_l):	_RMOVE(s, d, size, type_l); break; \
-		case sizeof(type_w):	_RMOVE(s, d, size, type_w); break; \
+		case 4:			_RMOVE(s, d, size, type_l); break; \
+		case 2: case 6:		_RMOVE(s, d, size, type_w); break; \
 		default:		_RMOVE(s, d, size, type_c); break; \
 	}
 
+/* main RAM to IO memory move */
+#define FAST_MRMOVE_TYPED(s, d, size, t)	\
+{ \
+	t *s1 = (s), *d1 = (d); \
+	register t tmp; \
+	while (size > 0) { \
+		tmp = *s1++; SET_CI; *d1++ = tmp; CLR_CI; size -= sizeof(t); \
+	} \
+}
+
+#define FAST_MRMOVE(s, d, size) \
+	switch (((type_u)(s) | (type_u)(d) | (size)) & (sizeof(type_u)-1)) { \
+	case 0:		FAST_MRMOVE_TYPED(s, d, size, type_u); break; \
+	case 4:		FAST_MRMOVE_TYPED(s, d, size, type_l); break; \
+	case 2: case 6:	FAST_MRMOVE_TYPED(s, d, size, type_w); break; \
+	default:	FAST_MRMOVE_TYPED(s, d, size, type_c); break; \
+	}
+
+/* fill IO memory with pattern */
+#define FAST_RFILL_TYPED(dst, size, pat, t) \
+{ \
+	t *d1 = (dst); \
+	register t tmp = 0; \
+	int i = sizeof(t); \
+	while (i-- > 0) { \
+		tmp <<= 8; tmp |= pat & 0xff; \
+	} \
+	SET_CI; \
+	while (size > 0) { \
+		*d1++ = tmp; size -= sizeof(t); \
+	} \
+	CLR_CI; \
+}
+
+#define FAST_RFILL(dst, size, pat) \
+	switch (((type_u)dst | size) & (sizeof(type_u)-1)) { \
+	case 0:		FAST_RFILL_TYPED(dst, size, pat, type_u); break; \
+	case 4:		FAST_RFILL_TYPED(dst, size, pat, type_l); break; \
+	case 2: case 6:	FAST_RFILL_TYPED(dst, size, pat, type_w); break; \
+	default:	FAST_RFILL_TYPED(dst, size, pat, type_c); break; \
+	}
+
 #endif
diff --git a/roms/SLOF/include/ppcp7/cache.h b/roms/SLOF/include/ppcp7/cache.h
index dc6837196..27975f09c 100644
--- a/roms/SLOF/include/ppcp7/cache.h
+++ b/roms/SLOF/include/ppcp7/cache.h
@@ -81,8 +81,8 @@ cache_inhibited_access(uint64_t, 64)
 #define _FASTMOVE(s, d, size) \
 	switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) { \
 	case 0:			_MOVE(s, d, size, type_u); break;	\
-	case sizeof(type_l):	_MOVE(s, d, size, type_l); break;	\
-	case sizeof(type_w):	_MOVE(s, d, size, type_w); break;	\
+	case 4:			_MOVE(s, d, size, type_l); break;	\
+	case 2: case 6:		_MOVE(s, d, size, type_w); break;	\
 	default:		_MOVE(s, d, size, type_c); break;	\
 	}
 
@@ -116,12 +116,26 @@ static inline void ci_rmove(void *dst, void *src, unsigned long esize,
 #define _FASTRMOVE(s, d, size) do {					      \
 		switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) {\
 		case 0:			ci_rmove(d,s,3,size>>3); break;	      \
-		case sizeof(type_l):	ci_rmove(d,s,2,size>>2); break;	      \
-		case sizeof(type_w):	ci_rmove(d,s,1,size>>1); break;	      \
+		case 4:			ci_rmove(d,s,2,size>>2); break;	      \
+		case 2: case 6:		ci_rmove(d,s,1,size>>1); break;	      \
 		default:		ci_rmove(d,s,0,size); break;	      \
 		}							      \
 	} while(0)
 
+#define FAST_MRMOVE(s, d, size) _FASTRMOVE(s, d, size)
+
+#define FAST_RFILL(dst, size, pat) do { \
+		type_u buf[64]; \
+		char *d = (char *)(dst); \
+		memset(buf, pat, size < sizeof(buf) ? size : sizeof(buf)); \
+		while (size > sizeof(buf)) { \
+			FAST_MRMOVE(buf, d, sizeof(buf)); \
+			d += sizeof(buf); \
+			size -= sizeof(buf); \
+		} \
+		FAST_MRMOVE(buf, d, size); \
+	} while(0)
+
 static inline uint16_t bswap16_load(uint64_t addr)
 {
 	unsigned int val;
diff --git a/roms/SLOF/lib/libusb/usb-hid.c b/roms/SLOF/lib/libusb/usb-hid.c
index f0cab8a69..ac6616aba 100644
--- a/roms/SLOF/lib/libusb/usb-hid.c
+++ b/roms/SLOF/lib/libusb/usb-hid.c
@@ -28,6 +28,10 @@
 #define HID_REQ_SET_IDLE                0x0A
 #define HID_REQ_SET_PROTOCOL            0x0B
 
+//key position for latin letters
+#define KEYP_LATIN_A 4
+#define KEYP_LATIN_Z 29
+
 //#define KEY_DEBUG
 
 /* HID SPEC - 7.2.6 Set_Protocol Request */
@@ -83,6 +87,8 @@ uint8_t set_leds;
 const uint8_t *key_std       = NULL;
 const uint8_t *key_std_shift = NULL;
 
+uint8_t ctrl; /* modifiers */
+
 /**
  * read character from Keyboard-Buffer
  *
@@ -111,6 +117,16 @@ static void write_key(uint8_t key)
 }
 
 /**
+ * Checks if keypos is a latin key
+ * @param  keypos
+ * @return -
+ */
+static bool is_latin(uint8_t keypos)
+{
+	return keypos >= KEYP_LATIN_A && keypos <= KEYP_LATIN_Z;
+}
+
+/**
  * Convert keyboard usage-ID to ANSI-Code
  *
  * @param   Ctrl=Modifier Byte
@@ -120,22 +136,24 @@ static void write_key(uint8_t key)
 static void get_char(uint8_t ctrl, uint8_t keypos)
 {
 	uint8_t ch;
+	bool caps = false;
 
 #ifdef KEY_DEBUG
 	printf("pos %02X\n", keypos);
 #endif
 
 	if (set_leds & LED_CAPS_LOCK)	                /* is CAPS Lock set ? */
-		ctrl |= MODIFIER_SHIFT;	                    /* simulate shift */
+		caps = true;
 
-	if (ctrl == 0) {
+	/* caps is a shift only for latin chars */
+	if ((!caps && ctrl == 0) || (caps && !is_latin(keypos))) {
 		ch = key_std[keypos];
 		if (ch != 0)
 			write_key(ch);
 		return;
 	}
 
-	if (ctrl & MODIFIER_SHIFT) {
+	if ((ctrl & MODIFIER_SHIFT) || caps) {
 		ch = key_std_shift[keypos];
 		if (ch != 0)
 			write_key(ch);
@@ -187,36 +205,38 @@ static void check_key_code(uint8_t *buf)
 					set_leds ^= LED_CAPS_LOCK;
 					break;
 
+				case 0x36:		                /*Shift pressed*/
+					ctrl |= MODIFIER_SHIFT;
+					break;
+				case 0xb6:		                /*Shift unpressed*/
+					ctrl &= ~MODIFIER_SHIFT;
+					break;
 				case 0x3a:	                        /* F1 */
 					write_key(0x1b);
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x31);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x50);
 					break;
 
 				case 0x3b:		                /* F2 */
 					write_key(0x1b);
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x32);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x51);
 					break;
 
 				case 0x3c:
 					write_key(0x1b);               /* F3 */
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x33);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x52);
 					break;
 
 				case 0x3d:
 					write_key(0x1b);		/* F4 */
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x34);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x53);
 					break;
 
 				case 0x3e:
@@ -254,7 +274,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x42:
 					write_key(0x1b);		/* F9 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x30);
 					write_key(0x7e);
 					break;
@@ -262,7 +282,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x43:
 					write_key(0x1b);	       /* F10 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x31);
 					write_key(0x7e);
 					break;
@@ -270,7 +290,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x44:
 					write_key(0x1b);	       /* F11 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x33);
 					write_key(0x7e);
 					break;
@@ -278,7 +298,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x45:
 					write_key(0x1b);	       /* F12 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x34);
 					write_key(0x7e);
 					break;
@@ -290,36 +310,34 @@ static void check_key_code(uint8_t *buf)
 				case 0x49:
 					write_key(0x1b);	       /* INS */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x7e);
 					break;
 
 				case 0x4a:
 					write_key(0x1b);	      /* HOME */
-					write_key(0x5b);
-					write_key(0x32);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x48);
 					break;
 
 				case 0x4b:
 					write_key(0x1b);	      /* PgUp */
 					write_key(0x5b);
-					write_key(0x33);
+					write_key(0x35);
 					write_key(0x7e);
 					break;
 
 				case 0x4c:
 					write_key(0x1b);	       /* DEL */
 					write_key(0x5b);
-					write_key(0x34);
+					write_key(0x33);
 					write_key(0x7e);
 					break;
 
 				case 0x4d:
 					write_key(0x1b);	       /* END */
-					write_key(0x5b);
-					write_key(0x35);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x46);
 					break;
 
 				case 0x4e:
@@ -443,11 +461,8 @@ unsigned char usb_key_available(void *dev)
 
 unsigned char usb_read_keyb(void *vdev)
 {
-	if (!vdev)
-		return false;
-
-	while (usb_poll_key(vdev)) {
-		/* loop for all pending keys */
-	}
-	return read_key();
+	if (usb_key_available(vdev))
+		return read_key();
+	else
+		return 0;
 }
diff --git a/roms/SLOF/lib/libusb/usb-xhci.c b/roms/SLOF/lib/libusb/usb-xhci.c
index 0c3d6e47f..7683c51d6 100644
--- a/roms/SLOF/lib/libusb/usb-xhci.c
+++ b/roms/SLOF/lib/libusb/usb-xhci.c
@@ -225,11 +225,11 @@ static void xhci_handle_cmd_completion(struct xhci_hcd *xhcd,
 		xhcd->slot_id = 0;
 }
 
-static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
-					uint32_t event_type)
+static uint64_t xhci_poll_event(struct xhci_hcd *xhcd,
+				uint32_t event_type)
 {
 	struct xhci_event_trb *event;
-	uint64_t val;
+	uint64_t val, retval = 0;
 	uint32_t flags, time;
 	int index;
 
@@ -244,7 +244,7 @@ static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
 		mb();
 		flags = le32_to_cpu(event->flags);
 		if (time < SLOF_GetTimer())
-			return NULL;
+			return 0;
 	}
 
 	mb();
@@ -273,6 +273,7 @@ static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
 		break;
 	}
 	xhcd->ering.deq = (uint64_t) (event + 1);
+	retval = le64_to_cpu(event->addr);
 
 	event->addr = 0;
 	event->status = 0;
@@ -289,7 +290,11 @@ static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
 	dprintf("Update start %x deq %x index %d\n",
 		xhcd->ering.trbs_dma, val, index/sizeof(*event));
 	write_reg64(&xhcd->run_regs->irs[0].erdp, val);
-	return event;
+
+	if (retval == 0)
+		return (uint64_t)event;
+	else
+		return retval;
 }
 
 static void xhci_send_cmd(struct xhci_hcd *xhcd, uint32_t field1,
@@ -388,10 +393,12 @@ static void xhci_init_seg(struct xhci_seg *seg, uint32_t size, uint32_t type)
 	seg->deq = (uint64_t)seg->trbs;
 	memset((void *)seg->trbs, 0, size);
 
-	link =(struct xhci_link_trb *) (seg->trbs + seg->size - 1);
-	link->addr = cpu_to_le64(seg->trbs_dma);
-	link->field2 = 0;
-	link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
+	if (type != TYPE_EVENT) {
+		link =(struct xhci_link_trb *) (seg->trbs + seg->size - 1);
+		link->addr = cpu_to_le64(seg->trbs_dma);
+		link->field2 = 0;
+		link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
+	}
 	return;
 }
 
@@ -616,6 +623,7 @@ static void xhci_free_dev(struct xhci_dev *xdev)
 {
 	xhci_free_seg(&xdev->bulk_in, XHCI_DATA_TRBS_SIZE);
 	xhci_free_seg(&xdev->bulk_out, XHCI_DATA_TRBS_SIZE);
+	xhci_free_seg(&xdev->intr, XHCI_INTR_TRBS_SIZE);
 	xhci_free_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE);
 	xhci_free_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE);
 	xhci_free_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE);
@@ -637,7 +645,25 @@ static bool usb3_dev_init(struct xhci_hcd *xhcd, uint32_t port)
 	return true;
 }
 
-static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
+static int xhci_device_present(uint32_t portsc, uint32_t usb_ver)
+{
+	if (usb_ver == USB_XHCI) {
+		/* Device present and enabled state */
+		if ((portsc & PORTSC_CCS) &&
+			(portsc & PORTSC_PP) &&
+			(portsc & PORTSC_PED)) {
+			return true;
+		}
+	} else if (usb_ver == USB_EHCI) {
+		/* Device present and in disabled state */
+		if ((portsc & PORTSC_CCS) && (portsc & PORTSC_CSC))
+			return true;
+	}
+	return false;
+}
+
+static int xhci_port_scan(struct xhci_hcd *xhcd,
+			uint32_t usb_ver)
 {
 	uint32_t num_ports, portsc, i;
 	struct xhci_op_regs *op;
@@ -645,7 +671,7 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	struct xhci_cap_regs *cap;
 	uint32_t xecp_off;
 	uint32_t *xecp_addr, *base;
-	uint32_t port_off = 1, port_cnt;
+	uint32_t port_off = 0, port_cnt;
 
 	dprintf("enter\n");
 
@@ -658,14 +684,14 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	base = (uint32_t *)cap;
 	while (xecp_off > 0) {
 		xecp_addr = base + xecp_off;
-		dprintf(stderr, "xecp_off %d %p %p \n", xecp_off, base, xecp_addr);
+		dprintf("xecp_off %d %p %p \n", xecp_off, base, xecp_addr);
 
 		if (XHCI_XECP_CAP_ID(read_reg32(xecp_addr)) == XHCI_XECP_CAP_SP &&
-		    XHCI_XECP_CAP_SP_MJ(read_reg32(xecp_addr)) == 3 &&
+		    XHCI_XECP_CAP_SP_MJ(read_reg32(xecp_addr)) == usb_ver &&
 		    XHCI_XECP_CAP_SP_MN(read_reg32(xecp_addr)) == 0) {
 			port_cnt = XHCI_XECP_CAP_SP_PC(read_reg32(xecp_addr + 2));
 			port_off = XHCI_XECP_CAP_SP_PO(read_reg32(xecp_addr + 2));
-			dprintf(stderr, "PortCount %d Portoffset %d\n", port_cnt, port_off);
+			dprintf("PortCount %d Portoffset %d\n", port_cnt, port_off);
 		}
 		base = xecp_addr;
 		xecp_off = XHCI_XECP_NEXT_PTR(read_reg32(xecp_addr));
@@ -675,10 +701,8 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	for (i = (port_off - 1); i < (port_off + port_cnt - 1); i++) {
 		prs = &op->prs[i];
 		portsc = read_reg32(&prs->portsc);
-		if ((portsc & PORTSC_CCS) &&
-			(portsc & PORTSC_PP) &&
-			(portsc & PORTSC_PED)) {
-			/* Device present and enabled */
+		if (xhci_device_present(portsc, usb_ver)) {
+			/* Device present */
 			dprintf("Device present on port %d\n", i);
 			/* Reset the port */
 			portsc = read_reg32(&prs->portsc);
@@ -701,6 +725,11 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	return true;
 }
 
+static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
+{
+	return xhci_port_scan(xhcd, USB_XHCI) | xhci_port_scan(xhcd, USB_EHCI);
+}
+
 static bool xhci_hcd_init(struct xhci_hcd *xhcd)
 {
 	struct xhci_op_regs *op;
@@ -868,6 +897,18 @@ static bool xhci_hcd_exit(struct xhci_hcd *xhcd)
 		SLOF_dma_map_out(xhcd->dcbaap_dma, (void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 		SLOF_dma_free((void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 	}
+
+	/*
+	 * QEMU implementation of XHCI doesn't implement halt
+	 * properly. It basically says that it's halted immediately
+	 * but doesn't actually terminate ongoing activities and
+	 * DMAs. This needs to be fixed in QEMU.
+	 *
+	 * For now, wait for 50ms grace time till qemu stops using
+	 * this device.
+	 */
+	SLOF_msleep(50);
+
 	return true;
 }
 
@@ -1079,18 +1120,17 @@ static inline struct xhci_seg *xhci_pipe_get_seg(struct usb_pipe *pipe)
 static inline void *xhci_get_trb(struct xhci_seg *seg)
 {
 	uint64_t val, enq;
-	uint32_t size;
+	int index;
 	struct xhci_link_trb *link;
 
 	enq = val = seg->enq;
 	val = val + XHCI_TRB_SIZE;
-	size = seg->size * XHCI_TRB_SIZE;
-        /* TRBs being a cyclic buffer, here we cycle back to beginning. */
-	if ((val % size) == 0) {
+	index = (enq - (uint64_t)seg->trbs) / XHCI_TRB_SIZE + 1;
+	dprintf("%s: enq %llx, val %llx %x\n", __func__, enq, val, index);
+	/* TRBs being a cyclic buffer, here we cycle back to beginning. */
+	if (index == (seg->size - 1)) {
+		dprintf("%s: rounding \n", __func__);
 		seg->enq = (uint64_t)seg->trbs;
-		enq = seg->enq;
-		seg->enq = seg->enq + XHCI_TRB_SIZE;
-		val = 0;
 		seg->cycle_state ^= seg->cycle_state;
 		link = (struct xhci_link_trb *) (seg->trbs + seg->size - 1);
 		link->addr = cpu_to_le64(seg->trbs_dma);
@@ -1105,6 +1145,12 @@ static inline void *xhci_get_trb(struct xhci_seg *seg)
 	return (void *)enq;
 }
 
+static uint64_t xhci_get_trb_phys(struct xhci_seg *seg, uint64_t trb)
+{
+	return seg->trbs_dma + (trb - (uint64_t)seg->trbs);
+}
+
+static int usb_kb = false;
 static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
 			void *data, int datalen)
 {
@@ -1114,7 +1160,8 @@ static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
 	struct xhci_transfer_trb *trb;
 	struct xhci_db_regs *dbr;
 	int ret = true;
-	uint32_t slot_id, epno;
+	uint32_t slot_id, epno, time;
+	uint64_t trb_phys, event_phys;
 
 	if (!pipe->dev || !pipe->dev->hcidev) {
 		dprintf(" NULL pointer\n");
@@ -1139,13 +1186,26 @@ static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
 	}
 
 	trb = xhci_get_trb(seg);
+	trb_phys = xhci_get_trb_phys(seg, (uint64_t)trb);
 	fill_normal_trb(trb, (void *)data, datalen);
 
 	epno = xhci_get_epno(pipe);
 	write_reg32(&dbr->db[slot_id], epno);
-	if (!xhci_poll_event(xhcd, 0)) {
-		dprintf("Bulk failed\n");
-		ret = false;
+
+	time = SLOF_GetTimer() + USB_TIMEOUT;
+	while (1) {
+		event_phys = xhci_poll_event(xhcd, 0);
+		if (event_phys == trb_phys) {
+			break;
+		} else if (event_phys == 0) { /* polling timed out */
+			ret = false;
+			break;
+		} else
+			usb_kb = true;
+
+		/* transfer timed out */
+		if (time < SLOF_GetTimer())
+			return false;
 	}
 	trb->addr = 0;
 	trb->len = 0;
@@ -1214,7 +1274,8 @@ static void xhci_init_bulk_ep(struct usb_dev *dev, struct usb_pipe *pipe)
 
 	if (!seg->trbs) {
 		if (!xhci_alloc_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK)) {
-			dprintf("Failed allocating seg\n");
+			printf("usb-xhci: allocation failed for bulk endpoint\n");
+			return;
 		}
 	} else {
 		xhci_init_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK);
@@ -1235,6 +1296,61 @@ static void xhci_init_bulk_ep(struct usb_dev *dev, struct usb_pipe *pipe)
 	xpipe->seg = seg;
 }
 
+static int xhci_get_pipe_intr(struct usb_pipe *pipe,
+			struct xhci_hcd *xhcd,
+			char *buf, size_t len)
+{
+	struct xhci_dev *xdev;
+	struct xhci_seg *seg;
+	struct xhci_pipe *xpipe;
+	struct xhci_control_ctx *ctrl;
+	struct xhci_ep_ctx *ep;
+	uint32_t x_epno, val, type;
+	struct usb_dev *dev;
+	struct xhci_transfer_trb *trb;
+
+	dev = pipe->dev;
+	if (dev->class != DEV_HID_KEYB)
+		return false;
+
+	xdev = dev->priv;
+	pipe->mps = 8;
+	seg = xhci_pipe_get_seg(pipe);
+	xpipe = xhci_pipe_get_xpipe(pipe);
+	type = EP_INT_IN;
+	seg = &xdev->intr;
+
+	if (!seg->trbs) {
+		if (!xhci_alloc_seg(seg, XHCI_INTR_TRBS_SIZE, TYPE_BULK)) {
+			printf("usb-xhci: allocation failed for interrupt endpoint\n");
+			return false;
+		}
+	} else {
+		xhci_init_seg(seg, XHCI_EVENT_TRBS_SIZE, TYPE_BULK);
+	}
+
+	xpipe->buf = buf;
+	xpipe->buf_phys = SLOF_dma_map_in(buf, len, false);
+	xpipe->buflen = len;
+
+	ctrl = xhci_get_control_ctx(&xdev->in_ctx);
+	x_epno = xhci_get_epno(pipe);
+	ep = xhci_get_ep_ctx(&xdev->in_ctx, xdev->ctx_size, x_epno);
+	val = EP_TYPE(type) | MAX_BURST(0) | ERROR_COUNT(3) |
+		MAX_PACKET_SIZE(pipe->mps);
+	ep->field2 = cpu_to_le32(val);
+	ep->deq_addr = cpu_to_le64(seg->trbs_dma | seg->cycle_state);
+	ep->field4 = cpu_to_le32(8);
+	ctrl->a_flags = cpu_to_le32(BIT(x_epno) | 0x1);
+	ctrl->d_flags = 0;
+	xhci_configure_ep(xhcd, xdev->slot_id, xdev->in_ctx.dma_addr);
+	xpipe->seg = seg;
+
+	trb = xhci_get_trb(seg);
+	fill_normal_trb(trb, (void *)xpipe->buf_phys, pipe->mps);
+	return true;
+}
+
 static struct usb_pipe* xhci_get_pipe(struct usb_dev *dev, struct usb_ep_descr *ep, char *buf, size_t len)
 {
 	struct xhci_hcd *xhcd;
@@ -1264,6 +1380,12 @@ static struct usb_pipe* xhci_get_pipe(struct usb_dev *dev, struct usb_ep_descr *
 	new->dir = (ep->bEndpointAddress & 0x80) >> 7;
 	new->epno = ep->bEndpointAddress & 0x0f;
 
+	if (new->type == USB_EP_TYPE_INTR) {
+		if (!xhci_get_pipe_intr(new, xhcd, buf, len)) {
+			printf("usb-xhci: %s alloc_intr failed  %p\n",
+				__func__, new);
+		}
+	}
 	if (new->type == USB_EP_TYPE_BULK)
 		xhci_init_bulk_ep(dev, new);
 
@@ -1284,6 +1406,10 @@ static void xhci_put_pipe(struct usb_pipe *pipe)
 	if (pipe->type == USB_EP_TYPE_BULK) {
 		xpipe = xhci_pipe_get_xpipe(pipe);
 		xpipe->seg = NULL;
+	} else if (pipe->type == USB_EP_TYPE_INTR) {
+		xpipe = xhci_pipe_get_xpipe(pipe);
+		SLOF_dma_map_out(xpipe->buf_phys, xpipe->buf, xpipe->buflen);
+		xpipe->seg = NULL;
 	}
 	if (xhcd->end)
 		xhcd->end->next = pipe;
@@ -1298,6 +1424,51 @@ static void xhci_put_pipe(struct usb_pipe *pipe)
 	dprintf("usb-xhci: %s exit\n", __func__);
 }
 
+static int xhci_poll_intr(struct usb_pipe *pipe, uint8_t *data)
+{
+	struct xhci_transfer_trb *trb;
+	struct xhci_seg *seg;
+	struct xhci_pipe *xpipe;
+	struct xhci_dev *xdev;
+	struct xhci_hcd *xhcd;
+	struct xhci_db_regs *dbr;
+	uint32_t x_epno;
+	uint8_t *buf, ret = 1;
+
+	if (!pipe || !pipe->dev || !pipe->dev->hcidev)
+		return 0;
+	xdev = pipe->dev->priv;
+	xhcd = (struct xhci_hcd *)pipe->dev->hcidev->priv;
+	x_epno = xhci_get_epno(pipe);
+	seg = xhci_pipe_get_seg(pipe);
+	xpipe = xhci_pipe_get_xpipe(pipe);
+
+	if (usb_kb == true) {
+		/* This event was consumed by bulk transfer */
+		usb_kb = false;
+		goto skip_poll;
+	}
+	buf = xpipe->buf;
+	memset(buf, 0, 8);
+
+	mb();
+	/* Ring the doorbell - x_epno */
+	dbr = xhcd->db_regs;
+	write_reg32(&dbr->db[xdev->slot_id], x_epno);
+	if (!xhci_poll_event(xhcd, 0)) {
+		printf("poll intr failed\n");
+		return 0;
+	}
+	mb();
+	memcpy(data, buf, 8);
+
+skip_poll:
+	trb = xhci_get_trb(seg);
+	fill_normal_trb(trb, (void *)xpipe->buf_phys, pipe->mps);
+	mb();
+	return ret;
+}
+
 struct usb_hcd_ops xhci_ops = {
 	.name          = "xhci-hcd",
 	.init          = xhci_init,
@@ -1305,6 +1476,7 @@ struct usb_hcd_ops xhci_ops = {
 	.usb_type      = USB_XHCI,
 	.get_pipe      = xhci_get_pipe,
 	.put_pipe      = xhci_put_pipe,
+	.poll_intr     = xhci_poll_intr,
 	.send_ctrl     = xhci_send_ctrl,
 	.transfer_bulk = xhci_transfer_bulk,
 	.next          = NULL,
diff --git a/roms/SLOF/lib/libusb/usb-xhci.h b/roms/SLOF/lib/libusb/usb-xhci.h
index faeb07ead..3fc7e7889 100644
--- a/roms/SLOF/lib/libusb/usb-xhci.h
+++ b/roms/SLOF/lib/libusb/usb-xhci.h
@@ -266,6 +266,7 @@ struct xhci_seg {
 #define XHCI_EVENT_TRBS_SIZE   4096
 #define XHCI_CONTROL_TRBS_SIZE 4096
 #define XHCI_DATA_TRBS_SIZE    4096
+#define XHCI_INTR_TRBS_SIZE    4096
 #define XHCI_ERST_NUM_SEGS     1
 
 #define XHCI_MAX_BULK_SIZE    0xF000
@@ -349,6 +350,7 @@ struct xhci_dev {
 	struct xhci_ctx in_ctx;
 	struct xhci_ctx out_ctx;
 	struct xhci_seg control;
+	struct xhci_seg intr;
 	struct xhci_seg bulk_in;
 	struct xhci_seg bulk_out;
 	uint32_t ctx_size;
@@ -381,6 +383,9 @@ struct xhci_hcd {
 struct xhci_pipe {
 	struct usb_pipe pipe;
 	struct xhci_seg *seg;
+	void *buf;
+	long buf_phys;
+	uint32_t buflen;
 };
 
 #endif	/* USB_XHCI_H */
diff --git a/roms/SLOF/make.rules b/roms/SLOF/make.rules
index aebc4e360..cbc63530a 100644
--- a/roms/SLOF/make.rules
+++ b/roms/SLOF/make.rules
@@ -19,8 +19,12 @@
 ARCH := $(shell uname -p)
 
 # Auto-detect ppc64
-ifeq ($(ARCH), ppc64)
-CROSS          = ""
+ifeq (ppc64,$(findstring ppc64,$(ARCH)))
+  ifeq ($(ARCH), ppc64le)
+    EXTRA_CC	= -mbig -mabi=elfv1
+    EXTRA_LD	= -mbig
+  endif
+CROSS		?=
 else
 CROSS		?= powerpc64-linux-
 endif
@@ -31,8 +35,8 @@ HOSTCC		?= gcc
 HOSTCFLAGS	= -g -Wall -W -O2 -I. -I../include
 DD		= dd
 
-ONLY_CC 	= $(CROSS)gcc -m$(CELLSIZE)
-ONLY_AS 	= $(CROSS)as -m$(CELLSIZE)
+ONLY_CC 	= $(CROSS)gcc -m$(CELLSIZE) $(EXTRA_CC)
+ONLY_AS 	= $(CROSS)as -m$(CELLSIZE) $(EXTRA_LD)
 ONLY_LD 	= $(CROSS)ld -melf$(CELLSIZE)ppc
 
 # Verbose level:
diff --git a/roms/SLOF/rtas/reloc.S b/roms/SLOF/rtas/reloc.S
index e24d293d4..1b5b59a68 100644
--- a/roms/SLOF/rtas/reloc.S
+++ b/roms/SLOF/rtas/reloc.S
@@ -61,7 +61,7 @@ _rtas_start:
 ._rtas_entry_offset:	.quad   rtas_entry-_rtas_start
 ._rtas_config_offset:	.quad   rtas_config-_rtas_start
 ._rtas_stack:	        .quad   .stack-_rtas_start+RTAS_STACKSIZE-0x60
-._rtas_toc:	        .quad   _got-_rtas_start+0x8000
+._rtas_toc:	        .quad   _got-_rtas_start
 
 .over:	
 	mflr r8			# gpr 8 is the base
diff --git a/roms/SLOF/rtas/rtas.lds b/roms/SLOF/rtas/rtas.lds
index a5ba1daaf..30b18dd26 100644
--- a/roms/SLOF/rtas/rtas.lds
+++ b/roms/SLOF/rtas/rtas.lds
@@ -28,7 +28,8 @@ SECTIONS {
 	}
 	.got :
 	{
-	  _got = .;
+	  . = ALIGN(256);
+	  _got = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
 	  *(.got .toc)
 	}  	
 	.reloc :
diff --git a/roms/SLOF/rtas/rtas_entry.S b/roms/SLOF/rtas/rtas_entry.S
index 74693aa48..424137bf5 100644
--- a/roms/SLOF/rtas/rtas_entry.S
+++ b/roms/SLOF/rtas/rtas_entry.S
@@ -39,7 +39,7 @@ rtas_entry:
 	bcl 20,31,.over		# branch to over 
 .base:	
 	.align  3
-..got:			.quad   _got-.base+0x8000
+..got:			.quad   _got-.base
 ..stack:		.quad   .stack+RTAS_STACKSIZE-0x60-.base
 .over:	
 	mflr r8			# gpr 8 is the base
diff --git a/roms/SLOF/slof/entry.S b/roms/SLOF/slof/entry.S
index dcff57ba0..d3d29f852 100644
--- a/roms/SLOF/slof/entry.S
+++ b/roms/SLOF/slof/entry.S
@@ -207,4 +207,13 @@ call_client:
 	li 3, -1 # client app return
 	blr
 
+
+	# Call another function via pointer in r6
+	# (arguments can be provided in r3 to r5)
+	# Destination function should jump back to lr
+C_ENTRY(call_c)
+	mtctr	r6
+	bctr
+
+
 	.lcomm	the_system_stack, STACKSIZE, 16
diff --git a/roms/SLOF/slof/fs/archsupport.fs b/roms/SLOF/slof/fs/archsupport.fs
index cc4668769..f564ab4e0 100644
--- a/roms/SLOF/slof/fs/archsupport.fs
+++ b/roms/SLOF/slof/fs/archsupport.fs
@@ -10,9 +10,9 @@
 \ *     IBM Corporation - initial implementation
 \ ****************************************************************************/
 
-\ Qemu supports max 256cpus, 32K will be able to accomodate the fdt changes if
-\ needed.
-8000 VALUE size
+\ 128KB FDT buffer size is enough to accommodate 255 CPU cores and 1TB of
+\ maxmem specification.
+20000 VALUE size
 : ibm,client-architecture-support         ( vec -- err? )
     \ Store require parameters in nvram
     \ to come back to right boot device
diff --git a/roms/SLOF/slof/fs/base.fs b/roms/SLOF/slof/fs/base.fs
index e71e087eb..03e77e54f 100644
--- a/roms/SLOF/slof/fs/base.fs
+++ b/roms/SLOF/slof/fs/base.fs
@@ -579,8 +579,6 @@ defer cursor-off ( -- )
 #include "debug.fs"
 \ provide 7.5.3.1 Dictionary search
 #include "dictionary.fs"
-\ block data access for IO devices - ought to be implemented in engine
-#include "rmove.fs"
 \ provide a simple run time preprocessor
 #include <preprocessor.fs>
 
diff --git a/roms/SLOF/slof/fs/boot.fs b/roms/SLOF/slof/fs/boot.fs
index 9a0ded0c2..a0fe29a1b 100644
--- a/roms/SLOF/slof/fs/boot.fs
+++ b/roms/SLOF/slof/fs/boot.fs
@@ -187,11 +187,6 @@ defer go ( -- )
       dup to my-self
       dup ihandle>phandle set-node
       -rot                              ( ihandle devstr len )
-      my-args nip 0= IF
-	 2dup 1- + c@ [char] : <> IF    \ Add : to device path if missing
-	    1+ strdup 2dup 1- + [char] : swap c!
-	 THEN
-      THEN
       encode-string s" bootpath" set-chosen
       $bootargs encode-string s" bootargs" set-chosen
       get-load-base s" load" 3 pick ['] $call-method CATCH IF
@@ -211,7 +206,7 @@ defer go ( -- )
 
 : parse-load ( "{devlist}" -- success )	\ Parse-execute boot-device list
    cr BEGIN parse-word dup WHILE
-	 ( de-alias ) do-load dup 0< IF drop 0 THEN IF
+	 de-alias do-load dup 0< IF drop 0 THEN IF
 	    state-valid @ IF ."   Successfully loaded" cr THEN
 	    true 0d parse strdup load-list 2! EXIT
 	 THEN
diff --git a/roms/SLOF/slof/fs/client.fs b/roms/SLOF/slof/fs/client.fs
index 1b2bb0326..7d537a668 100644
--- a/roms/SLOF/slof/fs/client.fs
+++ b/roms/SLOF/slof/fs/client.fs
@@ -282,6 +282,18 @@ ALSO client-voc DEFINITIONS
 ;
 
 \
+\ Standard for Boot, defined in 6.3.2.5:
+\
+: boot  ( zstr -- )
+   zcount
+   debug-client-interface? IF
+      ." ci: boot " 2dup type cr
+   THEN
+   " boot " 2swap $cat " boot-command" $setenv (nvupdate)
+   reset-all
+;
+
+\
 \ User Interface, defined in 6.3.2.6
 \
 : interpret ( ... zstr -- result ... )
diff --git a/roms/SLOF/slof/fs/fbuffer.fs b/roms/SLOF/slof/fs/fbuffer.fs
index 756f05a95..47046087d 100644
--- a/roms/SLOF/slof/fs/fbuffer.fs
+++ b/roms/SLOF/slof/fs/fbuffer.fs
@@ -19,6 +19,7 @@
 0 VALUE screen-height
 0 VALUE screen-width
 0 VALUE screen-depth
+0 VALUE screen-line-bytes
 0 VALUE window-top
 0 VALUE window-left
 
@@ -54,10 +55,10 @@
 : fb8-background inverse? ;
 : fb8-foreground inverse? invert ;
 
-: fb8-lines2bytes ( #lines -- #bytes ) char-height * screen-width * screen-depth * ;
+: fb8-lines2bytes ( #lines -- #bytes ) char-height * screen-line-bytes * ;
 : fb8-columns2bytes ( #columns -- #bytes ) char-width * screen-depth * ;
 : fb8-line2addr ( line# -- addr )
-	char-height * window-top + screen-width * screen-depth *
+	char-height * window-top + screen-line-bytes *
 	frame-buffer-adr + window-left screen-depth * +
 ;
 
@@ -98,9 +99,10 @@ CREATE bitmap-buffer 400 4 * allot
 
 : fb8-toggle-cursor ( -- )
 	line# fb8-line2addr column# fb8-columns2bytes +
-	char-height 0 ?DO
-		char-width screen-depth * 0 ?DO dup dup rb@ -1 xor swap rb! 1+ LOOP
-		screen-width screen-depth * + char-width screen-depth * -
+	char-height 2 - screen-line-bytes * +
+	2 0 ?DO
+		dup char-width screen-depth * invert-region
+		screen-line-bytes +
 	LOOP drop
 ;
 
@@ -110,7 +112,7 @@ CREATE bitmap-buffer 400 4 * allot
 	line# fb8-line2addr column# fb8-columns2bytes + ( bitmap-buf fb-addr )
 	char-height 0 ?DO
 		2dup char-width screen-depth * mrmove
-		screen-width screen-depth * + >r char-width screen-depth * + r>
+		screen-line-bytes + >r char-width screen-depth * + r>
 	LOOP 2drop
     ELSE 2drop r> 3drop THEN
 ;
@@ -135,12 +137,12 @@ CREATE bitmap-buffer 400 4 * allot
 		fb8-columns2bytes swap fb8-columns2bytes tuck -
 		over r@ tuck + rot char-height 0 ?DO
 			3dup rmove
-			-rot screen-width screen-depth * tuck + -rot + swap rot
+			-rot screen-line-bytes tuck + -rot + swap rot
 		LOOP
 		3drop r>
 	THEN
 	char-height 0 ?DO
-		dup 2 pick fb8-erase-block screen-width screen-depth * +
+		dup 2 pick fb8-erase-block screen-line-bytes +
 	LOOP
 	2drop
 ;
@@ -153,12 +155,12 @@ CREATE bitmap-buffer 400 4 * allot
 		fb8-columns2bytes swap fb8-columns2bytes tuck -
 		over r@ + 2dup + r> swap >r rot char-height 0 ?DO
 			3dup rmove
-			-rot screen-width screen-depth * tuck + -rot + swap rot
+			-rot screen-line-bytes tuck + -rot + swap rot
 		LOOP
 		3drop r> over -
 	THEN
 	char-height 0 ?DO
-		dup 2 pick fb8-erase-block screen-width screen-depth * +
+		dup 2 pick fb8-erase-block screen-line-bytes +
 	LOOP
 	2drop
 ;
@@ -166,13 +168,11 @@ CREATE bitmap-buffer 400 4 * allot
 : fb8-reset-screen ( -- ) ( Left as no-op by design ) ;
 
 : fb8-erase-screen ( -- )
-	frame-buffer-adr screen-height screen-width * screen-depth * fb8-erase-block
+	frame-buffer-adr screen-height screen-line-bytes * fb8-erase-block
 ;
 
 : fb8-invert-screen ( -- )
-	frame-buffer-adr screen-height screen-width * screen-depth * 2dup /x / 0 ?DO
-		dup rx@ -1 xor over rx! xa1+
-	LOOP 3drop
+	frame-buffer-adr screen-height screen-line-bytes * invert-region
 ;
 
 : fb8-blink-screen ( -- ) fb8-invert-screen fb8-invert-screen ;
@@ -180,6 +180,7 @@ CREATE bitmap-buffer 400 4 * allot
 : fb8-install ( width height #columns #lines -- )
 	1 to screen-depth
 	2swap  to screen-height  to screen-width
+	screen-width to screen-line-bytes
 	screen-#rows min to #lines
 	screen-#columns min to #columns
 	screen-height char-height #lines * - 2/ to window-top
@@ -201,6 +202,7 @@ CREATE bitmap-buffer 400 4 * allot
 	>r
 	fb8-install
 	r> to screen-depth
+	screen-width screen-depth * to screen-line-bytes
 ;
 
 
diff --git a/roms/SLOF/slof/fs/little-endian.fs b/roms/SLOF/slof/fs/little-endian.fs
index f2e4e8d42..6b4779ee0 100644
--- a/roms/SLOF/slof/fs/little-endian.fs
+++ b/roms/SLOF/slof/fs/little-endian.fs
@@ -17,6 +17,9 @@ here c@ ef = CONSTANT ?littleendian
 
 ?bigendian [IF]
 
+: x!-le >r xbflip r> x! ;
+: x@-le x@ xbflip ;
+
 : l!-le  >r lbflip r> l! ;
 : l@-le  l@ lbflip ;
 
@@ -47,6 +50,9 @@ here c@ ef = CONSTANT ?littleendian
 
 [ELSE]
 
+: x!-le x! ;
+: x@-le x@ ;
+
 : l!-le  l! ;
 : l@-le  l@ ;
 
diff --git a/roms/SLOF/slof/fs/packages/disk-label.fs b/roms/SLOF/slof/fs/packages/disk-label.fs
index fe1c25e7a..e034d6408 100644
--- a/roms/SLOF/slof/fs/packages/disk-label.fs
+++ b/roms/SLOF/slof/fs/packages/disk-label.fs
@@ -20,6 +20,7 @@ false VALUE debug-disk-label?
 \ If we ever want to put a large kernel with initramfs from a PREP partition
 \ we might need to increase this value. The default value is 65536 blocks (32MB)
 d# 65536 value max-prep-partition-blocks
+d# 4096 CONSTANT block-array-size
 
 s" disk-label" device-name
 
@@ -152,8 +153,8 @@ CONSTANT /gpt-part-entry
 : init-block ( -- )
    s" block-size" ['] $call-parent CATCH IF ABORT" parent has no block-size." THEN
    to block-size
-   d# 4096 alloc-mem
-   dup d# 4096 erase
+   block-array-size alloc-mem
+   dup block-array-size erase
    to block
    debug-disk-label? IF
       ." init-block: block-size=" block-size .d ." block=0x" block u. cr
@@ -178,7 +179,8 @@ CONSTANT /gpt-part-entry
 \ This word returns true if the currently loaded block has _NO_ GPT partition id
 : no-gpt? ( -- true|false )
    0 read-sector
-   1 partition>part-entry part-entry>id c@ ee <>
+   1 partition>part-entry part-entry>id c@ ee <> IF true EXIT THEN
+   block mbr>magic w@-le aa55 <>
 ;
 
 : pc-extended-partition? ( part-entry-addr -- true|false )
@@ -266,7 +268,10 @@ CONSTANT /gpt-part-entry
 
 : try-dos-partition ( -- okay? )
    \ Read partition table and check magic.
-   no-mbr? IF cr ." No DOS disk-label found." cr false EXIT THEN
+   no-mbr? IF
+       debug-disk-label? IF cr ." No DOS disk-label found." cr THEN
+       false EXIT
+   THEN
 
    count-dos-logical-partitions TO dos-logical-partitions
 
@@ -320,6 +325,14 @@ CONSTANT /gpt-part-entry
 
 \ Load from first active DOS boot partition.
 
+: fat-bootblock? ( addr -- flag )
+   \ byte 0-2 of the bootblock is a jump instruction in
+   \ all FAT filesystems.
+   \ e9 and eb are jump instructions in x86 assembler.
+   dup c@ e9 = IF drop true EXIT THEN
+   dup c@ eb = swap 2+ c@ 90 = and
+;
+
 \ NOTE: block-size is always 512 bytes for DOS partition tables.
 
 : load-from-dos-boot-partition ( addr -- size )
@@ -352,60 +365,103 @@ CONSTANT /gpt-part-entry
    drop 0
 ;
 
-\ Check for GPT PReP partition GUID
-9E1A2D38     CONSTANT GPT-PREP-PARTITION-1
-C612         CONSTANT GPT-PREP-PARTITION-2
-4316         CONSTANT GPT-PREP-PARTITION-3
-AA26         CONSTANT GPT-PREP-PARTITION-4
-8B49521E5A8B CONSTANT GPT-PREP-PARTITION-5
+\ Check for GPT PReP partition GUID. Only first 3 blocks are
+\ byte-swapped treating last two blocks as contigous for simplifying
+\ comparison
+9E1A2D38            CONSTANT GPT-PREP-PARTITION-1
+C612                CONSTANT GPT-PREP-PARTITION-2
+4316                CONSTANT GPT-PREP-PARTITION-3
+AA268B49521E5A8B    CONSTANT GPT-PREP-PARTITION-4
 
 : gpt-prep-partition? ( -- true|false )
-   block gpt-part-entry>part-type-guid l@-le GPT-PREP-PARTITION-1 = IF
-      block gpt-part-entry>part-type-guid 4 + w@-le
-      GPT-PREP-PARTITION-2 = IF
-         block gpt-part-entry>part-type-guid 6 + w@-le
-         GPT-PREP-PARTITION-3 = IF
-            block gpt-part-entry>part-type-guid 8 + w@
-            GPT-PREP-PARTITION-4 = IF
-               block gpt-part-entry>part-type-guid a + w@
-               block gpt-part-entry>part-type-guid c + l@ swap lxjoin
-               GPT-PREP-PARTITION-5 = IF
-                   TRUE EXIT
-               THEN
-            THEN
-         THEN
-      THEN
+   block gpt-part-entry>part-type-guid
+   dup l@-le     GPT-PREP-PARTITION-1 <> IF drop false EXIT THEN
+   dup 4 + w@-le GPT-PREP-PARTITION-2 <> IF drop false EXIT THEN
+   dup 6 + w@-le GPT-PREP-PARTITION-3 <> IF drop false EXIT THEN
+       8 + x@    GPT-PREP-PARTITION-4 =
+;
+
+\ Check for GPT MSFT BASIC DATA GUID - fat based
+EBD0A0A2            CONSTANT GPT-BASIC-DATA-PARTITION-1
+B9E5                CONSTANT GPT-BASIC-DATA-PARTITION-2
+4433                CONSTANT GPT-BASIC-DATA-PARTITION-3
+87C068B6B72699C7    CONSTANT GPT-BASIC-DATA-PARTITION-4
+
+: gpt-basic-data-partition? ( -- true|false )
+   block gpt-part-entry>part-type-guid
+   dup l@-le     GPT-BASIC-DATA-PARTITION-1 <> IF drop false EXIT THEN
+   dup 4 + w@-le GPT-BASIC-DATA-PARTITION-2 <> IF drop false EXIT THEN
+   dup 6 + w@-le GPT-BASIC-DATA-PARTITION-3 <> IF drop false EXIT THEN
+       8 + x@    GPT-BASIC-DATA-PARTITION-4 =
+;
+
+\
+\ GPT Signature
+\ ("EFI PART", 45h 46h 49h 20h 50h 41h 52h 54h)
+\
+4546492050415254 CONSTANT GPT-SIGNATURE
+
+\ The routine checks whether the protective MBR has GPT ID and then
+\ reads the gpt data from the sector. Also set the seek position and
+\ the partition size used in caller routines.
+
+: get-gpt-partition ( -- true|false )
+   no-gpt? IF false EXIT THEN
+   debug-disk-label? IF cr ." GPT partition found " cr  THEN
+   1 read-sector
+   block gpt>part-entry-lba x@-le
+   block-size * to seek-pos
+   block gpt>part-entry-size l@-le to gpt-part-size
+   gpt-part-size block-array-size > IF
+      cr ." GPT part size exceeds buffer allocated " cr
+      false exit
    THEN
-   FALSE
+   block gpt>signature x@ GPT-SIGNATURE =
 ;
 
 : load-from-gpt-prep-partition ( addr -- size )
-   no-gpt? IF drop FALSE EXIT THEN
-   debug-disk-label? IF
-      cr ." GPT partition found " cr
-   THEN
-   1 read-sector block gpt>part-entry-lba l@-le
-   block-size * to seek-pos
-   block gpt>part-entry-size l@-le to gpt-part-size
-   block gpt>num-part-entry l@-le dup 0= IF FALSE EXIT THEN
+   get-gpt-partition 0= IF false EXIT THEN
+   block gpt>num-part-entry l@-le dup 0= IF false exit THEN
    1+ 1 ?DO
       seek-pos 0 seek drop
       block gpt-part-size read drop gpt-prep-partition? IF
-         debug-disk-label? IF
-            ." GPT PReP partition found " cr
-         THEN
-         block gpt-part-entry>first-lba x@ xbflip
-         block gpt-part-entry>last-lba x@ xbflip
-         over - 1+                 ( addr offset len )
-         swap                      ( addr len offset )
-         block-size * to part-offset
-         0 0 seek drop             ( addr len )
-         block-size * read         ( size )
+         debug-disk-label? IF  ." GPT PReP partition found " cr THEN
+         block gpt-part-entry>first-lba x@-le ( addr first-lba )
+         block gpt-part-entry>last-lba x@-le  ( addr first-lba last-lba)
+         over - 1+                            ( addr first-lba blocks )
+         swap                                 ( addr blocks first-lba )
+         block-size * to part-offset          ( addr blocks )
+         0 0 seek drop                        ( addr blocks )
+         block-size * read                    ( size )
+         UNLOOP EXIT
+     THEN
+     seek-pos gpt-part-size + to seek-pos
+   LOOP
+   false
+;
+
+: try-gpt-dos-partition ( -- true|false )
+   get-gpt-partition 0= IF false EXIT THEN
+   block gpt>num-part-entry l@-le dup 0= IF false EXIT THEN
+   1+ 1 ?DO
+      seek-pos 0 seek drop
+      block gpt-part-size read drop
+      gpt-basic-data-partition? IF
+         debug-disk-label? IF ." GPT BASIC DATA partition found " cr THEN
+         block gpt-part-entry>first-lba x@-le       ( first-lba )
+         dup to part-start                          ( first-lba )
+         block gpt-part-entry>last-lba x@-le        ( first-lba last-lba )
+         over - 1+                                  ( first-lba s1 )
+         block-size * to part-size                  ( first-lba )
+         block-size * to part-offset                ( )
+         0 0 seek drop
+         block block-size read drop
+         block fat-bootblock?                    ( true|false )
          UNLOOP EXIT
       THEN
-      seek-pos gpt-part-size i * + to seek-pos
+      seek-pos gpt-part-size + to seek-pos
    LOOP
-   FALSE
+   false
 ;
 
 \ Extract the boot loader path from a bootinfo.txt file
@@ -493,7 +549,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 
    debug-disk-label? IF ." Trying CHRP boot " .s cr THEN
    1 disk-chrp-boot !
-   dup load-chrp-boot-file ?dup 0 <> IF .s cr nip EXIT THEN
+   dup load-chrp-boot-file ?dup 0 <> IF nip EXIT THEN
    0 disk-chrp-boot !
 
    debug-disk-label? IF ." Trying GPT boot " .s cr THEN
@@ -558,14 +614,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 : try-dos-files ( -- found? )
    no-mbr? IF false EXIT THEN
 
-   \ block 0 byte 0-2 is a jump instruction in all FAT
-   \ filesystems.
-   \ e9 and eb are jump instructions in x86 assembler.
-   block c@ e9 <> IF
-      block c@ eb <>
-      block 2+ c@ 90 <> or
-      IF false EXIT THEN
-   THEN
+   block fat-bootblock? 0= IF false EXIT THEN
    s" fat-files" (interpose-filesystem)
    true
 ;
@@ -600,6 +649,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 
 : try-partitions ( -- found? )
    try-dos-partition IF try-files EXIT THEN
+   try-gpt-dos-partition IF try-files EXIT THEN
    \ try-iso9660-partition IF try-files EXIT THEN
    \ ... more partition types here...
    false
@@ -610,7 +660,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 
 : close ( -- )
    debug-disk-label? IF ." Closing disk-label: block=0x" block u. ." block-size=" block-size .d cr THEN
-   block d# 4096 free-mem
+   block block-array-size free-mem
 ;
 
 
diff --git a/roms/SLOF/slof/fs/pci-scan.fs b/roms/SLOF/slof/fs/pci-scan.fs
index b8b9fe61f..2fdf0e8f5 100644
--- a/roms/SLOF/slof/fs/pci-scan.fs
+++ b/roms/SLOF/slof/fs/pci-scan.fs
@@ -110,10 +110,13 @@ here 100 allot CONSTANT pci-device-vec
         dup 100000 + pci-next-mem !             \ and write back with 1MB for bridge
         over 24 + rtas-config-w@                \ check if 64bit support
         1 and IF                                \ IF 64 bit support
-                2dup 20 rshift                  \ | keep upper 32 bits
-                swap 28 + rtas-config-l!        \ | and write it into the Base-Upper32-bits
-                pci-max-mem @ 20 rshift         \ | fetch max Limit address and keep upper 32 bits
-                2 pick 2C + rtas-config-l!      \ | and set the Limit
+                pci-next-mem64 @ 100000000 #aligned \ | read the current Value of 64-bit and align to 4GB boundary
+                dup 100000000 + pci-next-mem64 x!   \ | and write back with 1GB for bridge
+                2 pick swap                         \ |
+                20 rshift                           \ | keep upper 32 bits
+                swap 28 + rtas-config-l!            \ | and write it into the Base-Upper32-bits
+                pci-max-mem64 @ 20 rshift           \ | fetch max Limit address and keep upper 32 bits
+                2 pick 2C + rtas-config-l!          \ | and set the Limit
         THEN                                    \ FI
         10 rshift                               \ keep upper 16 bits
         pci-max-mem @ 1- FFFF0000 and or        \ and Insert mmem Limit (set it to max)
@@ -129,8 +132,12 @@ here 100 allot CONSTANT pci-device-vec
         1-                                      \ make limit one less than boundary
         over 24 + rtas-config-w@                \ check if 64bit support
         1 and IF                                \ IF 64 bit support
-                2dup 20 rshift                  \ | keep upper 32 bits
-                swap 2C + rtas-config-l!        \ | and write it into the Limit-Upper32-bits
+                pci-next-mem64 @ 100000000 #aligned \ | Reat current value of 64-bar and align at 4GB
+                dup pci-next-mem64 x!               \ | and write it back
+                1-                                  \ | make limite one less than boundary
+                2 pick swap                         \ |
+                20 rshift                           \ | keep upper 32 bits
+                swap 2C + rtas-config-l!            \ | and write it into the Limit-Upper32-bits
         THEN                                    \ FI
         FFFF0000 and                            \ keep upper 16 bits
         over 24 + rtas-config-l@ 0000FFFF and   \ fetch original Value
diff --git a/roms/SLOF/slof/fs/rmove.fs b/roms/SLOF/slof/fs/rmove.fs
deleted file mode 100644
index c28dba9c4..000000000
--- a/roms/SLOF/slof/fs/rmove.fs
+++ /dev/null
@@ -1,53 +0,0 @@
-\ *****************************************************************************
-\ * Copyright (c) 2004, 2008 IBM Corporation
-\ * All rights reserved.
-\ * This program and the accompanying materials
-\ * are made available under the terms of the BSD License
-\ * which accompanies this distribution, and is available at
-\ * http://www.opensource.org/licenses/bsd-license.php
-\ *
-\ * Contributors:
-\ *     IBM Corporation - initial implementation
-\ ****************************************************************************/
-
-defer '(r@)
-defer '(r!)
-1 VALUE /(r)
-
-
-\ The rest of the code already implemented in prim.in
-\ In the end all of this should be moved over there and this file terminated
-
-: (rfill) ( addr size pattern 'r! /r -- )
-	to /(r) to '(r!) ff and
-	dup 8 lshift or dup 10 lshift or dup 20 lshift or
-	-rot bounds ?do dup i '(r!) /(r) +loop drop
-;
-
-: (fwrmove) ( src dest size -- )
-	>r 0 -rot r> bounds ?do + dup '(r@) i '(r!) /(r) dup +loop 2drop
-;
-
-\ Move from main to device memory
-: mrmove ( src dest size -- )
-	3dup or or 7 AND CASE
-		0 OF ['] x@ ['] rx! /x ENDOF
-		4 OF ['] l@ ['] rl! /l ENDOF
-		2 OF ['] w@ ['] rw! /w ENDOF
-		dup OF ['] c@ ['] rb! /c ENDOF
-	ENDCASE
-	( We already know that source and destination do not overlap )
-	to /(r) to '(r!) to '(r@) (fwrmove)
-;
-
-: rfill ( addr size pattern -- )
-	3dup drop or 7 AND CASE
-		0 OF ['] rx! /x ENDOF
-		4 OF ['] rl! /l ENDOF
-		2 OF ['] rw! /w ENDOF
-		dup OF ['] rb! /c ENDOF
-	ENDCASE (rfill)
-;
-
-
-
diff --git a/roms/SLOF/slof/fs/terminal.fs b/roms/SLOF/slof/fs/terminal.fs
index 582bedeb3..dc82e7bf4 100644
--- a/roms/SLOF/slof/fs/terminal.fs
+++ b/roms/SLOF/slof/fs/terminal.fs
@@ -167,6 +167,7 @@ false VALUE stopcsi
 CREATE twtracebuf 4000 allot twtracebuf 4000 erase
 twtracebuf VALUE twbp
 0 VALUE twbc
+0 VALUE twtrace-enabled?
 
 : twtrace
 	twbc 4000 = IF 0 to twbc twtracebuf to twbp THEN
@@ -176,7 +177,7 @@ twtracebuf VALUE twbp
 : terminal-write ( addr len -- actual-len )
  	cursor-off
 	tuck bounds ?DO i c@
-		twtrace
+		twtrace-enabled? IF twtrace THEN
 		esc-on IF esc-process
 		ELSE CASE
 			1B OF true to esc-on ENDOF
diff --git a/roms/SLOF/slof/ppc64.c b/roms/SLOF/slof/ppc64.c
index 20d927069..619d95ec7 100644
--- a/roms/SLOF/slof/ppc64.c
+++ b/roms/SLOF/slof/ppc64.c
@@ -42,24 +42,7 @@ cell *the_heap_start = &the_heap[0];
 cell *the_heap_end = &the_heap[HEAP_SIZE / CELLSIZE];
 
 extern void io_putchar(unsigned char);
-
-
-static unsigned long __attribute__((noinline))
-call_c(cell arg0, cell arg1, cell arg2, cell entry)
-{
-	register unsigned long r3 asm("r3") = arg0.u;
-	register unsigned long r4 asm("r4") = arg1.u;
-	register unsigned long r5 asm("r5") = arg2.u;
-	register unsigned long r6 = entry.u         ;
-
-	asm volatile("mflr 31 ; mtctr %4 ; bctrl ; mtlr 31"
-		     : "=r" (r3)
-		     : "r" (r3), "r" (r4), "r" (r5), "r" (r6)
-		     : "ctr", "r6", "r7", "r8", "r9", "r10", "r11",
-		       "r12", "r13", "r31", "lr", "cc");
-
-	return r3;
-}
+extern unsigned long call_c(cell arg0, cell arg1, cell arg2, cell entry);
 
 
 long
diff --git a/roms/SLOF/slof/prim.code b/roms/SLOF/slof/prim.code
index 9fbed7168..bb9e036a9 100644
--- a/roms/SLOF/slof/prim.code
+++ b/roms/SLOF/slof/prim.code
@@ -520,6 +520,19 @@ PRIM(RMOVE)
 
 	MIRP
 
+PRIM(MRMOVE)
+	type_u size = TOS.u; POP;
+	void *d = TOS.a; POP;
+	void *s = TOS.a; POP;
+	FAST_MRMOVE(s, d, size);
+	MIRP
+
+PRIM(RFILL)
+	type_u pat = TOS.u; POP;
+	type_u size = TOS.u; POP;
+	void *dst = TOS.a; POP;
+	FAST_RFILL(dst, size, pat);
+	MIRP
 
 // String compare, case insensitive:
 // : string=ci  ( str1 len1 str2 len2 -- equal? )
diff --git a/roms/SLOF/slof/prim.in b/roms/SLOF/slof/prim.in
index 7a0d6a2ed..855f59262 100644
--- a/roms/SLOF/slof/prim.in
+++ b/roms/SLOF/slof/prim.in
@@ -104,8 +104,9 @@ cod(SEMICOLON)
 cod(EXECUTE)
 
 cod(MOVE)
-// cod(RMOVE64)
 cod(RMOVE)
+cod(MRMOVE)
+cod(RFILL)
 cod(ZCOUNT)
 con(HASH-SIZE HASHSIZE)
 cod(HASH)
diff --git a/roms/config.ipxe.general.h b/roms/config.ipxe.general.h
deleted file mode 100644
index 619ee4c15..000000000
--- a/roms/config.ipxe.general.h
+++ /dev/null
@@ -1,4 +0,0 @@
-#undef BANNER_TIMEOUT
-#define BANNER_TIMEOUT 30
-#undef ROM_BANNER_TIMEOUT
-#define ROM_BANNER_TIMEOUT 0
diff --git a/roms/ipxe/COPYING b/roms/ipxe/COPYING
index a43ea2126..342330bb9 100644
--- a/roms/ipxe/COPYING
+++ b/roms/ipxe/COPYING
@@ -1,339 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+In general iPXE files are licensed under the GPL.  For historical
+reasons, individual files may contain their own licence declarations.
+Most builds of iPXE do not contain all iPXE code (in particular, most
+builds will include only one driver), and so the overall licence can
+vary depending on what target you are building.
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-                          675 Mass Ave, Cambridge, MA 02139, USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
+The resultant applicable licence(s) for any particular build can be
+determined by using "make bin/xxxxxxx.yyy.licence"; for example:
 
-			    Preamble
+  make bin/rtl8139.rom.licence
 
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-			    NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-	Appendix: How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) 19yy  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) 19yy name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
-Public License instead of this License.
+to determine the resultant licence(s) for the build bin/rtl8139.rom
diff --git a/roms/ipxe/COPYING.GPLv2 b/roms/ipxe/COPYING.GPLv2
new file mode 100644
index 000000000..d159169d1
--- /dev/null
+++ b/roms/ipxe/COPYING.GPLv2
@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/roms/ipxe/COPYING.UBDL b/roms/ipxe/COPYING.UBDL
new file mode 100644
index 000000000..780ddcd77
--- /dev/null
+++ b/roms/ipxe/COPYING.UBDL
@@ -0,0 +1,59 @@
+UNMODIFIED BINARY DISTRIBUTION LICENCE
+
+
+PREAMBLE
+
+The GNU General Public License provides a legal guarantee that
+software covered by it remains free (in the sense of freedom, not
+price).  It achieves this guarantee by imposing obligations on anyone
+who chooses to distribute the software.
+
+Some of these obligations may be seen as unnecessarily burdensome.  In
+particular, when the source code for the software is already publicly
+and freely available, there is minimal value in imposing upon each
+distributor the obligation to provide the complete source code (or an
+equivalent written offer to provide the complete source code).
+
+This Licence allows for the distribution of unmodified binaries built
+from publicly available source code, without imposing the obligations
+of the GNU General Public License upon anyone who chooses to
+distribute only the unmodified binaries built from that source code.
+
+The extra permissions granted by this Licence apply only to unmodified
+binaries built from source code which has already been made available
+to the public in accordance with the terms of the GNU General Public
+Licence.  Nothing in this Licence allows for the creation of
+closed-source modified versions of the Program.  Any modified versions
+of the Program are subject to the usual terms and conditions of the
+GNU General Public License.
+
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+This Licence applies to any Program or other work which contains a
+notice placed by the copyright holder saying it may be distributed
+under the terms of this Unmodified Binary Distribution Licence.  All
+terms used in the text of this Licence are to be interpreted as they
+are used in version 2 of the GNU General Public License as published
+by the Free Software Foundation.
+
+If you have made this Program available to the public in both source
+code and executable form in accordance with the terms of the GNU
+General Public License as published by the Free Software Foundation;
+either version 2 of the License, or (at your option) any later
+version, then you are hereby granted an additional permission to use,
+copy, and distribute the unmodified executable form of this Program
+(the "Unmodified Binary") without restriction, including the right to
+permit persons to whom the Unmodified Binary is furnished to do
+likewise, subject to the following conditions:
+
+- when started running, the Program must display an announcement which
+  includes the details of your existing publication of the Program
+  made in accordance with the terms of the GNU General Public License.
+  For example, the Program could display the URL of the publicly
+  available source code from which the Unmodified Binary was built.
+
+- when exercising your right to grant permissions under this Licence,
+  you do not need to refer directly to the text of this Licence, but
+  you may not grant permissions beyond those granted to you by this
+  Licence.
diff --git a/roms/ipxe/COPYRIGHTS b/roms/ipxe/COPYRIGHTS
deleted file mode 100644
index 342330bb9..000000000
--- a/roms/ipxe/COPYRIGHTS
+++ /dev/null
@@ -1,12 +0,0 @@
-In general iPXE files are licensed under the GPL.  For historical
-reasons, individual files may contain their own licence declarations.
-Most builds of iPXE do not contain all iPXE code (in particular, most
-builds will include only one driver), and so the overall licence can
-vary depending on what target you are building.
-
-The resultant applicable licence(s) for any particular build can be
-determined by using "make bin/xxxxxxx.yyy.licence"; for example:
-
-  make bin/rtl8139.rom.licence
-
-to determine the resultant licence(s) for the build bin/rtl8139.rom
diff --git a/roms/ipxe/src/Makefile b/roms/ipxe/src/Makefile
index b742d1283..2a9cc9e8f 100644
--- a/roms/ipxe/src/Makefile
+++ b/roms/ipxe/src/Makefile
@@ -83,11 +83,13 @@ SRCDIRS		+= drivers/block
 SRCDIRS		+= drivers/nvs
 SRCDIRS		+= drivers/bitbash
 SRCDIRS		+= drivers/infiniband
+SRCDIRS		+= drivers/usb
 SRCDIRS		+= interface/pxe interface/efi interface/smbios
 SRCDIRS		+= interface/bofm
 SRCDIRS		+= interface/xen
+SRCDIRS		+= interface/hyperv
 SRCDIRS		+= tests
-SRCDIRS		+= crypto crypto/axtls crypto/matrixssl
+SRCDIRS		+= crypto crypto/mishmash
 SRCDIRS		+= hci hci/commands hci/tui
 SRCDIRS		+= hci/mucurses hci/mucurses/widgets
 SRCDIRS		+= hci/keymap
diff --git a/roms/ipxe/src/Makefile.housekeeping b/roms/ipxe/src/Makefile.housekeeping
index 1a75d3939..03800c8ef 100644
--- a/roms/ipxe/src/Makefile.housekeeping
+++ b/roms/ipxe/src/Makefile.housekeeping
@@ -157,17 +157,6 @@ SP_FLAGS := $(shell $(SP_TEST) && $(ECHO) '-fno-stack-protector')
 WORKAROUND_CFLAGS += $(SP_FLAGS)
 endif
 
-# Some widespread patched versions of gcc include -fPIE -Wl,-pie by
-# default.  Note that gcc will exit *successfully* if it fails to
-# recognise an option that starts with "no", so we have to test for
-# output on stderr instead of checking the exit status.
-#
-ifeq ($(CCTYPE),gcc)
-PIE_TEST = [ -z "`$(CC) -fno-PIE -nopie -x c -c /dev/null -o /dev/null 2>&1`" ]
-PIE_FLAGS := $(shell $(PIE_TEST) && $(ECHO) '-fno-PIE -nopie')
-WORKAROUND_CFLAGS += $(PIE_FLAGS)
-endif
-
 # gcc 4.4 generates .eh_frame sections by default, which distort the
 # output of "size".  Inhibit this.
 #
@@ -533,6 +522,7 @@ endif
 #
 COMPILE_c	= $(CC) $(CFLAGS) $(CFLAGS_c) $(OBJ_CFLAGS)
 RULE_c		= $(Q)$(COMPILE_c) -c $< -o $@ $(POST_O)
+RULE_c_to_ids.o = $(Q)$(ECHO_E) '$(OBJ_IDS_ASM_NL)' | $(ASSEMBLE_S) -o $@
 RULE_c_to_dbg%.o = $(Q)$(COMPILE_c) -DDBGLVL_MAX=$* -c $< -o $@ $(POST_O)
 RULE_c_to_c	= $(Q)$(COMPILE_c) -E -c $< > $@
 RULE_c_to_s	= $(Q)$(COMPILE_c) -S -g0 -c $< -o $@
@@ -543,7 +533,7 @@ RULE_S		= $(Q)$(PREPROCESS_S) $< | $(ASSEMBLE_S) -o $@
 RULE_S_to_dbg%.o = $(Q)$(PREPROCESS_S) -DDBGLVL_MAX=$* $< | $(ASSEMBLE_S) -o $@
 RULE_S_to_s	= $(Q)$(PREPROCESS_S) $< > $@
 
-DEBUG_TARGETS	+= dbg%.o c s
+GENERIC_TARGETS	+= ids.o dbg%.o c s
 
 # List of embedded images included in the last build of embedded.o.
 # This is needed in order to correctly rebuild embedded.o whenever the
@@ -775,8 +765,6 @@ define deps_template_parts
 	$(Q)$(CPP) $(CFLAGS) $(CFLAGS_$(2)) $(CFLAGS_$(3)) -DOBJECT=$(3) \
 		-Wno-error -M $(1) -MG -MP | \
 		sed 's/\.o\s*:/_DEPS +=/' > $(BIN)/deps/$(1).d
-	$(Q)$(if $(findstring drivers/,$(1)),\
-	   $(PERL) $(PARSEROM) $(1) >> $(BIN)/deps/$(1).d)
 endef
 
 # rules_template : generate rules for a given source file
@@ -796,7 +784,7 @@ $$(BIN)/$(3).o : $(1) $$(MAKEDEPS) $$(POST_O_DEPS) $$($(3)_DEPS)
 	$$(QM)$(ECHO) "  [BUILD] $$@"
 	$$(RULE_$(2))
 BOBJS += $$(BIN)/$(3).o
-$(foreach TGT,$(DEBUG_TARGETS),$(if $(RULE_$(2)_to_$(TGT)),$(NEWLINE)$(call rules_template_target,$(1),$(2),$(3),$(TGT))))
+$(foreach TGT,$(GENERIC_TARGETS),$(if $(RULE_$(2)_to_$(TGT)),$(NEWLINE)$(call rules_template_target,$(1),$(2),$(3),$(TGT))))
 $$(BIN)/deps/$(1).d : $$($(3)_DEPS)
 TAGS : $$($(3)_DEPS)
 endef
@@ -824,7 +812,7 @@ endef
 
 # Generate the dependency files
 #
-$(BIN)/deps/%.d : % $(MAKEDEPS) $(PARSEROM)
+$(BIN)/deps/%.d : % $(MAKEDEPS)
 	$(call deps_template_file,$<)
 
 # Calculate list of dependency files
@@ -866,10 +854,69 @@ endif
 endif
 endif
 
-# The following variables are created by the rules files
+# Files to be parsed using parserom.pl
+#
+ROM_SRCS	= $(foreach SRC,$(AUTO_SRCS),\
+		    $(if $(findstring drivers/,$(SRC)),$(SRC)))
+romsrcs :
+	@$(ECHO) $(ROM_SRCS)
+
+# List of files to be parsed using parserom.pl
+#
+ROM_SRCS_LIST	:= $(BIN)/.rom.list
+ifeq ($(wildcard $(ROM_SRCS_LIST)),)
+ROM_SRCS_OLD := <invalid>
+else
+ROM_SRCS_OLD := $(shell cat $(ROM_SRCS_LIST))
+endif
+ifneq ($(ROM_SRCS_OLD),$(ROM_SRCS))
+$(shell $(ECHO) "$(ROM_SRCS)" > $(ROM_SRCS_LIST))
+endif
+
+$(ROM_SRCS_LIST) : $(MAKEDEPS)
+
+VERYCLEANUP	+= $(ROM_SRCS_LIST)
+
+# ROM definition file
+#
+ROMDEFS		= $(BIN)/.rom.defs
+$(ROMDEFS) : $(ROM_SRCS) $(ROM_SRCS_LIST) $(PARSEROM) $(MAKEDEPS)
+	$(QM)$(ECHO) "  [PARSEROM]"
+	$(Q)$(PERL) $(PARSEROM) $(ROM_SRCS) > $@
+
+VERYCLEANUP	+= $(ROMDEFS)
+
+# Evaluate ROM definition file
+ifdef NEED_DEPS
+ifneq ($(ROM_SRCS),)
+-include $(ROMDEFS)
+endif
+endif
+
+# Device ID tables (using IDs from ROM definition file)
+#
+define obj_pci_id_asm
+	.section ".pci_devlist.$(1)", "a", @progbits
+	.globl pci_devlist_$(1)
+pci_devlist_$(1):
+	.short ( 0x$(1) & 0xffff )
+
+endef
+define obj_isa_id_asm
+endef
+OBJ_IDS_ASM	= $(foreach ROM,$(ROMS_$(OBJECT)),$(call obj_$(ROM_TYPE_$(ROM))_id_asm,$(ROM)))
+OBJ_IDS_ASM_NL	= $(subst $(NEWLINE),\n,$(OBJ_IDS_ASM))
+$(BIN)/%.ids :
+	@$(ECHO_E) '$(OBJ_IDS_ASM_NL)'
+
+BOBJS	+= $(patsubst %,$(BIN)/%.ids.o,$(DRIVERS))
+
+# The following variables are created by the autogenerated rules
 #
 bobjs :
 	@$(ECHO) $(BOBJS)
+drivers_% :
+	@$(ECHO) $(DRIVERS_$*)
 drivers :
 	@$(ECHO) $(DRIVERS)
 .PHONY : drivers
@@ -900,6 +947,11 @@ $(BIN)/NIC : $(AUTO_DEPS)
 	@perl -ne 'chomp; print "$$1\n" if /\# NIC\t(.*)$$/' $^ >> $@
 CLEANUP		+= $(BIN)/NIC	# Doesn't match the $(BIN)/*.* pattern
 
+# Select drivers to be included in the all-drivers build
+#
+DRIVERS_ipxe	= $(DRIVERS_net) $(DRIVERS_infiniband) \
+		  $(DRIVERS_xen) $(DRIVERS_hyperv)
+
 # Analyse a target name (e.g. "bin/dfe538--prism2_pci.rom.tmp") and
 # derive the variables:
 # 
@@ -908,7 +960,6 @@ CLEANUP		+= $(BIN)/NIC	# Doesn't match the $(BIN)/*.* pattern
 # TGT_DRIVERS  : the driver for each element (e.g. "rtl8139 prism2_pci")
 # TGT_ROM_NAME : the ROM name (e.g. "dfe538")
 #
-DRIVERS_ipxe	= $(DRIVERS)
 CARD_DRIVER	= $(firstword $(DRIVER_$(1)) $(1))
 TGT_ELEMENTS	= $(subst --, ,$(firstword $(subst ., ,$(notdir $@))))
 TGT_ROM_NAME	= $(firstword $(TGT_ELEMENTS))
@@ -941,6 +992,8 @@ TGT_PCI_DEVICE	= $(PCI_DEVICE_$(TGT_ROM_NAME))
 TGT_LD_DRIVERS	= $(subst -,_,$(patsubst %,obj_%,$(TGT_DRIVERS)))
 TGT_LD_IDS	= pci_vendor_id=$(firstword $(TGT_PCI_VENDOR) 0) \
 		  pci_device_id=$(firstword $(TGT_PCI_DEVICE) 0)
+TGT_LD_DEVLIST	= $(foreach ELEM,$(TGT_ELEMENTS),$(if $(PCI_VENDOR_$(ELEM)),\
+		    pci_devlist_$(patsubst 0x%,%,$(PCI_VENDOR_$(ELEM)))$(patsubst 0x%,%,$(PCI_DEVICE_$(ELEM)))))
 TGT_LD_ENTRY	= _$(TGT_PREFIX)_start
 
 # Calculate linker flags based on link-time options for the current
@@ -951,7 +1004,8 @@ TGT_LD_ENTRY	= _$(TGT_PREFIX)_start
 #		 "-u obj_zpciprefix -u obj_rtl8139 -u obj_prism2_pci
 #		  --defsym pci_vendor=0x1186 --defsym pci_device=0x1300")
 #
-TGT_LD_FLAGS	= $(foreach SYM,$(TGT_LD_ENTRY) $(TGT_LD_DRIVERS) obj_config,\
+TGT_LD_FLAGS	= $(foreach SYM,$(TGT_LD_ENTRY) $(TGT_LD_DRIVERS) \
+		    $(TGT_LD_DEVLIST) obj_config,\
 		    -u $(SYM) --defsym check_$(SYM)=$(SYM) ) \
 		  $(patsubst %,--defsym %,$(TGT_LD_IDS)) \
 		  -e $(TGT_LD_ENTRY)
@@ -981,6 +1035,7 @@ $(BIN)/%.info :
 	@$(ECHO)
 	@$(ECHO) 'LD driver symbols    : $(TGT_LD_DRIVERS)'
 	@$(ECHO) 'LD ID symbols        : $(TGT_LD_IDS)'
+	@$(ECHO) 'LD devlist symbols   : $(TGT_LD_DEVLIST)'
 	@$(ECHO) 'LD entry point       : $(TGT_LD_ENTRY)'
 	@$(ECHO)
 	@$(ECHO) 'LD target flags      : $(TGT_LD_FLAGS)'
@@ -1012,7 +1067,7 @@ BLIB		= $(BIN)/blib.a
 $(BLIB) : $(BLIB_OBJS) $(BLIB_LIST) $(MAKEDEPS)
 	$(Q)$(RM) $(BLIB)
 	$(QM)$(ECHO) "  [AR] $@"
-	$(Q)$(AR) r $@ $(BLIB_OBJS)
+	$(Q)$(AR) r $@ $(sort $(BLIB_OBJS))
 	$(Q)$(RANLIB) $@
 blib : $(BLIB)
 
@@ -1231,15 +1286,12 @@ endif # defined(BIN)
 #
 # The compression utilities
 #
-$(NRV2B) : util/nrv2b.c $(MAKEDEPS)
-	$(QM)$(ECHO) "  [HOSTCC] $@"
-	$(Q)$(HOST_CC) $(HOST_CFLAGS) -DENCODE -DDECODE -DMAIN -DVERBOSE \
-		       -DNDEBUG -DBITSIZE=32 -DENDIAN=0 -o $@ $<
-CLEANUP	+= $(NRV2B)
 
-$(ZBIN) : util/zbin.c util/nrv2b.c $(MAKEDEPS)
+ZBIN_LDFLAGS := -llzma
+
+$(ZBIN) : util/zbin.c $(MAKEDEPS)
 	$(QM)$(ECHO) "  [HOSTCC] $@"
-	$(Q)$(HOST_CC) $(HOST_CFLAGS) -o $@ $<
+	$(Q)$(HOST_CC) $(HOST_CFLAGS) $< $(ZBIN_LDFLAGS) -o $@
 CLEANUP += $(ZBIN)
 
 ###############################################################################
@@ -1319,31 +1371,6 @@ endif
 
 ###############################################################################
 #
-# Auto-incrementing build serial number.  Append "bs" to your list of
-# build targets to get a serial number printed at the end of the
-# build.  Enable -DBUILD_SERIAL in order to see it when the code runs.
-#
-BUILDSERIAL_H		= config/.buildserial.h
-BUILDSERIAL_NOW		= config/.buildserial.now
-BUILDSERIAL_NEXT	= config/.buildserial.next
-
-$(BUILDSERIAL_NOW) $(BUILDSERIAL_NEXT) :
-	$(ECHO) 1 > $@
-
-$(BUILDSERIAL_H) : $(BUILDSERIAL_NOW) $(BUILDSERIAL_NEXT)
-	$(ECHO) '#define BUILD_SERIAL_NUM $(shell cat $<)' > $@
-
-ifeq ($(filter bs,$(MAKECMDGOALS)),bs)
-$(shell diff -q $(BUILDSERIAL_NOW) $(BUILDSERIAL_NEXT) > /dev/null || \
-	cp -f $(BUILDSERIAL_NEXT) $(BUILDSERIAL_NOW))
-endif
-
-bs : $(BUILDSERIAL_NOW)
-	@$(ECHO) $$(( $(shell cat $<) + 1 )) > $(BUILDSERIAL_NEXT)
-	@$(ECHO) "Build serial number is $(shell cat $<)"
-
-###############################################################################
-#
 # Build the TAGS file(s) for emacs
 #
 TAGS :
diff --git a/roms/ipxe/src/arch/i386/Makefile b/roms/ipxe/src/arch/i386/Makefile
index 4925cc4e6..99f875314 100644
--- a/roms/ipxe/src/arch/i386/Makefile
+++ b/roms/ipxe/src/arch/i386/Makefile
@@ -69,6 +69,17 @@ CFLAGS		+= -fshort-wchar
 #
 CFLAGS			+= -Ui386
 
+# Some widespread patched versions of gcc include -fPIE -Wl,-pie by
+# default.  Note that gcc will exit *successfully* if it fails to
+# recognise an option that starts with "no", so we have to test for
+# output on stderr instead of checking the exit status.
+#
+ifeq ($(CCTYPE),gcc)
+PIE_TEST = [ -z "`$(CC) -fno-PIE -nopie -x c -c /dev/null -o /dev/null 2>&1`" ]
+PIE_FLAGS := $(shell $(PIE_TEST) && $(ECHO) '-fno-PIE -nopie')
+WORKAROUND_CFLAGS += $(PIE_FLAGS)
+endif
+
 # Define version string for lkrnprefix.S
 #
 CFLAGS_lkrnprefix	+= -DVERSION="\"$(VERSION)\""
diff --git a/roms/ipxe/src/arch/i386/core/basemem_packet.c b/roms/ipxe/src/arch/i386/core/basemem_packet.c
index 06ffa3bbd..9f5fbf330 100644
--- a/roms/ipxe/src/arch/i386/core/basemem_packet.c
+++ b/roms/ipxe/src/arch/i386/core/basemem_packet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/arch/i386/core/cachedhcp.c b/roms/ipxe/src/arch/i386/core/cachedhcp.c
index 3cac28e7d..a5c624035 100644
--- a/roms/ipxe/src/arch/i386/core/cachedhcp.c
+++ b/roms/ipxe/src/arch/i386/core/cachedhcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/arch/i386/core/gdbmach.c b/roms/ipxe/src/arch/i386/core/gdbmach.c
index 4d6897f7d..d92a4ac08 100644
--- a/roms/ipxe/src/arch/i386/core/gdbmach.c
+++ b/roms/ipxe/src/arch/i386/core/gdbmach.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/arch/i386/core/patch_cf.S b/roms/ipxe/src/arch/i386/core/patch_cf.S
index 97a62f494..4365563fe 100644
--- a/roms/ipxe/src/arch/i386/core/patch_cf.S
+++ b/roms/ipxe/src/arch/i386/core/patch_cf.S
@@ -14,9 +14,13 @@
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
diff --git a/roms/ipxe/src/arch/i386/core/pci_autoboot.c b/roms/ipxe/src/arch/i386/core/pci_autoboot.c
index a3eb1f97d..337598091 100644
--- a/roms/ipxe/src/arch/i386/core/pci_autoboot.c
+++ b/roms/ipxe/src/arch/i386/core/pci_autoboot.c
@@ -16,9 +16,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/device.h>
diff --git a/roms/ipxe/src/arch/i386/core/rdtsc_timer.c b/roms/ipxe/src/arch/i386/core/rdtsc_timer.c
index 2f31afc66..e720a239c 100644
--- a/roms/ipxe/src/arch/i386/core/rdtsc_timer.c
+++ b/roms/ipxe/src/arch/i386/core/rdtsc_timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -27,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <assert.h>
 #include <ipxe/timer.h>
-#include <ipxe/timer2.h>
+#include <ipxe/pit8254.h>
 
 /**
  * Number of TSC ticks per microsecond
@@ -56,10 +60,10 @@ static void rdtsc_udelay ( unsigned long usecs ) {
 			elapsed = ( currticks() - start );
 		} while ( elapsed < ( usecs * rdtsc_ticks_per_usec ) );
 	} else {
-		/* Not yet calibrated; use timer2 and calibrate
+		/* Not yet calibrated; use 8254 PIT and calibrate
 		 * based on result.
 		 */
-		timer2_udelay ( usecs );
+		pit8254_udelay ( usecs );
 		elapsed = ( currticks() - start );
 		rdtsc_ticks_per_usec = ( elapsed / usecs );
 		DBG ( "RDTSC timer calibrated: %ld ticks in %ld usecs "
diff --git a/roms/ipxe/src/arch/i386/core/relocate.c b/roms/ipxe/src/arch/i386/core/relocate.c
index 5fbf2d2c2..54ad387e4 100644
--- a/roms/ipxe/src/arch/i386/core/relocate.c
+++ b/roms/ipxe/src/arch/i386/core/relocate.c
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * The linker passes in the symbol _max_align, which is the alignment
diff --git a/roms/ipxe/src/arch/i386/core/runtime.c b/roms/ipxe/src/arch/i386/core/runtime.c
index 18ca7936e..d160fee04 100644
--- a/roms/ipxe/src/arch/i386/core/runtime.c
+++ b/roms/ipxe/src/arch/i386/core/runtime.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/core/setjmp.S b/roms/ipxe/src/arch/i386/core/setjmp.S
index 03727148c..81d3b4911 100644
--- a/roms/ipxe/src/arch/i386/core/setjmp.S
+++ b/roms/ipxe/src/arch/i386/core/setjmp.S
@@ -1,42 +1,64 @@
-/* setjmp and longjmp. Use of these functions is deprecated. */
-
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
 	.code32
-	
-/**************************************************************************
-SETJMP - Save stack context for non-local goto
-**************************************************************************/
+
+	/* Must match jmp_buf structure layout */
+	.struct	0
+env_retaddr:	.long	0
+env_stack:	.long	0
+env_ebx:	.long	0
+env_esi:	.long	0
+env_edi:	.long	0
+env_ebp:	.long	0
+	.previous
+
+/*
+ * Save stack context for non-local goto
+ */
 	.globl	setjmp
 setjmp:
-	movl	4(%esp),%ecx		/* jmpbuf */
-	movl	0(%esp),%edx		/* return address */
-	movl	%edx,0(%ecx)
-	movl	%ebx,4(%ecx)
-	movl	%esp,8(%ecx)
-	movl	%ebp,12(%ecx)
-	movl	%esi,16(%ecx)
-	movl	%edi,20(%ecx)
-	movl	$0,%eax
+	/* Get jmp_buf pointer in %edx */
+	movl	4(%esp),%edx
+	/* Save return address */
+	movl	0(%esp),%eax
+	movl	%eax, env_retaddr(%edx)
+	/* Save stack pointer */
+	movl	%esp, env_stack(%edx)
+	/* Save other registers */
+	movl	%ebx, env_ebx(%edx)
+	movl	%esi, env_esi(%edx)
+	movl	%edi, env_edi(%edx)
+	movl	%ebp, env_ebp(%edx)
+	/* Return 0 when returning as setjmp() */
+	xorl	%eax, %eax
 	ret
+	.size	setjmp, . - setjmp
 
-/**************************************************************************
-LONGJMP - Non-local jump to a saved stack context
-**************************************************************************/
+/*
+ * Non-local jump to a saved stack context
+ */
 	.globl	longjmp
 longjmp:
-	movl	4(%esp),%edx		/* jumpbuf */
-	movl	8(%esp),%eax		/* result */
-	movl	0(%edx),%ecx
-	movl	4(%edx),%ebx
-	movl	8(%edx),%esp
-	movl	12(%edx),%ebp
-	movl	16(%edx),%esi
-	movl	20(%edx),%edi
-	cmpl	$0,%eax
-	jne	1f
-	movl	$1,%eax
-1:	movl	%ecx,0(%esp)
+	/* Get jmp_buf pointer in %edx */
+	movl	4(%esp),%edx
+	/* Get result in %eax */
+	movl	8(%esp),%eax
+	/* Force result to non-zero */
+	testl	%eax, %eax
+	jnz	1f
+	incl	%eax
+1:	/* Restore stack pointer */
+	movl	env_stack(%edx), %esp
+	/* Restore other registers */
+	movl	env_ebx(%edx), %ebx
+	movl	env_esi(%edx), %esi
+	movl	env_edi(%edx), %edi
+	movl	env_ebp(%edx), %ebp
+	/* Replace return address on the new stack */
+	popl	%ecx	/* discard */
+	pushl	env_retaddr(%edx)
+	/* Return to setjmp() caller */
 	ret
+	.size	longjmp, . - longjmp
diff --git a/roms/ipxe/src/arch/i386/core/stack.S b/roms/ipxe/src/arch/i386/core/stack.S
index 737ec0eed..98f1cd9b9 100644
--- a/roms/ipxe/src/arch/i386/core/stack.S
+++ b/roms/ipxe/src/arch/i386/core/stack.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/roms/ipxe/src/arch/i386/core/stack16.S b/roms/ipxe/src/arch/i386/core/stack16.S
index 523f0288b..4bc6f081a 100644
--- a/roms/ipxe/src/arch/i386/core/stack16.S
+++ b/roms/ipxe/src/arch/i386/core/stack16.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/roms/ipxe/src/arch/i386/core/timer2.c b/roms/ipxe/src/arch/i386/core/timer2.c
deleted file mode 100644
index 077866562..000000000
--- a/roms/ipxe/src/arch/i386/core/timer2.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * arch/i386/core/i386_timer.c
- *
- * Use the "System Timer 2" to implement the udelay callback in
- * the BIOS timer driver. Also used to calibrate the clock rate
- * in the RTDSC timer driver.
- * 
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stddef.h>
-#include <ipxe/timer2.h>
-#include <ipxe/io.h>
-
-/* Timers tick over at this rate */
-#define TIMER2_TICKS_PER_SEC	1193180U
-
-/* Parallel Peripheral Controller Port B */
-#define	PPC_PORTB	0x61
-
-/* Meaning of the port bits */
-#define	PPCB_T2OUT	0x20	/* Bit 5 */
-#define	PPCB_SPKR	0x02	/* Bit 1 */
-#define	PPCB_T2GATE	0x01	/* Bit 0 */
-
-/* Ports for the 8254 timer chip */
-#define	TIMER2_PORT	0x42
-#define	TIMER_MODE_PORT	0x43
-
-/* Meaning of the mode bits */
-#define	TIMER0_SEL	0x00
-#define	TIMER1_SEL	0x40
-#define	TIMER2_SEL	0x80
-#define	READBACK_SEL	0xC0
-
-#define	LATCH_COUNT	0x00
-#define	LOBYTE_ACCESS	0x10
-#define	HIBYTE_ACCESS	0x20
-#define	WORD_ACCESS	0x30
-
-#define	MODE0		0x00
-#define	MODE1		0x02
-#define	MODE2		0x04
-#define	MODE3		0x06
-#define	MODE4		0x08
-#define	MODE5		0x0A
-
-#define	BINARY_COUNT	0x00
-#define	BCD_COUNT	0x01
-
-static void load_timer2 ( unsigned int ticks ) {
-	/*
-	 * Now let's take care of PPC channel 2
-	 *
-	 * Set the Gate high, program PPC channel 2 for mode 0,
-	 * (interrupt on terminal count mode), binary count,
-	 * load 5 * LATCH count, (LSB and MSB) to begin countdown.
-	 *
-	 * Note some implementations have a bug where the high bits byte
-	 * of channel 2 is ignored.
-	 */
-	/* Set up the timer gate, turn off the speaker */
-	/* Set the Gate high, disable speaker */
-	outb((inb(PPC_PORTB) & ~PPCB_SPKR) | PPCB_T2GATE, PPC_PORTB);
-	/* binary, mode 0, LSB/MSB, Ch 2 */
-	outb(TIMER2_SEL|WORD_ACCESS|MODE0|BINARY_COUNT, TIMER_MODE_PORT);
-	/* LSB of ticks */
-	outb(ticks & 0xFF, TIMER2_PORT);
-	/* MSB of ticks */
-	outb(ticks >> 8, TIMER2_PORT);
-}
-
-static int timer2_running ( void ) {
-	return ((inb(PPC_PORTB) & PPCB_T2OUT) == 0);
-}
-
-void timer2_udelay ( unsigned long usecs ) {
-	load_timer2 ( ( usecs * TIMER2_TICKS_PER_SEC ) / ( 1000 * 1000 ) );
-	while (timer2_running()) {
-		/* Do nothing */
-	}
-}
diff --git a/roms/ipxe/src/arch/i386/core/virtaddr.S b/roms/ipxe/src/arch/i386/core/virtaddr.S
index 5e5d77352..425591570 100644
--- a/roms/ipxe/src/arch/i386/core/virtaddr.S
+++ b/roms/ipxe/src/arch/i386/core/virtaddr.S
@@ -4,7 +4,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #include "librm.h"
 		
diff --git a/roms/ipxe/src/arch/i386/drivers/net/undi.c b/roms/ipxe/src/arch/i386/drivers/net/undi.c
index 2bc54824c..9820cf629 100644
--- a/roms/ipxe/src/arch/i386/drivers/net/undi.c
+++ b/roms/ipxe/src/arch/i386/drivers/net/undi.c
@@ -68,10 +68,6 @@ static int undipci_probe ( struct pci_device *pci ) {
 	struct undi_rom *undirom;
 	int rc;
 
-	/* Ignore non-network devices */
-	if ( PCI_BASE_CLASS ( pci->class ) != PCI_BASE_CLASS_NETWORK )
-		return -ENOTTY;
-
 	/* Allocate UNDI device structure */
 	undi = zalloc ( sizeof ( *undi ) );
 	if ( ! undi )
@@ -138,12 +134,13 @@ static void undipci_remove ( struct pci_device *pci ) {
 }
 
 static struct pci_device_id undipci_nics[] = {
-PCI_ROM ( 0xffff, 0xffff, "undipci", "UNDI (PCI)", 0 ),
+	PCI_ROM ( 0xffff, 0xffff, "undipci", "UNDI (PCI)", 0 ),
 };
 
 struct pci_driver undipci_driver __pci_driver_fallback = {
 	.ids = undipci_nics,
 	.id_count = ( sizeof ( undipci_nics ) / sizeof ( undipci_nics[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_NETWORK, PCI_ANY_ID, PCI_ANY_ID ),
 	.probe = undipci_probe,
 	.remove = undipci_remove,
 };
diff --git a/roms/ipxe/src/arch/i386/drivers/net/undiload.c b/roms/ipxe/src/arch/i386/drivers/net/undiload.c
index 77134dcb8..7160ee384 100644
--- a/roms/ipxe/src/arch/i386/drivers/net/undiload.c
+++ b/roms/ipxe/src/arch/i386/drivers/net/undiload.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/arch/i386/drivers/net/undionly.c b/roms/ipxe/src/arch/i386/drivers/net/undionly.c
index 028fac5d9..70dbe4bfd 100644
--- a/roms/ipxe/src/arch/i386/drivers/net/undionly.c
+++ b/roms/ipxe/src/arch/i386/drivers/net/undionly.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/arch/i386/drivers/net/undipreload.c b/roms/ipxe/src/arch/i386/drivers/net/undipreload.c
index 81d7a80eb..fca771843 100644
--- a/roms/ipxe/src/arch/i386/drivers/net/undipreload.c
+++ b/roms/ipxe/src/arch/i386/drivers/net/undipreload.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 #include <undipreload.h>
diff --git a/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c b/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c
index b23f2c356..6a46081aa 100644
--- a/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c
+++ b/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c b/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c
index bd73838b5..63413cdc1 100644
--- a/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c
+++ b/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <assert.h>
 #include <realmode.h>
@@ -39,6 +43,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ATTR_FCOL_YELLOW	0x06
 #define ATTR_FCOL_WHITE		0x07
 
+#define ATTR_BLINK		0x80
+
 #define ATTR_BCOL_MASK		0x70
 #define ATTR_BCOL_BLACK		0x00
 #define ATTR_BCOL_BLUE		0x10
@@ -137,8 +143,12 @@ static void bios_handle_sgr ( struct ansiesc_context *ctx __unused,
 			bios_attr = ATTR_DEFAULT;
 		} else if ( aspect == 1 ) {
 			bios_attr |= ATTR_BOLD;
+		} else if ( aspect == 5 ) {
+			bios_attr |= ATTR_BLINK;
 		} else if ( aspect == 22 ) {
 			bios_attr &= ~ATTR_BOLD;
+		} else if ( aspect == 25 ) {
+			bios_attr &= ~ATTR_BLINK;
 		} else if ( ( aspect >= 30 ) && ( aspect <= 39 ) ) {
 			bios_attr &= ~ATTR_FCOL_MASK;
 			bios_attr |= bios_attr_fcols[ aspect - 30 ];
diff --git a/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S b/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S
index cea17ef8e..d5d97b482 100644
--- a/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S
+++ b/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
diff --git a/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c b/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c
index e5f713728..15f4d772f 100644
--- a/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c
+++ b/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c
@@ -14,9 +14,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 #include <biosint.h>
diff --git a/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c b/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c
index 8f3069e18..253c601ff 100644
--- a/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c
+++ b/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c
@@ -14,9 +14,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <assert.h>
 #include <realmode.h>
diff --git a/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c b/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c
index 0937a7ce2..bcacecd6a 100644
--- a/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c
+++ b/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c b/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c
index 5c74b0431..20ec35d75 100644
--- a/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c
+++ b/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c b/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c
index 523724ab0..473b97f97 100644
--- a/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c
+++ b/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <ipxe/netdevice.h>
@@ -23,7 +27,7 @@
 #include <hci/ifmgmt_cmd.h>
 #include <pxe_call.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/image/bootsector.c b/roms/ipxe/src/arch/i386/image/bootsector.c
index 9a089e6bb..dba87613c 100644
--- a/roms/ipxe/src/arch/i386/image/bootsector.c
+++ b/roms/ipxe/src/arch/i386/image/bootsector.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/arch/i386/image/bzimage.c b/roms/ipxe/src/arch/i386/image/bzimage.c
index 4865c394c..a64206cd3 100644
--- a/roms/ipxe/src/arch/i386/image/bzimage.c
+++ b/roms/ipxe/src/arch/i386/image/bzimage.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/arch/i386/image/elfboot.c b/roms/ipxe/src/arch/i386/image/elfboot.c
index 0f6957f02..dc3568929 100644
--- a/roms/ipxe/src/arch/i386/image/elfboot.c
+++ b/roms/ipxe/src/arch/i386/image/elfboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <elf.h>
@@ -75,6 +79,27 @@ static int elfboot_exec ( struct image *image ) {
 }
 
 /**
+ * Check that ELF segment uses flat physical addressing
+ *
+ * @v image		ELF file
+ * @v phdr		ELF program header
+ * @v dest		Destination address
+ * @ret rc		Return status code
+ */
+static int elfboot_check_segment ( struct image *image, Elf_Phdr *phdr,
+				   physaddr_t dest ) {
+
+	/* Check that ELF segment uses flat physical addressing */
+	if ( phdr->p_vaddr != dest ) {
+		DBGC ( image, "ELF %p uses virtual addressing (phys %x, "
+		       "virt %x)\n", image, phdr->p_paddr, phdr->p_vaddr );
+		return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+/**
  * Probe ELF image
  *
  * @v image		ELF file
@@ -91,14 +116,24 @@ static int elfboot_probe ( struct image *image ) {
 		[EI_DATA]	= ELFDATA2LSB,
 		[EI_VERSION]	= EV_CURRENT,
 	};
+	physaddr_t entry;
+	physaddr_t max;
+	int rc;
 
 	/* Read ELF header */
 	copy_from_user ( &ehdr, image->data, 0, sizeof ( ehdr ) );
 	if ( memcmp ( ehdr.e_ident, e_ident, sizeof ( e_ident ) ) != 0 ) {
-		DBG ( "Invalid ELF identifier\n" );
+		DBGC ( image, "Invalid ELF identifier\n" );
 		return -ENOEXEC;
 	}
 
+	/* Check that this image uses flat physical addressing */
+	if ( ( rc = elf_segments ( image, &ehdr, elfboot_check_segment,
+				   &entry, &max ) ) != 0 ) {
+		DBGC ( image, "Unloadable ELF image\n" );
+		return rc;
+	}
+
 	return 0;
 }
 
diff --git a/roms/ipxe/src/arch/i386/image/initrd.c b/roms/ipxe/src/arch/i386/image/initrd.c
index eaba3a645..80c197417 100644
--- a/roms/ipxe/src/arch/i386/image/initrd.c
+++ b/roms/ipxe/src/arch/i386/image/initrd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <initrd.h>
diff --git a/roms/ipxe/src/arch/i386/image/multiboot.c b/roms/ipxe/src/arch/i386/image/multiboot.c
index 86b0bc12d..0c85df708 100644
--- a/roms/ipxe/src/arch/i386/image/multiboot.c
+++ b/roms/ipxe/src/arch/i386/image/multiboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/arch/i386/image/pxe_image.c b/roms/ipxe/src/arch/i386/image/pxe_image.c
index dc28f6082..5b0f6eb89 100644
--- a/roms/ipxe/src/arch/i386/image/pxe_image.c
+++ b/roms/ipxe/src/arch/i386/image/pxe_image.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -34,6 +38,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/netdevice.h>
 #include <ipxe/features.h>
 #include <ipxe/console.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/IndustryStandard/PeImage.h>
 
 FEATURE ( FEATURE_IMAGE, "PXE", DHCP_EB_FEATURE_PXE, 1 );
 
@@ -121,9 +127,45 @@ int pxe_probe ( struct image *image ) {
 	return 0;
 }
 
+/**
+ * Probe PXE image (with rejection of potential EFI images)
+ *
+ * @v image		PXE file
+ * @ret rc		Return status code
+ */
+int pxe_probe_no_mz ( struct image *image ) {
+	uint16_t magic;
+	int rc;
+
+	/* Probe PXE image */
+	if ( ( rc = pxe_probe ( image ) ) != 0 )
+		return rc;
+
+	/* Reject image with an "MZ" signature which may indicate an
+	 * EFI image incorrectly handed out to a BIOS system.
+	 */
+	if ( image->len >= sizeof ( magic ) ) {
+		copy_from_user ( &magic, image->data, 0, sizeof ( magic ) );
+		if ( magic == cpu_to_le16 ( EFI_IMAGE_DOS_SIGNATURE ) ) {
+			DBGC ( image, "IMAGE %p may be an EFI image\n",
+			       image );
+			return -ENOTTY;
+		}
+	}
+
+	return 0;
+}
+
 /** PXE image type */
-struct image_type pxe_image_type __image_type ( PROBE_PXE ) = {
-	.name = "PXE",
-	.probe = pxe_probe,
-	.exec = pxe_exec,
+struct image_type pxe_image_type[] __image_type ( PROBE_PXE ) = {
+	{
+		.name = "PXE-NBP",
+		.probe = pxe_probe_no_mz,
+		.exec = pxe_exec,
+	},
+	{
+		.name = "PXE-NBP (may be EFI?)",
+		.probe = pxe_probe,
+		.exec = pxe_exec,
+	},
 };
diff --git a/roms/ipxe/src/arch/i386/image/sdi.c b/roms/ipxe/src/arch/i386/image/sdi.c
index df1c3a868..fa2d0b73f 100644
--- a/roms/ipxe/src/arch/i386/image/sdi.c
+++ b/roms/ipxe/src/arch/i386/image/sdi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/arch/i386/include/basemem.h b/roms/ipxe/src/arch/i386/include/basemem.h
index c477c7fe2..01c2ea917 100644
--- a/roms/ipxe/src/arch/i386/include/basemem.h
+++ b/roms/ipxe/src/arch/i386/include/basemem.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/roms/ipxe/src/arch/i386/include/basemem_packet.h b/roms/ipxe/src/arch/i386/include/basemem_packet.h
index 3cb477671..def6dee31 100644
--- a/roms/ipxe/src/arch/i386/include/basemem_packet.h
+++ b/roms/ipxe/src/arch/i386/include/basemem_packet.h
@@ -1,7 +1,7 @@
 #ifndef BASEMEM_PACKET_H
 #define BASEMEM_PACKET_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bios.h b/roms/ipxe/src/arch/i386/include/bios.h
index 0754b1168..988bbc62b 100644
--- a/roms/ipxe/src/arch/i386/include/bios.h
+++ b/roms/ipxe/src/arch/i386/include/bios.h
@@ -1,7 +1,7 @@
 #ifndef BIOS_H
 #define BIOS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define BDA_SEG 0x0040
 #define BDA_EQUIPMENT_WORD 0x0010
diff --git a/roms/ipxe/src/arch/i386/include/biosint.h b/roms/ipxe/src/arch/i386/include/biosint.h
index ab466af3c..67d6a3811 100644
--- a/roms/ipxe/src/arch/i386/include/biosint.h
+++ b/roms/ipxe/src/arch/i386/include/biosint.h
@@ -6,7 +6,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/byteswap.h b/roms/ipxe/src/arch/i386/include/bits/byteswap.h
index 0d9cb967c..53b6a454d 100644
--- a/roms/ipxe/src/arch/i386/include/bits/byteswap.h
+++ b/roms/ipxe/src/arch/i386/include/bits/byteswap.h
@@ -9,7 +9,7 @@
 
 #include <stdint.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static inline __attribute__ (( always_inline, const )) uint16_t
 __bswap_variable_16 ( uint16_t x ) {
diff --git a/roms/ipxe/src/arch/i386/include/bits/compiler.h b/roms/ipxe/src/arch/i386/include/bits/compiler.h
index d48b4b385..87201135f 100644
--- a/roms/ipxe/src/arch/i386/include/bits/compiler.h
+++ b/roms/ipxe/src/arch/i386/include/bits/compiler.h
@@ -1,7 +1,10 @@
 #ifndef _BITS_COMPILER_H
 #define _BITS_COMPILER_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Dummy relocation type */
+#define RELOC_TYPE_NONE R_386_NONE
 
 #ifndef ASSEMBLY
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/endian.h b/roms/ipxe/src/arch/i386/include/bits/endian.h
deleted file mode 100644
index 841885424..000000000
--- a/roms/ipxe/src/arch/i386/include/bits/endian.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef ETHERBOOT_BITS_ENDIAN_H
-#define ETHERBOOT_BITS_ENDIAN_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#define __BYTE_ORDER __LITTLE_ENDIAN
-
-#endif /* ETHERBOOT_BITS_ENDIAN_H */
diff --git a/roms/ipxe/src/arch/i386/include/bits/entropy.h b/roms/ipxe/src/arch/i386/include/bits/entropy.h
index 6dcceec6d..bfeb5e3b5 100644
--- a/roms/ipxe/src/arch/i386/include/bits/entropy.h
+++ b/roms/ipxe/src/arch/i386/include/bits/entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/rtc_entropy.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/hyperv.h b/roms/ipxe/src/arch/i386/include/bits/hyperv.h
new file mode 100644
index 000000000..3565c8a83
--- /dev/null
+++ b/roms/ipxe/src/arch/i386/include/bits/hyperv.h
@@ -0,0 +1,72 @@
+#ifndef _BITS_HYPERV_H
+#define _BITS_HYPERV_H
+
+/** @file
+ *
+ * Hyper-V interface
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/**
+ * Issue hypercall
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v code		Call code
+ * @v in		Input parameters
+ * @v out		Output parameters
+ * @ret status		Status code
+ */
+static inline __attribute__ (( always_inline )) int
+hv_call ( struct hv_hypervisor *hv, unsigned int code, const void *in,
+	  void *out ) {
+	void *hypercall = hv->hypercall;
+	uint32_t in_phys;
+	uint32_t out_phys;
+	uint32_t discard_ecx;
+	uint32_t discard_edx;
+	uint16_t result;
+
+	in_phys = ( ( __builtin_constant_p ( in ) && ( in == NULL ) )
+		       ? 0 : virt_to_phys ( in ) );
+	out_phys = ( ( __builtin_constant_p ( out ) && ( out == NULL ) )
+		       ? 0 : virt_to_phys ( out ) );
+	__asm__ __volatile__ ( "call *%9"
+			       : "=a" ( result ), "=c" ( discard_ecx ),
+				 "=d" ( discard_edx )
+			       : "d" ( 0 ), "a" ( code ),
+				 "b" ( 0 ), "c" ( in_phys ),
+				 "D" ( 0 ), "S" ( out_phys ),
+				 "m" ( hypercall ) );
+	return result;
+}
+
+/**
+ * Set bit atomically
+ *
+ * @v bits		Bit field
+ * @v bit		Bit to set
+ */
+static inline __attribute__ (( always_inline )) void
+hv_set_bit ( void *bits, unsigned int bit ) {
+	struct {
+		uint32_t dword[ ( bit / 32 ) + 1 ];
+	} *dwords = bits;
+
+	/* Set bit using "lock bts".  Inform compiler that any memory
+	 * from the start of the bit field up to and including the
+	 * dword containing this bit may be modified.  (This is
+	 * overkill but shouldn't matter in practice since we're
+	 * unlikely to subsequently read other bits from the same bit
+	 * field.)
+	 */
+	__asm__ __volatile__ ( "lock bts %1, %0"
+			       : "+m" ( *dwords ) : "Ir" ( bit ) );
+}
+
+#endif /* _BITS_HYPERV_H */
diff --git a/roms/ipxe/src/arch/i386/include/bits/nap.h b/roms/ipxe/src/arch/i386/include/bits/nap.h
index 64066e6ab..e8bcfd13b 100644
--- a/roms/ipxe/src/arch/i386/include/bits/nap.h
+++ b/roms/ipxe/src/arch/i386/include/bits/nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_nap.h>
 #include <ipxe/efi/efix86_nap.h>
diff --git a/roms/ipxe/src/arch/i386/include/bits/profile.h b/roms/ipxe/src/arch/i386/include/bits/profile.h
index f3ee54ae9..e184d7b51 100644
--- a/roms/ipxe/src/arch/i386/include/bits/profile.h
+++ b/roms/ipxe/src/arch/i386/include/bits/profile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/reboot.h b/roms/ipxe/src/arch/i386/include/bits/reboot.h
index 5b09e95f7..803dacfe4 100644
--- a/roms/ipxe/src/arch/i386/include/bits/reboot.h
+++ b/roms/ipxe/src/arch/i386/include/bits/reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_reboot.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/sanboot.h b/roms/ipxe/src/arch/i386/include/bits/sanboot.h
index 9c77a4d42..f02d2e649 100644
--- a/roms/ipxe/src/arch/i386/include/bits/sanboot.h
+++ b/roms/ipxe/src/arch/i386/include/bits/sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_sanboot.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/smbios.h b/roms/ipxe/src/arch/i386/include/bits/smbios.h
index cc79eec51..2ab31e74b 100644
--- a/roms/ipxe/src/arch/i386/include/bits/smbios.h
+++ b/roms/ipxe/src/arch/i386/include/bits/smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_smbios.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/stdint.h b/roms/ipxe/src/arch/i386/include/bits/stdint.h
index 8edf13192..fe1f9946a 100644
--- a/roms/ipxe/src/arch/i386/include/bits/stdint.h
+++ b/roms/ipxe/src/arch/i386/include/bits/stdint.h
@@ -1,7 +1,7 @@
 #ifndef _BITS_STDINT_H
 #define _BITS_STDINT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 typedef __SIZE_TYPE__		size_t;
 typedef signed long		ssize_t;
diff --git a/roms/ipxe/src/arch/i386/include/bits/strings.h b/roms/ipxe/src/arch/i386/include/bits/strings.h
index 092bcb593..453545f00 100644
--- a/roms/ipxe/src/arch/i386/include/bits/strings.h
+++ b/roms/ipxe/src/arch/i386/include/bits/strings.h
@@ -1,7 +1,51 @@
 #ifndef _BITS_STRINGS_H
 #define _BITS_STRINGS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsl ( long value ) {
+	long lsb_minus_one;
+
+	/* If the input value is zero, the BSF instruction returns
+	 * ZF=0 and leaves an undefined value in the output register.
+	 * Perform this check in C rather than asm so that it can be
+	 * omitted in cases where the compiler is able to prove that
+	 * the input is non-zero.
+	 */
+	if ( value ) {
+		__asm__ ( "bsfl %1, %0"
+			  : "=r" ( lsb_minus_one )
+			  : "rm" ( value ) );
+		return ( lsb_minus_one + 1 );
+	} else {
+		return 0;
+	}
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsll ( long long value ){
+	unsigned long high = ( value >> 32 );
+	unsigned long low = ( value >> 0 );
+
+	if ( low ) {
+		return ( __ffsl ( low ) );
+	} else if ( high ) {
+		return ( 32 + __ffsl ( high ) );
+	} else {
+		return 0;
+	}
+}
 
 /**
  * Find last (i.e. most significant) set bit
@@ -13,7 +57,7 @@ static inline __attribute__ (( always_inline )) int __flsl ( long value ) {
 	long msb_minus_one;
 
 	/* If the input value is zero, the BSR instruction returns
-	 * ZF=1 and leaves an undefined value in the output register.
+	 * ZF=0 and leaves an undefined value in the output register.
 	 * Perform this check in C rather than asm so that it can be
 	 * omitted in cases where the compiler is able to prove that
 	 * the input is non-zero.
diff --git a/roms/ipxe/src/arch/i386/include/bits/time.h b/roms/ipxe/src/arch/i386/include/bits/time.h
index 24dd020e9..6a5d63d32 100644
--- a/roms/ipxe/src/arch/i386/include/bits/time.h
+++ b/roms/ipxe/src/arch/i386/include/bits/time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/rtc_time.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/timer.h b/roms/ipxe/src/arch/i386/include/bits/timer.h
index 50b676b77..f7d86d78c 100644
--- a/roms/ipxe/src/arch/i386/include/bits/timer.h
+++ b/roms/ipxe/src/arch/i386/include/bits/timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_timer.h>
 #include <ipxe/rdtsc_timer.h>
diff --git a/roms/ipxe/src/arch/i386/include/bits/uaccess.h b/roms/ipxe/src/arch/i386/include/bits/uaccess.h
index 2bb52e021..aac09ba95 100644
--- a/roms/ipxe/src/arch/i386/include/bits/uaccess.h
+++ b/roms/ipxe/src/arch/i386/include/bits/uaccess.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <librm.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bits/umalloc.h b/roms/ipxe/src/arch/i386/include/bits/umalloc.h
index 54fb006f0..113f16fd1 100644
--- a/roms/ipxe/src/arch/i386/include/bits/umalloc.h
+++ b/roms/ipxe/src/arch/i386/include/bits/umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/memtop_umalloc.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/bootsector.h b/roms/ipxe/src/arch/i386/include/bootsector.h
index 8730fbfcc..c5d35aae3 100644
--- a/roms/ipxe/src/arch/i386/include/bootsector.h
+++ b/roms/ipxe/src/arch/i386/include/bootsector.h
@@ -6,7 +6,7 @@
  * x86 bootsector image format
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int call_bootsector ( unsigned int segment, unsigned int offset,
 			     unsigned int drive );
diff --git a/roms/ipxe/src/arch/i386/include/bzimage.h b/roms/ipxe/src/arch/i386/include/bzimage.h
index 7e42e3188..4933ce5b1 100644
--- a/roms/ipxe/src/arch/i386/include/bzimage.h
+++ b/roms/ipxe/src/arch/i386/include/bzimage.h
@@ -1,7 +1,7 @@
 #ifndef _BZIMAGE_H
 #define _BZIMAGE_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h b/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h
index 184177219..c17c1ea5e 100644
--- a/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h
+++ b/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/fakee820.h b/roms/ipxe/src/arch/i386/include/fakee820.h
index 9d00fb670..552b1e48d 100644
--- a/roms/ipxe/src/arch/i386/include/fakee820.h
+++ b/roms/ipxe/src/arch/i386/include/fakee820.h
@@ -1,7 +1,7 @@
 #ifndef _FAKEE820_H
 #define _FAKEE820_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void fake_e820 ( void );
 extern void unfake_e820 ( void );
diff --git a/roms/ipxe/src/arch/i386/include/initrd.h b/roms/ipxe/src/arch/i386/include/initrd.h
index a5659f43c..ddb3e5a45 100644
--- a/roms/ipxe/src/arch/i386/include/initrd.h
+++ b/roms/ipxe/src/arch/i386/include/initrd.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/int13.h b/roms/ipxe/src/arch/i386/include/int13.h
index e337ca1d1..f82a583c6 100644
--- a/roms/ipxe/src/arch/i386/include/int13.h
+++ b/roms/ipxe/src/arch/i386/include/int13.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h b/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h
index 5b684c041..c9b82c1e5 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_PCBIOS
 #define NAP_PREFIX_pcbios
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h b/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h
index a0845328d..3f6df9073 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef REBOOT_PCBIOS
 #define REBOOT_PREFIX_pcbios
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h b/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h
index 689227b70..1a86b7d57 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SANBOOT_PCBIOS
 #define SANBOOT_PREFIX_pcbios
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h b/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h
index d8c7f648a..9f7f9c8ff 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SMBIOS_PCBIOS
 #define SMBIOS_PREFIX_pcbios
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h b/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h
index f9fc80412..6b88a623c 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_PCBIOS
 #define TIMER_PREFIX_pcbios
@@ -15,7 +15,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define TIMER_PREFIX_pcbios __pcbios_
 #endif
 
-#include <ipxe/timer2.h>
+#include <ipxe/pit8254.h>
 
 /**
  * Delay for a fixed number of microseconds
@@ -25,9 +25,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 static inline __always_inline void
 TIMER_INLINE ( pcbios, udelay ) ( unsigned long usecs ) {
 	/* BIOS timer is not high-resolution enough for udelay(), so
-	 * we use timer2
+	 * we use the 8254 Programmable Interval Timer.
 	 */
-	timer2_udelay ( usecs );
+	pit8254_udelay ( usecs );
 }
 
 /**
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h b/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h
index 3a9eb2495..6312adaa4 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h
@@ -10,7 +10,7 @@
  * for the PC-BIOS platform.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pxe_error.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h b/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h
index 72a0f714f..bc3d85506 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/vmware.h>
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h b/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h
index 001648fe5..dee055d16 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef UMALLOC_MEMTOP
 #define UMALLOC_PREFIX_memtop
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/msr.h b/roms/ipxe/src/arch/i386/include/ipxe/msr.h
index c88e26a39..5705318fd 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/msr.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/msr.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Read model-specific register
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h b/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h
index 472e14007..598f4bb08 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_RDTSC
 #define TIMER_PREFIX_rdtsc
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h b/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h
index 6c3cf2104..e214745d0 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h b/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h
index c0dfe3f88..cb8c7f49e 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIME_RTC
 #define TIME_PREFIX_rtc
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/timer2.h b/roms/ipxe/src/arch/i386/include/ipxe/timer2.h
deleted file mode 100644
index 322a3ed59..000000000
--- a/roms/ipxe/src/arch/i386/include/ipxe/timer2.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#ifndef _IPXE_TIMER2_H
-#define _IPXE_TIMER2_H
-
-/** @file
- *
- * Timer chip control
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-extern void timer2_udelay ( unsigned long usecs );
-
-#endif /* _IPXE_TIMER2_H */
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h b/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h
index 48cd6a7b7..efc8f2cb8 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/roms/ipxe/src/arch/i386/include/ipxe/vmware.h b/roms/ipxe/src/arch/i386/include/ipxe/vmware.h
index 2ac65f436..24f60a03a 100644
--- a/roms/ipxe/src/arch/i386/include/ipxe/vmware.h
+++ b/roms/ipxe/src/arch/i386/include/ipxe/vmware.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/librm.h b/roms/ipxe/src/arch/i386/include/librm.h
index c8ba72b53..a8a578a39 100644
--- a/roms/ipxe/src/arch/i386/include/librm.h
+++ b/roms/ipxe/src/arch/i386/include/librm.h
@@ -1,7 +1,7 @@
 #ifndef LIBRM_H
 #define LIBRM_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Segment selectors as used in our protected-mode GDTs.
  *
@@ -170,18 +170,6 @@ extern uint16_t __text16 ( rm_cs );
 extern uint16_t __text16 ( rm_ds );
 #define rm_ds __use_text16 ( rm_ds )
 
-/**
- * Convert segment:offset address to user buffer
- *
- * @v segment		Real-mode segment
- * @v offset		Real-mode offset
- * @ret buffer		User buffer
- */
-static inline __always_inline userptr_t
-real_to_user ( unsigned int segment, unsigned int offset ) {
-	return ( phys_to_user ( ( segment << 4 ) + offset ) );
-}
-
 extern uint16_t copy_user_to_rm_stack ( userptr_t data, size_t size );
 extern void remove_user_from_rm_stack ( userptr_t data, size_t size );
 
diff --git a/roms/ipxe/src/arch/i386/include/limits.h b/roms/ipxe/src/arch/i386/include/limits.h
index 031b6c57a..bb48b75ab 100644
--- a/roms/ipxe/src/arch/i386/include/limits.h
+++ b/roms/ipxe/src/arch/i386/include/limits.h
@@ -1,7 +1,7 @@
 #ifndef LIMITS_H
 #define LIMITS_H	1
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Number of bits in a `char' */
 #define CHAR_BIT	8
diff --git a/roms/ipxe/src/arch/i386/include/memsizes.h b/roms/ipxe/src/arch/i386/include/memsizes.h
index 7b217494a..f115f7574 100644
--- a/roms/ipxe/src/arch/i386/include/memsizes.h
+++ b/roms/ipxe/src/arch/i386/include/memsizes.h
@@ -1,7 +1,7 @@
 #ifndef _MEMSIZES_H
 #define _MEMSIZES_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <basemem.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/multiboot.h b/roms/ipxe/src/arch/i386/include/multiboot.h
index 44614c73a..ae09df6c7 100644
--- a/roms/ipxe/src/arch/i386/include/multiboot.h
+++ b/roms/ipxe/src/arch/i386/include/multiboot.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h b/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h
index a36d9cfa1..e07e4c192 100644
--- a/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h
+++ b/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/pnpbios.h b/roms/ipxe/src/arch/i386/include/pnpbios.h
index 4c20e73ed..d14873700 100644
--- a/roms/ipxe/src/arch/i386/include/pnpbios.h
+++ b/roms/ipxe/src/arch/i386/include/pnpbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* BIOS segment address */
 #define BIOS_SEG 0xf000
diff --git a/roms/ipxe/src/arch/i386/include/pxe.h b/roms/ipxe/src/arch/i386/include/pxe.h
index b95b0cce5..66d752683 100644
--- a/roms/ipxe/src/arch/i386/include/pxe.h
+++ b/roms/ipxe/src/arch/i386/include/pxe.h
@@ -1,7 +1,7 @@
 #ifndef PXE_H
 #define PXE_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include "pxe_types.h"
 #include "pxe_error.h"
diff --git a/roms/ipxe/src/arch/i386/include/pxe_api.h b/roms/ipxe/src/arch/i386/include/pxe_api.h
index e4396efb2..3110d26da 100644
--- a/roms/ipxe/src/arch/i386/include/pxe_api.h
+++ b/roms/ipxe/src/arch/i386/include/pxe_api.h
@@ -17,6 +17,10 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  * As an alternative, at your option, you may use this file under the
  * following terms, known as the "MIT license":
  *
@@ -49,7 +53,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include "pxe_types.h"
 
diff --git a/roms/ipxe/src/arch/i386/include/pxe_call.h b/roms/ipxe/src/arch/i386/include/pxe_call.h
index 45af46549..cbd548318 100644
--- a/roms/ipxe/src/arch/i386/include/pxe_call.h
+++ b/roms/ipxe/src/arch/i386/include/pxe_call.h
@@ -6,7 +6,7 @@
  * PXE API entry point
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pxe_api.h>
 #include <realmode.h>
diff --git a/roms/ipxe/src/arch/i386/include/pxe_error.h b/roms/ipxe/src/arch/i386/include/pxe_error.h
index a1398cbd4..51298e665 100644
--- a/roms/ipxe/src/arch/i386/include/pxe_error.h
+++ b/roms/ipxe/src/arch/i386/include/pxe_error.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @defgroup pxeerrors PXE error codes
diff --git a/roms/ipxe/src/arch/i386/include/pxe_types.h b/roms/ipxe/src/arch/i386/include/pxe_types.h
index db8214591..483666e33 100644
--- a/roms/ipxe/src/arch/i386/include/pxe_types.h
+++ b/roms/ipxe/src/arch/i386/include/pxe_types.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h> /* PXE status codes */
diff --git a/roms/ipxe/src/arch/i386/include/realmode.h b/roms/ipxe/src/arch/i386/include/realmode.h
index dafc5a32a..4defd3b97 100644
--- a/roms/ipxe/src/arch/i386/include/realmode.h
+++ b/roms/ipxe/src/arch/i386/include/realmode.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * Declaration of variables in .data16
@@ -65,6 +65,18 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /**
+ * Convert segment:offset address to user buffer
+ *
+ * @v segment		Real-mode segment
+ * @v offset		Real-mode offset
+ * @ret buffer		User buffer
+ */
+static inline __always_inline userptr_t
+real_to_user ( unsigned int segment, unsigned int offset ) {
+	return ( phys_to_user ( ( segment << 4 ) + offset ) );
+}
+
+/**
  * Copy data to base memory
  *
  * @v dest_seg		Destination segment
diff --git a/roms/ipxe/src/arch/i386/include/registers.h b/roms/ipxe/src/arch/i386/include/registers.h
index 06d236524..d9aa3c376 100644
--- a/roms/ipxe/src/arch/i386/include/registers.h
+++ b/roms/ipxe/src/arch/i386/include/registers.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/rtc.h b/roms/ipxe/src/arch/i386/include/rtc.h
index 2a6abbae5..6294b63e3 100644
--- a/roms/ipxe/src/arch/i386/include/rtc.h
+++ b/roms/ipxe/src/arch/i386/include/rtc.h
@@ -13,7 +13,7 @@
  *    http://wiki.osdev.org/CMOS
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pic8259.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/sdi.h b/roms/ipxe/src/arch/i386/include/sdi.h
index fc486402d..806c3f194 100644
--- a/roms/ipxe/src/arch/i386/include/sdi.h
+++ b/roms/ipxe/src/arch/i386/include/sdi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** SDI image header */
 struct sdi_header {
diff --git a/roms/ipxe/src/arch/i386/include/setjmp.h b/roms/ipxe/src/arch/i386/include/setjmp.h
index 5d3c11b69..fe1a9ef4d 100644
--- a/roms/ipxe/src/arch/i386/include/setjmp.h
+++ b/roms/ipxe/src/arch/i386/include/setjmp.h
@@ -1,40 +1,50 @@
-#ifndef ETHERBOOT_SETJMP_H
-#define ETHERBOOT_SETJMP_H
+#ifndef _SETJMP_H
+#define _SETJMP_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
 
 /** A jump buffer */
 typedef struct {
+	/** Saved return address */
 	uint32_t retaddr;
+	/** Saved stack pointer */
+	uint32_t stack;
+	/** Saved %ebx */
 	uint32_t ebx;
-	uint32_t esp;
-	uint32_t ebp;
+	/** Saved %esi */
 	uint32_t esi;
+	/** Saved %edi */
 	uint32_t edi;
+	/** Saved %ebp */
+	uint32_t ebp;
 } jmp_buf[1];
 
 /** A real-mode-extended jump buffer */
 typedef struct {
+	/** Jump buffer */
 	jmp_buf env;
-	uint16_t rm_ss;
-	uint16_t rm_sp;
+	/** Real-mode stack pointer */
+	segoff_t rm_stack;
 } rmjmp_buf[1];
 
-extern int __asmcall setjmp ( jmp_buf env );
-extern void __asmcall longjmp ( jmp_buf env, int val );
+extern int __asmcall __attribute__ (( returns_twice ))
+setjmp ( jmp_buf env );
+
+extern void __asmcall __attribute__ (( noreturn ))
+longjmp ( jmp_buf env, int val );
 
-#define rmsetjmp( _env ) ( {			\
-	(_env)->rm_ss = rm_ss;			\
-	(_env)->rm_sp = rm_sp;			\
-	setjmp ( (_env)->env ); } )		\
+#define rmsetjmp( _env ) ( {					\
+	(_env)->rm_stack.segment = rm_ss;			\
+	(_env)->rm_stack.offset = rm_sp;			\
+	setjmp ( (_env)->env ); } )				\
 
-#define rmlongjmp( _env, _val ) do {		\
-	rm_ss = (_env)->rm_ss;			\
-	rm_sp = (_env)->rm_sp;			\
-	longjmp ( (_env)->env, (_val) );	\
+#define rmlongjmp( _env, _val ) do {				\
+	rm_ss = (_env)->rm_stack.segment;			\
+	rm_sp = (_env)->rm_stack.offset;			\
+	longjmp ( (_env)->env, (_val) );			\
 	} while ( 0 )
 
-#endif /* ETHERBOOT_SETJMP_H */
+#endif /* _SETJMP_H */
diff --git a/roms/ipxe/src/arch/i386/include/undi.h b/roms/ipxe/src/arch/i386/include/undi.h
index 325fcbbf9..7a5624f93 100644
--- a/roms/ipxe/src/arch/i386/include/undi.h
+++ b/roms/ipxe/src/arch/i386/include/undi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef ASSEMBLY
 
diff --git a/roms/ipxe/src/arch/i386/include/undiload.h b/roms/ipxe/src/arch/i386/include/undiload.h
index 426830e8d..235e7a79e 100644
--- a/roms/ipxe/src/arch/i386/include/undiload.h
+++ b/roms/ipxe/src/arch/i386/include/undiload.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct undi_device;
 struct undi_rom;
diff --git a/roms/ipxe/src/arch/i386/include/undinet.h b/roms/ipxe/src/arch/i386/include/undinet.h
index c3c17c11a..2798c4466 100644
--- a/roms/ipxe/src/arch/i386/include/undinet.h
+++ b/roms/ipxe/src/arch/i386/include/undinet.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct undi_device;
 
diff --git a/roms/ipxe/src/arch/i386/include/undipreload.h b/roms/ipxe/src/arch/i386/include/undipreload.h
index de9b8fb52..57f493cec 100644
--- a/roms/ipxe/src/arch/i386/include/undipreload.h
+++ b/roms/ipxe/src/arch/i386/include/undipreload.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 #include <undi.h>
diff --git a/roms/ipxe/src/arch/i386/include/undirom.h b/roms/ipxe/src/arch/i386/include/undirom.h
index 86d7077b5..1c530118d 100644
--- a/roms/ipxe/src/arch/i386/include/undirom.h
+++ b/roms/ipxe/src/arch/i386/include/undirom.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pxe_types.h>
 
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/apm.c b/roms/ipxe/src/arch/i386/interface/pcbios/apm.c
index 3b13e1cd0..50b19cb81 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/apm.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/apm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c b/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c
index 1e7de756b..f1ba8297b 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c
@@ -1,7 +1,7 @@
 #include <ipxe/nap.h>
 #include <realmode.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Save power by halting the CPU until the next interrupt
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c b/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c
index 68546b2e5..10a1ecb89 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c b/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c
index dd7897e29..a8c0fc325 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c b/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c
index 65bbf9e01..3299c9aae 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c b/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c
index a193defa3..3b8e80438 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Hook INT vector
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/int13.c b/roms/ipxe/src/arch/i386/interface/pcbios/int13.c
index 1c7a8128f..f0450da90 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/int13.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/int13.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/int13con.c b/roms/ipxe/src/arch/i386/interface/pcbios/int13con.c
new file mode 100644
index 000000000..2414c6909
--- /dev/null
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/int13con.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <ipxe/console.h>
+#include <ipxe/init.h>
+#include <realmode.h>
+#include <int13.h>
+#include <config/console.h>
+
+/** @file
+ *
+ * INT13 disk log console
+ *
+ */
+
+/* Set default console usage if applicable */
+#if ! ( defined ( CONSOLE_INT13 ) && CONSOLE_EXPLICIT ( CONSOLE_INT13 ) )
+#undef CONSOLE_INT13
+#define CONSOLE_INT13 ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG )
+#endif
+
+/** Disk drive number */
+#define INT13CON_DRIVE 0x80
+
+/** Log partition type */
+#define INT13CON_PARTITION_TYPE 0xe0
+
+/** Maximum number of outstanding unwritten characters */
+#define INT13CON_MAX_UNWRITTEN 64
+
+/** Log partition header */
+struct int13con_header {
+	/** Magic signature */
+	char magic[10];
+} __attribute__ (( packed ));
+
+/** Log partition magic signature */
+#define INT13CON_MAGIC "iPXE LOG\n\n"
+
+/** Sector buffer */
+static uint8_t __bss16_array ( int13con_buffer, [INT13_BLKSIZE] );
+#define int13con_buffer __use_data16 ( int13con_buffer )
+
+/** Disk address packet */
+static struct int13_disk_address __bss16 ( int13con_address );
+#define int13con_address __use_data16 ( int13con_address )
+
+/** Current LBA */
+static uint64_t int13con_lba;
+
+/** Maximum LBA */
+static uint64_t int13con_max_lba;
+
+/** Current offset within sector */
+static size_t int13con_offset;
+
+/** Number of unwritten characters */
+static size_t int13con_unwritten;
+
+struct console_driver int13con __console_driver;
+
+/**
+ * Read/write disk sector
+ *
+ * @v op		Operation
+ * @v lba		Logical block address
+ * @ret rc		Return status code
+ */
+static int int13con_rw ( unsigned int op, uint64_t lba ) {
+	uint8_t error;
+
+	/* Construct disk address packet */
+	int13con_address.bufsize = sizeof ( int13con_address );
+	int13con_address.count = 1;
+	int13con_address.buffer.segment = rm_ds;
+	int13con_address.buffer.offset = __from_data16 ( int13con_buffer );
+	int13con_address.lba = lba;
+
+	/* Issue INT13 */
+	__asm__ ( REAL_CODE ( "int $0x13\n\t" )
+		  : "=a" ( error )
+		  : "0" ( op << 8 ), "d" ( INT13CON_DRIVE ),
+		    "S" ( __from_data16 ( &int13con_address ) ) );
+	if ( error ) {
+		DBG ( "INT13CON operation %04x failed: %02x\n",
+		      op, error );
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/**
+ * Write character to console
+ *
+ * @v character		Character
+ */
+static void int13con_putchar ( int character ) {
+	static int busy;
+	int rc;
+
+	/* Ignore if we are already mid-logging */
+	if ( busy )
+		return;
+	busy = 1;
+
+	/* Write character to buffer */
+	int13con_buffer[int13con_offset++] = character;
+	int13con_unwritten++;
+
+	/* Write sector to disk, if applicable */
+	if ( ( int13con_offset == INT13_BLKSIZE ) ||
+	     ( int13con_unwritten == INT13CON_MAX_UNWRITTEN ) ||
+	     ( character == '\n' ) ) {
+
+		/* Write sector to disk */
+		if ( ( rc = int13con_rw ( INT13_EXTENDED_WRITE,
+					  int13con_lba ) ) != 0 ) {
+			DBG ( "INT13CON could not write log\n" );
+			/* Ignore and continue; there's nothing we can do */
+		}
+
+		/* Reset count of unwritten characters */
+		int13con_unwritten = 0;
+	}
+
+	/* Move to next sector, if applicable */
+	if ( int13con_offset == INT13_BLKSIZE ) {
+
+		/* Disable console if we have run out of space */
+		if ( int13con_lba >= int13con_max_lba )
+			int13con.disabled = 1;
+
+		/* Clear log buffer */
+		memset ( int13con_buffer, 0, sizeof ( int13con_buffer ) );
+		int13con_offset = 0;
+
+		/* Move to next sector */
+		int13con_lba++;
+	}
+
+	/* Clear busy flag */
+	busy = 0;
+}
+
+/**
+ * Find log partition
+ *
+ * @ret rc		Return status code
+ */
+static int int13con_find ( void ) {
+	struct master_boot_record *mbr =
+		( ( struct master_boot_record * ) int13con_buffer );
+	struct int13con_header *hdr =
+		( ( struct int13con_header * ) int13con_buffer );
+	struct partition_table_entry part[4];
+	unsigned int i;
+	int rc;
+
+	/* Read MBR */
+	if ( ( rc = int13con_rw ( INT13_EXTENDED_READ, 0 ) ) != 0 ) {
+		DBG ( "INT13CON could not read MBR: %s\n", strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check MBR magic */
+	if ( mbr->magic != INT13_MBR_MAGIC ) {
+		DBG ( "INT13CON incorrect MBR magic\n" );
+		DBG2_HDA ( 0, mbr, sizeof ( *mbr ) );
+		return -EINVAL;
+	}
+
+	/* Look for magic partition */
+	memcpy ( part, mbr->partitions, sizeof ( part ) );
+	for ( i = 0 ; i < ( sizeof ( part ) / sizeof ( part[0] ) ) ; i++ ) {
+
+		/* Skip partitions of the wrong type */
+		if ( part[i].type != INT13CON_PARTITION_TYPE )
+			continue;
+
+		/* Read partition header */
+		if ( ( rc = int13con_rw ( INT13_EXTENDED_READ,
+					  part[i].start ) ) != 0 ) {
+			DBG ( "INT13CON partition %d could not read header: "
+			      "%s\n", ( i + 1 ), strerror ( rc ) );
+			continue;
+		}
+
+		/* Check partition header */
+		if ( memcmp ( hdr->magic, INT13CON_MAGIC,
+			      sizeof ( hdr->magic ) ) != 0 ) {
+			DBG ( "INT13CON partition %d bad magic\n", ( i + 1 ) );
+			DBG2_HDA ( 0, hdr, sizeof ( *hdr ) );
+			continue;
+		}
+
+		/* Found log partition */
+		DBG ( "INT13CON partition %d at [%08x,%08x)\n", ( i + 1 ),
+		      part[i].start, ( part[i].start + part[i].length ) );
+		int13con_lba = part[i].start;
+		int13con_max_lba = ( part[i].start + part[i].length - 1 );
+
+		/* Initialise log buffer */
+		memset ( &int13con_buffer[ sizeof ( *hdr ) ], 0,
+			 ( sizeof ( int13con_buffer ) - sizeof ( *hdr ) ) );
+		int13con_offset = sizeof ( hdr->magic );
+
+		return 0;
+	}
+
+	DBG ( "INT13CON found no log partition\n" );
+	return -ENOENT;
+}
+
+/**
+ * Initialise INT13 console
+ *
+ */
+static void int13con_init ( void ) {
+	uint8_t error;
+	uint16_t check;
+	unsigned int discard_c;
+	unsigned int discard_d;
+	int rc;
+
+	/* Check for INT13 extensions */
+	__asm__ __volatile__ ( REAL_CODE ( "int $0x13\n\t"
+					   "setc %%al\n\t" )
+			       : "=a" ( error ), "=b" ( check ),
+				 "=c" ( discard_c ), "=d" ( discard_d )
+			       : "0" ( INT13_EXTENSION_CHECK << 8 ),
+				 "1" ( 0x55aa ), "3" ( INT13CON_DRIVE ) );
+	if ( error || ( check != 0xaa55 ) ) {
+		DBG ( "INT13CON missing extensions (%02x,%04x)\n",
+		      error, check );
+		return;
+	}
+
+	/* Locate log partition */
+	if ( ( rc = int13con_find() ) != 0)
+		return;
+
+	/* Enable console */
+	int13con.disabled = 0;
+}
+
+/**
+ * INT13 console initialisation function
+ */
+struct init_fn int13con_init_fn __init_fn ( INIT_CONSOLE ) = {
+	.initialise = int13con_init,
+};
+
+/** INT13 console driver */
+struct console_driver int13con __console_driver = {
+	.putchar = int13con_putchar,
+	.disabled = CONSOLE_DISABLED,
+	.usage = CONSOLE_INT13,
+};
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c b/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c
index c382e3c36..957f8e324 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c b/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c
index 61873039f..34efa0b39 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/pci.h>
@@ -38,7 +42,11 @@ static int pcibios_num_bus ( void ) {
 	int discard_a, discard_D;
 	uint8_t max_bus;
 
-	__asm__ __volatile__ ( REAL_CODE ( "stc\n\t"
+	/* We issue this call using flat real mode, to work around a
+	 * bug in some HP BIOSes.
+	 */
+	__asm__ __volatile__ ( REAL_CODE ( "call flatten_real_mode\n\t"
+					   "stc\n\t"
 					   "int $0x1a\n\t"
 					   "jnc 1f\n\t"
 					   "xorw %%cx, %%cx\n\t"
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c b/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c
index fad421c2a..9aab03c03 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c b/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c
index 67041d4ca..cdbeac8d5 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c b/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c
index 2adc7b040..9cf2bf29e 100644
--- a/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c
+++ b/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c
index 657d47b6c..104313666 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 #include <ipxe/init.h>
@@ -342,6 +346,7 @@ int pxe_start_nbp ( void ) {
 	return 0;
 }
 
+REQUIRING_SYMBOL ( pxe_api_call );
 REQUIRE_OBJECT ( pxe_preboot );
 REQUIRE_OBJECT ( pxe_undi );
 REQUIRE_OBJECT ( pxe_udp );
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S b/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S
index 6274264ff..07852cd50 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c
index 9d1896507..f92dae0d1 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c
@@ -21,9 +21,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c
index 6e9610294..456ffb5fd 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c
@@ -31,9 +31,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 FEATURE ( FEATURE_MISC, "PXEXT", DHCP_EB_FEATURE_PXE_EXT, 2 );
 
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c
index 695af3b93..e6a2e072a 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/init.h>
 #include "pxe.h"
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c
index 534352b2b..6e09080bc 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c
@@ -22,9 +22,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -174,18 +178,16 @@ pxenv_get_cached_info ( struct s_PXENV_GET_CACHED_INFO *get_cached_info ) {
 	}
 	info = &cached_info[idx];
 
-	/* Construct cached version of packet, if not already constructed. */
-	if ( ! info->dhcphdr.op ) {
-		/* Construct DHCP packet */
-		creator = &pxe_dhcp_packet_creators[idx];
-		if ( ( rc = creator->create ( pxe_netdev, info,
-					      sizeof ( *info ) ) ) != 0 ) {
-			DBGC ( &pxe_netdev, " failed to build packet: %s\n",
-			       strerror ( rc ) );
-			goto err;
-		}
+	/* Construct DHCP packet */
+	creator = &pxe_dhcp_packet_creators[idx];
+	if ( ( rc = creator->create ( pxe_netdev, info,
+				      sizeof ( *info ) ) ) != 0 ) {
+		DBGC ( &pxe_netdev, " failed to build packet: %s\n",
+		       strerror ( rc ) );
+		goto err;
 	}
 
+	/* Copy packet (if applicable) */
 	len = get_cached_info->BufferSize;
 	if ( len == 0 ) {
 		/* Point client at our cached buffer.
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c
index f4801bad0..068d8a7b2 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c
@@ -21,9 +21,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -36,6 +40,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/xfer.h>
 #include <ipxe/open.h>
 #include <ipxe/process.h>
+#include <ipxe/uri.h>
+#include <realmode.h>
 #include <pxe.h>
 
 /** A PXE TFTP connection */
@@ -170,11 +176,10 @@ static struct pxe_tftp_connection pxe_tftp = {
  * @v blksize		Requested block size
  * @ret rc		Return status code
  */
-static int pxe_tftp_open ( uint32_t ipaddress, unsigned int port,
-			   const unsigned char *filename, size_t blksize,
-			   int sizeonly ) {
-	char uri_string[PXE_TFTP_URI_LEN];
+static int pxe_tftp_open ( IP4_t ipaddress, UDP_PORT_t port,
+			   UINT8_t *filename, UINT16_t blksize ) {
 	struct in_addr address;
+	struct uri *uri;
 	int rc;
 
 	/* Reset PXE TFTP connection structure */
@@ -185,19 +190,20 @@ static int pxe_tftp_open ( uint32_t ipaddress, unsigned int port,
 	pxe_tftp.blksize = blksize;
 	pxe_tftp.rc = -EINPROGRESS;
 
-	/* Construct URI string */
+	/* Construct URI */
 	address.s_addr = ipaddress;
-	if ( ! port )
-		port = htons ( TFTP_PORT );
-	snprintf ( uri_string, sizeof ( uri_string ), "tftp%s://%s:%d%s%s",
-		   sizeonly ? "size" : "", inet_ntoa ( address ),
-		   ntohs ( port ), ( ( filename[0] == '/' ) ? "" : "/" ),
-		   filename );
-	DBG ( " %s", uri_string );
+	DBG ( " %s", inet_ntoa ( address ) );
+	if ( port )
+		DBG ( ":%d", ntohs ( port ) );
+	DBG ( ":%s", filename );
+	uri = tftp_uri ( address, ntohs ( port ), ( ( char * ) filename ) );
+	if ( ! uri ) {
+		DBG ( " could not create URI\n" );
+		return -ENOMEM;
+	}
 
 	/* Open PXE TFTP connection */
-	if ( ( rc = xfer_open_uri_string ( &pxe_tftp.xfer,
-					   uri_string ) ) != 0 ) {
+	if ( ( rc = xfer_open_uri ( &pxe_tftp.xfer, uri ) ) != 0 ) {
 		DBG ( " could not open (%s)\n", strerror ( rc ) );
 		return rc;
 	}
@@ -259,8 +265,7 @@ static PXENV_EXIT_t pxenv_tftp_open ( struct s_PXENV_TFTP_OPEN *tftp_open ) {
 	if ( ( rc = pxe_tftp_open ( tftp_open->ServerIPAddress,
 				    tftp_open->TFTPPort,
 				    tftp_open->FileName,
-				    tftp_open->PacketSize,
-				    0) ) != 0 ) {
+				    tftp_open->PacketSize ) ) != 0 ) {
 		tftp_open->Status = PXENV_STATUS ( rc );
 		return PXENV_EXIT_FAILURE;
 	}
@@ -483,7 +488,7 @@ PXENV_EXIT_t pxenv_tftp_read_file ( struct s_PXENV_TFTP_READ_FILE
 
 	/* Open TFTP file */
 	if ( ( rc = pxe_tftp_open ( tftp_read_file->ServerIPAddress, 0,
-				    tftp_read_file->FileName, 0, 0 ) ) != 0 ) {
+				    tftp_read_file->FileName, 0 ) ) != 0 ) {
 		tftp_read_file->Status = PXENV_STATUS ( rc );
 		return PXENV_EXIT_FAILURE;
 	}
@@ -553,7 +558,7 @@ static PXENV_EXIT_t pxenv_tftp_get_fsize ( struct s_PXENV_TFTP_GET_FSIZE
 
 	/* Open TFTP file */
 	if ( ( rc = pxe_tftp_open ( tftp_get_fsize->ServerIPAddress, 0,
-				    tftp_get_fsize->FileName, 0, 1 ) ) != 0 ) {
+				    tftp_get_fsize->FileName, 0 ) ) != 0 ) {
 		tftp_get_fsize->Status = PXENV_STATUS ( rc );
 		return PXENV_EXIT_FAILURE;
 	}
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c
index 32bc39c8e..071cb59db 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c
@@ -11,6 +11,7 @@
 #include <ipxe/udp.h>
 #include <ipxe/uaccess.h>
 #include <ipxe/process.h>
+#include <realmode.h>
 #include <pxe.h>
 
 /*
@@ -30,9 +31,25 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** A PXE UDP pseudo-header */
+struct pxe_udp_pseudo_header {
+	/** Source IP address */
+	IP4_t src_ip;
+	/** Source port */
+	UDP_PORT_t s_port;
+	/** Destination IP address */
+	IP4_t dest_ip;
+	/** Destination port */
+	UDP_PORT_t d_port;
+} __attribute__ (( packed ));
 
 /** A PXE UDP connection */
 struct pxe_udp_connection {
@@ -40,8 +57,8 @@ struct pxe_udp_connection {
 	struct interface xfer;
 	/** Local address */
 	struct sockaddr_in local;
-	/** Current PXENV_UDP_READ parameter block */
-	struct s_PXENV_UDP_READ *pxenv_udp_read;
+	/** List of received packets */
+	struct list_head list;
 };
 
 /**
@@ -58,45 +75,38 @@ struct pxe_udp_connection {
 static int pxe_udp_deliver ( struct pxe_udp_connection *pxe_udp,
 			     struct io_buffer *iobuf,
 			     struct xfer_metadata *meta ) {
-	struct s_PXENV_UDP_READ *pxenv_udp_read = pxe_udp->pxenv_udp_read;
+	struct pxe_udp_pseudo_header *pshdr;
 	struct sockaddr_in *sin_src;
 	struct sockaddr_in *sin_dest;
-	userptr_t buffer;
-	size_t len;
-	int rc = 0;
-
-	if ( ! pxenv_udp_read ) {
-		DBG ( "PXE discarded UDP packet\n" );
-		rc = -ENOBUFS;
-		goto done;
-	}
-
-	/* Copy packet to buffer and record length */
-	buffer = real_to_user ( pxenv_udp_read->buffer.segment,
-				pxenv_udp_read->buffer.offset );
-	len = iob_len ( iobuf );
-	if ( len > pxenv_udp_read->buffer_size )
-		len = pxenv_udp_read->buffer_size;
-	copy_to_user ( buffer, 0, iobuf->data, len );
-	pxenv_udp_read->buffer_size = len;
+	int rc;
 
-	/* Fill in source/dest information */
+	/* Extract metadata */
 	assert ( meta );
 	sin_src = ( struct sockaddr_in * ) meta->src;
 	assert ( sin_src );
 	assert ( sin_src->sin_family == AF_INET );
-	pxenv_udp_read->src_ip = sin_src->sin_addr.s_addr;
-	pxenv_udp_read->s_port = sin_src->sin_port;
 	sin_dest = ( struct sockaddr_in * ) meta->dest;
 	assert ( sin_dest );
 	assert ( sin_dest->sin_family == AF_INET );
-	pxenv_udp_read->dest_ip = sin_dest->sin_addr.s_addr;
-	pxenv_udp_read->d_port = sin_dest->sin_port;
 
-	/* Mark as received */
-	pxe_udp->pxenv_udp_read = NULL;
+	/* Construct pseudo-header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, sizeof ( *pshdr ) ) ) != 0 ) {
+		DBG ( "PXE could not prepend pseudo-header\n" );
+		rc = -ENOMEM;
+		goto drop;
+	}
+	pshdr = iob_push ( iobuf, sizeof ( *pshdr ) );
+	pshdr->src_ip = sin_src->sin_addr.s_addr;
+	pshdr->s_port = sin_src->sin_port;
+	pshdr->dest_ip = sin_dest->sin_addr.s_addr;
+	pshdr->d_port = sin_dest->sin_port;
 
- done:
+	/* Add to queue */
+	list_add_tail ( &iobuf->list, &pxe_udp->list );
+
+	return 0;
+
+ drop:
 	free_iob ( iobuf );
 	return rc;
 }
@@ -116,6 +126,7 @@ static struct pxe_udp_connection pxe_udp = {
 	.local = {
 		.sin_family = AF_INET,
 	},
+	.list = LIST_HEAD_INIT ( pxe_udp.list ),
 };
 
 /**
@@ -205,11 +216,20 @@ static PXENV_EXIT_t pxenv_udp_open ( struct s_PXENV_UDP_OPEN *pxenv_udp_open ) {
  */
 static PXENV_EXIT_t
 pxenv_udp_close ( struct s_PXENV_UDP_CLOSE *pxenv_udp_close ) {
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
 	DBG ( "PXENV_UDP_CLOSE\n" );
 
 	/* Close UDP connection */
 	intf_restart ( &pxe_udp.xfer, 0 );
 
+	/* Discard any received packets */
+	list_for_each_entry_safe ( iobuf, tmp, &pxe_udp.list, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+
 	pxenv_udp_close->Status = PXENV_STATUS_SUCCESS;
 	return PXENV_EXIT_SUCCESS;
 }
@@ -365,20 +385,32 @@ pxenv_udp_write ( struct s_PXENV_UDP_WRITE *pxenv_udp_write ) {
 static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) {
 	struct in_addr dest_ip_wanted = { .s_addr = pxenv_udp_read->dest_ip };
 	struct in_addr dest_ip;
+	struct io_buffer *iobuf;
+	struct pxe_udp_pseudo_header *pshdr;
 	uint16_t d_port_wanted = pxenv_udp_read->d_port;
 	uint16_t d_port;
+	userptr_t buffer;
+	size_t len;
+
+	/* Try receiving a packet, if the queue is empty */
+	if ( list_empty ( &pxe_udp.list ) )
+		step();
 
-	/* Try receiving a packet */
-	pxe_udp.pxenv_udp_read = pxenv_udp_read;
-	step();
-	if ( pxe_udp.pxenv_udp_read ) {
+	/* Remove first packet from the queue */
+	iobuf = list_first_entry ( &pxe_udp.list, struct io_buffer, list );
+	if ( ! iobuf ) {
 		/* No packet received */
 		DBG2 ( "PXENV_UDP_READ\n" );
-		pxe_udp.pxenv_udp_read = NULL;
 		goto no_packet;
 	}
-	dest_ip.s_addr = pxenv_udp_read->dest_ip;
-	d_port = pxenv_udp_read->d_port;
+	list_del ( &iobuf->list );
+
+	/* Strip pseudo-header */
+	assert ( iob_len ( iobuf ) >= sizeof ( *pshdr ) );
+	pshdr = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *pshdr ) );
+	dest_ip.s_addr = pshdr->dest_ip;
+	d_port = pshdr->d_port;
 	DBG ( "PXENV_UDP_READ" );
 
 	/* Filter on destination address and/or port */
@@ -386,14 +418,29 @@ static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) {
 	     ( dest_ip_wanted.s_addr != dest_ip.s_addr ) ) {
 		DBG ( " wrong IP %s", inet_ntoa ( dest_ip ) );
 		DBG ( " (wanted %s)\n", inet_ntoa ( dest_ip_wanted ) );
-		goto no_packet;
+		goto drop;
 	}
 	if ( d_port_wanted && ( d_port_wanted != d_port ) ) {
 		DBG ( " wrong port %d", htons ( d_port ) );
 		DBG ( " (wanted %d)\n", htons ( d_port_wanted ) );
-		goto no_packet;
+		goto drop;
 	}
 
+	/* Copy packet to buffer and record length */
+	buffer = real_to_user ( pxenv_udp_read->buffer.segment,
+				pxenv_udp_read->buffer.offset );
+	len = iob_len ( iobuf );
+	if ( len > pxenv_udp_read->buffer_size )
+		len = pxenv_udp_read->buffer_size;
+	copy_to_user ( buffer, 0, iobuf->data, len );
+	pxenv_udp_read->buffer_size = len;
+
+	/* Fill in source/dest information */
+	pxenv_udp_read->src_ip = pshdr->src_ip;
+	pxenv_udp_read->s_port = pshdr->s_port;
+	pxenv_udp_read->dest_ip = pshdr->dest_ip;
+	pxenv_udp_read->d_port = pshdr->d_port;
+
 	DBG ( " %04x:%04x+%x %s:", pxenv_udp_read->buffer.segment,
 	      pxenv_udp_read->buffer.offset, pxenv_udp_read->buffer_size,
 	      inet_ntoa ( *( ( struct in_addr * ) &pxenv_udp_read->src_ip ) ));
@@ -401,9 +448,14 @@ static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) {
 	      inet_ntoa ( *( ( struct in_addr * ) &pxenv_udp_read->dest_ip ) ),
 	      ntohs ( pxenv_udp_read->d_port ) );
 
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+
 	pxenv_udp_read->Status = PXENV_STATUS_SUCCESS;
 	return PXENV_EXIT_SUCCESS;
 
+ drop:
+	free_iob ( iobuf );
  no_packet:
 	pxenv_udp_read->Status = PXENV_STATUS_FAILURE;
 	return PXENV_EXIT_FAILURE;
diff --git a/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c b/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c
index 29e586ed2..2eb68178a 100644
--- a/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c
+++ b/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c
@@ -21,9 +21,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c b/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c
index 1854501de..69d94c407 100644
--- a/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c
+++ b/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c
@@ -41,8 +41,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/image.h>
 #include <ipxe/version.h>
 #include <usr/imgmgmt.h>
-#include "config/console.h"
-#include "config/serial.h"
 
 /** The "SYSLINUX" version string */
 static char __bss16_array ( syslinux_version, [32] );
@@ -86,7 +84,6 @@ rmjmp_buf comboot_return;
 /* Mode flags set by INT 22h AX=0017h */
 static uint16_t comboot_graphics_mode = 0;
 
-
 /**
  * Print a string with a particular terminator
  */
@@ -261,8 +258,10 @@ static __asmcall void int21 ( struct i386_all_regs *ix86 ) {
 		break;
 
 	case 0x04: /* Write Character to Serial Port */
-		serial_putc ( ix86->regs.dl );
-		ix86->flags &= ~CF;
+		if ( serial_console.base ) {
+			uart_transmit ( &serial_console, ix86->regs.dl );
+			ix86->flags &= ~CF;
+		}
 		break;
 
 	case 0x09: /* Write DOS String to Console */
@@ -455,15 +454,16 @@ static __asmcall void int22 ( struct i386_all_regs *ix86 ) {
 		break;
 
 	case 0x000B: /* Get Serial Console Configuration */
-#if defined(CONSOLE_SERIAL) && !defined(COMPRESERVE)
-		ix86->regs.dx = COMCONSOLE;
-		ix86->regs.cx = 115200 / COMSPEED;
-		ix86->regs.bx = 0;
-#else
-		ix86->regs.dx = 0;
-#endif
+		if ( serial_console.base ) {
+			ix86->regs.dx = ( ( intptr_t ) serial_console.base );
+			ix86->regs.cx = serial_console.divisor;
+			ix86->regs.bx = 0;
+			ix86->flags &= ~CF;
+		}
+		break;
 
-		ix86->flags &= ~CF;
+	case 0x000C: /* Perform final cleanup */
+		shutdown_boot();
 		break;
 
 	case 0x000E: /* Get configuration file name */
@@ -712,3 +712,6 @@ void unhook_comboot_interrupts ( ) {
 	unhook_bios_interrupt ( 0x22, ( unsigned int ) int22_wrapper,
 				&int22_vector );
 }
+
+/* Avoid dragging in serial console support unconditionally */
+struct uart serial_console __attribute__ (( weak ));
diff --git a/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c b/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c
index 390fc5545..ef7ee8151 100644
--- a/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c
+++ b/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c b/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c
index c6b9fff12..f7df4f75b 100644
--- a/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c
+++ b/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/interface/vmware/vmware.c b/roms/ipxe/src/arch/i386/interface/vmware/vmware.c
index 8074e6118..a415465fb 100644
--- a/roms/ipxe/src/arch/i386/interface/vmware/vmware.c
+++ b/roms/ipxe/src/arch/i386/interface/vmware/vmware.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/i386/prefix/bootpart.S b/roms/ipxe/src/arch/i386/prefix/bootpart.S
index 968da1a38..6d0c6034a 100644
--- a/roms/ipxe/src/arch/i386/prefix/bootpart.S
+++ b/roms/ipxe/src/arch/i386/prefix/bootpart.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BOOT_SEG	0x07c0
 #define EXEC_SEG	0x0100
diff --git a/roms/ipxe/src/arch/i386/prefix/exeprefix.S b/roms/ipxe/src/arch/i386/prefix/exeprefix.S
index cb61287d3..5c648d51d 100644
--- a/roms/ipxe/src/arch/i386/prefix/exeprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/exeprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /* Initial temporary stack size */
 #define EXE_STACK_SIZE 0x400
diff --git a/roms/ipxe/src/arch/i386/prefix/hdprefix.S b/roms/ipxe/src/arch/i386/prefix/hdprefix.S
index 876bfe1be..1d012d80b 100644
--- a/roms/ipxe/src/arch/i386/prefix/hdprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/hdprefix.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
diff --git a/roms/ipxe/src/arch/i386/prefix/isaromprefix.S b/roms/ipxe/src/arch/i386/prefix/isaromprefix.S
index e28208089..fb49819ee 100644
--- a/roms/ipxe/src/arch/i386/prefix/isaromprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/isaromprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BUSTYPE "ISAR"
 #define _rom_start _isarom_start
diff --git a/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S b/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S
index 27ed231e7..6e43cd26a 100644
--- a/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S
@@ -5,12 +5,10 @@
  *****************************************************************************
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
-
-/* Since we have the whole stack, we can use cached DHCP information */
-REQUIRE_OBJECT ( pxeparent_dhcp )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /* Provide the PXENV_FILE_EXIT_HOOK API call */
+REQUIRING_SYMBOL ( _kkkpxe_start )
 REQUIRE_OBJECT ( pxe_exit_hook )
 
 #define PXELOADER_KEEP_UNDI
diff --git a/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S b/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S
index d177d7d62..3c17dbdb1 100644
--- a/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S
@@ -3,10 +3,7 @@
  *****************************************************************************
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
-
-/* Since we have the whole stack, we can use cached DHCP information */
-REQUEST_OBJECT ( pxeparent_dhcp )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PXELOADER_KEEP_UNDI
 #define PXELOADER_KEEP_PXE
diff --git a/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S b/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S
index c75608172..200006d83 100644
--- a/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S
@@ -3,7 +3,7 @@
  *****************************************************************************
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PXELOADER_KEEP_UNDI
 #define _pxe_start _kpxe_start
diff --git a/roms/ipxe/src/arch/i386/prefix/libprefix.S b/roms/ipxe/src/arch/i386/prefix/libprefix.S
index 7c1ece791..7d5c1ed53 100644
--- a/roms/ipxe/src/arch/i386/prefix/libprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/libprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
@@ -296,11 +300,9 @@ copy_bytes:
  * Zero bytes
  *
  * Parameters:
- *   %ds:esi : source address
  *   %es:edi : destination address
  *   %ecx : length
  * Returns:
- *   %ds:esi : next source address
  *   %es:edi : next destination address
  * Corrupts:
  *   None
@@ -396,8 +398,10 @@ process_bytes:
 	movw	%ax, %fs
 	movw	%ax, %gs
 
+#ifdef NDEBUG
 	/* Call memcpy()-like function */
 	call	*%bx
+#endif
 
 	/* Return to (flat) real mode */
 	movl	%cr0, %eax
@@ -411,6 +415,20 @@ process_bytes:
 	popw	%fs
 	popw	%gs
 
+#ifndef NDEBUG
+	/* Call memcpy()-like function in flat real mode (to allow for
+	 * debug output via INT 10).
+	 */
+	pushw	%ds
+	pushw	%es
+	xorw	%ax, %ax
+	movw	%ax, %ds
+	movw	%ax, %es
+	call	*%bx
+	popw	%es
+	popw	%ds
+#endif
+
 	/* Restore GDT */
 	data32 lgdt -8(%bp)
 	addw	$( 8 /* saved GDT */ + ( PM_DS + 8 ) /* GDT on stack */ ), %sp
@@ -442,11 +460,11 @@ process_bytes:
 
 	/* Convert %ds:esi and %es:edi back to physical addresses */
 	xorl	%eax, %eax
-	movw	%ds, %cx
+	movw    %ds, %ax
 	shll	$4, %eax
 	addl	%eax, %esi
 	xorl	%eax, %eax
-	movw	%es, %cx
+	movw    %es, %ax
 	shll	$4, %eax
 	addl	%eax, %edi
 
@@ -678,12 +696,21 @@ install:
 	.globl install_prealloc
 install_prealloc:
 	progress "install_prealloc:\n"
-	/* Save registers */
+	/* Save registers on external stack */
 	pushal
 	pushw	%ds
 	pushw	%es
 	cld			/* Sanity: clear the direction flag asap */
 
+	/* Switch to temporary stack in .bss16 */
+	pushw	%ss
+	popw	%ds
+	movl	%esp, %ecx
+	movw	%bx, %ss
+	movl	$_data16_memsz, %esp
+	pushw	%ds
+	pushl	%ecx
+
 	/* Set up %ds for (read-only) access to .prefix */
 	pushw	%cs
 	popw	%ds
@@ -710,6 +737,7 @@ install_prealloc:
 	popl	%esi
 
 #ifndef KEEP_IT_REAL
+
 	/* Access high memory by enabling the A20 gate.  (We will
 	 * already have 4GB segment limits as a result of calling
 	 * install_block.)
@@ -778,7 +806,7 @@ payload_death_message:
 	movzwl	%bx, %edi
 	shll	$4, %edi
 	movl	$_data16_filesz, %ecx
-	movl	$_data16_memsz, %edx
+	movl	$_data16_filesz, %edx	/* do not zero our temporary stack */
 	call	install_block		/* .data16 */
 
 	/* Set up %ds for access to .data16 */
@@ -787,11 +815,8 @@ payload_death_message:
 	/* Restore decompression temporary area physical address */
 	popl	%edi
 
-#ifdef KEEP_IT_REAL
-	/* Initialise libkir */
-	movw	%ax, (init_libkir_vector+2)
-	lcall	*init_libkir_vector
-#else
+#ifndef KEEP_IT_REAL
+
 	/* Find a suitable decompression temporary area, if none specified */
 	pushl	%eax
 	testl	%edi, %edi
@@ -823,6 +848,22 @@ payload_death_message:
 	call	install_block
 	popl	%edi
 
+#endif /* KEEP_IT_REAL */
+
+	/* Switch back to original stack and zero .bss16 */
+	addr32 lss %ss:(%esp), %esp
+	pushl	%edi
+	pushw	%es
+	movw	%bx, %es
+	movl	$_data16_filesz, %edi
+	movl	$_data16_memsz, %ecx
+	subl	%edi, %ecx
+	call	zero_bytes
+	popw	%es
+	popl	%edi
+
+#ifndef KEEP_IT_REAL
+
 	/* Initialise librm at current location */
 	progress "  init_librm\n"
 	movw	%ax, (init_librm_vector+2)
@@ -834,7 +875,6 @@ payload_death_message:
 	incb	memmap_post
 	decl	%ebp
 1:
-
 	/* Call relocate() to determine target address for relocation.
 	 * relocate() will return with %esi, %edi and %ecx set up
 	 * ready for the copy to the new location.
@@ -857,7 +897,14 @@ payload_death_message:
 	/* Initialise librm at new location */
 	progress "  init_librm\n"
 	lcall	*init_librm_vector
-#endif
+
+#else /* KEEP_IT_REAL */
+
+	/* Initialise libkir */
+	movw	%ax, (init_libkir_vector+2)
+	lcall	*init_libkir_vector
+
+#endif /* KEEP_IT_REAL */
 
 	/* Close access to payload */
 	progress "  close_payload\n"
diff --git a/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S b/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S
index 259bc6ba5..64135e14b 100644
--- a/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL_ANY )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BZI_LOAD_HIGH_ADDR 0x100000
 
diff --git a/roms/ipxe/src/arch/i386/prefix/mbr.S b/roms/ipxe/src/arch/i386/prefix/mbr.S
index adfe20410..a1e237de8 100644
--- a/roms/ipxe/src/arch/i386/prefix/mbr.S
+++ b/roms/ipxe/src/arch/i386/prefix/mbr.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
diff --git a/roms/ipxe/src/arch/i386/prefix/mromprefix.S b/roms/ipxe/src/arch/i386/prefix/mromprefix.S
index 4c94457c2..b636b92af 100644
--- a/roms/ipxe/src/arch/i386/prefix/mromprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/mromprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PCIBIOS_READ_CONFIG_WORD	0xb109
 #define PCIBIOS_READ_CONFIG_DWORD	0xb10a
@@ -463,6 +467,7 @@ pci_set_mem_access:
 	.org	0x00
 mromheader:
 	.word	0xaa55			/* BIOS extension signature */
+	.byte	0x01			/* Dummy size (BIOS bug workaround) */
 	.org	0x18
 	.word	mpciheader
 	.org	0x1a
diff --git a/roms/ipxe/src/arch/i386/prefix/nbiprefix.S b/roms/ipxe/src/arch/i386/prefix/nbiprefix.S
index 06e7df5b7..16c79566c 100644
--- a/roms/ipxe/src/arch/i386/prefix/nbiprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/nbiprefix.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.text
 	.arch i386
 	.code16
diff --git a/roms/ipxe/src/arch/i386/prefix/nullprefix.S b/roms/ipxe/src/arch/i386/prefix/nullprefix.S
index 032d41e0f..bd0ff339e 100644
--- a/roms/ipxe/src/arch/i386/prefix/nullprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/nullprefix.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.org	0
 	.text
 	.arch i386
diff --git a/roms/ipxe/src/arch/i386/prefix/pciromprefix.S b/roms/ipxe/src/arch/i386/prefix/pciromprefix.S
index 45ba31f50..5a5a49647 100644
--- a/roms/ipxe/src/arch/i386/prefix/pciromprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/pciromprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BUSTYPE "PCIR"
 #define _rom_start _pcirom_start
diff --git a/roms/ipxe/src/arch/i386/prefix/pxeprefix.S b/roms/ipxe/src/arch/i386/prefix/pxeprefix.S
index 6e29c7949..465ce4345 100644
--- a/roms/ipxe/src/arch/i386/prefix/pxeprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/pxeprefix.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PXENV_UNDI_SHUTDOWN		0x0005
 #define	PXENV_UNDI_GET_NIC_TYPE		0x0012
diff --git a/roms/ipxe/src/arch/i386/prefix/romprefix.S b/roms/ipxe/src/arch/i386/prefix/romprefix.S
index 7bc4fe8cd..18dda2b37 100644
--- a/roms/ipxe/src/arch/i386/prefix/romprefix.S
+++ b/roms/ipxe/src/arch/i386/prefix/romprefix.S
@@ -6,9 +6,10 @@
  * table so using a noticeable amount of stack space is a no-no.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #include <config/general.h>
+#include <config/branding.h>
 
 #define PNP_SIGNATURE ( '$' + ( 'P' << 8 ) + ( 'n' << 16 ) + ( 'P' << 24 ) )
 #define PMM_SIGNATURE ( '$' + ( 'P' << 8 ) + ( 'M' << 16 ) + ( 'M' << 24 ) )
@@ -90,7 +91,7 @@ pciheader:
 	.ascii	"PCIR"			/* Signature */
 	.word	pci_vendor_id		/* Vendor identification */ 
 	.word	pci_device_id		/* Device identification */
-	.word	0x0000			/* Device list pointer */
+	.word	( pci_devlist - pciheader ) /* Device list pointer */
 	.word	pciheader_len		/* PCI data structure length */
 	.byte	0x03			/* PCI data structure revision */
 	.byte	0x02, 0x00, 0x00	/* Class code */
@@ -106,6 +107,17 @@ pciheader_runtime_length:
 	.equ pciheader_len, . - pciheader
 	.size pciheader, . - pciheader
 
+	/* PCI additional device list (filled in by linker) */
+	.section ".pci_devlist.00000000", "a", @progbits
+pci_devlist:
+	.previous
+	.section ".pci_devlist.ffffffff", "a", @progbits
+pci_devlist_end:
+	.short	0x0000 /* List terminator */
+	.previous
+	/* Ensure that terminator is always present */
+	.reloc pciheader, RELOC_TYPE_NONE, pci_devlist_end
+
 	.section ".zinfo.fixup", "a", @progbits	/* Compressor fixups */
 	.ascii	ZINFO_TYPE_ADxW
 	.long	pciheader_image_length
@@ -573,7 +585,7 @@ get_pmm_decompress_to:
  * Note to hardware vendors:
  *
  * If you wish to brand this boot ROM, please do so by defining the
- * strings PRODUCT_NAME and PRODUCT_SHORT_NAME in config/general.h.
+ * strings PRODUCT_NAME and PRODUCT_SHORT_NAME in config/branding.h.
  *
  * While nothing in the GPL prevents you from removing all references
  * to iPXE or http://ipxe.org, we prefer you not to do so.
@@ -589,7 +601,10 @@ init_message:
 	.ascii	"\n"
 	.ascii	PRODUCT_NAME
 	.ascii	"\n"
-	.asciz	"iPXE (http://ipxe.org)"
+	.ascii	PRODUCT_SHORT_NAME
+	.ascii	" ("
+	.ascii	PRODUCT_URI
+	.asciz	")"
 	.size	init_message, . - init_message
 .ifeqs	BUSTYPE, "PCIR"
 init_message_pci:
@@ -771,7 +786,9 @@ exec:	/* Set %ds = %cs */
 
 	/* Store PCI bus:dev.fn, if applicable */
 .ifeqs	BUSTYPE, "PCIR"
+#ifdef AUTOBOOT_ROM_FILTER
 	movw	%ax, autoboot_busdevfn
+#endif /* AUTOBOOT_ROM_FILTER */
 .endif
 
 	/* Call main() */
@@ -870,3 +887,9 @@ wait_for_tick:
 	popl	%eax
 	ret
 	.size wait_for_tick, . - wait_for_tick
+
+/* Drag in objects via _rom_start */
+REQUIRING_SYMBOL ( _rom_start )
+
+/* Drag in ROM configuration */
+REQUIRE_OBJECT ( config_romprefix )
diff --git a/roms/ipxe/src/arch/i386/prefix/undiloader.S b/roms/ipxe/src/arch/i386/prefix/undiloader.S
index 74bb59041..5cace44b7 100644
--- a/roms/ipxe/src/arch/i386/prefix/undiloader.S
+++ b/roms/ipxe/src/arch/i386/prefix/undiloader.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.code16
diff --git a/roms/ipxe/src/arch/i386/prefix/unlzma.S b/roms/ipxe/src/arch/i386/prefix/unlzma.S
new file mode 100644
index 000000000..8d4b3c1a8
--- /dev/null
+++ b/roms/ipxe/src/arch/i386/prefix/unlzma.S
@@ -0,0 +1,942 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/****************************************************************************
+ *
+ * This file provides the decompress() and decompress16() functions
+ * which can be called in order to decompress an LZMA-compressed
+ * image.  The code is modelled on the public-domain "XZ Embedded"
+ * implementation as used by the Linux kernel.  Symbol names are
+ * chosen to match the XZ Embedded implementation where possible, for
+ * ease of reference.
+ *
+ * This code is optimised for size rather than speed, since the amount
+ * of data to be decompressed is trivially small by modern standards.
+ *
+ * The same basic assembly code is used to compile both decompress()
+ * and decompress16().
+ *
+ * Note that these functions require large amounts of stack space.
+ *
+ ****************************************************************************
+ */
+
+	.text
+	.arch i586
+	.section ".prefix.lib", "ax", @progbits
+
+#ifdef CODE16
+#define ADDR16
+#define ADDR32 addr32
+#define decompress decompress16
+	.code16
+#else /* CODE16 */
+#define ADDR16 addr16
+#define ADDR32
+	.code32
+#endif /* CODE16 */
+
+/****************************************************************************
+ * Debugging
+ ****************************************************************************
+ *
+ * This code will usually run in 16-bit protected mode, in which case
+ * only the 0xe9 debug port (present on some virtual machines) can be
+ * used.
+ *
+ * To debug on real hardware, build with DEBUG=libprefix.  This will
+ * cause this code to be called in flat real mode, and so DEBUG_INT10
+ * may be used.
+ */
+
+/* Enable debugging via 0xe9 debug port */
+#define DEBUG_E9 0
+
+/* Enable debugging via BIOS INT 10 (works only when in flat real mode) */
+#define DEBUG_INT10 0
+
+#if ( DEBUG_E9 || DEBUG_INT10 )
+	.macro	print_character, reg
+	pushfl
+	pushw	%ax
+	pushw	%bx
+	pushw	%bp
+	movb 	\reg, %al
+	movw	$0x0007, %bx
+	movb	$0x0e, %ah
+#if DEBUG_E9
+	outb	%al, $0xe9
+#endif
+#if DEBUG_INT10
+	cmpb	$('\n'), %al
+	jne	L\@
+	int	$0x10
+	movb	$('\r'), %al
+L\@:	int	$0x10
+#endif
+	popw	%bp
+	popw	%bx
+	popw	%ax
+	popfl
+	.endm
+
+	.macro	print_hex_nibble
+	pushfl
+	pushw	%ax
+	cmpb	$10, %al
+	sbb	$0x69, %al
+	das
+	print_character %al
+	popw	%ax
+	popfl
+	.endm
+
+	.macro	print_hex_byte, reg
+	pushfl
+	pushw	%ax
+	movb	\reg, %al
+	pushw	%ax
+	shrb	$4, %al
+	print_hex_nibble
+	popw	%ax
+	andb	$0x0f, %al
+	print_hex_nibble
+	popw	%ax
+	popfl
+	.endm
+
+	.macro	print_hex_word, reg
+	pushw	%ax
+	movw	\reg, %ax
+	print_hex_byte %ah
+	print_hex_byte %al
+	popw	%ax
+	.endm
+
+	.macro	print_hex_dword, reg
+	pushl	%eax
+	movl	\reg, %eax
+	rorl	$16, %eax
+	print_hex_word %ax
+	rorl	$16, %eax
+	print_hex_word %ax
+	popl	%eax
+	.endm
+#else
+	.macro	print_character, char
+	.endm
+	.macro	print_hex_byte, reg
+	.endm
+	.macro	print_hex_word, reg
+	.endm
+	.macro	print_hex_dword, reg
+	.endm
+#endif
+
+/****************************************************************************
+ * LZMA parameters and data structures
+ ****************************************************************************
+ */
+
+/* LZMA decompressor states (as used in XZ Embedded) */
+#define STATE_LIT_LIT 0x00
+#define STATE_MATCH_LIT_LIT 0x01
+#define STATE_REP_LIT_LIT 0x02
+#define STATE_SHORTREP_LIT_LIT 0x03
+#define STATE_MATCH_LIT 0x04
+#define STATE_REP_LIT 0x05
+#define STATE_SHORTREP_LIT 0x06
+#define STATE_LIT_MATCH 0x07
+#define STATE_LIT_LONGREP 0x08
+#define STATE_LIT_SHORTREP 0x09
+#define STATE_NONLIT_MATCH 0x0a
+#define STATE_NONLIT_REP 0x0b
+
+/* LZMA maximum decompressor state in which most recent symbol was a literal */
+#define STATE_LIT_MAX 0x06
+
+/* LZMA number of literal context bits ("lc=" parameter) */
+#define LZMA_LC 2
+
+	.struct	0
+lzma_len_dec:
+choice:		.word	0
+choice2:	.word	0
+low:		.rept	( 1 << 3 )
+		.word	0
+		.endr
+mid:		.rept	( 1 << 3 )
+		.word	0
+		.endr
+high:		.rept	( 1 << 8 )
+		.word	0
+		.endr
+	.equ	sizeof__lzma_len_dec, . - lzma_len_dec
+	.previous
+
+	.struct	0
+lzma_dec:
+out_start:	.long	0
+rc_code:	.long	0
+rc_range:	.long	0
+len:		.word	0
+reps:
+rep0:		.long	0
+rep1:		.long	0
+rep2:		.long	0
+rep3:		.long	0
+probs:
+is_match:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep:		.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep0:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep1:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep2:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep0_long:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+dist_slot:	.rept	( 4 * ( 1 << 6 ) )
+		.word	0
+		.endr
+dist_special:	.rept	( ( 1 << ( 14 / 2 ) ) - 14 )
+		.word	0
+		.endr
+dist_align:	.rept	( 1 << 4 )
+		.word	0
+		.endr
+match_len_dec:	.space	sizeof__lzma_len_dec
+rep_len_dec:	.space	sizeof__lzma_len_dec
+literal:	.rept	( ( 1 << LZMA_LC ) * 0x300 )
+		.word	0
+		.endr
+	.align	4
+	.equ	sizeof__lzma_dec, . - lzma_dec
+	.previous
+
+	/* Some binutils versions seem not to handle .struct/.previous */
+	.section ".prefix.lib", "ax", @progbits
+
+/*****************************************************************************
+ * Normalise range encoder
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : current range
+ *****************************************************************************
+ */
+rc_normalise:
+	/* Check if rc_range is less than 1<<24 */
+	testb	$0xff, (rc_range+3)(%ebp)
+	jnz	1f
+	/* If it is, shift in a new byte from the compressed input data */
+	shll	$8, rc_range(%ebp)
+	shll	$8, rc_code(%ebp)
+	ADDR32 lodsb
+	movb	%al, (rc_code+0)(%ebp)
+1:	/* Return current range */
+	movl	rc_range(%ebp), %eax
+	ret
+	.size	rc_normalise, . - rc_normalise
+
+/*****************************************************************************
+ * Decode single range-encoded bit using a probability estimate
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate pointer (offset from %ebp)
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   CF : decoded bit
+ *   ZF : inverse of decoded bit
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bit:
+	/* Preserve registers */
+	pushl	%eax
+	pushl	%edx
+	/* Perform normalisation */
+	call	rc_normalise
+	/* Calculate bound in %eax and probability estimate in %dx */
+	shrl	$11, %eax
+	movzwl	(%ebp,%ebx), %edx
+	mul	%edx /* will zero %edx */
+	movw	(%ebp,%ebx), %dx
+	/* Compare code against bound */
+	cmpl	%eax, rc_code(%ebp)
+	jae	2f
+1:	/* Code is less than bound */
+	movl	%eax, rc_range(%ebp)
+	negw	%dx
+	addw	$(1<<11), %dx
+	shrw	$5, %dx
+	addw	%dx, (%ebp,%ebx)
+	xorw	%ax, %ax	/* Clear CF, set ZF */
+	jmp	99f
+2:	/* Code is greater than or equal to bound */
+	subl	%eax, rc_range(%ebp)
+	subl	%eax, rc_code(%ebp)
+	shrw	$5, %dx
+	subw	%dx, (%ebp,%ebx)
+	incw	%dx		/* Clear ZF (%dx is 11-bit; can never wrap) */
+	stc			/* Set CF */
+99:	/* Restore registers and return */
+	popl	%edx
+	popl	%eax
+	ret
+	.size	rc_bit, . - rc_bit
+
+/*****************************************************************************
+ * Decode MSB-first bittree
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate set pointer (offset from %ebp)
+ *   %cx : number of bits to decode
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bittree
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bittree:
+	/* Preserve registers */
+	pushl	%edi
+	pushw	%cx
+	movl	%ebx, %edi
+	/* Initialise registers */
+	movl	$1, %eax
+1:	/* Decode bit */
+	leaw	(%edi,%eax,2), %bx	/* high word always zero anyway */
+	call	rc_bit
+	rclw	%ax
+	ADDR16 loop 1b
+	/* Restore registers, clear unwanted high bit of result, and return */
+	movl	%edi, %ebx
+	popw	%cx
+	popl	%edi
+	btrw	%cx, %ax
+	ret
+	.size	rc_bittree, . - rc_bittree
+
+/*****************************************************************************
+ * Decode LSB-first bittree
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate set pointer (offset from %ebp)
+ *   %cx : number of bits to decode
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bittree
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bittree_reverse:
+	/* Preserve registers */
+	pushw	%cx
+	/* Decode bittree */
+	call	rc_bittree
+1:	/* Reverse result */
+	rcrb	%al
+	rclb	%ah
+	ADDR16 loop 1b
+	shrw	$8, %ax
+	/* Restore registers and return */
+	popw	%cx
+	ret
+	.size	rc_bittree_reverse, . - rc_bittree_reverse
+
+/*****************************************************************************
+ * Decode MSB-first bittree with optional match byte
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate set pointer (offset from %ebp)
+ *   %cl : match byte
+ *   %ch : 1 to use match byte, 0 to ignore match byte
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bittree
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bittree_match:
+	/* Preserve registers */
+	pushl	%edi
+	pushw	%cx
+	pushw	%dx
+	movl	%ebx, %edi
+	/* Initialise registers */
+	movl	$1, %eax
+1:	/* Decode bit */
+	rolb	$1, %cl
+	movw	%cx, %dx
+	andb	%dh, %dl		/* match_bit in %dl */
+	movw	%dx, %bx
+	addb	%bl, %bh
+	xorb	%bl, %bl
+	addw	%ax, %bx		/* offset + match_bit + symbol */
+	leaw	(%edi,%ebx,2), %bx	/* high word always zero anyway */
+	call	rc_bit
+	rclw	%ax
+	movb	%al, %dh
+	notb	%dh
+	xorb	%dh, %dl
+	andb	%dl, %ch		/* offset &= ( match_bit ^ bit ) */
+	testb	%ah, %ah
+	jz	1b
+	/* Restore registers, clear unwanted high bit of result, and return */
+	movl	%edi, %ebx
+	popw	%dx
+	popw	%cx
+	popl	%edi
+	xorb	%ah, %ah
+	ret
+	.size	rc_bittree_match, . - rc_bittree_match
+
+/*****************************************************************************
+ * Decode direct bits (no probability estimates)
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %cx : number of bits to decode
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bits
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_direct:
+	/* Preserve registers */
+	pushl	%ebx
+	pushw	%cx
+	pushl	%edx
+	/* Initialise registers */
+	xorl	%edx, %edx
+1:	/* Perform normalisation */
+	call	rc_normalise
+	/* Decode bit */
+	shrl	$1, %eax
+	movl	%eax, rc_range(%ebp)
+	movl	rc_code(%ebp), %ebx
+	subl	%eax, %ebx
+	js	2f
+	movl	%ebx, rc_code(%ebp)
+2:	rcll	%ebx
+	rcll	%edx
+	xorb	$1, %dl
+	ADDR16 loop 1b
+	/* Restore registers and return */
+	movl	%edx, %eax
+	popl	%edx
+	popw	%cx
+	popl	%ebx
+	ret
+	.size	rc_direct, . - rc_direct
+
+/*****************************************************************************
+ * Decode an LZMA literal
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer (updated)
+ *   %edx : LZMA state
+ *   CF : end of payload marker found (always zero)
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ *
+ * Literals are coded as an eight-bit tree, using a match byte if the
+ * previous symbol was not a literal.
+ *
+ */
+lzma_literal:
+	/* Get most recent output byte, if available */
+	xorl	%ebx, %ebx
+	cmpl	%edi, out_start(%ebp)
+	je	1f
+	movb	%es:-1(%edi), %bh
+1:	/* Locate probability estimate set */
+	shrb	$( 8 - LZMA_LC ), %bh
+	shlb	$1, %bh
+	leaw	literal(%ebx,%ebx,2), %bx
+	/* Get match byte, if applicable */
+	xorw	%cx, %cx
+	cmpb	$STATE_LIT_MAX, %dl
+	jbe	1f
+	movl	rep0(%ebp), %eax
+	notl	%eax
+	movb	%es:(%edi,%eax), %cl
+	movb	$1, %ch
+1:	/* Decode bittree */
+	call	rc_bittree_match
+	/* Store output byte */
+	ADDR32 stosb
+	print_hex_byte %al
+	print_character $(' ')
+	/* Update LZMA state */
+	subb	$3, %dl
+	jns	1f
+	xorb	%dl, %dl
+1:	cmpb	$7, %dl
+	jb	1f
+	subb	$3, %dl
+1:	/* Clear CF and return */
+	clc
+	ret
+	.size	lzma_literal, . - lzma_literal
+
+/*****************************************************************************
+ * Decode an LZMA length
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : length parameter pointer (offset from %ebp)
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ * Corrupts:
+ *   %ebx
+ *****************************************************************************
+ *
+ * Lengths are encoded as:
+ *
+ *   "0" + 3 bits    : lengths 2-9 ("low")
+ *   "10" + 3 bits   : lengths 10-17 ("mid")
+ *   "11" + 8 bits   : lengths 18-273 ("high")
+ */
+lzma_len:
+	/* Preserve registers */
+	pushl	%eax
+	pushl	%ecx
+	pushl	%edi
+	movl	%ebx, %edi
+	/* Start by assuming three bits and a base length of 2 */
+	movw	$3, %cx
+	movw	$2, len(%ebp)
+	/* Check low-length choice bit */
+	leal	choice(%edi), %ebx
+	call	rc_bit
+	leal	low(%edi), %ebx
+	jz	1f
+	/* Check high-length choice bit */
+	leal	choice2(%edi), %ebx
+	call	rc_bit
+	leal	mid(%edi), %ebx
+	movb	$10, len(%ebp)
+	jz	1f
+	leal	high(%edi), %ebx
+	movb	$8, %cl
+	movb	$18, len(%ebp)
+1:	/* Get encoded length */
+	call	rc_bittree
+	addw	%ax, len(%ebp)
+	/* Restore registers and return */
+	movl	%edi, %ebx
+	popl	%edi
+	popl	%ecx
+	popl	%eax
+	ret
+	.size	lzma_len, . - lzma_len
+
+/*****************************************************************************
+ * Copy (possibly repeated) matched data
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %cl : repeated match distance index (for repeated matches)
+ *   %eax : match distance (for non-repeated matches)
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer
+ *   CF : match distance is out of range
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ */
+match:	/* Update repeated match list */
+	print_character $('[')
+	movl	$3, %ecx
+	jmp	1f
+match_rep:
+	print_character $('[')
+	print_character $('R')
+	print_hex_byte %cl
+	print_character $('=')
+	movzbl	%cl, %ecx
+	movl	reps(%ebp,%ecx,4), %eax
+	jcxz	2f
+1:	movl	(reps-4)(%ebp,%ecx,4), %ebx
+	movl	%ebx, reps(%ebp,%ecx,4)
+	loop	1b
+	movl	%eax, rep0(%ebp)
+2:	/* Preserve registers */
+	pushl	%esi
+	/* Get stored match length */
+	movzwl	len(%ebp), %ecx
+	print_hex_dword	%eax
+	print_character $('+')
+	print_hex_word %cx
+	print_character $(']')
+	print_character $(' ')
+	/* Abort with CF set if match distance is out of range */
+	movl	out_start(%ebp), %esi
+	negl	%esi
+	leal	-1(%edi,%esi), %esi
+	cmpl	%eax, %esi
+	jc	99f
+	/* Perform copy */
+	notl	%eax
+	leal	(%edi,%eax), %esi
+	ADDR32 es rep movsb
+99:	/* Restore registers and return */
+	popl	%esi
+	ret
+	.size	match, . - match
+
+/*****************************************************************************
+ * Decode an LZMA match
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ *   CF : end of payload marker found
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ *
+ * Matches are encoded as an LZMA length followed by a 6-bit "distance
+ * slot" code, 0-26 fixed-probability bits, and 0-5 context encoded
+ * bits.
+ */
+lzma_match:
+	/* Preserve registers */
+	pushl	%edi
+	/* Update LZMA state */
+	cmpb	$STATE_LIT_MAX, %dl
+	movb	$STATE_LIT_MATCH, %dl
+	jbe	1f
+	movb	$STATE_NONLIT_MATCH, %dl
+1:	/* Decode length */
+	movl	$match_len_dec, %ebx
+	call	lzma_len
+	/* Decode distance slot */
+	movw	len(%ebp), %bx
+	subw	$2, %bx
+	cmpw	$4, %bx
+	jb	1f
+	movw	$3, %bx
+1:	shlw	$7, %bx
+	addw	$dist_slot, %bx
+	movw	$6, %cx
+	call	rc_bittree
+	/* Distance slots 0-3 are literal distances */
+	cmpb	$4, %al
+	jb	99f
+	/* Determine initial bits: 10/11 for even/odd distance codes */
+	movl	%eax, %edi
+	andw	$1, %di
+	orw	$2, %di
+	/* Determine number of context-encoded bits */
+	movw	%ax, %cx
+	shrb	$1, %cl
+	decb	%cl
+	/* Select context to be used in absence of fixed-probability bits */
+	movl	%edi, %ebx
+	shlw	%cl, %bx
+	subw	%ax, %bx
+	leaw	(dist_special-2)(%ebx,%ebx), %bx
+	/* Decode fixed-probability bits, if any */
+	cmpb	$6, %cl
+	jb	1f
+	subb	$4, %cl
+	shll	%cl, %edi
+	call	rc_direct
+	orl	%eax, %edi
+	/* Select context to be used in presence of fixed-probability bits */
+	movb	$4, %cl
+	movl	$dist_align, %ebx
+1:	/* Decode context-encoded bits */
+	shll	%cl, %edi
+	call	rc_bittree_reverse
+	orl	%edi, %eax
+99:	/* Restore registers and tail-call */
+	popl	%edi
+	jmp	match
+	.size	lzma_match, . - lzma_match
+
+/*****************************************************************************
+ * Decode an LZMA repeated match
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ *   CF : end of payload marker found
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ *
+ * Repeated matches are encoded as:
+ *
+ *   "00"	 : shortrep0 (implicit length 1)
+ *   "01" + len  : longrep0
+ *   "10" + len  : longrep1
+ *   "110" + len : longrep2
+ *   "111" + len : longrep3
+ */
+lzma_rep_match:
+	/* Initially assume longrep0 */
+	movw	$(STATE_LIT_LONGREP << 8), %cx
+	/* Get is_rep0 bit */
+	leal	is_rep0(,%edx,2), %ebx
+	call	rc_bit
+	jnz	1f
+	/* Get is_rep0_long bit */
+	leal	is_rep0_long(,%edx,2), %ebx
+	call	rc_bit
+	jnz	98f
+	movw	$1, len(%ebp)
+	movb	$STATE_LIT_SHORTREP, %ch
+	jmp	99f
+1:	/* Get is_rep1 bit */
+	incb	%cl
+	leal	is_rep1(,%edx,2), %ebx
+	call	rc_bit
+	jz	98f
+	/* Get is_rep2 bit */
+	incb	%cl
+	leal	is_rep2(,%edx,2), %ebx
+	call	rc_bit
+	adcb	$0, %cl
+98:	/* Decode length */
+	movl	$rep_len_dec, %ebx
+	call	lzma_len
+99:	/* Update LZMA state */
+	cmpb	$STATE_LIT_MAX, %dl
+	movb	%ch, %dl
+	jbe	1f
+	movb	$STATE_NONLIT_REP, %dl
+1:	/* Tail call */
+	jmp	match_rep
+	.size	lzma_match, . - lzma_match
+
+/*****************************************************************************
+ * Decode one LZMA symbol
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer (updated)
+ *   %edx : LZMA state
+ *   CF : end of payload marker found
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ */
+lzma_decode:
+	/* Get is_match bit */
+	leal	is_match(,%edx,2), %ebx
+	call	rc_bit
+	jz	lzma_literal
+	/* Get is_rep bit */
+	leal	is_rep(,%edx,2), %ebx
+	call	rc_bit
+	jz	lzma_match
+	jmp	lzma_rep_match
+	.size	lzma_decode, . - lzma_decode
+
+/****************************************************************************
+ * Undo effect of branch-call-jump (BCJ) filter
+ *
+ * Parameters:
+ *   %es:%esi : start of uncompressed output data (note %es)
+ *   %es:%edi : end of uncompressed output data
+ * Returns:
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *   %edx
+ *   %esi
+ *****************************************************************************
+ */
+bcj_filter:
+	/* Store (negative) start of data in %edx */
+	movl	%esi, %edx
+	negl	%edx
+	/* Calculate limit in %ecx */
+	leal	-5(%edi,%edx), %ecx
+1:	/* Calculate offset in %ebx */
+	leal	(%esi,%edx), %ebx
+	/* Check for end of data */
+	cmpl	%ecx, %ebx
+	ja	99f
+	/* Check for an opcode which would be followed by a rel32 address */
+	ADDR32 es lodsb
+	andb	$0xfe, %al
+	cmpb	$0xe8, %al
+	jne	1b
+	/* Get current jump target value in %eax */
+	ADDR32 es lodsl
+	/* Convert absolute addresses in the range [0,limit) back to
+	 * relative addresses in the range [-offset,limit-offset).
+	 */
+	cmpl	%ecx, %eax
+	jae	2f
+	subl	%ebx,%es:-4(%esi)
+2:	/* Convert negative numbers in the range [-offset,0) back to
+	 * positive numbers in the range [limit-offset,limit).
+	 */
+	notl	%eax	/* Range is now [0,offset) */
+	cmpl	%ebx, %eax
+	jae	1b
+	addl	%ecx,%es:-4(%esi)
+	jmp	1b
+99:	/* Return */
+	ret
+	.size	bcj_filter, . - bcj_filter
+
+/****************************************************************************
+ * decompress (real-mode or 16/32-bit protected-mode near call)
+ *
+ * Decompress data
+ *
+ * Parameters (passed via registers):
+ *   %ds:%esi : Start of compressed input data
+ *   %es:%edi : Start of output buffer
+ * Returns:
+ *   %ds:%esi - End of compressed input data
+ *   %es:%edi - End of decompressed output data
+ *   All other registers are preserved
+ *
+ * NOTE: It would be possible to build a smaller version of the
+ * decompression code for -DKEEP_IT_REAL by using 16-bit registers
+ * where possible.
+ ****************************************************************************
+ */
+	.globl	decompress
+decompress:
+	/* Preserve registers */
+	pushl	%eax
+	pushl	%ebx
+	pushl	%ecx
+	pushl	%edx
+	pushl	%ebp
+	/* Allocate parameter block */
+	subl	$sizeof__lzma_dec, %esp
+	movl	%esp, %ebp
+	/* Zero parameter block and set all probabilities to 0.5 */
+	pushl	%edi
+	pushw	%es
+	pushw	%ss
+	popw	%es
+	movl	%ebp, %edi
+	xorl	%eax, %eax
+	movl	$( sizeof__lzma_dec / 4 ), %ecx
+	ADDR32 rep stosl
+	leal	probs(%ebp), %edi
+	movw	$( ( 1 << 11 ) / 2 ), %ax
+	movl	$( ( sizeof__lzma_dec - probs ) / 2 ), %ecx
+	ADDR32 rep stosw
+	popw	%es
+	popl	%edi
+	/* Initialise remaining parameters */
+	movl	%edi, out_start(%ebp)
+	print_character $('\n')
+	ADDR32 lodsb	/* discard initial byte */
+	print_hex_byte %al
+	ADDR32 lodsl
+	bswapl	%eax
+	print_hex_dword %eax
+	print_character $('\n')
+	movl	%eax, rc_code(%ebp)
+	decl	rc_range(%ebp)
+	movl	$STATE_LIT_LIT, %edx
+1:	/* Decompress until we reach end of buffer */
+	call	lzma_decode
+	jnc	1b
+	call	rc_normalise
+	print_character $('\n')
+	/* Undo BCJ filter */
+	pushl	%esi
+	movl	out_start(%ebp), %esi
+	call	bcj_filter
+	popl	%esi
+	/* Restore registers and return */
+	addl	$sizeof__lzma_dec, %esp
+	popl	%ebp
+	popl	%edx
+	popl	%ecx
+	popl	%ebx
+	popl	%eax
+	ret
+
+	/* Specify minimum amount of stack space required */
+	.globl	_min_decompress_stack
+	.equ	_min_decompress_stack, ( sizeof__lzma_dec + 512 /* margin */ )
diff --git a/roms/ipxe/src/arch/i386/prefix/unnrv2b16.S b/roms/ipxe/src/arch/i386/prefix/unlzma16.S
index b24c2846f..32b43f0dc 100644
--- a/roms/ipxe/src/arch/i386/prefix/unnrv2b16.S
+++ b/roms/ipxe/src/arch/i386/prefix/unlzma16.S
@@ -3,7 +3,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define CODE16
-#include "unnrv2b.S"
+#include "unlzma.S"
diff --git a/roms/ipxe/src/arch/i386/prefix/unnrv2b.S b/roms/ipxe/src/arch/i386/prefix/unnrv2b.S
deleted file mode 100644
index f5724c134..000000000
--- a/roms/ipxe/src/arch/i386/prefix/unnrv2b.S
+++ /dev/null
@@ -1,184 +0,0 @@
-/* 
- * Copyright (C) 1996-2002 Markus Franz Xaver Johannes Oberhumer
- *
- * This file is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * Originally this code was part of ucl the data compression library
- * for upx the ``Ultimate Packer of eXecutables''.
- *
- * - Converted to gas assembly, and refitted to work with etherboot.
- *   Eric Biederman 20 Aug 2002
- *
- * - Structure modified to be a subroutine call rather than an
- *   executable prefix.
- *   Michael Brown 30 Mar 2004
- *
- * - Modified to be compilable as either 16-bit or 32-bit code.
- *   Michael Brown 9 Mar 2005
- */
-
-FILE_LICENCE ( GPL2_OR_LATER )
-
-/****************************************************************************
- * This file provides the decompress() and decompress16() functions
- * which can be called in order to decompress an image compressed with
- * the nrv2b utility in src/util.
- *
- * These functions are designed to be called by the prefix.  They are
- * position-independent code.
- *
- * The same basic assembly code is used to compile both
- * decompress() and decompress16().
- ****************************************************************************
- */
-
-	.text
-	.arch i386
-	.section ".prefix.lib", "ax", @progbits
-
-#ifdef CODE16
-/****************************************************************************
- * decompress16 (real-mode near call, position independent)
- *
- * Decompress data in 16-bit mode
- *
- * Parameters (passed via registers):
- *   %ds:%esi - Start of compressed input data
- *   %es:%edi - Start of output buffer
- * Returns:
- *   %ds:%esi - End of compressed input data
- *   %es:%edi - End of decompressed output data
- *   All other registers are preserved
- *
- * NOTE: It would be possible to build a smaller version of the
- * decompression code for -DKEEP_IT_REAL by using
- *    #define REG(x) x
- * to use 16-bit registers where possible.  This would impose limits
- * that the compressed data size must be in the range [1,65533-%si]
- * and the uncompressed data size must be in the range [1,65536-%di]
- * (where %si and %di are the input values for those registers).  Note
- * particularly that the lower limit is 1, not 0, and that the upper
- * limit on the input (compressed) data really is 65533, since the
- * algorithm may read up to three bytes beyond the end of the input
- * data, since it reads dwords.
- ****************************************************************************
- */
-
-#define REG(x) e ## x
-#define ADDR32 addr32
-
-	.code16
-	.globl	decompress16
-decompress16:
-	
-#else /* CODE16 */
-
-/****************************************************************************
- * decompress (32-bit protected-mode near call, position independent)
- *
- * Parameters (passed via registers):
- *   %ds:%esi - Start of compressed input data
- *   %es:%edi - Start of output buffer
- * Returns:
- *   %ds:%esi - End of compressed input data
- *   %es:%edi - End of decompressed output data
- *   All other registers are preserved
- ****************************************************************************
- */
-
-#define REG(x) e ## x
-#define ADDR32
-	
-	.code32
-	.globl	decompress
-decompress:
-
-#endif /* CODE16 */
-
-#define xAX	REG(ax)
-#define xCX	REG(cx)
-#define xBP	REG(bp)
-#define xSI	REG(si)
-#define xDI	REG(di)
-
-	/* Save registers */
-	push	%xAX
-	pushl	%ebx
-	push	%xCX
-	push	%xBP
-	/* Do the decompression */
-	cld
-	xor	%xBP, %xBP
-	dec	%xBP		/* last_m_off = -1 */
-	jmp	dcl1_n2b
-	
-decompr_literals_n2b:
-	ADDR32 movsb
-decompr_loop_n2b:
-	addl	%ebx, %ebx
-	jnz	dcl2_n2b
-dcl1_n2b:
-	call	getbit32
-dcl2_n2b:
-	jc	decompr_literals_n2b
-	xor	%xAX, %xAX
-	inc	%xAX		/* m_off = 1 */
-loop1_n2b:
-	call	getbit1
-	adc	%xAX, %xAX	/* m_off = m_off*2 + getbit() */
-	call	getbit1
-	jnc	loop1_n2b	/* while(!getbit()) */
-	sub	$3, %xAX
-	jb	decompr_ebpeax_n2b	/* if (m_off == 2) goto decompr_ebpeax_n2b ? */
-	shl	$8, %xAX	
-	ADDR32 movb (%xSI), %al	/* m_off = (m_off - 3)*256 + src[ilen++] */
-	inc	%xSI
-	xor	$-1, %xAX
-	jz	decompr_end_n2b	/* if (m_off == 0xffffffff) goto decomp_end_n2b */
-	mov	%xAX, %xBP	/* last_m_off = m_off ?*/
-decompr_ebpeax_n2b:
-	xor	%xCX, %xCX
-	call	getbit1
-	adc	%xCX, %xCX	/* m_len = getbit() */
-	call	getbit1
-	adc	%xCX, %xCX	/* m_len = m_len*2 + getbit()) */
-	jnz	decompr_got_mlen_n2b	/* if (m_len == 0) goto decompr_got_mlen_n2b */
-	inc	%xCX		/* m_len++ */
-loop2_n2b:
-	call	getbit1	
-	adc	%xCX, %xCX	/* m_len = m_len*2 + getbit() */
-	call	getbit1
-	jnc	loop2_n2b	/* while(!getbit()) */
-	inc	%xCX
-	inc	%xCX		/* m_len += 2 */
-decompr_got_mlen_n2b:
-	cmp	$-0xd00, %xBP
-	adc	$1, %xCX	/* m_len = m_len + 1 + (last_m_off > 0xd00) */
-	push	%xSI
-	ADDR32 lea (%xBP,%xDI), %xSI	/* m_pos = dst + olen + -m_off  */
-	rep
-	es ADDR32 movsb		/* dst[olen++] = *m_pos++ while(m_len > 0) */
-	pop	%xSI
-	jmp	decompr_loop_n2b
-
-
-getbit1:
-	addl	%ebx, %ebx
-	jnz	1f
-getbit32:
-	ADDR32 movl (%xSI), %ebx
-	sub	$-4, %xSI	/* sets carry flag */
-	adcl	%ebx, %ebx
-1:
-	ret
-
-decompr_end_n2b:
-	/* Restore registers and return */
-	pop	%xBP
-	pop	%xCX
-	popl	%ebx
-	pop	%xAX
-	ret
diff --git a/roms/ipxe/src/arch/i386/prefix/usbdisk.S b/roms/ipxe/src/arch/i386/prefix/usbdisk.S
index fa7d1956e..9676406e2 100644
--- a/roms/ipxe/src/arch/i386/prefix/usbdisk.S
+++ b/roms/ipxe/src/arch/i386/prefix/usbdisk.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
@@ -6,18 +8,27 @@
 
 #include "mbr.S"
 
-/* Partition table: ZIP-compatible partition 4, 64 heads, 32 sectors/track */
+/* Partition table: 64 heads, 32 sectors/track (ZIP-drive compatible) */
 	.org 446
 	.space 16
 	.space 16
-	.space 16
-	.byte 0x80, 0x01, 0x01, 0x00
-	.byte 0xeb, 0x3f, 0x20, 0x01
+	/* Partition 3: log partition (for CONSOLE_INT13) */
+	.byte 0x00, 0x01, 0x01, 0x00
+	.byte 0xe0, 0x3f, 0x20, 0x00
 	.long 0x00000020
-	.long 0x00000fe0
+	.long 0x000007e0
+	/* Partition 4: boot partition */
+	.byte 0x80, 0x00, 0x01, 0x01
+	.byte 0xeb, 0x3f, 0x20, 0x02
+	.long 0x00000800
+	.long 0x00001000
 
 	.org 510
 	.byte 0x55, 0xaa
 
-/* Skip to start of partition */
+/* Skip to start of log partition */
 	.org 32 * 512
+	.ascii "iPXE LOG\n\n"
+
+/* Skip to start of boot partition */
+	.org 2048 * 512
diff --git a/roms/ipxe/src/arch/i386/scripts/i386.lds b/roms/ipxe/src/arch/i386/scripts/i386.lds
index 98f95cb23..38c89e14b 100644
--- a/roms/ipxe/src/arch/i386/scripts/i386.lds
+++ b/roms/ipxe/src/arch/i386/scripts/i386.lds
@@ -27,6 +27,13 @@ SECTIONS {
     PROVIDE ( _max_align = 16 );
 
     /*
+     * Allow decompressor to require a minimum amount of temporary stack
+     * space.
+     *
+     */
+    PROVIDE ( _min_decompress_stack = 0 );
+
+    /*
      * The prefix
      *
      */
@@ -34,6 +41,7 @@ SECTIONS {
     .prefix 0x0 : AT ( _prefix_lma ) {
 	_prefix = .;
 	*(.prefix)
+	*(SORT(.pci_devlist.*))
 	*(.prefix.*)
 	_mprefix = .;
     } .bss.prefix (NOLOAD) : AT ( _end_lma ) {
@@ -87,6 +95,7 @@ SECTIONS {
 	*(.bss16.*)
 	*(.stack16)
 	*(.stack16.*)
+	. = MAX ( ., _mdata16 + _min_decompress_stack );
 	_edata16 = .;
     }
     _data16_filesz	= ABSOLUTE ( _mdata16 ) - ABSOLUTE ( _data16 );
diff --git a/roms/ipxe/src/arch/i386/transitions/liba20.S b/roms/ipxe/src/arch/i386/transitions/liba20.S
index 684697525..6c1e1f62f 100644
--- a/roms/ipxe/src/arch/i386/transitions/liba20.S
+++ b/roms/ipxe/src/arch/i386/transitions/liba20.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/roms/ipxe/src/arch/i386/transitions/libkir.S b/roms/ipxe/src/arch/i386/transitions/libkir.S
index 1176fcced..fa9459d52 100644
--- a/roms/ipxe/src/arch/i386/transitions/libkir.S
+++ b/roms/ipxe/src/arch/i386/transitions/libkir.S
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /****************************************************************************
  * This file defines libkir: an interface between external and
diff --git a/roms/ipxe/src/arch/i386/transitions/librm.S b/roms/ipxe/src/arch/i386/transitions/librm.S
index 2e447b030..863e22415 100644
--- a/roms/ipxe/src/arch/i386/transitions/librm.S
+++ b/roms/ipxe/src/arch/i386/transitions/librm.S
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /* Drag in local definitions */
 #include "librm.h"
diff --git a/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c b/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c
index cc4765de2..becb02677 100644
--- a/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c
+++ b/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/profile.h>
@@ -103,7 +103,7 @@ void init_idt ( void ) {
 				( uint32_t ) vec->next );
 		set_interrupt_vector ( intr, vec );
 	}
-	DBGC ( &intr_vec[0], "INTn vector at %p+%xn (phys %#lx+%xn)\n",
+	DBGC ( &intr_vec[0], "INTn vector at %p+%zxn (phys %#lx+%zxn)\n",
 	       intr_vec, sizeof ( intr_vec[0] ),
 	       virt_to_phys ( intr_vec ), sizeof ( intr_vec[0] ) );
 
diff --git a/roms/ipxe/src/arch/i386/transitions/librm_test.c b/roms/ipxe/src/arch/i386/transitions/librm_test.c
index e07cfccdd..f1a517eda 100644
--- a/roms/ipxe/src/arch/i386/transitions/librm_test.c
+++ b/roms/ipxe/src/arch/i386/transitions/librm_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -114,4 +118,5 @@ struct self_test librm_test __self_test = {
 	.exec = librm_test_exec,
 };
 
+REQUIRING_SYMBOL ( librm_test );
 REQUIRE_OBJECT ( test );
diff --git a/roms/ipxe/src/arch/x86/Makefile b/roms/ipxe/src/arch/x86/Makefile
index e555587df..98c49b98d 100644
--- a/roms/ipxe/src/arch/x86/Makefile
+++ b/roms/ipxe/src/arch/x86/Makefile
@@ -9,9 +9,14 @@ SRCDIRS		+= arch/x86/interface/efi
 SRCDIRS		+= arch/x86/prefix
 SRCDIRS		+= arch/x86/hci/commands
 SRCDIRS		+= arch/x86/drivers/xen
+SRCDIRS		+= arch/x86/drivers/hyperv
 
 # breaks building some of the linux-related objects
 CFLAGS		+= -Ulinux
 
 # disable valgrind
 CFLAGS		+= -DNVALGRIND
+
+# Include Hyper-V driver in the all-drivers build
+#
+DRIVERS_hyperv += hyperv
diff --git a/roms/ipxe/src/arch/x86/Makefile.efi b/roms/ipxe/src/arch/x86/Makefile.efi
index 13a69d9f7..f73bc7d5d 100644
--- a/roms/ipxe/src/arch/x86/Makefile.efi
+++ b/roms/ipxe/src/arch/x86/Makefile.efi
@@ -17,7 +17,7 @@ NON_AUTO_MEDIA	+= efirom
 
 # Include SNP driver in the all-drivers build
 #
-DRIVERS += snp
+DRIVERS_net += snp
 
 # Rules for building EFI files
 #
diff --git a/roms/ipxe/src/arch/x86/core/cpuid.c b/roms/ipxe/src/arch/x86/core/cpuid.c
index 5908f4419..bc5a6c68c 100644
--- a/roms/ipxe/src/arch/x86/core/cpuid.c
+++ b/roms/ipxe/src/arch/x86/core/cpuid.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/cpuid.h>
diff --git a/roms/ipxe/src/arch/x86/core/cpuid_settings.c b/roms/ipxe/src/arch/x86/core/cpuid_settings.c
index 42dea9336..08bd3918a 100644
--- a/roms/ipxe/src/arch/x86/core/cpuid_settings.c
+++ b/roms/ipxe/src/arch/x86/core/cpuid_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/arch/x86/core/debugcon.c b/roms/ipxe/src/arch/x86/core/debugcon.c
index 263cb4af1..60de61f55 100644
--- a/roms/ipxe/src/arch/x86/core/debugcon.c
+++ b/roms/ipxe/src/arch/x86/core/debugcon.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/x86/core/pcidirect.c b/roms/ipxe/src/arch/x86/core/pcidirect.c
index dbc8317b8..9b8e6b1d9 100644
--- a/roms/ipxe/src/arch/x86/core/pcidirect.c
+++ b/roms/ipxe/src/arch/x86/core/pcidirect.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/io.h>
 #include <ipxe/pci.h>
diff --git a/roms/ipxe/src/arch/i386/core/pic8259.c b/roms/ipxe/src/arch/x86/core/pic8259.c
index 0a9ea2e03..0a9ea2e03 100644
--- a/roms/ipxe/src/arch/i386/core/pic8259.c
+++ b/roms/ipxe/src/arch/x86/core/pic8259.c
diff --git a/roms/ipxe/src/arch/x86/core/pit8254.c b/roms/ipxe/src/arch/x86/core/pit8254.c
new file mode 100644
index 000000000..da2099263
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/core/pit8254.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/io.h>
+#include <ipxe/pit8254.h>
+
+/** @file
+ *
+ * 8254 Programmable Interval Timer
+ *
+ */
+
+/**
+ * Delay for a fixed number of timer ticks using the speaker channel
+ *
+ * @v ticks		Number of timer ticks for which to delay
+ */
+void pit8254_speaker_delay ( unsigned int ticks ) {
+	uint8_t spkr;
+	uint8_t cmd;
+	uint8_t low;
+	uint8_t high;
+
+	/* Sanity check */
+	assert ( ticks <= 0xffff );
+
+	/* Disable speaker, set speaker channel gate input high */
+	spkr = inb ( PIT8254_SPKR );
+	spkr &= ~PIT8254_SPKR_ENABLE;
+	spkr |= PIT8254_SPKR_GATE;
+	outb ( spkr, PIT8254_SPKR );
+
+	/* Program speaker channel to "interrupt" on terminal count */
+	cmd = ( PIT8254_CMD_CHANNEL ( PIT8254_CH_SPKR ) |
+		PIT8254_CMD_ACCESS_LOHI | PIT8254_CMD_OP_TERMINAL |
+		PIT8254_CMD_BINARY );
+	low = ( ( ticks >> 0 ) & 0xff );
+	high = ( ( ticks >> 8 ) & 0xff );
+	outb ( cmd, PIT8254_CMD );
+	outb ( low, PIT8254_DATA ( PIT8254_CH_SPKR ) );
+	outb ( high, PIT8254_DATA ( PIT8254_CH_SPKR ) );
+
+	/* Wait for channel to "interrupt" */
+	do {
+		spkr = inb ( PIT8254_SPKR );
+	} while ( ! ( spkr & PIT8254_SPKR_OUT ) );
+}
diff --git a/roms/ipxe/src/arch/x86/core/vram_settings.c b/roms/ipxe/src/arch/x86/core/vram_settings.c
new file mode 100644
index 000000000..9c169b40c
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/core/vram_settings.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/uaccess.h>
+#include <ipxe/settings.h>
+
+/** @file
+ *
+ * Video RAM dump
+ *
+ */
+
+/** Video RAM base address */
+#define VRAM_BASE 0xb8000
+
+/** Video RAM length */
+#define VRAM_LEN \
+	( 80 /* columns */ * 25 /* rows */ * 2 /* bytes per character */ )
+
+/**
+ * Fetch video RAM setting
+ *
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int vram_fetch ( void *data, size_t len ) {
+	userptr_t vram = phys_to_user ( VRAM_BASE );
+
+	/* Copy video RAM */
+	if ( len > VRAM_LEN )
+		len = VRAM_LEN;
+	copy_from_user ( data, vram, 0, len );
+
+	return VRAM_LEN;
+}
+
+/** Video RAM setting */
+const struct setting vram_setting __setting ( SETTING_MISC, vram ) = {
+	.name = "vram",
+	.description = "Video RAM",
+	.type = &setting_type_base64,
+	.scope = &builtin_scope,
+};
+
+/** Video RAM built-in setting */
+struct builtin_setting vram_builtin_setting __builtin_setting = {
+	.setting = &vram_setting,
+	.fetch = vram_fetch,
+};
diff --git a/roms/ipxe/src/arch/x86/core/x86_bigint.c b/roms/ipxe/src/arch/x86/core/x86_bigint.c
index 418ac2309..6413b2fa8 100644
--- a/roms/ipxe/src/arch/x86/core/x86_bigint.c
+++ b/roms/ipxe/src/arch/x86/core/x86_bigint.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/arch/x86/core/x86_io.c b/roms/ipxe/src/arch/x86/core/x86_io.c
index 9b2d2d935..3081fa8b9 100644
--- a/roms/ipxe/src/arch/x86/core/x86_io.c
+++ b/roms/ipxe/src/arch/x86/core/x86_io.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/io.h>
 #include <ipxe/x86_io.h>
diff --git a/roms/ipxe/src/arch/x86/core/x86_string.c b/roms/ipxe/src/arch/x86/core/x86_string.c
index d48347c96..7d5e4a5f1 100644
--- a/roms/ipxe/src/arch/x86/core/x86_string.c
+++ b/roms/ipxe/src/arch/x86/core/x86_string.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 /** @file
@@ -23,7 +27,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 
@@ -104,87 +108,3 @@ void * __memmove ( void *dest, const void *src, size_t len ) {
 		return __memcpy_reverse ( dest, src, len );
 	}
 }
-
-/**
- * Swap memory areas
- *
- * @v dest		Destination address
- * @v src		Source address
- * @v len		Length
- * @ret dest		Destination address
- */
-void * memswap ( void *dest, void *src, size_t len ) {
-	size_t discard_c;
-	int discard;
-
-	__asm__ __volatile__ ( "\n1:\n\t"
-			       "dec %2\n\t"
-			       "js 2f\n\t"
-			       "movb (%0,%2), %b3\n\t"
-			       "xchgb (%1,%2), %b3\n\t"
-			       "movb %b3, (%0,%2)\n\t"
-			       "jmp 1b\n\t"
-			       "2:\n\t"
-			       : "=r" ( src ), "=r" ( dest ),
-				 "=&c" ( discard_c ), "=&q" ( discard )
-			       : "0" ( src ), "1" ( dest ), "2" ( len )
-			       : "memory" );
-
-	return dest;
-}
-
-/**
- * Calculate length of string
- *
- * @v string		String
- * @ret len		Length (excluding NUL)
- */
-size_t strlen ( const char *string ) {
-	const char *discard_D;
-	size_t len_plus_one;
-
-	__asm__ __volatile__ ( "repne scasb\n\t"
-			       "not %1\n\t"
-			       : "=&D" ( discard_D ), "=&c" ( len_plus_one )
-			       : "0" ( string ), "1" ( -1UL ), "a" ( 0 ) );
-
-	return ( len_plus_one - 1 );
-}
-
-/**
- * Compare strings (up to a specified length)
- *
- * @v str1		First string
- * @v str2		Second string
- * @v len		Maximum length
- * @ret diff		Difference
- */
-int strncmp ( const char *str1, const char *str2, size_t len ) {
-	const void *discard_S;
-	const void *discard_D;
-	size_t discard_c;
-	int diff;
-
-	__asm__ __volatile__ ( "\n1:\n\t"
-			       "dec %2\n\t"
-			       "js 2f\n\t"
-			       "lodsb\n\t"
-			       "scasb\n\t"
-			       "jne 3f\n\t"
-			       "testb %b3, %b3\n\t"
-			       "jnz 1b\n\t"
-			       /* Equal */
-			       "\n2:\n\t"
-			       "xor %3, %3\n\t"
-			       "jmp 4f\n\t"
-			       /* Not equal; CF indicates difference */
-			       "\n3:\n\t"
-			       "sbb %3, %3\n\t"
-			       "orb $1, %b3\n\t"
-			       "\n4:\n\t"
-			       : "=&S" ( discard_S ), "=&D" ( discard_D ),
-				 "=&c" ( discard_c ), "=&a" ( diff )
-			       : "0" ( str1 ), "1" ( str2 ), "2" ( len ) );
-
-	return diff;
-}
diff --git a/roms/ipxe/src/arch/x86/core/x86_tcpip.c b/roms/ipxe/src/arch/x86/core/x86_tcpip.c
index 8a4ce5152..88042f5f7 100644
--- a/roms/ipxe/src/arch/x86/core/x86_tcpip.c
+++ b/roms/ipxe/src/arch/x86/core/x86_tcpip.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/arch/x86/core/x86_uart.c b/roms/ipxe/src/arch/x86/core/x86_uart.c
new file mode 100644
index 000000000..e455775bf
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/core/x86_uart.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+#include <errno.h>
+#include <ipxe/uart.h>
+
+/** UART port bases */
+static uint16_t uart_base[] = {
+	[COM1] = 0x3f8,
+	[COM2] = 0x2f8,
+	[COM3] = 0x3e8,
+	[COM4] = 0x2e8,
+};
+
+/**
+ * Select UART port
+ *
+ * @v uart		UART
+ * @v port		Port number, or 0 to disable
+ * @ret rc		Return status code
+ */
+int uart_select ( struct uart *uart, unsigned int port ) {
+	int rc;
+
+	/* Set new UART base */
+	if ( port >= ( sizeof ( uart_base ) / sizeof ( uart_base[0] ) ) ) {
+		rc = -ENODEV;
+		goto err;
+	}
+	uart->base = ( ( void * ) ( intptr_t ) uart_base[port] );
+
+	/* Check that UART exists */
+	if ( ( rc = uart_exists ( uart ) ) != 0 )
+		goto err;
+
+	return 0;
+
+ err:
+	uart->base = NULL;
+	return rc;
+}
diff --git a/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.c b/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.c
new file mode 100644
index 000000000..f73829bd5
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.c
@@ -0,0 +1,597 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Hyper-V driver
+ *
+ */
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <pic8259.h>
+#include <ipxe/malloc.h>
+#include <ipxe/device.h>
+#include <ipxe/cpuid.h>
+#include <ipxe/msr.h>
+#include <ipxe/hyperv.h>
+#include <ipxe/vmbus.h>
+#include "hyperv.h"
+
+/** Maximum time to wait for a message response
+ *
+ * This is a policy decision.
+ */
+#define HV_MESSAGE_MAX_WAIT_MS 1000
+
+/**
+ * Convert a Hyper-V status code to an iPXE status code
+ *
+ * @v status		Hyper-V status code
+ * @ret rc		iPXE status code (before negation)
+ */
+#define EHV( status ) EPLATFORM ( EINFO_EPLATFORM, (status) )
+
+/**
+ * Allocate zeroed pages
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v ...		Page addresses to fill in, terminated by NULL
+ * @ret rc		Return status code
+ */
+__attribute__ (( sentinel )) int
+hv_alloc_pages ( struct hv_hypervisor *hv, ... ) {
+	va_list args;
+	void **page;
+	int i;
+
+	/* Allocate and zero pages */
+	va_start ( args, hv );
+	for ( i = 0 ; ( ( page = va_arg ( args, void ** ) ) != NULL ); i++ ) {
+		*page = malloc_dma ( PAGE_SIZE, PAGE_SIZE );
+		if ( ! *page )
+			goto err_alloc;
+		memset ( *page, 0, PAGE_SIZE );
+	}
+	va_end ( args );
+
+	return 0;
+
+ err_alloc:
+	va_end ( args );
+	va_start ( args, hv );
+	for ( ; i >= 0 ; i-- ) {
+		page = va_arg ( args, void ** );
+		free_dma ( *page, PAGE_SIZE );
+	}
+	va_end ( args );
+	return -ENOMEM;
+}
+
+/**
+ * Free pages
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v ...		Page addresses, terminated by NULL
+ */
+__attribute__ (( sentinel )) void
+hv_free_pages ( struct hv_hypervisor *hv, ... ) {
+	va_list args;
+	void *page;
+
+	va_start ( args, hv );
+	while ( ( page = va_arg ( args, void * ) ) != NULL )
+		free_dma ( page, PAGE_SIZE );
+	va_end ( args );
+}
+
+/**
+ * Allocate message buffer
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_alloc_message ( struct hv_hypervisor *hv ) {
+
+	/* Allocate buffer.  Must be aligned to at least 8 bytes and
+	 * must not cross a page boundary, so align on its own size.
+	 */
+	hv->message = malloc_dma ( sizeof ( *hv->message ),
+				   sizeof ( *hv->message ) );
+	if ( ! hv->message )
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * Free message buffer
+ *
+ * @v hv		Hyper-V hypervisor
+ */
+static void hv_free_message ( struct hv_hypervisor *hv ) {
+
+	/* Free buffer */
+	free_dma ( hv->message, sizeof ( *hv->message ) );
+}
+
+/**
+ * Check whether or not we are running in Hyper-V
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_check_hv ( struct hv_hypervisor *hv ) {
+	struct x86_features features;
+	uint32_t interface_id;
+	uint32_t discard_ebx;
+	uint32_t discard_ecx;
+	uint32_t discard_edx;
+	uint32_t available;
+	uint32_t permissions;
+
+	/* Check for presence of a hypervisor (not necessarily Hyper-V) */
+	x86_features ( &features );
+	if ( ! ( features.intel.ecx & CPUID_FEATURES_INTEL_ECX_HYPERVISOR ) ) {
+		DBGC ( hv, "HV %p not running in a hypervisor\n", hv );
+		return -ENODEV;
+	}
+
+	/* Check that hypervisor is Hyper-V */
+	cpuid ( HV_CPUID_INTERFACE_ID, &interface_id, &discard_ebx,
+		&discard_ecx, &discard_edx );
+	if ( interface_id != HV_INTERFACE_ID ) {
+		DBGC ( hv, "HV %p not running in Hyper-V (interface ID "
+		       "%#08x)\n", hv, interface_id );
+		return -ENODEV;
+	}
+
+	/* Check that required features and privileges are available */
+	cpuid ( HV_CPUID_FEATURES, &available, &permissions, &discard_ecx,
+		&discard_edx );
+	if ( ! ( available & HV_FEATURES_AVAIL_HYPERCALL_MSR ) ) {
+		DBGC ( hv, "HV %p has no hypercall MSRs (features %08x:%08x)\n",
+		       hv, available, permissions );
+		return -ENODEV;
+	}
+	if ( ! ( available & HV_FEATURES_AVAIL_SYNIC_MSR ) ) {
+		DBGC ( hv, "HV %p has no SynIC MSRs (features %08x:%08x)\n",
+		       hv, available, permissions );
+		return -ENODEV;
+	}
+	if ( ! ( permissions & HV_FEATURES_PERM_POST_MESSAGES ) ) {
+		DBGC ( hv, "HV %p cannot post messages (features %08x:%08x)\n",
+		       hv, available, permissions );
+		return -EACCES;
+	}
+	if ( ! ( permissions & HV_FEATURES_PERM_SIGNAL_EVENTS ) ) {
+		DBGC ( hv, "HV %p cannot signal events (features %08x:%08x)",
+		       hv, available, permissions );
+		return -EACCES;
+	}
+
+	return 0;
+}
+
+/**
+ * Map hypercall page
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_map_hypercall ( struct hv_hypervisor *hv ) {
+	union {
+		struct {
+			uint32_t ebx;
+			uint32_t ecx;
+			uint32_t edx;
+		} __attribute__ (( packed ));
+		char text[ 13 /* "bbbbccccdddd" + NUL */ ];
+	} vendor_id;
+	uint32_t build;
+	uint32_t version;
+	uint32_t discard_eax;
+	uint32_t discard_ecx;
+	uint32_t discard_edx;
+	uint64_t guest_os_id;
+	uint64_t hypercall;
+
+	/* Report guest OS identity */
+	guest_os_id = rdmsr ( HV_X64_MSR_GUEST_OS_ID );
+	if ( guest_os_id != 0 ) {
+		DBGC ( hv, "HV %p guest OS ID MSR already set to %#08llx\n",
+		       hv, guest_os_id );
+		return -EBUSY;
+	}
+	guest_os_id = HV_GUEST_OS_ID_IPXE;
+	DBGC2 ( hv, "HV %p guest OS ID MSR is %#08llx\n", hv, guest_os_id );
+	wrmsr ( HV_X64_MSR_GUEST_OS_ID, guest_os_id );
+
+	/* Get hypervisor system identity (for debugging) */
+	cpuid ( HV_CPUID_VENDOR_ID, &discard_eax, &vendor_id.ebx,
+		&vendor_id.ecx, &vendor_id.edx );
+	vendor_id.text[ sizeof ( vendor_id.text ) - 1 ] = '\0';
+	cpuid ( HV_CPUID_HYPERVISOR_ID, &build, &version, &discard_ecx,
+		&discard_edx );
+	DBGC ( hv, "HV %p detected \"%s\" version %d.%d build %d\n", hv,
+	       vendor_id.text, ( version >> 16 ), ( version & 0xffff ), build );
+
+	/* Map hypercall page */
+	hypercall = rdmsr ( HV_X64_MSR_HYPERCALL );
+	hypercall &= ( PAGE_SIZE - 1 );
+	hypercall |= ( virt_to_phys ( hv->hypercall ) | HV_HYPERCALL_ENABLE );
+	DBGC2 ( hv, "HV %p hypercall MSR is %#08llx\n", hv, hypercall );
+	wrmsr ( HV_X64_MSR_HYPERCALL, hypercall );
+
+	return 0;
+}
+
+/**
+ * Unmap hypercall page
+ *
+ * @v hv		Hyper-V hypervisor
+ */
+static void hv_unmap_hypercall ( struct hv_hypervisor *hv ) {
+	uint64_t hypercall;
+	uint64_t guest_os_id;
+
+	/* Unmap the hypercall page */
+	hypercall = rdmsr ( HV_X64_MSR_HYPERCALL );
+	hypercall &= ( ( PAGE_SIZE - 1 ) & ~HV_HYPERCALL_ENABLE );
+	DBGC2 ( hv, "HV %p hypercall MSR is %#08llx\n", hv, hypercall );
+	wrmsr ( HV_X64_MSR_HYPERCALL, hypercall );
+
+	/* Reset the guest OS identity */
+	guest_os_id = 0;
+	DBGC2 ( hv, "HV %p guest OS ID MSR is %#08llx\n", hv, guest_os_id );
+	wrmsr ( HV_X64_MSR_GUEST_OS_ID, guest_os_id );
+}
+
+/**
+ * Map synthetic interrupt controller
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_map_synic ( struct hv_hypervisor *hv ) {
+	uint64_t simp;
+	uint64_t siefp;
+	uint64_t scontrol;
+
+	/* Map SynIC message page */
+	simp = rdmsr ( HV_X64_MSR_SIMP );
+	simp &= ( PAGE_SIZE - 1 );
+	simp |= ( virt_to_phys ( hv->synic.message ) | HV_SIMP_ENABLE );
+	DBGC2 ( hv, "HV %p SIMP MSR is %#08llx\n", hv, simp );
+	wrmsr ( HV_X64_MSR_SIMP, simp );
+
+	/* Map SynIC event page */
+	siefp = rdmsr ( HV_X64_MSR_SIEFP );
+	siefp &= ( PAGE_SIZE - 1 );
+	siefp |= ( virt_to_phys ( hv->synic.event ) | HV_SIEFP_ENABLE );
+	DBGC2 ( hv, "HV %p SIEFP MSR is %#08llx\n", hv, siefp );
+	wrmsr ( HV_X64_MSR_SIEFP, siefp );
+
+	/* Enable SynIC */
+	scontrol = rdmsr ( HV_X64_MSR_SCONTROL );
+	scontrol |= HV_SCONTROL_ENABLE;
+	DBGC2 ( hv, "HV %p SCONTROL MSR is %#08llx\n", hv, scontrol );
+	wrmsr ( HV_X64_MSR_SCONTROL, scontrol );
+
+	return 0;
+}
+
+/**
+ * Unmap synthetic interrupt controller
+ *
+ * @v hv		Hyper-V hypervisor
+ */
+static void hv_unmap_synic ( struct hv_hypervisor *hv ) {
+	uint64_t scontrol;
+	uint64_t siefp;
+	uint64_t simp;
+
+	/* Disable SynIC */
+	scontrol = rdmsr ( HV_X64_MSR_SCONTROL );
+	scontrol &= ~HV_SCONTROL_ENABLE;
+	DBGC2 ( hv, "HV %p SCONTROL MSR is %#08llx\n", hv, scontrol );
+	wrmsr ( HV_X64_MSR_SCONTROL, scontrol );
+
+	/* Unmap SynIC event page */
+	siefp = rdmsr ( HV_X64_MSR_SIEFP );
+	siefp &= ( ( PAGE_SIZE - 1 ) & ~HV_SIEFP_ENABLE );
+	DBGC2 ( hv, "HV %p SIEFP MSR is %#08llx\n", hv, siefp );
+	wrmsr ( HV_X64_MSR_SIEFP, siefp );
+
+	/* Unmap SynIC message page */
+	simp = rdmsr ( HV_X64_MSR_SIMP );
+	simp &= ( ( PAGE_SIZE - 1 ) & ~HV_SIMP_ENABLE );
+	DBGC2 ( hv, "HV %p SIMP MSR is %#08llx\n", hv, simp );
+	wrmsr ( HV_X64_MSR_SIMP, simp );
+}
+
+/**
+ * Enable synthetic interrupt
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v sintx		Synthetic interrupt number
+ */
+void hv_enable_sint ( struct hv_hypervisor *hv, unsigned int sintx ) {
+	unsigned long msr = HV_X64_MSR_SINT ( sintx );
+	uint64_t sint;
+
+	/* Enable synthetic interrupt
+	 *
+	 * We have to enable the interrupt, otherwise messages will
+	 * not be delivered (even though the documentation implies
+	 * that polling for messages is possible).  We enable AutoEOI
+	 * and hook the interrupt to the obsolete IRQ13 (FPU
+	 * exception) vector, which will be implemented as a no-op.
+	 */
+	sint = rdmsr ( msr );
+	sint &= ~( HV_SINT_MASKED | HV_SINT_VECTOR_MASK );
+	sint |= ( HV_SINT_AUTO_EOI |
+		  HV_SINT_VECTOR ( IRQ_INT ( 13 /* See comment above */ ) ) );
+	DBGC2 ( hv, "HV %p SINT%d MSR is %#08llx\n", hv, sintx, sint );
+	wrmsr ( msr, sint );
+}
+
+/**
+ * Disable synthetic interrupt
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v sintx		Synthetic interrupt number
+ */
+void hv_disable_sint ( struct hv_hypervisor *hv, unsigned int sintx ) {
+	unsigned long msr = HV_X64_MSR_SINT ( sintx );
+	uint64_t sint;
+
+	/* Disable synthetic interrupt */
+	sint = rdmsr ( msr );
+	sint &= ~HV_SINT_AUTO_EOI;
+	sint |= HV_SINT_MASKED;
+	DBGC2 ( hv, "HV %p SINT%d MSR is %#08llx\n", hv, sintx, sint );
+	wrmsr ( msr, sint );
+}
+
+/**
+ * Post message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v id		Connection ID
+ * @v type		Message type
+ * @v data		Message
+ * @v len		Length of message
+ * @ret rc		Return status code
+ */
+int hv_post_message ( struct hv_hypervisor *hv, unsigned int id,
+		      unsigned int type, const void *data, size_t len ) {
+	struct hv_post_message *msg = &hv->message->posted;
+	int status;
+	int rc;
+
+	/* Sanity check */
+	assert ( len <= sizeof ( msg->data ) );
+
+	/* Construct message */
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->id = cpu_to_le32 ( id );
+	msg->type = cpu_to_le32 ( type );
+	msg->len = cpu_to_le32 ( len );
+	memcpy ( msg->data, data, len );
+	DBGC2 ( hv, "HV %p connection %d posting message type %#08x:\n",
+		hv, id, type );
+	DBGC2_HDA ( hv, 0, msg->data, len );
+
+	/* Post message */
+	if ( ( status = hv_call ( hv, HV_POST_MESSAGE, msg, NULL ) ) != 0 ) {
+		rc = -EHV ( status );
+		DBGC ( hv, "HV %p could not post message to %#08x: %s\n",
+		       hv, id, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Wait for received message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v sintx		Synthetic interrupt number
+ * @ret rc		Return status code
+ */
+int hv_wait_for_message ( struct hv_hypervisor *hv, unsigned int sintx ) {
+	struct hv_message *msg = &hv->message->received;
+	struct hv_message *src = &hv->synic.message[sintx];
+	unsigned int retries;
+	size_t len;
+
+	/* Wait for message to arrive */
+	for ( retries = 0 ; retries < HV_MESSAGE_MAX_WAIT_MS ; retries++ ) {
+
+		/* Check for message */
+		if ( src->type ) {
+
+			/* Copy message */
+			memset ( msg, 0, sizeof ( *msg ) );
+			len = src->len;
+			assert ( len <= sizeof ( *msg ) );
+			memcpy ( msg, src,
+				 ( offsetof ( typeof ( *msg ), data ) + len ) );
+			DBGC2 ( hv, "HV %p SINT%d received message type "
+				"%#08x:\n", hv, sintx,
+				le32_to_cpu ( msg->type ) );
+			DBGC2_HDA ( hv, 0, msg->data, len );
+
+			/* Consume message */
+			src->type = 0;
+
+			return 0;
+		}
+
+		/* Trigger message delivery */
+		wrmsr ( HV_X64_MSR_EOM, 0 );
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( hv, "HV %p SINT%d timed out waiting for message\n",
+	       hv, sintx );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Signal event
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v id		Connection ID
+ * @v flag		Flag number
+ * @ret rc		Return status code
+ */
+int hv_signal_event ( struct hv_hypervisor *hv, unsigned int id,
+		      unsigned int flag ) {
+	struct hv_signal_event *event = &hv->message->signalled;
+	int status;
+	int rc;
+
+	/* Construct event */
+	memset ( event, 0, sizeof ( *event ) );
+	event->id = cpu_to_le32 ( id );
+	event->flag = cpu_to_le16 ( flag );
+
+	/* Signal event */
+	if ( ( status = hv_call ( hv, HV_SIGNAL_EVENT, event, NULL ) ) != 0 ) {
+		rc = -EHV ( status );
+		DBGC ( hv, "HV %p could not signal event to %#08x: %s\n",
+		       hv, id, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Probe root device
+ *
+ * @v rootdev		Root device
+ * @ret rc		Return status code
+ */
+static int hv_probe ( struct root_device *rootdev ) {
+	struct hv_hypervisor *hv;
+	int rc;
+
+	/* Allocate and initialise structure */
+	hv = zalloc ( sizeof ( *hv ) );
+	if ( ! hv ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Check we are running in Hyper-V */
+	if ( ( rc = hv_check_hv ( hv ) ) != 0 )
+		goto err_check_hv;
+
+	/* Allocate pages */
+	if ( ( rc = hv_alloc_pages ( hv, &hv->hypercall, &hv->synic.message,
+				     &hv->synic.event, NULL ) ) != 0 )
+		goto err_alloc_pages;
+
+	/* Allocate message buffer */
+	if ( ( rc = hv_alloc_message ( hv ) ) != 0 )
+		goto err_alloc_message;
+
+	/* Map hypercall page */
+	if ( ( rc = hv_map_hypercall ( hv ) ) != 0 )
+		goto err_map_hypercall;
+
+	/* Map synthetic interrupt controller */
+	if ( ( rc = hv_map_synic ( hv ) ) != 0 )
+		goto err_map_synic;
+
+	/* Probe Hyper-V devices */
+	if ( ( rc = vmbus_probe ( hv, &rootdev->dev ) ) != 0 )
+		goto err_vmbus_probe;
+
+	rootdev_set_drvdata ( rootdev, hv );
+	return 0;
+
+	vmbus_remove ( hv, &rootdev->dev );
+ err_vmbus_probe:
+	hv_unmap_synic ( hv );
+ err_map_synic:
+	hv_unmap_hypercall ( hv );
+ err_map_hypercall:
+	hv_free_message ( hv );
+ err_alloc_message:
+	hv_free_pages ( hv, hv->hypercall, hv->synic.message, hv->synic.event,
+			NULL );
+ err_alloc_pages:
+ err_check_hv:
+	free ( hv );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove root device
+ *
+ * @v rootdev		Root device
+ */
+static void hv_remove ( struct root_device *rootdev ) {
+	struct hv_hypervisor *hv = rootdev_get_drvdata ( rootdev );
+
+	vmbus_remove ( hv, &rootdev->dev );
+	hv_unmap_synic ( hv );
+	hv_unmap_hypercall ( hv );
+	hv_free_message ( hv );
+	hv_free_pages ( hv, hv->hypercall, hv->synic.message, hv->synic.event,
+			NULL );
+	free ( hv );
+}
+
+/** Hyper-V root device driver */
+static struct root_driver hv_root_driver = {
+	.probe = hv_probe,
+	.remove = hv_remove,
+};
+
+/** Hyper-V root device */
+struct root_device hv_root_device __root_device = {
+	.dev = { .name = "Hyper-V" },
+	.driver = &hv_root_driver,
+};
+
+/* Drag in objects via hv_root_device */
+REQUIRING_SYMBOL ( hv_root_device );
+
+/* Drag in netvsc driver */
+REQUIRE_OBJECT ( netvsc );
diff --git a/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.h b/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.h
new file mode 100644
index 000000000..0d09beb44
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.h
@@ -0,0 +1,57 @@
+#ifndef _HYPERV_H
+#define _HYPERV_H
+
+/** @file
+ *
+ * Hyper-V driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Get vendor identification */
+#define HV_CPUID_VENDOR_ID 0x40000000UL
+
+/** Get interface identification */
+#define HV_CPUID_INTERFACE_ID 0x40000001UL
+
+/** Get hypervisor identification */
+#define HV_CPUID_HYPERVISOR_ID 0x40000002UL
+
+/** Get hypervisor features */
+#define HV_CPUID_FEATURES 0x40000003UL
+
+/** SynIC MSRs are available */
+#define HV_FEATURES_AVAIL_SYNIC_MSR 0x00000004UL
+
+/** Hypercall MSRs are available */
+#define HV_FEATURES_AVAIL_HYPERCALL_MSR 0x00000020UL
+
+/** Guest may post messages */
+#define HV_FEATURES_PERM_POST_MESSAGES 0x00000010UL
+
+/** Guest may signal events */
+#define HV_FEATURES_PERM_SIGNAL_EVENTS 0x00000020UL
+
+/** Guest OS identity MSR */
+#define HV_X64_MSR_GUEST_OS_ID 0x40000000UL
+
+/** Hypercall page MSR */
+#define HV_X64_MSR_HYPERCALL 0x40000001UL
+
+/** SynIC control MSR */
+#define HV_X64_MSR_SCONTROL 0x40000080UL
+
+/** SynIC event flags page MSR */
+#define HV_X64_MSR_SIEFP 0x40000082UL
+
+/** SynIC message page MSR */
+#define HV_X64_MSR_SIMP 0x40000083UL
+
+/** SynIC end of message MSR */
+#define HV_X64_MSR_EOM 0x40000084UL
+
+/** SynIC interrupt source MSRs */
+#define HV_X64_MSR_SINT(x) ( 0x40000090UL + (x) )
+
+#endif /* _HYPERV_H */
diff --git a/roms/ipxe/src/arch/x86/drivers/xen/hvm.c b/roms/ipxe/src/arch/x86/drivers/xen/hvm.c
index 7406ca68d..7ac32d54c 100644
--- a/roms/ipxe/src/arch/x86/drivers/xen/hvm.c
+++ b/roms/ipxe/src/arch/x86/drivers/xen/hvm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
@@ -492,5 +496,8 @@ struct pci_driver hvm_driver __pci_driver = {
 	.remove = hvm_remove,
 };
 
+/* Drag in objects via hvm_driver */
+REQUIRING_SYMBOL ( hvm_driver );
+
 /* Drag in netfront driver */
 REQUIRE_OBJECT ( netfront );
diff --git a/roms/ipxe/src/arch/x86/drivers/xen/hvm.h b/roms/ipxe/src/arch/x86/drivers/xen/hvm.h
index 325d20d66..72ed94f6d 100644
--- a/roms/ipxe/src/arch/x86/drivers/xen/hvm.h
+++ b/roms/ipxe/src/arch/x86/drivers/xen/hvm.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/xen.h>
diff --git a/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c b/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c
index c4e35d179..d73ce2a3e 100644
--- a/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c
+++ b/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/arch/x86/include/bits/bigint.h b/roms/ipxe/src/arch/x86/include/bits/bigint.h
index d3449af5a..c9bb6ea45 100644
--- a/roms/ipxe/src/arch/x86/include/bits/bigint.h
+++ b/roms/ipxe/src/arch/x86/include/bits/bigint.h
@@ -6,7 +6,7 @@
  * Big integer support
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/arch/x86/include/bits/endian.h b/roms/ipxe/src/arch/x86/include/bits/endian.h
new file mode 100644
index 000000000..85718cfdd
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/include/bits/endian.h
@@ -0,0 +1,8 @@
+#ifndef _BITS_ENDIAN_H
+#define _BITS_ENDIAN_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#define __BYTE_ORDER __LITTLE_ENDIAN
+
+#endif /* _BITS_ENDIAN_H */
diff --git a/roms/ipxe/src/arch/x86/include/bits/errfile.h b/roms/ipxe/src/arch/x86/include/bits/errfile.h
index 624575621..0d1617d20 100644
--- a/roms/ipxe/src/arch/x86/include/bits/errfile.h
+++ b/roms/ipxe/src/arch/x86/include/bits/errfile.h
@@ -1,7 +1,7 @@
 #ifndef _BITS_ERRFILE_H
 #define _BITS_ERRFILE_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @addtogroup errfile Error file identifiers
@@ -21,6 +21,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_guestinfo	( ERRFILE_ARCH | ERRFILE_CORE | 0x000a0000 )
 #define ERRFILE_apm		( ERRFILE_ARCH | ERRFILE_CORE | 0x000b0000 )
 #define ERRFILE_vesafb		( ERRFILE_ARCH | ERRFILE_CORE | 0x000c0000 )
+#define ERRFILE_int13con	( ERRFILE_ARCH | ERRFILE_CORE | 0x000d0000 )
 
 #define ERRFILE_bootsector     ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_bzimage	       ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x00010000 )
@@ -46,9 +47,12 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_timer_rdtsc   ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_timer_bios    ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00010000 )
 #define ERRFILE_hvm	      ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00020000 )
+#define ERRFILE_hyperv	      ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00030000 )
+#define ERRFILE_x86_uart      ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00040000 )
 
 #define ERRFILE_cpuid_cmd      ( ERRFILE_ARCH | ERRFILE_OTHER | 0x00000000 )
 #define ERRFILE_cpuid_settings ( ERRFILE_ARCH | ERRFILE_OTHER | 0x00010000 )
+#define ERRFILE_efi_entropy    ( ERRFILE_ARCH | ERRFILE_OTHER | 0x00020000 )
 
 /** @} */
 
diff --git a/roms/ipxe/src/arch/x86/include/bits/io.h b/roms/ipxe/src/arch/x86/include/bits/io.h
index cb1b67a6f..60c2e3edf 100644
--- a/roms/ipxe/src/arch/x86/include/bits/io.h
+++ b/roms/ipxe/src/arch/x86/include/bits/io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/x86_io.h>
 
diff --git a/roms/ipxe/src/arch/x86/include/bits/pci_io.h b/roms/ipxe/src/arch/x86/include/bits/pci_io.h
index 01b12326e..b41e562ee 100644
--- a/roms/ipxe/src/arch/x86/include/bits/pci_io.h
+++ b/roms/ipxe/src/arch/x86/include/bits/pci_io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/pcibios.h>
 #include <ipxe/pcidirect.h>
diff --git a/roms/ipxe/src/arch/x86/include/bits/string.h b/roms/ipxe/src/arch/x86/include/bits/string.h
index dce994983..c26fe30d5 100644
--- a/roms/ipxe/src/arch/x86/include/bits/string.h
+++ b/roms/ipxe/src/arch/x86/include/bits/string.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -28,8 +32,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#define __HAVE_ARCH_MEMCPY
-
 extern void * __memcpy ( void *dest, const void *src, size_t len );
 extern void * __memcpy_reverse ( void *dest, const void *src, size_t len );
 
@@ -169,8 +171,6 @@ memcpy ( void *dest, const void *src, size_t len ) {
 	}
 }
 
-#define __HAVE_ARCH_MEMMOVE
-
 extern void * __memmove ( void *dest, const void *src, size_t len );
 
 /**
@@ -196,8 +196,6 @@ memmove ( void *dest, const void *src, size_t len ) {
 	}
 }
 
-#define __HAVE_ARCH_MEMSET
-
 /**
  * Fill memory region
  *
@@ -206,7 +204,8 @@ memmove ( void *dest, const void *src, size_t len ) {
  * @v len		Length
  * @ret dest		Destination address
  */
-static inline void * memset ( void *dest, int fill, size_t len ) {
+static inline __attribute__ (( always_inline )) void *
+__memset ( void *dest, int fill, size_t len ) {
 	void *discard_D;
 	size_t discard_c;
 
@@ -217,16 +216,129 @@ static inline void * memset ( void *dest, int fill, size_t len ) {
 	return dest;
 }
 
-#define __HAVE_ARCH_MEMSWAP
+/**
+ * Fill memory region with zero (where length is a compile-time constant)
+ *
+ * @v dest		Destination address
+ * @v len		Length
+ * @ret dest		Destination address
+ */
+static inline __attribute__ (( always_inline )) void *
+__constant_memset_zero ( void *dest, size_t len ) {
+	union {
+		uint32_t u32[2];
+		uint16_t u16[4];
+		uint8_t  u8[8];
+	} __attribute__ (( __may_alias__ )) *dest_u = dest;
+	void *edi;
+	uint32_t eax;
+
+	switch ( len ) {
+	case 0 : /* 0 bytes */
+		return dest;
+
+	/* Single-register moves.  Almost certainly better than a
+	 * string operation.  We can avoid clobbering any registers,
+	 * we can reuse a zero that happens to already be in a
+	 * register, and we can optimise away the code entirely if the
+	 * memset() is used to clear a region which then gets
+	 * immediately overwritten.
+	 */
+	case 1 : /* 3 bytes */
+		dest_u->u8[0] = 0;
+		return dest;
+	case 2: /* 5 bytes */
+		dest_u->u16[0] = 0;
+		return dest;
+	case 4: /* 6 bytes */
+		dest_u->u32[0] = 0;
+		return dest;
+
+	/* Double-register moves.  Very probably better than a string
+	 * operation.
+	 */
+	case 3 : /* 9 bytes */
+		dest_u->u16[0] = 0;
+		dest_u->u8[2]  = 0;
+		return dest;
+	case 5 : /* 10 bytes */
+		dest_u->u32[0] = 0;
+		dest_u->u8[4]  = 0;
+		return dest;
+	case 6 : /* 12 bytes */
+		dest_u->u32[0] = 0;
+		dest_u->u16[2] = 0;
+		return dest;
+	case 8 : /* 13 bytes */
+		dest_u->u32[0] = 0;
+		dest_u->u32[1] = 0;
+		return dest;
+	}
+
+	/* As with memcpy(), we can potentially save space by using
+	 * multiple single-byte "stos" instructions instead of loading
+	 * up ecx and using "rep stosb".
+	 *
+	 * "load ecx, rep movsb" is 7 bytes, plus an average of 1 byte
+	 * to allow for saving/restoring ecx 50% of the time.
+	 *
+	 * "stosl" and "stosb" are 1 byte each, "stosw" is two bytes.
+	 *
+	 * The calculations are therefore the same as for memcpy(),
+	 * giving a cutoff point of around 26 bytes.
+	 */
 
-extern void * memswap ( void *dest, void *src, size_t len );
+	edi = dest;
+	eax = 0;
+
+	if ( len >= 26 )
+		return __memset ( dest, 0, len );
 
-#define __HAVE_ARCH_STRNCMP
+	if ( len >= 6*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 5*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 4*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 3*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 2*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 1*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( ( len % 4 ) >= 2 )
+		__asm__ __volatile__ ( "stosw" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( ( len % 2 ) >= 1 )
+		__asm__ __volatile__ ( "stosb" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
 
-extern int strncmp ( const char *str1, const char *str2, size_t len );
+	return dest;
+}
 
-#define __HAVE_ARCH_STRLEN
+/**
+ * Fill memory region
+ *
+ * @v dest		Destination address
+ * @v fill		Fill pattern
+ * @v len		Length
+ * @ret dest		Destination address
+ */
+static inline __attribute__ (( always_inline )) void *
+memset ( void *dest, int fill, size_t len ) {
 
-extern size_t strlen ( const char *string );
+	if ( __builtin_constant_p ( fill ) && ( fill == 0 ) &&
+	     __builtin_constant_p ( len ) ) {
+		return __constant_memset_zero ( dest, len );
+	} else {
+		return __memset ( dest, fill, len );
+	}
+}
 
 #endif /* X86_BITS_STRING_H */
diff --git a/roms/ipxe/src/arch/x86/include/bits/tcpip.h b/roms/ipxe/src/arch/x86/include/bits/tcpip.h
index a4b335eb1..5c2baffcf 100644
--- a/roms/ipxe/src/arch/x86/include/bits/tcpip.h
+++ b/roms/ipxe/src/arch/x86/include/bits/tcpip.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern uint16_t x86_tcpip_continue_chksum ( uint16_t partial,
 					    const void *data, size_t len );
diff --git a/roms/ipxe/src/arch/x86/include/bits/uart.h b/roms/ipxe/src/arch/x86/include/bits/uart.h
new file mode 100644
index 000000000..e09cd3f4c
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/include/bits/uart.h
@@ -0,0 +1,41 @@
+#ifndef _BITS_UART_H
+#define _BITS_UART_H
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/**
+ * Write to UART register
+ *
+ * @v uart		UART
+ * @v addr		Register address
+ * @v data		Data
+ */
+static inline __attribute__ (( always_inline )) void
+uart_write ( struct uart *uart, unsigned int addr, uint8_t data ) {
+	outb ( data, ( uart->base + addr ) );
+}
+
+/**
+ * Read from UART register
+ *
+ * @v uart		UART
+ * @v addr		Register address
+ * @ret data		Data
+ */
+static inline __attribute__ (( always_inline )) uint8_t
+uart_read ( struct uart *uart, unsigned int addr ) {
+	return inb ( uart->base + addr );
+}
+
+extern int uart_select ( struct uart *uart, unsigned int port );
+
+#endif /* _BITS_UART_H */
diff --git a/roms/ipxe/src/arch/x86/include/bits/xen.h b/roms/ipxe/src/arch/x86/include/bits/xen.h
index dbccf1b77..fc065ea38 100644
--- a/roms/ipxe/src/arch/x86/include/bits/xen.h
+++ b/roms/ipxe/src/arch/x86/include/bits/xen.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Hypercall registers */
 #ifdef __x86_64__
@@ -161,4 +161,23 @@ xen_hypercall_5 ( struct xen_hypervisor *xen, unsigned int hypercall,
 	return retval;
 }
 
+/**
+ * Test and clear pending event
+ *
+ * @v xen		Xen hypervisor
+ * @v port		Event channel port
+ * @ret pending		Event was pending
+ */
+static inline __attribute__ (( always_inline )) uint8_t
+xenevent_pending ( struct xen_hypervisor *xen, evtchn_port_t port ) {
+	uint8_t pending;
+
+	__asm__ __volatile__ ( "lock btr %2, %0\n\t"
+			       "setc %1\n\t"
+			       : "+m" ( xen->shared->evtchn_pending ),
+				 "=a"  ( pending )
+			       : "Ir" ( port ) );
+	return pending;
+}
+
 #endif /* _BITS_XEN_H */
diff --git a/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h b/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h
index 2f78dfca1..da85d0b88 100644
--- a/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h
+++ b/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
@@ -39,6 +39,9 @@ struct x86_features {
 /** Get standard features */
 #define CPUID_FEATURES 0x00000001UL
 
+/** Hypervisor is present */
+#define CPUID_FEATURES_INTEL_ECX_HYPERVISOR 0x80000000UL
+
 /** Get largest extended function */
 #define CPUID_AMD_MAX_FN 0x80000000UL
 
diff --git a/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h b/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h
index e85a272b3..1a391c9b6 100644
--- a/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h
+++ b/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_EFIX86
 #define NAP_PREFIX_efix86
diff --git a/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h b/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h
index 36af7fcde..7e1bcd814 100644
--- a/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h
+++ b/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef PCIAPI_PCBIOS
 #define PCIAPI_PREFIX_pcbios
diff --git a/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h b/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h
index 7fa7c4fa7..d924f2f20 100644
--- a/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h
+++ b/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h
@@ -1,7 +1,7 @@
 #ifndef _PCIDIRECT_H
 #define _PCIDIRECT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/io.h>
diff --git a/roms/ipxe/src/arch/x86/include/ipxe/pit8254.h b/roms/ipxe/src/arch/x86/include/ipxe/pit8254.h
new file mode 100644
index 000000000..00b0ab164
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/include/ipxe/pit8254.h
@@ -0,0 +1,81 @@
+#ifndef _IPXE_PIT8254_H
+#define _IPXE_PIT8254_H
+
+/** @file
+ *
+ * 8254 Programmable Interval Timer
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** IRQ0 channel */
+#define PIT8254_CH_IRQ0 0
+
+/** PC speaker channel */
+#define PIT8254_CH_SPKR 2
+
+/** Timer frequency (1.193182MHz) */
+#define PIT8254_HZ 1193182UL
+
+/** Data port */
+#define PIT8254_DATA(channel) ( 0x40 + (channel) )
+
+/** Mode/command register */
+#define PIT8254_CMD 0x43
+
+/** Select channel */
+#define PIT8254_CMD_CHANNEL(channel) ( (channel) << 6 )
+
+/** Access modes */
+#define PIT8254_CMD_ACCESS_LATCH 0x00	/**< Latch count value command */
+#define PIT8254_CMD_ACCESS_LO	0x10	/**< Low byte only */
+#define PIT8254_CMD_ACCESS_HI	0x20	/**< High byte only */
+#define PIT8254_CMD_ACCESS_LOHI	0x30	/**< Low-byte, high-byte pair */
+
+/* Operating modes */
+#define PIT8254_CMD_OP_TERMINAL	0x00	/**< Interrupt on terminal count */
+#define PIT8254_CMD_OP_ONESHOT	0x02	/**< Hardware re-triggerable one-shot */
+#define PIT8254_CMD_OP_RATE	0x04	/**< Rate generator */
+#define PIT8254_CMD_OP_SQUARE	0x06	/**< Square wave generator */
+#define PIT8254_CMD_OP_SWSTROBE	0x08	/**< Software triggered strobe */
+#define PIT8254_CMD_OP_HWSTROBE	0x0a	/**< Hardware triggered strobe */
+#define PIT8254_CMD_OP_RATE2	0x0c	/**< Rate generator (duplicate) */
+#define PIT8254_CMD_OP_SQUARE2	0x0e	/**< Square wave generator (duplicate)*/
+
+/** Binary mode */
+#define PIT8254_CMD_BINARY 0x00
+
+/** BCD mode */
+#define PIT8254_CMD_BCD 0x01
+
+/** PC speaker control register */
+#define PIT8254_SPKR 0x61
+
+/** PC speaker channel gate */
+#define PIT8254_SPKR_GATE 0x01
+
+/** PC speaker enabled */
+#define PIT8254_SPKR_ENABLE 0x02
+
+/** PC speaker channel output */
+#define PIT8254_SPKR_OUT 0x20
+
+extern void pit8254_speaker_delay ( unsigned int ticks );
+
+/**
+ * Delay for a fixed number of microseconds
+ *
+ * @v usecs		Number of microseconds for which to delay
+ */
+static inline __attribute__ (( always_inline )) void
+pit8254_udelay ( unsigned long usecs ) {
+
+	/* Delays are invariably compile-time constants; force the
+	 * multiplication and division to take place at compilation
+	 * time rather than runtime.
+	 */
+	pit8254_speaker_delay ( ( usecs * PIT8254_HZ ) / 1000000 );
+}
+
+#endif /* _IPXE_PIT8254_H */
diff --git a/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h b/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h
index 9e68f4e78..5214e9fbb 100644
--- a/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h
+++ b/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h
@@ -15,7 +15,7 @@
  * physically fit into a machine with such an old CPU anyway.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef IOAPI_X86
 #define IOAPI_PREFIX_x86
diff --git a/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h b/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h
index e83fd9d87..d60905f22 100644
--- a/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h
+++ b/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _LINUX_DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE(GPL2_OR_LATER_OR_UBDL);
 
 #include <ipxe/dhcp.h>
 
diff --git a/roms/ipxe/src/arch/i386/include/pic8259.h b/roms/ipxe/src/arch/x86/include/pic8259.h
index a07e97d30..f02e62909 100644
--- a/roms/ipxe/src/arch/i386/include/pic8259.h
+++ b/roms/ipxe/src/arch/x86/include/pic8259.h
@@ -4,16 +4,13 @@
  * Initially written by Michael Brown (mcb30).
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef PIC8259_H
 #define PIC8259_H
 
 #include <ipxe/io.h>
 
-/* For segoff_t */
-#include "realmode.h"
-
 #define IRQ_PIC_CUTOFF 8
 
 /* 8259 register locations */
diff --git a/roms/ipxe/src/arch/x86/interface/efi/efi_entropy.c b/roms/ipxe/src/arch/x86/interface/efi/efi_entropy.c
new file mode 100644
index 000000000..a54bd12e6
--- /dev/null
+++ b/roms/ipxe/src/arch/x86/interface/efi/efi_entropy.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <errno.h>
+#include <ipxe/entropy.h>
+#include <ipxe/crc32.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/Protocol/Rng.h>
+
+/** @file
+ *
+ * EFI entropy source
+ *
+ */
+
+/** Random number generator protocol */
+static EFI_RNG_PROTOCOL *efirng;
+EFI_REQUEST_PROTOCOL ( EFI_RNG_PROTOCOL, &efirng );
+
+/** Minimum number of bytes to request from RNG
+ *
+ * The UEFI spec states (for no apparently good reason) that "When a
+ * Deterministic Random Bit Generator (DRBG) is used on the output of
+ * a (raw) entropy source, its security level must be at least 256
+ * bits."  The EDK2 codebase (mis)interprets this to mean that the
+ * call to GetRNG() should fail if given a buffer less than 32 bytes.
+ *
+ * Incidentally, nothing in the EFI RNG protocol provides any way to
+ * report the actual amount of entropy returned by GetRNG().
+ */
+#define EFI_ENTROPY_RNG_LEN 32
+
+/** Time (in 100ns units) to delay waiting for timer tick
+ *
+ * In theory, UEFI allows us to specify a trigger time of zero to
+ * simply wait for the next timer tick.  In practice, specifying zero
+ * seems to often return immediately, which produces almost no
+ * entropy.  Specify a delay of 1000ns to try to force an existent
+ * delay.
+ */
+#define EFI_ENTROPY_TRIGGER_TIME 10
+
+/** Event used to wait for timer tick */
+static EFI_EVENT tick;
+
+/**
+ * Enable entropy gathering
+ *
+ * @ret rc		Return status code
+ */
+static int efi_entropy_enable ( void ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_STATUS efirc;
+	int rc;
+
+	DBGC ( &tick, "ENTROPY %s RNG protocol\n",
+	       ( efirng ? "has" : "has no" ) );
+
+	/* Create timer tick event */
+	if ( ( efirc = bs->CreateEvent ( EVT_TIMER, TPL_NOTIFY, NULL, NULL,
+					 &tick ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not create event: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Disable entropy gathering
+ *
+ */
+static void efi_entropy_disable ( void ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+
+	/* Close timer tick event */
+	bs->CloseEvent ( tick );
+}
+
+/**
+ * Wait for a timer tick
+ *
+ * @ret low		TSC low-order bits, or negative error
+ */
+static int efi_entropy_tick ( void ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	UINTN index;
+	uint16_t low;
+	uint32_t discard_d;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Wait for next timer tick */
+	if ( ( efirc = bs->SetTimer ( tick, TimerRelative,
+				      EFI_ENTROPY_TRIGGER_TIME ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not set timer: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+	if ( ( efirc = bs->WaitForEvent ( 1, &tick, &index ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not wait for timer tick: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Get current TSC low-order bits */
+	__asm__ __volatile__ ( "rdtsc" : "=a" ( low ), "=d" ( discard_d ) );
+
+	return low;
+}
+
+/**
+ * Get noise sample from timer ticks
+ *
+ * @ret noise		Noise sample
+ * @ret rc		Return status code
+ */
+static int efi_get_noise_ticks ( noise_sample_t *noise ) {
+	int before;
+	int after;
+	int rc;
+
+	/* Wait for a timer tick */
+	before = efi_entropy_tick();
+	if ( before < 0 ) {
+		rc = before;
+		return rc;
+	}
+
+	/* Wait for another timer tick */
+	after = efi_entropy_tick();
+	if ( after < 0 ) {
+		rc = after;
+		return rc;
+	}
+
+	/* Use TSC delta as noise sample */
+	*noise = ( after - before );
+
+	return 0;
+}
+
+/**
+ * Get noise sample from RNG protocol
+ *
+ * @ret noise		Noise sample
+ * @ret rc		Return status code
+ */
+static int efi_get_noise_rng ( noise_sample_t *noise ) {
+	uint8_t buf[EFI_ENTROPY_RNG_LEN];
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Fail if we have no EFI RNG protocol */
+	if ( ! efirng )
+		return -ENOTSUP;
+
+	/* Get the minimum allowed number of random bytes */
+	if ( ( efirc = efirng->GetRNG ( efirng, NULL, EFI_ENTROPY_RNG_LEN,
+					buf ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not read from RNG: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Reduce random bytes to a single noise sample.  This seems
+	 * like overkill, but we have no way of knowing how much
+	 * entropy is actually present in the bytes returned by the
+	 * RNG protocol.
+	 */
+	*noise = crc32_le ( 0, buf, sizeof ( buf ) );
+
+	return 0;
+}
+
+/**
+ * Get noise sample
+ *
+ * @ret noise		Noise sample
+ * @ret rc		Return status code
+ */
+static int efi_get_noise ( noise_sample_t *noise ) {
+	int rc;
+
+	/* Try RNG first, falling back to timer ticks */
+	if ( ( ( rc = efi_get_noise_rng ( noise ) ) != 0 ) &&
+	     ( ( rc = efi_get_noise_ticks ( noise ) ) != 0 ) )
+		return rc;
+
+	return 0;
+}
+
+PROVIDE_ENTROPY_INLINE ( efi, min_entropy_per_sample );
+PROVIDE_ENTROPY ( efi, entropy_enable, efi_entropy_enable );
+PROVIDE_ENTROPY ( efi, entropy_disable, efi_entropy_disable );
+PROVIDE_ENTROPY ( efi, get_noise, efi_get_noise );
diff --git a/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c b/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c
index b05421fab..3ebf0bd68 100644
--- a/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c
+++ b/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/nap.h>
 #include <ipxe/efi/efi.h>
diff --git a/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c b/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c
index 3daefd00a..4fbb19ff7 100644
--- a/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c
+++ b/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c
@@ -21,7 +21,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <stdlib.h>
 #include <ipxe/init.h>
+#include <ipxe/device.h>
 #include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_driver.h>
 
 /**
  * EFI entry point
@@ -44,3 +46,36 @@ EFI_STATUS EFIAPI _efidrv_start ( EFI_HANDLE image_handle,
 
 	return 0;
 }
+
+/**
+ * Probe EFI root bus
+ *
+ * @v rootdev		EFI root device
+ */
+static int efi_probe ( struct root_device *rootdev __unused ) {
+
+	/* Do nothing */
+	return 0;
+}
+
+/**
+ * Remove EFI root bus
+ *
+ * @v rootdev		EFI root device
+ */
+static void efi_remove ( struct root_device *rootdev __unused ) {
+
+	efi_driver_disconnect_all();
+}
+
+/** EFI root device driver */
+static struct root_driver efi_root_driver = {
+	.probe = efi_probe,
+	.remove = efi_remove,
+};
+
+/** EFI root device */
+struct root_device efi_root_device __root_device = {
+	.dev = { .name = "EFI" },
+	.driver = &efi_root_driver,
+};
diff --git a/roms/ipxe/src/arch/x86/prefix/efiprefix.c b/roms/ipxe/src/arch/x86/prefix/efiprefix.c
index b0bf99c65..18b931e68 100644
--- a/roms/ipxe/src/arch/x86/prefix/efiprefix.c
+++ b/roms/ipxe/src/arch/x86/prefix/efiprefix.c
@@ -26,6 +26,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_snp.h>
 #include <ipxe/efi/efi_autoboot.h>
+#include <ipxe/efi/efi_watchdog.h>
 
 /**
  * EFI entry point
@@ -49,6 +50,9 @@ EFI_STATUS EFIAPI _efi_start ( EFI_HANDLE image_handle,
 	/* Claim SNP devices for use by iPXE */
 	efi_snp_claim();
 
+	/* Start watchdog holdoff timer */
+	efi_watchdog_start();
+
 	/* Call to main() */
 	if ( ( rc = main() ) != 0 ) {
 		efirc = EFIRC ( rc );
@@ -56,6 +60,7 @@ EFI_STATUS EFIAPI _efi_start ( EFI_HANDLE image_handle,
 	}
 
  err_main:
+	efi_watchdog_stop();
 	efi_snp_release();
 	efi_loaded_image->Unload ( image_handle );
 	efi_driver_reconnect_all();
diff --git a/roms/ipxe/src/arch/x86_64/Makefile b/roms/ipxe/src/arch/x86_64/Makefile
index b687f3407..48c0aa1af 100644
--- a/roms/ipxe/src/arch/x86_64/Makefile
+++ b/roms/ipxe/src/arch/x86_64/Makefile
@@ -40,6 +40,7 @@ endif
 
 # x86_64-specific directories containing source files
 #
+SRCDIRS		+= arch/x86_64/core
 SRCDIRS		+= arch/x86_64/prefix
 
 # Include common x86 Makefile
diff --git a/roms/ipxe/src/arch/x86_64/core/setjmp.S b/roms/ipxe/src/arch/x86_64/core/setjmp.S
new file mode 100644
index 000000000..e43200d7b
--- /dev/null
+++ b/roms/ipxe/src/arch/x86_64/core/setjmp.S
@@ -0,0 +1,65 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+	.text
+	.code64
+
+	/* Must match jmp_buf structure layout */
+	.struct	0
+env_retaddr:	.quad	0
+env_stack:	.quad	0
+env_rbx:	.quad	0
+env_rbp:	.quad	0
+env_r12:	.quad	0
+env_r13:	.quad	0
+env_r14:	.quad	0
+env_r15:	.quad	0
+	.previous
+
+/*
+ * Save stack context for non-local goto
+ */
+	.globl	setjmp
+setjmp:
+	/* Save return address */
+	movq	0(%rsp), %rax
+	movq	%rax, env_retaddr(%rdi)
+	/* Save stack pointer */
+	movq	%rsp, env_stack(%rdi)
+	/* Save other registers */
+	movq	%rbx, env_rbx(%rdi)
+	movq	%rbp, env_rbp(%rdi)
+	movq	%r12, env_r12(%rdi)
+	movq	%r13, env_r13(%rdi)
+	movq	%r14, env_r14(%rdi)
+	movq	%r15, env_r15(%rdi)
+	/* Return 0 when returning as setjmp() */
+	xorq	%rax, %rax
+	ret
+	.size	setjmp, . - setjmp
+
+/*
+ * Non-local jump to a saved stack context
+ */
+	.globl	longjmp
+longjmp:
+	/* Get result in %rax */
+	movq	%rsi, %rax
+	/* Force result to non-zero */
+	testq	%rax, %rax
+	jnz	1f
+	incq	%rax
+1:	/* Restore stack pointer */
+	movq	env_stack(%rdi), %rsp
+	/* Restore other registers */
+	movq	env_rbx(%rdi), %rbx
+	movq	env_rbp(%rdi), %rbp
+	movq	env_r12(%rdi), %r12
+	movq	env_r13(%rdi), %r13
+	movq	env_r14(%rdi), %r14
+	movq	env_r15(%rdi), %r15
+	/* Replace return address on the new stack */
+	popq	%rcx	/* discard */
+	pushq	env_retaddr(%rdi)
+	/* Return to setjmp() caller */
+	ret
+	.size	longjmp, . - longjmp
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h b/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h
index 2e472d98a..d8c5098ef 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h
@@ -9,7 +9,7 @@
 
 #include <stdint.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static inline __attribute__ (( always_inline, const )) uint16_t
 __bswap_variable_16 ( uint16_t x ) {
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/compiler.h b/roms/ipxe/src/arch/x86_64/include/bits/compiler.h
index 51a7eaae2..f70b2e517 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/compiler.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/compiler.h
@@ -1,6 +1,9 @@
 #ifndef _BITS_COMPILER_H
 #define _BITS_COMPILER_H
 
+/** Dummy relocation type */
+#define RELOC_TYPE_NONE R_X86_64_NONE
+
 #ifndef ASSEMBLY
 
 /** Declare a function with standard calling conventions */
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/endian.h b/roms/ipxe/src/arch/x86_64/include/bits/endian.h
deleted file mode 100644
index 413e702db..000000000
--- a/roms/ipxe/src/arch/x86_64/include/bits/endian.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef ETHERBOOT_BITS_ENDIAN_H
-#define ETHERBOOT_BITS_ENDIAN_H
-
-#define __BYTE_ORDER __LITTLE_ENDIAN
-
-#endif /* ETHERBOOT_BITS_ENDIAN_H */
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/entropy.h b/roms/ipxe/src/arch/x86_64/include/bits/entropy.h
index 9c64c833b..a9b3bc10e 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/entropy.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/entropy.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_ENTROPY_H */
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/hyperv.h b/roms/ipxe/src/arch/x86_64/include/bits/hyperv.h
new file mode 100644
index 000000000..845c182f7
--- /dev/null
+++ b/roms/ipxe/src/arch/x86_64/include/bits/hyperv.h
@@ -0,0 +1,75 @@
+#ifndef _BITS_HYPERV_H
+#define _BITS_HYPERV_H
+
+/** @file
+ *
+ * Hyper-V interface
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/**
+ * Issue hypercall
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v code		Call code
+ * @v in		Input parameters
+ * @v out		Output parameters
+ * @ret status		Status code
+ */
+static inline __attribute__ (( always_inline )) int
+hv_call ( struct hv_hypervisor *hv, unsigned int code, const void *in,
+	  void *out ) {
+	void *hypercall = hv->hypercall;
+	register uint64_t rcx asm ( "rcx" );
+	register uint64_t rdx asm ( "rdx" );
+	register uint64_t r8 asm ( "r8" );
+	uint64_t in_phys;
+	uint64_t out_phys;
+	uint16_t result;
+
+	in_phys = ( ( __builtin_constant_p ( in ) && ( in == NULL ) )
+		       ? 0 : virt_to_phys ( in ) );
+	out_phys = ( ( __builtin_constant_p ( out ) && ( out == NULL ) )
+		       ? 0 : virt_to_phys ( out ) );
+	rcx = code;
+	rdx = in_phys;
+	r8 = out_phys;
+	__asm__ __volatile__ ( "call *%4"
+			       : "=a" ( result ), "+r" ( rcx ), "+r" ( rdx ),
+				 "+r" ( r8 )
+			       : "m" ( hypercall )
+			       : "r9", "r10", "r11", "xmm0", "xmm1", "xmm2",
+				 "xmm3", "xmm4", "xmm5" );
+	return result;
+}
+
+/**
+ * Set bit atomically
+ *
+ * @v bits		Bit field
+ * @v bit		Bit to set
+ */
+static inline __attribute__ (( always_inline )) void
+hv_set_bit ( void *bits, unsigned int bit ) {
+	struct {
+		uint64_t qword[ ( bit / 64 ) + 1 ];
+	} *qwords = bits;
+
+	/* Set bit using "lock bts".  Inform compiler that any memory
+	 * from the start of the bit field up to and including the
+	 * qword containing this bit may be modified.  (This is
+	 * overkill but shouldn't matter in practice since we're
+	 * unlikely to subsequently read other bits from the same bit
+	 * field.)
+	 */
+	__asm__ __volatile__ ( "lock bts %1, %0"
+			       : "+m" ( *qwords ) : "Ir" ( bit ) );
+}
+
+#endif /* _BITS_HYPERV_H */
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/profile.h b/roms/ipxe/src/arch/x86_64/include/bits/profile.h
index 6fc16d84b..b7c74fbe7 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/profile.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/profile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/reboot.h b/roms/ipxe/src/arch/x86_64/include/bits/reboot.h
index f1bce0540..f9bcd6a7b 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/reboot.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/reboot.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_REBOOT_H */
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h b/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h
index d33d03cbe..dcab830f6 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_SANBOOT_H */
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/strings.h b/roms/ipxe/src/arch/x86_64/include/bits/strings.h
index 6ee99a500..3b7911f3b 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/strings.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/strings.h
@@ -1,7 +1,43 @@
 #ifndef _BITS_STRINGS_H
 #define _BITS_STRINGS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsll ( long long value ){
+	long long lsb_minus_one;
+
+	/* If the input value is zero, the BSF instruction returns
+	 * ZF=0 and leaves an undefined value in the output register.
+	 * Perform this check in C rather than asm so that it can be
+	 * omitted in cases where the compiler is able to prove that
+	 * the input is non-zero.
+	 */
+	if ( value ) {
+		__asm__ ( "bsfq %1, %0"
+			  : "=r" ( lsb_minus_one )
+			  : "rm" ( value ) );
+		return ( lsb_minus_one + 1 );
+	} else {
+		return 0;
+	}
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsl ( long value ) {
+
+	return __ffsll ( value );
+}
 
 /**
  * Find last (i.e. most significant) set bit
@@ -13,7 +49,7 @@ static inline __attribute__ (( always_inline )) int __flsll ( long long value ){
 	long long msb_minus_one;
 
 	/* If the input value is zero, the BSR instruction returns
-	 * ZF=1 and leaves an undefined value in the output register.
+	 * ZF=0 and leaves an undefined value in the output register.
 	 * Perform this check in C rather than asm so that it can be
 	 * omitted in cases where the compiler is able to prove that
 	 * the input is non-zero.
diff --git a/roms/ipxe/src/arch/x86_64/include/bits/time.h b/roms/ipxe/src/arch/x86_64/include/bits/time.h
index 59b355359..aa74fac8c 100644
--- a/roms/ipxe/src/arch/x86_64/include/bits/time.h
+++ b/roms/ipxe/src/arch/x86_64/include/bits/time.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_TIME_H */
diff --git a/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h b/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h
index 9a4790fdc..6511c1ad3 100644
--- a/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h
+++ b/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 
diff --git a/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h b/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h
index a5816ac35..316243b69 100644
--- a/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h
+++ b/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Read model-specific register
diff --git a/roms/ipxe/src/arch/x86_64/include/setjmp.h b/roms/ipxe/src/arch/x86_64/include/setjmp.h
new file mode 100644
index 000000000..69835d9fa
--- /dev/null
+++ b/roms/ipxe/src/arch/x86_64/include/setjmp.h
@@ -0,0 +1,34 @@
+#ifndef _SETJMP_H
+#define _SETJMP_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** A jump buffer */
+typedef struct {
+	/** Saved return address */
+	uint64_t retaddr;
+	/** Saved stack pointer */
+	uint64_t stack;
+	/** Saved %rbx */
+	uint64_t rbx;
+	/** Saved %rbp */
+	uint64_t rbp;
+	/** Saved %r12 */
+	uint64_t r12;
+	/** Saved %r13 */
+	uint64_t r13;
+	/** Saved %r14 */
+	uint64_t r14;
+	/** Saved %r15 */
+	uint64_t r15;
+} jmp_buf[1];
+
+extern int __asmcall __attribute__ (( returns_twice ))
+setjmp ( jmp_buf env );
+
+extern void __asmcall __attribute__ (( noreturn ))
+longjmp ( jmp_buf env, int val );
+
+#endif /* _SETJMP_H */
diff --git a/roms/ipxe/src/config/.gitignore b/roms/ipxe/src/config/.gitignore
deleted file mode 100644
index 8e94f32fe..000000000
--- a/roms/ipxe/src/config/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-.buildserial.*
diff --git a/roms/ipxe/src/config/branding.h b/roms/ipxe/src/config/branding.h
new file mode 100644
index 000000000..73f00af95
--- /dev/null
+++ b/roms/ipxe/src/config/branding.h
@@ -0,0 +1,174 @@
+#ifndef CONFIG_BRANDING_H
+#define CONFIG_BRANDING_H
+
+/** @file
+ *
+ * Branding configuration
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/*
+ * Branding
+ *
+ * Vendors may use these strings to add their own branding to iPXE.
+ * PRODUCT_NAME is displayed prior to any iPXE branding in startup
+ * messages, and PRODUCT_SHORT_NAME is used where a brief product
+ * label is required (e.g. in BIOS boot selection menus).
+ *
+ * To minimise end-user confusion, it's probably a good idea to either
+ * make PRODUCT_SHORT_NAME a substring of PRODUCT_NAME or leave it as
+ * "iPXE".
+ *
+ */
+#define PRODUCT_NAME ""
+#define PRODUCT_SHORT_NAME "iPXE"
+#define PRODUCT_URI "http://ipxe.org"
+
+/*
+ * Tag line
+ *
+ * If your PRODUCT_SHORT_NAME is longer than the four characters used
+ * by "iPXE", then the standard tag line "Open Source Network Boot
+ * Firmware" is unlikely to fit neatly onto the screen.
+ */
+#define PRODUCT_TAG_LINE "Open Source Network Boot Firmware"
+
+/*
+ * Error messages
+ *
+ * iPXE error messages comprise a summary error message
+ * (e.g. "Permission denied") and a 32-bit error number.  This number
+ * is incorporated into an error URI such as
+ *
+ *   "No such file or directory (http://ipxe.org/2d0c613b)"
+ *
+ * or
+ *
+ *   "Operation not supported (http://ipxe.org/3c092003)"
+ *
+ * Users may browse to the URI within the error message, which is
+ * provided by a database running on the iPXE web site
+ * (http://ipxe.org).  This database provides details for all possible
+ * errors generated by iPXE, including:
+ *
+ * - the detailed error message (e.g. "Not an OCSP signing
+ *   certificate") to complement the summary message (e.g. "Permission
+ *   denied") which is compiled into the iPXE binary.
+ *
+ * - an instruction to the user to upgrade, if the error cannot be
+ *   generated by the latest version of iPXE.
+ *
+ * - hints on how to fix the error (e.g. "This error indicates that
+ *   the file was not found on the TFTP server.  Check that you can
+ *   retrieve the file using an alternative TFTP client, such as
+ *   tftp-hpa on Linux.")
+ *
+ * - details of which source file within the iPXE codebase generated
+ *   the error.
+ *
+ * - a direct link to the line(s) of code which generated the error.
+ *
+ * If you have a customer support team and would like your customers
+ * to contact your support team for all problems, instead of using the
+ * existing support infrastructure provided by http://ipxe.org, then
+ * you may define a custom URI to be included within error messages.
+ *
+ * Note that the custom URI is a printf() format string which must
+ * include a format specifier for the 32-bit error number.
+ */
+#define PRODUCT_ERROR_URI "http://ipxe.org/%08x"
+
+/*
+ * Command help messages
+ *
+ * iPXE command help messages include a URI constructed from the
+ * command name, such as
+ *
+ *   "See http://ipxe.org/cmd/vcreate for further information"
+ *
+ * The iPXE web site includes documentation for the commands provided
+ * by the iPXE shell, including:
+ *
+ * - details of the command syntax (e.g. "vcreate --tag <tag>
+ *   [--priority <priority>] <trunk interface>").
+ *
+ * - example usages of the command (e.g. "vcreate --tag 123 net0")
+ *
+ * - a formal description of the command (e.g. "Create a VLAN network
+ *   interface on an existing trunk network interface. The new network
+ *   interface will be named by appending a hyphen and the VLAN tag
+ *   value to the trunk network interface name.")
+ *
+ * - details of the possible exit statuses from the command.
+ *
+ * - links to documentation for related commands (e.g. "vdestroy")
+ *
+ * - links to documentation for relevant build options (e.g. "VLAN_CMD").
+ *
+ * - general hints and tips on using the command.
+ *
+ * If you want to provide your own documentation for all of the
+ * commands provided by the iPXE shell, rather than using the existing
+ * support infrastructure provided by http://ipxe.org, then you may
+ * define a custom URI to be included within command help messages.
+ *
+ * Note that the custom URI is a printf() format string which must
+ * include a format specifier for the command name.
+ *
+ * [ Please also note that the existing documentation is licensed
+ *   under Creative Commons terms which require attribution to the
+ *   iPXE project and prohibit the alteration or removal of any
+ *   references to "iPXE". ]
+ */
+#define PRODUCT_COMMAND_URI "http://ipxe.org/cmd/%s"
+
+/*
+ * Setting help messages
+ *
+ * iPXE setting help messages include a URI constructed from the
+ * setting name, such as
+ *
+ *   "http://ipxe.org/cfg/initiator-iqn"
+ *
+ * The iPXE web site includes documentation for the settings used by
+ * iPXE, including:
+ *
+ * - details of the corresponding DHCP option number.
+ *
+ * - details of the corresponding ISC dhcpd option name.
+ *
+ * - examples of using the setting from the iPXE command line, or in
+ *   iPXE scripts.
+ *
+ * - examples of configuring the setting via a DHCP server.
+ *
+ * - a formal description of the setting.
+ *
+ * - links to documentation for related settings.
+ *
+ * - links to documentation for relevant build options.
+ *
+ * - general notes about the setting.
+ *
+ * If you want to provide your own documentation for all of the
+ * settings used by iPXE, rather than using the existing support
+ * infrastructure provided by http://ipxe.org, then you may define a
+ * custom URI to be included within setting help messages.
+ *
+ * Note that the custom URI is a printf() format string which must
+ * include a format specifier for the setting name.
+ *
+ * [ Please also note that the existing documentation is licensed
+ *   under Creative Commons terms which require attribution to the
+ *   iPXE project and prohibit the alteration or removal of any
+ *   references to "iPXE". ]
+ */
+#define PRODUCT_SETTING_URI "http://ipxe.org/cfg/%s"
+
+#include <config/local/branding.h>
+
+#endif /* CONFIG_BRANDING_H */
diff --git a/roms/ipxe/src/config/colour.h b/roms/ipxe/src/config/colour.h
index 57d20c1db..98198f12f 100644
--- a/roms/ipxe/src/config/colour.h
+++ b/roms/ipxe/src/config/colour.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define COLOR_NORMAL_FG		COLOR_WHITE
 #define COLOR_NORMAL_BG		COLOR_BLUE
diff --git a/roms/ipxe/src/config/config.c b/roms/ipxe/src/config/config.c
index 6c8b9551a..1dd912c1d 100644
--- a/roms/ipxe/src/config/config.c
+++ b/roms/ipxe/src/config/config.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 #include <config/console.h>
@@ -30,33 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * in the final iPXE executable built.
  */
 
-/*
- * Build ID string calculations
- *
- */
-#undef XSTR
-#undef STR
-#define XSTR(s) STR(s)
-#define STR(s) #s
-
-#ifdef BUILD_SERIAL
-#include "config/.buildserial.h"
-#define BUILD_SERIAL_STR " #" XSTR(BUILD_SERIAL_NUM)
-#else
-#define BUILD_SERIAL_STR ""
-#endif
-
-#ifdef BUILD_ID
-#define BUILD_ID_STR " " BUILD_ID
-#else
-#define BUILD_ID_STR ""
-#endif
-
-#if defined(BUILD_ID) || defined(BUILD_SERIAL)
-#define BUILD_STRING " [build" BUILD_ID_STR BUILD_SERIAL_STR "]"
-#else
-#define BUILD_STRING ""
-#endif
+PROVIDE_REQUIRING_SYMBOL();
 
 /*
  * Drag in all requested console types
@@ -67,7 +55,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 REQUIRE_OBJECT ( bios_console );
 #endif
 #ifdef CONSOLE_SERIAL
-REQUIRE_OBJECT ( serial_console );
+REQUIRE_OBJECT ( serial );
 #endif
 #ifdef CONSOLE_DIRECT_VGA
 REQUIRE_OBJECT ( video_subr );
@@ -96,6 +84,9 @@ REQUIRE_OBJECT ( debugcon );
 #ifdef CONSOLE_VESAFB
 REQUIRE_OBJECT ( vesafb );
 #endif
+#ifdef CONSOLE_INT13
+REQUIRE_OBJECT ( int13con );
+#endif
 
 /*
  * Drag in all requested network protocols
@@ -149,6 +140,9 @@ REQUIRE_OBJECT ( slam );
 #ifdef SANBOOT_PROTO_ISCSI
 REQUIRE_OBJECT ( iscsi );
 #endif
+#ifdef SANBOOT_PROTO_HTTP
+REQUIRE_OBJECT ( httpblock );
+#endif
 
 /*
  * Drag in all requested resolvers
@@ -349,6 +343,9 @@ REQUIRE_OBJECT ( cpuid_settings );
 #ifdef MEMMAP_SETTINGS
 REQUIRE_OBJECT ( memmap_settings );
 #endif
+#ifdef VRAM_SETTINGS
+REQUIRE_OBJECT ( vram_settings );
+#endif
 
 /*
  * Drag in selected keyboard map
diff --git a/roms/ipxe/src/config/config_crypto.c b/roms/ipxe/src/config/config_crypto.c
new file mode 100644
index 000000000..1e125d8ab
--- /dev/null
+++ b/roms/ipxe/src/config/config_crypto.c
@@ -0,0 +1,76 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/crypto.h>
+
+/** @file
+ *
+ * Cryptographic configuration
+ *
+ * Cryptographic configuration is slightly messy since we need to drag
+ * in objects based on combinations of build options.
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/* RSA and MD5 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_MD5 )
+REQUIRE_OBJECT ( rsa_md5 );
+#endif
+
+/* RSA and SHA-1 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( rsa_sha1 );
+#endif
+
+/* RSA and SHA-224 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA224 )
+REQUIRE_OBJECT ( rsa_sha224 );
+#endif
+
+/* RSA and SHA-256 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( rsa_sha256 );
+#endif
+
+/* RSA and SHA-384 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( rsa_sha384 );
+#endif
+
+/* RSA and SHA-512 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA512 )
+REQUIRE_OBJECT ( rsa_sha512 );
+#endif
+
+/* RSA, AES-CBC, and SHA-1 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
+    defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( rsa_aes_cbc_sha1 );
+#endif
+
+/* RSA, AES-CBC, and SHA-256 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
+    defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( rsa_aes_cbc_sha256 );
+#endif
diff --git a/roms/ipxe/src/config/config_ethernet.c b/roms/ipxe/src/config/config_ethernet.c
index d13bd6144..de7a07c57 100644
--- a/roms/ipxe/src/config/config_ethernet.c
+++ b/roms/ipxe/src/config/config_ethernet.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in Ethernet-specific protocols
  */
@@ -24,3 +40,6 @@ REQUIRE_OBJECT ( aoe );
 #ifdef NET_PROTO_FCOE
 REQUIRE_OBJECT ( fcoe );
 #endif
+#ifdef NET_PROTO_STP
+REQUIRE_OBJECT ( stp );
+#endif
diff --git a/roms/ipxe/src/config/config_fc.c b/roms/ipxe/src/config/config_fc.c
index 414646994..33fc9462a 100644
--- a/roms/ipxe/src/config/config_fc.c
+++ b/roms/ipxe/src/config/config_fc.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in Fibre Channel-specific commands
  *
diff --git a/roms/ipxe/src/config/config_http.c b/roms/ipxe/src/config/config_http.c
new file mode 100644
index 000000000..3f198d228
--- /dev/null
+++ b/roms/ipxe/src/config/config_http.c
@@ -0,0 +1,45 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/general.h>
+
+/** @file
+ *
+ * HTTP extensions
+ *
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/*
+ * Drag in HTTP extensions
+ */
+#ifdef HTTP_AUTH_BASIC
+REQUIRE_OBJECT ( httpbasic );
+#endif
+#ifdef HTTP_AUTH_DIGEST
+REQUIRE_OBJECT ( httpdigest );
+#endif
+#ifdef HTTP_ENC_PEERDIST
+REQUIRE_OBJECT ( peerdist );
+#endif
diff --git a/roms/ipxe/src/config/config_infiniband.c b/roms/ipxe/src/config/config_infiniband.c
index 432e621d0..a742e7559 100644
--- a/roms/ipxe/src/config/config_infiniband.c
+++ b/roms/ipxe/src/config/config_infiniband.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in Infiniband-specific protocols
  */
diff --git a/roms/ipxe/src/config/config_net80211.c b/roms/ipxe/src/config/config_net80211.c
index b33c363b1..343617548 100644
--- a/roms/ipxe/src/config/config_net80211.c
+++ b/roms/ipxe/src/config/config_net80211.c
@@ -1,8 +1,18 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -15,6 +25,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in 802.11-specific commands
  *
diff --git a/roms/ipxe/src/config/config_romprefix.c b/roms/ipxe/src/config/config_romprefix.c
index 85f1e78ab..21921b867 100644
--- a/roms/ipxe/src/config/config_romprefix.c
+++ b/roms/ipxe/src/config/config_romprefix.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Provide UNDI loader if PXE stack is requested
  *
diff --git a/roms/ipxe/src/config/config_route.c b/roms/ipxe/src/config/config_route.c
index 33e18cdd3..c0b4ee91d 100644
--- a/roms/ipxe/src/config/config_route.c
+++ b/roms/ipxe/src/config/config_route.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in routing management for relevant protocols
  *
diff --git a/roms/ipxe/src/config/config_usb.c b/roms/ipxe/src/config/config_usb.c
new file mode 100644
index 000000000..dc0e6e6af
--- /dev/null
+++ b/roms/ipxe/src/config/config_usb.c
@@ -0,0 +1,52 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/usb.h>
+
+/** @file
+ *
+ * USB configuration options
+ *
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/*
+ * Drag in USB controllers
+ */
+#ifdef USB_HCD_XHCI
+REQUIRE_OBJECT ( xhci );
+#endif
+#ifdef USB_HCD_EHCI
+REQUIRE_OBJECT ( ehci );
+#endif
+#ifdef USB_HCD_UHCI
+REQUIRE_OBJECT ( uhci );
+#endif
+
+/*
+ * Drag in USB peripherals
+ */
+#ifdef USB_KEYBOARD
+REQUIRE_OBJECT ( usbkbd );
+#endif
diff --git a/roms/ipxe/src/config/console.h b/roms/ipxe/src/config/console.h
index 908ec5a0b..ffa5cf50d 100644
--- a/roms/ipxe/src/config/console.h
+++ b/roms/ipxe/src/config/console.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
@@ -23,6 +23,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 //#define	CONSOLE_VMWARE		/* VMware logfile console */
 //#define	CONSOLE_DEBUGCON	/* Debug port console */
 //#define	CONSOLE_VESAFB		/* VESA framebuffer console */
+//#define	CONSOLE_INT13		/* INT13 disk log console */
 
 #define	KEYBOARD_MAP	us
 
diff --git a/roms/ipxe/src/config/crypto.h b/roms/ipxe/src/config/crypto.h
index 1e021b0fb..bccfc04b8 100644
--- a/roms/ipxe/src/config/crypto.h
+++ b/roms/ipxe/src/config/crypto.h
@@ -7,7 +7,40 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** RSA public-key algorithm */
+#define CRYPTO_PUBKEY_RSA
+
+/** AES-CBC block cipher */
+#define CRYPTO_CIPHER_AES_CBC
+
+/** MD5 digest algorithm
+ *
+ * Note that use of MD5 is implicit when using TLSv1.1 or earlier.
+ */
+#define CRYPTO_DIGEST_MD5
+
+/** SHA-1 digest algorithm
+ *
+ * Note that use of SHA-1 is implicit when using TLSv1.1 or earlier.
+ */
+#define CRYPTO_DIGEST_SHA1
+
+/** SHA-224 digest algorithm */
+#define CRYPTO_DIGEST_SHA224
+
+/** SHA-256 digest algorithm
+ *
+ * Note that use of SHA-256 is implicit when using TLSv1.2.
+ */
+#define CRYPTO_DIGEST_SHA256
+
+/** SHA-384 digest algorithm */
+#define CRYPTO_DIGEST_SHA384
+
+/** SHA-512 digest algorithm */
+#define CRYPTO_DIGEST_SHA512
 
 /** Margin of error (in seconds) allowed in signed timestamps
  *
diff --git a/roms/ipxe/src/config/defaults.h b/roms/ipxe/src/config/defaults.h
index 389c0b07b..32d6dbcce 100644
--- a/roms/ipxe/src/config/defaults.h
+++ b/roms/ipxe/src/config/defaults.h
@@ -1,7 +1,7 @@
 #ifndef CONFIG_DEFAULTS_H
 #define CONFIG_DEFAULTS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define CONFIG_DEFAULTS(_platform) <config/defaults/_platform.h>
 
diff --git a/roms/ipxe/src/config/defaults/efi.h b/roms/ipxe/src/config/defaults/efi.h
index 4276d9366..cdf41c54d 100644
--- a/roms/ipxe/src/config/defaults/efi.h
+++ b/roms/ipxe/src/config/defaults/efi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define UACCESS_EFI
 #define IOAPI_X86
@@ -19,8 +19,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define SMBIOS_EFI
 #define SANBOOT_NULL
 #define BOFM_EFI
-#define ENTROPY_NULL
-#define TIME_NULL
+#define ENTROPY_EFI
+#define TIME_EFI
 #define REBOOT_EFI
 
 #define	IMAGE_EFI		/* EFI image support */
diff --git a/roms/ipxe/src/config/defaults/pcbios.h b/roms/ipxe/src/config/defaults/pcbios.h
index 7debc8d2f..3ed8343ce 100644
--- a/roms/ipxe/src/config/defaults/pcbios.h
+++ b/roms/ipxe/src/config/defaults/pcbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define UACCESS_LIBRM
 #define IOAPI_X86
@@ -35,6 +35,12 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define	SANBOOT_PROTO_AOE	/* AoE protocol */
 #define	SANBOOT_PROTO_IB_SRP	/* Infiniband SCSI RDMA protocol */
 #define	SANBOOT_PROTO_FCP	/* Fibre Channel protocol */
+#define SANBOOT_PROTO_HTTP	/* HTTP SAN protocol */
+
+#define	USB_HCD_XHCI		/* xHCI USB host controller */
+#define	USB_HCD_EHCI		/* EHCI USB host controller */
+#define	USB_HCD_UHCI		/* UHCI USB host controller */
+#define	USB_KEYBOARD		/* USB keyboards */
 
 #define	REBOOT_CMD		/* Reboot command */
 #define	CPUID_CMD		/* x86 CPU feature detection command */
diff --git a/roms/ipxe/src/config/dhcp.h b/roms/ipxe/src/config/dhcp.h
new file mode 100644
index 000000000..49fe16b92
--- /dev/null
+++ b/roms/ipxe/src/config/dhcp.h
@@ -0,0 +1,87 @@
+#ifndef CONFIG_DHCP_H
+#define CONFIG_DHCP_H
+
+/** @file
+ *
+ * DHCP configuration
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/*
+ * DHCP and PXE Boot Server timeout parameters
+ *
+ * Initial and final timeout for DHCP discovery
+ *
+ * The PXE spec indicates discover request are sent 4 times, with
+ * timeouts of 4, 8, 16, 32 seconds.  iPXE by default uses 1, 2, 4, 8.
+ */
+#define DHCP_DISC_START_TIMEOUT_SEC	1
+#define DHCP_DISC_END_TIMEOUT_SEC	10
+//#define DHCP_DISC_START_TIMEOUT_SEC	4	/* as per PXE spec */
+//#define DHCP_DISC_END_TIMEOUT_SEC	32	/* as per PXE spec */
+
+/*
+ * ProxyDHCP offers are given precedence by continue to wait for them
+ * after a valid DHCPOFFER is received.  We'll wait through this
+ * timeout for it.  The PXE spec indicates waiting through the 4 & 8
+ * second timeouts, iPXE by default stops after 2.
+ */
+#define DHCP_DISC_PROXY_TIMEOUT_SEC	2
+//#define DHCP_DISC_PROXY_TIMEOUT_SEC	11	/* as per PXE spec */
+
+/*
+ * Per the PXE spec, requests are also tried 4 times, but at timeout
+ * intervals of 1, 2, 3, 4 seconds.  To adapt this to an exponential
+ * backoff timer, we can either do 1, 2, 4, 8, ie. 4 retires with a
+ * longer interval or start at 0 (0.25s) for 0.25, 0.5, 1, 2, 4,
+ * ie. one extra try and shorter initial timeouts.  iPXE by default
+ * does a combination of both, starting at 0 and going through the 8
+ * second timeout.
+ */
+#define DHCP_REQ_START_TIMEOUT_SEC	0
+#define DHCP_REQ_END_TIMEOUT_SEC	10
+//#define DHCP_REQ_END_TIMEOUT_SEC	4	/* as per PXE spec */
+
+/*
+ * A ProxyDHCP offer without PXE options also goes through a request
+ * phase using these same parameters, but note the early break below.
+ */
+#define DHCP_PROXY_START_TIMEOUT_SEC	0
+#define DHCP_PROXY_END_TIMEOUT_SEC	10
+//#define DHCP_PROXY_END_TIMEOUT_SEC	8	/* as per PXE spec */
+
+/*
+ * A ProxyDHCP request timeout should not induce a failure condition,
+ * so we always want to break before the above set of timers expire.
+ * The iPXE default value of 2 breaks at the first timeout after 2
+ * seconds, which will be after the 2 second timeout.
+ */
+#define DHCP_REQ_PROXY_TIMEOUT_SEC	2
+//#define DHCP_REQ_PROXY_TIMEOUT_SEC	7	/* as per PXE spec */
+
+/*
+ * Per the PXE spec, a PXE boot server request is also be retried 4
+ * times at timeouts of 1, 2, 3, 4.  iPXE uses the same timeouts as
+ * discovery, 1, 2, 4, 8, but will move on to the next server if
+ * available after an elapsed time greater than 3 seconds, therefore
+ * effectively only sending 3 tries at timeouts of 1, 2, 4.
+ */
+#define PXEBS_START_TIMEOUT_SEC		1
+#define PXEBS_END_TIMEOUT_SEC		10
+//#define PXEBS_START_TIMEOUT_SEC	0	/* as per PXE spec */
+//#define PXEBS_END_TIMEOUT_SEC		8	/* as per PXE spec */
+
+/*
+ * Increment to the next PXE Boot server, if available, after this
+ * this much time has elapsed.
+ */
+#define PXEBS_MAX_TIMEOUT_SEC		3
+//#define PXEBS_MAX_TIMEOUT_SEC		7	/* as per PXE spec */
+
+#include <config/local/dhcp.h>
+
+#endif /* CONFIG_DHCP_H */
diff --git a/roms/ipxe/src/config/entropy.h b/roms/ipxe/src/config/entropy.h
index 7de2f6737..c79060fd5 100644
--- a/roms/ipxe/src/config/entropy.h
+++ b/roms/ipxe/src/config/entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/roms/ipxe/src/config/fault.h b/roms/ipxe/src/config/fault.h
new file mode 100644
index 000000000..5024a8ff3
--- /dev/null
+++ b/roms/ipxe/src/config/fault.h
@@ -0,0 +1,34 @@
+#ifndef CONFIG_FAULT_H
+#define CONFIG_FAULT_H
+
+/** @file
+ *
+ * Fault injection
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/* Drop every N transmitted or received network packets */
+#define	NETDEV_DISCARD_RATE 0
+
+/* Drop every N transmitted or received PeerDist discovery packets */
+#define PEERDISC_DISCARD_RATE 0
+
+/* Annul every N PeerDist download attempts */
+#define PEERBLK_ANNUL_RATE 0
+
+/* Stall every N PeerDist download attempts */
+#define PEERBLK_STALL_RATE 0
+
+/* Abort every N PeerDist download attempts */
+#define PEERBLK_ABORT_RATE 0
+
+/* Corrupt every N received PeerDist packets */
+#define PEERBLK_CORRUPT_RATE 0
+
+#include <config/local/fault.h>
+
+#endif /* CONFIG_FAULT_H */
diff --git a/roms/ipxe/src/config/general.h b/roms/ipxe/src/config/general.h
index 539203457..ee15f6bf1 100644
--- a/roms/ipxe/src/config/general.h
+++ b/roms/ipxe/src/config/general.h
@@ -7,27 +7,11 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
 /*
- * Branding
- *
- * Vendors may use these strings to add their own branding to iPXE.
- * PRODUCT_NAME is displayed prior to any iPXE branding in startup
- * messages, and PRODUCT_SHORT_NAME is used where a brief product
- * label is required (e.g. in BIOS boot selection menus).
- *
- * To minimise end-user confusion, it's probably a good idea to either
- * make PRODUCT_SHORT_NAME a substring of PRODUCT_NAME or leave it as
- * "iPXE".
- *
- */
-#define PRODUCT_NAME ""
-#define PRODUCT_SHORT_NAME "iPXE"
-
-/*
  * Banner timeout configuration
  *
  * This controls the timeout for the "Press Ctrl-B for the iPXE
@@ -53,6 +37,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define	NET_PROTO_IPV4		/* IPv4 protocol */
 #undef	NET_PROTO_IPV6		/* IPv6 protocol */
 #undef	NET_PROTO_FCOE		/* Fibre Channel over Ethernet protocol */
+#define	NET_PROTO_STP		/* Spanning Tree protocol */
 
 /*
  * PXE support
@@ -82,6 +67,15 @@ FILE_LICENCE ( GPL2_OR_LATER );
 //#undef	SANBOOT_PROTO_AOE	/* AoE protocol */
 //#undef	SANBOOT_PROTO_IB_SRP	/* Infiniband SCSI RDMA protocol */
 //#undef	SANBOOT_PROTO_FCP	/* Fibre Channel protocol */
+//#undef	SANBOOT_PROTO_HTTP	/* HTTP SAN protocol */
+
+/*
+ * HTTP extensions
+ *
+ */
+#define HTTP_AUTH_BASIC		/* Basic authentication */
+#define HTTP_AUTH_DIGEST	/* Digest authentication */
+//#define HTTP_ENC_PEERDIST	/* PeerDist content encoding */
 
 /*
  * 802.11 cryptosystems and handshaking protocols
@@ -156,6 +150,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 #undef	NONPNP_HOOK_INT19	/* Hook INT19 on non-PnP BIOSes */
+#define	AUTOBOOT_ROM_FILTER	/* Autoboot only devices matching our ROM */
 
 /*
  * Error message tables to include
@@ -170,7 +165,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#define	NETDEV_DISCARD_RATE 0	/* Drop every N packets (0=>no drop) */
 #undef	BUILD_SERIAL		/* Include an automatic build serial
 				 * number.  Add "bs" to the list of
 				 * make targets.  For example:
@@ -181,6 +175,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #undef	GDBSERIAL		/* Remote GDB debugging over serial */
 #undef	GDBUDP			/* Remote GDB debugging over UDP
 				 * (both may be set) */
+//#define EFI_DOWNGRADE_UX	/* Downgrade UEFI user experience */
 
 #include <config/named.h>
 #include NAMED_CONFIG(general.h)
diff --git a/roms/ipxe/src/config/ioapi.h b/roms/ipxe/src/config/ioapi.h
index ce19c6d71..abe5a50ce 100644
--- a/roms/ipxe/src/config/ioapi.h
+++ b/roms/ipxe/src/config/ioapi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/roms/ipxe/src/config/named.h b/roms/ipxe/src/config/named.h
index 36efdabdd..ddde6f0a6 100644
--- a/roms/ipxe/src/config/named.h
+++ b/roms/ipxe/src/config/named.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* config/<name>/<header>.h */
 #ifdef CONFIG
diff --git a/roms/ipxe/src/config/nap.h b/roms/ipxe/src/config/nap.h
index 187af4289..e4fe97964 100644
--- a/roms/ipxe/src/config/nap.h
+++ b/roms/ipxe/src/config/nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/tests/qapi-schema/alternate-good.err b/roms/ipxe/src/config/qemu/colour.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/alternate-good.err
+++ b/roms/ipxe/src/config/qemu/colour.h
diff --git a/tests/qapi-schema/data-array-empty.out b/roms/ipxe/src/config/qemu/console.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/data-array-empty.out
+++ b/roms/ipxe/src/config/qemu/console.h
diff --git a/tests/qapi-schema/data-array-unknown.out b/roms/ipxe/src/config/qemu/crypto.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/data-array-unknown.out
+++ b/roms/ipxe/src/config/qemu/crypto.h
diff --git a/roms/ipxe/src/config/qemu/general.h b/roms/ipxe/src/config/qemu/general.h
new file mode 100644
index 000000000..30f60d3f7
--- /dev/null
+++ b/roms/ipxe/src/config/qemu/general.h
@@ -0,0 +1,10 @@
+/* Disable entry during POST */
+#undef ROM_BANNER_TIMEOUT
+#define ROM_BANNER_TIMEOUT 0
+
+/* Extend banner timeout */
+#undef BANNER_TIMEOUT
+#define BANNER_TIMEOUT 30
+
+/* Work around missing EFI_PXE_BASE_CODE_PROTOCOL */
+#define EFI_DOWNGRADE_UX
diff --git a/tests/qapi-schema/data-int.out b/roms/ipxe/src/config/qemu/serial.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/data-int.out
+++ b/roms/ipxe/src/config/qemu/serial.h
diff --git a/tests/qapi-schema/data-member-array-bad.out b/roms/ipxe/src/config/qemu/settings.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/data-member-array-bad.out
+++ b/roms/ipxe/src/config/qemu/settings.h
diff --git a/tests/qapi-schema/data-member-array.err b/roms/ipxe/src/config/qemu/sideband.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/data-member-array.err
+++ b/roms/ipxe/src/config/qemu/sideband.h
diff --git a/tests/qapi-schema/data-member-unknown.out b/roms/ipxe/src/config/qemu/usb.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/data-member-unknown.out
+++ b/roms/ipxe/src/config/qemu/usb.h
diff --git a/roms/ipxe/src/config/reboot.h b/roms/ipxe/src/config/reboot.h
index 240ef87be..2d1648e7b 100644
--- a/roms/ipxe/src/config/reboot.h
+++ b/roms/ipxe/src/config/reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/roms/ipxe/src/config/sanboot.h b/roms/ipxe/src/config/sanboot.h
index 1d7f5f177..ccc4bda1f 100644
--- a/roms/ipxe/src/config/sanboot.h
+++ b/roms/ipxe/src/config/sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/roms/ipxe/src/config/serial.h b/roms/ipxe/src/config/serial.h
index 08368efdb..27040dc54 100644
--- a/roms/ipxe/src/config/serial.h
+++ b/roms/ipxe/src/config/serial.h
@@ -13,11 +13,6 @@
 
 FILE_LICENCE ( GPL2_OR_LATER );
 
-#define	COM1		0x3f8
-#define	COM2		0x2f8
-#define	COM3		0x3e8
-#define	COM4		0x2e8
-
 #define	COMCONSOLE	COM1		/* I/O port address */
 
 /* Keep settings from a previous user of the serial port (e.g. lilo or
diff --git a/roms/ipxe/src/config/settings.h b/roms/ipxe/src/config/settings.h
index 42fe9cc81..01feaaa87 100644
--- a/roms/ipxe/src/config/settings.h
+++ b/roms/ipxe/src/config/settings.h
@@ -7,12 +7,13 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define	PCI_SETTINGS	/* PCI device settings */
 //#define	CPUID_SETTINGS	/* CPUID settings */
 //#define	MEMMAP_SETTINGS	/* Memory map settings */
 //#define	VMWARE_SETTINGS	/* VMware GuestInfo settings */
+//#define	VRAM_SETTINGS	/* Video RAM dump settings */
 
 #include <config/named.h>
 #include NAMED_CONFIG(settings.h)
diff --git a/roms/ipxe/src/config/sideband.h b/roms/ipxe/src/config/sideband.h
index 039bb5d09..dd704f9bb 100644
--- a/roms/ipxe/src/config/sideband.h
+++ b/roms/ipxe/src/config/sideband.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 //#define	CONFIG_BOFM	/* IBM's BladeCenter Open Fabric Manager */
 
diff --git a/roms/ipxe/src/config/time.h b/roms/ipxe/src/config/time.h
index 0576211fd..678f6f864 100644
--- a/roms/ipxe/src/config/time.h
+++ b/roms/ipxe/src/config/time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/roms/ipxe/src/config/timer.h b/roms/ipxe/src/config/timer.h
index abd669851..5a54d398c 100644
--- a/roms/ipxe/src/config/timer.h
+++ b/roms/ipxe/src/config/timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/roms/ipxe/src/config/umalloc.h b/roms/ipxe/src/config/umalloc.h
index 245c6b4aa..832dd21d1 100644
--- a/roms/ipxe/src/config/umalloc.h
+++ b/roms/ipxe/src/config/umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/roms/ipxe/src/config/usb.h b/roms/ipxe/src/config/usb.h
new file mode 100644
index 000000000..52e82eaad
--- /dev/null
+++ b/roms/ipxe/src/config/usb.h
@@ -0,0 +1,33 @@
+#ifndef CONFIG_USB_H
+#define CONFIG_USB_H
+
+/** @file
+ *
+ * USB configuration
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/*
+ * USB host controllers (all enabled by default)
+ *
+ */
+//#undef	USB_HCD_XHCI		/* xHCI USB host controller */
+//#undef	USB_HCD_EHCI		/* EHCI USB host controller */
+//#undef	USB_HCD_UHCI		/* UHCI USB host controller */
+
+/*
+ * USB peripherals
+ *
+ */
+//#undef	USB_KEYBOARD		/* USB keyboards */
+
+#include <config/named.h>
+#include NAMED_CONFIG(usb.h)
+#include <config/local/usb.h>
+#include LOCAL_NAMED_CONFIG(usb.h)
+
+#endif /* CONFIG_USB_H */
diff --git a/roms/ipxe/src/config/vbox/general.h b/roms/ipxe/src/config/vbox/general.h
index 27d15daf2..06b45f1a8 100644
--- a/roms/ipxe/src/config/vbox/general.h
+++ b/roms/ipxe/src/config/vbox/general.h
@@ -1,25 +1,17 @@
 /* Disabled from config/defaults/pcbios.h */
 
-#undef IMAGE_ELF
 #undef SANBOOT_PROTO_ISCSI
 #undef SANBOOT_PROTO_AOE
 #undef SANBOOT_PROTO_IB_SRP
 #undef SANBOOT_PROTO_FCP
-#undef REBOOT_CMD
-#undef CPUID_CMD
 
 /* Disabled from config/general.h */
 
-#undef DOWNLOAD_PROTO_HTTP
 #undef CRYPTO_80211_WEP
 #undef CRYPTO_80211_WPA
 #undef CRYPTO_80211_WPA2
 #undef IWMGMT_CMD
-#undef FCMGMT_CMD
-#undef SANBOOT_CMD
 #undef MENU_CMD
-#undef LOGIN_CMD
-#undef SYNC_CMD
 
 /* Ensure ROM banner is not displayed */
 
diff --git a/tests/qapi-schema/data-unknown.out b/roms/ipxe/src/config/vbox/usb.h
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/data-unknown.out
+++ b/roms/ipxe/src/config/vbox/usb.h
diff --git a/roms/ipxe/src/core/acpi.c b/roms/ipxe/src/core/acpi.c
index 330f50631..b0ccfa78d 100644
--- a/roms/ipxe/src/core/acpi.c
+++ b/roms/ipxe/src/core/acpi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/acpi.h>
diff --git a/roms/ipxe/src/core/ansicol.c b/roms/ipxe/src/core/ansicol.c
index 142a00f8d..ddf9ba77c 100644
--- a/roms/ipxe/src/core/ansicol.c
+++ b/roms/ipxe/src/core/ansicol.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/core/ansicoldef.c b/roms/ipxe/src/core/ansicoldef.c
index dd89f3b70..6d8598e11 100644
--- a/roms/ipxe/src/core/ansicoldef.c
+++ b/roms/ipxe/src/core/ansicoldef.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/core/ansiesc.c b/roms/ipxe/src/core/ansiesc.c
index ca9a73ce0..7f545db0e 100644
--- a/roms/ipxe/src/core/ansiesc.c
+++ b/roms/ipxe/src/core/ansiesc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <assert.h>
diff --git a/roms/ipxe/src/core/asprintf.c b/roms/ipxe/src/core/asprintf.c
index 03cf45cfc..00edf8e11 100644
--- a/roms/ipxe/src/core/asprintf.c
+++ b/roms/ipxe/src/core/asprintf.c
@@ -4,7 +4,7 @@
 #include <stdio.h>
 #include <errno.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Write a formatted string to newly allocated memory.
diff --git a/roms/ipxe/src/core/assert.c b/roms/ipxe/src/core/assert.c
index 0791ea7b9..294e766be 100644
--- a/roms/ipxe/src/core/assert.c
+++ b/roms/ipxe/src/core/assert.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/base16.c b/roms/ipxe/src/core/base16.c
index bf9cc21bb..f9e0f3364 100644
--- a/roms/ipxe/src/core/base16.c
+++ b/roms/ipxe/src/core/base16.c
@@ -15,14 +15,20 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
-#include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
+#include <ipxe/string.h>
+#include <ipxe/vsprintf.h>
 #include <ipxe/base16.h>
 
 /** @file
@@ -32,48 +38,42 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /**
- * Base16-encode data
+ * Encode hexadecimal string (with optional byte separator character)
  *
+ * @v separator		Byte separator character, or 0 for no separator
  * @v raw		Raw data
- * @v len		Length of raw data
- * @v encoded		Buffer for encoded string
- *
- * The buffer must be the correct length for the encoded string.  Use
- * something like
- *
- *     char buf[ base16_encoded_len ( len ) + 1 ];
- *
- * (the +1 is for the terminating NUL) to provide a buffer of the
- * correct size.
+ * @v raw_len		Length of raw data
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Encoded length
  */
-void base16_encode ( const uint8_t *raw, size_t len, char *encoded ) {
-	const uint8_t *raw_bytes = raw;
-	char *encoded_bytes = encoded;
-	size_t remaining = len;
-
-	/* Encode each byte */
-	for ( ; remaining-- ; encoded_bytes += 2 ) {
-		sprintf ( encoded_bytes, "%02x", *(raw_bytes++) );
+size_t hex_encode ( char separator, const void *raw, size_t raw_len,
+		    char *data, size_t len ) {
+	const uint8_t *bytes = raw;
+	const char delimiter[2] = { separator, '\0' };
+	size_t used = 0;
+	unsigned int i;
+
+	if ( len )
+		data[0] = 0; /* Ensure that a terminating NUL exists */
+	for ( i = 0 ; i < raw_len ; i++ ) {
+		used += ssnprintf ( ( data + used ), ( len - used ),
+				    "%s%02x", ( used ? delimiter : "" ),
+				    bytes[i] );
 	}
-
-	/* Ensure terminating NUL exists even if length was zero */
-	*encoded_bytes = '\0';
-
-	DBG ( "Base16-encoded to \"%s\":\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( strlen ( encoded ) == base16_encoded_len ( len ) );
+	return used;
 }
 
 /**
- * Decode hexadecimal string
+ * Decode hexadecimal string (with optional byte separator character)
  *
- * @v encoded		Encoded string
  * @v separator		Byte separator character, or 0 for no separator
+ * @v encoded		Encoded string
  * @v data		Buffer
  * @v len		Length of buffer
  * @ret len		Length of data, or negative error
  */
-int hex_decode ( const char *encoded, char separator, void *data, size_t len ) {
+int hex_decode ( char separator, const char *encoded, void *data, size_t len ) {
 	uint8_t *out = data;
 	unsigned int count = 0;
 	unsigned int sixteens;
@@ -87,13 +87,13 @@ int hex_decode ( const char *encoded, char separator, void *data, size_t len ) {
 
 		/* Extract digits.  Note that either digit may be NUL,
 		 * which would be interpreted as an invalid value by
-		 * strtoul_charval(); there is therefore no need for an
+		 * digit_value(); there is therefore no need for an
 		 * explicit end-of-string check.
 		 */
-		sixteens = strtoul_charval ( *(encoded++) );
+		sixteens = digit_value ( *(encoded++) );
 		if ( sixteens >= 16 )
 			return -EINVAL;
-		units = strtoul_charval ( *(encoded++) );
+		units = digit_value ( *(encoded++) );
 		if ( units >= 16 )
 			return -EINVAL;
 
@@ -105,31 +105,3 @@ int hex_decode ( const char *encoded, char separator, void *data, size_t len ) {
 	}
 	return count;
 }
-
-/**
- * Base16-decode data
- *
- * @v encoded		Encoded string
- * @v raw		Raw data
- * @ret len		Length of raw data, or negative error
- *
- * The buffer must be large enough to contain the decoded data.  Use
- * something like
- *
- *     char buf[ base16_decoded_max_len ( encoded ) ];
- *
- * to provide a buffer of the correct size.
- */
-int base16_decode ( const char *encoded, uint8_t *raw ) {
-	int len;
-
-	len = hex_decode ( encoded, 0, raw, -1UL );
-	if ( len < 0 )
-		return len;
-
-	DBG ( "Base16-decoded \"%s\" to:\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( len <= ( int ) base16_decoded_max_len ( encoded ) );
-
-	return len;
-}
diff --git a/roms/ipxe/src/core/base64.c b/roms/ipxe/src/core/base64.c
index bdaf70957..e452f7d41 100644
--- a/roms/ipxe/src/core/base64.c
+++ b/roms/ipxe/src/core/base64.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -39,80 +43,73 @@ static const char base64[64] =
  * Base64-encode data
  *
  * @v raw		Raw data
- * @v len		Length of raw data
- * @v encoded		Buffer for encoded string
- *
- * The buffer must be the correct length for the encoded string.  Use
- * something like
- *
- *     char buf[ base64_encoded_len ( len ) + 1 ];
- *
- * (the +1 is for the terminating NUL) to provide a buffer of the
- * correct size.
+ * @v raw_len		Length of raw data
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Encoded length
  */
-void base64_encode ( const uint8_t *raw, size_t len, char *encoded ) {
+size_t base64_encode ( const void *raw, size_t raw_len, char *data,
+		       size_t len ) {
 	const uint8_t *raw_bytes = ( ( const uint8_t * ) raw );
-	uint8_t *encoded_bytes = ( ( uint8_t * ) encoded );
-	size_t raw_bit_len = ( 8 * len );
+	size_t raw_bit_len = ( 8 * raw_len );
+	size_t used = 0;
 	unsigned int bit;
 	unsigned int byte;
 	unsigned int shift;
 	unsigned int tmp;
 
-	for ( bit = 0 ; bit < raw_bit_len ; bit += 6 ) {
+	for ( bit = 0 ; bit < raw_bit_len ; bit += 6, used++ ) {
 		byte = ( bit / 8 );
 		shift = ( bit % 8 );
 		tmp = ( raw_bytes[byte] << shift );
-		if ( ( byte + 1 ) < len )
+		if ( ( byte + 1 ) < raw_len )
 			tmp |= ( raw_bytes[ byte + 1 ] >> ( 8 - shift ) );
 		tmp = ( ( tmp >> 2 ) & 0x3f );
-		*(encoded_bytes++) = base64[tmp];
+		if ( used < len )
+			data[used] = base64[tmp];
+	}
+	for ( ; ( bit % 8 ) != 0 ; bit += 6, used++ ) {
+		if ( used < len )
+			data[used] = '=';
 	}
-	for ( ; ( bit % 8 ) != 0 ; bit += 6 )
-		*(encoded_bytes++) = '=';
-	*(encoded_bytes++) = '\0';
+	if ( used < len )
+		data[used] = '\0';
+	if ( len )
+		data[ len - 1 ] = '\0'; /* Ensure terminator exists */
 
-	DBG ( "Base64-encoded to \"%s\":\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( strlen ( encoded ) == base64_encoded_len ( len ) );
+	return used;
 }
 
 /**
  * Base64-decode string
  *
  * @v encoded		Encoded string
- * @v raw		Raw data
- * @ret len		Length of raw data, or negative error
- *
- * The buffer must be large enough to contain the decoded data.  Use
- * something like
- *
- *     char buf[ base64_decoded_max_len ( encoded ) ];
- *
- * to provide a buffer of the correct size.
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of data, or negative error
  */
-int base64_decode ( const char *encoded, uint8_t *raw ) {
-	const uint8_t *encoded_bytes = ( ( const uint8_t * ) encoded );
-	uint8_t *raw_bytes = ( ( uint8_t * ) raw );
-	uint8_t encoded_byte;
+int base64_decode ( const char *encoded, void *data, size_t len ) {
+	const char *in = encoded;
+	uint8_t *out = data;
+	uint8_t in_char;
 	char *match;
-	int decoded;
+	int in_bits;
 	unsigned int bit = 0;
 	unsigned int pad_count = 0;
-	size_t len;
+	size_t offset;
 
-	/* Zero the raw data */
-	memset ( raw, 0, base64_decoded_max_len ( encoded ) );
+	/* Zero the output buffer */
+	memset ( data, 0, len );
 
 	/* Decode string */
-	while ( ( encoded_byte = *(encoded_bytes++) ) ) {
+	while ( ( in_char = *(in++) ) ) {
 
 		/* Ignore whitespace characters */
-		if ( isspace ( encoded_byte ) )
+		if ( isspace ( in_char ) )
 			continue;
 
 		/* Process pad characters */
-		if ( encoded_byte == '=' ) {
+		if ( in_char == '=' ) {
 			if ( pad_count >= 2 ) {
 				DBG ( "Base64-encoded string \"%s\" has too "
 				      "many pad characters\n", encoded );
@@ -129,18 +126,22 @@ int base64_decode ( const char *encoded, uint8_t *raw ) {
 		}
 
 		/* Process normal characters */
-		match = strchr ( base64, encoded_byte );
+		match = strchr ( base64, in_char );
 		if ( ! match ) {
 			DBG ( "Base64-encoded string \"%s\" contains invalid "
-			      "character '%c'\n", encoded, encoded_byte );
+			      "character '%c'\n", encoded, in_char );
 			return -EINVAL;
 		}
-		decoded = ( match - base64 );
+		in_bits = ( match - base64 );
 
 		/* Add to raw data */
-		decoded <<= 2;
-		raw_bytes[ bit / 8 ] |= ( decoded >> ( bit % 8 ) );
-		raw_bytes[ bit / 8 + 1 ] |= ( decoded << ( 8 - ( bit % 8 ) ) );
+		in_bits <<= 2;
+		offset = ( bit / 8 );
+		if ( offset < len )
+			out[offset] |= ( in_bits >> ( bit % 8 ) );
+		offset++;
+		if ( offset < len )
+			out[offset] |= ( in_bits << ( 8 - ( bit % 8 ) ) );
 		bit += 6;
 	}
 
@@ -150,12 +151,7 @@ int base64_decode ( const char *encoded, uint8_t *raw ) {
 		      "%d\n", encoded, bit );
 		return -EINVAL;
 	}
-	len = ( bit / 8 );
-
-	DBG ( "Base64-decoded \"%s\" to:\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( len <= base64_decoded_max_len ( encoded ) );
 
 	/* Return length in bytes */
-	return ( len );
+	return ( bit / 8 );
 }
diff --git a/roms/ipxe/src/core/basename.c b/roms/ipxe/src/core/basename.c
index b534a7886..f4f929517 100644
--- a/roms/ipxe/src/core/basename.c
+++ b/roms/ipxe/src/core/basename.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/core/bitmap.c b/roms/ipxe/src/core/bitmap.c
index 0d1152327..2aac33870 100644
--- a/roms/ipxe/src/core/bitmap.c
+++ b/roms/ipxe/src/core/bitmap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/bitmap.h>
diff --git a/roms/ipxe/src/core/blockdev.c b/roms/ipxe/src/core/blockdev.c
index 9d118cb2f..c219d9673 100644
--- a/roms/ipxe/src/core/blockdev.c
+++ b/roms/ipxe/src/core/blockdev.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/interface.h>
diff --git a/roms/ipxe/src/core/blocktrans.c b/roms/ipxe/src/core/blocktrans.c
new file mode 100644
index 000000000..3f32f9cf8
--- /dev/null
+++ b/roms/ipxe/src/core/blocktrans.c
@@ -0,0 +1,261 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Block device translator
+ *
+ */
+
+#include <stdlib.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/xfer.h>
+#include <ipxe/blockdev.h>
+#include <ipxe/blocktrans.h>
+
+/**
+ * Reallocate block device translator data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v len		New length (or zero to free buffer)
+ * @ret rc		Return status code
+ */
+static int blktrans_xferbuf_realloc ( struct xfer_buffer *xferbuf,
+				      size_t len ) {
+	struct block_translator *blktrans =
+		container_of ( xferbuf, struct block_translator, xferbuf );
+
+	/* Record length, if applicable */
+	if ( blktrans->buffer ) {
+
+		/* We have a (non-reallocatable) data buffer */
+		return -ENOTSUP;
+
+	} else {
+
+		/* Record length (for block device capacity) */
+		xferbuf->len = len;
+		return 0;
+	}
+}
+
+/**
+ * Write data to block device translator data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to copy
+ * @v len		Length of data
+ */
+static void blktrans_xferbuf_write ( struct xfer_buffer *xferbuf, size_t offset,
+				     const void *data, size_t len ) {
+	struct block_translator *blktrans =
+		container_of ( xferbuf, struct block_translator, xferbuf );
+
+	/* Write data to buffer, if applicable */
+	if ( blktrans->buffer ) {
+
+		/* Write data to buffer */
+		copy_to_user ( blktrans->buffer, offset, data, len );
+
+	} else {
+
+		/* Sanity check */
+		assert ( len == 0 );
+	}
+}
+
+/**
+ * Read data from block device translator data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to read
+ * @v len		Length of data
+ */
+static void blktrans_xferbuf_read ( struct xfer_buffer *xferbuf, size_t offset,
+				    void *data, size_t len ) {
+	struct block_translator *blktrans =
+		container_of ( xferbuf, struct block_translator, xferbuf );
+
+	/* Read data from buffer, if applicable */
+	if ( blktrans->buffer ) {
+
+		/* Read data from buffer */
+		copy_from_user ( data, blktrans->buffer, offset, len );
+
+	} else {
+
+		/* Sanity check */
+		assert ( len == 0 );
+	}
+}
+
+/** Block device translator data transfer buffer operations */
+static struct xfer_buffer_operations blktrans_xferbuf_operations = {
+	.realloc = blktrans_xferbuf_realloc,
+	.write = blktrans_xferbuf_write,
+	.read = blktrans_xferbuf_read,
+};
+
+/**
+ * Close block device translator
+ *
+ * @v blktrans		Block device translator
+ * @v rc		Reason for close
+ */
+static void blktrans_close ( struct block_translator *blktrans, int rc ) {
+	struct block_device_capacity capacity;
+
+	/* Report block device capacity, if applicable */
+	if ( ( rc == 0 ) && ( blktrans->blksize ) ) {
+
+		/* Construct block device capacity */
+		capacity.blocks =
+			( blktrans->xferbuf.len / blktrans->blksize );
+		capacity.blksize = blktrans->blksize;
+		capacity.max_count = -1U;
+
+		/* Report block device capacity */
+		block_capacity ( &blktrans->block, &capacity );
+	}
+
+	/* Shut down interfaces */
+	intf_shutdown ( &blktrans->xfer, rc );
+	intf_shutdown ( &blktrans->block, rc );
+}
+
+/**
+ * Deliver data
+ *
+ * @v blktrans		Block device translator
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int blktrans_deliver ( struct block_translator *blktrans,
+			      struct io_buffer *iobuf,
+			      struct xfer_metadata *meta ) {
+	int rc;
+
+	/* Deliver to buffer */
+	if ( ( rc = xferbuf_deliver ( &blktrans->xferbuf, iob_disown ( iobuf ),
+				      meta ) ) != 0 ) {
+		DBGC ( blktrans, "BLKTRANS %p could not deliver: %s\n",
+		       blktrans, strerror ( rc ) );
+		goto err;
+	}
+
+	return 0;
+
+ err:
+	blktrans_close ( blktrans, rc );
+	return rc;
+}
+
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v blktrans		Block device translator
+ * @ret xferbuf		Data transfer buffer
+ */
+static struct xfer_buffer *
+blktrans_buffer ( struct block_translator *blktrans ) {
+
+	return &blktrans->xferbuf;
+}
+
+/** Block device translator block device interface operations */
+static struct interface_operation blktrans_block_operations[] = {
+	INTF_OP ( intf_close, struct block_translator *, blktrans_close ),
+};
+
+/** Block device translator block device interface descriptor */
+static struct interface_descriptor blktrans_block_desc =
+	INTF_DESC_PASSTHRU ( struct block_translator, block,
+			     blktrans_block_operations, xfer );
+
+/** Block device translator data transfer interface operations */
+static struct interface_operation blktrans_xfer_operations[] = {
+	INTF_OP ( xfer_deliver, struct block_translator *, blktrans_deliver ),
+	INTF_OP ( xfer_buffer, struct block_translator *, blktrans_buffer ),
+	INTF_OP ( intf_close, struct block_translator *, blktrans_close ),
+};
+
+/** Block device translator data transfer interface descriptor */
+static struct interface_descriptor blktrans_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct block_translator, xfer,
+			     blktrans_xfer_operations, block );
+
+/**
+ * Insert block device translator
+ *
+ * @v block		Block device interface
+ * @v buffer		Data buffer (or UNULL)
+ * @v size		Length of data buffer, or block size
+ * @ret rc		Return status code
+ */
+int block_translate ( struct interface *block, userptr_t buffer, size_t size ) {
+	struct block_translator *blktrans;
+	int rc;
+
+	/* Allocate and initialise structure */
+	blktrans = zalloc ( sizeof ( *blktrans ) );
+	if ( ! blktrans ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	ref_init ( &blktrans->refcnt, NULL );
+	intf_init ( &blktrans->block, &blktrans_block_desc, &blktrans->refcnt );
+	intf_init ( &blktrans->xfer, &blktrans_xfer_desc, &blktrans->refcnt );
+	blktrans->xferbuf.op = &blktrans_xferbuf_operations;
+	blktrans->buffer = buffer;
+	if ( buffer ) {
+		blktrans->xferbuf.len = size;
+	} else {
+		blktrans->blksize = size;
+	}
+
+	/* Attach to interfaces, mortalise self, and return */
+	assert ( block->dest != &null_intf );
+	intf_plug_plug ( &blktrans->xfer, block->dest );
+	intf_plug_plug ( &blktrans->block, block );
+	ref_put ( &blktrans->refcnt );
+
+	DBGC2 ( blktrans, "BLKTRANS %p created", blktrans );
+	if ( buffer ) {
+		DBGC2 ( blktrans, " for %#lx+%#zx",
+			user_to_phys ( buffer, 0 ), size );
+	}
+	DBGC2 ( blktrans, "\n" );
+	return 0;
+
+	ref_put ( &blktrans->refcnt );
+ err_alloc:
+	return rc;
+}
diff --git a/roms/ipxe/src/core/console.c b/roms/ipxe/src/core/console.c
index 141d8f0f0..7fd00036f 100644
--- a/roms/ipxe/src/core/console.c
+++ b/roms/ipxe/src/core/console.c
@@ -5,7 +5,7 @@
 
 /** @file */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Current console usage */
 int console_usage = CONSOLE_USAGE_STDOUT;
diff --git a/roms/ipxe/src/core/cpio.c b/roms/ipxe/src/core/cpio.c
index 3a5f4d2b6..080c72daf 100644
--- a/roms/ipxe/src/core/cpio.c
+++ b/roms/ipxe/src/core/cpio.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/ctype.c b/roms/ipxe/src/core/ctype.c
index c812346a0..891af71ea 100644
--- a/roms/ipxe/src/core/ctype.c
+++ b/roms/ipxe/src/core/ctype.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -31,11 +35,12 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /**
  * Check to see if character is a space
  *
- * @v c			Character
+ * @v character		Character
  * @ret isspace		Character is a space
  */
-int isspace ( int c ) {
-	switch ( c ) {
+int isspace ( int character ) {
+
+	switch ( character ) {
 	case ' ' :
 	case '\f' :
 	case '\n' :
diff --git a/roms/ipxe/src/core/cwuri.c b/roms/ipxe/src/core/cwuri.c
index 5865552a0..612f0b179 100644
--- a/roms/ipxe/src/core/cwuri.c
+++ b/roms/ipxe/src/core/cwuri.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/uri.h>
diff --git a/roms/ipxe/src/core/debug.c b/roms/ipxe/src/core/debug.c
index 7ded47089..def5d8b09 100644
--- a/roms/ipxe/src/core/debug.c
+++ b/roms/ipxe/src/core/debug.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdint.h>
diff --git a/roms/ipxe/src/core/debug_md5.c b/roms/ipxe/src/core/debug_md5.c
index f049ac757..d0dbad9ed 100644
--- a/roms/ipxe/src/core/debug_md5.c
+++ b/roms/ipxe/src/core/debug_md5.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdint.h>
diff --git a/roms/ipxe/src/core/device.c b/roms/ipxe/src/core/device.c
index 330f95c5a..77d7b719b 100644
--- a/roms/ipxe/src/core/device.c
+++ b/roms/ipxe/src/core/device.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/list.h>
diff --git a/roms/ipxe/src/core/downloader.c b/roms/ipxe/src/core/downloader.c
index ec69db6b1..d745f3617 100644
--- a/roms/ipxe/src/core/downloader.c
+++ b/roms/ipxe/src/core/downloader.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <errno.h>
@@ -29,7 +33,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/uaccess.h>
 #include <ipxe/umalloc.h>
 #include <ipxe/image.h>
-#include <ipxe/profile.h>
+#include <ipxe/xferbuf.h>
 #include <ipxe/downloader.h>
 
 /** @file
@@ -38,14 +42,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-/** Receive profiler */
-static struct profiler downloader_rx_profiler __profiler =
-	{ .name = "downloader.rx" };
-
-/** Data copy profiler */
-static struct profiler downloader_copy_profiler __profiler =
-	{ .name = "downloader.copy" };
-
 /** A downloader */
 struct downloader {
 	/** Reference count for this object */
@@ -58,8 +54,8 @@ struct downloader {
 
 	/** Image to contain downloaded file */
 	struct image *image;
-	/** Current position within image buffer */
-	size_t pos;
+	/** Data transfer buffer */
+	struct xfer_buffer buffer;
 };
 
 /**
@@ -92,42 +88,14 @@ static void downloader_finished ( struct downloader *downloader, int rc ) {
 			 downloader->image->name, strerror ( rc ) );
 	}
 
+	/* Update image length */
+	downloader->image->len = downloader->buffer.len;
+
 	/* Shut down interfaces */
 	intf_shutdown ( &downloader->xfer, rc );
 	intf_shutdown ( &downloader->job, rc );
 }
 
-/**
- * Ensure that download buffer is large enough for the specified size
- *
- * @v downloader	Downloader
- * @v len		Required minimum size
- * @ret rc		Return status code
- */
-static int downloader_ensure_size ( struct downloader *downloader,
-				    size_t len ) {
-	userptr_t new_buffer;
-
-	/* If buffer is already large enough, do nothing */
-	if ( len <= downloader->image->len )
-		return 0;
-
-	DBGC ( downloader, "Downloader %p extending to %zd bytes\n",
-	       downloader, len );
-
-	/* Extend buffer */
-	new_buffer = urealloc ( downloader->image->data, len );
-	if ( ! new_buffer ) {
-		DBGC ( downloader, "Downloader %p could not extend buffer to "
-		       "%zd bytes\n", downloader, len );
-		return -ENOSPC;
-	}
-	downloader->image->data = new_buffer;
-	downloader->image->len = len;
-
-	return 0;
-}
-
 /****************************************************************************
  *
  * Job control interface
@@ -148,8 +116,8 @@ static int downloader_progress ( struct downloader *downloader,
 	 * arrive out of order (e.g. with multicast protocols), but
 	 * it's a reasonable first approximation.
 	 */
-	progress->completed = downloader->pos;
-	progress->total = downloader->image->len;
+	progress->completed = downloader->buffer.pos;
+	progress->total = downloader->buffer.len;
 
 	return 0;
 }
@@ -171,44 +139,37 @@ static int downloader_progress ( struct downloader *downloader,
 static int downloader_xfer_deliver ( struct downloader *downloader,
 				     struct io_buffer *iobuf,
 				     struct xfer_metadata *meta ) {
-	size_t len;
-	size_t max;
 	int rc;
 
-	/* Start profiling */
-	profile_start ( &downloader_rx_profiler );
-
-	/* Calculate new buffer position */
-	if ( meta->flags & XFER_FL_ABS_OFFSET )
-		downloader->pos = 0;
-	downloader->pos += meta->offset;
-
-	/* Ensure that we have enough buffer space for this data */
-	len = iob_len ( iobuf );
-	max = ( downloader->pos + len );
-	if ( ( rc = downloader_ensure_size ( downloader, max ) ) != 0 )
-		goto done;
-
-	/* Copy data to buffer */
-	profile_start ( &downloader_copy_profiler );
-	copy_to_user ( downloader->image->data, downloader->pos,
-		       iobuf->data, len );
-	profile_stop ( &downloader_copy_profiler );
-
-	/* Update current buffer position */
-	downloader->pos += len;
-
- done:
-	free_iob ( iobuf );
-	if ( rc != 0 )
-		downloader_finished ( downloader, rc );
-	profile_stop ( &downloader_rx_profiler );
+	/* Add data to buffer */
+	if ( ( rc = xferbuf_deliver ( &downloader->buffer, iob_disown ( iobuf ),
+				      meta ) ) != 0 )
+		goto err_deliver;
+
+	return 0;
+
+ err_deliver:
+	downloader_finished ( downloader, rc );
 	return rc;
 }
 
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v downloader	Downloader
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ */
+static struct xfer_buffer *
+downloader_xfer_buffer ( struct downloader *downloader ) {
+
+	/* Provide direct access to underlying data transfer buffer */
+	return &downloader->buffer;
+}
+
 /** Downloader data transfer interface operations */
 static struct interface_operation downloader_xfer_operations[] = {
 	INTF_OP ( xfer_deliver, struct downloader *, downloader_xfer_deliver ),
+	INTF_OP ( xfer_buffer, struct downloader *, downloader_xfer_buffer ),
 	INTF_OP ( intf_close, struct downloader *, downloader_finished ),
 };
 
@@ -262,6 +223,7 @@ int create_downloader ( struct interface *job, struct image *image ) {
 	intf_init ( &downloader->xfer, &downloader_xfer_desc,
 		    &downloader->refcnt );
 	downloader->image = image_get ( image );
+	xferbuf_umalloc_init ( &downloader->buffer, &image->data );
 
 	/* Instantiate child objects and attach to our interfaces */
 	if ( ( rc = xfer_open_uri ( &downloader->xfer, image->uri ) ) != 0 )
diff --git a/roms/ipxe/src/core/edd.c b/roms/ipxe/src/core/edd.c
index d574ea6c0..a50b74ab1 100644
--- a/roms/ipxe/src/core/edd.c
+++ b/roms/ipxe/src/core/edd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/interface.h>
diff --git a/roms/ipxe/src/core/errno.c b/roms/ipxe/src/core/errno.c
index 06905561f..5de15bb92 100644
--- a/roms/ipxe/src/core/errno.c
+++ b/roms/ipxe/src/core/errno.c
@@ -1,6 +1,6 @@
 #include <errno.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/exec.c b/roms/ipxe/src/core/exec.c
index 1c85705ae..2c2ade0a5 100644
--- a/roms/ipxe/src/core/exec.c
+++ b/roms/ipxe/src/core/exec.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/core/fault.c b/roms/ipxe/src/core/fault.c
new file mode 100644
index 000000000..63d3ccacf
--- /dev/null
+++ b/roms/ipxe/src/core/fault.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <errno.h>
+#include <ipxe/fault.h>
+
+/** @file
+ *
+ * Fault injection
+ *
+ */
+
+/**
+ * Inject fault with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (must be non-zero)
+ * @ret rc		Return status code
+ */
+int inject_fault_nonzero ( unsigned int rate ) {
+
+	/* Do nothing unless we want to inject a fault now */
+	if ( ( random() % rate ) != 0 )
+		return 0;
+
+	/* Generate error number here so that faults can be injected
+	 * into files that don't themselves have error file
+	 * identifiers (via errfile.h).
+	 */
+	return -EFAULT;
+}
+
+/**
+ * Corrupt data with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (must be non-zero)
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+void inject_corruption_nonzero ( unsigned int rate, const void *data,
+				 size_t len ) {
+	uint8_t *writable;
+	size_t offset;
+
+	/* Do nothing if we have no data to corrupt */
+	if ( ! len )
+		return;
+
+	/* Do nothing unless we want to inject a fault now */
+	if ( ! inject_fault_nonzero ( rate ) )
+		return;
+
+	/* Get a writable pointer to the nominally read-only data */
+	writable = ( ( uint8_t * ) data );
+
+	/* Pick a random victim byte and zap it */
+	offset = ( random() % len );
+	writable[offset] ^= random();
+}
diff --git a/roms/ipxe/src/core/fbcon.c b/roms/ipxe/src/core/fbcon.c
index 72d6a6789..6d8b0086d 100644
--- a/roms/ipxe/src/core/fbcon.c
+++ b/roms/ipxe/src/core/fbcon.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/fnrec.c b/roms/ipxe/src/core/fnrec.c
index 3453c8b6a..0430817f8 100644
--- a/roms/ipxe/src/core/fnrec.c
+++ b/roms/ipxe/src/core/fnrec.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/core/gdbserial.c b/roms/ipxe/src/core/gdbserial.c
index 6f78c88bf..0983f2557 100644
--- a/roms/ipxe/src/core/gdbserial.c
+++ b/roms/ipxe/src/core/gdbserial.c
@@ -15,35 +15,105 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <assert.h>
-#include <ipxe/serial.h>
+#include <ipxe/uart.h>
 #include <ipxe/gdbstub.h>
 #include <ipxe/gdbserial.h>
+#include <config/serial.h>
+
+/* UART port number */
+#ifdef COMCONSOLE
+#define GDBSERIAL_PORT COMCONSOLE
+#else
+#define GDBSERIAL_PORT 0
+#endif
+
+/* UART baud rate */
+#ifdef COMPRESERVE
+#define GDBSERIAL_BAUD 0
+#else
+#define GDBSERIAL_BAUD COMSPEED
+#endif
+
+/* UART line control register value */
+#ifdef COMPRESERVE
+#define GDBSERIAL_LCR 0
+#else
+#define GDBSERIAL_LCR UART_LCR_WPS ( COMDATA, COMPARITY, COMSTOP )
+#endif
+
+/** GDB serial UART */
+static struct uart gdbserial_uart;
 
 struct gdb_transport serial_gdb_transport __gdb_transport;
 
 static size_t gdbserial_recv ( char *buf, size_t len ) {
+
 	assert ( len > 0 );
-	buf [ 0 ] = serial_getc();
+	while ( ! uart_data_ready ( &gdbserial_uart ) ) {}
+	buf[0] = uart_receive ( &gdbserial_uart );
 	return 1;
 }
 
 static void gdbserial_send ( const char *buf, size_t len ) {
+
 	while ( len-- > 0 ) {
-		serial_putc ( *buf++ );
+		uart_transmit ( &gdbserial_uart, *buf++ );
 	}
 }
 
+static int gdbserial_init ( int argc, char **argv ) {
+	unsigned int port;
+	char *endp;
+
+	if ( argc == 0 ) {
+		port = GDBSERIAL_PORT;
+	} else if ( argc == 1 ) {
+		port = strtoul ( argv[0], &endp, 10 );
+		if ( *endp ) {
+			printf ( "serial: invalid port\n" );
+			return 1;
+		}
+	} else {
+		printf ( "serial: syntax <port>\n" );
+		return 1;
+	}
+
+	if ( ! gdbserial_configure ( port, GDBSERIAL_BAUD, GDBSERIAL_LCR ) ) {
+		printf ( "serial: unable to configure\n" );
+		return 1;
+	}
+
+	return 0;
+}
+
 struct gdb_transport serial_gdb_transport __gdb_transport = {
 	.name = "serial",
+	.init = gdbserial_init,
 	.recv = gdbserial_recv,
 	.send = gdbserial_send,
 };
 
-struct gdb_transport *gdbserial_configure ( void ) {
+struct gdb_transport * gdbserial_configure ( unsigned int port,
+					     unsigned int baud, uint8_t lcr ) {
+	int rc;
+
+	if ( ( rc = uart_select ( &gdbserial_uart, port ) ) != 0 )
+		return NULL;
+
+	if ( ( rc = uart_init ( &gdbserial_uart, baud, lcr ) ) != 0 )
+		return NULL;
+
 	return &serial_gdb_transport;
 }
diff --git a/roms/ipxe/src/core/gdbstub.c b/roms/ipxe/src/core/gdbstub.c
index af06118b2..6ad52d1a6 100644
--- a/roms/ipxe/src/core/gdbstub.c
+++ b/roms/ipxe/src/core/gdbstub.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/core/gdbudp.c b/roms/ipxe/src/core/gdbudp.c
index 5977547c8..e4613d137 100644
--- a/roms/ipxe/src/core/gdbudp.c
+++ b/roms/ipxe/src/core/gdbudp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <string.h>
diff --git a/roms/ipxe/src/core/getkey.c b/roms/ipxe/src/core/getkey.c
index d69cfb44b..0f0f8b7c3 100644
--- a/roms/ipxe/src/core/getkey.c
+++ b/roms/ipxe/src/core/getkey.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ctype.h>
 #include <ipxe/console.h>
diff --git a/roms/ipxe/src/core/getopt.c b/roms/ipxe/src/core/getopt.c
index abc1edd6c..e6c3948d1 100644
--- a/roms/ipxe/src/core/getopt.c
+++ b/roms/ipxe/src/core/getopt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/core/image.c b/roms/ipxe/src/core/image.c
index ec4480238..529e3d72c 100644
--- a/roms/ipxe/src/core/image.c
+++ b/roms/ipxe/src/core/image.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
@@ -154,6 +158,32 @@ int image_set_cmdline ( struct image *image, const char *cmdline ) {
 }
 
 /**
+ * Determine image type
+ *
+ * @v image		Executable image
+ * @ret rc		Return status code
+ */
+static int image_probe ( struct image *image ) {
+	struct image_type *type;
+	int rc;
+
+	/* Try each type in turn */
+	for_each_table_entry ( type, IMAGE_TYPES ) {
+		if ( ( rc = type->probe ( image ) ) == 0 ) {
+			image->type = type;
+			DBGC ( image, "IMAGE %s is %s\n",
+			       image->name, type->name );
+			break;
+		}
+		DBGC ( image, "IMAGE %s is not %s: %s\n", image->name,
+		       type->name, strerror ( rc ) );
+	}
+
+	DBGC ( image, "IMAGE %s format not recognised\n", image->name );
+	return -ENOTSUP;
+}
+
+/**
  * Register executable image
  *
  * @v image		Executable image
@@ -185,6 +215,14 @@ int register_image ( struct image *image ) {
 	       image->name, user_to_phys ( image->data, 0 ),
 	       user_to_phys ( image->data, image->len ) );
 
+	/* Try to detect image type, if applicable.  Ignore failures,
+	 * since we expect to handle some unrecognised images
+	 * (e.g. kernel initrds, multiboot modules, random files
+	 * provided via our EFI virtual filesystem, etc).
+	 */
+	if ( ! image->type )
+		image_probe ( image );
+
 	return 0;
 }
 
@@ -223,36 +261,6 @@ struct image * find_image ( const char *name ) {
 }
 
 /**
- * Determine image type
- *
- * @v image		Executable image
- * @ret rc		Return status code
- */
-int image_probe ( struct image *image ) {
-	struct image_type *type;
-	int rc;
-
-	/* Succeed if we already have a type */
-	if ( image->type )
-		return 0;
-
-	/* Try each type in turn */
-	for_each_table_entry ( type, IMAGE_TYPES ) {
-		if ( ( rc = type->probe ( image ) ) == 0 ) {
-			image->type = type;
-			DBGC ( image, "IMAGE %s is %s\n",
-			       image->name, type->name );
-			return 0;
-		}
-		DBGC ( image, "IMAGE %s is not %s: %s\n", image->name,
-		       type->name, strerror ( rc ) );
-	}
-
-	DBGC ( image, "IMAGE %s format not recognised\n", image->name );
-	return -ENOEXEC;
-}
-
-/**
  * Execute image
  *
  * @v image		Executable image
@@ -284,9 +292,11 @@ int image_exec ( struct image *image ) {
 	 */
 	current_image = image_get ( image );
 
-	/* Check that this image can be selected for execution */
-	if ( ( rc = image_select ( image ) ) != 0 )
+	/* Check that this image can be executed */
+	if ( ! ( image->type && image->type->exec ) ) {
+		rc = -ENOEXEC;
 		goto err;
+	}
 
 	/* Check that image is trusted (if applicable) */
 	if ( require_trusted_images && ! ( image->flags & IMAGE_TRUSTED ) ) {
@@ -378,8 +388,8 @@ int image_replace ( struct image *replacement ) {
 	}
 
 	/* Check that the replacement image can be executed */
-	if ( ( rc = image_probe ( replacement ) ) != 0 )
-		return rc;
+	if ( ! ( replacement->type && replacement->type->exec ) )
+		return -ENOEXEC;
 
 	/* Clear any existing replacement */
 	image_put ( image->replacement );
@@ -400,16 +410,13 @@ int image_replace ( struct image *replacement ) {
  */
 int image_select ( struct image *image ) {
 	struct image *tmp;
-	int rc;
 
 	/* Unselect all other images */
 	for_each_image ( tmp )
 		tmp->flags &= ~IMAGE_SELECTED;
 
 	/* Check that this image can be executed */
-	if ( ( rc = image_probe ( image ) ) != 0 )
-		return rc;
-	if ( ! image->type->exec )
+	if ( ! ( image->type && image->type->exec ) )
 		return -ENOEXEC;
 
 	/* Mark image as selected */
@@ -468,9 +475,7 @@ int image_pixbuf ( struct image *image, struct pixel_buffer **pixbuf ) {
 	int rc;
 
 	/* Check that this image can be used to create a pixel buffer */
-	if ( ( rc = image_probe ( image ) ) != 0 )
-		return rc;
-	if ( ! image->type->pixbuf )
+	if ( ! ( image->type && image->type->pixbuf ) )
 		return -ENOTSUP;
 
 	/* Try creating pixel buffer */
diff --git a/roms/ipxe/src/core/init.c b/roms/ipxe/src/core/init.c
index 7ea0730fa..d91e44669 100644
--- a/roms/ipxe/src/core/init.c
+++ b/roms/ipxe/src/core/init.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 #include <ipxe/console.h>
diff --git a/roms/ipxe/src/core/interface.c b/roms/ipxe/src/core/interface.c
index 62f4621db..ba148c13d 100644
--- a/roms/ipxe/src/core/interface.c
+++ b/roms/ipxe/src/core/interface.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/interface.h>
@@ -307,3 +311,28 @@ void intf_restart ( struct interface *intf, int rc ) {
 	 */
 	intf->desc = desc;
 }
+
+/**
+ * Poke an object interface
+ *
+ * @v intf		Object interface
+ * @v type		Operation type
+ *
+ * This is a helper function to implement methods which take no
+ * parameters and return nothing.
+ */
+void intf_poke ( struct interface *intf,
+		 void ( type ) ( struct interface *intf ) ) {
+	struct interface *dest;
+	intf_poke_TYPE ( void * ) *op =
+		intf_get_dest_op_untyped ( intf, type, &dest );
+	void *object = intf_object ( dest );
+
+	if ( op ) {
+		op ( object );
+	} else {
+		/* Default is to do nothing */
+	}
+
+	intf_put ( dest );
+}
diff --git a/roms/ipxe/src/core/iobuf.c b/roms/ipxe/src/core/iobuf.c
index afc91d150..3e52ada4f 100644
--- a/roms/ipxe/src/core/iobuf.c
+++ b/roms/ipxe/src/core/iobuf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <strings.h>
@@ -200,3 +204,33 @@ struct io_buffer * iob_concatenate ( struct list_head *list ) {
 
 	return concatenated;
 }
+
+/**
+ * Split I/O buffer
+ *
+ * @v iobuf		I/O buffer
+ * @v len		Length to split into a new I/O buffer
+ * @ret split		New I/O buffer, or NULL on allocation failure
+ *
+ * Split the first @c len bytes of the existing I/O buffer into a
+ * separate I/O buffer.  The resulting buffers are likely to have no
+ * headroom or tailroom.
+ *
+ * If this call fails, then the original buffer will be unmodified.
+ */
+struct io_buffer * iob_split ( struct io_buffer *iobuf, size_t len ) {
+	struct io_buffer *split;
+
+	/* Sanity checks */
+	assert ( len <= iob_len ( iobuf ) );
+
+	/* Allocate new I/O buffer */
+	split = alloc_iob ( len );
+	if ( ! split )
+		return NULL;
+
+	/* Copy in data */
+	memcpy ( iob_put ( split, len ), iobuf->data, len );
+	iob_pull ( iobuf, len );
+	return split;
+}
diff --git a/roms/ipxe/src/core/isqrt.c b/roms/ipxe/src/core/isqrt.c
index 35c918d19..c4d0571e7 100644
--- a/roms/ipxe/src/core/isqrt.c
+++ b/roms/ipxe/src/core/isqrt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/job.c b/roms/ipxe/src/core/job.c
index 674bec8b5..65df80056 100644
--- a/roms/ipxe/src/core/job.c
+++ b/roms/ipxe/src/core/job.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/core/linebuf.c b/roms/ipxe/src/core/linebuf.c
index 8fb2f86a7..c197e383c 100644
--- a/roms/ipxe/src/core/linebuf.c
+++ b/roms/ipxe/src/core/linebuf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -39,7 +43,18 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * @ret line		Buffered line, or NULL if no line ready to read
  */
 char * buffered_line ( struct line_buffer *linebuf ) {
-	return ( linebuf->ready ? linebuf->data : NULL );
+	char *line = &linebuf->data[ linebuf->len ];
+
+	/* Fail unless we have a newly completed line to retrieve */
+	if ( ( linebuf->len == 0 ) || ( linebuf->consumed == 0 ) ||
+	     ( *(--line) != '\0' ) )
+		return NULL;
+
+	/* Identify start of line */
+	while ( ( line > linebuf->data ) && ( line[-1] != '\0' ) )
+		line--;
+
+	return line;
 }
 
 /**
@@ -48,10 +63,11 @@ char * buffered_line ( struct line_buffer *linebuf ) {
  * @v linebuf		Line buffer
  */
 void empty_line_buffer ( struct line_buffer *linebuf ) {
+
 	free ( linebuf->data );
 	linebuf->data = NULL;
 	linebuf->len = 0;
-	linebuf->ready = 0;
+	linebuf->consumed = 0;
 }
 
 /**
@@ -72,16 +88,13 @@ void empty_line_buffer ( struct line_buffer *linebuf ) {
  * should call empty_line_buffer() before freeing a @c struct @c
  * line_buffer.
  */
-ssize_t line_buffer ( struct line_buffer *linebuf,
-		      const char *data, size_t len ) {
+int line_buffer ( struct line_buffer *linebuf, const char *data, size_t len ) {
 	const char *eol;
 	size_t consume;
 	size_t new_len;
 	char *new_data;
-
-	/* Free any completed line from previous iteration */
-	if ( linebuf->ready )
-		empty_line_buffer ( linebuf );
+	char *lf;
+	char *cr;
 
 	/* Search for line terminator */
 	if ( ( eol = memchr ( data, '\n', len ) ) ) {
@@ -90,6 +103,10 @@ ssize_t line_buffer ( struct line_buffer *linebuf,
 		consume = len;
 	}
 
+	/* Reject any embedded NULs within the data to be consumed */
+	if ( memchr ( data, '\0', consume ) )
+		return -EINVAL;
+
 	/* Reallocate data buffer and copy in new data */
 	new_len = ( linebuf->len + consume );
 	new_data = realloc ( linebuf->data, ( new_len + 1 ) );
@@ -100,13 +117,27 @@ ssize_t line_buffer ( struct line_buffer *linebuf,
 	linebuf->data = new_data;
 	linebuf->len = new_len;
 
-	/* If we have reached end of line, trim the line and mark as ready */
+	/* If we have reached end of line, terminate the line */
 	if ( eol ) {
-		linebuf->data[--linebuf->len] = '\0'; /* trim NL */
-		if ( linebuf->data[linebuf->len - 1] == '\r' )
-			linebuf->data[--linebuf->len] = '\0'; /* trim CR */
-		linebuf->ready = 1;
+
+		/* Overwrite trailing LF (which must exist at this point) */
+		assert ( linebuf->len > 0 );
+		lf = &linebuf->data[ linebuf->len - 1 ];
+		assert ( *lf == '\n' );
+		*lf = '\0';
+
+		/* Trim (and overwrite) trailing CR, if present */
+		if ( linebuf->len > 1 ) {
+			cr = ( lf - 1 );
+			if ( *cr == '\r' ) {
+				linebuf->len--;
+				*cr = '\0';
+			}
+		}
 	}
 
+	/* Record consumed length */
+	linebuf->consumed = consume;
+
 	return consume;
 }
diff --git a/roms/ipxe/src/core/lineconsole.c b/roms/ipxe/src/core/lineconsole.c
index 1b6791cf3..bb3bfafc9 100644
--- a/roms/ipxe/src/core/lineconsole.c
+++ b/roms/ipxe/src/core/lineconsole.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/list.c b/roms/ipxe/src/core/list.c
index 77579d69a..5175c84ec 100644
--- a/roms/ipxe/src/core/list.c
+++ b/roms/ipxe/src/core/list.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/log.c b/roms/ipxe/src/core/log.c
index f160b4fc8..c08e4bb9b 100644
--- a/roms/ipxe/src/core/log.c
+++ b/roms/ipxe/src/core/log.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/main.c b/roms/ipxe/src/core/main.c
index db09e4c39..638dea9cf 100644
--- a/roms/ipxe/src/core/main.c
+++ b/roms/ipxe/src/core/main.c
@@ -12,7 +12,7 @@ Literature dealing with the network protocols:
 
 **************************************************************************/
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdio.h>
@@ -26,6 +26,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * @ret rc		Return status code
  */
 __asmcall int main ( void ) {
+	int rc;
 
 	/* Perform one-time-only initialisation (e.g. heap) */
 	initialise();
@@ -35,9 +36,11 @@ __asmcall int main ( void ) {
 	startup();
 	printf ( "ok\n" );
 
-	ipxe ( NULL );
+	/* Attempt to boot */
+	if ( ( rc = ipxe ( NULL ) ) != 0 )
+		goto err_ipxe;
 
+ err_ipxe:
 	shutdown_exit();
-
-	return 0;
+	return rc;
 }
diff --git a/roms/ipxe/src/core/malloc.c b/roms/ipxe/src/core/malloc.c
index d9c07495d..b120c0325 100644
--- a/roms/ipxe/src/core/malloc.c
+++ b/roms/ipxe/src/core/malloc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
@@ -106,6 +110,7 @@ static char heap[HEAP_SIZE] __attribute__ (( aligned ( __alignof__(void *) )));
 static inline void valgrind_make_blocks_defined ( void ) {
 	struct memory_block *block;
 
+	/* Do nothing unless running under Valgrind */
 	if ( RUNNING_ON_VALGRIND <= 0 )
 		return;
 
@@ -147,6 +152,7 @@ static inline void valgrind_make_blocks_noaccess ( void ) {
 	struct memory_block *block;
 	struct memory_block *prev = NULL;
 
+	/* Do nothing unless running under Valgrind */
 	if ( RUNNING_ON_VALGRIND <= 0 )
 		return;
 
@@ -267,24 +273,25 @@ static void discard_all_cache ( void ) {
 void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 	struct memory_block *block;
 	size_t align_mask;
+	size_t actual_size;
 	size_t pre_size;
 	ssize_t post_size;
 	struct memory_block *pre;
 	struct memory_block *post;
-	struct memory_block *ptr;
+	void *ptr;
 
 	/* Sanity checks */
 	assert ( size != 0 );
 	assert ( ( align == 0 ) || ( ( align & ( align - 1 ) ) == 0 ) );
-
 	valgrind_make_blocks_defined();
 	check_blocks();
 
 	/* Round up size to multiple of MIN_MEMBLOCK_SIZE and
 	 * calculate alignment mask.
 	 */
-	size = ( size + MIN_MEMBLOCK_SIZE - 1 ) & ~( MIN_MEMBLOCK_SIZE - 1 );
-	align_mask = ( align - 1 ) | ( MIN_MEMBLOCK_SIZE - 1 );
+	actual_size = ( ( size + MIN_MEMBLOCK_SIZE - 1 ) &
+			~( MIN_MEMBLOCK_SIZE - 1 ) );
+	align_mask = ( ( align - 1 ) | ( MIN_MEMBLOCK_SIZE - 1 ) );
 
 	DBGC2 ( &heap, "Allocating %#zx (aligned %#zx+%zx)\n",
 		size, align, offset );
@@ -293,7 +300,7 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 		list_for_each_entry ( block, &free_blocks, list ) {
 			pre_size = ( ( offset - virt_to_phys ( block ) )
 				     & align_mask );
-			post_size = ( block->size - pre_size - size );
+			post_size = ( block->size - pre_size - actual_size );
 			if ( post_size >= 0 ) {
 				/* Split block into pre-block, block, and
 				 * post-block.  After this split, the "pre"
@@ -302,7 +309,7 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 				 */
 				pre   = block;
 				block = ( ( ( void * ) pre   ) + pre_size );
-				post  = ( ( ( void * ) block ) + size     );
+				post  = ( ( ( void * ) block ) + actual_size );
 				DBGC2 ( &heap, "[%p,%p) -> [%p,%p) + [%p,%p)\n",
 					pre, ( ( ( void * ) pre ) + pre->size ),
 					pre, block, post,
@@ -313,8 +320,8 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 				 * the heap).
 				 */
 				if ( (size_t) post_size >= MIN_MEMBLOCK_SIZE ) {
-					VALGRIND_MAKE_MEM_DEFINED ( post,
-							     sizeof ( *post ) );
+					VALGRIND_MAKE_MEM_UNDEFINED
+						( post, sizeof ( *post ) );
 					post->size = post_size;
 					list_add ( &post->list, &pre->list );
 				}
@@ -328,14 +335,18 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 				 * it is too small, which can happen only at
 				 * the very start of the heap.
 				 */
-				if ( pre_size < MIN_MEMBLOCK_SIZE )
+				if ( pre_size < MIN_MEMBLOCK_SIZE ) {
 					list_del ( &pre->list );
+					VALGRIND_MAKE_MEM_NOACCESS
+						( pre, sizeof ( *pre ) );
+				}
 				/* Update total free memory */
-				freemem -= size;
+				freemem -= actual_size;
 				/* Return allocated block */
 				DBGC2 ( &heap, "Allocated [%p,%p)\n", block,
 					( ( ( void * ) block ) + size ) );
 				ptr = block;
+				VALGRIND_MAKE_MEM_UNDEFINED ( ptr, size );
 				goto done;
 			}
 		}
@@ -368,13 +379,16 @@ void free_memblock ( void *ptr, size_t size ) {
 	struct memory_block *freeing;
 	struct memory_block *block;
 	struct memory_block *tmp;
+	size_t actual_size;
 	ssize_t gap_before;
 	ssize_t gap_after = -1;
 
 	/* Allow for ptr==NULL */
 	if ( ! ptr )
 		return;
+	VALGRIND_MAKE_MEM_NOACCESS ( ptr, size );
 
+	/* Sanity checks */
 	valgrind_make_blocks_defined();
 	check_blocks();
 
@@ -382,9 +396,10 @@ void free_memblock ( void *ptr, size_t size ) {
 	 * would have used.
 	 */
 	assert ( size != 0 );
-	size = ( size + MIN_MEMBLOCK_SIZE - 1 ) & ~( MIN_MEMBLOCK_SIZE - 1 );
+	actual_size = ( ( size + MIN_MEMBLOCK_SIZE - 1 ) &
+			~( MIN_MEMBLOCK_SIZE - 1 ) );
 	freeing = ptr;
-	VALGRIND_MAKE_MEM_DEFINED ( freeing, sizeof ( *freeing ) );
+	VALGRIND_MAKE_MEM_UNDEFINED ( freeing, sizeof ( *freeing ) );
 	DBGC2 ( &heap, "Freeing [%p,%p)\n",
 		freeing, ( ( ( void * ) freeing ) + size ) );
 
@@ -392,7 +407,7 @@ void free_memblock ( void *ptr, size_t size ) {
 	if ( ASSERTING ) {
 		list_for_each_entry ( block, &free_blocks, list ) {
 			if ( ( ( ( void * ) block ) <
-			       ( ( void * ) freeing + size ) ) &&
+			       ( ( void * ) freeing + actual_size ) ) &&
 			     ( ( void * ) freeing <
 			       ( ( void * ) block + block->size ) ) ) {
 				assert ( 0 );
@@ -407,7 +422,7 @@ void free_memblock ( void *ptr, size_t size ) {
 	}
 
 	/* Insert/merge into free list */
-	freeing->size = size;
+	freeing->size = actual_size;
 	list_for_each_entry_safe ( block, tmp, &free_blocks, list ) {
 		/* Calculate gaps before and after the "freeing" block */
 		gap_before = ( ( ( void * ) freeing ) - 
@@ -421,8 +436,10 @@ void free_memblock ( void *ptr, size_t size ) {
 				( ( ( void * ) freeing ) + freeing->size ),
 				block,
 				( ( ( void * ) freeing ) + freeing->size ) );
-			block->size += size;
+			block->size += actual_size;
 			list_del ( &block->list );
+			VALGRIND_MAKE_MEM_NOACCESS ( freeing,
+						     sizeof ( *freeing ) );
 			freeing = block;
 		}
 		/* Stop processing as soon as we reach a following block */
@@ -444,10 +461,11 @@ void free_memblock ( void *ptr, size_t size ) {
 			( ( ( void * ) block ) + block->size ) );
 		freeing->size += block->size;
 		list_del ( &block->list );
+		VALGRIND_MAKE_MEM_NOACCESS ( block, sizeof ( *block ) );
 	}
 
 	/* Update free memory counter */
-	freemem += size;
+	freemem += actual_size;
 
 	check_blocks();
 	valgrind_make_blocks_noaccess();
@@ -490,9 +508,9 @@ void * realloc ( void *old_ptr, size_t new_size ) {
 		new_block = alloc_memblock ( new_total_size, 1, 0 );
 		if ( ! new_block )
 			return NULL;
-		VALGRIND_MAKE_MEM_UNDEFINED ( new_block, offsetof ( struct autosized_block, data ) );
 		new_block->size = new_total_size;
-		VALGRIND_MAKE_MEM_NOACCESS ( new_block, offsetof ( struct autosized_block, data ) );
+		VALGRIND_MAKE_MEM_NOACCESS ( &new_block->size,
+					     sizeof ( new_block->size ) );
 		new_ptr = &new_block->data;
 		VALGRIND_MALLOCLIKE_BLOCK ( new_ptr, new_size, 0, 0 );
 	}
@@ -505,16 +523,16 @@ void * realloc ( void *old_ptr, size_t new_size ) {
 	if ( old_ptr && ( old_ptr != NOWHERE ) ) {
 		old_block = container_of ( old_ptr, struct autosized_block,
 					   data );
-		VALGRIND_MAKE_MEM_DEFINED ( old_block, offsetof ( struct autosized_block, data ) );
+		VALGRIND_MAKE_MEM_DEFINED ( &old_block->size,
+					    sizeof ( old_block->size ) );
 		old_total_size = old_block->size;
 		assert ( old_total_size != 0 );
 		old_size = ( old_total_size -
 			     offsetof ( struct autosized_block, data ) );
 		memcpy ( new_ptr, old_ptr,
 			 ( ( old_size < new_size ) ? old_size : new_size ) );
-		free_memblock ( old_block, old_total_size );
-		VALGRIND_MAKE_MEM_NOACCESS ( old_block, offsetof ( struct autosized_block, data ) );
 		VALGRIND_FREELIKE_BLOCK ( old_ptr, 0 );
+		free_memblock ( old_block, old_total_size );
 	}
 
 	if ( ASSERTED ) {
@@ -611,6 +629,7 @@ void mpopulate ( void *start, size_t len ) {
  */
 static void init_heap ( void ) {
 	VALGRIND_MAKE_MEM_NOACCESS ( heap, sizeof ( heap ) );
+	VALGRIND_MAKE_MEM_NOACCESS ( &free_blocks, sizeof ( free_blocks ) );
 	mpopulate ( heap, sizeof ( heap ) );
 }
 
diff --git a/roms/ipxe/src/core/memblock.c b/roms/ipxe/src/core/memblock.c
index 1fd89b871..aecddc22c 100644
--- a/roms/ipxe/src/core/memblock.c
+++ b/roms/ipxe/src/core/memblock.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/memmap_settings.c b/roms/ipxe/src/core/memmap_settings.c
index 0f6d0abf5..fab3e5f3a 100644
--- a/roms/ipxe/src/core/memmap_settings.c
+++ b/roms/ipxe/src/core/memmap_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/core/menu.c b/roms/ipxe/src/core/menu.c
index 8d42e1f83..ab5b0c7f5 100644
--- a/roms/ipxe/src/core/menu.c
+++ b/roms/ipxe/src/core/menu.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/misc.c b/roms/ipxe/src/core/misc.c
deleted file mode 100644
index eaceddfea..000000000
--- a/roms/ipxe/src/core/misc.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/**************************************************************************
-MISC Support Routines
-**************************************************************************/
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stdlib.h>
-#include <ctype.h>
-#include <byteswap.h>
-#include <ipxe/in.h>
-#include <ipxe/timer.h>
-
-/**************************************************************************
-INET_ATON - Convert an ascii x.x.x.x to binary form
-**************************************************************************/
-int inet_aton ( const char *cp, struct in_addr *inp ) {
-	const char *p = cp;
-	const char *digits_start;
-	unsigned long ip = 0;
-	unsigned long val;
-	int j;
-	for(j = 0; j <= 3; j++) {
-		digits_start = p;
-		val = strtoul(p, ( char ** ) &p, 10);
-		if ((p == digits_start) || (val > 255)) return 0;
-		if ( ( j < 3 ) && ( *(p++) != '.' ) ) return 0;
-		ip = (ip << 8) | val;
-	}
-	if ( *p == '\0' ) {
-		inp->s_addr = htonl(ip);
-		return 1;
-	}
-	return 0;
-}
-
-unsigned int strtoul_charval ( unsigned int charval ) {
-
-	if ( charval >= 'a' ) {
-		charval = ( charval - 'a' + 10 );
-	} else if ( charval >= 'A' ) {
-		charval = ( charval - 'A' + 10 );
-	} else if ( charval <= '9' ) {
-		charval = ( charval - '0' );
-	}
-
-	return charval;
-}
-
-unsigned long strtoul ( const char *p, char **endp, int base ) {
-	unsigned long ret = 0;
-	int negative = 0;
-	unsigned int charval;
-
-	while ( isspace ( *p ) )
-		p++;
-
-	if ( *p == '-' ) {
-		negative = 1;
-		p++;
-	}
-
-	base = strtoul_base ( &p, base );
-
-	while ( 1 ) {
-		charval = strtoul_charval ( *p );
-		if ( charval >= ( unsigned int ) base )
-			break;
-		ret = ( ( ret * base ) + charval );
-		p++;
-	}
-
-	if ( negative )
-		ret = -ret;
-
-	if ( endp )
-		*endp = ( char * ) p;
-
-	return ( ret );
-}
-
-/*
- * Local variables:
- *  c-basic-offset: 8
- * End:
- */
diff --git a/roms/ipxe/src/core/monojob.c b/roms/ipxe/src/core/monojob.c
index 820fa31dc..817f21b2c 100644
--- a/roms/ipxe/src/core/monojob.c
+++ b/roms/ipxe/src/core/monojob.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/core/null_reboot.c b/roms/ipxe/src/core/null_reboot.c
index a3d5b2ef8..7be5612a3 100644
--- a/roms/ipxe/src/core/null_reboot.c
+++ b/roms/ipxe/src/core/null_reboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/core/null_sanboot.c b/roms/ipxe/src/core/null_sanboot.c
index 18c0dea84..2f7522c6c 100644
--- a/roms/ipxe/src/core/null_sanboot.c
+++ b/roms/ipxe/src/core/null_sanboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/sanboot.h>
diff --git a/roms/ipxe/src/core/null_time.c b/roms/ipxe/src/core/null_time.c
index 506c70b52..90041a456 100644
--- a/roms/ipxe/src/core/null_time.c
+++ b/roms/ipxe/src/core/null_time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/nvo.c b/roms/ipxe/src/core/nvo.c
index e135d2b41..d2c9b5e73 100644
--- a/roms/ipxe/src/core/nvo.c
+++ b/roms/ipxe/src/core/nvo.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/core/open.c b/roms/ipxe/src/core/open.c
index b479c2975..9d665ffda 100644
--- a/roms/ipxe/src/core/open.c
+++ b/roms/ipxe/src/core/open.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <string.h>
diff --git a/roms/ipxe/src/core/params.c b/roms/ipxe/src/core/params.c
index 93b834419..e1f66acca 100644
--- a/roms/ipxe/src/core/params.c
+++ b/roms/ipxe/src/core/params.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/parseopt.c b/roms/ipxe/src/core/parseopt.c
index d268c0594..66f60158c 100644
--- a/roms/ipxe/src/core/parseopt.c
+++ b/roms/ipxe/src/core/parseopt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
@@ -32,6 +36,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/params.h>
 #include <ipxe/timer.h>
 #include <ipxe/parseopt.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -343,7 +348,7 @@ void print_usage ( struct command_descriptor *cmd, char **argv ) {
 	}
 	if ( cmd->usage )
 		printf ( " %s", cmd->usage );
-	printf ( "\n\nSee http://ipxe.org/cmd/%s for further information\n",
+	printf ( "\n\nSee " PRODUCT_COMMAND_URI " for further information\n",
 		 argv[0] );
 }
 
diff --git a/roms/ipxe/src/core/pending.c b/roms/ipxe/src/core/pending.c
index 7bb0c2e00..96d0cf197 100644
--- a/roms/ipxe/src/core/pending.c
+++ b/roms/ipxe/src/core/pending.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/process.h>
diff --git a/roms/ipxe/src/core/pinger.c b/roms/ipxe/src/core/pinger.c
index 31ea2ce1c..0ff7bb9f2 100644
--- a/roms/ipxe/src/core/pinger.c
+++ b/roms/ipxe/src/core/pinger.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/roms/ipxe/src/core/pixbuf.c b/roms/ipxe/src/core/pixbuf.c
index 48f8e9f9a..41e18f8dc 100644
--- a/roms/ipxe/src/core/pixbuf.c
+++ b/roms/ipxe/src/core/pixbuf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/core/pool.c b/roms/ipxe/src/core/pool.c
new file mode 100644
index 000000000..0163405f7
--- /dev/null
+++ b/roms/ipxe/src/core/pool.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Pooled connections
+ *
+ */
+
+#include <assert.h>
+#include <ipxe/pool.h>
+
+/**
+ * Recycle this connection after closing
+ *
+ * @v intf		Data transfer interface
+ */
+void pool_recycle ( struct interface *intf ) {
+
+	intf_poke ( intf, pool_recycle );
+}
+
+/**
+ * Reopen a defunct connection
+ *
+ * @v intf		Data transfer interface
+ */
+void pool_reopen ( struct interface *intf ) {
+
+	intf_poke ( intf, pool_reopen );
+}
+
+/**
+ * Add connection to pool
+ *
+ * @v pool		Pooled connection
+ * @v list		List of pooled connections
+ * @v expiry		Expiry time
+ */
+void pool_add ( struct pooled_connection *pool, struct list_head *list,
+		unsigned long expiry ) {
+
+	/* Sanity check */
+	assert ( list_empty ( &pool->list ) );
+	assert ( ! timer_running ( &pool->timer ) );
+
+	/* Add to list of pooled connections */
+	list_add_tail ( &pool->list, list );
+
+	/* Start expiry timer */
+	start_timer_fixed ( &pool->timer, expiry );
+}
+
+/**
+ * Remove connection from pool
+ *
+ * @v pool		Pooled connection
+ */
+void pool_del ( struct pooled_connection *pool ) {
+
+	/* Remove from list of pooled connections */
+	list_del ( &pool->list );
+	INIT_LIST_HEAD ( &pool->list );
+
+	/* Stop expiry timer */
+	stop_timer ( &pool->timer );
+
+	/* Mark as a freshly recycled connection */
+	pool->flags = POOL_RECYCLED;
+}
+
+/**
+ * Close expired pooled connection
+ *
+ * @v timer		Expiry timer
+ * @v over		Failure indicator
+ */
+void pool_expired ( struct retry_timer *timer, int over __unused ) {
+	struct pooled_connection *pool =
+		container_of ( timer, struct pooled_connection, timer );
+
+	/* Sanity check */
+	assert ( ! list_empty ( &pool->list ) );
+
+	/* Remove from connection pool */
+	list_del ( &pool->list );
+	INIT_LIST_HEAD ( &pool->list );
+
+	/* Close expired connection */
+	pool->expired ( pool );
+}
diff --git a/roms/ipxe/src/core/posix_io.c b/roms/ipxe/src/core/posix_io.c
index 8460d0f51..35b52beeb 100644
--- a/roms/ipxe/src/core/posix_io.c
+++ b/roms/ipxe/src/core/posix_io.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/roms/ipxe/src/core/process.c b/roms/ipxe/src/core/process.c
index d341a2c37..69852c416 100644
--- a/roms/ipxe/src/core/process.c
+++ b/roms/ipxe/src/core/process.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/init.h>
diff --git a/roms/ipxe/src/core/profile.c b/roms/ipxe/src/core/profile.c
index 150e6b273..1075047b9 100644
--- a/roms/ipxe/src/core/profile.c
+++ b/roms/ipxe/src/core/profile.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/core/random.c b/roms/ipxe/src/core/random.c
index 8824dca3a..a74175a79 100644
--- a/roms/ipxe/src/core/random.c
+++ b/roms/ipxe/src/core/random.c
@@ -4,7 +4,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <ipxe/timer.h>
diff --git a/roms/ipxe/src/core/refcnt.c b/roms/ipxe/src/core/refcnt.c
index 68a86120e..47c975a0b 100644
--- a/roms/ipxe/src/core/refcnt.c
+++ b/roms/ipxe/src/core/refcnt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/core/resolv.c b/roms/ipxe/src/core/resolv.c
index d59a8c0ad..1e3182b0b 100644
--- a/roms/ipxe/src/core/resolv.c
+++ b/roms/ipxe/src/core/resolv.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/core/serial.c b/roms/ipxe/src/core/serial.c
index 7e4460ab9..4ce025519 100644
--- a/roms/ipxe/src/core/serial.c
+++ b/roms/ipxe/src/core/serial.c
@@ -1,259 +1,184 @@
 /*
- * The serial port interface routines implement a simple polled i/o
- * interface to a standard serial port.  Due to the space restrictions
- * for the boot blocks, no BIOS support is used (since BIOS requires
- * expensive real/protected mode switches), instead the rudimentary
- * BIOS support is duplicated here.
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
  *
- * The base address and speed for the i/o port are passed from the
- * Makefile in the COMCONSOLE and CONSPEED preprocessor macros.  The
- * line control parameters are currently hard-coded to 8 bits, no
- * parity, 1 stop bit (8N1).  This can be changed in init_serial().
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Serial console
+ *
+ */
 
-#include "stddef.h"
+#include <stddef.h>
 #include <ipxe/init.h>
-#include <ipxe/io.h>
-#include <unistd.h>
+#include <ipxe/uart.h>
+#include <ipxe/console.h>
 #include <ipxe/serial.h>
-#include "config/serial.h"
-
-/* Set default values if none specified */
+#include <config/console.h>
+#include <config/serial.h>
 
-#ifndef COMCONSOLE
-#define COMCONSOLE	0x3f8
+/* Set default console usage if applicable */
+#if ! ( defined ( CONSOLE_SERIAL ) && CONSOLE_EXPLICIT ( CONSOLE_SERIAL ) )
+#undef CONSOLE_SERIAL
+#define CONSOLE_SERIAL ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG )
 #endif
 
-#ifndef COMSPEED
-#define COMSPEED	9600
-#endif
-
-#ifndef COMDATA
-#define COMDATA		8
+/* UART port number */
+#ifdef COMCONSOLE
+#define CONSOLE_PORT COMCONSOLE
+#else
+#define CONSOLE_PORT 0
 #endif
 
-#ifndef COMPARITY
-#define COMPARITY	0
+/* UART baud rate */
+#ifdef COMPRESERVE
+#define CONSOLE_BAUD 0
+#else
+#define CONSOLE_BAUD COMSPEED
 #endif
 
-#ifndef COMSTOP
-#define COMSTOP		1
+/* UART line control register value */
+#ifdef COMPRESERVE
+#define CONSOLE_LCR 0
+#else
+#define CONSOLE_LCR UART_LCR_WPS ( COMDATA, COMPARITY, COMSTOP )
 #endif
 
-#undef UART_BASE
-#define UART_BASE ( COMCONSOLE )
-
-#undef UART_BAUD
-#define UART_BAUD ( COMSPEED )
+/** Serial console UART */
+struct uart serial_console;
 
-#if ((115200%UART_BAUD) != 0)
-#error Bad ttys0 baud rate
-#endif
-
-#define COMBRD (115200/UART_BAUD)
+/**
+ * Print a character to serial console
+ *
+ * @v character		Character to be printed
+ */
+static void serial_putchar ( int character ) {
 
-/* Line Control Settings */
-#define UART_LCS ( ( ( (COMDATA) - 5 )	<< 0 ) | \
-		   ( ( (COMPARITY) )	<< 3 ) | \
-		   ( ( (COMSTOP) - 1 )	<< 2 ) )
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return;
 
-/* Data */
-#define UART_RBR 0x00
-#define UART_TBR 0x00
+	/* Transmit character */
+	uart_transmit ( &serial_console, character );
+}
 
-/* Control */
-#define UART_IER 0x01
-#define UART_IIR 0x02
-#define UART_FCR 0x02
-#define UART_LCR 0x03
-#define UART_MCR 0x04
-#define UART_DLL 0x00
-#define UART_DLM 0x01
+/**
+ * Get character from serial console
+ *
+ * @ret character	Character read from console
+ */
+static int serial_getchar ( void ) {
+	uint8_t data;
 
-/* Status */
-#define UART_LSR 0x05
-#define  UART_LSR_TEMPT 0x40	/* Transmitter empty */
-#define  UART_LSR_THRE  0x20	/* Transmit-hold-register empty */
-#define  UART_LSR_BI	0x10	/* Break interrupt indicator */
-#define  UART_LSR_FE	0x08	/* Frame error indicator */
-#define  UART_LSR_PE	0x04	/* Parity error indicator */
-#define  UART_LSR_OE	0x02	/* Overrun error indicator */
-#define  UART_LSR_DR	0x01	/* Receiver data ready */
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return 0;
 
-#define UART_MSR 0x06
-#define UART_SCR 0x07
+	/* Wait for data to be ready */
+	while ( ! uart_data_ready ( &serial_console ) ) {}
 
-#if defined(UART_MEM)
-#define uart_readb(addr) readb((addr))
-#define uart_writeb(val,addr) writeb((val),(addr))
-#else
-#define uart_readb(addr) inb((addr))
-#define uart_writeb(val,addr) outb((val),(addr))
-#endif
+	/* Receive data */
+	data = uart_receive ( &serial_console );
 
-/* Boolean for the state of serial driver initialization */
-int serial_initialized = 0;
+	/* Strip any high bit and convert DEL to backspace */
+	data &= 0x7f;
+	if ( data == 0x7f )
+		data = 0x08;
 
-/*
- * void serial_putc(int ch);
- *	Write character `ch' to port UART_BASE.
- */
-void serial_putc ( int ch ) {
-	int i;
-	int status;
-	i = 1000; /* timeout */
-	while(--i > 0) {
-		status = uart_readb(UART_BASE + UART_LSR);
-		if (status & UART_LSR_THRE) { 
-			/* TX buffer emtpy */
-			uart_writeb(ch, UART_BASE + UART_TBR);
-			break;
-		}
-		mdelay(2);
-	}
+	return data;
 }
 
-/*
- * int serial_getc(void);
- *	Read a character from port UART_BASE.
- */
-int serial_getc ( void ) {
-	int status;
-	int ch;
-	do {
-		status = uart_readb(UART_BASE + UART_LSR);
-	} while((status & 1) == 0);
-	ch = uart_readb(UART_BASE + UART_RBR);	/* fetch (first) character */
-	ch &= 0x7f;				/* remove any parity bits we get */
-	if (ch == 0x7f) {			/* Make DEL... look like BS */
-		ch = 0x08;
-	}
-	return ch;
-}
-
-/*
- * int serial_ischar(void);
- *       If there is a character in the input buffer of port UART_BASE,
- *       return nonzero; otherwise return 0.
+/**
+ * Check for character ready to read from serial console
+ *
+ * @ret True		Character available to read
+ * @ret False		No character available to read
  */
-int serial_ischar ( void ) {
-	int status;
-	status = uart_readb(UART_BASE + UART_LSR);	/* line status reg; */
-	return status & 1;		/* rx char available */
-}
+static int serial_iskey ( void ) {
 
-/*
- * int serial_init(void);
- *	Initialize port UART_BASE to speed COMSPEED, line settings 8N1.
- */
-static void serial_init ( void ) {
-	int status;
-	int divisor, lcs;
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return 0;
 
-	DBG ( "Serial port %#x initialising\n", UART_BASE );
+	/* Check UART */
+	return uart_data_ready ( &serial_console );
+}
 
-	divisor = COMBRD;
-	lcs = UART_LCS;
+/** Serial console */
+struct console_driver serial_console_driver __console_driver = {
+	.putchar = serial_putchar,
+	.getchar = serial_getchar,
+	.iskey = serial_iskey,
+	.usage = CONSOLE_SERIAL,
+};
 
+/** Initialise serial console */
+static void serial_init ( void ) {
+	int rc;
 
-#ifdef COMPRESERVE
-	lcs = uart_readb(UART_BASE + UART_LCR) & 0x7f;
-	uart_writeb(0x80 | lcs, UART_BASE + UART_LCR);
-	divisor = (uart_readb(UART_BASE + UART_DLM) << 8) | uart_readb(UART_BASE + UART_DLL);
-	uart_writeb(lcs, UART_BASE + UART_LCR);
-#endif
+	/* Do nothing if we have no default port */
+	if ( ! CONSOLE_PORT )
+		return;
 
-	/* Set Baud Rate Divisor to COMSPEED, and test to see if the
-	 * serial port appears to be present.
-	 */
-	uart_writeb(0x80 | lcs, UART_BASE + UART_LCR);
-	uart_writeb(0xaa, UART_BASE + UART_DLL);
-	if (uart_readb(UART_BASE + UART_DLL) != 0xaa) {
-		DBG ( "Serial port %#x UART_DLL failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(0x55, UART_BASE + UART_DLL);
-	if (uart_readb(UART_BASE + UART_DLL) != 0x55) {
-		DBG ( "Serial port %#x UART_DLL failed\n", UART_BASE );
-		goto out;
+	/* Select UART */
+	if ( ( rc = uart_select ( &serial_console, CONSOLE_PORT ) ) != 0 ) {
+		DBG ( "Could not select UART %d: %s\n",
+		      CONSOLE_PORT, strerror ( rc ) );
+		return;
 	}
-	uart_writeb(divisor & 0xff, UART_BASE + UART_DLL);
-	if (uart_readb(UART_BASE + UART_DLL) != (divisor & 0xff)) {
-		DBG ( "Serial port %#x UART_DLL failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(0xaa, UART_BASE + UART_DLM);
-	if (uart_readb(UART_BASE + UART_DLM) != 0xaa) {
-		DBG ( "Serial port %#x UART_DLM failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(0x55, UART_BASE + UART_DLM);
-	if (uart_readb(UART_BASE + UART_DLM) != 0x55) {
-		DBG ( "Serial port %#x UART_DLM failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb((divisor >> 8) & 0xff, UART_BASE + UART_DLM);
-	if (uart_readb(UART_BASE + UART_DLM) != ((divisor >> 8) & 0xff)) {
-		DBG ( "Serial port %#x UART_DLM failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(lcs, UART_BASE + UART_LCR);
-	
-	/* disable interrupts */
-	uart_writeb(0x0, UART_BASE + UART_IER);
 
-	/* enable fifos */
-	uart_writeb(0x01, UART_BASE + UART_FCR);
+	/* Initialise UART */
+	if ( ( rc = uart_init ( &serial_console, CONSOLE_BAUD,
+				CONSOLE_LCR ) ) != 0 ) {
+		DBG ( "Could not initialise UART %d baud %d LCR %#02x: %s\n",
+		      CONSOLE_PORT, CONSOLE_BAUD, CONSOLE_LCR, strerror ( rc ));
+		return;
+	}
+}
 
-	/* Set clear to send, so flow control works... */
-	uart_writeb((1<<1), UART_BASE + UART_MCR);
+/**
+ * Shut down serial console
+ *
+ * @v flags		Shutdown flags
+ */
+static void serial_shutdown ( int flags __unused ) {
 
-	/* Flush the input buffer. */
-	do {
-		/* rx buffer reg
-		 * throw away (unconditionally the first time)
-		 */
-	        (void) uart_readb(UART_BASE + UART_RBR);
-		/* line status reg */
-		status = uart_readb(UART_BASE + UART_LSR);
-	} while(status & UART_LSR_DR);
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return;
 
-	/* Note that serial support has been initialized */
-	serial_initialized = 1;
- out:
-	return;
-}
+	/* Flush any pending output */
+	uart_flush ( &serial_console );
 
-/*
- * void serial_fini(void);
- *	Cleanup our use of the serial port, in particular flush the
- *	output buffer so we don't accidentially lose characters.
- */
-static void serial_fini ( int flags __unused ) {
-	int i, status;
-	/* Flush the output buffer to avoid dropping characters,
-	 * if we are reinitializing the serial port.
-	 */
-	i = 10000; /* timeout */
-	do {
-		status = uart_readb(UART_BASE + UART_LSR);
-	} while((--i > 0) && !(status & UART_LSR_TEMPT));
-	/* Don't mark it as disabled; it's still usable */
+	/* Leave console enabled; it's still usable */
 }
 
-/**
- * Serial driver initialisation function
- *
- * Initialise serial port early on so that it is available to capture
- * early debug messages.
- */
-struct init_fn serial_init_fn __init_fn ( INIT_SERIAL ) = {
+/** Serial console initialisation function */
+struct init_fn serial_console_init_fn __init_fn ( INIT_CONSOLE ) = {
 	.initialise = serial_init,
 };
 
-/** Serial driver startup function */
+/** Serial console startup function */
 struct startup_fn serial_startup_fn __startup_fn ( STARTUP_EARLY ) = {
-	.shutdown = serial_fini,
+	.shutdown = serial_shutdown,
 };
diff --git a/roms/ipxe/src/core/serial_console.c b/roms/ipxe/src/core/serial_console.c
deleted file mode 100644
index de9b84ca7..000000000
--- a/roms/ipxe/src/core/serial_console.c
+++ /dev/null
@@ -1,42 +0,0 @@
-#include <ipxe/init.h>
-#include <ipxe/serial.h>
-#include <ipxe/console.h>
-#include <config/console.h>
-
-/** @file
- *
- * Serial console
- *
- */
-
-/* Set default console usage if applicable */
-#if ! ( defined ( CONSOLE_SERIAL ) && CONSOLE_EXPLICIT ( CONSOLE_SERIAL ) )
-#undef CONSOLE_SERIAL
-#define CONSOLE_SERIAL ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG )
-#endif
-
-struct console_driver serial_console __console_driver;
-
-static void serial_console_init ( void ) {
-	/*
-	 * Check if serial driver initialization is done.
-	 * If so, it's time to enable the serial console.
-	 */
-	if ( serial_initialized )
-		serial_console.disabled = 0;
-}
-
-struct console_driver serial_console __console_driver = {
-	.putchar = serial_putc,
-	.getchar = serial_getc,
-	.iskey = serial_ischar,
-	.disabled = CONSOLE_DISABLED,
-	.usage = CONSOLE_SERIAL,
-};
-
-/**
- * Serial console initialisation function
- */
-struct init_fn serial_console_init_fn __init_fn ( INIT_CONSOLE ) = {
-	.initialise = serial_console_init,
-};
diff --git a/roms/ipxe/src/core/settings.c b/roms/ipxe/src/core/settings.c
index 5e16b27d0..12e6c7d61 100644
--- a/roms/ipxe/src/core/settings.c
+++ b/roms/ipxe/src/core/settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -35,6 +39,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/uuid.h>
 #include <ipxe/uri.h>
 #include <ipxe/base16.h>
+#include <ipxe/base64.h>
 #include <ipxe/pci.h>
 #include <ipxe/init.h>
 #include <ipxe/version.h>
@@ -337,17 +342,20 @@ struct settings * autovivify_child_settings ( struct settings *parent,
  */
 const char * settings_name ( struct settings *settings ) {
 	static char buf[16];
-	char tmp[ sizeof ( buf ) ];
+	char tmp[ 1 /* '.' */ + sizeof ( buf ) ];
 
 	/* Find target settings block */
 	settings = settings_target ( settings );
 
 	/* Construct name */
-	for ( buf[2] = buf[0] = 0 ; settings ; settings = settings->parent ) {
-		memcpy ( tmp, buf, sizeof ( tmp ) );
-		snprintf ( buf, sizeof ( buf ), ".%s%s", settings->name, tmp );
+	buf[0] = '\0';
+	tmp[0] = '\0';
+	for ( ; settings->parent ; settings = settings->parent ) {
+		memcpy ( ( tmp + 1 ), buf, ( sizeof ( tmp ) - 1 ) );
+		snprintf ( buf, sizeof ( buf ), "%s%s", settings->name, tmp );
+		tmp[0] = '.';
 	}
-	return ( buf + 2 );
+	return buf;
 }
 
 /**
@@ -499,10 +507,10 @@ int register_settings ( struct settings *settings, struct settings *parent,
  */
 void unregister_settings ( struct settings *settings ) {
 	struct settings *child;
-	struct settings *tmp;
 
 	/* Unregister child settings */
-	list_for_each_entry_safe ( child, tmp, &settings->children, siblings ) {
+	while ( ( child = list_first_entry ( &settings->children,
+					     struct settings, siblings ) ) ) {
 		unregister_settings ( child );
 	}
 
@@ -1999,32 +2007,6 @@ const struct setting_type setting_type_uint32 __setting_type =
 	SETTING_TYPE_UINT ( SETTING_TYPE_INT32 );
 
 /**
- * Format hex string setting value
- *
- * @v delimiter		Byte delimiter
- * @v raw		Raw setting value
- * @v raw_len		Length of raw setting value
- * @v buf		Buffer to contain formatted value
- * @v len		Length of buffer
- * @ret len		Length of formatted value, or negative error
- */
-static int format_hex_setting ( const char *delimiter, const void *raw,
-				size_t raw_len, char *buf, size_t len ) {
-	const uint8_t *bytes = raw;
-	int used = 0;
-	unsigned int i;
-
-	if ( len )
-		buf[0] = 0; /* Ensure that a terminating NUL exists */
-	for ( i = 0 ; i < raw_len ; i++ ) {
-		used += ssnprintf ( ( buf + used ), ( len - used ),
-				    "%s%02x", ( used ? delimiter : "" ),
-				    bytes[i] );
-	}
-	return used;
-}
-
-/**
  * Parse hex string setting value (using colon delimiter)
  *
  * @v type		Setting type
@@ -2036,7 +2018,7 @@ static int format_hex_setting ( const char *delimiter, const void *raw,
  */
 static int parse_hex_setting ( const struct setting_type *type __unused,
 			       const char *value, void *buf, size_t len ) {
-	return hex_decode ( value, ':', buf, len );
+	return hex_decode ( ':', value, buf, len );
 }
 
 /**
@@ -2052,7 +2034,7 @@ static int parse_hex_setting ( const struct setting_type *type __unused,
 static int format_hex_colon_setting ( const struct setting_type *type __unused,
 				      const void *raw, size_t raw_len,
 				      char *buf, size_t len ) {
-	return format_hex_setting ( ":", raw, raw_len, buf, len );
+	return hex_encode ( ':', raw, raw_len, buf, len );
 }
 
 /**
@@ -2068,7 +2050,7 @@ static int format_hex_colon_setting ( const struct setting_type *type __unused,
 static int parse_hex_hyphen_setting ( const struct setting_type *type __unused,
 				      const char *value, void *buf,
 				      size_t len ) {
-	return hex_decode ( value, '-', buf, len );
+	return hex_decode ( '-', value, buf, len );
 }
 
 /**
@@ -2084,7 +2066,7 @@ static int parse_hex_hyphen_setting ( const struct setting_type *type __unused,
 static int format_hex_hyphen_setting ( const struct setting_type *type __unused,
 				       const void *raw, size_t raw_len,
 				       char *buf, size_t len ) {
-	return format_hex_setting ( "-", raw, raw_len, buf, len );
+	return hex_encode ( '-', raw, raw_len, buf, len );
 }
 
 /**
@@ -2099,7 +2081,7 @@ static int format_hex_hyphen_setting ( const struct setting_type *type __unused,
  */
 static int parse_hex_raw_setting ( const struct setting_type *type __unused,
 				   const char *value, void *buf, size_t len ) {
-	return hex_decode ( value, 0, buf, len );
+	return hex_decode ( 0, value, buf, len );
 }
 
 /**
@@ -2115,7 +2097,7 @@ static int parse_hex_raw_setting ( const struct setting_type *type __unused,
 static int format_hex_raw_setting ( const struct setting_type *type __unused,
 				    const void *raw, size_t raw_len,
 				    char *buf, size_t len ) {
-	return format_hex_setting ( "", raw, raw_len, buf, len );
+	return hex_encode ( 0, raw, raw_len, buf, len );
 }
 
 /** A hex-string setting (colon-delimited) */
@@ -2140,6 +2122,46 @@ const struct setting_type setting_type_hexraw __setting_type = {
 };
 
 /**
+ * Parse Base64-encoded setting value
+ *
+ * @v type		Setting type
+ * @v value		Formatted setting value
+ * @v buf		Buffer to contain raw value
+ * @v len		Length of buffer
+ * @v size		Integer size, in bytes
+ * @ret len		Length of raw value, or negative error
+ */
+static int parse_base64_setting ( const struct setting_type *type __unused,
+				  const char *value, void *buf, size_t len ) {
+
+	return base64_decode ( value, buf, len );
+}
+
+/**
+ * Format Base64-encoded setting value
+ *
+ * @v type		Setting type
+ * @v raw		Raw setting value
+ * @v raw_len		Length of raw setting value
+ * @v buf		Buffer to contain formatted value
+ * @v len		Length of buffer
+ * @ret len		Length of formatted value, or negative error
+ */
+static int format_base64_setting ( const struct setting_type *type __unused,
+				   const void *raw, size_t raw_len,
+				   char *buf, size_t len ) {
+
+	return base64_encode ( raw, raw_len, buf, len );
+}
+
+/** A Base64-encoded setting */
+const struct setting_type setting_type_base64 __setting_type = {
+	.name = "base64",
+	.parse = parse_base64_setting,
+	.format = format_base64_setting,
+};
+
+/**
  * Format UUID setting value
  *
  * @v type		Setting type
diff --git a/roms/ipxe/src/core/string.c b/roms/ipxe/src/core/string.c
index e53c283c2..3e658e54e 100644
--- a/roms/ipxe/src/core/string.c
+++ b/roms/ipxe/src/core/string.c
@@ -1,353 +1,501 @@
 /*
- *  Copyright (C) 1991, 1992  Linus Torvalds
- *  Copyright (C) 2004 Tobias Lorenz
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
  *
- *  string handling functions
- *  based on linux/lib/string.c
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-FILE_LICENCE ( GPL2_ONLY );
-
-/*
- * stupid library routines.. The optimized versions should generally be found
- * as inline code in <asm-xx/string.h>
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
  *
- * These are buggy as well..
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  *
- * * Fri Jun 25 1999, Ingo Oeser <ioe@informatik.tu-chemnitz.de>
- * -  Added strsep() which will replace strtok() soon (because strsep() is
- *    reentrant and should be faster). Use only strsep() in new code, please.
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
- 
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
 
-/* *** FROM string.c *** */
+/** @file
+ *
+ * String functions
+ *
+ */
 
-#ifndef __HAVE_ARCH_STRCPY
 /**
- * strcpy - Copy a %NUL terminated string
- * @dest: Where to copy the string to
- * @src: Where to copy the string from
+ * Fill memory region
+ *
+ * @v dest		Destination region
+ * @v character		Fill character
+ * @v len		Length
+ * @ret dest		Destination region
  */
-char * strcpy(char * dest,const char *src)
-{
-	char *tmp = dest;
+void * generic_memset ( void *dest, int character, size_t len ) {
+	uint8_t *dest_bytes = dest;
 
-	while ((*dest++ = *src++) != '\0')
-		/* nothing */;
-	return tmp;
+	while ( len-- )
+		*(dest_bytes++) = character;
+	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRNCPY
 /**
- * strncpy - Copy a length-limited, %NUL-terminated string
- * @dest: Where to copy the string to
- * @src: Where to copy the string from
- * @count: The maximum number of bytes to copy
+ * Copy memory region
  *
- * Note that unlike userspace strncpy, this does not %NUL-pad the buffer.
- * However, the result is not %NUL-terminated if the source exceeds
- * @count bytes.
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ * @ret dest		Destination region
  */
-char * strncpy(char * dest,const char *src,size_t count)
-{
-	char *tmp = dest;
-
-	while (count-- && (*dest++ = *src++) != '\0')
-		/* nothing */;
+void * generic_memcpy ( void *dest, const void *src, size_t len ) {
+	const uint8_t *src_bytes = src;
+	uint8_t *dest_bytes = dest;
 
-	return tmp;
+	while ( len-- )
+		*(dest_bytes++) = *(src_bytes++);
+	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCAT
 /**
- * strcat - Append one %NUL-terminated string to another
- * @dest: The string to be appended to
- * @src: The string to append to it
+ * Copy (possibly overlapping) memory region
+ *
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ * @ret dest		Destination region
  */
-char * strcat(char * dest, const char * src)
-{
-	char *tmp = dest;
-
-	while (*dest)
-		dest++;
-	while ((*dest++ = *src++) != '\0')
-		;
+void * generic_memmove ( void *dest, const void *src, size_t len ) {
+	const uint8_t *src_bytes = ( src + len );
+	uint8_t *dest_bytes = ( dest + len );
+
+	if ( dest < src )
+		return memcpy ( dest, src, len );
+	while ( len-- )
+		*(--dest_bytes) = *(--src_bytes);
+	return dest;
+}
 
-	return tmp;
+/**
+ * Compare memory regions
+ *
+ * @v first		First region
+ * @v second		Second region
+ * @v len		Length
+ * @ret diff		Difference
+ */
+int memcmp ( const void *first, const void *second, size_t len ) {
+	const uint8_t *first_bytes = first;
+	const uint8_t *second_bytes = second;
+	int diff;
+
+	while ( len-- ) {
+		diff = ( *(second_bytes++) - *(first_bytes++) );
+		if ( diff )
+			return diff;
+	}
+	return 0;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCMP
 /**
- * strcmp - Compare two strings
- * @cs: One string
- * @ct: Another string
+ * Find character within a memory region
+ *
+ * @v src		Source region
+ * @v character		Character to find
+ * @v len		Length
+ * @ret found		Found character, or NULL if not found
  */
-int strcmp(const char * cs,const char * ct)
-{
-	register signed char __res;
+void * memchr ( const void *src, int character, size_t len ) {
+	const uint8_t *src_bytes = src;
 
-	while (1) {
-		if ((__res = *cs - *ct++) != 0 || !*cs++)
-			break;
+	for ( ; len-- ; src_bytes++ ) {
+		if ( *src_bytes == character )
+			return ( ( void * ) src_bytes );
 	}
-
-	return __res;
+	return NULL;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRNCMP
 /**
- * strncmp - Compare two length-limited strings
- * @cs: One string
- * @ct: Another string
- * @count: The maximum number of bytes to compare
+ * Swap memory regions
+ *
+ * @v first		First region
+ * @v second		Second region
+ * @v len		Length
+ * @ret first		First region
  */
-int strncmp(const char * cs,const char * ct,size_t count)
-{
-	register signed char __res = 0;
-
-	while (count) {
-		if ((__res = *cs - *ct++) != 0 || !*cs++)
-			break;
-		count--;
+void * memswap ( void *first, void *second, size_t len ) {
+	uint8_t *first_bytes = first;
+	uint8_t *second_bytes = second;
+	uint8_t temp;
+
+	for ( ; len-- ; first_bytes++, second_bytes++ ) {
+		temp = *first_bytes;
+		*first_bytes = *second_bytes;
+		*second_bytes = temp;
 	}
+	return first;
+}
+
+/**
+ * Compare strings
+ *
+ * @v first		First string
+ * @v second		Second string
+ * @ret diff		Difference
+ */
+int strcmp ( const char *first, const char *second ) {
 
-	return __res;
+	return strncmp ( first, second, ~( ( size_t ) 0 ) );
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCASECMP
-int strcasecmp(const char *a, const char *b)
-{
-	while (*a && *b && (*a & ~0x20) == (*b & ~0x20)) {a++; b++; }
-	return((*a & ~0x20) - (*b & ~0x20));
+/**
+ * Compare strings
+ *
+ * @v first		First string
+ * @v second		Second string
+ * @v max		Maximum length to compare
+ * @ret diff		Difference
+ */
+int strncmp ( const char *first, const char *second, size_t max ) {
+	const uint8_t *first_bytes = ( ( const uint8_t * ) first );
+	const uint8_t *second_bytes = ( ( const uint8_t * ) second );
+	int diff;
+
+	for ( ; max-- ; first_bytes++, second_bytes++ ) {
+		diff = ( *second_bytes - *first_bytes );
+		if ( diff )
+			return diff;
+		if ( ! *first_bytes )
+			return 0;
+	}
+	return 0;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCHR
 /**
- * strchr - Find the first occurrence of a character in a string
- * @s: The string to be searched
- * @c: The character to search for
+ * Compare case-insensitive strings
+ *
+ * @v first		First string
+ * @v second		Second string
+ * @ret diff		Difference
  */
-char * strchr(const char * s, int c)
-{
-	for(; *s != (char) c; ++s)
-		if (*s == '\0')
-			return NULL;
-	return (char *) s;
+int strcasecmp ( const char *first, const char *second ) {
+	const uint8_t *first_bytes = ( ( const uint8_t * ) first );
+	const uint8_t *second_bytes = ( ( const uint8_t * ) second );
+	int diff;
+
+	for ( ; ; first_bytes++, second_bytes++ ) {
+		diff = ( toupper ( *second_bytes ) -
+			 toupper ( *first_bytes ) );
+		if ( diff )
+			return diff;
+		if ( ! *first_bytes )
+			return 0;
+	}
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRRCHR
 /**
- * strrchr - Find the last occurrence of a character in a string
- * @s: The string to be searched
- * @c: The character to search for
+ * Get length of string
+ *
+ * @v src		String
+ * @ret len		Length
  */
-char * strrchr(const char * s, int c)
-{
-       const char *p = s + strlen(s);
-       do {
-           if (*p == (char)c)
-               return (char *)p;
-       } while (--p >= s);
-       return NULL;
+size_t strlen ( const char *src ) {
+
+	return strnlen ( src, ~( ( size_t ) 0 ) );
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRLEN
 /**
- * strlen - Find the length of a string
- * @s: The string to be sized
+ * Get length of string
+ *
+ * @v src		String
+ * @v max		Maximum length
+ * @ret len		Length
  */
-size_t strlen(const char * s)
-{
-	const char *sc;
+size_t strnlen ( const char *src, size_t max ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	size_t len = 0;
 
-	for (sc = s; *sc != '\0'; ++sc)
-		/* nothing */;
-	return sc - s;
+	while ( max-- && *(src_bytes++) )
+		len++;
+	return len;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRNLEN
 /**
- * strnlen - Find the length of a length-limited string
- * @s: The string to be sized
- * @count: The maximum number of bytes to search
+ * Find character within a string
+ *
+ * @v src		String
+ * @v character		Character to find
+ * @ret found		Found character, or NULL if not found
  */
-size_t strnlen(const char * s, size_t count)
-{
-	const char *sc;
+char * strchr ( const char *src, int character ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
 
-	for (sc = s; count-- && *sc != '\0'; ++sc)
-		/* nothing */;
-	return sc - s;
+	for ( ; ; src_bytes++ ) {
+		if ( *src_bytes == character )
+			return ( ( char * ) src_bytes );
+		if ( ! *src_bytes )
+			return NULL;
+	}
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMSET
 /**
- * memset - Fill a region of memory with the given value
- * @s: Pointer to the start of the area.
- * @c: The byte to fill the area with
- * @count: The size of the area.
+ * Find rightmost character within a string
  *
- * Do not use memset() to access IO space, use memset_io() instead.
+ * @v src		String
+ * @v character		Character to find
+ * @ret found		Found character, or NULL if not found
  */
-void * memset(void * s,int c,size_t count)
-{
-	char *xs = (char *) s;
+char * strrchr ( const char *src, int character ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	const uint8_t *start = src_bytes;
+
+	while ( *src_bytes )
+		src_bytes++;
+	for ( src_bytes-- ; src_bytes >= start ; src_bytes-- ) {
+		if ( *src_bytes == character )
+			return ( ( char * ) src_bytes );
+	}
+	return NULL;
+}
 
-	while (count--)
-		*xs++ = c;
+/**
+ * Find substring
+ *
+ * @v haystack		String
+ * @v needle		Substring
+ * @ret found		Found substring, or NULL if not found
+ */
+char * strstr ( const char *haystack, const char *needle ) {
+	size_t len = strlen ( needle );
 
-	return s;
+	for ( ; *haystack ; haystack++ ) {
+		if ( memcmp ( haystack, needle, len ) == 0 )
+			return ( ( char * ) haystack );
+	}
+	return NULL;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMCPY
 /**
- * memcpy - Copy one area of memory to another
- * @dest: Where to copy to
- * @src: Where to copy from
- * @count: The size of the area.
+ * Copy string
  *
- * You should not use this function to access IO space, use memcpy_toio()
- * or memcpy_fromio() instead.
+ * @v dest		Destination string
+ * @v src		Source string
+ * @ret dest		Destination string
  */
-void * memcpy(void * dest,const void *src,size_t count)
-{
-	char *tmp = (char *) dest, *s = (char *) src;
+char * strcpy ( char *dest, const char *src ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	uint8_t *dest_bytes = ( ( uint8_t * ) dest );
+
+	/* We cannot use strncpy(), since that would pad the destination */
+	for ( ; ; src_bytes++, dest_bytes++ ) {
+		*dest_bytes = *src_bytes;
+		if ( ! *dest_bytes )
+			break;
+	}
+	return dest;
+}
 
-	while (count--)
-		*tmp++ = *s++;
+/**
+ * Copy string
+ *
+ * @v dest		Destination string
+ * @v src		Source string
+ * @v max		Maximum length
+ * @ret dest		Destination string
+ */
+char * strncpy ( char *dest, const char *src, size_t max ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	uint8_t *dest_bytes = ( ( uint8_t * ) dest );
 
+	for ( ; max ; max--, src_bytes++, dest_bytes++ ) {
+		*dest_bytes = *src_bytes;
+		if ( ! *dest_bytes )
+			break;
+	}
+	while ( max-- )
+		*(dest_bytes++) = '\0';
 	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMMOVE
 /**
- * memmove - Copy one area of memory to another
- * @dest: Where to copy to
- * @src: Where to copy from
- * @count: The size of the area.
+ * Concatenate string
  *
- * Unlike memcpy(), memmove() copes with overlapping areas.
+ * @v dest		Destination string
+ * @v src		Source string
+ * @ret dest		Destination string
  */
-void * memmove(void * dest,const void *src,size_t count)
-{
-	char *tmp, *s;
-
-	if (dest <= src) {
-		tmp = (char *) dest;
-		s = (char *) src;
-		while (count--)
-			*tmp++ = *s++;
-		}
-	else {
-		tmp = (char *) dest + count;
-		s = (char *) src + count;
-		while (count--)
-			*--tmp = *--s;
-		}
+char * strcat ( char *dest, const char *src ) {
 
+	strcpy ( ( dest + strlen ( dest ) ), src );
 	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMCMP
 /**
- * memcmp - Compare two areas of memory
- * @cs: One area of memory
- * @ct: Another area of memory
- * @count: The size of the area.
+ * Duplicate string
+ *
+ * @v src		Source string
+ * @ret dup		Duplicated string, or NULL if allocation failed
  */
-int memcmp(const void * cs,const void * ct,size_t count)
-{
-	const unsigned char *su1, *su2;
-	int res = 0;
+char * strdup ( const char *src ) {
 
-	for( su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--)
-		if ((res = *su1 - *su2) != 0)
-			break;
-	return res;
+	return strndup ( src, ~( ( size_t ) 0 ) );
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRSTR
 /**
- * strstr - Find the first substring in a %NUL terminated string
- * @s1: The string to be searched
- * @s2: The string to search for
+ * Duplicate string
+ *
+ * @v src		Source string
+ * @v max		Maximum length
+ * @ret dup		Duplicated string, or NULL if allocation failed
  */
-char * strstr(const char * s1,const char * s2)
-{
-	int l1, l2;
-
-	l2 = strlen(s2);
-	if (!l2)
-		return (char *) s1;
-	l1 = strlen(s1);
-	while (l1 >= l2) {
-		l1--;
-		if (!memcmp(s1,s2,l2))
-			return (char *) s1;
-		s1++;
-	}
-	return NULL;
+char * strndup ( const char *src, size_t max ) {
+	size_t len = strnlen ( src, max );
+        char *dup;
+
+        dup = malloc ( len + 1 /* NUL */ );
+        if ( dup ) {
+		memcpy ( dup, src, len );
+		dup[len] = '\0';
+        }
+        return dup;
+}
+
+/**
+ * Calculate digit value
+ *
+ * @v character		Digit character
+ * @ret digit		Digit value
+ *
+ * Invalid digits will be returned as a value greater than or equal to
+ * the numeric base.
+ */
+unsigned int digit_value ( unsigned int character ) {
+
+	if ( character >= 'a' )
+		return ( character - ( 'a' - 10 ) );
+	if ( character >= 'A' )
+		return ( character - ( 'A' - 10 ) );
+	if ( character <= '9' )
+		return ( character - '0' );
+	return character;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMCHR
 /**
- * memchr - Find a character in an area of memory.
- * @s: The memory area
- * @c: The byte to search for
- * @n: The size of the area.
+ * Preprocess string for strtoul() or strtoull()
  *
- * returns the address of the first occurrence of @c, or %NULL
- * if @c is not found
+ * @v string		String
+ * @v negate		Final value should be negated
+ * @v base		Numeric base
+ * @ret string		Remaining string
  */
-void * memchr(const void *s, int c, size_t n)
-{
-	const unsigned char *p = s;
-	while (n-- != 0) {
-        	if ((unsigned char)c == *p++) {
-			return (void *)(p-1);
+static const char * strtoul_pre ( const char *string, int *negate, int *base ) {
+
+	/* Skip any leading whitespace */
+	while ( isspace ( *string ) )
+		string++;
+
+	/* Process arithmetic sign, if present */
+	*negate = 0;
+	if ( *string == '-' ) {
+		string++;
+		*negate = 1;
+	} else if ( *string == '+' ) {
+		string++;
+	}
+
+	/* Process base, if present */
+	if ( *base == 0 ) {
+		*base = 10;
+		if ( *string == '0' ) {
+			string++;
+			*base = 8;
+			if ( ( *string & ~0x20 ) == 'X' ) {
+				string++;
+				*base = 16;
+			}
 		}
 	}
-	return NULL;
+
+	return string;
 }
 
-#endif
+/**
+ * Convert string to numeric value
+ *
+ * @v string		String
+ * @v endp		End pointer (or NULL)
+ * @v base		Numeric base (or zero to autodetect)
+ * @ret value		Numeric value
+ */
+unsigned long strtoul ( const char *string, char **endp, int base ) {
+	unsigned long value = 0;
+	unsigned int digit;
+	int negate;
+
+	/* Preprocess string */
+	string = strtoul_pre ( string, &negate, &base );
+
+	/* Process digits */
+	for ( ; ; string++ ) {
+		digit = digit_value ( *string );
+		if ( digit >= ( unsigned int ) base )
+			break;
+		value = ( ( value * base ) + digit );
+	}
+
+	/* Negate value if, applicable */
+	if ( negate )
+		value = -value;
 
-char * strndup(const char *s, size_t n)
-{
-        size_t len = strnlen(s,n);
-        char *new;
+	/* Fill in end pointer, if applicable */
+	if ( endp )
+		*endp = ( ( char * ) string );
 
-        new = malloc(len+1);
-        if (new) {
-                new[len] = '\0';
-                memcpy(new,s,len);
-        }
-        return new;
+	return value;
 }
 
-char * strdup(const char *s) {
-	return strndup(s, ~((size_t)0));
+/**
+ * Convert string to numeric value
+ *
+ * @v string		String
+ * @v endp		End pointer (or NULL)
+ * @v base		Numeric base (or zero to autodetect)
+ * @ret value		Numeric value
+ */
+unsigned long long strtoull ( const char *string, char **endp, int base ) {
+	unsigned long long value = 0;
+	unsigned int digit;
+	int negate;
+
+	/* Preprocess string */
+	string = strtoul_pre ( string, &negate, &base );
+
+	/* Process digits */
+	for ( ; ; string++ ) {
+		digit = digit_value ( *string );
+		if ( digit >= ( unsigned int ) base )
+			break;
+		value = ( ( value * base ) + digit );
+	}
+
+	/* Negate value if, applicable */
+	if ( negate )
+		value = -value;
+
+	/* Fill in end pointer, if applicable */
+	if ( endp )
+		*endp = ( ( char * ) string );
+
+	return value;
 }
diff --git a/roms/ipxe/src/core/stringextra.c b/roms/ipxe/src/core/stringextra.c
index 0a509852e..18ffc6301 100644
--- a/roms/ipxe/src/core/stringextra.c
+++ b/roms/ipxe/src/core/stringextra.c
@@ -38,122 +38,6 @@ FILE_LICENCE ( GPL2_ONLY );
 
 /* *** FROM string.c *** */
 
-#ifndef __HAVE_ARCH_STRNICMP
-/**
- * strnicmp - Case insensitive, length-limited string comparison
- * @s1: One string
- * @s2: The other string
- * @len: the maximum number of characters to compare
- */
-int strnicmp(const char *s1, const char *s2, size_t len)
-{
-	/* Yes, Virginia, it had better be unsigned */
-	unsigned char c1, c2;
-
-	c1 = 0;	c2 = 0;
-	if (len) {
-		do {
-			c1 = *s1; c2 = *s2;
-			s1++; s2++;
-			if (!c1)
-				break;
-			if (!c2)
-				break;
-			if (c1 == c2)
-				continue;
-			c1 = tolower(c1);
-			c2 = tolower(c2);
-			if (c1 != c2)
-				break;
-		} while (--len);
-	}
-	return (int)c1 - (int)c2;
-}
-#endif
-
-char * ___strtok;
-
-#ifndef __HAVE_ARCH_STRNCAT
-/**
- * strncat - Append a length-limited, %NUL-terminated string to another
- * @dest: The string to be appended to
- * @src: The string to append to it
- * @count: The maximum numbers of bytes to copy
- *
- * Note that in contrast to strncpy, strncat ensures the result is
- * terminated.
- */
-char * strncat(char *dest, const char *src, size_t count)
-{
-	char *tmp = dest;
-
-	if (count) {
-		while (*dest)
-			dest++;
-		while ((*dest++ = *src++)) {
-			if (--count == 0) {
-				*dest = '\0';
-				break;
-			}
-		}
-	}
-
-	return tmp;
-}
-#endif
-
-#ifndef __HAVE_ARCH_STRSPN
-/**
- * strspn - Calculate the length of the initial substring of @s which only
- * 	contain letters in @accept
- * @s: The string to be searched
- * @accept: The string to search for
- */
-size_t strspn(const char *s, const char *accept)
-{
-	const char *p;
-	const char *a;
-	size_t count = 0;
-
-	for (p = s; *p != '\0'; ++p) {
-		for (a = accept; *a != '\0'; ++a) {
-			if (*p == *a)
-				break;
-		}
-		if (*a == '\0')
-			return count;
-		++count;
-	}
-
-	return count;
-}
-#endif
-
-#ifndef __HAVE_ARCH_STRCSPN
-/**
- * strcspn - Calculate the length of the initial substring of @s which only
- * 	contain letters not in @reject
- * @s: The string to be searched
- * @accept: The string to search for
- */
-size_t strcspn(const char *s, const char *reject)
-{
-	const char *p;
-	const char *r;
-	size_t count = 0;
-
-	for (p = s; *p != '\0'; ++p) {
-		for (r = reject; *r != '\0'; ++r) {
-			if (*p == *r)
-				return count;
-		}
-		++count;
-	}
-
-	return count;
-}
-#endif
-
 #ifndef __HAVE_ARCH_STRPBRK
 /**
  * strpbrk - Find the first occurrence of a set of characters
@@ -174,35 +58,6 @@ char * strpbrk(const char * cs,const char * ct)
 }
 #endif
 
-#ifndef __HAVE_ARCH_STRTOK
-/**
- * strtok - Split a string into tokens
- * @s: The string to be searched
- * @ct: The characters to search for
- *
- * WARNING: strtok is deprecated, use strsep instead.
- */
-char * strtok(char * s,const char * ct)
-{
-	char *sbegin, *send;
-
-	sbegin  = s ? s : ___strtok;
-	if (!sbegin) {
-		return NULL;
-	}
-	sbegin += strspn(sbegin,ct);
-	if (*sbegin == '\0') {
-		___strtok = NULL;
-		return( NULL );
-	}
-	send = strpbrk( sbegin, ct);
-	if (send && *send != '\0')
-		*send++ = '\0';
-	___strtok = send;
-	return (sbegin);
-}
-#endif
-
 #ifndef __HAVE_ARCH_STRSEP
 /**
  * strsep - Split a string into tokens
@@ -230,46 +85,3 @@ char * strsep(char **s, const char *ct)
 	return sbegin;
 }
 #endif
-
-#ifndef __HAVE_ARCH_BCOPY
-/**
- * bcopy - Copy one area of memory to another
- * @src: Where to copy from
- * @dest: Where to copy to
- * @count: The size of the area.
- *
- * Note that this is the same as memcpy(), with the arguments reversed.
- * memcpy() is the standard, bcopy() is a legacy BSD function.
- *
- * You should not use this function to access IO space, use memcpy_toio()
- * or memcpy_fromio() instead.
- */
-char * bcopy(const char * src, char * dest, int count)
-{
-	return memmove(dest,src,count);
-}
-#endif
-
-#ifndef __HAVE_ARCH_MEMSCAN
-/**
- * memscan - Find a character in an area of memory.
- * @addr: The memory area
- * @c: The byte to search for
- * @size: The size of the area.
- *
- * returns the address of the first occurrence of @c, or 1 byte past
- * the area if @c is not found
- */
-void * memscan(const void * addr, int c, size_t size)
-{
-	unsigned char * p = (unsigned char *) addr;
-
-	while (size) {
-		if (*p == c)
-			return (void *) p;
-		p++;
-		size--;
-	}
-  	return (void *) p;
-}
-#endif
diff --git a/roms/ipxe/src/core/strtoull.c b/roms/ipxe/src/core/strtoull.c
deleted file mode 100644
index 00986eef0..000000000
--- a/roms/ipxe/src/core/strtoull.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (C) 2006 Michael Brown <mbrown@fensystems.co.uk>
- * Copyright (C) 2010 Piotr Jaroszyński <p.jaroszynski@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stdlib.h>
-#include <ctype.h>
-
-/*
- * Despite being exactly the same as strtoul() except the long long instead of
- * long it ends up being much bigger so provide a separate implementation in a
- * separate object so that it won't be linked in if not used.
- */
-unsigned long long strtoull ( const char *p, char **endp, int base ) {
-	unsigned long long ret = 0;
-	int negative = 0;
-	unsigned int charval;
-
-	while ( isspace ( *p ) )
-		p++;
-
-	if ( *p == '-' ) {
-		negative = 1;
-		p++;
-	}
-
-	base = strtoul_base ( &p, base );
-
-	while ( 1 ) {
-		charval = strtoul_charval ( *p );
-		if ( charval >= ( unsigned int ) base )
-			break;
-		ret = ( ( ret * base ) + charval );
-		p++;
-	}
-
-	if ( negative )
-		ret = -ret;
-
-	if ( endp )
-		*endp = ( char * ) p;
-
-	return ( ret );
-}
diff --git a/roms/ipxe/src/core/time.c b/roms/ipxe/src/core/time.c
index f70e1981d..29a924ebe 100644
--- a/roms/ipxe/src/core/time.c
+++ b/roms/ipxe/src/core/time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <time.h>
 
diff --git a/roms/ipxe/src/core/timer.c b/roms/ipxe/src/core/timer.c
index 18c2b2849..dbd89f12b 100644
--- a/roms/ipxe/src/core/timer.c
+++ b/roms/ipxe/src/core/timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <unistd.h>
 
diff --git a/roms/ipxe/src/core/uart.c b/roms/ipxe/src/core/uart.c
new file mode 100644
index 000000000..b85fe0767
--- /dev/null
+++ b/roms/ipxe/src/core/uart.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/uart.h>
+
+/** Timeout for transmit holding register to become empty */
+#define UART_THRE_TIMEOUT_MS 100
+
+/** Timeout for transmitter to become empty */
+#define UART_TEMT_TIMEOUT_MS 1000
+
+/**
+ * Transmit data
+ *
+ * @v uart		UART
+ * @v data		Data
+ */
+void uart_transmit ( struct uart *uart, uint8_t data ) {
+	unsigned int i;
+	uint8_t lsr;
+
+	/* Wait for transmitter holding register to become empty */
+	for ( i = 0 ; i < UART_THRE_TIMEOUT_MS ; i++ ) {
+		lsr = uart_read ( uart, UART_LSR );
+		if ( lsr & UART_LSR_THRE )
+			break;
+		mdelay ( 1 );
+	}
+
+	/* Transmit data (even if we timed out) */
+	uart_write ( uart, UART_THR, data );
+}
+
+/**
+ * Flush data
+ *
+ * @v uart		UART
+ */
+void uart_flush ( struct uart *uart ) {
+	unsigned int i;
+	uint8_t lsr;
+
+	/* Wait for transmitter and receiver to become empty */
+	for ( i = 0 ; i < UART_TEMT_TIMEOUT_MS ; i++ ) {
+		uart_read ( uart, UART_RBR );
+		lsr = uart_read ( uart, UART_LSR );
+		if ( ( lsr & UART_LSR_TEMT ) && ! ( lsr & UART_LSR_DR ) )
+			break;
+	}
+}
+
+/**
+ * Check for existence of UART
+ *
+ * @v uart		UART
+ * @ret rc		Return status code
+ */
+int uart_exists ( struct uart *uart ) {
+
+	/* Fail if no UART port is defined */
+	if ( ! uart->base )
+		return -ENODEV;
+
+	/* Fail if UART scratch register seems not to be present */
+	uart_write ( uart, UART_SCR, 0x18 );
+	if ( uart_read ( uart, UART_SCR ) != 0x18 )
+		return -ENODEV;
+	uart_write ( uart, UART_SCR, 0xae );
+	if ( uart_read ( uart, UART_SCR ) != 0xae )
+		return -ENODEV;
+
+	return 0;
+}
+
+/**
+ * Initialise UART
+ *
+ * @v uart		UART
+ * @v baud		Baud rate, or zero to leave unchanged
+ * @v lcr		Line control register value, or zero to leave unchanged
+ * @ret rc		Return status code
+ */
+int uart_init ( struct uart *uart, unsigned int baud, uint8_t lcr ) {
+	uint8_t dlm;
+	uint8_t dll;
+	int rc;
+
+	/* Check for existence of UART */
+	if ( ( rc = uart_exists ( uart ) ) != 0 )
+		return rc;
+
+	/* Configure divisor and line control register, if applicable */
+	if ( ! lcr )
+		lcr = uart_read ( uart, UART_LCR );
+	uart->lcr = lcr;
+	uart_write ( uart, UART_LCR, ( lcr | UART_LCR_DLAB ) );
+	if ( baud ) {
+		uart->divisor = ( UART_MAX_BAUD / baud );
+		dlm = ( ( uart->divisor >> 8 ) & 0xff );
+		dll = ( ( uart->divisor >> 0 ) & 0xff );
+		uart_write ( uart, UART_DLM, dlm );
+		uart_write ( uart, UART_DLL, dll );
+	} else {
+		dlm = uart_read ( uart, UART_DLM );
+		dll = uart_read ( uart, UART_DLL );
+		uart->divisor = ( ( dlm << 8 ) | dll );
+	}
+	uart_write ( uart, UART_LCR, ( lcr & ~UART_LCR_DLAB ) );
+
+	/* Disable interrupts */
+	uart_write ( uart, UART_IER, 0 );
+
+	/* Enable FIFOs */
+	uart_write ( uart, UART_FCR, UART_FCR_FE );
+
+	/* Assert DTR and RTS */
+	uart_write ( uart, UART_MCR, ( UART_MCR_DTR | UART_MCR_RTS ) );
+
+	/* Flush any stale data */
+	uart_flush ( uart );
+
+	return 0;
+}
diff --git a/roms/ipxe/src/core/uri.c b/roms/ipxe/src/core/uri.c
index 9ec21cee4..3b5f270fe 100644
--- a/roms/ipxe/src/core/uri.c
+++ b/roms/ipxe/src/core/uri.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -661,6 +665,7 @@ struct uri * resolve_uri ( const struct uri *base_uri,
  * Construct TFTP URI from next-server and filename
  *
  * @v next_server	Next-server address
+ * @v port		Port number, or zero to use the default port
  * @v filename		Filename
  * @ret uri		URI, or NULL on failure
  *
@@ -669,12 +674,18 @@ struct uri * resolve_uri ( const struct uri *base_uri,
  * generic URI parser.  We provide a mechanism for directly
  * constructing a TFTP URI from the next-server and filename.
  */
-struct uri * tftp_uri ( struct in_addr next_server, const char *filename ) {
+struct uri * tftp_uri ( struct in_addr next_server, unsigned int port,
+			const char *filename ) {
+	char buf[ 6 /* "65535" + NUL */ ];
 	struct uri uri;
 
 	memset ( &uri, 0, sizeof ( uri ) );
 	uri.scheme = "tftp";
 	uri.host = inet_ntoa ( next_server );
+	if ( port ) {
+		snprintf ( buf, sizeof ( buf ), "%d", port );
+		uri.port = buf;
+	}
 	uri.path = filename;
 	return uri_dup ( &uri );
 }
diff --git a/roms/ipxe/src/core/uuid.c b/roms/ipxe/src/core/uuid.c
index 27a249da8..b8d21de17 100644
--- a/roms/ipxe/src/core/uuid.c
+++ b/roms/ipxe/src/core/uuid.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/core/version.c b/roms/ipxe/src/core/version.c
index 1e1e9daca..c984335c2 100644
--- a/roms/ipxe/src/core/version.c
+++ b/roms/ipxe/src/core/version.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -29,6 +33,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/features.h>
 #include <ipxe/version.h>
 #include <config/general.h>
+#include <config/branding.h>
 
 /**
  * Create wide-character version of string
diff --git a/roms/ipxe/src/core/vsprintf.c b/roms/ipxe/src/core/vsprintf.c
index 54811b11b..cb3bec5dd 100644
--- a/roms/ipxe/src/core/vsprintf.c
+++ b/roms/ipxe/src/core/vsprintf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
diff --git a/roms/ipxe/src/core/wchar.c b/roms/ipxe/src/core/wchar.c
index 7fabca470..860322820 100644
--- a/roms/ipxe/src/core/wchar.c
+++ b/roms/ipxe/src/core/wchar.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/core/xfer.c b/roms/ipxe/src/core/xfer.c
index 8d4bc9f53..112fee1bf 100644
--- a/roms/ipxe/src/core/xfer.c
+++ b/roms/ipxe/src/core/xfer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdlib.h>
@@ -134,18 +138,8 @@ size_t xfer_window ( struct interface *intf ) {
  * generating an xfer_window_changed() message.
  */
 void xfer_window_changed ( struct interface *intf ) {
-	struct interface *dest;
-	xfer_window_changed_TYPE ( void * ) *op =
-		intf_get_dest_op ( intf, xfer_window_changed, &dest );
-	void *object = intf_object ( dest );
 
-	if ( op ) {
-		op ( object );
-	} else {
-		/* Default is to do nothing */
-	}
-
-	intf_put ( dest );
+	intf_poke ( intf, xfer_window_changed );
 }
 
 /**
@@ -365,3 +359,34 @@ int xfer_seek ( struct interface *intf, off_t offset ) {
 
 	return xfer_deliver ( intf, iobuf, &meta );
 }
+
+/**
+ * Check that data is delivered strictly in order
+ *
+ * @v meta		Data transfer metadata
+ * @v pos		Current position
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+int xfer_check_order ( struct xfer_metadata *meta, size_t *pos, size_t len ) {
+	size_t new_pos;
+
+	/* Allow out-of-order zero-length packets (as used by xfer_seek()) */
+	if ( len == 0 )
+		return 0;
+
+	/* Calculate position of this delivery */
+	new_pos = *pos;
+	if ( meta->flags & XFER_FL_ABS_OFFSET )
+		new_pos = 0;
+	new_pos += meta->offset;
+
+	/* Fail if delivery position is not equal to current position */
+	if ( new_pos != *pos )
+		return -EPROTO;
+
+	/* Update current position */
+	*pos += len;
+
+	return 0;
+}
diff --git a/roms/ipxe/src/core/xferbuf.c b/roms/ipxe/src/core/xferbuf.c
index a0457feee..240118557 100644
--- a/roms/ipxe/src/core/xferbuf.c
+++ b/roms/ipxe/src/core/xferbuf.c
@@ -15,15 +15,21 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
 #include <ipxe/xfer.h>
 #include <ipxe/iobuf.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/profile.h>
 #include <ipxe/xferbuf.h>
 
 /** @file
@@ -32,14 +38,26 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+/** Data delivery profiler */
+static struct profiler xferbuf_deliver_profiler __profiler =
+	{ .name = "xferbuf.deliver" };
+
+/** Data write profiler */
+static struct profiler xferbuf_write_profiler __profiler =
+	{ .name = "xferbuf.write" };
+
+/** Data read profiler */
+static struct profiler xferbuf_read_profiler __profiler =
+	{ .name = "xferbuf.read" };
+
 /**
- * Finish using data transfer buffer
+ * Free data transfer buffer
  *
  * @v xferbuf		Data transfer buffer
  */
-void xferbuf_done ( struct xfer_buffer *xferbuf ) {
-	free ( xferbuf->data );
-	xferbuf->data = NULL;
+void xferbuf_free ( struct xfer_buffer *xferbuf ) {
+
+	xferbuf->op->realloc ( xferbuf, 0 );
 	xferbuf->len = 0;
 	xferbuf->pos = 0;
 }
@@ -52,26 +70,78 @@ void xferbuf_done ( struct xfer_buffer *xferbuf ) {
  * @ret rc		Return status code
  */
 static int xferbuf_ensure_size ( struct xfer_buffer *xferbuf, size_t len ) {
-	void *new_data;
+	int rc;
 
 	/* If buffer is already large enough, do nothing */
 	if ( len <= xferbuf->len )
 		return 0;
 
 	/* Extend buffer */
-	new_data = realloc ( xferbuf->data, len );
-	if ( ! new_data ) {
+	if ( ( rc = xferbuf->op->realloc ( xferbuf, len ) ) != 0 ) {
 		DBGC ( xferbuf, "XFERBUF %p could not extend buffer to "
-		       "%zd bytes\n", xferbuf, len );
-		return -ENOSPC;
+		       "%zd bytes: %s\n", xferbuf, len, strerror ( rc ) );
+		return rc;
 	}
-	xferbuf->data = new_data;
 	xferbuf->len = len;
 
 	return 0;
 }
 
 /**
+ * Write to data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to write
+ * @v len		Length of data
+ */
+int xferbuf_write ( struct xfer_buffer *xferbuf, size_t offset,
+		    const void *data, size_t len ) {
+	size_t max_len;
+	int rc;
+
+	/* Check for overflow */
+	max_len = ( offset + len );
+	if ( max_len < offset )
+		return -EOVERFLOW;
+
+	/* Ensure buffer is large enough to contain this write */
+	if ( ( rc = xferbuf_ensure_size ( xferbuf, max_len ) ) != 0 )
+		return rc;
+
+	/* Copy data to buffer */
+	profile_start ( &xferbuf_write_profiler );
+	xferbuf->op->write ( xferbuf, offset, data, len );
+	profile_stop ( &xferbuf_write_profiler );
+
+	return 0;
+}
+
+/**
+ * Read from data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to write
+ * @v len		Length of data
+ */
+int xferbuf_read ( struct xfer_buffer *xferbuf, size_t offset,
+		   void *data, size_t len ) {
+
+	/* Check that read is within buffer range */
+	if ( ( offset > xferbuf->len ) ||
+	     ( len > ( xferbuf->len - offset ) ) )
+		return -ENOENT;
+
+	/* Copy data from buffer */
+	profile_start ( &xferbuf_read_profiler );
+	xferbuf->op->read ( xferbuf, offset, data, len );
+	profile_stop ( &xferbuf_read_profiler );
+
+	return 0;
+}
+
+/**
  * Add received data to data transfer buffer
  *
  * @v xferbuf		Data transfer buffer
@@ -81,28 +151,174 @@ static int xferbuf_ensure_size ( struct xfer_buffer *xferbuf, size_t len ) {
  */
 int xferbuf_deliver ( struct xfer_buffer *xferbuf, struct io_buffer *iobuf,
 		      struct xfer_metadata *meta ) {
-	size_t len;
-	size_t max;
+	size_t len = iob_len ( iobuf );
+	size_t pos;
 	int rc;
 
+	/* Start profiling */
+	profile_start ( &xferbuf_deliver_profiler );
+
 	/* Calculate new buffer position */
+	pos = xferbuf->pos;
 	if ( meta->flags & XFER_FL_ABS_OFFSET )
-		xferbuf->pos = 0;
-	xferbuf->pos += meta->offset;
+		pos = 0;
+	pos += meta->offset;
 
-	/* Ensure that we have enough buffer space for this data */
-	len = iob_len ( iobuf );
-	max = ( xferbuf->pos + len );
-	if ( ( rc = xferbuf_ensure_size ( xferbuf, max ) ) != 0 )
+	/* Write data to buffer */
+	if ( ( rc = xferbuf_write ( xferbuf, pos, iobuf->data, len ) ) != 0 )
 		goto done;
 
-	/* Copy data to buffer */
-	memcpy ( ( xferbuf->data + xferbuf->pos ), iobuf->data, len );
-
 	/* Update current buffer position */
-	xferbuf->pos += len;
+	xferbuf->pos = ( pos + len );
 
  done:
 	free_iob ( iobuf );
+	profile_stop ( &xferbuf_deliver_profiler );
 	return rc;
 }
+
+/**
+ * Reallocate malloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v len		New length (or zero to free buffer)
+ * @ret rc		Return status code
+ */
+static int xferbuf_malloc_realloc ( struct xfer_buffer *xferbuf, size_t len ) {
+	void *new_data;
+
+	new_data = realloc ( xferbuf->data, len );
+	if ( ! new_data )
+		return -ENOSPC;
+	xferbuf->data = new_data;
+	return 0;
+}
+
+/**
+ * Write data to malloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to copy
+ * @v len		Length of data
+ */
+static void xferbuf_malloc_write ( struct xfer_buffer *xferbuf, size_t offset,
+				   const void *data, size_t len ) {
+
+	memcpy ( ( xferbuf->data + offset ), data, len );
+}
+
+/**
+ * Read data from malloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to read
+ * @v len		Length of data
+ */
+static void xferbuf_malloc_read ( struct xfer_buffer *xferbuf, size_t offset,
+				  void *data, size_t len ) {
+
+	memcpy ( data, ( xferbuf->data + offset ), len );
+}
+
+/** malloc()-based data buffer operations */
+struct xfer_buffer_operations xferbuf_malloc_operations = {
+	.realloc = xferbuf_malloc_realloc,
+	.write = xferbuf_malloc_write,
+	.read = xferbuf_malloc_read,
+};
+
+/**
+ * Reallocate umalloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v len		New length (or zero to free buffer)
+ * @ret rc		Return status code
+ */
+static int xferbuf_umalloc_realloc ( struct xfer_buffer *xferbuf, size_t len ) {
+	userptr_t *udata = xferbuf->data;
+	userptr_t new_udata;
+
+	new_udata = urealloc ( *udata, len );
+	if ( ! new_udata )
+		return -ENOSPC;
+	*udata = new_udata;
+	return 0;
+}
+
+/**
+ * Write data to umalloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to copy
+ * @v len		Length of data
+ */
+static void xferbuf_umalloc_write ( struct xfer_buffer *xferbuf, size_t offset,
+				    const void *data, size_t len ) {
+	userptr_t *udata = xferbuf->data;
+
+	copy_to_user ( *udata, offset, data, len );
+}
+
+/**
+ * Read data from umalloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to read
+ * @v len		Length of data
+ */
+static void xferbuf_umalloc_read ( struct xfer_buffer *xferbuf, size_t offset,
+				   void *data, size_t len ) {
+	userptr_t *udata = xferbuf->data;
+
+	copy_from_user ( data, *udata, offset, len );
+}
+
+/** umalloc()-based data buffer operations */
+struct xfer_buffer_operations xferbuf_umalloc_operations = {
+	.realloc = xferbuf_umalloc_realloc,
+	.write = xferbuf_umalloc_write,
+	.read = xferbuf_umalloc_read,
+};
+
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v interface		Data transfer interface
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ *
+ * This call will check that the xfer_buffer() handler belongs to the
+ * destination interface which also provides xfer_deliver() for this
+ * interface.
+ *
+ * This is done to prevent accidental accesses to a data transfer
+ * buffer which may be located behind a non-transparent datapath via a
+ * series of pass-through interfaces.
+ */
+struct xfer_buffer * xfer_buffer ( struct interface *intf ) {
+	struct interface *dest;
+	xfer_buffer_TYPE ( void * ) *op =
+		intf_get_dest_op ( intf, xfer_buffer, &dest );
+	void *object = intf_object ( dest );
+	struct interface *xfer_deliver_dest;
+	struct xfer_buffer *xferbuf;
+
+	/* Check that this operation is provided by the same interface
+	 * which handles xfer_deliver().
+	 */
+	( void ) intf_get_dest_op ( intf, xfer_deliver, &xfer_deliver_dest );
+
+	if ( op && ( dest == xfer_deliver_dest ) ) {
+		xferbuf = op ( object );
+	} else {
+		/* Default is to not have a data transfer buffer */
+		xferbuf = NULL;
+	}
+
+	intf_put ( xfer_deliver_dest );
+	intf_put ( dest );
+	return xferbuf;
+}
diff --git a/roms/ipxe/src/crypto/aes.c b/roms/ipxe/src/crypto/aes.c
new file mode 100644
index 000000000..b9e206bfb
--- /dev/null
+++ b/roms/ipxe/src/crypto/aes.c
@@ -0,0 +1,808 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * AES algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/rotate.h>
+#include <ipxe/crypto.h>
+#include <ipxe/ecb.h>
+#include <ipxe/cbc.h>
+#include <ipxe/aes.h>
+
+/** AES strides
+ *
+ * These are the strides (modulo 16) used to walk through the AES
+ * input state bytes in order of byte position after [Inv]ShiftRows.
+ */
+enum aes_stride {
+	/** Input stride for ShiftRows
+	 *
+	 *    0 4 8 c
+	 *     \ \ \
+	 *    1 5 9 d
+	 *     \ \ \
+	 *    2 6 a e
+	 *     \ \ \
+	 *    3 7 b f
+	 */
+	AES_STRIDE_SHIFTROWS = +5,
+	/** Input stride for InvShiftRows
+	 *
+	 *    0 4 8 c
+	 *     / / /
+	 *    1 5 9 d
+	 *     / / /
+	 *    2 6 a e
+	 *     / / /
+	 *    3 7 b f
+	 */
+	AES_STRIDE_INVSHIFTROWS = -3,
+};
+
+/** A single AES lookup table entry
+ *
+ * This represents the product (in the Galois field GF(2^8)) of an
+ * eight-byte vector multiplier with a single scalar multiplicand.
+ *
+ * The vector multipliers used for AES will be {1,1,1,3,2,1,1,3} for
+ * MixColumns and {1,9,13,11,14,9,13,11} for InvMixColumns.  This
+ * allows for the result of multiplying any single column of the
+ * [Inv]MixColumns matrix by a scalar value to be obtained simply by
+ * extracting the relevant four-byte subset from the lookup table
+ * entry.
+ *
+ * For example, to find the result of multiplying the second column of
+ * the MixColumns matrix by the scalar value 0x80:
+ *
+ * MixColumns column[0]: {			      2,    1,    1,    3 }
+ * MixColumns column[1]: {			3,    2,    1,    1	  }
+ * MixColumns column[2]: {		  1,    3,    2,    1		  }
+ * MixColumns column[3]: {	    1,    1,    3,    2			  }
+ * Vector multiplier:	 {    1,    1,    1,    3,    2,    1,    1,    3 }
+ * Scalar multiplicand:	   0x80
+ * Lookup table entry:	 { 0x80, 0x80, 0x80, 0x9b, 0x1b, 0x80, 0x80, 0x9b }
+ *
+ * The second column of the MixColumns matrix is {3,2,1,1}.  The
+ * product of this column with the scalar value 0x80 can be obtained
+ * by extracting the relevant four-byte subset of the lookup table
+ * entry:
+ *
+ * MixColumns column[1]: {			3,    2,    1,    1	  }
+ * Vector multiplier:	 {    1,    1,    1,    3,    2,    1,    1,    3 }
+ * Lookup table entry:	 { 0x80, 0x80, 0x80, 0x9b, 0x1b, 0x80, 0x80, 0x9b }
+ * Product:		 {		     0x9b, 0x1b, 0x80, 0x80	  }
+ *
+ * The column lookups require only seven bytes of the eight-byte
+ * entry: the remaining (first) byte is used to hold the scalar
+ * multiplicand itself (i.e. the first byte of the vector multiplier
+ * is always chosen to be 1).
+ */
+union aes_table_entry {
+	/** Viewed as an array of bytes */
+	uint8_t byte[8];
+} __attribute__ (( packed ));
+
+/** An AES lookup table
+ *
+ * This represents the products (in the Galois field GF(2^8)) of a
+ * constant eight-byte vector multiplier with all possible 256 scalar
+ * multiplicands.
+ *
+ * The entries are indexed by the AES [Inv]SubBytes S-box output
+ * values (denoted S(N)).  This allows for the result of multiplying
+ * any single column of the [Inv]MixColumns matrix by S(N) to be
+ * obtained simply by extracting the relevant four-byte subset from
+ * the Nth table entry.  For example:
+ *
+ * Input byte (N):	   0x3a
+ * SubBytes output S(N):   0x80
+ * MixColumns column[1]: {			3,    2,    1,    1	  }
+ * Vector multiplier:	 {    1,    1,    1,    3,    2,    1,    1,    3 }
+ * Table entry[0x3a]:	 { 0x80, 0x80, 0x80, 0x9b, 0x1b, 0x80, 0x80, 0x9b }
+ * Product:		 {		     0x9b, 0x1b, 0x80, 0x80	  }
+ *
+ * Since the first byte of the eight-byte vector multiplier is always
+ * chosen to be 1, the value of S(N) may be lookup up by extracting
+ * the first byte of the Nth table entry.
+ */
+struct aes_table {
+	/** Table entries, indexed by S(N) */
+	union aes_table_entry entry[256];
+} __attribute__ (( aligned ( 8 ) ));
+
+/** AES MixColumns lookup table */
+static struct aes_table aes_mixcolumns;
+
+/** AES InvMixColumns lookup table */
+static struct aes_table aes_invmixcolumns;
+
+/**
+ * Multiply [Inv]MixColumns matrix column by scalar multiplicand
+ *
+ * @v entry		AES lookup table entry for scalar multiplicand
+ * @v column		[Inv]MixColumns matrix column index
+ * @ret product		Product of matrix column with scalar multiplicand
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_entry_column ( const union aes_table_entry *entry, unsigned int column ) {
+	const union {
+		uint8_t byte;
+		uint32_t column;
+	} __attribute__ (( may_alias )) *product;
+
+	/* Locate relevant four-byte subset */
+	product = container_of ( &entry->byte[ 4 - column ],
+				 typeof ( *product ), byte );
+
+	/* Extract this four-byte subset */
+	return product->column;
+}
+
+/**
+ * Multiply [Inv]MixColumns matrix column by S-boxed input byte
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v offset		Output byte offset (after [Inv]ShiftRows)
+ * @ret product		Product of matrix column with S(input byte)
+ *
+ * Note that the specified offset is not the offset of the input byte;
+ * it is the offset of the output byte which corresponds to the input
+ * byte.  This output byte offset is used to calculate both the input
+ * byte offset and to select the appropriate matric column.
+ *
+ * With a compile-time constant offset, this function will optimise
+ * down to a single "movzbl" (to extract the input byte) and will
+ * generate a single x86 memory reference expression which can then be
+ * used directly within a single "xorl" instruction.
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_column ( const struct aes_table *table, size_t stride,
+	     const union aes_matrix *in, size_t offset ) {
+	const union aes_table_entry *entry;
+	unsigned int byte;
+
+	/* Extract input byte corresponding to this output byte offset
+	 * (i.e. perform [Inv]ShiftRows).
+	 */
+	byte = in->byte[ ( stride * offset ) & 0xf ];
+
+	/* Locate lookup table entry for this input byte (i.e. perform
+	 * [Inv]SubBytes).
+	 */
+	entry = &table->entry[byte];
+
+	/* Multiply appropriate matrix column by this input byte
+	 * (i.e. perform [Inv]MixColumns).
+	 */
+	return aes_entry_column ( entry, ( offset & 0x3 ) );
+}
+
+/**
+ * Calculate intermediate round output column
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v key		AES round key
+ * @v column		Column index
+ * @ret output		Output column value
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_output ( const struct aes_table *table, size_t stride,
+	     const union aes_matrix *in, const union aes_matrix *key,
+	     unsigned int column ) {
+	size_t offset = ( column * 4 );
+
+	/* Perform [Inv]ShiftRows, [Inv]SubBytes, [Inv]MixColumns, and
+	 * AddRoundKey for this column.  The loop is unrolled to allow
+	 * for the required compile-time constant optimisations.
+	 */
+	return ( aes_column ( table, stride, in, ( offset + 0 ) ) ^
+		 aes_column ( table, stride, in, ( offset + 1 ) ) ^
+		 aes_column ( table, stride, in, ( offset + 2 ) ) ^
+		 aes_column ( table, stride, in, ( offset + 3 ) ) ^
+		 key->column[column] );
+}
+
+/**
+ * Perform a single intermediate round
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		AES round key
+ */
+static inline __attribute__ (( always_inline )) void
+aes_round ( const struct aes_table *table, size_t stride,
+	    const union aes_matrix *in, union aes_matrix *out,
+	    const union aes_matrix *key ) {
+
+	/* Perform [Inv]ShiftRows, [Inv]SubBytes, [Inv]MixColumns, and
+	 * AddRoundKey for all columns.  The loop is unrolled to allow
+	 * for the required compile-time constant optimisations.
+	 */
+	out->column[0] = aes_output ( table, stride, in, key, 0 );
+	out->column[1] = aes_output ( table, stride, in, key, 1 );
+	out->column[2] = aes_output ( table, stride, in, key, 2 );
+	out->column[3] = aes_output ( table, stride, in, key, 3 );
+}
+
+/**
+ * Perform encryption intermediate rounds
+ *
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		Round keys
+ * @v rounds		Number of rounds (must be odd)
+ *
+ * This function is deliberately marked as non-inlinable to ensure
+ * maximal availability of registers for GCC's register allocator,
+ * which has a tendency to otherwise spill performance-critical
+ * registers to the stack.
+ */
+static __attribute__ (( noinline )) void
+aes_encrypt_rounds ( union aes_matrix *in, union aes_matrix *out,
+		     const union aes_matrix *key, unsigned int rounds ) {
+	union aes_matrix *tmp;
+
+	/* Perform intermediate rounds */
+	do {
+		/* Perform one intermediate round */
+		aes_round ( &aes_mixcolumns, AES_STRIDE_SHIFTROWS,
+			    in, out, key++ );
+
+		/* Swap input and output states for next round */
+		tmp = in;
+		in = out;
+		out = tmp;
+
+	} while ( --rounds );
+}
+
+/**
+ * Perform decryption intermediate rounds
+ *
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		Round keys
+ * @v rounds		Number of rounds (must be odd)
+ *
+ * As with aes_encrypt_rounds(), this function is deliberately marked
+ * as non-inlinable.
+ *
+ * This function could potentially use the same binary code as is used
+ * for encryption.  To compensate for the difference between ShiftRows
+ * and InvShiftRows, half of the input byte offsets would have to be
+ * modifiable at runtime (half by an offset of +4/-4, half by an
+ * offset of -4/+4 for ShiftRows/InvShiftRows).  This can be
+ * accomplished in x86 assembly within the number of available
+ * registers, but GCC's register allocator struggles to do so,
+ * resulting in a significant performance decrease due to registers
+ * being spilled to the stack.  We therefore use two separate but very
+ * similar binary functions based on the same C source.
+ */
+static __attribute__ (( noinline )) void
+aes_decrypt_rounds ( union aes_matrix *in, union aes_matrix *out,
+		     const union aes_matrix *key, unsigned int rounds ) {
+	union aes_matrix *tmp;
+
+	/* Perform intermediate rounds */
+	do {
+		/* Perform one intermediate round */
+		aes_round ( &aes_invmixcolumns, AES_STRIDE_INVSHIFTROWS,
+			    in, out, key++ );
+
+		/* Swap input and output states for next round */
+		tmp = in;
+		in = out;
+		out = tmp;
+
+	} while ( --rounds );
+}
+
+/**
+ * Perform standalone AddRoundKey
+ *
+ * @v state		AES state
+ * @v key		AES round key
+ */
+static inline __attribute__ (( always_inline )) void
+aes_addroundkey ( union aes_matrix *state, const union aes_matrix *key ) {
+
+	state->column[0] ^= key->column[0];
+	state->column[1] ^= key->column[1];
+	state->column[2] ^= key->column[2];
+	state->column[3] ^= key->column[3];
+}
+
+/**
+ * Perform final round
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		AES round key
+ */
+static void aes_final ( const struct aes_table *table, size_t stride,
+			const union aes_matrix *in, union aes_matrix *out,
+			const union aes_matrix *key ) {
+	const union aes_table_entry *entry;
+	unsigned int byte;
+	size_t out_offset;
+	size_t in_offset;
+
+	/* Perform [Inv]ShiftRows and [Inv]SubBytes */
+	for ( out_offset = 0, in_offset = 0 ; out_offset < 16 ;
+	      out_offset++, in_offset = ( ( in_offset + stride ) & 0xf ) ) {
+
+		/* Extract input byte (i.e. perform [Inv]ShiftRows) */
+		byte = in->byte[in_offset];
+
+		/* Locate lookup table entry for this input byte
+		 * (i.e. perform [Inv]SubBytes).
+		 */
+		entry = &table->entry[byte];
+
+		/* Store output byte */
+		out->byte[out_offset] = entry->byte[0];
+	}
+
+	/* Perform AddRoundKey */
+	aes_addroundkey ( out, key );
+}
+
+/**
+ * Encrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to encrypt
+ * @v dst		Buffer for encrypted data
+ * @v len		Length of data
+ */
+static void aes_encrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct aes_context *aes = ctx;
+	union aes_matrix buffer[2];
+	union aes_matrix *in = &buffer[0];
+	union aes_matrix *out = &buffer[1];
+	unsigned int rounds = aes->rounds;
+
+	/* Sanity check */
+	assert ( len == sizeof ( *in ) );
+
+	/* Initialise input state */
+	memcpy ( in, src, sizeof ( *in ) );
+
+	/* Perform initial round (AddRoundKey) */
+	aes_addroundkey ( in, &aes->encrypt.key[0] );
+
+	/* Perform intermediate rounds (ShiftRows, SubBytes,
+	 * MixColumns, AddRoundKey).
+	 */
+	aes_encrypt_rounds ( in, out, &aes->encrypt.key[1], ( rounds - 2 ) );
+	in = out;
+
+	/* Perform final round (ShiftRows, SubBytes, AddRoundKey) */
+	out = dst;
+	aes_final ( &aes_mixcolumns, AES_STRIDE_SHIFTROWS, in, out,
+		    &aes->encrypt.key[ rounds - 1 ] );
+}
+
+/**
+ * Decrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to decrypt
+ * @v dst		Buffer for decrypted data
+ * @v len		Length of data
+ */
+static void aes_decrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct aes_context *aes = ctx;
+	union aes_matrix buffer[2];
+	union aes_matrix *in = &buffer[0];
+	union aes_matrix *out = &buffer[1];
+	unsigned int rounds = aes->rounds;
+
+	/* Sanity check */
+	assert ( len == sizeof ( *in ) );
+
+	/* Initialise input state */
+	memcpy ( in, src, sizeof ( *in ) );
+
+	/* Perform initial round (AddRoundKey) */
+	aes_addroundkey ( in, &aes->decrypt.key[0] );
+
+	/* Perform intermediate rounds (InvShiftRows, InvSubBytes,
+	 * InvMixColumns, AddRoundKey).
+	 */
+	aes_decrypt_rounds ( in, out, &aes->decrypt.key[1], ( rounds - 2 ) );
+	in = out;
+
+	/* Perform final round (InvShiftRows, InvSubBytes, AddRoundKey) */
+	out = dst;
+	aes_final ( &aes_invmixcolumns, AES_STRIDE_INVSHIFTROWS, in, out,
+		    &aes->decrypt.key[ rounds - 1 ] );
+}
+
+/**
+ * Multiply a polynomial by (x) modulo (x^8 + x^4 + x^3 + x^2 + 1) in GF(2^8)
+ *
+ * @v poly		Polynomial to be multiplied
+ * @ret result		Result
+ */
+static __attribute__ (( const )) unsigned int aes_double ( unsigned int poly ) {
+
+	/* Multiply polynomial by (x), placing the resulting x^8
+	 * coefficient in the LSB (i.e. rotate byte left by one).
+	 */
+	poly = rol8 ( poly, 1 );
+
+	/* If coefficient of x^8 (in LSB) is non-zero, then reduce by
+	 * subtracting (x^8 + x^4 + x^3 + x^2 + 1) in GF(2^8).
+	 */
+	if ( poly & 0x01 ) {
+		poly ^= 0x01; /* Subtract x^8 (currently in LSB) */
+		poly ^= 0x1b; /* Subtract (x^4 + x^3 + x^2 + 1) */
+	}
+
+	return poly;
+}
+
+/**
+ * Fill in MixColumns lookup table entry
+ *
+ * @v entry		AES lookup table entry for scalar multiplicand
+ *
+ * The MixColumns lookup table vector multiplier is {1,1,1,3,2,1,1,3}.
+ */
+static void aes_mixcolumns_entry ( union aes_table_entry *entry ) {
+	unsigned int scalar_x_1;
+	unsigned int scalar_x;
+	unsigned int scalar;
+
+	/* Retrieve scalar multiplicand */
+	scalar = entry->byte[0];
+	entry->byte[1] = scalar;
+	entry->byte[2] = scalar;
+	entry->byte[5] = scalar;
+	entry->byte[6] = scalar;
+
+	/* Calculate scalar multiplied by (x) */
+	scalar_x = aes_double ( scalar );
+	entry->byte[4] = scalar_x;
+
+	/* Calculate scalar multiplied by (x + 1) */
+	scalar_x_1 = ( scalar_x ^ scalar );
+	entry->byte[3] = scalar_x_1;
+	entry->byte[7] = scalar_x_1;
+}
+
+/**
+ * Fill in InvMixColumns lookup table entry
+ *
+ * @v entry		AES lookup table entry for scalar multiplicand
+ *
+ * The InvMixColumns lookup table vector multiplier is {1,9,13,11,14,9,13,11}.
+ */
+static void aes_invmixcolumns_entry ( union aes_table_entry *entry ) {
+	unsigned int scalar_x3_x2_x;
+	unsigned int scalar_x3_x2_1;
+	unsigned int scalar_x3_x2;
+	unsigned int scalar_x3_x_1;
+	unsigned int scalar_x3_1;
+	unsigned int scalar_x3;
+	unsigned int scalar_x2;
+	unsigned int scalar_x;
+	unsigned int scalar;
+
+	/* Retrieve scalar multiplicand */
+	scalar = entry->byte[0];
+
+	/* Calculate scalar multiplied by (x) */
+	scalar_x = aes_double ( scalar );
+
+	/* Calculate scalar multiplied by (x^2) */
+	scalar_x2 = aes_double ( scalar_x );
+
+	/* Calculate scalar multiplied by (x^3) */
+	scalar_x3 = aes_double ( scalar_x2 );
+
+	/* Calculate scalar multiplied by (x^3 + 1) */
+	scalar_x3_1 = ( scalar_x3 ^ scalar );
+	entry->byte[1] = scalar_x3_1;
+	entry->byte[5] = scalar_x3_1;
+
+	/* Calculate scalar multiplied by (x^3 + x + 1) */
+	scalar_x3_x_1 = ( scalar_x3_1 ^ scalar_x );
+	entry->byte[3] = scalar_x3_x_1;
+	entry->byte[7] = scalar_x3_x_1;
+
+	/* Calculate scalar multiplied by (x^3 + x^2) */
+	scalar_x3_x2 = ( scalar_x3 ^ scalar_x2 );
+
+	/* Calculate scalar multiplied by (x^3 + x^2 + 1) */
+	scalar_x3_x2_1 = ( scalar_x3_x2 ^ scalar );
+	entry->byte[2] = scalar_x3_x2_1;
+	entry->byte[6] = scalar_x3_x2_1;
+
+	/* Calculate scalar multiplied by (x^3 + x^2 + x) */
+	scalar_x3_x2_x = ( scalar_x3_x2 ^ scalar_x );
+	entry->byte[4] = scalar_x3_x2_x;
+}
+
+/**
+ * Generate AES lookup tables
+ *
+ */
+static void aes_generate ( void ) {
+	union aes_table_entry *entry;
+	union aes_table_entry *inventry;
+	unsigned int poly = 0x01;
+	unsigned int invpoly = 0x01;
+	unsigned int transformed;
+	unsigned int i;
+
+	/* Iterate over non-zero values of GF(2^8) using generator (x + 1) */
+	do {
+
+		/* Multiply polynomial by (x + 1) */
+		poly ^= aes_double ( poly );
+
+		/* Divide inverse polynomial by (x + 1).  This code
+		 * fragment is taken directly from the Wikipedia page
+		 * on the Rijndael S-box.  An explanation of why it
+		 * works would be greatly appreciated.
+		 */
+		invpoly ^= ( invpoly << 1 );
+		invpoly ^= ( invpoly << 2 );
+		invpoly ^= ( invpoly << 4 );
+		if ( invpoly & 0x80 )
+			invpoly ^= 0x09;
+		invpoly &= 0xff;
+
+		/* Apply affine transformation */
+		transformed = ( 0x63 ^ invpoly ^ rol8 ( invpoly, 1 ) ^
+				rol8 ( invpoly, 2 ) ^ rol8 ( invpoly, 3 ) ^
+				rol8 ( invpoly, 4 ) );
+
+		/* Populate S-box (within MixColumns lookup table) */
+		aes_mixcolumns.entry[poly].byte[0] = transformed;
+
+	} while ( poly != 0x01 );
+
+	/* Populate zeroth S-box entry (which has no inverse) */
+	aes_mixcolumns.entry[0].byte[0] = 0x63;
+
+	/* Fill in MixColumns and InvMixColumns lookup tables */
+	for ( i = 0 ; i < 256 ; i++ ) {
+
+		/* Fill in MixColumns lookup table entry */
+		entry = &aes_mixcolumns.entry[i];
+		aes_mixcolumns_entry ( entry );
+
+		/* Populate inverse S-box (within InvMixColumns lookup table) */
+		inventry = &aes_invmixcolumns.entry[ entry->byte[0] ];
+		inventry->byte[0] = i;
+
+		/* Fill in InvMixColumns lookup table entry */
+		aes_invmixcolumns_entry ( inventry );
+	}
+}
+
+/**
+ * Rotate key column
+ *
+ * @v column		Key column
+ * @ret column		Updated key column
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_key_rotate ( uint32_t column ) {
+
+	return ( ( __BYTE_ORDER == __LITTLE_ENDIAN ) ?
+		 ror32 ( column, 8 ) : rol32 ( column, 8 ) );
+}
+
+/**
+ * Apply S-box to key column
+ *
+ * @v column		Key column
+ * @ret column		Updated key column
+ */
+static uint32_t aes_key_sbox ( uint32_t column ) {
+	unsigned int i;
+	uint8_t byte;
+
+	for ( i = 0 ; i < 4 ; i++ ) {
+		byte = ( column & 0xff );
+		byte = aes_mixcolumns.entry[byte].byte[0];
+		column = ( ( column & ~0xff ) | byte );
+		column = rol32 ( column, 8 );
+	}
+	return column;
+}
+
+/**
+ * Apply schedule round constant to key column
+ *
+ * @v column		Key column
+ * @v rcon		Round constant
+ * @ret column		Updated key column
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_key_rcon ( uint32_t column, unsigned int rcon ) {
+
+	return ( ( __BYTE_ORDER == __LITTLE_ENDIAN ) ?
+		 ( column ^ rcon ) : ( column ^ ( rcon << 24 ) ) );
+}
+
+/**
+ * Set key
+ *
+ * @v ctx		Context
+ * @v key		Key
+ * @v keylen		Key length
+ * @ret rc		Return status code
+ */
+static int aes_setkey ( void *ctx, const void *key, size_t keylen ) {
+	struct aes_context *aes = ctx;
+	union aes_matrix *enc;
+	union aes_matrix *dec;
+	union aes_matrix temp;
+	union aes_matrix zero;
+	unsigned int rcon = 0x01;
+	unsigned int rounds;
+	size_t offset = 0;
+	uint32_t *prev;
+	uint32_t *next;
+	uint32_t *end;
+	uint32_t tmp;
+
+	/* Generate lookup tables, if not already done */
+	if ( ! aes_mixcolumns.entry[0].byte[0] )
+		aes_generate();
+
+	/* Validate key length and calculate number of intermediate rounds */
+	switch ( keylen ) {
+	case ( 128 / 8 ) :
+		rounds = 11;
+		break;
+	case ( 192 / 8 ) :
+		rounds = 13;
+		break;
+	case ( 256 / 8 ) :
+		rounds = 15;
+		break;
+	default:
+		DBGC ( aes, "AES %p unsupported key length (%zd bits)\n",
+		       aes, ( keylen * 8 ) );
+		return -EINVAL;
+	}
+	aes->rounds = rounds;
+	enc = aes->encrypt.key;
+	end = enc[rounds].column;
+
+	/* Copy raw key */
+	memcpy ( enc, key, keylen );
+	prev = enc->column;
+	next = ( ( ( void * ) prev ) + keylen );
+	tmp = next[-1];
+
+	/* Construct expanded key */
+	while ( next < end ) {
+
+		/* If this is the first column of an expanded key
+		 * block, or the middle column of an AES-256 key
+		 * block, then apply the S-box.
+		 */
+		if ( ( offset == 0 ) || ( ( offset | keylen ) == 48 ) )
+			tmp = aes_key_sbox ( tmp );
+
+		/* If this is the first column of an expanded key
+		 * block then rotate and apply the round constant.
+		 */
+		if ( offset == 0 ) {
+			tmp = aes_key_rotate ( tmp );
+			tmp = aes_key_rcon ( tmp, rcon );
+			rcon = aes_double ( rcon );
+		}
+
+		/* XOR with previous key column */
+		tmp ^= *prev;
+
+		/* Store column */
+		*next = tmp;
+
+		/* Move to next column */
+		offset += sizeof ( *next );
+		if ( offset == keylen )
+			offset = 0;
+		next++;
+		prev++;
+	}
+	DBGC2 ( aes, "AES %p expanded %zd-bit key:\n", aes, ( keylen * 8 ) );
+	DBGC2_HDA ( aes, 0, &aes->encrypt, ( rounds * sizeof ( *enc ) ) );
+
+	/* Convert to decryption key */
+	memset ( &zero, 0, sizeof ( zero ) );
+	dec = &aes->decrypt.key[ rounds - 1 ];
+	memcpy ( dec--, enc++, sizeof ( *dec ) );
+	while ( dec > aes->decrypt.key ) {
+		/* Perform InvMixColumns (by reusing the encryption
+		 * final-round code to perform ShiftRows+SubBytes and
+		 * reusing the decryption intermediate-round code to
+		 * perform InvShiftRows+InvSubBytes+InvMixColumns, all
+		 * with a zero encryption key).
+		 */
+		aes_final ( &aes_mixcolumns, AES_STRIDE_SHIFTROWS,
+			    enc++, &temp, &zero );
+		aes_decrypt_rounds ( &temp, dec--, &zero, 1 );
+	}
+	memcpy ( dec--, enc++, sizeof ( *dec ) );
+	DBGC2 ( aes, "AES %p inverted %zd-bit key:\n", aes, ( keylen * 8 ) );
+	DBGC2_HDA ( aes, 0, &aes->decrypt, ( rounds * sizeof ( *dec ) ) );
+
+	return 0;
+}
+
+/**
+ * Set initialisation vector
+ *
+ * @v ctx		Context
+ * @v iv		Initialisation vector
+ */
+static void aes_setiv ( void *ctx __unused, const void *iv __unused ) {
+	/* Nothing to do */
+}
+
+/** Basic AES algorithm */
+struct cipher_algorithm aes_algorithm = {
+	.name = "aes",
+	.ctxsize = sizeof ( struct aes_context ),
+	.blocksize = AES_BLOCKSIZE,
+	.setkey = aes_setkey,
+	.setiv = aes_setiv,
+	.encrypt = aes_encrypt,
+	.decrypt = aes_decrypt,
+};
+
+/* AES in Electronic Codebook mode */
+ECB_CIPHER ( aes_ecb, aes_ecb_algorithm,
+	     aes_algorithm, struct aes_context, AES_BLOCKSIZE );
+
+/* AES in Cipher Block Chaining mode */
+CBC_CIPHER ( aes_cbc, aes_cbc_algorithm,
+	     aes_algorithm, struct aes_context, AES_BLOCKSIZE );
diff --git a/roms/ipxe/src/crypto/asn1.c b/roms/ipxe/src/crypto/asn1.c
index 6d880704f..aca12bf30 100644
--- a/roms/ipxe/src/crypto/asn1.c
+++ b/roms/ipxe/src/crypto/asn1.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/roms/ipxe/src/crypto/axtls/aes.c b/roms/ipxe/src/crypto/axtls/aes.c
deleted file mode 100644
index bd99a7097..000000000
--- a/roms/ipxe/src/crypto/axtls/aes.c
+++ /dev/null
@@ -1,457 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/**
- * AES implementation - this is a small code version. There are much faster
- * versions around but they are much larger in size (i.e. they use large 
- * submix tables).
- */
-
-#include <string.h>
-#include "os_port.h"
-#include "crypto.h"
-
-/* all commented out in skeleton mode */
-#ifndef CONFIG_SSL_SKELETON_MODE
-
-#define rot1(x) (((x) << 24) | ((x) >> 8))
-#define rot2(x) (((x) << 16) | ((x) >> 16))
-#define rot3(x) (((x) <<  8) | ((x) >> 24))
-
-/* 
- * This cute trick does 4 'mul by two' at once.  Stolen from
- * Dr B. R. Gladman <brg@gladman.uk.net> but I'm sure the u-(u>>7) is
- * a standard graphics trick
- * The key to this is that we need to xor with 0x1b if the top bit is set.
- * a 1xxx xxxx   0xxx 0xxx First we mask the 7bit,
- * b 1000 0000   0000 0000 then we shift right by 7 putting the 7bit in 0bit,
- * c 0000 0001   0000 0000 we then subtract (c) from (b)
- * d 0111 1111   0000 0000 and now we and with our mask
- * e 0001 1011   0000 0000
- */
-#define mt  0x80808080
-#define ml  0x7f7f7f7f
-#define mh  0xfefefefe
-#define mm  0x1b1b1b1b
-#define mul2(x,t)	((t)=((x)&mt), \
-			((((x)+(x))&mh)^(((t)-((t)>>7))&mm)))
-
-#define inv_mix_col(x,f2,f4,f8,f9) (\
-			(f2)=mul2(x,f2), \
-			(f4)=mul2(f2,f4), \
-			(f8)=mul2(f4,f8), \
-			(f9)=(x)^(f8), \
-			(f8)=((f2)^(f4)^(f8)), \
-			(f2)^=(f9), \
-			(f4)^=(f9), \
-			(f8)^=rot3(f2), \
-			(f8)^=rot2(f4), \
-			(f8)^rot1(f9))
-
-/*
- * AES S-box
- */
-static const uint8_t aes_sbox[256] =
-{
-	0x63,0x7C,0x77,0x7B,0xF2,0x6B,0x6F,0xC5,
-	0x30,0x01,0x67,0x2B,0xFE,0xD7,0xAB,0x76,
-	0xCA,0x82,0xC9,0x7D,0xFA,0x59,0x47,0xF0,
-	0xAD,0xD4,0xA2,0xAF,0x9C,0xA4,0x72,0xC0,
-	0xB7,0xFD,0x93,0x26,0x36,0x3F,0xF7,0xCC,
-	0x34,0xA5,0xE5,0xF1,0x71,0xD8,0x31,0x15,
-	0x04,0xC7,0x23,0xC3,0x18,0x96,0x05,0x9A,
-	0x07,0x12,0x80,0xE2,0xEB,0x27,0xB2,0x75,
-	0x09,0x83,0x2C,0x1A,0x1B,0x6E,0x5A,0xA0,
-	0x52,0x3B,0xD6,0xB3,0x29,0xE3,0x2F,0x84,
-	0x53,0xD1,0x00,0xED,0x20,0xFC,0xB1,0x5B,
-	0x6A,0xCB,0xBE,0x39,0x4A,0x4C,0x58,0xCF,
-	0xD0,0xEF,0xAA,0xFB,0x43,0x4D,0x33,0x85,
-	0x45,0xF9,0x02,0x7F,0x50,0x3C,0x9F,0xA8,
-	0x51,0xA3,0x40,0x8F,0x92,0x9D,0x38,0xF5,
-	0xBC,0xB6,0xDA,0x21,0x10,0xFF,0xF3,0xD2,
-	0xCD,0x0C,0x13,0xEC,0x5F,0x97,0x44,0x17,
-	0xC4,0xA7,0x7E,0x3D,0x64,0x5D,0x19,0x73,
-	0x60,0x81,0x4F,0xDC,0x22,0x2A,0x90,0x88,
-	0x46,0xEE,0xB8,0x14,0xDE,0x5E,0x0B,0xDB,
-	0xE0,0x32,0x3A,0x0A,0x49,0x06,0x24,0x5C,
-	0xC2,0xD3,0xAC,0x62,0x91,0x95,0xE4,0x79,
-	0xE7,0xC8,0x37,0x6D,0x8D,0xD5,0x4E,0xA9,
-	0x6C,0x56,0xF4,0xEA,0x65,0x7A,0xAE,0x08,
-	0xBA,0x78,0x25,0x2E,0x1C,0xA6,0xB4,0xC6,
-	0xE8,0xDD,0x74,0x1F,0x4B,0xBD,0x8B,0x8A,
-	0x70,0x3E,0xB5,0x66,0x48,0x03,0xF6,0x0E,
-	0x61,0x35,0x57,0xB9,0x86,0xC1,0x1D,0x9E,
-	0xE1,0xF8,0x98,0x11,0x69,0xD9,0x8E,0x94,
-	0x9B,0x1E,0x87,0xE9,0xCE,0x55,0x28,0xDF,
-	0x8C,0xA1,0x89,0x0D,0xBF,0xE6,0x42,0x68,
-	0x41,0x99,0x2D,0x0F,0xB0,0x54,0xBB,0x16,
-};
-
-/*
- * AES is-box
- */
-static const uint8_t aes_isbox[256] = 
-{
-    0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38,
-    0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb,
-    0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87,
-    0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb,
-    0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d,
-    0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e,
-    0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2,
-    0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25,
-    0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16,
-    0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92,
-    0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda,
-    0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84,
-    0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a,
-    0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06,
-    0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02,
-    0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b,
-    0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea,
-    0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73,
-    0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85,
-    0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e,
-    0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89,
-    0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b,
-    0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20,
-    0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4,
-    0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31,
-    0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f,
-    0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d,
-    0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef,
-    0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0,
-    0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61,
-    0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26,
-    0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d
-};
-
-static const unsigned char Rcon[30]=
-{
-	0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80,
-	0x1b,0x36,0x6c,0xd8,0xab,0x4d,0x9a,0x2f,
-	0x5e,0xbc,0x63,0xc6,0x97,0x35,0x6a,0xd4,
-	0xb3,0x7d,0xfa,0xef,0xc5,0x91,
-};
-
-/* ----- static functions ----- */
-static void AES_encrypt(const AES_CTX *ctx, uint32_t *data);
-static void AES_decrypt(const AES_CTX *ctx, uint32_t *data);
-
-/* Perform doubling in Galois Field GF(2^8) using the irreducible polynomial
-   x^8+x^4+x^3+x+1 */
-static unsigned char AES_xtime(uint32_t x)
-{
-	return (x&0x80) ? (x<<1)^0x1b : x<<1;
-}
-
-/**
- * Set up AES with the key/iv and cipher size.
- */
-void AES_set_key(AES_CTX *ctx, const uint8_t *key, 
-        const uint8_t *iv, AES_MODE mode)
-{
-    int i, ii;
-    uint32_t *W, tmp, tmp2;
-    const unsigned char *ip;
-    int words;
-
-    switch (mode)
-    {
-        case AES_MODE_128:
-            i = 10;
-            words = 4;
-            break;
-
-        case AES_MODE_256:
-            i = 14;
-            words = 8;
-            break;
-
-        default:        /* fail silently */
-            return;
-    }
-
-    ctx->rounds = i;
-    ctx->key_size = words;
-    W = ctx->ks;
-    for (i = 0; i < words; i+=2)
-    {
-        W[i+0]=	((uint32_t)key[ 0]<<24)|
-            ((uint32_t)key[ 1]<<16)|
-            ((uint32_t)key[ 2]<< 8)|
-            ((uint32_t)key[ 3]    );
-        W[i+1]=	((uint32_t)key[ 4]<<24)|
-            ((uint32_t)key[ 5]<<16)|
-            ((uint32_t)key[ 6]<< 8)|
-            ((uint32_t)key[ 7]    );
-        key += 8;
-    }
-
-    ip = Rcon;
-    ii = 4 * (ctx->rounds+1);
-    for (i = words; i<ii; i++)
-    {
-        tmp = W[i-1];
-
-        if ((i % words) == 0)
-        {
-            tmp2 =(uint32_t)aes_sbox[(tmp    )&0xff]<< 8;
-            tmp2|=(uint32_t)aes_sbox[(tmp>> 8)&0xff]<<16;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>16)&0xff]<<24;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>24)     ];
-            tmp=tmp2^(((unsigned int)*ip)<<24);
-            ip++;
-        }
-
-        if ((words == 8) && ((i % words) == 4))
-        {
-            tmp2 =(uint32_t)aes_sbox[(tmp    )&0xff]    ;
-            tmp2|=(uint32_t)aes_sbox[(tmp>> 8)&0xff]<< 8;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>16)&0xff]<<16;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>24)     ]<<24;
-            tmp=tmp2;
-        }
-
-        W[i]=W[i-words]^tmp;
-    }
-
-    /* copy the iv across */
-    memcpy(ctx->iv, iv, 16);
-}
-
-/**
- * Change a key for decryption.
- */
-void AES_convert_key(AES_CTX *ctx)
-{
-    int i;
-    uint32_t *k,w,t1,t2,t3,t4;
-
-    k = ctx->ks;
-    k += 4;
-
-    for (i= ctx->rounds*4; i > 4; i--)
-    {
-        w= *k;
-        w = inv_mix_col(w,t1,t2,t3,t4);
-        *k++ =w;
-    }
-}
-
-/**
- * Encrypt a byte sequence (with a block size 16) using the AES cipher.
- */
-void AES_cbc_encrypt(AES_CTX *ctx, const uint8_t *msg, uint8_t *out, int length)
-{
-    int i;
-    uint32_t tin[4], tout[4], iv[4];
-
-    memcpy(iv, ctx->iv, AES_IV_SIZE);
-    for (i = 0; i < 4; i++)
-        tout[i] = ntohl(iv[i]);
-
-    for (length -= AES_BLOCKSIZE; length >= 0; length -= AES_BLOCKSIZE)
-    {
-        uint32_t msg_32[4];
-        uint32_t out_32[4];
-        memcpy(msg_32, msg, AES_BLOCKSIZE);
-        msg += AES_BLOCKSIZE;
-
-        for (i = 0; i < 4; i++)
-            tin[i] = ntohl(msg_32[i])^tout[i];
-
-        AES_encrypt(ctx, tin);
-
-        for (i = 0; i < 4; i++)
-        {
-            tout[i] = tin[i];
-            out_32[i] = htonl(tout[i]);
-        }
-
-        memcpy(out, out_32, AES_BLOCKSIZE);
-        out += AES_BLOCKSIZE;
-    }
-
-    for (i = 0; i < 4; i++)
-        iv[i] = htonl(tout[i]);
-    memcpy(ctx->iv, iv, AES_IV_SIZE);
-}
-
-/**
- * Decrypt a byte sequence (with a block size 16) using the AES cipher.
- */
-void AES_cbc_decrypt(AES_CTX *ctx, const uint8_t *msg, uint8_t *out, int length)
-{
-    int i;
-    uint32_t tin[4], xor[4], tout[4], data[4], iv[4];
-
-    memcpy(iv, ctx->iv, AES_IV_SIZE);
-    for (i = 0; i < 4; i++)
-        xor[i] = ntohl(iv[i]);
-
-    for (length -= 16; length >= 0; length -= 16)
-    {
-        uint32_t msg_32[4];
-        uint32_t out_32[4];
-        memcpy(msg_32, msg, AES_BLOCKSIZE);
-        msg += AES_BLOCKSIZE;
-
-        for (i = 0; i < 4; i++)
-        {
-            tin[i] = ntohl(msg_32[i]);
-            data[i] = tin[i];
-        }
-
-        AES_decrypt(ctx, data);
-
-        for (i = 0; i < 4; i++)
-        {
-            tout[i] = data[i]^xor[i];
-            xor[i] = tin[i];
-            out_32[i] = htonl(tout[i]);
-        }
-
-        memcpy(out, out_32, AES_BLOCKSIZE);
-        out += AES_BLOCKSIZE;
-    }
-
-    for (i = 0; i < 4; i++)
-        iv[i] = htonl(xor[i]);
-    memcpy(ctx->iv, iv, AES_IV_SIZE);
-}
-
-/**
- * Encrypt a single block (16 bytes) of data
- */
-static void AES_encrypt(const AES_CTX *ctx, uint32_t *data)
-{
-    /* To make this code smaller, generate the sbox entries on the fly.
-     * This will have a really heavy effect upon performance.
-     */
-    uint32_t tmp[4];
-    uint32_t tmp1, old_a0, a0, a1, a2, a3, row;
-    int curr_rnd;
-    int rounds = ctx->rounds; 
-    const uint32_t *k = ctx->ks;
-
-    /* Pre-round key addition */
-    for (row = 0; row < 4; row++)
-        data[row] ^= *(k++);
-
-    /* Encrypt one block. */
-    for (curr_rnd = 0; curr_rnd < rounds; curr_rnd++)
-    {
-        /* Perform ByteSub and ShiftRow operations together */
-        for (row = 0; row < 4; row++)
-        {
-            a0 = (uint32_t)aes_sbox[(data[row%4]>>24)&0xFF];
-            a1 = (uint32_t)aes_sbox[(data[(row+1)%4]>>16)&0xFF];
-            a2 = (uint32_t)aes_sbox[(data[(row+2)%4]>>8)&0xFF]; 
-            a3 = (uint32_t)aes_sbox[(data[(row+3)%4])&0xFF];
-
-            /* Perform MixColumn iff not last round */
-            if (curr_rnd < (rounds - 1))
-            {
-                tmp1 = a0 ^ a1 ^ a2 ^ a3;
-                old_a0 = a0;
-                a0 ^= tmp1 ^ AES_xtime(a0 ^ a1);
-                a1 ^= tmp1 ^ AES_xtime(a1 ^ a2);
-                a2 ^= tmp1 ^ AES_xtime(a2 ^ a3);
-                a3 ^= tmp1 ^ AES_xtime(a3 ^ old_a0);
-            }
-
-            tmp[row] = ((a0 << 24) | (a1 << 16) | (a2 << 8) | a3);
-        }
-
-        /* KeyAddition - note that it is vital that this loop is separate from
-           the MixColumn operation, which must be atomic...*/ 
-        for (row = 0; row < 4; row++)
-            data[row] = tmp[row] ^ *(k++);
-    }
-}
-
-/**
- * Decrypt a single block (16 bytes) of data
- */
-static void AES_decrypt(const AES_CTX *ctx, uint32_t *data)
-{ 
-    uint32_t tmp[4];
-    uint32_t xt0,xt1,xt2,xt3,xt4,xt5,xt6;
-    uint32_t a0, a1, a2, a3, row;
-    int curr_rnd;
-    int rounds = ctx->rounds;
-    const uint32_t *k = ctx->ks + ((rounds+1)*4);
-
-    /* pre-round key addition */
-    for (row=4; row > 0;row--)
-        data[row-1] ^= *(--k);
-
-    /* Decrypt one block */
-    for (curr_rnd = 0; curr_rnd < rounds; curr_rnd++)
-    {
-        /* Perform ByteSub and ShiftRow operations together */
-        for (row = 4; row > 0; row--)
-        {
-            a0 = aes_isbox[(data[(row+3)%4]>>24)&0xFF];
-            a1 = aes_isbox[(data[(row+2)%4]>>16)&0xFF];
-            a2 = aes_isbox[(data[(row+1)%4]>>8)&0xFF];
-            a3 = aes_isbox[(data[row%4])&0xFF];
-
-            /* Perform MixColumn iff not last round */
-            if (curr_rnd<(rounds-1))
-            {
-                /* The MDS cofefficients (0x09, 0x0B, 0x0D, 0x0E)
-                   are quite large compared to encryption; this 
-                   operation slows decryption down noticeably. */
-                xt0 = AES_xtime(a0^a1);
-                xt1 = AES_xtime(a1^a2);
-                xt2 = AES_xtime(a2^a3);
-                xt3 = AES_xtime(a3^a0);
-                xt4 = AES_xtime(xt0^xt1);
-                xt5 = AES_xtime(xt1^xt2);
-                xt6 = AES_xtime(xt4^xt5);
-
-                xt0 ^= a1^a2^a3^xt4^xt6;
-                xt1 ^= a0^a2^a3^xt5^xt6;
-                xt2 ^= a0^a1^a3^xt4^xt6;
-                xt3 ^= a0^a1^a2^xt5^xt6;
-                tmp[row-1] = ((xt0<<24)|(xt1<<16)|(xt2<<8)|xt3);
-            }
-            else
-                tmp[row-1] = ((a0<<24)|(a1<<16)|(a2<<8)|a3);
-        }
-
-        for (row = 4; row > 0; row--)
-            data[row-1] = tmp[row-1] ^ *(--k);
-    }
-}
-
-#endif
diff --git a/roms/ipxe/src/crypto/axtls/bigint.h b/roms/ipxe/src/crypto/axtls/bigint.h
deleted file mode 100644
index 1f38c53d6..000000000
--- a/roms/ipxe/src/crypto/axtls/bigint.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef BIGINT_HEADER
-#define BIGINT_HEADER
-
-#include "crypto.h"
-
-BI_CTX *bi_initialize(void);
-void bi_terminate(BI_CTX *ctx);
-void bi_permanent(bigint *bi);
-void bi_depermanent(bigint *bi);
-void bi_clear_cache(BI_CTX *ctx);
-void bi_free(BI_CTX *ctx, bigint *bi);
-bigint *bi_copy(bigint *bi);
-bigint *bi_clone(BI_CTX *ctx, const bigint *bi);
-void bi_export(BI_CTX *ctx, bigint *bi, uint8_t *data, int size);
-bigint *bi_import(BI_CTX *ctx, const uint8_t *data, int len);
-bigint *int_to_bi(BI_CTX *ctx, comp i);
-
-/* the functions that actually do something interesting */
-bigint *bi_add(BI_CTX *ctx, bigint *bia, bigint *bib);
-bigint *bi_subtract(BI_CTX *ctx, bigint *bia, 
-        bigint *bib, int *is_negative);
-bigint *bi_divide(BI_CTX *ctx, bigint *bia, bigint *bim, int is_mod);
-bigint *bi_multiply(BI_CTX *ctx, bigint *bia, bigint *bib);
-bigint *bi_mod_power(BI_CTX *ctx, bigint *bi, bigint *biexp);
-bigint *bi_mod_power2(BI_CTX *ctx, bigint *bi, bigint *bim, bigint *biexp);
-int bi_compare(bigint *bia, bigint *bib);
-void bi_set_mod(BI_CTX *ctx, bigint *bim, int mod_offset);
-void bi_free_mod(BI_CTX *ctx, int mod_offset);
-
-#ifdef CONFIG_SSL_FULL_MODE
-void bi_print(const char *label, bigint *bi);
-bigint *bi_str_import(BI_CTX *ctx, const char *data);
-#endif
-
-/**
- * @def bi_mod
- * Find the residue of B. bi_set_mod() must be called before hand.
- */
-#define bi_mod(A, B)      bi_divide(A, B, ctx->bi_mod[ctx->mod_offset], 1)
-
-/**
- * bi_residue() is technically the same as bi_mod(), but it uses the
- * appropriate reduction technique (which is bi_mod() when doing classical
- * reduction).
- */
-#if defined(CONFIG_BIGINT_MONTGOMERY)
-#define bi_residue(A, B)         bi_mont(A, B)
-bigint *bi_mont(BI_CTX *ctx, bigint *bixy);
-#elif defined(CONFIG_BIGINT_BARRETT)
-#define bi_residue(A, B)         bi_barrett(A, B)
-bigint *bi_barrett(BI_CTX *ctx, bigint *bi);
-#else /* if defined(CONFIG_BIGINT_CLASSICAL) */
-#define bi_residue(A, B)         bi_mod(A, B)
-#endif
-
-#ifdef CONFIG_BIGINT_SQUARE
-bigint *bi_square(BI_CTX *ctx, bigint *bi);
-#else
-#define bi_square(A, B)     bi_multiply(A, bi_copy(B), B)
-#endif
-
-#ifdef CONFIG_BIGINT_CRT
-bigint *bi_crt(BI_CTX *ctx, bigint *bi,
-        bigint *dP, bigint *dQ,
-        bigint *p, bigint *q,
-        bigint *qInv);
-#endif
-
-#endif
diff --git a/roms/ipxe/src/crypto/axtls/bigint_impl.h b/roms/ipxe/src/crypto/axtls/bigint_impl.h
deleted file mode 100644
index 09d8550ea..000000000
--- a/roms/ipxe/src/crypto/axtls/bigint_impl.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef BIGINT_IMPL_HEADER
-#define BIGINT_IMPL_HEADER
-
-/* Maintain a number of precomputed variables when doing reduction */
-#define BIGINT_M_OFFSET     0    /**< Normal modulo offset. */
-#ifdef CONFIG_BIGINT_CRT
-#define BIGINT_P_OFFSET     1    /**< p modulo offset. */
-#define BIGINT_Q_OFFSET     2    /**< q module offset. */
-#define BIGINT_NUM_MODS     3    /**< The number of modulus constants used. */
-#else
-#define BIGINT_NUM_MODS     1    
-#endif
-
-/* Architecture specific functions for big ints */
-#if defined(CONFIG_INTEGER_8BIT)
-#define COMP_RADIX          256U       /**< Max component + 1 */
-#define COMP_MAX            0xFFFFU/**< (Max dbl comp -1) */
-#define COMP_BIT_SIZE       8   /**< Number of bits in a component. */
-#define COMP_BYTE_SIZE      1   /**< Number of bytes in a component. */
-#define COMP_NUM_NIBBLES    2   /**< Used For diagnostics only. */
-typedef uint8_t comp;	        /**< A single precision component. */
-typedef uint16_t long_comp;     /**< A double precision component. */
-typedef int16_t slong_comp;     /**< A signed double precision component. */
-#elif defined(CONFIG_INTEGER_16BIT)
-#define COMP_RADIX          65536U       /**< Max component + 1 */
-#define COMP_MAX            0xFFFFFFFFU/**< (Max dbl comp -1) */
-#define COMP_BIT_SIZE       16  /**< Number of bits in a component. */
-#define COMP_BYTE_SIZE      2   /**< Number of bytes in a component. */
-#define COMP_NUM_NIBBLES    4   /**< Used For diagnostics only. */
-typedef uint16_t comp;	        /**< A single precision component. */
-typedef uint32_t long_comp;     /**< A double precision component. */
-typedef int32_t slong_comp;     /**< A signed double precision component. */
-#else /* regular 32 bit */
-#ifdef WIN32
-#define COMP_RADIX          4294967296i64         
-#define COMP_MAX            0xFFFFFFFFFFFFFFFFui64
-#else
-#define COMP_RADIX          4294967296ULL         /**< Max component + 1 */
-#define COMP_MAX            0xFFFFFFFFFFFFFFFFULL/**< (Max dbl comp -1) */
-#endif
-#define COMP_BIT_SIZE       32  /**< Number of bits in a component. */
-#define COMP_BYTE_SIZE      4   /**< Number of bytes in a component. */
-#define COMP_NUM_NIBBLES    8   /**< Used For diagnostics only. */
-typedef uint32_t comp;	        /**< A single precision component. */
-typedef uint64_t long_comp;     /**< A double precision component. */
-typedef int64_t slong_comp;     /**< A signed double precision component. */
-#endif
-
-/**
- * @struct  _bigint
- * @brief A big integer basic object
- */
-struct _bigint
-{
-    struct _bigint* next;       /**< The next bigint in the cache. */
-    short size;                 /**< The number of components in this bigint. */
-    short max_comps;            /**< The heapsize allocated for this bigint */
-    int refs;                   /**< An internal reference count. */
-    comp* comps;                /**< A ptr to the actual component data */
-};
-
-typedef struct _bigint bigint;  /**< An alias for _bigint */
-
-/**
- * Maintains the state of the cache, and a number of variables used in 
- * reduction.
- */
-typedef struct /**< A big integer "session" context. */
-{
-    bigint *active_list;                    /**< Bigints currently used. */
-    bigint *free_list;                      /**< Bigints not used. */
-    bigint *bi_radix;                       /**< The radix used. */
-    bigint *bi_mod[BIGINT_NUM_MODS];        /**< modulus */
-
-#if defined(CONFIG_BIGINT_MONTGOMERY)
-    bigint *bi_RR_mod_m[BIGINT_NUM_MODS];   /**< R^2 mod m */
-    bigint *bi_R_mod_m[BIGINT_NUM_MODS];    /**< R mod m */
-    comp N0_dash[BIGINT_NUM_MODS];
-#elif defined(CONFIG_BIGINT_BARRETT)
-    bigint *bi_mu[BIGINT_NUM_MODS];         /**< Storage for mu */
-#endif
-    bigint *bi_normalised_mod[BIGINT_NUM_MODS]; /**< Normalised mod storage. */
-    bigint **g;                 /**< Used by sliding-window. */
-    int window;                 /**< The size of the sliding window */
-    int active_count;           /**< Number of active bigints. */
-    int free_count;             /**< Number of free bigints. */
-
-#ifdef CONFIG_BIGINT_MONTGOMERY
-    uint8_t use_classical;      /**< Use classical reduction. */
-#endif
-    uint8_t mod_offset;         /**< The mod offset we are using */
-} BI_CTX;
-
-#ifndef WIN32
-#define max(a,b) ((a)>(b)?(a):(b))  /**< Find the maximum of 2 numbers. */
-#define min(a,b) ((a)<(b)?(a):(b))  /**< Find the minimum of 2 numbers. */
-#endif
-
-#define PERMANENT           0x7FFF55AA  /**< A magic number for permanents. */
-
-#endif
diff --git a/roms/ipxe/src/crypto/axtls/config.h b/roms/ipxe/src/crypto/axtls/config.h
deleted file mode 100644
index 32fa3bf03..000000000
--- a/roms/ipxe/src/crypto/axtls/config.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef AXTLS_CONFIG_H
-#define AXTLS_CONFIG_H
-
-/**
- * @file config.h
- *
- * Trick the axtls code into building within our build environment.
- */
-
-#define CONFIG_SSL_ENABLE_CLIENT 1
-#define CONFIG_BIGINT_CLASSICAL 1
-
-#endif
diff --git a/roms/ipxe/src/crypto/axtls/crypto.h b/roms/ipxe/src/crypto/axtls/crypto.h
deleted file mode 100644
index 2c4cda4de..000000000
--- a/roms/ipxe/src/crypto/axtls/crypto.h
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/**
- * @file crypto.h
- */
-
-#ifndef HEADER_CRYPTO_H
-#define HEADER_CRYPTO_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "config.h"
-#include "bigint_impl.h"
-#include "bigint.h"
-
-#ifndef STDCALL
-#define STDCALL
-#endif
-#ifndef EXP_FUNC
-#define EXP_FUNC
-#endif
-
-
-/* enable features based on a 'super-set' capbaility. */
-#if defined(CONFIG_SSL_FULL_MODE)
-#define CONFIG_SSL_ENABLE_CLIENT
-#define CONFIG_SSL_CERT_VERIFICATION
-#elif defined(CONFIG_SSL_ENABLE_CLIENT)
-#define CONFIG_SSL_CERT_VERIFICATION
-#endif
-
-/**************************************************************************
- * AES declarations 
- **************************************************************************/
-
-#define AES_MAXROUNDS			14
-#define AES_BLOCKSIZE           16
-#define AES_IV_SIZE             16
-
-typedef struct aes_key_st 
-{
-    uint16_t rounds;
-    uint16_t key_size;
-    uint32_t ks[(AES_MAXROUNDS+1)*8];
-    uint8_t iv[AES_IV_SIZE];
-} AES_CTX;
-
-typedef enum
-{
-    AES_MODE_128,
-    AES_MODE_256
-} AES_MODE;
-
-void AES_set_key(AES_CTX *ctx, const uint8_t *key, 
-        const uint8_t *iv, AES_MODE mode);
-void AES_cbc_encrypt(AES_CTX *ctx, const uint8_t *msg, 
-        uint8_t *out, int length);
-void AES_cbc_decrypt(AES_CTX *ks, const uint8_t *in, uint8_t *out, int length);
-void AES_convert_key(AES_CTX *ctx);
-
-/**************************************************************************
- * RC4 declarations 
- **************************************************************************/
-
-typedef struct 
-{
-    uint8_t x, y, m[256];
-} RC4_CTX;
-
-void RC4_setup(RC4_CTX *s, const uint8_t *key, int length);
-void RC4_crypt(RC4_CTX *s, const uint8_t *msg, uint8_t *data, int length);
-
-/**************************************************************************
- * SHA1 declarations 
- **************************************************************************/
-
-#define SHA1_SIZE   20
-
-/*
- *  This structure will hold context information for the SHA-1
- *  hashing operation
- */
-typedef struct 
-{
-    uint32_t Intermediate_Hash[SHA1_SIZE/4]; /* Message Digest */
-    uint32_t Length_Low;            /* Message length in bits */
-    uint32_t Length_High;           /* Message length in bits */
-    uint16_t Message_Block_Index;   /* Index into message block array   */
-    uint8_t Message_Block[64];      /* 512-bit message blocks */
-} SHA1_CTX;
-
-void SHA1_Init(SHA1_CTX *);
-void SHA1_Update(SHA1_CTX *, const uint8_t * msg, int len);
-void SHA1_Final(uint8_t *digest, SHA1_CTX *);
-
-/**************************************************************************
- * MD2 declarations
- **************************************************************************/
-
-#define MD2_SIZE 16
-
-typedef struct
-{
-    unsigned char cksum[16];    /* checksum of the data block */
-    unsigned char state[48];    /* intermediate digest state */
-    unsigned char buffer[16];   /* data block being processed */
-    int left;                   /* amount of data in buffer */
-} MD2_CTX;
-
-EXP_FUNC void STDCALL MD2_Init(MD2_CTX *ctx);
-EXP_FUNC void STDCALL MD2_Update(MD2_CTX *ctx, const uint8_t *input, int ilen);
-EXP_FUNC void STDCALL MD2_Final(uint8_t *digest, MD2_CTX *ctx);
-
-/**************************************************************************
- * MD5 declarations
- **************************************************************************/
-
-#define MD5_SIZE    16
-
-typedef struct 
-{
-  uint32_t state[4];        /* state (ABCD) */
-  uint32_t count[2];        /* number of bits, modulo 2^64 (lsb first) */
-  uint8_t buffer[64];       /* input buffer */
-} MD5_CTX;
-
-EXP_FUNC void STDCALL MD5_Init(MD5_CTX *);
-EXP_FUNC void STDCALL MD5_Update(MD5_CTX *, const uint8_t *msg, int len);
-EXP_FUNC void STDCALL MD5_Final(uint8_t *digest, MD5_CTX *);
-
-/**************************************************************************
- * HMAC declarations 
- **************************************************************************/
-void hmac_md5(const uint8_t *msg, int length, const uint8_t *key, 
-        int key_len, uint8_t *digest);
-void hmac_sha1(const uint8_t *msg, int length, const uint8_t *key, 
-        int key_len, uint8_t *digest);
-
-/**************************************************************************
- * RSA declarations 
- **************************************************************************/
-
-typedef struct 
-{
-    bigint *m;              /* modulus */
-    bigint *e;              /* public exponent */
-    bigint *d;              /* private exponent */
-#ifdef CONFIG_BIGINT_CRT
-    bigint *p;              /* p as in m = pq */
-    bigint *q;              /* q as in m = pq */
-    bigint *dP;             /* d mod (p-1) */
-    bigint *dQ;             /* d mod (q-1) */
-    bigint *qInv;           /* q^-1 mod p */
-#endif
-    int num_octets;
-    BI_CTX *bi_ctx;
-} RSA_CTX;
-
-void RSA_priv_key_new(RSA_CTX **rsa_ctx, 
-        const uint8_t *modulus, int mod_len,
-        const uint8_t *pub_exp, int pub_len,
-        const uint8_t *priv_exp, int priv_len
-#ifdef CONFIG_BIGINT_CRT
-      , const uint8_t *p, int p_len,
-        const uint8_t *q, int q_len,
-        const uint8_t *dP, int dP_len,
-        const uint8_t *dQ, int dQ_len,
-        const uint8_t *qInv, int qInv_len
-#endif
-        );
-void RSA_pub_key_new(RSA_CTX **rsa_ctx, 
-        const uint8_t *modulus, int mod_len,
-        const uint8_t *pub_exp, int pub_len);
-void RSA_free(RSA_CTX *ctx);
-int RSA_decrypt(const RSA_CTX *ctx, const uint8_t *in_data, uint8_t *out_data,
-        int is_decryption);
-bigint *RSA_private(const RSA_CTX *c, bigint *bi_msg);
-#if defined(CONFIG_SSL_CERT_VERIFICATION) || defined(CONFIG_SSL_GENERATE_X509_CERT)
-bigint *RSA_sign_verify(BI_CTX *ctx, const uint8_t *sig, int sig_len,
-        bigint *modulus, bigint *pub_exp);
-bigint *RSA_public(const RSA_CTX * c, bigint *bi_msg);
-int RSA_encrypt(const RSA_CTX *ctx, const uint8_t *in_data, uint16_t in_len, 
-        uint8_t *out_data, int is_signing);
-void RSA_print(const RSA_CTX *ctx);
-#endif
-
-/**************************************************************************
- * RNG declarations
- **************************************************************************/
-EXP_FUNC void STDCALL RNG_initialize(const uint8_t *seed_buf, int size);
-EXP_FUNC void STDCALL RNG_terminate(void);
-EXP_FUNC void STDCALL get_random(int num_rand_bytes, uint8_t *rand_data);
-void get_random_NZ(int num_rand_bytes, uint8_t *rand_data);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif 
diff --git a/roms/ipxe/src/crypto/axtls/os_port.h b/roms/ipxe/src/crypto/axtls/os_port.h
deleted file mode 100644
index 76313e204..000000000
--- a/roms/ipxe/src/crypto/axtls/os_port.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#ifndef AXTLS_OS_PORT_H
-#define AXTLS_OS_PORT_H
-
-/**
- * @file os_port.h
- *
- * Trick the axtls code into building within our build environment.
- */
-
-#include <stdint.h>
-#include <byteswap.h>
-
-/** All imported axTLS files are licensed using the three-clause BSD licence */
-FILE_LICENCE ( BSD3 );
-
-/** We can't actually abort, since we are effectively a kernel... */
-#define abort() assert ( 0 )
-
-/** rsa.c uses alloca() */
-#define alloca( size ) __builtin_alloca ( size )
-
-#include <ipxe/random_nz.h>
-static inline void get_random_NZ ( int num_rand_bytes, uint8_t *rand_data ) {
-	/* AXTLS does not check for failures when generating random
-	 * data.  Rely on the fact that get_random_nz() does not
-	 * request prediction resistance (and so cannot introduce new
-	 * failures) and therefore any potential failure must already
-	 * have been encountered by e.g. tls_generate_random(), which
-	 * does check for failures.
-	 */
-	get_random_nz ( rand_data, num_rand_bytes );
-}
-
-/* Expose AES_encrypt() and AES_decrypt() in aes.o */
-#define aes 1
-#if OBJECT
-
-struct aes_key_st;
-
-static void AES_encrypt ( const struct aes_key_st *ctx, uint32_t *data );
-static void AES_decrypt ( const struct aes_key_st *ctx, uint32_t *data );
-
-void axtls_aes_encrypt ( void *ctx, uint32_t *data ) {
-	AES_encrypt ( ctx, data );
-}
-
-void axtls_aes_decrypt ( void *ctx, uint32_t *data ) {
-	AES_decrypt ( ctx, data );
-}
-
-#endif
-#undef aes
-
-#endif 
diff --git a/roms/ipxe/src/crypto/axtls_aes.c b/roms/ipxe/src/crypto/axtls_aes.c
deleted file mode 100644
index 7f93c0ed7..000000000
--- a/roms/ipxe/src/crypto/axtls_aes.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <string.h>
-#include <errno.h>
-#include <assert.h>
-#include <byteswap.h>
-#include <ipxe/crypto.h>
-#include <ipxe/cbc.h>
-#include <ipxe/aes.h>
-#include "crypto/axtls/crypto.h"
-
-/** @file
- *
- * AES algorithm
- *
- */
-
-/**
- * Set key
- *
- * @v ctx		Context
- * @v key		Key
- * @v keylen		Key length
- * @ret rc		Return status code
- */
-static int aes_setkey ( void *ctx, const void *key, size_t keylen ) {
-	struct aes_context *aes_ctx = ctx;
-	AES_MODE mode;
-	void *iv;
-
-	switch ( keylen ) {
-	case ( 128 / 8 ):
-		mode = AES_MODE_128;
-		break;
-	case ( 256 / 8 ):
-		mode = AES_MODE_256;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* IV is not a relevant concept at this stage; use a dummy
-	 * value that will have no side-effects.
-	 */
-	iv = &aes_ctx->axtls_ctx.iv;
-
-	AES_set_key ( &aes_ctx->axtls_ctx, key, iv, mode );
-
-	aes_ctx->decrypting = 0;
-
-	return 0;
-}
-
-/**
- * Set initialisation vector
- *
- * @v ctx		Context
- * @v iv		Initialisation vector
- */
-static void aes_setiv ( void *ctx __unused, const void *iv __unused ) {
-	/* Nothing to do */
-}
-
-/**
- * Call AXTLS' AES_encrypt() or AES_decrypt() functions
- *
- * @v axtls_ctx		AXTLS AES context
- * @v src		Data to process
- * @v dst		Buffer for output
- * @v func		AXTLS AES function to call
- */
-static void aes_call_axtls ( AES_CTX *axtls_ctx, const void *src, void *dst,
-			     void ( * func ) ( const AES_CTX *axtls_ctx,
-					       uint32_t *data ) ){
-	const uint32_t *srcl = src;
-	uint32_t *dstl = dst;
-	unsigned int i;
-
-	/* AXTLS' AES_encrypt() and AES_decrypt() functions both
-	 * expect to deal with an array of four dwords in host-endian
-	 * order.
-	 */
-	for ( i = 0 ; i < 4 ; i++ )
-		dstl[i] = ntohl ( srcl[i] );
-	func ( axtls_ctx, dstl );
-	for ( i = 0 ; i < 4 ; i++ )
-		dstl[i] = htonl ( dstl[i] );
-}
-
-/**
- * Encrypt data
- *
- * @v ctx		Context
- * @v src		Data to encrypt
- * @v dst		Buffer for encrypted data
- * @v len		Length of data
- */
-static void aes_encrypt ( void *ctx, const void *src, void *dst,
-			  size_t len ) {
-	struct aes_context *aes_ctx = ctx;
-
-	assert ( len == AES_BLOCKSIZE );
-	if ( aes_ctx->decrypting )
-		assert ( 0 );
-	aes_call_axtls ( &aes_ctx->axtls_ctx, src, dst, axtls_aes_encrypt );
-}
-
-/**
- * Decrypt data
- *
- * @v ctx		Context
- * @v src		Data to decrypt
- * @v dst		Buffer for decrypted data
- * @v len		Length of data
- */
-static void aes_decrypt ( void *ctx, const void *src, void *dst,
-			  size_t len ) {
-	struct aes_context *aes_ctx = ctx;
-
-	assert ( len == AES_BLOCKSIZE );
-	if ( ! aes_ctx->decrypting ) {
-		AES_convert_key ( &aes_ctx->axtls_ctx );
-		aes_ctx->decrypting = 1;
-	}
-	aes_call_axtls ( &aes_ctx->axtls_ctx, src, dst, axtls_aes_decrypt );
-}
-
-/** Basic AES algorithm */
-struct cipher_algorithm aes_algorithm = {
-	.name = "aes",
-	.ctxsize = sizeof ( struct aes_context ),
-	.blocksize = AES_BLOCKSIZE,
-	.setkey = aes_setkey,
-	.setiv = aes_setiv,
-	.encrypt = aes_encrypt,
-	.decrypt = aes_decrypt,
-};
-
-/* AES with cipher-block chaining */
-CBC_CIPHER ( aes_cbc, aes_cbc_algorithm,
-	     aes_algorithm, struct aes_context, AES_BLOCKSIZE );
diff --git a/roms/ipxe/src/crypto/bigint.c b/roms/ipxe/src/crypto/bigint.c
index 340128e2f..50f320302 100644
--- a/roms/ipxe/src/crypto/bigint.c
+++ b/roms/ipxe/src/crypto/bigint.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/crypto/cbc.c b/roms/ipxe/src/crypto/cbc.c
index 9bf0e8b49..0ba17ee48 100644
--- a/roms/ipxe/src/crypto/cbc.c
+++ b/roms/ipxe/src/crypto/cbc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <assert.h>
diff --git a/roms/ipxe/src/crypto/certstore.c b/roms/ipxe/src/crypto/certstore.c
index 77cf6ebb6..503ce499e 100644
--- a/roms/ipxe/src/crypto/certstore.c
+++ b/roms/ipxe/src/crypto/certstore.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/crypto/chap.c b/roms/ipxe/src/crypto/chap.c
index db64371c7..c90c16def 100644
--- a/roms/ipxe/src/crypto/chap.c
+++ b/roms/ipxe/src/crypto/chap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/crypto/cms.c b/roms/ipxe/src/crypto/cms.c
index b4a41de6c..bc2148e8a 100644
--- a/roms/ipxe/src/crypto/cms.c
+++ b/roms/ipxe/src/crypto/cms.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/crypto_null.c b/roms/ipxe/src/crypto/crypto_null.c
index ba05f7269..15a1c538b 100644
--- a/roms/ipxe/src/crypto/crypto_null.c
+++ b/roms/ipxe/src/crypto/crypto_null.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/crypto/deflate.c b/roms/ipxe/src/crypto/deflate.c
index 91a489961..e1c87d5fe 100644
--- a/roms/ipxe/src/crypto/deflate.c
+++ b/roms/ipxe/src/crypto/deflate.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <strings.h>
diff --git a/roms/ipxe/src/crypto/drbg.c b/roms/ipxe/src/crypto/drbg.c
index 9e0175d25..5c8b5e612 100644
--- a/roms/ipxe/src/crypto/drbg.c
+++ b/roms/ipxe/src/crypto/drbg.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/ecb.c b/roms/ipxe/src/crypto/ecb.c
new file mode 100644
index 000000000..3c9cf340c
--- /dev/null
+++ b/roms/ipxe/src/crypto/ecb.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/crypto.h>
+#include <ipxe/ecb.h>
+
+/** @file
+ *
+ * Electronic codebook (ECB)
+ *
+ */
+
+/**
+ * Encrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to encrypt
+ * @v dst		Buffer for encrypted data
+ * @v len		Length of data
+ * @v raw_cipher	Underlying cipher algorithm
+ */
+void ecb_encrypt ( void *ctx, const void *src, void *dst, size_t len,
+		   struct cipher_algorithm *raw_cipher ) {
+	size_t blocksize = raw_cipher->blocksize;
+
+	assert ( ( len % blocksize ) == 0 );
+
+	while ( len ) {
+		cipher_encrypt ( raw_cipher, ctx, src, dst, blocksize );
+		dst += blocksize;
+		src += blocksize;
+		len -= blocksize;
+	}
+}
+
+/**
+ * Decrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to decrypt
+ * @v dst		Buffer for decrypted data
+ * @v len		Length of data
+ * @v raw_cipher	Underlying cipher algorithm
+ */
+void ecb_decrypt ( void *ctx, const void *src, void *dst, size_t len,
+		   struct cipher_algorithm *raw_cipher ) {
+	size_t blocksize = raw_cipher->blocksize;
+
+	assert ( ( len % blocksize ) == 0 );
+
+	while ( len ) {
+		cipher_decrypt ( raw_cipher, ctx, src, dst, blocksize );
+		dst += blocksize;
+		src += blocksize;
+		len -= blocksize;
+	}
+}
diff --git a/roms/ipxe/src/crypto/entropy.c b/roms/ipxe/src/crypto/entropy.c
index c7045840e..5acbc0258 100644
--- a/roms/ipxe/src/crypto/entropy.c
+++ b/roms/ipxe/src/crypto/entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/hash_df.c b/roms/ipxe/src/crypto/hash_df.c
index adf1d87e4..c1417e683 100644
--- a/roms/ipxe/src/crypto/hash_df.c
+++ b/roms/ipxe/src/crypto/hash_df.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/hmac.c b/roms/ipxe/src/crypto/hmac.c
index e9459198c..95a46195c 100644
--- a/roms/ipxe/src/crypto/hmac.c
+++ b/roms/ipxe/src/crypto/hmac.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/crypto/hmac_drbg.c b/roms/ipxe/src/crypto/hmac_drbg.c
index 1e5f732e2..6c1d5deb2 100644
--- a/roms/ipxe/src/crypto/hmac_drbg.c
+++ b/roms/ipxe/src/crypto/hmac_drbg.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/md5.c b/roms/ipxe/src/crypto/md5.c
index 122c7d59e..f9738b0ac 100644
--- a/roms/ipxe/src/crypto/md5.c
+++ b/roms/ipxe/src/crypto/md5.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha1.c b/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha1.c
new file mode 100644
index 000000000..06722c0e1
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha1.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha1.h>
+#include <ipxe/tls.h>
+
+/** TLS_RSA_WITH_AES_128_CBC_SHA cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite (03) = {
+	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA ),
+	.key_len = ( 128 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+};
+
+/** TLS_RSA_WITH_AES_256_CBC_SHA cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_256_cbc_sha __tls_cipher_suite (04) = {
+	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA ),
+	.key_len = ( 256 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+};
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha256.c b/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha256.c
new file mode 100644
index 000000000..c609eacea
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha256.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite(01)={
+	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+};
+
+/** TLS_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite(02)={
+	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA256 ),
+	.key_len = ( 256 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+};
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_md5.c b/roms/ipxe/src/crypto/mishmash/rsa_md5.c
new file mode 100644
index 000000000..ac828ac11
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_md5.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/md5.h>
+#include <ipxe/asn1.h>
+
+/** "md5WithRSAEncryption" object identifier */
+static uint8_t oid_md5_with_rsa_encryption[] =
+	{ ASN1_OID_MD5WITHRSAENCRYPTION };
+
+/** "md5WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm md5_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "md5WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &md5_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_md5_with_rsa_encryption ),
+};
+
+/** MD5 digestInfo prefix */
+static const uint8_t rsa_md5_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( MD5_DIGEST_SIZE, ASN1_OID_MD5 ) };
+
+/** MD5 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_md5_prefix __rsa_digestinfo_prefix = {
+	.digest = &md5_algorithm,
+	.data = rsa_md5_prefix_data,
+	.len = sizeof ( rsa_md5_prefix_data ),
+};
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_sha1.c b/roms/ipxe/src/crypto/mishmash/rsa_sha1.c
new file mode 100644
index 000000000..39424bf2d
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_sha1.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha1.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha1WithRSAEncryption" object identifier */
+static uint8_t oid_sha1_with_rsa_encryption[] =
+	{ ASN1_OID_SHA1WITHRSAENCRYPTION };
+
+/** "sha1WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha1_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha1WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha1_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha1_with_rsa_encryption ),
+};
+
+/** SHA-1 digestInfo prefix */
+static const uint8_t rsa_sha1_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA1_DIGEST_SIZE, ASN1_OID_SHA1 ) };
+
+/** SHA-1 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha1_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha1_algorithm,
+	.data = rsa_sha1_prefix_data,
+	.len = sizeof ( rsa_sha1_prefix_data ),
+};
+
+/** RSA with SHA-1 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha1 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA1_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha1_algorithm,
+};
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_sha224.c b/roms/ipxe/src/crypto/mishmash/rsa_sha224.c
new file mode 100644
index 000000000..5e8755aab
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_sha224.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha256.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha224WithRSAEncryption" object identifier */
+static uint8_t oid_sha224_with_rsa_encryption[] =
+	{ ASN1_OID_SHA224WITHRSAENCRYPTION };
+
+/** "sha224WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha224_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha224WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha224_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha224_with_rsa_encryption ),
+};
+
+/** SHA-224 digestInfo prefix */
+static const uint8_t rsa_sha224_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA224_DIGEST_SIZE, ASN1_OID_SHA224 ) };
+
+/** SHA-224 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha224_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha224_algorithm,
+	.data = rsa_sha224_prefix_data,
+	.len = sizeof ( rsa_sha224_prefix_data ),
+};
+
+/** RSA with SHA-224 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha224 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA224_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha224_algorithm,
+};
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_sha256.c b/roms/ipxe/src/crypto/mishmash/rsa_sha256.c
new file mode 100644
index 000000000..b44af5f19
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_sha256.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha256.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha256WithRSAEncryption" object identifier */
+static uint8_t oid_sha256_with_rsa_encryption[] =
+	{ ASN1_OID_SHA256WITHRSAENCRYPTION };
+
+/** "sha256WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha256_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha256WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha256_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha256_with_rsa_encryption ),
+};
+
+/** SHA-256 digestInfo prefix */
+static const uint8_t rsa_sha256_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA256_DIGEST_SIZE, ASN1_OID_SHA256 ) };
+
+/** SHA-256 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha256_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha256_algorithm,
+	.data = rsa_sha256_prefix_data,
+	.len = sizeof ( rsa_sha256_prefix_data ),
+};
+
+/** RSA with SHA-256 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha256 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA256_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha256_algorithm,
+};
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_sha384.c b/roms/ipxe/src/crypto/mishmash/rsa_sha384.c
new file mode 100644
index 000000000..af22a2bf0
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_sha384.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha512.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha384WithRSAEncryption" object identifier */
+static uint8_t oid_sha384_with_rsa_encryption[] =
+	{ ASN1_OID_SHA384WITHRSAENCRYPTION };
+
+/** "sha384WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha384_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha384WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha384_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha384_with_rsa_encryption ),
+};
+
+/** SHA-384 digestInfo prefix */
+static const uint8_t rsa_sha384_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA384_DIGEST_SIZE, ASN1_OID_SHA384 ) };
+
+/** SHA-384 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha384_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha384_algorithm,
+	.data = rsa_sha384_prefix_data,
+	.len = sizeof ( rsa_sha384_prefix_data ),
+};
+
+/** RSA with SHA-384 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha384 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA384_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha384_algorithm,
+};
diff --git a/roms/ipxe/src/crypto/mishmash/rsa_sha512.c b/roms/ipxe/src/crypto/mishmash/rsa_sha512.c
new file mode 100644
index 000000000..29ee15493
--- /dev/null
+++ b/roms/ipxe/src/crypto/mishmash/rsa_sha512.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha512.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha512WithRSAEncryption" object identifier */
+static uint8_t oid_sha512_with_rsa_encryption[] =
+	{ ASN1_OID_SHA512WITHRSAENCRYPTION };
+
+/** "sha512WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha512_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha512WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha512_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512_with_rsa_encryption ),
+};
+
+/** SHA-512 digestInfo prefix */
+static const uint8_t rsa_sha512_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA512_DIGEST_SIZE, ASN1_OID_SHA512 ) };
+
+/** SHA-512 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha512_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha512_algorithm,
+	.data = rsa_sha512_prefix_data,
+	.len = sizeof ( rsa_sha512_prefix_data ),
+};
+
+/** RSA with SHA-512 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha512 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA512_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha512_algorithm,
+};
diff --git a/roms/ipxe/src/crypto/null_entropy.c b/roms/ipxe/src/crypto/null_entropy.c
index c56d5e76f..d1e1a6f73 100644
--- a/roms/ipxe/src/crypto/null_entropy.c
+++ b/roms/ipxe/src/crypto/null_entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/ocsp.c b/roms/ipxe/src/crypto/ocsp.c
index 66e47c57e..5df55bc96 100644
--- a/roms/ipxe/src/crypto/ocsp.c
+++ b/roms/ipxe/src/crypto/ocsp.c
@@ -233,7 +233,7 @@ static int ocsp_uri_string ( struct ocsp_check *ocsp ) {
 		goto err_path_base64;
 	}
 	base64_encode ( ocsp->request.builder.data, ocsp->request.builder.len,
-			path_base64_string );
+			path_base64_string, path_len );
 
 	/* URI-encode the Base64-encoded request */
 	memset ( &path_uri, 0, sizeof ( path_uri ) );
diff --git a/roms/ipxe/src/crypto/privkey.c b/roms/ipxe/src/crypto/privkey.c
index e010649c0..a6043bd1e 100644
--- a/roms/ipxe/src/crypto/privkey.c
+++ b/roms/ipxe/src/crypto/privkey.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/crypto/random_nz.c b/roms/ipxe/src/crypto/random_nz.c
index f1d2e187d..5fe576e05 100644
--- a/roms/ipxe/src/crypto/random_nz.c
+++ b/roms/ipxe/src/crypto/random_nz.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/rbg.c b/roms/ipxe/src/crypto/rbg.c
index e2d06978c..943b288c3 100644
--- a/roms/ipxe/src/crypto/rbg.c
+++ b/roms/ipxe/src/crypto/rbg.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/rootcert.c b/roms/ipxe/src/crypto/rootcert.c
index ae28905ac..00ea1647e 100644
--- a/roms/ipxe/src/crypto/rootcert.c
+++ b/roms/ipxe/src/crypto/rootcert.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <ipxe/crypto.h>
diff --git a/roms/ipxe/src/crypto/rsa.c b/roms/ipxe/src/crypto/rsa.c
index 0ab7b2ad3..36109280d 100644
--- a/roms/ipxe/src/crypto/rsa.c
+++ b/roms/ipxe/src/crypto/rsa.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -28,9 +32,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/crypto.h>
 #include <ipxe/bigint.h>
 #include <ipxe/random_nz.h>
-#include <ipxe/md5.h>
-#include <ipxe/sha1.h>
-#include <ipxe/sha256.h>
 #include <ipxe/rsa.h>
 
 /** @file
@@ -49,18 +50,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /** "rsaEncryption" object identifier */
 static uint8_t oid_rsa_encryption[] = { ASN1_OID_RSAENCRYPTION };
 
-/** "md5WithRSAEncryption" object identifier */
-static uint8_t oid_md5_with_rsa_encryption[] =
-	{ ASN1_OID_MD5WITHRSAENCRYPTION };
-
-/** "sha1WithRSAEncryption" object identifier */
-static uint8_t oid_sha1_with_rsa_encryption[] =
-	{ ASN1_OID_SHA1WITHRSAENCRYPTION };
-
-/** "sha256WithRSAEncryption" object identifier */
-static uint8_t oid_sha256_with_rsa_encryption[] =
-	{ ASN1_OID_SHA256WITHRSAENCRYPTION };
-
 /** "rsaEncryption" OID-identified algorithm */
 struct asn1_algorithm rsa_encryption_algorithm __asn1_algorithm = {
 	.name = "rsaEncryption",
@@ -69,63 +58,6 @@ struct asn1_algorithm rsa_encryption_algorithm __asn1_algorithm = {
 	.oid = ASN1_OID_CURSOR ( oid_rsa_encryption ),
 };
 
-/** "md5WithRSAEncryption" OID-identified algorithm */
-struct asn1_algorithm md5_with_rsa_encryption_algorithm __asn1_algorithm = {
-	.name = "md5WithRSAEncryption",
-	.pubkey = &rsa_algorithm,
-	.digest = &md5_algorithm,
-	.oid = ASN1_OID_CURSOR ( oid_md5_with_rsa_encryption ),
-};
-
-/** "sha1WithRSAEncryption" OID-identified algorithm */
-struct asn1_algorithm sha1_with_rsa_encryption_algorithm __asn1_algorithm = {
-	.name = "sha1WithRSAEncryption",
-	.pubkey = &rsa_algorithm,
-	.digest = &sha1_algorithm,
-	.oid = ASN1_OID_CURSOR ( oid_sha1_with_rsa_encryption ),
-};
-
-/** "sha256WithRSAEncryption" OID-identified algorithm */
-struct asn1_algorithm sha256_with_rsa_encryption_algorithm __asn1_algorithm = {
-	.name = "sha256WithRSAEncryption",
-	.pubkey = &rsa_algorithm,
-	.digest = &sha256_algorithm,
-	.oid = ASN1_OID_CURSOR ( oid_sha256_with_rsa_encryption ),
-};
-
-/** MD5 digestInfo prefix */
-static const uint8_t rsa_md5_prefix_data[] =
-	{ RSA_DIGESTINFO_PREFIX ( MD5_DIGEST_SIZE, ASN1_OID_MD5 ) };
-
-/** SHA-1 digestInfo prefix */
-static const uint8_t rsa_sha1_prefix_data[] =
-	{ RSA_DIGESTINFO_PREFIX ( SHA1_DIGEST_SIZE, ASN1_OID_SHA1 ) };
-
-/** SHA-256 digestInfo prefix */
-static const uint8_t rsa_sha256_prefix_data[] =
-	{ RSA_DIGESTINFO_PREFIX ( SHA256_DIGEST_SIZE, ASN1_OID_SHA256 ) };
-
-/** MD5 digestInfo prefix */
-struct rsa_digestinfo_prefix rsa_md5_prefix __rsa_digestinfo_prefix = {
-	.digest = &md5_algorithm,
-	.data = rsa_md5_prefix_data,
-	.len = sizeof ( rsa_md5_prefix_data ),
-};
-
-/** SHA-1 digestInfo prefix */
-struct rsa_digestinfo_prefix rsa_sha1_prefix __rsa_digestinfo_prefix = {
-	.digest = &sha1_algorithm,
-	.data = rsa_sha1_prefix_data,
-	.len = sizeof ( rsa_sha1_prefix_data ),
-};
-
-/** SHA-256 digestInfo prefix */
-struct rsa_digestinfo_prefix rsa_sha256_prefix __rsa_digestinfo_prefix = {
-	.digest = &sha256_algorithm,
-	.data = rsa_sha256_prefix_data,
-	.len = sizeof ( rsa_sha256_prefix_data ),
-};
-
 /**
  * Identify RSA prefix
  *
diff --git a/roms/ipxe/src/crypto/sha1.c b/roms/ipxe/src/crypto/sha1.c
index e1bef669e..51866f4b7 100644
--- a/roms/ipxe/src/crypto/sha1.c
+++ b/roms/ipxe/src/crypto/sha1.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/crypto/sha224.c b/roms/ipxe/src/crypto/sha224.c
new file mode 100644
index 000000000..be25f24e9
--- /dev/null
+++ b/roms/ipxe/src/crypto/sha224.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-224 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha256.h>
+
+/** SHA-224 initial digest values */
+static const struct sha256_digest sha224_init_digest = {
+	.h = {
+		cpu_to_be32 ( 0xc1059ed8 ),
+		cpu_to_be32 ( 0x367cd507 ),
+		cpu_to_be32 ( 0x3070dd17 ),
+		cpu_to_be32 ( 0xf70e5939 ),
+		cpu_to_be32 ( 0xffc00b31 ),
+		cpu_to_be32 ( 0x68581511 ),
+		cpu_to_be32 ( 0x64f98fa7 ),
+		cpu_to_be32 ( 0xbefa4fa4 ),
+	},
+};
+
+/**
+ * Initialise SHA-224 algorithm
+ *
+ * @v ctx		SHA-224 context
+ */
+static void sha224_init ( void *ctx ) {
+	struct sha256_context *context = ctx;
+
+	sha256_family_init ( context, &sha224_init_digest, SHA224_DIGEST_SIZE );
+}
+
+/** SHA-224 algorithm */
+struct digest_algorithm sha224_algorithm = {
+	.name		= "sha224",
+	.ctxsize	= sizeof ( struct sha256_context ),
+	.blocksize	= sizeof ( union sha256_block ),
+	.digestsize	= SHA224_DIGEST_SIZE,
+	.init		= sha224_init,
+	.update		= sha256_update,
+	.final		= sha256_final,
+};
+
+/** "sha224" object identifier */
+static uint8_t oid_sha224[] = { ASN1_OID_SHA224 };
+
+/** "sha224" OID-identified algorithm */
+struct asn1_algorithm oid_sha224_algorithm __asn1_algorithm = {
+	.name = "sha224",
+	.digest = &sha224_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha224 ),
+};
diff --git a/roms/ipxe/src/crypto/sha256.c b/roms/ipxe/src/crypto/sha256.c
index 36e02b3c2..0360d8d16 100644
--- a/roms/ipxe/src/crypto/sha256.c
+++ b/roms/ipxe/src/crypto/sha256.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -47,11 +51,11 @@ struct sha256_variables {
 	uint32_t f;
 	uint32_t g;
 	uint32_t h;
-	uint32_t w[64];
+	uint32_t w[SHA256_ROUNDS];
 } __attribute__ (( packed ));
 
 /** SHA-256 constants */
-static const uint32_t k[64] = {
+static const uint32_t k[SHA256_ROUNDS] = {
 	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
 	0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
 	0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
@@ -65,6 +69,37 @@ static const uint32_t k[64] = {
 	0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
 };
 
+/** SHA-256 initial digest values */
+static const struct sha256_digest sha256_init_digest = {
+	.h = {
+		cpu_to_be32 ( 0x6a09e667 ),
+		cpu_to_be32 ( 0xbb67ae85 ),
+		cpu_to_be32 ( 0x3c6ef372 ),
+		cpu_to_be32 ( 0xa54ff53a ),
+		cpu_to_be32 ( 0x510e527f ),
+		cpu_to_be32 ( 0x9b05688c ),
+		cpu_to_be32 ( 0x1f83d9ab ),
+		cpu_to_be32 ( 0x5be0cd19 ),
+	},
+};
+
+/**
+ * Initialise SHA-256 family algorithm
+ *
+ * @v context		SHA-256 context
+ * @v init		Initial digest values
+ * @v digestsize	Digest size
+ */
+void sha256_family_init ( struct sha256_context *context,
+			  const struct sha256_digest *init,
+			  size_t digestsize ) {
+
+	context->len = 0;
+	context->digestsize = digestsize;
+	memcpy ( &context->ddd.dd.digest, init,
+		 sizeof ( context->ddd.dd.digest ) );
+}
+
 /**
  * Initialise SHA-256 algorithm
  *
@@ -73,15 +108,8 @@ static const uint32_t k[64] = {
 static void sha256_init ( void *ctx ) {
 	struct sha256_context *context = ctx;
 
-	context->ddd.dd.digest.h[0] = cpu_to_be32 ( 0x6a09e667 );
-	context->ddd.dd.digest.h[1] = cpu_to_be32 ( 0xbb67ae85 );
-	context->ddd.dd.digest.h[2] = cpu_to_be32 ( 0x3c6ef372 );
-	context->ddd.dd.digest.h[3] = cpu_to_be32 ( 0xa54ff53a );
-	context->ddd.dd.digest.h[4] = cpu_to_be32 ( 0x510e527f );
-	context->ddd.dd.digest.h[5] = cpu_to_be32 ( 0x9b05688c );
-	context->ddd.dd.digest.h[6] = cpu_to_be32 ( 0x1f83d9ab );
-	context->ddd.dd.digest.h[7] = cpu_to_be32 ( 0x5be0cd19 );
-	context->len = 0;
+	sha256_family_init ( context, &sha256_init_digest,
+			     sizeof ( struct sha256_digest ) );
 }
 
 /**
@@ -139,7 +167,7 @@ static void sha256_digest ( struct sha256_context *context ) {
 	}
 
 	/* Initialise w[16..63] */
-	for ( i = 16 ; i < 64 ; i++ ) {
+	for ( i = 16 ; i < SHA256_ROUNDS ; i++ ) {
 		s0 = ( ror32 ( w[i-15], 7 ) ^ ror32 ( w[i-15], 18 ) ^
 		       ( w[i-15] >> 3 ) );
 		s1 = ( ror32 ( w[i-2], 17 ) ^ ror32 ( w[i-2], 19 ) ^
@@ -148,7 +176,7 @@ static void sha256_digest ( struct sha256_context *context ) {
 	}
 
 	/* Main loop */
-	for ( i = 0 ; i < 64 ; i++ ) {
+	for ( i = 0 ; i < SHA256_ROUNDS ; i++ ) {
 		s0 = ( ror32 ( *a, 2 ) ^ ror32 ( *a, 13 ) ^ ror32 ( *a, 22 ) );
 		maj = ( ( *a & *b ) ^ ( *a & *c ) ^ ( *b & *c ) );
 		t2 = ( s0 + maj );
@@ -186,7 +214,7 @@ static void sha256_digest ( struct sha256_context *context ) {
  * @v data		Data
  * @v len		Length of data
  */
-static void sha256_update ( void *ctx, const void *data, size_t len ) {
+void sha256_update ( void *ctx, const void *data, size_t len ) {
 	struct sha256_context *context = ctx;
 	const uint8_t *byte = data;
 	size_t offset;
@@ -209,7 +237,7 @@ static void sha256_update ( void *ctx, const void *data, size_t len ) {
  * @v ctx		SHA-256 context
  * @v out		Output buffer
  */
-static void sha256_final ( void *ctx, void *out ) {
+void sha256_final ( void *ctx, void *out ) {
 	struct sha256_context *context = ctx;
 	uint64_t len_bits;
 	uint8_t pad;
@@ -230,8 +258,7 @@ static void sha256_final ( void *ctx, void *out ) {
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
 
 	/* Copy out final digest */
-	memcpy ( out, &context->ddd.dd.digest,
-		 sizeof ( context->ddd.dd.digest ) );
+	memcpy ( out, &context->ddd.dd.digest, context->digestsize );
 }
 
 /** SHA-256 algorithm */
diff --git a/roms/ipxe/src/crypto/sha384.c b/roms/ipxe/src/crypto/sha384.c
new file mode 100644
index 000000000..017751826
--- /dev/null
+++ b/roms/ipxe/src/crypto/sha384.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-384 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-384 initial digest values */
+static const struct sha512_digest sha384_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0xcbbb9d5dc1059ed8ULL ),
+		cpu_to_be64 ( 0x629a292a367cd507ULL ),
+		cpu_to_be64 ( 0x9159015a3070dd17ULL ),
+		cpu_to_be64 ( 0x152fecd8f70e5939ULL ),
+		cpu_to_be64 ( 0x67332667ffc00b31ULL ),
+		cpu_to_be64 ( 0x8eb44a8768581511ULL ),
+		cpu_to_be64 ( 0xdb0c2e0d64f98fa7ULL ),
+		cpu_to_be64 ( 0x47b5481dbefa4fa4ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-384 algorithm
+ *
+ * @v ctx		SHA-384 context
+ */
+static void sha384_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha384_init_digest, SHA384_DIGEST_SIZE );
+}
+
+/** SHA-384 algorithm */
+struct digest_algorithm sha384_algorithm = {
+	.name		= "sha384",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= SHA384_DIGEST_SIZE,
+	.init		= sha384_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha384" object identifier */
+static uint8_t oid_sha384[] = { ASN1_OID_SHA384 };
+
+/** "sha384" OID-identified algorithm */
+struct asn1_algorithm oid_sha384_algorithm __asn1_algorithm = {
+	.name = "sha384",
+	.digest = &sha384_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha384 ),
+};
diff --git a/roms/ipxe/src/crypto/sha512.c b/roms/ipxe/src/crypto/sha512.c
new file mode 100644
index 000000000..814f44563
--- /dev/null
+++ b/roms/ipxe/src/crypto/sha512.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <byteswap.h>
+#include <assert.h>
+#include <ipxe/rotate.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-512 variables */
+struct sha512_variables {
+	/* This layout matches that of struct sha512_digest_data,
+	 * allowing for efficient endianness-conversion,
+	 */
+	uint64_t a;
+	uint64_t b;
+	uint64_t c;
+	uint64_t d;
+	uint64_t e;
+	uint64_t f;
+	uint64_t g;
+	uint64_t h;
+	uint64_t w[SHA512_ROUNDS];
+} __attribute__ (( packed ));
+
+/** SHA-512 constants */
+static const uint64_t k[SHA512_ROUNDS] = {
+	0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL,
+	0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL,
+	0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL,
+	0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL,
+	0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL,
+	0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL,
+	0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL,
+	0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL,
+	0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL,
+	0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL,
+	0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL,
+	0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL,
+	0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL,
+	0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL,
+	0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL,
+	0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL,
+	0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL,
+	0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL,
+	0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL,
+	0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL,
+	0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL,
+	0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL,
+	0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL,
+	0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL,
+	0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL,
+	0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
+	0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL
+};
+
+/** SHA-512 initial digest values */
+static const struct sha512_digest sha512_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0x6a09e667f3bcc908ULL ),
+		cpu_to_be64 ( 0xbb67ae8584caa73bULL ),
+		cpu_to_be64 ( 0x3c6ef372fe94f82bULL ),
+		cpu_to_be64 ( 0xa54ff53a5f1d36f1ULL ),
+		cpu_to_be64 ( 0x510e527fade682d1ULL ),
+		cpu_to_be64 ( 0x9b05688c2b3e6c1fULL ),
+		cpu_to_be64 ( 0x1f83d9abfb41bd6bULL ),
+		cpu_to_be64 ( 0x5be0cd19137e2179ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-512 family algorithm
+ *
+ * @v context		SHA-512 context
+ * @v init		Initial digest values
+ * @v digestsize	Digest size
+ */
+void sha512_family_init ( struct sha512_context *context,
+			  const struct sha512_digest *init,
+			  size_t digestsize ) {
+
+	context->len = 0;
+	context->digestsize = digestsize;
+	memcpy ( &context->ddq.dd.digest, init,
+		 sizeof ( context->ddq.dd.digest ) );
+}
+
+/**
+ * Initialise SHA-512 algorithm
+ *
+ * @v ctx		SHA-512 context
+ */
+static void sha512_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha512_init_digest,
+			     sizeof ( struct sha512_digest ) );
+}
+
+/**
+ * Calculate SHA-512 digest of accumulated data
+ *
+ * @v context		SHA-512 context
+ */
+static void sha512_digest ( struct sha512_context *context ) {
+        union {
+		union sha512_digest_data_qwords ddq;
+		struct sha512_variables v;
+	} u;
+	uint64_t *a = &u.v.a;
+	uint64_t *b = &u.v.b;
+	uint64_t *c = &u.v.c;
+	uint64_t *d = &u.v.d;
+	uint64_t *e = &u.v.e;
+	uint64_t *f = &u.v.f;
+	uint64_t *g = &u.v.g;
+	uint64_t *h = &u.v.h;
+	uint64_t *w = u.v.w;
+	uint64_t s0;
+	uint64_t s1;
+	uint64_t maj;
+	uint64_t t1;
+	uint64_t t2;
+	uint64_t ch;
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 );
+	linker_assert ( &u.ddq.dd.digest.h[0] == a, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[1] == b, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[2] == c, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[3] == d, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[4] == e, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[5] == f, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[6] == g, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[7] == h, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.data.qword[0] == w, sha512_bad_layout );
+
+	DBGC ( context, "SHA512 digesting:\n" );
+	DBGC_HDA ( context, 0, &context->ddq.dd.digest,
+		   sizeof ( context->ddq.dd.digest ) );
+	DBGC_HDA ( context, context->len, &context->ddq.dd.data,
+		   sizeof ( context->ddq.dd.data ) );
+
+	/* Convert h[0..7] to host-endian, and initialise a, b, c, d,
+	 * e, f, g, h, and w[0..15]
+	 */
+	for ( i = 0 ; i < ( sizeof ( u.ddq.qword ) /
+			    sizeof ( u.ddq.qword[0] ) ) ; i++ ) {
+		be64_to_cpus ( &context->ddq.qword[i] );
+		u.ddq.qword[i] = context->ddq.qword[i];
+	}
+
+	/* Initialise w[16..79] */
+	for ( i = 16 ; i < SHA512_ROUNDS ; i++ ) {
+		s0 = ( ror64 ( w[i-15], 1 ) ^ ror64 ( w[i-15], 8 ) ^
+		       ( w[i-15] >> 7 ) );
+		s1 = ( ror64 ( w[i-2], 19 ) ^ ror64 ( w[i-2], 61 ) ^
+		       ( w[i-2] >> 6 ) );
+		w[i] = ( w[i-16] + s0 + w[i-7] + s1 );
+	}
+
+	/* Main loop */
+	for ( i = 0 ; i < SHA512_ROUNDS ; i++ ) {
+		s0 = ( ror64 ( *a, 28 ) ^ ror64 ( *a, 34 ) ^ ror64 ( *a, 39 ) );
+		maj = ( ( *a & *b ) ^ ( *a & *c ) ^ ( *b & *c ) );
+		t2 = ( s0 + maj );
+		s1 = ( ror64 ( *e, 14 ) ^ ror64 ( *e, 18 ) ^ ror64 ( *e, 41 ) );
+		ch = ( ( *e & *f ) ^ ( (~*e) & *g ) );
+		t1 = ( *h + s1 + ch + k[i] + w[i] );
+		*h = *g;
+		*g = *f;
+		*f = *e;
+		*e = ( *d + t1 );
+		*d = *c;
+		*c = *b;
+		*b = *a;
+		*a = ( t1 + t2 );
+		DBGC2 ( context, "%2d : %016llx %016llx %016llx %016llx "
+			"%016llx %016llx %016llx %016llx\n",
+			i, *a, *b, *c, *d, *e, *f, *g, *h );
+	}
+
+	/* Add chunk to hash and convert back to big-endian */
+	for ( i = 0 ; i < 8 ; i++ ) {
+		context->ddq.dd.digest.h[i] =
+			cpu_to_be64 ( context->ddq.dd.digest.h[i] +
+				      u.ddq.dd.digest.h[i] );
+	}
+
+	DBGC ( context, "SHA512 digested:\n" );
+	DBGC_HDA ( context, 0, &context->ddq.dd.digest,
+		   sizeof ( context->ddq.dd.digest ) );
+}
+
+/**
+ * Accumulate data with SHA-512 algorithm
+ *
+ * @v ctx		SHA-512 context
+ * @v data		Data
+ * @v len		Length of data
+ */
+void sha512_update ( void *ctx, const void *data, size_t len ) {
+	struct sha512_context *context = ctx;
+	const uint8_t *byte = data;
+	size_t offset;
+
+	/* Accumulate data a byte at a time, performing the digest
+	 * whenever we fill the data buffer
+	 */
+	while ( len-- ) {
+		offset = ( context->len % sizeof ( context->ddq.dd.data ) );
+		context->ddq.dd.data.byte[offset] = *(byte++);
+		context->len++;
+		if ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 )
+			sha512_digest ( context );
+	}
+}
+
+/**
+ * Generate SHA-512 digest
+ *
+ * @v ctx		SHA-512 context
+ * @v out		Output buffer
+ */
+void sha512_final ( void *ctx, void *out ) {
+	struct sha512_context *context = ctx;
+	uint64_t len_bits_hi;
+	uint64_t len_bits_lo;
+	uint8_t pad;
+
+	/* Record length before pre-processing */
+	len_bits_hi = 0;
+	len_bits_lo = cpu_to_be64 ( ( ( uint64_t ) context->len ) * 8 );
+
+	/* Pad with a single "1" bit followed by as many "0" bits as required */
+	pad = 0x80;
+	do {
+		sha512_update ( ctx, &pad, sizeof ( pad ) );
+		pad = 0x00;
+	} while ( ( context->len % sizeof ( context->ddq.dd.data ) ) !=
+		  offsetof ( typeof ( context->ddq.dd.data ), final.len_hi ) );
+
+	/* Append length (in bits) */
+	sha512_update ( ctx, &len_bits_hi, sizeof ( len_bits_hi ) );
+	sha512_update ( ctx, &len_bits_lo, sizeof ( len_bits_lo ) );
+	assert ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 );
+
+	/* Copy out final digest */
+	memcpy ( out, &context->ddq.dd.digest, context->digestsize );
+}
+
+/** SHA-512 algorithm */
+struct digest_algorithm sha512_algorithm = {
+	.name		= "sha512",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= sizeof ( struct sha512_digest ),
+	.init		= sha512_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha512" object identifier */
+static uint8_t oid_sha512[] = { ASN1_OID_SHA512 };
+
+/** "sha512" OID-identified algorithm */
+struct asn1_algorithm oid_sha512_algorithm __asn1_algorithm = {
+	.name = "sha512",
+	.digest = &sha512_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512 ),
+};
diff --git a/roms/ipxe/src/crypto/sha512_224.c b/roms/ipxe/src/crypto/sha512_224.c
new file mode 100644
index 000000000..8c37b566b
--- /dev/null
+++ b/roms/ipxe/src/crypto/sha512_224.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512/224 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-512/224 initial digest values */
+static const struct sha512_digest sha512_224_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0x8c3d37c819544da2ULL ),
+		cpu_to_be64 ( 0x73e1996689dcd4d6ULL ),
+		cpu_to_be64 ( 0x1dfab7ae32ff9c82ULL ),
+		cpu_to_be64 ( 0x679dd514582f9fcfULL ),
+		cpu_to_be64 ( 0x0f6d2b697bd44da8ULL ),
+		cpu_to_be64 ( 0x77e36f7304c48942ULL ),
+		cpu_to_be64 ( 0x3f9d85a86a1d36c8ULL ),
+		cpu_to_be64 ( 0x1112e6ad91d692a1ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-512/224 algorithm
+ *
+ * @v ctx		SHA-512/224 context
+ */
+static void sha512_224_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha512_224_init_digest,
+			     SHA512_224_DIGEST_SIZE );
+}
+
+/** SHA-512/224 algorithm */
+struct digest_algorithm sha512_224_algorithm = {
+	.name		= "sha512/224",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= SHA512_224_DIGEST_SIZE,
+	.init		= sha512_224_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha512_224" object identifier */
+static uint8_t oid_sha512_224[] = { ASN1_OID_SHA512_224 };
+
+/** "sha512_224" OID-identified algorithm */
+struct asn1_algorithm oid_sha512_224_algorithm __asn1_algorithm = {
+	.name = "sha512/224",
+	.digest = &sha512_224_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512_224 ),
+};
diff --git a/roms/ipxe/src/crypto/sha512_256.c b/roms/ipxe/src/crypto/sha512_256.c
new file mode 100644
index 000000000..f8afaf3e3
--- /dev/null
+++ b/roms/ipxe/src/crypto/sha512_256.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512/256 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-512/256 initial digest values */
+static const struct sha512_digest sha512_256_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0x22312194fc2bf72cULL ),
+		cpu_to_be64 ( 0x9f555fa3c84c64c2ULL ),
+		cpu_to_be64 ( 0x2393b86b6f53b151ULL ),
+		cpu_to_be64 ( 0x963877195940eabdULL ),
+		cpu_to_be64 ( 0x96283ee2a88effe3ULL ),
+		cpu_to_be64 ( 0xbe5e1e2553863992ULL ),
+		cpu_to_be64 ( 0x2b0199fc2c85b8aaULL ),
+		cpu_to_be64 ( 0x0eb72ddc81c52ca2ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-512/256 algorithm
+ *
+ * @v ctx		SHA-512/256 context
+ */
+static void sha512_256_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha512_256_init_digest,
+			     SHA512_256_DIGEST_SIZE );
+}
+
+/** SHA-512/256 algorithm */
+struct digest_algorithm sha512_256_algorithm = {
+	.name		= "sha512/256",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= SHA512_256_DIGEST_SIZE,
+	.init		= sha512_256_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha512_256" object identifier */
+static uint8_t oid_sha512_256[] = { ASN1_OID_SHA512_256 };
+
+/** "sha512_256" OID-identified algorithm */
+struct asn1_algorithm oid_sha512_256_algorithm __asn1_algorithm = {
+	.name = "sha512/256",
+	.digest = &sha512_256_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512_256 ),
+};
diff --git a/roms/ipxe/src/crypto/x509.c b/roms/ipxe/src/crypto/x509.c
index 4a02dad14..43a4ca17a 100644
--- a/roms/ipxe/src/crypto/x509.c
+++ b/roms/ipxe/src/crypto/x509.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
@@ -139,7 +143,8 @@ const char * x509_name ( struct x509_certificate *cert ) {
 	} else {
 		/* Certificate has no commonName: use SHA-1 fingerprint */
 		x509_fingerprint ( cert, digest, fingerprint );
-		base16_encode ( fingerprint, sizeof ( fingerprint ), buf );
+		base16_encode ( fingerprint, sizeof ( fingerprint ),
+				buf, sizeof ( buf ) );
 	}
 	return buf;
 }
@@ -1761,5 +1766,11 @@ int x509_validate_chain ( struct x509_chain *chain, time_t time,
 	return -EACCES_USELESS;
 }
 
+/* Drag in objects via x509_validate() */
+REQUIRING_SYMBOL ( x509_validate );
+
 /* Drag in certificate store */
 REQUIRE_OBJECT ( certstore );
+
+/* Drag in crypto configuration */
+REQUIRE_OBJECT ( config_crypto );
diff --git a/roms/ipxe/src/drivers/bitbash/bitbash.c b/roms/ipxe/src/drivers/bitbash/bitbash.c
index 23ca30356..9b24f716c 100644
--- a/roms/ipxe/src/drivers/bitbash/bitbash.c
+++ b/roms/ipxe/src/drivers/bitbash/bitbash.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bitbash.h>
 
diff --git a/roms/ipxe/src/drivers/bitbash/i2c_bit.c b/roms/ipxe/src/drivers/bitbash/i2c_bit.c
index decc8d80e..707d9447d 100644
--- a/roms/ipxe/src/drivers/bitbash/i2c_bit.c
+++ b/roms/ipxe/src/drivers/bitbash/i2c_bit.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/roms/ipxe/src/drivers/bitbash/spi_bit.c b/roms/ipxe/src/drivers/bitbash/spi_bit.c
index 1b39d72fa..04fddc20b 100644
--- a/roms/ipxe/src/drivers/bitbash/spi_bit.c
+++ b/roms/ipxe/src/drivers/bitbash/spi_bit.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/roms/ipxe/src/drivers/block/ata.c b/roms/ipxe/src/drivers/block/ata.c
index c9b87c20c..b1c6855a0 100644
--- a/roms/ipxe/src/drivers/block/ata.c
+++ b/roms/ipxe/src/drivers/block/ata.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/drivers/block/scsi.c b/roms/ipxe/src/drivers/block/scsi.c
index 64d692986..fd5f82b9f 100644
--- a/roms/ipxe/src/drivers/block/scsi.c
+++ b/roms/ipxe/src/drivers/block/scsi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/drivers/bus/cdc.c b/roms/ipxe/src/drivers/bus/cdc.c
new file mode 100644
index 000000000..373a03072
--- /dev/null
+++ b/roms/ipxe/src/drivers/bus/cdc.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <ipxe/usb.h>
+#include <ipxe/cdc.h>
+
+/** @file
+ *
+ * USB Communications Device Class (CDC)
+ *
+ */
+
+/**
+ * Locate CDC union functional descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @ret desc		Union functional descriptor, or NULL if not found
+ */
+struct cdc_union_descriptor *
+cdc_union_descriptor ( struct usb_configuration_descriptor *config,
+		       struct usb_interface_descriptor *interface ) {
+	struct cdc_union_descriptor *desc;
+
+	for_each_interface_descriptor ( desc, config, interface ) {
+		if ( ( desc->header.type == USB_CS_INTERFACE_DESCRIPTOR ) &&
+		     ( desc->subtype == CDC_SUBTYPE_UNION ) )
+			return desc;
+	}
+	return NULL;
+}
diff --git a/roms/ipxe/src/drivers/bus/pci.c b/roms/ipxe/src/drivers/bus/pci.c
index 4a8d00b54..6fbedd940 100644
--- a/roms/ipxe/src/drivers/bus/pci.c
+++ b/roms/ipxe/src/drivers/bus/pci.c
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -58,8 +62,8 @@ static unsigned long pci_bar ( struct pci_device *pci, unsigned int reg ) {
 	uint32_t high;
 
 	pci_read_config_dword ( pci, reg, &low );
-	if ( ( low & (PCI_BASE_ADDRESS_SPACE|PCI_BASE_ADDRESS_MEM_TYPE_MASK) )
-	     == (PCI_BASE_ADDRESS_SPACE_MEMORY|PCI_BASE_ADDRESS_MEM_TYPE_64) ){
+	if ( ( low & (PCI_BASE_ADDRESS_SPACE_IO|PCI_BASE_ADDRESS_MEM_TYPE_MASK))
+	     == PCI_BASE_ADDRESS_MEM_TYPE_64 ) {
 		pci_read_config_dword ( pci, reg + 4, &high );
 		if ( high ) {
 			if ( sizeof ( unsigned long ) > sizeof ( uint32_t ) ) {
@@ -93,10 +97,10 @@ unsigned long pci_bar_start ( struct pci_device *pci, unsigned int reg ) {
 	unsigned long bar;
 
 	bar = pci_bar ( pci, reg );
-	if ( (bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_MEMORY ){
-		return ( bar & PCI_BASE_ADDRESS_MEM_MASK );
+	if ( bar & PCI_BASE_ADDRESS_SPACE_IO ) {
+		return ( bar & ~PCI_BASE_ADDRESS_IO_MASK );
 	} else {
-		return ( bar & PCI_BASE_ADDRESS_IO_MASK );
+		return ( bar & ~PCI_BASE_ADDRESS_MEM_MASK );
 	}
 }
 
@@ -122,11 +126,11 @@ static void pci_read_bases ( struct pci_device *pci ) {
 		if ( bar & PCI_BASE_ADDRESS_SPACE_IO ) {
 			if ( ! pci->ioaddr )
 				pci->ioaddr = 
-					( bar & PCI_BASE_ADDRESS_IO_MASK );
+					( bar & ~PCI_BASE_ADDRESS_IO_MASK );
 		} else {
 			if ( ! pci->membase )
 				pci->membase =
-					( bar & PCI_BASE_ADDRESS_MEM_MASK );
+					( bar & ~PCI_BASE_ADDRESS_MEM_MASK );
 			/* Skip next BAR if 64-bit */
 			if ( bar & PCI_BASE_ADDRESS_MEM_TYPE_64 )
 				reg += 4;
@@ -181,7 +185,7 @@ int pci_read_config ( struct pci_device *pci ) {
 		pci->busdevfn = PCI_FIRST_FUNC ( pci->busdevfn );
 		pci_read_config_byte ( pci, PCI_HEADER_TYPE, &hdrtype );
 		pci->busdevfn = busdevfn;
-		if ( ! ( hdrtype & 0x80 ) )
+		if ( ! ( hdrtype & PCI_HEADER_TYPE_MULTI ) )
 			return -ENODEV;
 	}
 
@@ -253,6 +257,8 @@ int pci_find_driver ( struct pci_device *pci ) {
 	unsigned int i;
 
 	for_each_table_entry ( driver, PCI_DRIVERS ) {
+		if ( ( driver->class.class ^ pci->class ) & driver->class.mask )
+			continue;
 		for ( i = 0 ; i < driver->id_count ; i++ ) {
 			id = &driver->ids[i];
 			if ( ( id->vendor != PCI_ANY_ID ) &&
@@ -334,14 +340,15 @@ static int pcibus_probe ( struct root_device *rootdev ) {
 
 		/* Look for a driver */
 		if ( ( rc = pci_find_driver ( pci ) ) != 0 ) {
-			DBGC ( pci, PCI_FMT " (%04x:%04x) has no driver\n",
-			       PCI_ARGS ( pci ), pci->vendor, pci->device );
+			DBGC ( pci, PCI_FMT " (%04x:%04x class %06x) has no "
+			       "driver\n", PCI_ARGS ( pci ), pci->vendor,
+			       pci->device, pci->class );
 			continue;
 		}
 
 		/* Add to device hierarchy */
 		pci->dev.parent = &rootdev->dev;
-		list_add ( &pci->dev.siblings, &rootdev->dev.children);
+		list_add ( &pci->dev.siblings, &rootdev->dev.children );
 
 		/* Look for a driver */
 		if ( ( rc = pci_probe ( pci ) ) == 0 ) {
diff --git a/roms/ipxe/src/drivers/bus/pci_settings.c b/roms/ipxe/src/drivers/bus/pci_settings.c
index db20452e0..1cb9fa5a3 100644
--- a/roms/ipxe/src/drivers/bus/pci_settings.c
+++ b/roms/ipxe/src/drivers/bus/pci_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/drivers/bus/pcibackup.c b/roms/ipxe/src/drivers/bus/pcibackup.c
index 6b592e893..fecad8192 100644
--- a/roms/ipxe/src/drivers/bus/pcibackup.c
+++ b/roms/ipxe/src/drivers/bus/pcibackup.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/pci.h>
diff --git a/roms/ipxe/src/drivers/bus/pciextra.c b/roms/ipxe/src/drivers/bus/pciextra.c
index c4417e0cb..82287fb86 100644
--- a/roms/ipxe/src/drivers/bus/pciextra.c
+++ b/roms/ipxe/src/drivers/bus/pciextra.c
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/pci.h>
@@ -26,7 +26,7 @@ int pci_find_capability ( struct pci_device *pci, int cap ) {
 		return 0;
 
 	pci_read_config_byte ( pci, PCI_HEADER_TYPE, &hdr_type );
-	switch ( hdr_type & 0x7F ) {
+	switch ( hdr_type & PCI_HEADER_TYPE_MASK ) {
 	case PCI_HEADER_TYPE_NORMAL:
 	case PCI_HEADER_TYPE_BRIDGE:
 	default:
@@ -38,13 +38,13 @@ int pci_find_capability ( struct pci_device *pci, int cap ) {
 	}
 	while ( ttl-- && pos >= 0x40 ) {
 		pos &= ~3;
-		pci_read_config_byte ( pci, pos + PCI_CAP_LIST_ID, &id );
+		pci_read_config_byte ( pci, pos + PCI_CAP_ID, &id );
 		DBG ( "PCI Capability: %d\n", id );
 		if ( id == 0xff )
 			break;
 		if ( id == cap )
 			return pos;
-		pci_read_config_byte ( pci, pos + PCI_CAP_LIST_NEXT, &pos );
+		pci_read_config_byte ( pci, pos + PCI_CAP_NEXT, &pos );
 	}
 	return 0;
 }
@@ -76,9 +76,9 @@ unsigned long pci_bar_size ( struct pci_device *pci, unsigned int reg ) {
 	/* Restore the original command register. This reenables decoding. */
 	pci_write_config_word ( pci, PCI_COMMAND, cmd );
 	if ( start & PCI_BASE_ADDRESS_SPACE_IO ) {
-		size &= PCI_BASE_ADDRESS_IO_MASK;
+		size &= ~PCI_BASE_ADDRESS_IO_MASK;
 	} else {
-		size &= PCI_BASE_ADDRESS_MEM_MASK;
+		size &= ~PCI_BASE_ADDRESS_MEM_MASK;
 	}
 	/* Find the lowest bit set */
 	size = size & ~( size - 1 );
diff --git a/roms/ipxe/src/drivers/bus/pcivpd.c b/roms/ipxe/src/drivers/bus/pcivpd.c
index 0b7a879fe..243b1f779 100644
--- a/roms/ipxe/src/drivers/bus/pcivpd.c
+++ b/roms/ipxe/src/drivers/bus/pcivpd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/drivers/bus/usb.c b/roms/ipxe/src/drivers/bus/usb.c
new file mode 100644
index 000000000..2019e3341
--- /dev/null
+++ b/roms/ipxe/src/drivers/bus/usb.c
@@ -0,0 +1,2128 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/usb.h>
+#include <ipxe/cdc.h>
+
+/** @file
+ *
+ * Universal Serial Bus (USB)
+ *
+ */
+
+/** List of USB buses */
+struct list_head usb_buses = LIST_HEAD_INIT ( usb_buses );
+
+/** List of changed ports */
+static struct list_head usb_changed = LIST_HEAD_INIT ( usb_changed );
+
+/** List of halted endpoints */
+static struct list_head usb_halted = LIST_HEAD_INIT ( usb_halted );
+
+/******************************************************************************
+ *
+ * Utility functions
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get USB speed name (for debugging)
+ *
+ * @v speed		Speed
+ * @ret name		Speed name
+ */
+static inline const char * usb_speed_name ( unsigned int speed ) {
+	static const char *exponents[4] = { "", "k", "M", "G" };
+	static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
+	unsigned int mantissa;
+	unsigned int exponent;
+
+	/* Extract mantissa and exponent */
+	mantissa = USB_SPEED_MANTISSA ( speed );
+	exponent = USB_SPEED_EXPONENT ( speed );
+
+	/* Name speed */
+	switch ( speed ) {
+	case USB_SPEED_NONE:		return "DETACHED";
+	case USB_SPEED_LOW:		return "low";
+	case USB_SPEED_FULL:		return "full";
+	case USB_SPEED_HIGH:		return "high";
+	case USB_SPEED_SUPER:		return "super";
+	default:
+		snprintf ( buf, sizeof ( buf ), "%d%sbps",
+			   mantissa, exponents[exponent] );
+		return buf;
+	}
+}
+
+/**
+ * Transcribe USB BCD-coded value (for debugging)
+ *
+ * @v bcd		BCD-coded value
+ * @ret string		Transcribed value
+ */
+static inline const char * usb_bcd ( uint16_t bcd ) {
+	static char buf[ 6 /* "xx.xx" + NUL */ ];
+	uint8_t high = ( bcd >> 8 );
+	uint8_t low = ( bcd >> 0 );
+
+	snprintf ( buf, sizeof ( buf ), "%x.%02x", high, low );
+	return buf;
+}
+
+/******************************************************************************
+ *
+ * USB descriptors
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Locate USB interface association descriptor
+ *
+ * @v config		Configuraton descriptor
+ * @v first		First interface number
+ * @ret desc		Interface association descriptor, or NULL if not found
+ */
+static struct usb_interface_association_descriptor *
+usb_interface_association_descriptor ( struct usb_configuration_descriptor
+								       *config,
+				       unsigned int first ) {
+	struct usb_interface_association_descriptor *desc;
+
+	/* Find a matching interface association descriptor */
+	for_each_config_descriptor ( desc, config ) {
+		if ( ( desc->header.type ==
+		       USB_INTERFACE_ASSOCIATION_DESCRIPTOR ) &&
+		     ( desc->first == first ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Locate USB interface descriptor
+ *
+ * @v config		Configuraton descriptor
+ * @v interface		Interface number
+ * @v alternate		Alternate setting
+ * @ret desc		Interface descriptor, or NULL if not found
+ */
+struct usb_interface_descriptor *
+usb_interface_descriptor ( struct usb_configuration_descriptor *config,
+			   unsigned int interface, unsigned int alternate ) {
+	struct usb_interface_descriptor *desc;
+
+	/* Find a matching interface descriptor */
+	for_each_config_descriptor ( desc, config ) {
+		if ( ( desc->header.type == USB_INTERFACE_DESCRIPTOR ) &&
+		     ( desc->interface == interface ) &&
+		     ( desc->alternate == alternate ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Locate USB endpoint descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @v type		Endpoint (internal) type
+ * @v index		Endpoint index
+ * @ret desc		Descriptor, or NULL if not found
+ */
+struct usb_endpoint_descriptor *
+usb_endpoint_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface,
+			  unsigned int type, unsigned int index ) {
+	struct usb_endpoint_descriptor *desc;
+	unsigned int attributes = ( type & USB_ENDPOINT_ATTR_TYPE_MASK );
+	unsigned int direction = ( type & USB_DIR_IN );
+
+	/* Find a matching endpoint descriptor */
+	for_each_interface_descriptor ( desc, config, interface ) {
+		if ( ( desc->header.type == USB_ENDPOINT_DESCRIPTOR ) &&
+		     ( ( desc->attributes &
+			 USB_ENDPOINT_ATTR_TYPE_MASK ) == attributes ) &&
+		     ( ( desc->endpoint & USB_DIR_IN ) == direction ) &&
+		     ( index-- == 0 ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Locate USB endpoint companion descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v desc		Endpoint descriptor
+ * @ret descx		Companion descriptor, or NULL if not found
+ */
+struct usb_endpoint_companion_descriptor *
+usb_endpoint_companion_descriptor ( struct usb_configuration_descriptor *config,
+				    struct usb_endpoint_descriptor *desc ) {
+	struct usb_endpoint_companion_descriptor *descx;
+
+	/* Get companion descriptor, if present */
+	descx = container_of ( usb_next_descriptor ( &desc->header ),
+			       struct usb_endpoint_companion_descriptor,
+			       header );
+	return ( ( usb_is_within_config ( config, &descx->header ) &&
+		   descx->header.type == USB_ENDPOINT_COMPANION_DESCRIPTOR )
+		 ? descx : NULL );
+}
+
+/******************************************************************************
+ *
+ * USB endpoint
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get USB endpoint name (for debugging)
+ *
+ * @v ep		USB endpoint
+ * @ret name		Endpoint name
+ */
+const char * usb_endpoint_name ( struct usb_endpoint *ep ) {
+	static char buf[ 9 /* "EPxx OUT" + NUL */ ];
+	unsigned int address = ep->address;
+
+	snprintf ( buf, sizeof ( buf ), "EP%d%s",
+		   ( address & USB_ENDPOINT_MAX ),
+		   ( address ?
+		     ( ( address & USB_ENDPOINT_IN ) ? " IN" : " OUT" ) : "" ));
+	return buf;
+}
+
+/**
+ * Describe USB endpoint from device configuration
+ *
+ * @v ep		USB endpoint
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @v type		Endpoint (internal) type
+ * @v index		Endpoint index
+ * @ret rc		Return status code
+ */
+int usb_endpoint_described ( struct usb_endpoint *ep,
+			     struct usb_configuration_descriptor *config,
+			     struct usb_interface_descriptor *interface,
+			     unsigned int type, unsigned int index ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *port = usb->port;
+	struct usb_endpoint_descriptor *desc;
+	struct usb_endpoint_companion_descriptor *descx;
+	unsigned int sizes;
+	unsigned int burst;
+	unsigned int interval;
+	size_t mtu;
+
+	/* Locate endpoint descriptor */
+	desc = usb_endpoint_descriptor ( config, interface, type, index );
+	if ( ! desc )
+		return -ENOENT;
+
+	/* Locate companion descriptor, if any */
+	descx = usb_endpoint_companion_descriptor ( config, desc );
+
+	/* Calculate MTU and burst size */
+	sizes = le16_to_cpu ( desc->sizes );
+	mtu = USB_ENDPOINT_MTU ( sizes );
+	burst = ( descx ? descx->burst : USB_ENDPOINT_BURST ( sizes ) );
+
+	/* Calculate interval */
+	if ( ( type & USB_ENDPOINT_ATTR_TYPE_MASK ) ==
+	     USB_ENDPOINT_ATTR_INTERRUPT ) {
+		if ( port->speed >= USB_SPEED_HIGH ) {
+			/* 2^(desc->interval-1) is a microframe count */
+			interval = ( 1 << ( desc->interval - 1 ) );
+		} else {
+			/* desc->interval is a (whole) frame count */
+			interval = ( desc->interval << 3 );
+		}
+	} else {
+		/* desc->interval is a microframe count */
+		interval = desc->interval;
+	}
+
+	/* Describe endpoint */
+	usb_endpoint_describe ( ep, desc->endpoint, desc->attributes,
+				mtu, burst, interval );
+	return 0;
+}
+
+/**
+ * Open USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+int usb_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int idx = USB_ENDPOINT_IDX ( ep->address );
+	int rc;
+
+	/* Populate host controller operations */
+	ep->host = &usb->port->hub->bus->op->endpoint;
+
+	/* Add to endpoint list */
+	if ( usb->ep[idx] != NULL ) {
+		DBGC ( usb, "USB %s %s is already open\n",
+		       usb->name, usb_endpoint_name ( ep ) );
+		rc = -EALREADY;
+		goto err_already;
+	}
+	usb->ep[idx] = ep;
+	INIT_LIST_HEAD ( &ep->halted );
+
+	/* Open endpoint */
+	if ( ( rc = ep->host->open ( ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not open: %s\n", usb->name,
+		       usb_endpoint_name ( ep ), strerror ( rc ) );
+		goto err_open;
+	}
+	ep->open = 1;
+
+	DBGC2 ( usb, "USB %s %s opened with MTU %zd, burst %d, interval %d\n",
+		usb->name, usb_endpoint_name ( ep ), ep->mtu, ep->burst,
+		ep->interval );
+	return 0;
+
+	ep->open = 0;
+	ep->host->close ( ep );
+ err_open:
+	usb->ep[idx] = NULL;
+ err_already:
+	if ( ep->max )
+		usb_flush ( ep );
+	return rc;
+}
+
+/**
+ * Clear transaction translator (if applicable)
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int usb_endpoint_clear_tt ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *tt;
+	int rc;
+
+	/* Do nothing if this is a periodic endpoint */
+	if ( ep->attributes & USB_ENDPOINT_ATTR_PERIODIC )
+		return 0;
+
+	/* Do nothing if this endpoint is not behind a transaction translator */
+	tt = usb_transaction_translator ( usb );
+	if ( ! tt )
+		return 0;
+
+	/* Clear transaction translator buffer */
+	if ( ( rc = tt->hub->driver->clear_tt ( tt->hub, tt, ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not clear transaction translator: "
+		       "%s\n", usb->name, usb_endpoint_name ( ep ),
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Close USB endpoint
+ *
+ * @v ep		USB endpoint
+ */
+void usb_endpoint_close ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int idx = USB_ENDPOINT_IDX ( ep->address );
+
+	/* Sanity checks */
+	assert ( usb->ep[idx] == ep );
+
+	/* Close endpoint */
+	ep->open = 0;
+	ep->host->close ( ep );
+	assert ( ep->fill == 0 );
+
+	/* Remove from endpoint list */
+	usb->ep[idx] = NULL;
+	list_del ( &ep->halted );
+
+	/* Discard any recycled buffers, if applicable */
+	if ( ep->max )
+		usb_flush ( ep );
+
+	/* Clear transaction translator, if applicable */
+	usb_endpoint_clear_tt ( ep );
+}
+
+/**
+ * Reset USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int usb_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int type;
+	int rc;
+
+	/* Sanity check */
+	assert ( ! list_empty ( &ep->halted ) );
+
+	/* Reset endpoint */
+	if ( ( rc = ep->host->reset ( ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not reset: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	/* Clear transaction translator, if applicable */
+	if ( ( rc = usb_endpoint_clear_tt ( ep ) ) != 0 )
+		return rc;
+
+	/* Clear endpoint halt, if applicable */
+	type = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	if ( ( type != USB_ENDPOINT_ATTR_CONTROL ) &&
+	     ( ( rc = usb_clear_feature ( usb, USB_RECIP_ENDPOINT,
+					  USB_ENDPOINT_HALT,
+					  ep->address ) ) != 0 ) ) {
+		DBGC ( usb, "USB %s %s could not clear endpoint halt: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	/* Remove from list of halted endpoints */
+	list_del ( &ep->halted );
+	INIT_LIST_HEAD ( &ep->halted );
+
+	DBGC ( usb, "USB %s %s reset\n",
+	       usb->name, usb_endpoint_name ( ep ) );
+	return 0;
+}
+
+/**
+ * Update endpoint MTU
+ *
+ * @v ep		USB endpoint
+ * @v mtu		New MTU
+ * @ret rc		Return status code
+ */
+static int usb_endpoint_mtu ( struct usb_endpoint *ep, size_t mtu ) {
+	struct usb_device *usb = ep->usb;
+	int rc;
+
+	/* Update MTU */
+	ep->mtu = mtu;
+	if ( ( rc = ep->host->mtu ( ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not update MTU: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Enqueue USB message transfer
+ *
+ * @v ep		USB endpoint
+ * @v request		Request
+ * @v value		Value parameter
+ * @v index		Index parameter
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * The I/O buffer must have sufficient headroom to contain a setup
+ * packet.
+ */
+int usb_message ( struct usb_endpoint *ep, unsigned int request,
+		  unsigned int value, unsigned int index,
+		  struct io_buffer *iobuf ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *port = usb->port;
+	struct usb_setup_packet *packet;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Sanity check */
+	assert ( iob_headroom ( iobuf ) >= sizeof ( *packet ) );
+
+	/* Fail immediately if device has been unplugged */
+	if ( port->speed == USB_SPEED_NONE )
+		return -ENODEV;
+
+	/* Reset endpoint if required */
+	if ( ( ! list_empty ( &ep->halted ) ) &&
+	     ( ( rc = usb_endpoint_reset ( ep ) ) != 0 ) )
+		return rc;
+
+	/* Zero input data buffer (if applicable) */
+	if ( request & USB_DIR_IN )
+		memset ( iobuf->data, 0, len );
+
+	/* Construct setup packet */
+	packet = iob_push ( iobuf, sizeof ( *packet ) );
+	packet->request = cpu_to_le16 ( request );
+	packet->value = cpu_to_le16 ( value );
+	packet->index = cpu_to_le16 ( index );
+	packet->len = cpu_to_le16 ( len );
+
+	/* Enqueue message transfer */
+	if ( ( rc = ep->host->message ( ep, iobuf ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not enqueue message transfer: "
+		       "%s\n", usb->name, usb_endpoint_name ( ep ),
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Increment fill level */
+	ep->fill++;
+
+	return 0;
+}
+
+/**
+ * Enqueue USB stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+int usb_stream ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+		 int terminate ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *port = usb->port;
+	int rc;
+
+	/* Fail immediately if device has been unplugged */
+	if ( port->speed == USB_SPEED_NONE )
+		return -ENODEV;
+
+	/* Reset endpoint if required */
+	if ( ( ! list_empty ( &ep->halted ) ) &&
+	     ( ( rc = usb_endpoint_reset ( ep ) ) != 0 ) )
+		return rc;
+
+	/* Enqueue stream transfer */
+	if ( ( rc = ep->host->stream ( ep, iobuf, terminate ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not enqueue stream transfer: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	/* Increment fill level */
+	ep->fill++;
+
+	return 0;
+}
+
+/**
+ * Complete transfer (possibly with error)
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+void usb_complete_err ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			int rc ) {
+	struct usb_device *usb = ep->usb;
+
+	/* Decrement fill level */
+	assert ( ep->fill > 0 );
+	ep->fill--;
+
+	/* Schedule reset, if applicable */
+	if ( ( rc != 0 ) && ep->open ) {
+		DBGC ( usb, "USB %s %s completion failed: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		list_del ( &ep->halted );
+		list_add_tail ( &ep->halted, &usb_halted );
+	}
+
+	/* Report completion */
+	ep->driver->complete ( ep, iobuf, rc );
+}
+
+/******************************************************************************
+ *
+ * Endpoint refilling
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Prefill endpoint recycled buffer list
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+int usb_prefill ( struct usb_endpoint *ep ) {
+	struct io_buffer *iobuf;
+	size_t len = ( ep->len ? ep->len : ep->mtu );
+	unsigned int fill;
+	int rc;
+
+	/* Sanity checks */
+	assert ( ep->fill == 0 );
+	assert ( ep->max > 0 );
+	assert ( list_empty ( &ep->recycled ) );
+
+	/* Fill recycled buffer list */
+	for ( fill = 0 ; fill < ep->max ; fill++ ) {
+
+		/* Allocate I/O buffer */
+		iobuf = alloc_iob ( len );
+		if ( ! iobuf ) {
+			rc = -ENOMEM;
+			goto err_alloc;
+		}
+
+		/* Add to recycled buffer list */
+		list_add_tail ( &iobuf->list, &ep->recycled );
+	}
+
+	return 0;
+
+ err_alloc:
+	usb_flush ( ep );
+	return rc;
+}
+
+/**
+ * Refill endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+int usb_refill ( struct usb_endpoint *ep ) {
+	struct io_buffer *iobuf;
+	size_t len = ( ep->len ? ep->len : ep->mtu );
+	int rc;
+
+	/* Sanity checks */
+	assert ( ep->open );
+	assert ( ep->max > 0 );
+
+	/* Refill endpoint */
+	while ( ep->fill < ep->max ) {
+
+		/* Get or allocate buffer */
+		if ( list_empty ( &ep->recycled ) ) {
+			/* Recycled buffer list is empty; allocate new buffer */
+			iobuf = alloc_iob ( len );
+			if ( ! iobuf )
+				return -ENOMEM;
+		} else {
+			/* Get buffer from recycled buffer list */
+			iobuf = list_first_entry ( &ep->recycled,
+						   struct io_buffer, list );
+			assert ( iobuf != NULL );
+			list_del ( &iobuf->list );
+		}
+
+		/* Reset buffer to maximum size */
+		assert ( iob_len ( iobuf ) <= len );
+		iob_put ( iobuf, ( len - iob_len ( iobuf ) ) );
+
+		/* Enqueue buffer */
+		if ( ( rc = usb_stream ( ep, iobuf, 0 ) ) != 0 ) {
+			list_add ( &iobuf->list, &ep->recycled );
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Discard endpoint recycled buffer list
+ *
+ * @v ep		USB endpoint
+ */
+void usb_flush ( struct usb_endpoint *ep ) {
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
+	/* Sanity checks */
+	assert ( ! ep->open );
+	assert ( ep->max > 0 );
+
+	/* Free all I/O buffers */
+	list_for_each_entry_safe ( iobuf, tmp, &ep->recycled, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+}
+
+/******************************************************************************
+ *
+ * Control endpoint
+ *
+ ******************************************************************************
+ */
+
+/** USB control transfer pseudo-header */
+struct usb_control_pseudo_header {
+	/** Completion status */
+	int rc;
+};
+
+/**
+ * Complete USB control transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void usb_control_complete ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf, int rc ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_control_pseudo_header *pshdr;
+
+	/* Record completion status in buffer */
+	pshdr = iob_push ( iobuf, sizeof ( *pshdr ) );
+	pshdr->rc = rc;
+
+	/* Add to list of completed I/O buffers */
+	list_add_tail ( &iobuf->list, &usb->complete );
+}
+
+/** USB control endpoint driver operations */
+static struct usb_endpoint_driver_operations usb_control_operations = {
+	.complete = usb_control_complete,
+};
+
+/**
+ * Issue USB control transaction
+ *
+ * @v usb		USB device
+ * @v request		Request
+ * @v value		Value parameter
+ * @v index		Index parameter
+ * @v data		Data buffer (if any)
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+int usb_control ( struct usb_device *usb, unsigned int request,
+		  unsigned int value, unsigned int index, void *data,
+		  size_t len ) {
+	struct usb_bus *bus = usb->port->hub->bus;
+	struct usb_endpoint *ep = &usb->control;
+	struct io_buffer *iobuf;
+	struct io_buffer *cmplt;
+	union {
+		struct usb_setup_packet setup;
+		struct usb_control_pseudo_header pshdr;
+	} *headroom;
+	struct usb_control_pseudo_header *pshdr;
+	unsigned int i;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = alloc_iob ( sizeof ( *headroom ) + len );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	iob_reserve ( iobuf, sizeof ( *headroom ) );
+	iob_put ( iobuf, len );
+	if ( request & USB_DIR_IN ) {
+		memset ( data, 0, len );
+	} else {
+		memcpy ( iobuf->data, data, len );
+	}
+
+	/* Enqueue message */
+	if ( ( rc = usb_message ( ep, request, value, index, iobuf ) ) != 0 )
+		goto err_message;
+
+	/* Wait for completion */
+	for ( i = 0 ; i < USB_CONTROL_MAX_WAIT_MS ; i++ ) {
+
+		/* Poll bus */
+		usb_poll ( bus );
+
+		/* Check for completion */
+		while ( ( cmplt = list_first_entry ( &usb->complete,
+						     struct io_buffer,
+						     list ) ) ) {
+
+			/* Remove from completion list */
+			list_del ( &cmplt->list );
+
+			/* Extract and strip completion status */
+			pshdr = cmplt->data;
+			iob_pull ( cmplt, sizeof ( *pshdr ) );
+			rc = pshdr->rc;
+
+			/* Discard stale completions */
+			if ( cmplt != iobuf ) {
+				DBGC ( usb, "USB %s stale control completion: "
+				       "%s\n", usb->name, strerror ( rc ) );
+				DBGC_HDA ( usb, 0, cmplt->data,
+					   iob_len ( cmplt ) );
+				free_iob ( cmplt );
+				continue;
+			}
+
+			/* Fail immediately if completion was in error */
+			if ( rc != 0 ) {
+				DBGC ( usb, "USB %s control %04x:%04x:%04x "
+				       "failed: %s\n", usb->name, request,
+				       value, index, strerror ( rc ) );
+				free_iob ( cmplt );
+				return rc;
+			}
+
+			/* Copy completion to data buffer, if applicable */
+			assert ( iob_len ( cmplt ) <= len );
+			if ( request & USB_DIR_IN )
+				memcpy ( data, cmplt->data, iob_len ( cmplt ) );
+			free_iob ( cmplt );
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( usb, "USB %s timed out waiting for control %04x:%04x:%04x\n",
+	       usb->name, request, value, index );
+	return -ETIMEDOUT;
+
+ err_message:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Get USB string descriptor
+ *
+ * @v usb		USB device
+ * @v index		String index
+ * @v language		Language ID
+ * @v buf		Data buffer
+ * @v len		Length of buffer
+ * @ret len		String length (excluding NUL), or negative error
+ */
+int usb_get_string_descriptor ( struct usb_device *usb, unsigned int index,
+				unsigned int language, char *buf, size_t len ) {
+	size_t max = ( len ? ( len - 1 /* NUL */ ) : 0 );
+	struct {
+		struct usb_descriptor_header header;
+		uint16_t character[max];
+	} __attribute__ (( packed )) *desc;
+	unsigned int actual;
+	unsigned int i;
+	int rc;
+
+	/* Allocate buffer for string */
+	desc = malloc ( sizeof ( *desc ) );
+	if ( ! desc ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Get descriptor */
+	if ( ( rc = usb_get_descriptor ( usb, 0, USB_STRING_DESCRIPTOR, index,
+					 language, &desc->header,
+					 sizeof ( *desc ) ) ) != 0 )
+		goto err_get_descriptor;
+
+	/* Copy to buffer */
+	actual = ( ( desc->header.len - sizeof ( desc->header ) ) /
+		   sizeof ( desc->character[0] ) );
+	for ( i = 0 ; ( ( i < actual ) && ( i < max ) ) ; i++ )
+		buf[i] = le16_to_cpu ( desc->character[i] );
+	if ( len )
+		buf[i] = '\0';
+
+	/* Free buffer */
+	free ( desc );
+
+	return actual;
+
+ err_get_descriptor:
+	free ( desc );
+ err_alloc:
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * USB device driver
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Describe USB function
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @v first		First interface number
+ * @ret rc		Return status code
+ */
+static int usb_function ( struct usb_function *func,
+			  struct usb_configuration_descriptor *config,
+			  unsigned int first ) {
+	struct usb_device *usb = func->usb;
+	struct usb_interface_association_descriptor *association;
+	struct usb_interface_descriptor *interface;
+	struct cdc_union_descriptor *cdc_union;
+	unsigned int i;
+
+	/* First, look for an interface association descriptor */
+	association = usb_interface_association_descriptor ( config, first );
+	if ( association ) {
+
+		/* Sanity check */
+		if ( association->count > config->interfaces ) {
+			DBGC ( usb, "USB %s has invalid association [%d-%d)\n",
+			       func->name, association->first,
+			       ( association->first + association->count ) );
+			return -ERANGE;
+		}
+
+		/* Describe function */
+		memcpy ( &func->class, &association->class,
+			 sizeof ( func->class ) );
+		func->count = association->count;
+		for ( i = 0 ; i < association->count ; i++ )
+			func->interface[i] = ( association->first + i );
+		return 0;
+	}
+
+	/* Next, look for an interface descriptor */
+	interface = usb_interface_descriptor ( config, first, 0 );
+	if ( ! interface ) {
+		DBGC ( usb, "USB %s has no interface descriptor\n",
+		       func->name );
+		return -ENOENT;
+	}
+
+	/* Describe function */
+	memcpy ( &func->class, &interface->class, sizeof ( func->class ) );
+	func->count = 1;
+	func->interface[0] = first;
+
+	/* Look for a CDC union descriptor, if applicable */
+	if ( ( func->class.class == USB_CLASS_CDC ) &&
+	     ( cdc_union = cdc_union_descriptor ( config, interface ) ) ) {
+
+		/* Determine interface count */
+		func->count = ( ( cdc_union->header.len -
+				  offsetof ( typeof ( *cdc_union ),
+					     interface[0] ) ) /
+				sizeof ( cdc_union->interface[0] ) );
+		if ( func->count > config->interfaces ) {
+			DBGC ( usb, "USB %s has invalid union functional "
+			       "descriptor with %d interfaces\n",
+			       func->name, func->count );
+			return -ERANGE;
+		}
+
+		/* Describe function */
+		for ( i = 0 ; i < func->count ; i++ )
+			func->interface[i] = cdc_union->interface[i];
+
+		return 0;
+	}
+
+	return 0;
+}
+
+/**
+ * Check for a USB device ID match
+ *
+ * @v func		USB function
+ * @v id		Device ID
+ * @ret matches		Device ID matches
+ */
+static int
+usb_device_id_matches ( struct usb_function *func, struct usb_device_id *id ) {
+
+	return ( ( ( id->vendor == func->dev.desc.vendor ) ||
+		   ( id->vendor == USB_ANY_ID ) ) &&
+		 ( ( id->product == func->dev.desc.device ) ||
+		   ( id->product == USB_ANY_ID ) ) &&
+		 ( id->class.class == func->class.class ) &&
+		 ( id->class.subclass == func->class.subclass ) &&
+		 ( id->class.protocol == func->class.protocol ) );
+}
+
+/**
+ * Probe USB device driver
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usb_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct usb_driver *driver;
+	struct usb_device_id *id;
+	unsigned int i;
+	int rc;
+
+	/* Look for a matching driver */
+	for_each_table_entry ( driver, USB_DRIVERS ) {
+		for ( i = 0 ; i < driver->id_count ; i++ ) {
+
+			/* Check for a matching ID */
+			id = &driver->ids[i];
+			if ( ! usb_device_id_matches ( func, id ) )
+				continue;
+
+			/* Probe driver */
+			if ( ( rc = driver->probe ( func, config ) ) != 0 ) {
+				DBGC ( usb, "USB %s failed to probe driver %s: "
+				       "%s\n", func->name, id->name,
+				       strerror ( rc ) );
+				/* Continue trying other drivers */
+				continue;
+			}
+
+			/* Record driver */
+			func->driver = driver;
+			func->dev.driver_name = id->name;
+			return 0;
+		}
+	}
+
+	/* No driver found */
+	DBGC ( usb, "USB %s %04x:%04x class %d:%d:%d has no driver\n",
+	       func->name, func->dev.desc.vendor, func->dev.desc.device,
+	       func->class.class, func->class.subclass, func->class.protocol );
+	return -ENOENT;
+}
+
+/**
+ * Remove USB device driver
+ *
+ * @v func		USB function
+ */
+static void usb_remove ( struct usb_function *func ) {
+
+	/* Remove driver */
+	func->driver->remove ( func );
+}
+
+/**
+ * Probe all USB device drivers
+ *
+ * @v usb		USB device
+ * @v config		Configuration descriptor
+ */
+static void
+usb_probe_all ( struct usb_device *usb,
+		struct usb_configuration_descriptor *config ) {
+	struct usb_bus *bus = usb->port->hub->bus;
+	struct usb_function *func;
+	uint8_t used[config->interfaces];
+	unsigned int first;
+	unsigned int i;
+	int rc;
+
+	/* Identify each function in turn */
+	memset ( used, 0, sizeof ( used ) );
+	for ( first = 0 ; first < config->interfaces ; first++ ) {
+
+		/* Skip interfaces already used */
+		if ( used[first] )
+			continue;
+
+		/* Allocate and initialise structure */
+		func = zalloc ( sizeof ( *func ) +
+				( config->interfaces *
+				  sizeof ( func->interface[0] ) ) );
+		if ( ! func )
+			goto err_alloc;
+		func->name = func->dev.name;
+		func->usb = usb;
+		func->dev.desc.bus_type = BUS_TYPE_USB;
+		func->dev.desc.location = usb->address;
+		func->dev.desc.vendor = le16_to_cpu ( usb->device.vendor );
+		func->dev.desc.device = le16_to_cpu ( usb->device.product );
+		snprintf ( func->dev.name, sizeof ( func->dev.name ),
+			   "%s-%d.%d", usb->name, config->config, first );
+		INIT_LIST_HEAD ( &func->dev.children );
+		func->dev.parent = bus->dev;
+
+		/* Identify function */
+		if ( ( rc = usb_function ( func, config, first ) ) != 0 )
+			goto err_function;
+		assert ( func->count <= config->interfaces );
+
+		/* Mark interfaces as used */
+		for ( i = 0 ; i < func->count ; i++ ) {
+			if ( func->interface[i] >= config->interfaces ) {
+				DBGC ( usb, "USB %s has invalid interface %d\n",
+				       func->name, func->interface[i] );
+				goto err_interface;
+			}
+			used[ func->interface[i] ] = 1;
+		}
+
+		/* Probe device driver */
+		if ( ( rc = usb_probe ( func, config ) ) != 0 )
+			goto err_probe;
+		DBGC ( usb, "USB %s %04x:%04x class %d:%d:%d interfaces ",
+		       func->name, func->dev.desc.vendor, func->dev.desc.device,
+		       func->class.class, func->class.subclass,
+		       func->class.protocol );
+		for ( i = 0 ; i < func->count ; i++ )
+			DBGC ( usb, "%s%d", ( i ? "," : "" ),
+			       func->interface[i] );
+		DBGC ( usb, " using driver %s\n", func->dev.driver_name );
+
+		/* Add to list of functions */
+		list_add ( &func->list, &usb->functions );
+
+		/* Add to device hierarchy */
+		list_add_tail ( &func->dev.siblings, &bus->dev->children );
+
+		continue;
+
+		list_del ( &func->dev.siblings );
+		list_del ( &func->list );
+		usb_remove ( func );
+	err_probe:
+		free ( func );
+	err_alloc:
+	err_interface:
+	err_function:
+		/* Continue registering other functions */
+		continue;
+	}
+}
+
+/**
+ * Remove all device drivers
+ *
+ * @v usb		USB device
+ */
+static void usb_remove_all ( struct usb_device *usb ) {
+	struct usb_function *func;
+	struct usb_function *tmp;
+
+	/* Remove all functions */
+	list_for_each_entry_safe ( func, tmp, &usb->functions, list ) {
+
+		/* Remove device driver */
+		usb_remove ( func );
+
+		/* Remove from device hierarchy */
+		assert ( list_empty ( &func->dev.children ) );
+		list_del ( &func->dev.siblings );
+
+		/* Remove from list of functions */
+		list_del ( &func->list );
+
+		/* Free function */
+		free ( func );
+	}
+}
+
+/**
+ * Select USB device configuration
+ *
+ * @v usb		USB device
+ * @v index		Configuration index
+ * @ret rc		Return status code
+ */
+static int usb_configure ( struct usb_device *usb, unsigned int index ) {
+	struct usb_configuration_descriptor partial;
+	struct usb_configuration_descriptor *config;
+	size_t len;
+	int rc;
+
+	/* Read first part of configuration descriptor to get size */
+	if ( ( rc = usb_get_config_descriptor ( usb, index, &partial,
+						sizeof ( partial ) ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get configuration descriptor %d: "
+		       "%s\n", usb->name, index, strerror ( rc ) );
+		goto err_get_partial;
+	}
+	len = le16_to_cpu ( partial.len );
+	if ( len < sizeof ( partial ) ) {
+		DBGC ( usb, "USB %s underlength configuraton descriptor %d\n",
+		       usb->name, index );
+		rc = -EINVAL;
+		goto err_partial_len;
+	}
+
+	/* Allocate buffer for whole configuration descriptor */
+	config = malloc ( len );
+	if ( ! config ) {
+		rc = -ENOMEM;
+		goto err_alloc_config;
+	}
+
+	/* Read whole configuration descriptor */
+	if ( ( rc = usb_get_config_descriptor ( usb, index, config,
+						len ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get configuration descriptor %d: "
+		       "%s\n", usb->name, index, strerror ( rc ) );
+		goto err_get_config_descriptor;
+	}
+	if ( config->len != partial.len ) {
+		DBGC ( usb, "USB %s bad configuration descriptor %d length\n",
+		       usb->name, index );
+		rc = -EINVAL;
+		goto err_config_len;
+	}
+
+	/* Set configuration */
+	if ( ( rc = usb_set_configuration ( usb, config->config ) ) != 0){
+		DBGC ( usb, "USB %s could not set configuration %d: %s\n",
+		       usb->name, config->config, strerror ( rc ) );
+		goto err_set_configuration;
+	}
+
+	/* Probe USB device drivers */
+	usb_probe_all ( usb, config );
+
+	/* Free configuration descriptor */
+	free ( config );
+
+	return 0;
+
+	usb_remove_all ( usb );
+	usb_set_configuration ( usb, 0 );
+ err_set_configuration:
+ err_config_len:
+ err_get_config_descriptor:
+	free ( config );
+ err_alloc_config:
+ err_partial_len:
+ err_get_partial:
+	return rc;
+}
+
+/**
+ * Clear USB device configuration
+ *
+ * @v usb		USB device
+ */
+static void usb_deconfigure ( struct usb_device *usb ) {
+	unsigned int i;
+
+	/* Remove device drivers */
+	usb_remove_all ( usb );
+
+	/* Sanity checks */
+	for ( i = 0 ; i < ( sizeof ( usb->ep ) / sizeof ( usb->ep[0] ) ) ; i++){
+		if ( i != USB_ENDPOINT_IDX ( USB_EP0_ADDRESS ) )
+			assert ( usb->ep[i] == NULL );
+	}
+
+	/* Clear device configuration */
+	usb_set_configuration ( usb, 0 );
+}
+
+/**
+ * Find and select a supported USB device configuration
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int usb_configure_any ( struct usb_device *usb ) {
+	unsigned int index;
+	int rc = -ENOENT;
+
+	/* Attempt all configuration indexes */
+	for ( index = 0 ; index < usb->device.configurations ; index++ ) {
+
+		/* Attempt this configuration index */
+		if ( ( rc = usb_configure ( usb, index ) ) != 0 )
+			continue;
+
+		/* If we have no drivers, then try the next configuration */
+		if ( list_empty ( &usb->functions ) ) {
+			rc = -ENOTSUP;
+			usb_deconfigure ( usb );
+			continue;
+		}
+
+		return 0;
+	}
+
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * USB device
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate USB device
+ *
+ * @v port		USB port
+ * @ret usb		USB device, or NULL on allocation failure
+ */
+static struct usb_device * alloc_usb ( struct usb_port *port ) {
+	struct usb_hub *hub = port->hub;
+	struct usb_bus *bus = hub->bus;
+	struct usb_device *usb;
+
+	/* Allocate and initialise structure */
+	usb = zalloc ( sizeof ( *usb ) );
+	if ( ! usb )
+		return NULL;
+	snprintf ( usb->name, sizeof ( usb->name ), "%s%c%d", hub->name,
+		   ( hub->usb ? '.' : '-' ), port->address );
+	usb->port = port;
+	INIT_LIST_HEAD ( &usb->functions );
+	usb->host = &bus->op->device;
+	usb_endpoint_init ( &usb->control, usb, &usb_control_operations );
+	INIT_LIST_HEAD ( &usb->complete );
+
+	return usb;
+}
+
+/**
+ * Register USB device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int register_usb ( struct usb_device *usb ) {
+	struct usb_port *port = usb->port;
+	struct usb_hub *hub = port->hub;
+	struct usb_bus *bus = hub->bus;
+	unsigned int protocol;
+	size_t mtu;
+	int rc;
+
+	/* Add to port */
+	if ( port->usb != NULL ) {
+		DBGC ( hub, "USB hub %s port %d is already registered to %s\n",
+		       hub->name, port->address, port->usb->name );
+		rc = -EALREADY;
+		goto err_already;
+	}
+	port->usb = usb;
+
+	/* Add to bus device list */
+	list_add_tail ( &usb->list, &bus->devices );
+
+	/* Enable device */
+	if ( ( rc = hub->driver->enable ( hub, port ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s port %d could not enable: %s\n",
+		       hub->name, port->address, strerror ( rc ) );
+		goto err_enable;
+	}
+
+	/* Allow recovery interval since port may have been reset */
+	mdelay ( USB_RESET_RECOVER_DELAY_MS );
+
+	/* Get device speed */
+	if ( ( rc = hub->driver->speed ( hub, port ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s port %d could not get speed: %s\n",
+		       hub->name, port->address, strerror ( rc ) );
+		goto err_speed;
+	}
+	DBGC2 ( usb, "USB %s attached as %s-speed device\n",
+		usb->name, usb_speed_name ( port->speed ) );
+
+	/* Open device */
+	if ( ( rc = usb->host->open ( usb ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not open: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Describe control endpoint */
+	mtu = USB_EP0_DEFAULT_MTU ( port->speed );
+	usb_endpoint_describe ( &usb->control, USB_EP0_ADDRESS,
+				USB_EP0_ATTRIBUTES, mtu, USB_EP0_BURST,
+				USB_EP0_INTERVAL );
+
+	/* Open control endpoint */
+	if ( ( rc = usb_endpoint_open ( &usb->control ) ) != 0 )
+		goto err_open_control;
+	assert ( usb_endpoint ( usb, USB_EP0_ADDRESS ) == &usb->control );
+
+	/* Assign device address */
+	if ( ( rc = usb->host->address ( usb ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not set address: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_address;
+	}
+	DBGC2 ( usb, "USB %s assigned address %d\n", usb->name, usb->address );
+
+	/* Allow recovery interval after Set Address command */
+	mdelay ( USB_SET_ADDRESS_RECOVER_DELAY_MS );
+
+	/* Read first part of device descriptor to get EP0 MTU */
+	if ( ( rc = usb_get_mtu ( usb, &usb->device ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get MTU: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_get_mtu;
+	}
+
+	/* Calculate EP0 MTU */
+	protocol = le16_to_cpu ( usb->device.protocol );
+	mtu = ( ( protocol < USB_PROTO_3_0 ) ?
+		usb->device.mtu : ( 1 << usb->device.mtu ) );
+	DBGC2 ( usb, "USB %s has control MTU %zd (guessed %zd)\n",
+		usb->name, mtu, usb->control.mtu );
+
+	/* Update MTU */
+	if ( ( rc = usb_endpoint_mtu ( &usb->control, mtu ) ) != 0 )
+		goto err_mtu;
+
+	/* Read whole device descriptor */
+	if ( ( rc = usb_get_device_descriptor ( usb, &usb->device ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get device descriptor: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_get_device_descriptor;
+	}
+	DBGC ( usb, "USB %s addr %d %04x:%04x class %d:%d:%d (v%s, %s-speed, "
+	       "MTU %zd)\n", usb->name, usb->address,
+	       le16_to_cpu ( usb->device.vendor ),
+	       le16_to_cpu ( usb->device.product ), usb->device.class.class,
+	       usb->device.class.subclass, usb->device.class.protocol,
+	       usb_bcd ( le16_to_cpu ( usb->device.protocol ) ),
+	       usb_speed_name ( port->speed ), usb->control.mtu );
+
+	/* Configure device */
+	if ( ( rc = usb_configure_any ( usb ) ) != 0 )
+		goto err_configure_any;
+
+	return 0;
+
+	usb_deconfigure ( usb );
+ err_configure_any:
+ err_get_device_descriptor:
+ err_mtu:
+ err_get_mtu:
+ err_address:
+	usb_endpoint_close ( &usb->control );
+ err_open_control:
+	usb->host->close ( usb );
+ err_open:
+ err_speed:
+	hub->driver->disable ( hub, port );
+ err_enable:
+	list_del ( &usb->list );
+	port->usb = NULL;
+ err_already:
+	return rc;
+}
+
+/**
+ * Unregister USB device
+ *
+ * @v usb		USB device
+ */
+static void unregister_usb ( struct usb_device *usb ) {
+	struct usb_port *port = usb->port;
+	struct usb_hub *hub = port->hub;
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
+	/* Sanity checks */
+	assert ( port->usb == usb );
+
+	/* Clear device configuration */
+	usb_deconfigure ( usb );
+
+	/* Close control endpoint */
+	usb_endpoint_close ( &usb->control );
+
+	/* Discard any stale control completions */
+	list_for_each_entry_safe ( iobuf, tmp, &usb->complete, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+
+	/* Close device */
+	usb->host->close ( usb );
+
+	/* Disable port */
+	hub->driver->disable ( hub, port );
+
+	/* Remove from bus device list */
+	list_del ( &usb->list );
+
+	/* Remove from port */
+	port->usb = NULL;
+}
+
+/**
+ * Free USB device
+ *
+ * @v usb		USB device
+ */
+static void free_usb ( struct usb_device *usb ) {
+	unsigned int i;
+
+	/* Sanity checks */
+	for ( i = 0 ; i < ( sizeof ( usb->ep ) / sizeof ( usb->ep[0] ) ) ; i++ )
+		assert ( usb->ep[i] == NULL );
+	assert ( list_empty ( &usb->functions ) );
+	assert ( list_empty ( &usb->complete ) );
+
+	/* Free device */
+	free ( usb );
+}
+
+/******************************************************************************
+ *
+ * USB device hotplug event handling
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Handle newly attached USB device
+ *
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int usb_attached ( struct usb_port *port ) {
+	struct usb_device *usb;
+	int rc;
+
+	/* Mark port as attached */
+	port->attached = 1;
+
+	/* Sanity checks */
+	assert ( port->usb == NULL );
+
+	/* Allocate USB device */
+	usb = alloc_usb ( port );
+	if ( ! usb ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Register USB device */
+	if ( ( rc = register_usb ( usb ) ) != 0 )
+		goto err_register;
+
+	return 0;
+
+	unregister_usb ( usb );
+ err_register:
+	free_usb ( usb );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Handle newly detached USB device
+ *
+ * @v port		USB port
+ */
+static void usb_detached ( struct usb_port *port ) {
+	struct usb_device *usb = port->usb;
+
+	/* Mark port as detached */
+	port->attached = 0;
+
+	/* Do nothing if we have no USB device */
+	if ( ! usb )
+		return;
+
+	/* Unregister USB device */
+	unregister_usb ( usb );
+
+	/* Free USB device */
+	free_usb ( usb );
+}
+
+/**
+ * Handle newly attached or detached USB device
+ *
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int usb_hotplugged ( struct usb_port *port ) {
+	struct usb_hub *hub = port->hub;
+	int rc;
+
+	/* Get current port speed */
+	if ( ( rc = hub->driver->speed ( hub, port ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s port %d could not get speed: %s\n",
+		       hub->name, port->address, strerror ( rc ) );
+		goto err_speed;
+	}
+
+	/* Detach device, if applicable */
+	if ( port->attached && ( port->disconnected || ! port->speed ) )
+		usb_detached ( port );
+
+	/* Attach device, if applicable */
+	if ( port->speed && ( ! port->attached ) &&
+	     ( ( rc = usb_attached ( port ) ) != 0 ) )
+		goto err_attached;
+
+ err_attached:
+ err_speed:
+	/* Clear any recorded disconnections */
+	port->disconnected = 0;
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * USB process
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Report port status change
+ *
+ * @v port		USB port
+ */
+void usb_port_changed ( struct usb_port *port ) {
+
+	/* Record hub port status change */
+	list_del ( &port->changed );
+	list_add_tail ( &port->changed, &usb_changed );
+}
+
+/**
+ * Handle newly attached or detached USB device
+ *
+ */
+static void usb_hotplug ( void ) {
+	struct usb_port *port;
+
+	/* Handle any changed ports, allowing for the fact that the
+	 * port list may change as we perform hotplug actions.
+	 */
+	while ( ! list_empty ( &usb_changed ) ) {
+
+		/* Get first changed port */
+		port = list_first_entry ( &usb_changed, struct usb_port,
+					  changed );
+		assert ( port != NULL );
+
+		/* Remove from list of changed ports */
+		list_del ( &port->changed );
+		INIT_LIST_HEAD ( &port->changed );
+
+		/* Perform appropriate hotplug action */
+		usb_hotplugged ( port );
+	}
+}
+
+/**
+ * USB process
+ *
+ * @v process		USB process
+ */
+static void usb_step ( struct process *process __unused ) {
+	struct usb_bus *bus;
+	struct usb_endpoint *ep;
+
+	/* Poll all buses */
+	for_each_usb_bus ( bus )
+		usb_poll ( bus );
+
+	/* Attempt to reset first halted endpoint in list, if any.  We
+	 * do not attempt to process the complete list, since this
+	 * would require extra code to allow for the facts that the
+	 * halted endpoint list may change as we do so, and that
+	 * resetting an endpoint may fail.
+	 */
+	if ( ( ep = list_first_entry ( &usb_halted, struct usb_endpoint,
+				       halted ) ) != NULL )
+		usb_endpoint_reset ( ep );
+
+	/* Handle any changed ports */
+	usb_hotplug();
+}
+
+/** USB process */
+PERMANENT_PROCESS ( usb_process, usb_step );
+
+/******************************************************************************
+ *
+ * USB hub
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate USB hub
+ *
+ * @v bus		USB bus
+ * @v usb		Underlying USB device, if any
+ * @v ports		Number of ports
+ * @v driver		Hub driver operations
+ * @ret hub		USB hub, or NULL on allocation failure
+ */
+struct usb_hub * alloc_usb_hub ( struct usb_bus *bus, struct usb_device *usb,
+				 unsigned int ports,
+				 struct usb_hub_driver_operations *driver ) {
+	struct usb_hub *hub;
+	struct usb_port *port;
+	unsigned int i;
+
+	/* Allocate and initialise structure */
+	hub = zalloc ( sizeof ( *hub ) + ( ports * sizeof ( hub->port[0] ) ) );
+	if ( ! hub )
+		return NULL;
+	hub->name = ( usb ? usb->name : bus->name );
+	hub->bus = bus;
+	hub->usb = usb;
+	if ( usb )
+		hub->protocol = usb->port->protocol;
+	hub->ports = ports;
+	hub->driver = driver;
+	hub->host = &bus->op->hub;
+
+	/* Initialise port list */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		port->hub = hub;
+		port->address = i;
+		if ( usb )
+			port->protocol = usb->port->protocol;
+		INIT_LIST_HEAD ( &port->changed );
+	}
+
+	return hub;
+}
+
+/**
+ * Register USB hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+int register_usb_hub ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct usb_port *port;
+	unsigned int i;
+	int rc;
+
+	/* Add to hub list */
+	list_add_tail ( &hub->list, &bus->hubs );
+
+	/* Open hub (host controller) */
+	if ( ( rc = hub->host->open ( hub ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s could not open: %s\n",
+		       hub->name, strerror ( rc ) );
+		goto err_host_open;
+	}
+
+	/* Open hub (driver) */
+	if ( ( rc = hub->driver->open ( hub ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s could not open: %s\n",
+		       hub->name, strerror ( rc ) );
+		goto err_driver_open;
+	}
+
+	/* Delay to allow ports to stabilise */
+	mdelay ( USB_PORT_DELAY_MS );
+
+	/* Mark all ports as changed */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		usb_port_changed ( port );
+	}
+
+	/* Some hubs seem to defer reporting device connections until
+	 * their interrupt endpoint is polled for the first time.
+	 * Poll the bus once now in order to pick up any such
+	 * connections.
+	 */
+	usb_poll ( bus );
+
+	return 0;
+
+	hub->driver->close ( hub );
+ err_driver_open:
+	hub->host->close ( hub );
+ err_host_open:
+	list_del ( &hub->list );
+	return rc;
+}
+
+/**
+ * Unregister USB hub
+ *
+ * @v hub		USB hub
+ */
+void unregister_usb_hub ( struct usb_hub *hub ) {
+	struct usb_port *port;
+	unsigned int i;
+
+	/* Detach all devices */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		if ( port->attached )
+			usb_detached ( port );
+	}
+
+	/* Close hub (driver) */
+	hub->driver->close ( hub );
+
+	/* Close hub (host controller) */
+	hub->host->close ( hub );
+
+	/* Cancel any pending port status changes */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		list_del ( &port->changed );
+		INIT_LIST_HEAD ( &port->changed );
+	}
+
+	/* Remove from hub list */
+	list_del ( &hub->list );
+}
+
+/**
+ * Free USB hub
+ *
+ * @v hub		USB hub
+ */
+void free_usb_hub ( struct usb_hub *hub ) {
+	struct usb_port *port;
+	unsigned int i;
+
+	/* Sanity checks */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		assert ( ! port->attached );
+		assert ( port->usb == NULL );
+		assert ( list_empty ( &port->changed ) );
+	}
+
+	/* Free hub */
+	free ( hub );
+}
+
+/******************************************************************************
+ *
+ * USB bus
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate USB bus
+ *
+ * @v dev		Underlying hardware device
+ * @v ports		Number of root hub ports
+ * @v mtu		Largest transfer allowed on the bus
+ * @v op		Host controller operations
+ * @ret bus		USB bus, or NULL on allocation failure
+ */
+struct usb_bus * alloc_usb_bus ( struct device *dev, unsigned int ports,
+				 size_t mtu, struct usb_host_operations *op ) {
+	struct usb_bus *bus;
+
+	/* Allocate and initialise structure */
+	bus = zalloc ( sizeof ( *bus ) );
+	if ( ! bus )
+		goto err_alloc_bus;
+	bus->name = dev->name;
+	bus->dev = dev;
+	bus->mtu = mtu;
+	bus->op = op;
+	INIT_LIST_HEAD ( &bus->devices );
+	INIT_LIST_HEAD ( &bus->hubs );
+	bus->host = &bus->op->bus;
+
+	/* Allocate root hub */
+	bus->hub = alloc_usb_hub ( bus, NULL, ports, &op->root );
+	if ( ! bus->hub )
+		goto err_alloc_hub;
+
+	return bus;
+
+	free_usb_hub ( bus->hub );
+ err_alloc_hub:
+	free ( bus );
+ err_alloc_bus:
+	return NULL;
+}
+
+/**
+ * Register USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+int register_usb_bus ( struct usb_bus *bus ) {
+	int rc;
+
+	/* Sanity checks */
+	assert ( bus->hub != NULL );
+
+	/* Open bus */
+	if ( ( rc = bus->host->open ( bus ) ) != 0 )
+		goto err_open;
+
+	/* Add to list of USB buses */
+	list_add_tail ( &bus->list, &usb_buses );
+
+	/* Register root hub */
+	if ( ( rc = register_usb_hub ( bus->hub ) ) != 0 )
+		goto err_register_hub;
+
+	/* Attach any devices already present */
+	usb_hotplug();
+
+	return 0;
+
+	unregister_usb_hub ( bus->hub );
+ err_register_hub:
+	list_del ( &bus->list );
+	bus->host->close ( bus );
+ err_open:
+	return rc;
+}
+
+/**
+ * Unregister USB bus
+ *
+ * @v bus		USB bus
+ */
+void unregister_usb_bus ( struct usb_bus *bus ) {
+
+	/* Sanity checks */
+	assert ( bus->hub != NULL );
+
+	/* Unregister root hub */
+	unregister_usb_hub ( bus->hub );
+
+	/* Remove from list of USB buses */
+	list_del ( &bus->list );
+
+	/* Close bus */
+	bus->host->close ( bus );
+
+	/* Sanity checks */
+	assert ( list_empty ( &bus->devices ) );
+	assert ( list_empty ( &bus->hubs ) );
+}
+
+/**
+ * Free USB bus
+ *
+ * @v bus		USB bus
+ */
+void free_usb_bus ( struct usb_bus *bus ) {
+	struct usb_endpoint *ep;
+	struct usb_port *port;
+
+	/* Sanity checks */
+	assert ( list_empty ( &bus->devices ) );
+	assert ( list_empty ( &bus->hubs ) );
+	list_for_each_entry ( ep, &usb_halted, halted )
+		assert ( ep->usb->port->hub->bus != bus );
+	list_for_each_entry ( port, &usb_changed, changed )
+		assert ( port->hub->bus != bus );
+
+	/* Free root hub */
+	free_usb_hub ( bus->hub );
+
+	/* Free bus */
+	free ( bus );
+}
+
+/**
+ * Find USB bus by device location
+ *
+ * @v bus_type		Bus type
+ * @v location		Bus location
+ * @ret bus		USB bus, or NULL
+ */
+struct usb_bus * find_usb_bus_by_location ( unsigned int bus_type,
+					    unsigned int location ) {
+	struct usb_bus *bus;
+
+	for_each_usb_bus ( bus ) {
+		if ( ( bus->dev->desc.bus_type == bus_type ) &&
+		     ( bus->dev->desc.location == location ) )
+			return bus;
+	}
+
+	return NULL;
+}
+
+/******************************************************************************
+ *
+ * USB address assignment
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate device address
+ *
+ * @v bus		USB bus
+ * @ret address		Device address, or negative error
+ */
+int usb_alloc_address ( struct usb_bus *bus ) {
+	unsigned int address;
+
+	/* Find first free device address */
+	address = ffsll ( ~bus->addresses );
+	if ( ! address )
+		return -ENOENT;
+
+	/* Mark address as used */
+	bus->addresses |= ( 1ULL << ( address - 1 ) );
+
+	return address;
+}
+
+/**
+ * Free device address
+ *
+ * @v bus		USB bus
+ * @v address		Device address
+ */
+void usb_free_address ( struct usb_bus *bus, unsigned int address ) {
+
+	/* Sanity check */
+	assert ( address > 0 );
+	assert ( bus->addresses & ( 1ULL << ( address - 1 ) ) );
+
+	/* Mark address as free */
+	bus->addresses &= ~( 1ULL << ( address - 1 ) );
+}
+
+/******************************************************************************
+ *
+ * USB bus topology
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get USB route string
+ *
+ * @v usb		USB device
+ * @ret route		USB route string
+ */
+unsigned int usb_route_string ( struct usb_device *usb ) {
+	struct usb_device *parent;
+	unsigned int route;
+
+	/* Navigate up to root hub, constructing route string as we go */
+	for ( route = 0 ; ( parent = usb->port->hub->usb ) ; usb = parent ) {
+		route <<= 4;
+		route |= ( ( usb->port->address > 0xf ) ?
+			   0xf : usb->port->address );
+	}
+
+	return route;
+}
+
+/**
+ * Get USB depth
+ *
+ * @v usb		USB device
+ * @ret depth		Hub depth
+ */
+unsigned int usb_depth ( struct usb_device *usb ) {
+	struct usb_device *parent;
+	unsigned int depth;
+
+	/* Navigate up to root hub, constructing depth as we go */
+	for ( depth = 0 ; ( parent = usb->port->hub->usb ) ; usb = parent )
+		depth++;
+
+	return depth;
+}
+
+/**
+ * Get USB root hub port
+ *
+ * @v usb		USB device
+ * @ret port		Root hub port
+ */
+struct usb_port * usb_root_hub_port ( struct usb_device *usb ) {
+	struct usb_device *parent;
+
+	/* Navigate up to root hub */
+	while ( ( parent = usb->port->hub->usb ) )
+		usb = parent;
+
+	return usb->port;
+}
+
+/**
+ * Get USB transaction translator
+ *
+ * @v usb		USB device
+ * @ret port		Transaction translator port, or NULL
+ */
+struct usb_port * usb_transaction_translator ( struct usb_device *usb ) {
+	struct usb_device *parent;
+
+	/* Navigate up to root hub.  If we find a low-speed or
+	 * full-speed port with a higher-speed parent device, then
+	 * that port is the transaction translator.
+	 */
+	for ( ; ( parent = usb->port->hub->usb ) ; usb = parent ) {
+		if ( ( usb->port->speed <= USB_SPEED_FULL ) &&
+		     ( parent->port->speed > USB_SPEED_FULL ) )
+			return usb->port;
+	}
+
+	return NULL;
+}
+
+/* Drag in objects via register_usb_bus() */
+REQUIRING_SYMBOL ( register_usb_bus );
+
+/* Drag in USB configuration */
+REQUIRE_OBJECT ( config_usb );
+
+/* Drag in hub driver */
+REQUIRE_OBJECT ( usbhub );
diff --git a/roms/ipxe/src/drivers/infiniband/arbel.c b/roms/ipxe/src/drivers/infiniband/arbel.c
index 1a56ff9af..2a6c32dec 100644
--- a/roms/ipxe/src/drivers/infiniband/arbel.c
+++ b/roms/ipxe/src/drivers/infiniband/arbel.c
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/drivers/infiniband/arbel.h b/roms/ipxe/src/drivers/infiniband/arbel.h
index c0303f1bc..73394cd9a 100644
--- a/roms/ipxe/src/drivers/infiniband/arbel.h
+++ b/roms/ipxe/src/drivers/infiniband/arbel.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/roms/ipxe/src/drivers/infiniband/linda.c b/roms/ipxe/src/drivers/infiniband/linda.c
index 4afda1208..a6ae9f529 100644
--- a/roms/ipxe/src/drivers/infiniband/linda.c
+++ b/roms/ipxe/src/drivers/infiniband/linda.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/drivers/infiniband/linda.h b/roms/ipxe/src/drivers/infiniband/linda.h
index 72ce70868..46a920a17 100644
--- a/roms/ipxe/src/drivers/infiniband/linda.h
+++ b/roms/ipxe/src/drivers/infiniband/linda.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/drivers/infiniband/qib7322.c b/roms/ipxe/src/drivers/infiniband/qib7322.c
index 9979b346e..e22f2349a 100644
--- a/roms/ipxe/src/drivers/infiniband/qib7322.c
+++ b/roms/ipxe/src/drivers/infiniband/qib7322.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/drivers/infiniband/qib7322.h b/roms/ipxe/src/drivers/infiniband/qib7322.h
index 63abe221b..72797b240 100644
--- a/roms/ipxe/src/drivers/infiniband/qib7322.h
+++ b/roms/ipxe/src/drivers/infiniband/qib7322.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/drivers/net/amd8111e.h b/roms/ipxe/src/drivers/net/amd8111e.h
index 2000df158..8ecd159af 100644
--- a/roms/ipxe/src/drivers/net/amd8111e.h
+++ b/roms/ipxe/src/drivers/net/amd8111e.h
@@ -16,6 +16,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  * USA
 
 Module Name:
@@ -36,7 +40,7 @@ Revision History:
 	3.0.1
 */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef _AMD811E_H
 #define _AMD811E_H
diff --git a/roms/ipxe/src/drivers/net/ath/ath9k/ani.h b/roms/ipxe/src/drivers/net/ath/ath9k/ani.h
index dbd4d4d5b..ba87ba0fd 100644
--- a/roms/ipxe/src/drivers/net/ath/ath9k/ani.h
+++ b/roms/ipxe/src/drivers/net/ath/ath9k/ani.h
@@ -125,7 +125,7 @@ struct ar5416AniState {
 	u8 mrcCCKOff;
 	u8 spurImmunityLevel;
 	u8 firstepLevel;
-	u8 ofdmWeakSigDetectOff;
+	u8 ofdmWeakSigDetect;
 	u8 cckWeakSigThreshold;
 	u32 listenTime;
 	int32_t rssiThrLow;
diff --git a/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c b/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c
index ff7df497f..76ca79cba 100644
--- a/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c
+++ b/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c
@@ -177,7 +177,7 @@ static void ath9k_hw_ani_ofdm_err_trigger_old(struct ath_hw *ah)
 
 	rssi = BEACON_RSSI(ah);
 	if (rssi > aniState->rssiThrHigh) {
-		if (!aniState->ofdmWeakSigDetectOff) {
+		if (aniState->ofdmWeakSigDetect) {
 			if (ath9k_hw_ani_control(ah,
 					 ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 					 0)) {
@@ -192,7 +192,7 @@ static void ath9k_hw_ani_ofdm_err_trigger_old(struct ath_hw *ah)
 			return;
 		}
 	} else if (rssi > aniState->rssiThrLow) {
-		if (aniState->ofdmWeakSigDetectOff)
+		if (!aniState->ofdmWeakSigDetect)
 			ath9k_hw_ani_control(ah,
 				     ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 				     1);
@@ -202,7 +202,7 @@ static void ath9k_hw_ani_ofdm_err_trigger_old(struct ath_hw *ah)
 		return;
 	} else {
 		if ((ah->dev->channels + ah->dev->channel)->band == NET80211_BAND_2GHZ) {
-			if (!aniState->ofdmWeakSigDetectOff)
+			if (aniState->ofdmWeakSigDetect)
 				ath9k_hw_ani_control(ah,
 				     ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 				     0);
@@ -360,7 +360,7 @@ static void ath9k_hw_ani_lower_immunity_old(struct ath_hw *ah)
 	if (rssi > aniState->rssiThrHigh) {
 		/* XXX: Handle me */
 	} else if (rssi > aniState->rssiThrLow) {
-		if (aniState->ofdmWeakSigDetectOff) {
+		if (!aniState->ofdmWeakSigDetect) {
 			if (ath9k_hw_ani_control(ah,
 				 ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 				 1) == 1)
@@ -436,9 +436,9 @@ static void ath9k_ani_reset_old(struct ath_hw *ah)
 	if (aniState->spurImmunityLevel != 0)
 		ath9k_hw_ani_control(ah, ATH9K_ANI_SPUR_IMMUNITY_LEVEL,
 				     aniState->spurImmunityLevel);
-	if (aniState->ofdmWeakSigDetectOff)
+	if (!aniState->ofdmWeakSigDetect)
 		ath9k_hw_ani_control(ah, ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
-				     !aniState->ofdmWeakSigDetectOff);
+				     aniState->ofdmWeakSigDetect);
 	if (aniState->cckWeakSigThreshold)
 		ath9k_hw_ani_control(ah, ATH9K_ANI_CCK_WEAK_SIGNAL_THR,
 				     aniState->cckWeakSigThreshold);
@@ -709,8 +709,8 @@ void ath9k_hw_ani_init(struct ath_hw *ah)
 
 		ani->rssiThrHigh = ATH9K_ANI_RSSI_THR_HIGH;
 		ani->rssiThrLow = ATH9K_ANI_RSSI_THR_LOW;
-		ani->ofdmWeakSigDetectOff =
-			!ATH9K_ANI_USE_OFDM_WEAK_SIG;
+		ani->ofdmWeakSigDetect =
+			ATH9K_ANI_USE_OFDM_WEAK_SIG;
 		ani->cckNoiseImmunityLevel = ATH9K_ANI_CCK_DEF_LEVEL;
 	}
 
diff --git a/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c b/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c
index 60e87e9e2..2b6c133cb 100644
--- a/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c
+++ b/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c
@@ -1141,12 +1141,12 @@ static int ar5008_hw_ani_control_old(struct ath_hw *ah,
 			REG_CLR_BIT(ah, AR_PHY_SFCORR_LOW,
 				    AR_PHY_SFCORR_LOW_USE_SELF_CORR_LOW);
 
-		if (!on != aniState->ofdmWeakSigDetectOff) {
+		if (on != aniState->ofdmWeakSigDetect) {
 			if (on)
 				ah->stats.ast_ani_ofdmon++;
 			else
 				ah->stats.ast_ani_ofdmoff++;
-			aniState->ofdmWeakSigDetectOff = !on;
+			aniState->ofdmWeakSigDetect = on;
 		}
 		break;
 	}
@@ -1215,10 +1215,10 @@ static int ar5008_hw_ani_control_old(struct ath_hw *ah,
 
 	DBG2("ath9k: ANI parameters:\n");
 	DBG2(
-		"noiseImmunityLevel=%d, spurImmunityLevel=%d, ofdmWeakSigDetectOff=%d\n",
+		"noiseImmunityLevel=%d, spurImmunityLevel=%d, ofdmWeakSigDetect=%d\n",
 		aniState->noiseImmunityLevel,
 		aniState->spurImmunityLevel,
-		!aniState->ofdmWeakSigDetectOff);
+		aniState->ofdmWeakSigDetect);
 	DBG2(
 		"cckWeakSigThreshold=%d, firstepLevel=%d, listenTime=%d\n",
 		aniState->cckWeakSigThreshold,
@@ -1307,18 +1307,18 @@ static int ar5008_hw_ani_control_new(struct ath_hw *ah,
 			REG_CLR_BIT(ah, AR_PHY_SFCORR_LOW,
 				    AR_PHY_SFCORR_LOW_USE_SELF_CORR_LOW);
 
-		if (!on != aniState->ofdmWeakSigDetectOff) {
+		if (on != aniState->ofdmWeakSigDetect) {
 			DBG2("ath9k: "
 				"** ch %d: ofdm weak signal: %s=>%s\n",
 				chan->channel,
-				!aniState->ofdmWeakSigDetectOff ?
+				aniState->ofdmWeakSigDetect ?
 				"on" : "off",
 				on ? "on" : "off");
 			if (on)
 				ah->stats.ast_ani_ofdmon++;
 			else
 				ah->stats.ast_ani_ofdmoff++;
-			aniState->ofdmWeakSigDetectOff = !on;
+			aniState->ofdmWeakSigDetect = on;
 		}
 		break;
 	}
@@ -1467,7 +1467,7 @@ static int ar5008_hw_ani_control_new(struct ath_hw *ah,
 	DBG2("ath9k: "
 		"ANI parameters: SI=%d, ofdmWS=%s FS=%d MRCcck=%s listenTime=%d ofdmErrs=%d cckErrs=%d\n",
 		aniState->spurImmunityLevel,
-		!aniState->ofdmWeakSigDetectOff ? "on" : "off",
+		aniState->ofdmWeakSigDetect ? "on" : "off",
 		aniState->firstepLevel,
 		!aniState->mrcCCKOff ? "on" : "off",
 		aniState->listenTime,
@@ -1554,7 +1554,7 @@ static void ar5008_hw_ani_cache_ini_regs(struct ath_hw *ah)
 	/* these levels just got reset to defaults by the INI */
 	aniState->spurImmunityLevel = ATH9K_ANI_SPUR_IMMUNE_LVL_NEW;
 	aniState->firstepLevel = ATH9K_ANI_FIRSTEP_LVL_NEW;
-	aniState->ofdmWeakSigDetectOff = !ATH9K_ANI_USE_OFDM_WEAK_SIG;
+	aniState->ofdmWeakSigDetect = ATH9K_ANI_USE_OFDM_WEAK_SIG;
 	aniState->mrcCCKOff = 1; /* not available on pre AR9003 */
 }
 
diff --git a/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c b/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c
index 6103040ab..2244b775a 100644
--- a/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c
+++ b/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c
@@ -859,18 +859,18 @@ static int ar9003_hw_ani_control(struct ath_hw *ah,
 			REG_CLR_BIT(ah, AR_PHY_SFCORR_LOW,
 				    AR_PHY_SFCORR_LOW_USE_SELF_CORR_LOW);
 
-		if (!on != aniState->ofdmWeakSigDetectOff) {
+		if (on != aniState->ofdmWeakSigDetect) {
 			DBG2("ath9k: "
 				"** ch %d: ofdm weak signal: %s=>%s\n",
 				chan->channel,
-				!aniState->ofdmWeakSigDetectOff ?
+				aniState->ofdmWeakSigDetect ?
 				"on" : "off",
 				on ? "on" : "off");
 			if (on)
 				ah->stats.ast_ani_ofdmon++;
 			else
 				ah->stats.ast_ani_ofdmoff++;
-			aniState->ofdmWeakSigDetectOff = !on;
+			aniState->ofdmWeakSigDetect = on;
 		}
 		break;
 	}
@@ -1013,7 +1013,7 @@ static int ar9003_hw_ani_control(struct ath_hw *ah,
 			      AR_PHY_MRC_CCK_ENABLE, is_on);
 		REG_RMW_FIELD(ah, AR_PHY_MRC_CCK_CTRL,
 			      AR_PHY_MRC_CCK_MUX_REG, is_on);
-		if (!is_on != aniState->mrcCCKOff) {
+		if (!(is_on != aniState->mrcCCKOff)) {
 			DBG2("ath9k: "
 				"** ch %d: MRC CCK: %s=>%s\n",
 				chan->channel,
@@ -1037,7 +1037,7 @@ static int ar9003_hw_ani_control(struct ath_hw *ah,
 	DBG2("ath9k: "
 		"ANI parameters: SI=%d, ofdmWS=%s FS=%d MRCcck=%s listenTime=%d ofdmErrs=%d cckErrs=%d\n",
 		aniState->spurImmunityLevel,
-		!aniState->ofdmWeakSigDetectOff ? "on" : "off",
+		aniState->ofdmWeakSigDetect ? "on" : "off",
 		aniState->firstepLevel,
 		!aniState->mrcCCKOff ? "on" : "off",
 		aniState->listenTime,
@@ -1137,7 +1137,7 @@ static void ar9003_hw_ani_cache_ini_regs(struct ath_hw *ah)
 	/* these levels just got reset to defaults by the INI */
 	aniState->spurImmunityLevel = ATH9K_ANI_SPUR_IMMUNE_LVL_NEW;
 	aniState->firstepLevel = ATH9K_ANI_FIRSTEP_LVL_NEW;
-	aniState->ofdmWeakSigDetectOff = !ATH9K_ANI_USE_OFDM_WEAK_SIG;
+	aniState->ofdmWeakSigDetect = ATH9K_ANI_USE_OFDM_WEAK_SIG;
 	aniState->mrcCCKOff = !ATH9K_ANI_ENABLE_MRC_CCK;
 }
 
diff --git a/roms/ipxe/src/drivers/net/atl1e.c b/roms/ipxe/src/drivers/net/atl1e.c
index 1ff0f0d10..d010d8c4a 100644
--- a/roms/ipxe/src/drivers/net/atl1e.c
+++ b/roms/ipxe/src/drivers/net/atl1e.c
@@ -224,7 +224,7 @@ static int atl1e_sw_init(struct atl1e_adapter *adapter)
 	adapter->link_duplex = FULL_DUPLEX;
 
 	/* PCI config space info */
-	pci_read_config_byte(pdev, PCI_REVISION_ID, &rev_id);
+	pci_read_config_byte(pdev, PCI_REVISION, &rev_id);
 
 	phy_status_data = AT_READ_REG(hw, REG_PHY_STATUS);
 	/* nic type */
diff --git a/roms/ipxe/src/drivers/net/davicom.c b/roms/ipxe/src/drivers/net/davicom.c
index a4870a729..9d3d8b915 100644
--- a/roms/ipxe/src/drivers/net/davicom.c
+++ b/roms/ipxe/src/drivers/net/davicom.c
@@ -340,6 +340,7 @@ static void davicom_media_chk(struct nic * nic __unused)
   csr6 = 0x00200000;	/* SF */
   outl(csr6, ioaddr + CSR6);
 
+#define PCI_VENDOR_ID_DAVICOM		0x1282
 #define	PCI_DEVICE_ID_DM9009		0x9009
   if (vendor == PCI_VENDOR_ID_DAVICOM && dev_id == PCI_DEVICE_ID_DM9009) {
     /* Set to 10BaseT mode for DM9009 */
diff --git a/roms/ipxe/src/drivers/net/dm96xx.c b/roms/ipxe/src/drivers/net/dm96xx.c
new file mode 100644
index 000000000..58d8dd964
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/dm96xx.c
@@ -0,0 +1,671 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include "dm96xx.h"
+
+/** @file
+ *
+ * Davicom DM96xx USB Ethernet driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Register operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Reset device
+ *
+ * @v dm96xx		DM96xx device
+ * @ret rc		Return status code
+ */
+static int dm96xx_reset ( struct dm96xx_device *dm96xx ) {
+	int ncr;
+	int rc;
+
+	/* Reset device */
+	if ( ( rc = dm96xx_write_register ( dm96xx, DM96XX_NCR,
+					    DM96XX_NCR_RST ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not reset: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Wait for reset to complete */
+	udelay ( DM96XX_RESET_DELAY_US );
+
+	/* Check that reset has completed */
+	ncr = dm96xx_read_register ( dm96xx, DM96XX_NCR );
+	if ( ncr < 0 ) {
+		rc = ncr;
+		DBGC ( dm96xx, "DM96XX %p failed to reset: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+	if ( ncr & DM96XX_NCR_RST ) {
+		DBGC ( dm96xx, "DM96XX %p failed to reset (NCR=%#02x)\n",
+		       dm96xx, ncr );
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/**
+ * Read MAC address
+ *
+ * @v dm96xx		DM96xx device
+ * @v mac		MAC address to fill in
+ * @ret rc		Return status code
+ */
+static int dm96xx_read_mac ( struct dm96xx_device *dm96xx, uint8_t *mac ) {
+	int rc;
+
+	/* Read MAC address */
+	if ( ( rc = dm96xx_read_registers ( dm96xx, DM96XX_PAR, mac,
+					    ETH_ALEN ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not read MAC address: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Write MAC address
+ *
+ * @v dm96xx		DM96xx device
+ * @v mac		MAC address
+ * @ret rc		Return status code
+ */
+static int dm96xx_write_mac ( struct dm96xx_device *dm96xx, uint8_t *mac ) {
+	int rc;
+
+	/* Write MAC address */
+	if ( ( rc = dm96xx_write_registers ( dm96xx, DM96XX_PAR, mac,
+					     ETH_ALEN ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not write MAC address: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Update link status based on network status register
+ *
+ * @v dm96xx		DM96xx device
+ * @v nsr		Network status register
+ */
+static void dm96xx_link_nsr ( struct dm96xx_device *dm96xx, unsigned int nsr ) {
+	struct net_device *netdev = dm96xx->netdev;
+
+	if ( nsr & DM96XX_NSR_LINKST ) {
+		if ( ! netdev_link_ok ( netdev ) )
+			netdev_link_up ( netdev );
+	} else {
+		if ( netdev_link_ok ( netdev ) )
+			netdev_link_down ( netdev );
+	}
+}
+
+/**
+ * Get link status
+ *
+ * @v dm96xx		DM96xx device
+ * @ret rc		Return status code
+ */
+static int dm96xx_check_link ( struct dm96xx_device *dm96xx ) {
+	int nsr;
+	int rc;
+
+	/* Read network status register */
+	nsr = dm96xx_read_register ( dm96xx, DM96XX_NSR );
+	if ( nsr < 0 ) {
+		rc = nsr;
+		DBGC ( dm96xx, "DM96XX %p could not read network status: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Update link status */
+	dm96xx_link_nsr ( dm96xx, nsr );
+
+	return 0;
+}
+
+/**
+ * Set DM9601-compatible RX header mode
+ *
+ * @v dm96xx		DM96xx device
+ * @ret rc		Return status code
+ */
+static int dm96xx_rx_mode ( struct dm96xx_device *dm96xx ) {
+	int chipr;
+	int mode_ctl;
+	int rc;
+
+	/* Get chip revision */
+	chipr = dm96xx_read_register ( dm96xx, DM96XX_CHIPR );
+	if ( chipr < 0 ) {
+		rc = chipr;
+		DBGC ( dm96xx, "DM96XX %p could not read chip revision: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Do nothing if device is a DM9601 anyway */
+	if ( chipr == DM96XX_CHIPR_9601 )
+		return 0;
+
+	/* Read current mode control */
+	mode_ctl = dm96xx_read_register ( dm96xx, DM96XX_MODE_CTL );
+	if ( mode_ctl < 0 ) {
+		rc = mode_ctl;
+		DBGC ( dm96xx, "DM96XX %p could not read mode control: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Write mode control */
+	mode_ctl &= ~DM96XX_MODE_CTL_MODE;
+	if ( ( rc = dm96xx_write_register ( dm96xx, DM96XX_MODE_CTL,
+					    mode_ctl ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not write mode control: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void dm96xx_intr_complete ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf, int rc ) {
+	struct dm96xx_device *dm96xx = container_of ( ep, struct dm96xx_device,
+						      usbnet.intr );
+	struct net_device *netdev = dm96xx->netdev;
+	struct dm96xx_interrupt *intr;
+	size_t len = iob_len ( iobuf );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto done;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p interrupt failed: %s\n",
+		       dm96xx, strerror ( rc ) );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, rc );
+		goto done;
+	}
+
+	/* Extract message header */
+	if ( len < sizeof ( *intr ) ) {
+		DBGC ( dm96xx, "DM96XX %p underlength interrupt:\n", dm96xx );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, -EINVAL );
+		goto done;
+	}
+	intr = iobuf->data;
+
+	/* Update link status */
+	dm96xx_link_nsr ( dm96xx, intr->nsr );
+
+ done:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations dm96xx_intr_operations = {
+	.complete = dm96xx_intr_complete,
+};
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void dm96xx_in_complete ( struct usb_endpoint *ep,
+				 struct io_buffer *iobuf, int rc ) {
+	struct dm96xx_device *dm96xx = container_of ( ep, struct dm96xx_device,
+						      usbnet.in );
+	struct net_device *netdev = dm96xx->netdev;
+	struct dm96xx_rx_header *header;
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open ) {
+		free_iob ( iobuf );
+		return;
+	}
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p bulk IN failed: %s\n",
+		       dm96xx, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < ( sizeof ( *header ) + 4 /* CRC */ ) ) {
+		DBGC ( dm96xx, "DM96XX %p underlength bulk IN\n", dm96xx );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto err;
+	}
+
+	/* Strip header and CRC */
+	header = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *header ) );
+	iob_unput ( iobuf, 4 /* CRC */ );
+
+	/* Check status */
+	if ( header->rsr & ~DM96XX_RSR_MF ) {
+		DBGC ( dm96xx, "DM96XX %p receive error %02x:\n",
+		       dm96xx, header->rsr );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EIO;
+		goto err;
+	}
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+	return;
+
+ err:
+	/* Hand off to network stack */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations dm96xx_in_operations = {
+	.complete = dm96xx_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v dm96xx		DM96xx device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int dm96xx_out_transmit ( struct dm96xx_device *dm96xx,
+				 struct io_buffer *iobuf ) {
+	struct dm96xx_tx_header *header;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Prepend header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, sizeof ( *header ) ) ) != 0 )
+		return rc;
+	header = iob_push ( iobuf, sizeof ( *header ) );
+	header->len = cpu_to_le16 ( len );
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &dm96xx->usbnet.out, iobuf, 0 ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void dm96xx_out_complete ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int rc ) {
+	struct dm96xx_device *dm96xx = container_of ( ep, struct dm96xx_device,
+						      usbnet.out );
+	struct net_device *netdev = dm96xx->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations dm96xx_out_operations = {
+	.complete = dm96xx_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int dm96xx_open ( struct net_device *netdev ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+	unsigned int rcr;
+	int rc;
+
+	/* Set DM9601-compatible RX header mode */
+	if ( ( rc = dm96xx_rx_mode ( dm96xx ) ) != 0 )
+		goto err_rx_mode;
+
+	/* Write MAC address */
+	if ( ( rc = dm96xx_write_mac ( dm96xx, netdev->ll_addr ) ) != 0 )
+		goto err_write_mac;
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &dm96xx->usbnet ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not open: %s\n",
+		       dm96xx, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Set receive filters */
+	rcr = ( DM96XX_RCR_ALL | DM96XX_RCR_RUNT | DM96XX_RCR_PRMSC |
+		DM96XX_RCR_RXEN );
+	if ( ( rc = dm96xx_write_register ( dm96xx, DM96XX_RCR, rcr ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not write receive filters: "
+		       "%s\n", dm96xx, strerror ( rc ) );
+		goto err_write_rcr;
+	}
+
+	/* Update link status */
+	if ( ( rc = dm96xx_check_link ( dm96xx ) ) != 0 )
+		goto err_check_link;
+
+	return 0;
+
+ err_check_link:
+ err_write_rcr:
+	usbnet_close ( &dm96xx->usbnet );
+ err_open:
+ err_write_mac:
+ err_rx_mode:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void dm96xx_close ( struct net_device *netdev ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &dm96xx->usbnet );
+
+	/* Reset device */
+	dm96xx_reset ( dm96xx );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int dm96xx_transmit ( struct net_device *netdev,
+			     struct io_buffer *iobuf ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = dm96xx_out_transmit ( dm96xx, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void dm96xx_poll ( struct net_device *netdev ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( dm96xx->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &dm96xx->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+}
+
+/** DM96xx network device operations */
+static struct net_device_operations dm96xx_operations = {
+	.open		= dm96xx_open,
+	.close		= dm96xx_close,
+	.transmit	= dm96xx_transmit,
+	.poll		= dm96xx_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int dm96xx_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct dm96xx_device *dm96xx;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *dm96xx ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &dm96xx_operations );
+	netdev->dev = &func->dev;
+	dm96xx = netdev->priv;
+	memset ( dm96xx, 0, sizeof ( *dm96xx ) );
+	dm96xx->usb = usb;
+	dm96xx->bus = usb->port->hub->bus;
+	dm96xx->netdev = netdev;
+	usbnet_init ( &dm96xx->usbnet, func, &dm96xx_intr_operations,
+		      &dm96xx_in_operations, &dm96xx_out_operations );
+	usb_refill_init ( &dm96xx->usbnet.intr, 0, DM96XX_INTR_MAX_FILL );
+	usb_refill_init ( &dm96xx->usbnet.in, DM96XX_IN_MTU,
+			  DM96XX_IN_MAX_FILL );
+	DBGC ( dm96xx, "DM96XX %p on %s\n", dm96xx, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &dm96xx->usbnet, config ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not describe: %s\n",
+		       dm96xx, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Reset device */
+	if ( ( rc = dm96xx_reset ( dm96xx ) ) != 0 )
+		goto err_reset;
+
+	/* Read MAC address */
+	if ( ( rc = dm96xx_read_mac ( dm96xx, netdev->hw_addr ) ) != 0 )
+		goto err_read_mac;
+
+	/* Get initial link status */
+	if ( ( rc = dm96xx_check_link ( dm96xx ) ) != 0 )
+		goto err_check_link;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, netdev );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_check_link:
+ err_read_mac:
+ err_reset:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void dm96xx_remove ( struct usb_function *func ) {
+	struct net_device *netdev = usb_func_get_drvdata ( func );
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** DM96xx device IDs */
+static struct usb_device_id dm96xx_ids[] = {
+	{
+		.name = "dm9601-corega",
+		.vendor = 0x07aa,
+		.product = 0x9601,
+	},
+	{
+		.name = "dm9601",
+		.vendor = 0x0a46,
+		.product = 0x9601,
+	},
+	{
+		.name = "zt6688",
+		.vendor = 0x0a46,
+		.product = 0x6688,
+	},
+	{
+		.name = "st268",
+		.vendor = 0x0a46,
+		.product = 0x0268,
+	},
+	{
+		.name = "adm8515",
+		.vendor = 0x0a46,
+		.product = 0x8515,
+	},
+	{
+		.name = "dm9601-hirose",
+		.vendor = 0x0a47,
+		.product = 0x9601,
+	},
+	{
+		.name = "dm9601-8101",
+		.vendor = 0x0fe6,
+		.product = 0x8101,
+	},
+	{
+		.name = "dm9601-9700",
+		.vendor = 0x0fe6,
+		.product = 0x9700,
+	},
+	{
+		.name = "dm9000e",
+		.vendor = 0x0a46,
+		.product = 0x9000,
+	},
+	{
+		.name = "dm9620",
+		.vendor = 0x0a46,
+		.product = 0x9620,
+	},
+	{
+		.name = "dm9621A",
+		.vendor = 0x0a46,
+		.product = 0x9621,
+	},
+	{
+		.name = "dm9622",
+		.vendor = 0x0a46,
+		.product = 0x9622,
+	},
+	{
+		.name = "dm962Oa",
+		.vendor = 0x0a46,
+		.product = 0x0269,
+	},
+	{
+		.name = "dm9621a",
+		.vendor = 0x0a46,
+		.product = 0x1269,
+	},
+};
+
+/** Davicom DM96xx driver */
+struct usb_driver dm96xx_driver __usb_driver = {
+	.ids = dm96xx_ids,
+	.id_count = ( sizeof ( dm96xx_ids ) / sizeof ( dm96xx_ids[0] ) ),
+	.probe = dm96xx_probe,
+	.remove = dm96xx_remove,
+};
diff --git a/roms/ipxe/src/drivers/net/dm96xx.h b/roms/ipxe/src/drivers/net/dm96xx.h
new file mode 100644
index 000000000..43a1a4e30
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/dm96xx.h
@@ -0,0 +1,194 @@
+#ifndef _DM96XX_H
+#define _DM96XX_H
+
+/** @file
+ *
+ * Davicom DM96xx USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/if_ether.h>
+
+/** Read register(s) */
+#define DM96XX_READ_REGISTER					\
+	( USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0x00 ) )
+
+/** Write register(s) */
+#define DM96XX_WRITE_REGISTER					\
+	( USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0x01 ) )
+
+/** Write single register */
+#define DM96XX_WRITE1_REGISTER					\
+	( USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0x03 ) )
+
+/** Network control register */
+#define DM96XX_NCR 0x00
+#define DM96XX_NCR_RST		0x01	/**< Software reset */
+
+/** Network status register */
+#define DM96XX_NSR 0x01
+#define DM96XX_NSR_LINKST	0x40	/**< Link status */
+
+/** Receive control register */
+#define DM96XX_RCR 0x05
+#define DM96XX_RCR_ALL		0x08	/**< Pass all multicast */
+#define DM96XX_RCR_RUNT		0x04	/**< Pass runt packet */
+#define DM96XX_RCR_PRMSC	0x02	/**< Promiscuous mode */
+#define DM96XX_RCR_RXEN		0x01	/**< RX enable */
+
+/** Receive status register */
+#define DM96XX_RSR 0x06
+#define DM96XX_RSR_MF		0x40	/**< Multicast frame */
+
+/** PHY address registers */
+#define DM96XX_PAR 0x10
+
+/** Chip revision register */
+#define DM96XX_CHIPR 0x2c
+#define DM96XX_CHIPR_9601	0x00	/**< DM9601 */
+#define DM96XX_CHIPR_9620	0x01	/**< DM9620 */
+
+/** RX header control/status register (DM9620+ only) */
+#define DM96XX_MODE_CTL 0x91
+#define DM96XX_MODE_CTL_MODE	0x80	/**< 4-byte header mode */
+
+/** DM96xx interrupt data */
+struct dm96xx_interrupt {
+	/** Network status register */
+	uint8_t nsr;
+	/** Transmit status registers */
+	uint8_t tsr[2];
+	/** Receive status register */
+	uint8_t rsr;
+	/** Receive overflow counter register */
+	uint8_t rocr;
+	/** Receive packet counter */
+	uint8_t rxc;
+	/** Transmit packet counter */
+	uint8_t txc;
+	/** General purpose register */
+	uint8_t gpr;
+} __attribute__ (( packed ));
+
+/** DM96xx receive header */
+struct dm96xx_rx_header {
+	/** Packet status */
+	uint8_t rsr;
+	/** Packet length (excluding this header, including CRC) */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** DM96xx transmit header */
+struct dm96xx_tx_header {
+	/** Packet length (excluding this header) */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** A DM96xx network device */
+struct dm96xx_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+};
+
+/**
+ * Read registers
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @v data		Data buffer
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_read_registers ( struct dm96xx_device *dm96xx, unsigned int offset,
+			void *data, size_t len ) {
+
+	return usb_control ( dm96xx->usb, DM96XX_READ_REGISTER, 0, offset,
+			     data, len );
+}
+
+/**
+ * Read register
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @ret value		Register value, or negative error
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_read_register ( struct dm96xx_device *dm96xx, unsigned int offset ) {
+	uint8_t value;
+	int rc;
+
+	if ( ( rc = dm96xx_read_registers ( dm96xx, offset, &value,
+					    sizeof ( value ) ) ) != 0 )
+		return rc;
+	return value;
+}
+
+/**
+ * Write registers
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @v data		Data buffer
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_write_registers ( struct dm96xx_device *dm96xx, unsigned int offset,
+			 void *data, size_t len ) {
+
+	return usb_control ( dm96xx->usb, DM96XX_WRITE_REGISTER, 0, offset,
+			     data, len );
+}
+
+/**
+ * Write register
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @v value		Register value
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_write_register ( struct dm96xx_device *dm96xx, unsigned int offset,
+			uint8_t value ) {
+
+	return usb_control ( dm96xx->usb, DM96XX_WRITE1_REGISTER, value,
+			     offset, NULL, 0 );
+}
+
+/** Reset delay (in microseconds) */
+#define DM96XX_RESET_DELAY_US 10
+
+/** Interrupt maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define DM96XX_INTR_MAX_FILL 2
+
+/** Bulk IN maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define DM96XX_IN_MAX_FILL 8
+
+/** Bulk IN buffer size */
+#define DM96XX_IN_MTU					\
+	( 4 /* DM96xx header */ + ETH_FRAME_LEN +	\
+	  4 /* possible VLAN header */ + 4 /* CRC */ )
+
+#endif /* _DM96XX_H */
diff --git a/roms/ipxe/src/drivers/net/dmfe.c b/roms/ipxe/src/drivers/net/dmfe.c
index aae40fce7..2ea0d2b2b 100644
--- a/roms/ipxe/src/drivers/net/dmfe.c
+++ b/roms/ipxe/src/drivers/net/dmfe.c
@@ -462,7 +462,7 @@ static int dmfe_probe ( struct nic *nic, struct pci_device *pci ) {
 	       pci->id->name, pci->vendor, pci->device);
 
 	/* Read Chip revision */
-	pci_read_config_dword(pci, PCI_REVISION_ID, &dev_rev);
+	pci_read_config_dword(pci, PCI_REVISION, &dev_rev);
 	dprintf(("Revision %lX\n", dev_rev));
 
 	/* point to private storage */
diff --git a/roms/ipxe/src/drivers/net/ecm.c b/roms/ipxe/src/drivers/net/ecm.c
new file mode 100644
index 000000000..8c84ea9e9
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/ecm.c
@@ -0,0 +1,520 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <errno.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/base16.h>
+#include <ipxe/profile.h>
+#include <ipxe/usb.h>
+#include "ecm.h"
+
+/** @file
+ *
+ * CDC-ECM USB Ethernet driver
+ *
+ */
+
+/** Interrupt completion profiler */
+static struct profiler ecm_intr_profiler __profiler =
+	{ .name = "ecm.intr" };
+
+/** Bulk IN completion profiler */
+static struct profiler ecm_in_profiler __profiler =
+	{ .name = "ecm.in" };
+
+/** Bulk OUT profiler */
+static struct profiler ecm_out_profiler __profiler =
+	{ .name = "ecm.out" };
+
+/******************************************************************************
+ *
+ * Ethernet functional descriptor
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Locate Ethernet functional descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @ret desc		Descriptor, or NULL if not found
+ */
+struct ecm_ethernet_descriptor *
+ecm_ethernet_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface ) {
+	struct ecm_ethernet_descriptor *desc;
+
+	for_each_interface_descriptor ( desc, config, interface ) {
+		if ( ( desc->header.type == USB_CS_INTERFACE_DESCRIPTOR ) &&
+		     ( desc->subtype == CDC_SUBTYPE_ETHERNET ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Get hardware MAC address
+ *
+ * @v usb		USB device
+ * @v desc		Ethernet functional descriptor
+ * @v hw_addr		Hardware address to fill in
+ * @ret rc		Return status code
+ */
+int ecm_fetch_mac ( struct usb_device *usb,
+		    struct ecm_ethernet_descriptor *desc, uint8_t *hw_addr ) {
+	char buf[ base16_encoded_len ( ETH_ALEN ) + 1 /* NUL */ ];
+	int len;
+	int rc;
+
+	/* Fetch MAC address string */
+	len = usb_get_string_descriptor ( usb, desc->mac, 0, buf,
+					  sizeof ( buf ) );
+	if ( len < 0 ) {
+		rc = len;
+		return rc;
+	}
+
+	/* Sanity check */
+	if ( len != ( ( int ) ( sizeof ( buf ) - 1 /* NUL */ ) ) )
+		return -EINVAL;
+
+	/* Decode MAC address */
+	len = base16_decode ( buf, hw_addr, ETH_ALEN );
+	if ( len < 0 ) {
+		rc = len;
+		return rc;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * CDC-ECM communications interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ecm_intr_complete ( struct usb_endpoint *ep,
+				struct io_buffer *iobuf, int rc ) {
+	struct ecm_device *ecm = container_of ( ep, struct ecm_device,
+						usbnet.intr );
+	struct net_device *netdev = ecm->netdev;
+	struct usb_setup_packet *message;
+	size_t len = iob_len ( iobuf );
+
+	/* Profile completions */
+	profile_start ( &ecm_intr_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Drop packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( ecm, "ECM %p interrupt failed: %s\n",
+		       ecm, strerror ( rc ) );
+		DBGC_HDA ( ecm, 0, iobuf->data, iob_len ( iobuf ) );
+		goto error;
+	}
+
+	/* Extract message header */
+	if ( len < sizeof ( *message ) ) {
+		DBGC ( ecm, "ECM %p underlength interrupt:\n", ecm );
+		DBGC_HDA ( ecm, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto error;
+	}
+	message = iobuf->data;
+
+	/* Parse message header */
+	switch ( message->request ) {
+
+	case cpu_to_le16 ( CDC_NETWORK_CONNECTION ) :
+		if ( message->value && ! netdev_link_ok ( netdev ) ) {
+			DBGC ( ecm, "ECM %p link up\n", ecm );
+			netdev_link_up ( netdev );
+		} else if ( netdev_link_ok ( netdev ) && ! message->value ) {
+			DBGC ( ecm, "ECM %p link down\n", ecm );
+			netdev_link_down ( netdev );
+		}
+		break;
+
+	case cpu_to_le16 ( CDC_CONNECTION_SPEED_CHANGE ) :
+		/* Ignore */
+		break;
+
+	default:
+		DBGC ( ecm, "ECM %p unrecognised interrupt:\n", ecm );
+		DBGC_HDA ( ecm, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -ENOTSUP;
+		goto error;
+	}
+
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+	profile_stop ( &ecm_intr_profiler );
+
+	return;
+
+ error:
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+ ignore:
+	free_iob ( iobuf );
+	return;
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations ecm_intr_operations = {
+	.complete = ecm_intr_complete,
+};
+
+/******************************************************************************
+ *
+ * CDC-ECM data interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ecm_in_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			      int rc ) {
+	struct ecm_device *ecm = container_of ( ep, struct ecm_device,
+						usbnet.in );
+	struct net_device *netdev = ecm->netdev;
+
+	/* Profile receive completions */
+	profile_start ( &ecm_in_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( ecm, "ECM %p bulk IN failed: %s\n",
+		       ecm, strerror ( rc ) );
+		goto error;
+	}
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+
+	profile_stop ( &ecm_in_profiler );
+	return;
+
+ error:
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+ ignore:
+	free_iob ( iobuf );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations ecm_in_operations = {
+	.complete = ecm_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v ecm		CDC-ECM device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ecm_out_transmit ( struct ecm_device *ecm,
+			      struct io_buffer *iobuf ) {
+	int rc;
+
+	/* Profile transmissions */
+	profile_start ( &ecm_out_profiler );
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &ecm->usbnet.out, iobuf, 1 ) ) != 0 )
+		return rc;
+
+	profile_stop ( &ecm_out_profiler );
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ecm_out_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			       int rc ) {
+	struct ecm_device *ecm = container_of ( ep, struct ecm_device,
+						usbnet.out );
+	struct net_device *netdev = ecm->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations ecm_out_operations = {
+	.complete = ecm_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int ecm_open ( struct net_device *netdev ) {
+	struct ecm_device *ecm = netdev->priv;
+	struct usb_device *usb = ecm->usb;
+	unsigned int filter;
+	int rc;
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &ecm->usbnet ) ) != 0 ) {
+		DBGC ( ecm, "ECM %p could not open: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Set packet filter */
+	filter = ( ECM_PACKET_TYPE_PROMISCUOUS |
+		   ECM_PACKET_TYPE_ALL_MULTICAST |
+		   ECM_PACKET_TYPE_DIRECTED |
+		   ECM_PACKET_TYPE_BROADCAST );
+	if ( ( rc = usb_control ( usb, ECM_SET_ETHERNET_PACKET_FILTER,
+				  filter, ecm->usbnet.comms, NULL, 0 ) ) != 0 ){
+		DBGC ( ecm, "ECM %p could not set packet filter: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_set_filter;
+	}
+
+	return 0;
+
+ err_set_filter:
+	usbnet_close ( &ecm->usbnet );
+ err_open:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void ecm_close ( struct net_device *netdev ) {
+	struct ecm_device *ecm = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &ecm->usbnet );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ecm_transmit ( struct net_device *netdev,
+			  struct io_buffer *iobuf ) {
+	struct ecm_device *ecm = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = ecm_out_transmit ( ecm, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void ecm_poll ( struct net_device *netdev ) {
+	struct ecm_device *ecm = netdev->priv;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( ecm->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &ecm->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+}
+
+/** CDC-ECM network device operations */
+static struct net_device_operations ecm_operations = {
+	.open		= ecm_open,
+	.close		= ecm_close,
+	.transmit	= ecm_transmit,
+	.poll		= ecm_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int ecm_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct ecm_device *ecm;
+	struct usb_interface_descriptor *comms;
+	struct ecm_ethernet_descriptor *ethernet;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *ecm ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &ecm_operations );
+	netdev->dev = &func->dev;
+	ecm = netdev->priv;
+	memset ( ecm, 0, sizeof ( *ecm ) );
+	ecm->usb = usb;
+	ecm->bus = usb->port->hub->bus;
+	ecm->netdev = netdev;
+	usbnet_init ( &ecm->usbnet, func, &ecm_intr_operations,
+		      &ecm_in_operations, &ecm_out_operations );
+	usb_refill_init ( &ecm->usbnet.intr, 0, ECM_INTR_MAX_FILL );
+	usb_refill_init ( &ecm->usbnet.in, ECM_IN_MTU, ECM_IN_MAX_FILL );
+	DBGC ( ecm, "ECM %p on %s\n", ecm, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &ecm->usbnet, config ) ) != 0 ) {
+		DBGC ( ecm, "ECM %p could not describe: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Locate Ethernet descriptor */
+	comms = usb_interface_descriptor ( config, ecm->usbnet.comms, 0 );
+	assert ( comms != NULL );
+	ethernet = ecm_ethernet_descriptor ( config, comms );
+	if ( ! ethernet ) {
+		DBGC ( ecm, "ECM %p has no Ethernet descriptor\n", ecm );
+		rc = -EINVAL;
+		goto err_ethernet;
+	}
+
+	/* Fetch MAC address */
+	if ( ( rc = ecm_fetch_mac ( usb, ethernet, netdev->hw_addr ) ) != 0 ) {
+		DBGC ( ecm, "ECM %p could not fetch MAC address: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_fetch_mac;
+	}
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, ecm );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_fetch_mac:
+ err_ethernet:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void ecm_remove ( struct usb_function *func ) {
+	struct ecm_device *ecm = usb_func_get_drvdata ( func );
+	struct net_device *netdev = ecm->netdev;
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** CDC-ECM device IDs */
+static struct usb_device_id ecm_ids[] = {
+	{
+		.name = "cdc-ecm",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_CDC,
+			.subclass = USB_SUBCLASS_CDC_ECM,
+			.protocol = 0,
+		},
+	},
+};
+
+/** CDC-ECM driver */
+struct usb_driver ecm_driver __usb_driver = {
+	.ids = ecm_ids,
+	.id_count = ( sizeof ( ecm_ids ) / sizeof ( ecm_ids[0] ) ),
+	.probe = ecm_probe,
+	.remove = ecm_remove,
+};
diff --git a/roms/ipxe/src/drivers/net/ecm.h b/roms/ipxe/src/drivers/net/ecm.h
new file mode 100644
index 000000000..83d324bdc
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/ecm.h
@@ -0,0 +1,93 @@
+#ifndef _ECM_H
+#define _ECM_H
+
+/** @file
+ *
+ * CDC-ECM USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/cdc.h>
+
+/** CDC-ECM subclass */
+#define USB_SUBCLASS_CDC_ECM 0x06
+
+/** Set Ethernet packet filter */
+#define ECM_SET_ETHERNET_PACKET_FILTER					\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x43 ) )
+
+/** Ethernet packet types */
+enum ecm_ethernet_packet_filter {
+	/** Promiscuous mode */
+	ECM_PACKET_TYPE_PROMISCUOUS = 0x0001,
+	/** All multicast packets */
+	ECM_PACKET_TYPE_ALL_MULTICAST = 0x0002,
+	/** Unicast packets */
+	ECM_PACKET_TYPE_DIRECTED = 0x0004,
+	/** Broadcast packets */
+	ECM_PACKET_TYPE_BROADCAST = 0x0008,
+	/** Specified multicast packets */
+	ECM_PACKET_TYPE_MULTICAST = 0x0010,
+};
+
+/** An Ethernet Functional Descriptor */
+struct ecm_ethernet_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Descriptor subtype */
+	uint8_t subtype;
+	/** MAC address string */
+	uint8_t mac;
+	/** Ethernet statistics bitmap */
+	uint32_t statistics;
+	/** Maximum segment size */
+	uint16_t mtu;
+	/** Multicast filter configuration */
+	uint16_t mcast;
+	/** Number of wake-on-LAN filters */
+	uint8_t wol;
+} __attribute__ (( packed ));
+
+/** A CDC-ECM network device */
+struct ecm_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+};
+
+/** Interrupt maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define ECM_INTR_MAX_FILL 2
+
+/** Bulk IN maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define ECM_IN_MAX_FILL 8
+
+/** Bulk IN buffer size
+ *
+ * This is a policy decision.
+ */
+#define ECM_IN_MTU ( ETH_FRAME_LEN + 4 /* possible VLAN header */ )
+
+extern struct ecm_ethernet_descriptor *
+ecm_ethernet_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface );
+extern int ecm_fetch_mac ( struct usb_device *usb,
+			   struct ecm_ethernet_descriptor *desc,
+			   uint8_t *hw_addr );
+
+#endif /* _ECM_H */
diff --git a/roms/ipxe/src/drivers/net/eepro.c b/roms/ipxe/src/drivers/net/eepro.c
index 909482bcc..97b4c4061 100644
--- a/roms/ipxe/src/drivers/net/eepro.c
+++ b/roms/ipxe/src/drivers/net/eepro.c
@@ -27,8 +27,18 @@ has 34 pins, the top row of 2 are not used.
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -591,9 +601,9 @@ static int eepro_probe ( struct nic *nic, struct isa_device *isa ) {
 		l_eepro = 0;
 		name = "Intel 82595-based LAN card";
 	}
-	station_addr.saddr[0] = swap16(station_addr.saddr[0]);
-	station_addr.saddr[1] = swap16(station_addr.saddr[1]);
-	station_addr.saddr[2] = swap16(station_addr.saddr[2]);
+	station_addr.saddr[0] = bswap_16(station_addr.saddr[0]);
+	station_addr.saddr[1] = bswap_16(station_addr.saddr[1]);
+	station_addr.saddr[2] = bswap_16(station_addr.saddr[2]);
 	for (i = 0; i < ETH_ALEN; i++) {
 		nic->node_addr[i] = station_addr.caddr[i];
 	}
diff --git a/roms/ipxe/src/drivers/net/eepro100.c b/roms/ipxe/src/drivers/net/eepro100.c
index ede0a1a4b..1046cda39 100644
--- a/roms/ipxe/src/drivers/net/eepro100.c
+++ b/roms/ipxe/src/drivers/net/eepro100.c
@@ -1136,7 +1136,6 @@ PCI_ROM(0x8086, 0x2449, "82562em",       "Intel EtherExpressPro100 82562EM", 0),
 PCI_ROM(0x8086, 0x2459, "82562-1",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x245d, "82562-2",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x1050, "82562ez",       "Intel 82562EZ Network Connection", 0),
-PCI_ROM(0x8086, 0x1051, "eepro100-1051", "Intel 82801EB/ER (ICH5/ICH5R) Chipset Ethernet Controller", 0),
 PCI_ROM(0x8086, 0x1065, "82562-3",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x5200, "eepro100-5200", "Intel EtherExpress PRO/100 Intelligent Server", 0),
 PCI_ROM(0x8086, 0x5201, "eepro100-5201", "Intel EtherExpress PRO/100 Intelligent Server", 0),
diff --git a/roms/ipxe/src/drivers/net/efi/nii.c b/roms/ipxe/src/drivers/net/efi/nii.c
index d0d7da95a..b91848f5c 100644
--- a/roms/ipxe/src/drivers/net/efi/nii.c
+++ b/roms/ipxe/src/drivers/net/efi/nii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <strings.h>
@@ -168,6 +172,9 @@ struct nii_nic {
 	/** Saved task priority level */
 	EFI_TPL saved_tpl;
 
+	/** Media status is supported */
+	int media;
+
 	/** Current transmit buffer */
 	struct io_buffer *txbuf;
 	/** Current receive buffer */
@@ -408,6 +415,13 @@ static int nii_issue_cpb_db ( struct nii_nic *nii, unsigned int op, void *cpb,
 	cdb.IFnum = nii->nii->IfNum;
 
 	/* Issue command */
+	DBGC2 ( nii, "NII %s issuing %02x:%04x ifnum %d%s%s\n",
+		nii->dev.name, cdb.OpCode, cdb.OpFlags, cdb.IFnum,
+		( cpb ? " cpb" : "" ), ( db ? " db" : "" ) );
+	if ( cpb )
+		DBGC2_HD ( nii, cpb, cpb_len );
+	if ( db )
+		DBGC2_HD ( nii, db, db_len );
 	nii->issue ( ( intptr_t ) &cdb );
 
 	/* Check completion status */
@@ -552,6 +566,7 @@ static int nii_get_init_info ( struct nii_nic *nii,
 	nii->buffer_len = db.MemoryRequired;
 	nii->mtu = ( db.FrameDataLen + db.MediaHeaderLen );
 	netdev->max_pkt_len = nii->mtu;
+	nii->media = ( stat & PXE_STATFLAGS_GET_STATUS_NO_MEDIA_SUPPORTED );
 
 	return 0;
 }
@@ -560,10 +575,12 @@ static int nii_get_init_info ( struct nii_nic *nii,
  * Initialise UNDI
  *
  * @v nii		NII NIC
+ * @v flags		Flags
  * @ret rc		Return status code
  */
-static int nii_initialise ( struct nii_nic *nii ) {
+static int nii_initialise_flags ( struct nii_nic *nii, unsigned int flags ) {
 	PXE_CPB_INITIALIZE cpb;
+	PXE_DB_INITIALIZE db;
 	unsigned int op;
 	int stat;
 	int rc;
@@ -580,10 +597,13 @@ static int nii_initialise ( struct nii_nic *nii ) {
 	cpb.MemoryAddr = ( ( intptr_t ) nii->buffer );
 	cpb.MemoryLength = nii->buffer_len;
 
+	/* Construct data block */
+	memset ( &db, 0, sizeof ( db ) );
+
 	/* Issue command */
-	op = NII_OP ( PXE_OPCODE_INITIALIZE,
-		      PXE_OPFLAGS_INITIALIZE_DO_NOT_DETECT_CABLE );
-	if ( ( stat = nii_issue_cpb ( nii, op, &cpb, sizeof ( cpb ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_INITIALIZE, flags );
+	if ( ( stat = nii_issue_cpb_db ( nii, op, &cpb, sizeof ( cpb ),
+					 &db, sizeof ( db ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not initialise: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -599,6 +619,36 @@ static int nii_initialise ( struct nii_nic *nii ) {
 }
 
 /**
+ * Initialise UNDI
+ *
+ * @v nii		NII NIC
+ * @ret rc		Return status code
+ */
+static int nii_initialise ( struct nii_nic *nii ) {
+	unsigned int flags;
+
+	/* Initialise UNDI */
+	flags = PXE_OPFLAGS_INITIALIZE_DO_NOT_DETECT_CABLE;
+	return nii_initialise_flags ( nii, flags );
+}
+
+/**
+ * Initialise UNDI and detect cable
+ *
+ * @v nii		NII NIC
+ * @ret rc		Return status code
+ */
+static int nii_initialise_and_detect ( struct nii_nic *nii ) {
+	unsigned int flags;
+
+	/* Initialise UNDI and detect cable.  This is required to work
+	 * around bugs in some Emulex NII drivers.
+	 */
+	flags = PXE_OPFLAGS_INITIALIZE_DETECT_CABLE;
+	return nii_initialise_flags ( nii, flags );
+}
+
+/**
  * Shut down UNDI
  *
  * @v nii		NII NIC
@@ -630,6 +680,7 @@ static void nii_shutdown ( struct nii_nic *nii ) {
 static int nii_get_station_address ( struct nii_nic *nii,
 				     struct net_device *netdev ) {
 	PXE_DB_STATION_ADDRESS db;
+	unsigned int op;
 	int stat;
 	int rc;
 
@@ -638,8 +689,9 @@ static int nii_get_station_address ( struct nii_nic *nii,
 		goto err_initialise;
 
 	/* Issue command */
-	if ( ( stat = nii_issue_db ( nii, PXE_OPCODE_STATION_ADDRESS, &db,
-				     sizeof ( db ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_STATION_ADDRESS,
+		      PXE_OPFLAGS_STATION_ADDRESS_READ );
+	if ( ( stat = nii_issue_db ( nii, op, &db, sizeof ( db ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not get station address: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -669,18 +721,25 @@ static int nii_get_station_address ( struct nii_nic *nii,
  */
 static int nii_set_station_address ( struct nii_nic *nii,
 				     struct net_device *netdev ) {
+	uint32_t implementation = nii->undi->Implementation;
 	PXE_CPB_STATION_ADDRESS cpb;
+	unsigned int op;
 	int stat;
 	int rc;
 
+	/* Fail if setting station address is unsupported */
+	if ( ! ( implementation & PXE_ROMID_IMP_STATION_ADDR_SETTABLE ) )
+		return -ENOTSUP;
+
 	/* Construct parameter block */
 	memset ( &cpb, 0, sizeof ( cpb ) );
 	memcpy ( cpb.StationAddr, netdev->ll_addr,
 		 netdev->ll_protocol->ll_addr_len );
 
 	/* Issue command */
-	if ( ( stat = nii_issue_cpb ( nii, PXE_OPCODE_STATION_ADDRESS,
-				      &cpb, sizeof ( cpb ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_STATION_ADDRESS,
+	              PXE_OPFLAGS_STATION_ADDRESS_WRITE );
+	if ( ( stat = nii_issue_cpb ( nii, op, &cpb, sizeof ( cpb ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not set station address: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -697,21 +756,28 @@ static int nii_set_station_address ( struct nii_nic *nii,
  * @ret rc		Return status code
  */
 static int nii_set_rx_filters ( struct nii_nic *nii ) {
+	uint32_t implementation = nii->undi->Implementation;
+	unsigned int flags;
 	unsigned int op;
 	int stat;
 	int rc;
 
+	/* Construct receive filter set */
+	flags = ( PXE_OPFLAGS_RECEIVE_FILTER_ENABLE |
+		  PXE_OPFLAGS_RECEIVE_FILTER_UNICAST );
+	if ( implementation & PXE_ROMID_IMP_BROADCAST_RX_SUPPORTED )
+		flags |= PXE_OPFLAGS_RECEIVE_FILTER_BROADCAST;
+	if ( implementation & PXE_ROMID_IMP_PROMISCUOUS_RX_SUPPORTED )
+		flags |= PXE_OPFLAGS_RECEIVE_FILTER_PROMISCUOUS;
+	if ( implementation & PXE_ROMID_IMP_PROMISCUOUS_MULTICAST_RX_SUPPORTED )
+		flags |= PXE_OPFLAGS_RECEIVE_FILTER_ALL_MULTICAST;
+
 	/* Issue command */
-	op = NII_OP ( PXE_OPCODE_RECEIVE_FILTERS,
-		      ( PXE_OPFLAGS_RECEIVE_FILTER_ENABLE |
-			PXE_OPFLAGS_RECEIVE_FILTER_UNICAST |
-			PXE_OPFLAGS_RECEIVE_FILTER_BROADCAST |
-			PXE_OPFLAGS_RECEIVE_FILTER_PROMISCUOUS |
-			PXE_OPFLAGS_RECEIVE_FILTER_ALL_MULTICAST ) );
+	op = NII_OP ( PXE_OPCODE_RECEIVE_FILTERS, flags );
 	if ( ( stat = nii_issue ( nii, op ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
-		DBGC ( nii, "NII %s could not set receive filters: %s\n",
-		       nii->dev.name, strerror ( rc ) );
+		DBGC ( nii, "NII %s could not set receive filters %#04x: %s\n",
+		       nii->dev.name, flags, strerror ( rc ) );
 		return rc;
 	}
 
@@ -729,6 +795,7 @@ static int nii_transmit ( struct net_device *netdev,
 			  struct io_buffer *iobuf ) {
 	struct nii_nic *nii = netdev->priv;
 	PXE_CPB_TRANSMIT cpb;
+	unsigned int op;
 	int stat;
 	int rc;
 
@@ -745,8 +812,10 @@ static int nii_transmit ( struct net_device *netdev,
 	cpb.MediaheaderLen = netdev->ll_protocol->ll_header_len;
 
 	/* Transmit packet */
-	if ( ( stat = nii_issue_cpb ( nii, PXE_OPCODE_TRANSMIT, &cpb,
-				      sizeof ( cpb ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_TRANSMIT,
+		      ( PXE_OPFLAGS_TRANSMIT_WHOLE |
+			PXE_OPFLAGS_TRANSMIT_DONT_BLOCK ) );
+	if ( ( stat = nii_issue_cpb ( nii, op, &cpb, sizeof ( cpb ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not transmit: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -772,12 +841,7 @@ static void nii_poll_tx ( struct net_device *netdev, unsigned int stat ) {
 		return;
 
 	/* Sanity check */
-	if ( ! nii->txbuf ) {
-		DBGC ( nii, "NII %s reported spurious TX completion\n",
-		       nii->dev.name );
-		netdev_tx_err ( netdev, NULL, -EPIPE );
-		return;
-	}
+	assert ( nii->txbuf != NULL );
 
 	/* Complete transmission */
 	iobuf = nii->txbuf;
@@ -869,11 +933,14 @@ static void nii_poll ( struct net_device *netdev ) {
 	int stat;
 	int rc;
 
+	/* Construct data block */
+	memset ( &db, 0, sizeof ( db ) );
+
 	/* Get status */
 	op = NII_OP ( PXE_OPCODE_GET_STATUS,
 		      ( PXE_OPFLAGS_GET_INTERRUPT_STATUS |
-			PXE_OPFLAGS_GET_TRANSMITTED_BUFFERS |
-			PXE_OPFLAGS_GET_MEDIA_STATUS ) );
+			( nii->txbuf ? PXE_OPFLAGS_GET_TRANSMITTED_BUFFERS : 0)|
+			( nii->media ? PXE_OPFLAGS_GET_MEDIA_STATUS : 0 ) ) );
 	if ( ( stat = nii_issue_db ( nii, op, &db, sizeof ( db ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not get status: %s\n",
@@ -882,13 +949,15 @@ static void nii_poll ( struct net_device *netdev ) {
 	}
 
 	/* Process any TX completions */
-	nii_poll_tx ( netdev, stat );
+	if ( nii->txbuf )
+		nii_poll_tx ( netdev, stat );
 
 	/* Process any RX completions */
 	nii_poll_rx ( netdev );
 
 	/* Check for link state changes */
-	nii_poll_link ( netdev, stat );
+	if ( nii->media )
+		nii_poll_link ( netdev, stat );
 }
 
 /**
@@ -901,8 +970,18 @@ static int nii_open ( struct net_device *netdev ) {
 	struct nii_nic *nii = netdev->priv;
 	int rc;
 
-	/* Initialise NIC */
-	if ( ( rc = nii_initialise ( nii ) ) != 0 )
+	/* Initialise NIC
+	 *
+	 * Some Emulex NII drivers have a bug which prevents packets
+	 * from being sent or received unless we specifically ask it
+	 * to detect cable presence during initialisation.  Work
+	 * around these buggy drivers by requesting cable detection at
+	 * this point, even though we don't care about link state here
+	 * (and would prefer to have the NIC initialise even if no
+	 * cable is present, to match the behaviour of all other iPXE
+	 * drivers).
+	 */
+	if ( ( rc = nii_initialise_and_detect ( nii ) ) != 0 )
 		goto err_initialise;
 
 	/* Attempt to set station address */
@@ -1023,8 +1102,9 @@ int nii_start ( struct efi_device *efidev ) {
 		nii->issue = ( ( ( void * ) nii->undi ) +
 			       nii->undi->EntryPoint );
 	}
-	DBGC ( nii, "NII %s using UNDI v%x.%x at %p entry %p\n", nii->dev.name,
-	       nii->nii->MajorVer, nii->nii->MinorVer, nii->undi, nii->issue );
+	DBGC ( nii, "NII %s using UNDI v%x.%x at %p entry %p impl %#08x\n",
+	       nii->dev.name, nii->nii->MajorVer, nii->nii->MinorVer,
+	       nii->undi, nii->issue, nii->undi->Implementation );
 
 	/* Open PCI I/O protocols and locate BARs */
 	if ( ( rc = nii_pci_open ( nii ) ) != 0 )
@@ -1048,6 +1128,10 @@ int nii_start ( struct efi_device *efidev ) {
 	DBGC ( nii, "NII %s registered as %s for %p %s\n", nii->dev.name,
 	       netdev->name, device, efi_handle_name ( device ) );
 
+	/* Set initial link state (if media detection is not supported) */
+	if ( ! nii->media )
+		netdev_link_up ( netdev );
+
 	return 0;
 
 	unregister_netdev ( netdev );
diff --git a/roms/ipxe/src/drivers/net/efi/nii.h b/roms/ipxe/src/drivers/net/efi/nii.h
index de0ac687b..c10be9db5 100644
--- a/roms/ipxe/src/drivers/net/efi/nii.h
+++ b/roms/ipxe/src/drivers/net/efi/nii.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct efi_device;
 
diff --git a/roms/ipxe/src/drivers/net/efi/snp.c b/roms/ipxe/src/drivers/net/efi/snp.c
index 2b5fc8618..acfcfba9f 100644
--- a/roms/ipxe/src/drivers/net/efi/snp.c
+++ b/roms/ipxe/src/drivers/net/efi/snp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/efi/efi.h>
diff --git a/roms/ipxe/src/drivers/net/efi/snponly.c b/roms/ipxe/src/drivers/net/efi/snponly.c
index 99f264bca..73abfdbf4 100644
--- a/roms/ipxe/src/drivers/net/efi/snponly.c
+++ b/roms/ipxe/src/drivers/net/efi/snponly.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/drivers/net/etherfabric.c b/roms/ipxe/src/drivers/net/etherfabric.c
index 5e0efb1e1..29d117443 100644
--- a/roms/ipxe/src/drivers/net/etherfabric.c
+++ b/roms/ipxe/src/drivers/net/etherfabric.c
@@ -3176,7 +3176,7 @@ falcon_probe_nic_variant ( struct efab_nic *efab, struct pci_device *pci )
 	uint8_t revision;
 
 	/* PCI revision */
-	pci_read_config_byte ( pci, PCI_CLASS_REVISION, &revision );
+	pci_read_config_byte ( pci, PCI_REVISION, &revision );
 	efab->pci_revision = revision;
 
 	/* Asic vs FPGA */
diff --git a/roms/ipxe/src/drivers/net/forcedeth.c b/roms/ipxe/src/drivers/net/forcedeth.c
index d8ece9a7a..79938cbbb 100644
--- a/roms/ipxe/src/drivers/net/forcedeth.c
+++ b/roms/ipxe/src/drivers/net/forcedeth.c
@@ -1749,10 +1749,8 @@ forcedeth_map_regs ( struct forcedeth_private *priv )
 	for ( reg = PCI_BASE_ADDRESS_0; reg <= PCI_BASE_ADDRESS_5; reg += 4 ) {
 		pci_read_config_dword ( priv->pci_dev, reg, &bar );
 
-		if ( ( ( bar & PCI_BASE_ADDRESS_SPACE ) ==
-			 PCI_BASE_ADDRESS_SPACE_MEMORY ) &&
-		       ( pci_bar_size ( priv->pci_dev, reg ) >=
-			 register_size ) ) {
+		if ( ( ! ( bar & PCI_BASE_ADDRESS_SPACE_IO ) ) &&
+		     ( pci_bar_size ( priv->pci_dev, reg ) >= register_size ) ){
 			addr = pci_bar_start ( priv->pci_dev, reg );
 			break;
 		}
diff --git a/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c b/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c
index aace5ad56..fc7021c38 100644
--- a/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c
+++ b/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c
@@ -461,7 +461,7 @@ static int __devinit igbvf_sw_init ( struct igbvf_adapter *adapter )
         hw->vendor_id = pdev->vendor;
         hw->device_id = pdev->device;
 
-        pci_read_config_byte ( pdev, PCI_REVISION_ID, &hw->revision_id );
+        pci_read_config_byte ( pdev, PCI_REVISION, &hw->revision_id );
 
         pci_read_config_word ( pdev, PCI_COMMAND, &hw->bus.pci_cmd_word );
 
diff --git a/roms/ipxe/src/drivers/net/intel.c b/roms/ipxe/src/drivers/net/intel.c
index a89f947b2..6309e9aa5 100644
--- a/roms/ipxe/src/drivers/net/intel.c
+++ b/roms/ipxe/src/drivers/net/intel.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -248,32 +252,6 @@ static int intel_fetch_mac ( struct intel_nic *intel, uint8_t *hw_addr ) {
 
 /******************************************************************************
  *
- * Diagnostics
- *
- ******************************************************************************
- */
-
-/**
- * Dump diagnostic information
- *
- * @v intel		Intel device
- */
-static void __attribute__ (( unused )) intel_diag ( struct intel_nic *intel ) {
-
-	DBGC ( intel, "INTEL %p TX %04x(%02x)/%04x(%02x) "
-	       "RX %04x(%02x)/%04x(%02x)\n", intel,
-	       ( intel->tx.cons & 0xffff ),
-	       readl ( intel->regs + intel->tx.reg + INTEL_xDH ),
-	       ( intel->tx.prod & 0xffff ),
-	       readl ( intel->regs + intel->tx.reg + INTEL_xDT ),
-	       ( intel->rx.cons & 0xffff ),
-	       readl ( intel->regs + intel->rx.reg + INTEL_xDH ),
-	       ( intel->rx.prod & 0xffff ),
-	       readl ( intel->regs + intel->rx.reg + INTEL_xDT ) );
-}
-
-/******************************************************************************
- *
  * Device reset
  *
  ******************************************************************************
@@ -371,6 +349,67 @@ static void intel_check_link ( struct net_device *netdev ) {
 
 /******************************************************************************
  *
+ * Descriptors
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Populate transmit descriptor
+ *
+ * @v tx		Transmit descriptor
+ * @v addr		Data buffer address
+ * @v len		Length of data
+ */
+void intel_describe_tx ( struct intel_descriptor *tx, physaddr_t addr,
+			 size_t len ) {
+
+	/* Populate transmit descriptor */
+	tx->address = cpu_to_le64 ( addr );
+	tx->length = cpu_to_le16 ( len );
+	tx->flags = 0;
+	tx->command = ( INTEL_DESC_CMD_RS | INTEL_DESC_CMD_IFCS |
+			INTEL_DESC_CMD_EOP );
+	tx->status = 0;
+}
+
+/**
+ * Populate advanced transmit descriptor
+ *
+ * @v tx		Transmit descriptor
+ * @v addr		Data buffer address
+ * @v len		Length of data
+ */
+void intel_describe_tx_adv ( struct intel_descriptor *tx, physaddr_t addr,
+			     size_t len ) {
+
+	/* Populate advanced transmit descriptor */
+	tx->address = cpu_to_le64 ( addr );
+	tx->length = cpu_to_le16 ( len );
+	tx->flags = INTEL_DESC_FL_DTYP_DATA;
+	tx->command = ( INTEL_DESC_CMD_DEXT | INTEL_DESC_CMD_RS |
+			INTEL_DESC_CMD_IFCS | INTEL_DESC_CMD_EOP );
+	tx->status = cpu_to_le32 ( INTEL_DESC_STATUS_PAYLEN ( len ) );
+}
+
+/**
+ * Populate receive descriptor
+ *
+ * @v rx		Receive descriptor
+ * @v addr		Data buffer address
+ * @v len		Length of data
+ */
+void intel_describe_rx ( struct intel_descriptor *rx, physaddr_t addr,
+			 size_t len __unused ) {
+
+	/* Populate transmit descriptor */
+	rx->address = cpu_to_le64 ( addr );
+	rx->length = 0;
+	rx->status = 0;
+}
+
+/******************************************************************************
+ *
  * Network device interface
  *
  ******************************************************************************
@@ -479,10 +518,7 @@ void intel_refill_rx ( struct intel_nic *intel ) {
 
 		/* Populate receive descriptor */
 		address = virt_to_bus ( iobuf->data );
-		rx->address = cpu_to_le64 ( address );
-		rx->length = 0;
-		rx->status = 0;
-		rx->errors = 0;
+		intel->rx.describe ( rx, address, 0 );
 
 		/* Record I/O buffer */
 		assert ( intel->rx_iobuf[rx_idx] == NULL );
@@ -568,6 +604,13 @@ static int intel_open ( struct net_device *netdev ) {
 	/* Update link state */
 	intel_check_link ( netdev );
 
+	/* Apply required errata */
+	if ( intel->flags & INTEL_VMWARE ) {
+		DBGC ( intel, "INTEL %p applying VMware errata workaround\n",
+		       intel );
+		intel->force_icr = INTEL_IRQ_RXT0;
+	}
+
 	return 0;
 
 	intel_destroy_ring ( intel, &intel->rx );
@@ -617,6 +660,7 @@ int intel_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
 	unsigned int tx_idx;
 	unsigned int tx_tail;
 	physaddr_t address;
+	size_t len;
 
 	/* Get next transmit descriptor */
 	if ( ( intel->tx.prod - intel->tx.cons ) >= INTEL_TX_FILL ) {
@@ -629,11 +673,8 @@ int intel_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
 
 	/* Populate transmit descriptor */
 	address = virt_to_bus ( iobuf->data );
-	tx->address = cpu_to_le64 ( address );
-	tx->length = cpu_to_le16 ( iob_len ( iobuf ) );
-	tx->command = ( INTEL_DESC_CMD_RS | INTEL_DESC_CMD_IFCS |
-			INTEL_DESC_CMD_EOP );
-	tx->status = 0;
+	len = iob_len ( iobuf );
+	intel->tx.describe ( tx, address, len );
 	wmb();
 
 	/* Notify card that there are packets ready to transmit */
@@ -644,7 +685,7 @@ int intel_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
 
 	DBGC2 ( intel, "INTEL %p TX %d is [%llx,%llx)\n", intel, tx_idx,
 		( ( unsigned long long ) address ),
-		( ( unsigned long long ) address + iob_len ( iobuf ) ) );
+		( ( unsigned long long ) address + len ) );
 
 	return 0;
 }
@@ -667,7 +708,7 @@ void intel_poll_tx ( struct net_device *netdev ) {
 		tx = &intel->tx.desc[tx_idx];
 
 		/* Stop if descriptor is still in use */
-		if ( ! ( tx->status & INTEL_DESC_STATUS_DD ) )
+		if ( ! ( tx->status & cpu_to_le32 ( INTEL_DESC_STATUS_DD ) ) )
 			return;
 
 		DBGC2 ( intel, "INTEL %p TX %d complete\n", intel, tx_idx );
@@ -698,7 +739,7 @@ void intel_poll_rx ( struct net_device *netdev ) {
 		rx = &intel->rx.desc[rx_idx];
 
 		/* Stop if descriptor is still in use */
-		if ( ! ( rx->status & INTEL_DESC_STATUS_DD ) )
+		if ( ! ( rx->status & cpu_to_le32 ( INTEL_DESC_STATUS_DD ) ) )
 			return;
 
 		/* Populate I/O buffer */
@@ -708,10 +749,10 @@ void intel_poll_rx ( struct net_device *netdev ) {
 		iob_put ( iobuf, len );
 
 		/* Hand off to network stack */
-		if ( rx->errors ) {
+		if ( rx->status & cpu_to_le32 ( INTEL_DESC_STATUS_RXE ) ) {
 			DBGC ( intel, "INTEL %p RX %d error (length %zd, "
-			       "errors %02x)\n",
-			       intel, rx_idx, len, rx->errors );
+			       "status %08x)\n", intel, rx_idx, len,
+			       le32_to_cpu ( rx->status ) );
 			netdev_rx_err ( netdev, iobuf, -EIO );
 		} else {
 			DBGC2 ( intel, "INTEL %p RX %d complete (length %zd)\n",
@@ -736,6 +777,7 @@ static void intel_poll ( struct net_device *netdev ) {
 	icr = readl ( intel->regs + INTEL_ICR );
 	profile_stop ( &intel_vm_poll_profiler );
 	profile_exclude ( &intel_vm_poll_profiler );
+	icr |= intel->force_icr;
 	if ( ! icr )
 		return;
 
@@ -755,6 +797,14 @@ static void intel_poll ( struct net_device *netdev ) {
 	if ( icr & INTEL_IRQ_LSC )
 		intel_check_link ( netdev );
 
+	/* Check for unexpected interrupts */
+	if ( icr & ~( INTEL_IRQ_TXDW | INTEL_IRQ_TXQE | INTEL_IRQ_LSC |
+		      INTEL_IRQ_RXDMT0 | INTEL_IRQ_RXT0 | INTEL_IRQ_RXO ) ) {
+		DBGC ( intel, "INTEL %p unexpected ICR %08x\n", intel, icr );
+		/* Report as a TX error */
+		netdev_tx_err ( netdev, NULL, -ENOTSUP );
+	}
+
 	/* Refill RX ring */
 	intel_refill_rx ( intel );
 }
@@ -817,8 +867,10 @@ static int intel_probe ( struct pci_device *pci ) {
 	memset ( intel, 0, sizeof ( *intel ) );
 	intel->port = PCI_FUNC ( pci->busdevfn );
 	intel->flags = pci->id->driver_data;
-	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTEL_TD );
-	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTEL_RD );
+	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTEL_TD,
+			  intel_describe_tx );
+	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTEL_RD,
+			  intel_describe_rx );
 
 	/* Fix up PCI device */
 	adjust_pci_device ( pci );
@@ -895,7 +947,7 @@ static struct pci_device_id intel_nics[] = {
 	PCI_ROM ( 0x8086, 0x100c, "82544gc", "82544GC (Copper)", 0 ),
 	PCI_ROM ( 0x8086, 0x100d, "82544gc-l", "82544GC (LOM)", 0 ),
 	PCI_ROM ( 0x8086, 0x100e, "82540em", "82540EM", 0 ),
-	PCI_ROM ( 0x8086, 0x100f, "82545em", "82545EM (Copper)", 0 ),
+	PCI_ROM ( 0x8086, 0x100f, "82545em", "82545EM (Copper)", INTEL_VMWARE ),
 	PCI_ROM ( 0x8086, 0x1010, "82546eb", "82546EB (Copper)", 0 ),
 	PCI_ROM ( 0x8086, 0x1011, "82545em-f", "82545EM (Fiber)", 0 ),
 	PCI_ROM ( 0x8086, 0x1012, "82546eb-f", "82546EB (Fiber)", 0 ),
@@ -998,6 +1050,12 @@ static struct pci_device_id intel_nics[] = {
 	PCI_ROM ( 0x8086, 0x1533, "i210", "I210", 0 ),
 	PCI_ROM ( 0x8086, 0x153a, "i217lm", "I217-LM", 0 ),
 	PCI_ROM ( 0x8086, 0x153b, "i217v", "I217-V", 0 ),
+	PCI_ROM ( 0x8086, 0x1559, "i218v", "I218-V", 0),
+	PCI_ROM ( 0x8086, 0x155a, "i218lm", "I218-LM", 0),
+	PCI_ROM ( 0x8086, 0x15a0, "i218lm-2", "I218-LM", 0 ),
+	PCI_ROM ( 0x8086, 0x15a1, "i218v-2", "I218-V", 0 ),
+	PCI_ROM ( 0x8086, 0x15a2, "i218lm-3", "I218-LM", 0 ),
+	PCI_ROM ( 0x8086, 0x15a3, "i218v-3", "I218-V", 0 ),
 	PCI_ROM ( 0x8086, 0x294c, "82566dc-2", "82566DC-2", 0 ),
 	PCI_ROM ( 0x8086, 0x2e6e, "cemedia", "CE Media Processor", 0 ),
 };
diff --git a/roms/ipxe/src/drivers/net/intel.h b/roms/ipxe/src/drivers/net/intel.h
index 8c4479bb4..ce9e3f467 100644
--- a/roms/ipxe/src/drivers/net/intel.h
+++ b/roms/ipxe/src/drivers/net/intel.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/if_ether.h>
@@ -22,33 +22,38 @@ struct intel_descriptor {
 	uint64_t address;
 	/** Length */
 	uint16_t length;
-	/** Reserved */
-	uint8_t reserved_a;
+	/** Flags */
+	uint8_t flags;
 	/** Command */
 	uint8_t command;
 	/** Status */
-	uint8_t status;
-	/** Errors */
-	uint8_t errors;
-	/** Reserved */
-	uint16_t reserved_b;
+	uint32_t status;
 } __attribute__ (( packed ));
 
-/** Packet descriptor command bits */
-enum intel_descriptor_command {
-	/** Report status */
-	INTEL_DESC_CMD_RS = 0x08,
-	/** Insert frame checksum (CRC) */
-	INTEL_DESC_CMD_IFCS = 0x02,
-	/** End of packet */
-	INTEL_DESC_CMD_EOP = 0x01,
-};
+/** Descriptor type */
+#define INTEL_DESC_FL_DTYP( dtyp ) ( (dtyp) << 4 )
+#define INTEL_DESC_FL_DTYP_DATA INTEL_DESC_FL_DTYP ( 0x03 )
 
-/** Packet descriptor status bits */
-enum intel_descriptor_status {
-	/** Descriptor done */
-	INTEL_DESC_STATUS_DD = 0x01,
-};
+/** Descriptor extension */
+#define INTEL_DESC_CMD_DEXT 0x20
+
+/** Report status */
+#define INTEL_DESC_CMD_RS 0x08
+
+/** Insert frame checksum (CRC) */
+#define INTEL_DESC_CMD_IFCS 0x02
+
+/** End of packet */
+#define INTEL_DESC_CMD_EOP 0x01
+
+/** Descriptor done */
+#define INTEL_DESC_STATUS_DD 0x00000001UL
+
+/** Receive error */
+#define INTEL_DESC_STATUS_RXE 0x00000100UL
+
+/** Payload length */
+#define INTEL_DESC_STATUS_PAYLEN( len ) ( (len) << 14 )
 
 /** Device Control Register */
 #define INTEL_CTRL 0x00000UL
@@ -91,7 +96,9 @@ enum intel_descriptor_status {
 /** Interrupt Cause Read Register */
 #define INTEL_ICR 0x000c0UL
 #define INTEL_IRQ_TXDW		0x00000001UL	/**< Transmit descriptor done */
+#define INTEL_IRQ_TXQE		0x00000002UL	/**< Transmit queue empty */
 #define INTEL_IRQ_LSC		0x00000004UL	/**< Link status change */
+#define INTEL_IRQ_RXDMT0	0x00000010UL	/**< Receive queue low */
 #define INTEL_IRQ_RXT0		0x00000080UL	/**< Receive timer */
 #define INTEL_IRQ_RXO		0x00000400UL	/**< Receive overrun */
 
@@ -207,6 +214,15 @@ struct intel_ring {
 	unsigned int reg;
 	/** Length (in bytes) */
 	size_t len;
+
+	/** Populate descriptor
+	 *
+	 * @v desc		Descriptor
+	 * @v addr		Data buffer address
+	 * @v len		Length of data
+	 */
+	void ( * describe ) ( struct intel_descriptor *desc, physaddr_t addr,
+			      size_t len );
 };
 
 /**
@@ -215,12 +231,39 @@ struct intel_ring {
  * @v ring		Descriptor ring
  * @v count		Number of descriptors
  * @v reg		Descriptor register block
+ * @v describe		Method to populate descriptor
  */
 static inline __attribute__ (( always_inline)) void
-intel_init_ring ( struct intel_ring *ring, unsigned int count,
-		  unsigned int reg ) {
+intel_init_ring ( struct intel_ring *ring, unsigned int count, unsigned int reg,
+		  void ( * describe ) ( struct intel_descriptor *desc,
+					physaddr_t addr, size_t len ) ) {
+
 	ring->len = ( count * sizeof ( ring->desc[0] ) );
 	ring->reg = reg;
+	ring->describe = describe;
+}
+
+/** An Intel virtual function mailbox */
+struct intel_mailbox {
+	/** Mailbox control register */
+	unsigned int ctrl;
+	/** Mailbox memory base */
+	unsigned int mem;
+};
+
+/**
+ * Initialise mailbox
+ *
+ * @v mbox		Mailbox
+ * @v ctrl		Mailbox control register
+ * @v mem		Mailbox memory register base
+ */
+static inline __attribute__ (( always_inline )) void
+intel_init_mbox ( struct intel_mailbox *mbox, unsigned int ctrl,
+		  unsigned int mem ) {
+
+	mbox->ctrl = ctrl;
+	mbox->mem = mem;
 }
 
 /** An Intel network card */
@@ -231,6 +274,8 @@ struct intel_nic {
 	unsigned int port;
 	/** Flags */
 	unsigned int flags;
+	/** Forced interrupts */
+	unsigned int force_icr;
 
 	/** EEPROM */
 	struct nvs_device eeprom;
@@ -239,6 +284,9 @@ struct intel_nic {
 	/** EEPROM address shift */
 	unsigned int eerd_addr_shift;
 
+	/** Mailbox */
+	struct intel_mailbox mbox;
+
 	/** Transmit descriptor ring */
 	struct intel_ring tx;
 	/** Receive descriptor ring */
@@ -251,8 +299,35 @@ struct intel_nic {
 enum intel_flags {
 	/** PBS/PBA errata workaround required */
 	INTEL_PBS_ERRATA = 0x0001,
+	/** VMware missing interrupt workaround required */
+	INTEL_VMWARE = 0x0002,
 };
 
+/**
+ * Dump diagnostic information
+ *
+ * @v intel		Intel device
+ */
+static inline void intel_diag ( struct intel_nic *intel ) {
+
+	DBGC ( intel, "INTEL %p TX %04x(%02x)/%04x(%02x) "
+	       "RX %04x(%02x)/%04x(%02x)\n", intel,
+	       ( intel->tx.cons & 0xffff ),
+	       readl ( intel->regs + intel->tx.reg + INTEL_xDH ),
+	       ( intel->tx.prod & 0xffff ),
+	       readl ( intel->regs + intel->tx.reg + INTEL_xDT ),
+	       ( intel->rx.cons & 0xffff ),
+	       readl ( intel->regs + intel->rx.reg + INTEL_xDH ),
+	       ( intel->rx.prod & 0xffff ),
+	       readl ( intel->regs + intel->rx.reg + INTEL_xDT ) );
+}
+
+extern void intel_describe_tx ( struct intel_descriptor *tx,
+				physaddr_t addr, size_t len );
+extern void intel_describe_tx_adv ( struct intel_descriptor *tx,
+				    physaddr_t addr, size_t len );
+extern void intel_describe_rx ( struct intel_descriptor *rx,
+				physaddr_t addr, size_t len );
 extern int intel_create_ring ( struct intel_nic *intel,
 			       struct intel_ring *ring );
 extern void intel_destroy_ring ( struct intel_nic *intel,
diff --git a/roms/ipxe/src/drivers/net/intelvf.c b/roms/ipxe/src/drivers/net/intelvf.c
new file mode 100644
index 000000000..ac6fea745
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/intelvf.c
@@ -0,0 +1,340 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/io.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include "intelvf.h"
+
+/** @file
+ *
+ * Intel 10/100/1000 virtual function network card driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Mailbox messages
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Write message to mailbox
+ *
+ * @v intel		Intel device
+ * @v msg		Message
+ */
+static void intelvf_mbox_write ( struct intel_nic *intel,
+				 const union intelvf_msg *msg ) {
+	unsigned int i;
+
+	/* Write message */
+	DBGC2 ( intel, "INTEL %p sending message", intel );
+	for ( i = 0 ; i < ( sizeof ( *msg ) / sizeof ( msg->dword[0] ) ) ; i++){
+		DBGC2 ( intel, "%c%08x", ( i ? ':' : ' ' ), msg->dword[i] );
+		writel ( msg->dword[i], ( intel->regs + intel->mbox.mem +
+					  ( i * sizeof ( msg->dword[0] ) ) ) );
+	}
+	DBGC2 ( intel, "\n" );
+}
+
+/**
+ * Read message from mailbox
+ *
+ * @v intel		Intel device
+ * @v msg		Message
+ */
+static void intelvf_mbox_read ( struct intel_nic *intel,
+				union intelvf_msg *msg ) {
+	unsigned int i;
+
+	/* Read message */
+	DBGC2 ( intel, "INTEL %p received message", intel );
+	for ( i = 0 ; i < ( sizeof ( *msg ) / sizeof ( msg->dword[0] ) ) ; i++){
+		msg->dword[i] = readl ( intel->regs + intel->mbox.mem +
+					( i * sizeof ( msg->dword[0] ) ) );
+		DBGC2 ( intel, "%c%08x", ( i ? ':' : ' ' ), msg->dword[i] );
+	}
+	DBGC2 ( intel, "\n" );
+}
+
+/**
+ * Poll mailbox
+ *
+ * @v intel		Intel device
+ * @ret rc		Return status code
+ *
+ * Note that polling the mailbox may fail if the underlying PF is
+ * reset.
+ */
+int intelvf_mbox_poll ( struct intel_nic *intel ) {
+	struct intel_mailbox *mbox = &intel->mbox;
+	union intelvf_msg msg;
+	uint32_t ctrl;
+
+	/* Get mailbox status */
+	ctrl = readl ( intel->regs + mbox->ctrl );
+
+	/* Fail if a reset is in progress */
+	if ( ctrl & INTELVF_MBCTRL_RSTI )
+		return -EPIPE;
+
+	/* Acknowledge (and ignore) any received messages */
+	if ( ctrl & INTELVF_MBCTRL_PFSTS ) {
+		intelvf_mbox_read ( intel, &msg );
+		writel ( INTELVF_MBCTRL_ACK, intel->regs + mbox->ctrl );
+	}
+
+	return 0;
+}
+
+/**
+ * Wait for PF reset to complete
+ *
+ * @v intel		Intel device
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_wait ( struct intel_nic *intel ) {
+	unsigned int i;
+	int rc;
+
+	/* Wait until a poll completes successfully */
+	for ( i = 0 ; i < INTELVF_MBOX_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for successful poll */
+		if ( ( rc = intelvf_mbox_poll ( intel ) ) == 0 )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( intel, "INTEL %p timed out waiting for reset\n", intel );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Send/receive mailbox message
+ *
+ * @v intel		Intel device
+ * @v msg		Message buffer
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_msg ( struct intel_nic *intel, union intelvf_msg *msg ) {
+	struct intel_mailbox *mbox = &intel->mbox;
+	uint32_t ctrl;
+	uint32_t seen = 0;
+	unsigned int i;
+
+	/* Sanity check */
+	assert ( ! ( msg->hdr & INTELVF_MSG_RESPONSE ) );
+
+	/* Handle mailbox */
+	for ( i = 0 ; i < INTELVF_MBOX_MAX_WAIT_MS ; i++ ) {
+
+		/* Attempt to claim mailbox, if we have not yet sent
+		 * our message.
+		 */
+		if ( ! ( seen & INTELVF_MBCTRL_VFU ) )
+			writel ( INTELVF_MBCTRL_VFU, intel->regs + mbox->ctrl );
+
+		/* Get mailbox status and record observed flags */
+		ctrl = readl ( intel->regs + mbox->ctrl );
+		seen |= ctrl;
+
+		/* If a reset is in progress, clear VFU and abort */
+		if ( ctrl & INTELVF_MBCTRL_RSTI ) {
+			writel ( 0, intel->regs + mbox->ctrl );
+			return -EPIPE;
+		}
+
+		/* Write message to mailbox, if applicable.  This
+		 * potentially overwrites a message sent by the PF (if
+		 * the PF has simultaneously released PFU (thus
+		 * allowing our VFU) and asserted PFSTS), but that
+		 * doesn't really matter since there are no
+		 * unsolicited PF->VF messages that require the actual
+		 * message content to be observed.
+		 */
+		if ( ctrl & INTELVF_MBCTRL_VFU )
+			intelvf_mbox_write ( intel, msg );
+
+		/* Read message from mailbox, if applicable. */
+		if ( ( seen & INTELVF_MBCTRL_VFU ) &&
+		     ( seen & INTELVF_MBCTRL_PFACK ) &&
+		     ( ctrl & INTELVF_MBCTRL_PFSTS ) )
+			intelvf_mbox_read ( intel, msg );
+
+		/* Acknowledge received message (if applicable),
+		 * release VFU lock, and send message (if applicable).
+		 */
+		ctrl = ( ( ( ctrl & INTELVF_MBCTRL_PFSTS ) ?
+			   INTELVF_MBCTRL_ACK : 0 ) |
+			 ( ( ctrl & INTELVF_MBCTRL_VFU ) ?
+			   INTELVF_MBCTRL_REQ : 0 ) );
+		writel ( ctrl, intel->regs + mbox->ctrl );
+
+		/* Exit successfully if we have received a response */
+		if ( msg->hdr & INTELVF_MSG_RESPONSE ) {
+
+			/* Sanity check */
+			assert ( seen & INTELVF_MBCTRL_VFU );
+			assert ( seen & INTELVF_MBCTRL_PFACK );
+			assert ( seen & INTELVF_MBCTRL_PFSTS );
+
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( intel, "INTEL %p timed out waiting for mailbox (seen %08x)\n",
+	       intel, seen );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Send reset message and get initial MAC address
+ *
+ * @v intel		Intel device
+ * @v hw_addr		Hardware address to fill in, or NULL
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_reset ( struct intel_nic *intel, uint8_t *hw_addr ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send reset message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELVF_MSG_TYPE_RESET;
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p reset failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELVF_MSG_TYPE_RESET ) {
+		DBGC ( intel, "INTEL %p reset unexpected response:\n", intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Fill in MAC address, if applicable */
+	if ( hw_addr ) {
+		if ( msg.hdr & INTELVF_MSG_ACK ) {
+			memcpy ( hw_addr, msg.mac.mac, sizeof ( msg.mac.mac ) );
+			DBGC ( intel, "INTEL %p reset assigned MAC address "
+			       "%s\n", intel, eth_ntoa ( hw_addr ) );
+		} else {
+			eth_random_addr ( hw_addr );
+			DBGC ( intel, "INTEL %p reset generated MAC address "
+			       "%s\n", intel, eth_ntoa ( hw_addr ) );
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Send set MAC address message
+ *
+ * @v intel		Intel device
+ * @v ll_addr		Link-layer address
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_set_mac ( struct intel_nic *intel, const uint8_t *ll_addr ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send set MAC address message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELVF_MSG_TYPE_SET_MAC;
+	memcpy ( msg.mac.mac, ll_addr, sizeof ( msg.mac.mac ) );
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p set MAC address failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELVF_MSG_TYPE_SET_MAC ) {
+		DBGC ( intel, "INTEL %p set MAC address unexpected response:\n",
+		       intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Check that we were allowed to set the MAC address */
+	if ( ! ( msg.hdr & INTELVF_MSG_ACK ) ) {
+		DBGC ( intel, "INTEL %p set MAC address refused\n", intel );
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+/**
+ * Send set MTU message
+ *
+ * @v intel		Intel device
+ * @v mtu		Maximum packet size
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_set_mtu ( struct intel_nic *intel, size_t mtu ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send set MTU message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELVF_MSG_TYPE_SET_MTU;
+	msg.mtu.mtu = mtu;
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p set MTU failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELVF_MSG_TYPE_SET_MTU ) {
+		DBGC ( intel, "INTEL %p set MTU unexpected response:\n",
+		       intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Check that we were allowed to set the MTU */
+	if ( ! ( msg.hdr & INTELVF_MSG_ACK ) ) {
+		DBGC ( intel, "INTEL %p set MTU refused\n", intel );
+		return -EPERM;
+	}
+
+	return 0;
+}
diff --git a/roms/ipxe/src/drivers/net/intelvf.h b/roms/ipxe/src/drivers/net/intelvf.h
new file mode 100644
index 000000000..d2f98d874
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/intelvf.h
@@ -0,0 +1,109 @@
+#ifndef _INTELVF_H
+#define _INTELVF_H
+
+/** @file
+ *
+ * Intel 10/100/1000 virtual function network card driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include "intel.h"
+
+/** Intel VF BAR size */
+#define INTELVF_BAR_SIZE ( 16 * 1024 )
+
+/** Mailbox Control Register */
+#define INTELVF_MBCTRL 0x0c40UL
+#define INTELVF_MBCTRL_REQ	0x00000001UL	/**< Request for PF ready */
+#define INTELVF_MBCTRL_ACK	0x00000002UL	/**< PF message received */
+#define INTELVF_MBCTRL_VFU	0x00000004UL	/**< Buffer taken by VF */
+#define INTELVF_MBCTRL_PFU	0x00000008UL	/**< Buffer taken to PF */
+#define INTELVF_MBCTRL_PFSTS	0x00000010UL	/**< PF wrote a message */
+#define INTELVF_MBCTRL_PFACK	0x00000020UL	/**< PF acknowledged message */
+#define INTELVF_MBCTRL_RSTI	0x00000040UL	/**< PF reset in progress */
+#define INTELVF_MBCTRL_RSTD	0x00000080UL	/**< PF reset complete */
+
+/** Mailbox Memory Register Base */
+#define INTELVF_MBMEM 0x0800UL
+
+/** Reset mailbox message */
+#define INTELVF_MSG_TYPE_RESET 0x00000001UL
+
+/** Set MAC address mailbox message */
+#define INTELVF_MSG_TYPE_SET_MAC 0x00000002UL
+
+/** Set MTU mailbox message */
+#define INTELVF_MSG_TYPE_SET_MTU 0x00000005UL
+
+/** Control ("ping") mailbox message */
+#define INTELVF_MSG_TYPE_CONTROL 0x00000100UL
+
+/** Message type mask */
+#define INTELVF_MSG_TYPE_MASK 0x0000ffffUL
+
+/** Message NACK flag */
+#define INTELVF_MSG_NACK 0x40000000UL
+
+/** Message ACK flag */
+#define INTELVF_MSG_ACK 0x80000000UL
+
+/** Message is a response */
+#define INTELVF_MSG_RESPONSE ( INTELVF_MSG_ACK | INTELVF_MSG_NACK )
+
+/** MAC address mailbox message */
+struct intelvf_msg_mac {
+	/** Message header */
+	uint32_t hdr;
+	/** MAC address */
+	uint8_t mac[ETH_ALEN];
+	/** Alignment padding */
+	uint8_t reserved[ (-ETH_ALEN) & 0x3 ];
+} __attribute__ (( packed ));
+
+/** Version number mailbox message */
+struct intelvf_msg_version {
+	/** Message header */
+	uint32_t hdr;
+	/** API version */
+	uint32_t version;
+} __attribute__ (( packed ));
+
+/** MTU mailbox message */
+struct intelvf_msg_mtu {
+	/** Message header */
+	uint32_t hdr;
+	/** Maximum packet size */
+	uint32_t mtu;
+} __attribute__ (( packed ));
+
+/** Mailbox message */
+union intelvf_msg {
+	/** Message header */
+	uint32_t hdr;
+	/** MAC address message */
+	struct intelvf_msg_mac mac;
+	/** Version number message */
+	struct intelvf_msg_version version;
+	/** MTU message */
+	struct intelvf_msg_mtu mtu;
+	/** Raw dwords */
+	uint32_t dword[0];
+};
+
+/** Maximum time to wait for mailbox message
+ *
+ * This is a policy decision.
+ */
+#define INTELVF_MBOX_MAX_WAIT_MS 500
+
+extern int intelvf_mbox_msg ( struct intel_nic *intel, union intelvf_msg *msg );
+extern int intelvf_mbox_poll ( struct intel_nic *intel );
+extern int intelvf_mbox_wait ( struct intel_nic *intel );
+extern int intelvf_mbox_reset ( struct intel_nic *intel, uint8_t *hw_addr );
+extern int intelvf_mbox_set_mac ( struct intel_nic *intel,
+				  const uint8_t *ll_addr );
+extern int intelvf_mbox_set_mtu ( struct intel_nic *intel, size_t mtu );
+
+#endif /* _INTELVF_H */
diff --git a/roms/ipxe/src/drivers/net/intelx.c b/roms/ipxe/src/drivers/net/intelx.c
index d69900e41..982b74f12 100644
--- a/roms/ipxe/src/drivers/net/intelx.c
+++ b/roms/ipxe/src/drivers/net/intelx.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -392,8 +396,10 @@ static int intelx_probe ( struct pci_device *pci ) {
 	netdev->dev = &pci->dev;
 	memset ( intel, 0, sizeof ( *intel ) );
 	intel->port = PCI_FUNC ( pci->busdevfn );
-	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTELX_TD );
-	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTELX_RD );
+	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTELX_TD,
+			  intel_describe_tx );
+	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTELX_RD,
+			  intel_describe_rx );
 
 	/* Fix up PCI device */
 	adjust_pci_device ( pci );
@@ -458,10 +464,15 @@ static void intelx_remove ( struct pci_device *pci ) {
 
 /** PCI device IDs */
 static struct pci_device_id intelx_nics[] = {
-	PCI_ROM ( 0x8086, 0x10fb, "82599", "82599", 0 ),
-	PCI_ROM ( 0x8086, 0x1528, "x540at2", "X540-AT2", 0 ),
-	PCI_ROM ( 0x8086, 0x154d, "x520", "X520", 0 ),
-	PCI_ROM ( 0x8086, 0x1557, "82599", "82599", 0 ),
+	PCI_ROM ( 0x8086, 0x10f7, "82599-kx4", "82599 (KX/KX4)", 0 ),
+	PCI_ROM ( 0x8086, 0x10f8, "82599-combo-backplane", "82599 (combined backplane; KR/KX4/KX)", 0 ),
+	PCI_ROM ( 0x8086, 0x10f9, "82599-cx4", "82599 (CX4)", 0 ),
+	PCI_ROM ( 0x8086, 0x10fb, "82599-sfp", "82599 (SFI/SFP+)", 0 ),
+	PCI_ROM ( 0x8086, 0x10fc, "82599-xaui", "82599 (XAUI/BX4)", 0 ),
+	PCI_ROM ( 0x8086, 0x1528, "x540t", "X540-AT2/X540-BT2", 0 ),
+	PCI_ROM ( 0x8086, 0x154d, "82599-sfp-sf2", "82599 (SFI/SFP+)", 0 ),
+	PCI_ROM ( 0x8086, 0x1557, "82599en-sfp", "82599 (Single Port SFI Only)", 0 ),
+	PCI_ROM ( 0x8086, 0x1560, "x540t1", "X540-AT2/X540-BT2 (with single port NVM)", 0 ),
 };
 
 /** PCI driver */
diff --git a/roms/ipxe/src/drivers/net/intelx.h b/roms/ipxe/src/drivers/net/intelx.h
index 60bb294d5..6383dfcad 100644
--- a/roms/ipxe/src/drivers/net/intelx.h
+++ b/roms/ipxe/src/drivers/net/intelx.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/if_ether.h>
diff --git a/roms/ipxe/src/drivers/net/intelxvf.c b/roms/ipxe/src/drivers/net/intelxvf.c
new file mode 100644
index 000000000..05e34c127
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/intelxvf.c
@@ -0,0 +1,466 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/io.h>
+#include <ipxe/pci.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include "intelxvf.h"
+
+/** @file
+ *
+ * Intel 10 Gigabit Ethernet virtual function network card driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Diagnostics
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Dump statistics
+ *
+ * @v intel		Intel device
+ */
+static __attribute__ (( unused )) void
+intelxvf_stats ( struct intel_nic *intel ) {
+
+	DBGC ( intel, "INTEL %p TX %d (%#x%08x) RX %d (%#x%08x) multi %d\n",
+	       intel, readl ( intel->regs + INTELXVF_GPTC ),
+	       readl ( intel->regs + INTELXVF_GOTCH ),
+	       readl ( intel->regs + INTELXVF_GOTCL ),
+	       readl ( intel->regs + INTELXVF_GPRC ),
+	       readl ( intel->regs + INTELXVF_GORCH ),
+	       readl ( intel->regs + INTELXVF_GORCL ),
+	       readl ( intel->regs + INTELXVF_MPRC ) );
+}
+
+/******************************************************************************
+ *
+ * Device reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Reset hardware
+ *
+ * @v intel		Intel device
+ */
+static void intelxvf_reset ( struct intel_nic *intel ) {
+
+	/* Perform a function-level reset */
+	writel ( INTELXVF_CTRL_RST, intel->regs + INTELXVF_CTRL );
+}
+
+/******************************************************************************
+ *
+ * Link state
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Check link state
+ *
+ * @v netdev		Network device
+ */
+static void intelxvf_check_link ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t links;
+
+	/* Read link status */
+	links = readl ( intel->regs + INTELXVF_LINKS );
+	DBGC ( intel, "INTEL %p link status is %08x\n", intel, links );
+
+	/* Update network device */
+	if ( links & INTELXVF_LINKS_UP ) {
+		netdev_link_up ( netdev );
+	} else {
+		netdev_link_down ( netdev );
+	}
+}
+
+/******************************************************************************
+ *
+ * Mailbox messages
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Send negotiate API version message
+ *
+ * @v intel		Intel device
+ * @v version		Requested version
+ * @ret rc		Return status code
+ */
+static int intelxvf_mbox_version ( struct intel_nic *intel,
+				   unsigned int version ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send set MTU message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELXVF_MSG_TYPE_VERSION;
+	msg.version.version = version;
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p negotiate API version failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELXVF_MSG_TYPE_VERSION ){
+		DBGC ( intel, "INTEL %p negotiate API version unexpected "
+		       "response:\n", intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Check that this version is supported */
+	if ( ! ( msg.hdr & INTELVF_MSG_ACK ) ) {
+		DBGC ( intel, "INTEL %p negotiate API version failed\n",
+		       intel );
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int intelxvf_open ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t srrctl;
+	uint32_t dca_rxctrl;
+	int rc;
+
+	/* Reset the function */
+	intelxvf_reset ( intel );
+
+	/* Notify PF that reset is complete */
+	if ( ( rc = intelvf_mbox_reset ( intel, NULL ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p could not reset: %s\n",
+		       intel, strerror ( rc ) );
+		goto err_mbox_reset;
+	}
+
+	/* Negotiate API version 1.1.  If we do not negotiate at least
+	 * this version, then the RX datapath will remain disabled if
+	 * the PF has jumbo frames enabled.
+	 *
+	 * Ignore failures, since the host may not actually support
+	 * v1.1.
+	 */
+	intelxvf_mbox_version ( intel, INTELXVF_MSG_VERSION_1_1 );
+
+	/* Set MAC address */
+	if ( ( rc = intelvf_mbox_set_mac ( intel, netdev->ll_addr ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p could not set MAC address: %s\n",
+		       intel, strerror ( rc ) );
+		goto err_mbox_set_mac;
+	}
+
+	/* Set MTU */
+	if ( ( rc = intelvf_mbox_set_mtu ( intel, netdev->max_pkt_len ) ) != 0){
+		DBGC ( intel, "INTEL %p could not set MTU %zd: %s\n",
+		       intel, netdev->max_pkt_len, strerror ( rc ) );
+		goto err_mbox_set_mtu;
+	}
+
+	/* Create transmit descriptor ring */
+	if ( ( rc = intel_create_ring ( intel, &intel->tx ) ) != 0 )
+		goto err_create_tx;
+
+	/* Create receive descriptor ring */
+	if ( ( rc = intel_create_ring ( intel, &intel->rx ) ) != 0 )
+		goto err_create_rx;
+
+	/* Allocate interrupt vectors */
+	writel ( ( INTELXVF_IVAR_RX0_DEFAULT | INTELXVF_IVAR_RX0_VALID |
+		   INTELXVF_IVAR_TX0_DEFAULT | INTELXVF_IVAR_TX0_VALID ),
+		 intel->regs + INTELXVF_IVAR );
+	writel ( ( INTELXVF_IVARM_MBOX_DEFAULT | INTELXVF_IVARM_MBOX_VALID ),
+		 intel->regs + INTELXVF_IVARM );
+
+	/* Configure receive buffer sizes and set receive descriptor type */
+	srrctl = readl ( intel->regs + INTELXVF_SRRCTL );
+	srrctl &= ~( INTELXVF_SRRCTL_BSIZE_MASK |
+		     INTELXVF_SRRCTL_DESCTYPE_MASK );
+	srrctl |= ( INTELXVF_SRRCTL_BSIZE_DEFAULT |
+		    INTELXVF_SRRCTL_DESCTYPE_DEFAULT );
+	writel ( srrctl, intel->regs + INTELXVF_SRRCTL );
+
+	/* Clear "must-be-zero" bit for direct cache access (DCA).  We
+	 * leave DCA disabled anyway, but if we do not clear this bit
+	 * then the received packets contain garbage data.
+	 */
+	dca_rxctrl = readl ( intel->regs + INTELXVF_DCA_RXCTRL );
+	dca_rxctrl &= ~INTELXVF_DCA_RXCTRL_MUST_BE_ZERO;
+	writel ( dca_rxctrl, intel->regs + INTELXVF_DCA_RXCTRL );
+
+	/* Fill receive ring */
+	intel_refill_rx ( intel );
+
+	/* Update link state */
+	intelxvf_check_link ( netdev );
+
+	return 0;
+
+	intel_destroy_ring ( intel, &intel->rx );
+ err_create_rx:
+	intel_destroy_ring ( intel, &intel->tx );
+ err_create_tx:
+ err_mbox_set_mtu:
+ err_mbox_set_mac:
+ err_mbox_reset:
+	intelxvf_reset ( intel );
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void intelxvf_close ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+
+	/* Destroy receive descriptor ring */
+	intel_destroy_ring ( intel, &intel->rx );
+
+	/* Discard any unused receive buffers */
+	intel_empty_rx ( intel );
+
+	/* Destroy transmit descriptor ring */
+	intel_destroy_ring ( intel, &intel->tx );
+
+	/* Reset the function */
+	intelxvf_reset ( intel );
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void intelxvf_poll ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t eicr;
+	int rc;
+
+	/* Check for and acknowledge interrupts */
+	eicr = readl ( intel->regs + INTELXVF_EICR );
+	if ( ! eicr )
+		return;
+
+	/* Poll for TX completions, if applicable */
+	if ( eicr & INTELXVF_EIRQ_TX0 )
+		intel_poll_tx ( netdev );
+
+	/* Poll for RX completions, if applicable */
+	if ( eicr & INTELXVF_EIRQ_RX0 )
+		intel_poll_rx ( netdev );
+
+	/* Poll for mailbox messages, if applicable */
+	if ( eicr & INTELXVF_EIRQ_MBOX ) {
+
+		/* Poll mailbox */
+		if ( ( rc = intelvf_mbox_poll ( intel ) ) != 0 ) {
+			DBGC ( intel, "INTEL %p mailbox poll failed!\n",
+			       intel );
+			netdev_rx_err ( netdev, NULL, rc );
+		}
+
+		/* Update link state */
+		intelxvf_check_link ( netdev );
+	}
+
+	/* Refill RX ring */
+	intel_refill_rx ( intel );
+}
+
+/**
+ * Enable or disable interrupts
+ *
+ * @v netdev		Network device
+ * @v enable		Interrupts should be enabled
+ */
+static void intelxvf_irq ( struct net_device *netdev, int enable ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t mask;
+
+	mask = ( INTELXVF_EIRQ_MBOX | INTELXVF_EIRQ_TX0 | INTELXVF_EIRQ_RX0 );
+	if ( enable ) {
+		writel ( mask, intel->regs + INTELXVF_EIMS );
+	} else {
+		writel ( mask, intel->regs + INTELXVF_EIMC );
+	}
+}
+
+/** Network device operations */
+static struct net_device_operations intelxvf_operations = {
+	.open		= intelxvf_open,
+	.close		= intelxvf_close,
+	.transmit	= intel_transmit,
+	.poll		= intelxvf_poll,
+	.irq		= intelxvf_irq,
+};
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int intelxvf_probe ( struct pci_device *pci ) {
+	struct net_device *netdev;
+	struct intel_nic *intel;
+	int rc;
+
+	/* Allocate and initialise net device */
+	netdev = alloc_etherdev ( sizeof ( *intel ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &intelxvf_operations );
+	intel = netdev->priv;
+	pci_set_drvdata ( pci, netdev );
+	netdev->dev = &pci->dev;
+	memset ( intel, 0, sizeof ( *intel ) );
+	intel_init_mbox ( &intel->mbox, INTELXVF_MBCTRL, INTELXVF_MBMEM );
+	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTELXVF_TD,
+			  intel_describe_tx_adv );
+	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTELXVF_RD,
+			  intel_describe_rx );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Map registers */
+	intel->regs = ioremap ( pci->membase, INTELVF_BAR_SIZE );
+	if ( ! intel->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Reset the function */
+	intelxvf_reset ( intel );
+
+	/* Send reset message and fetch MAC address */
+	if ( ( rc = intelvf_mbox_reset ( intel, netdev->hw_addr ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p could not reset and fetch MAC: %s\n",
+		       intel, strerror ( rc ) );
+		goto err_mbox_reset;
+	}
+
+	/* Reset the function (since we will not respond to Control
+	 * ("ping") mailbox messages until the network device is opened.
+	 */
+	intelxvf_reset ( intel );
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register_netdev;
+
+	/* Set initial link state */
+	intelxvf_check_link ( netdev );
+
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register_netdev:
+ err_mbox_reset:
+	intelxvf_reset ( intel );
+	iounmap ( intel->regs );
+ err_ioremap:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void intelxvf_remove ( struct pci_device *pci ) {
+	struct net_device *netdev = pci_get_drvdata ( pci );
+	struct intel_nic *intel = netdev->priv;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+
+	/* Reset the NIC */
+	intelxvf_reset ( intel );
+
+	/* Free network device */
+	iounmap ( intel->regs );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** PCI device IDs */
+static struct pci_device_id intelxvf_nics[] = {
+	PCI_ROM ( 0x8086, 0x10ed, "82599-vf", "82599 VF", 0 ),
+	PCI_ROM ( 0x8086, 0x1515, "x540-vf", "X540 VF", 0 ),
+	PCI_ROM ( 0x8086, 0x1565, "x550-vf", "X550 VF", 0 ),
+	PCI_ROM ( 0x8086, 0x15a8, "x552-vf", "X552 VF", 0 ),
+};
+
+/** PCI driver */
+struct pci_driver intelxvf_driver __pci_driver = {
+	.ids = intelxvf_nics,
+	.id_count = ( sizeof ( intelxvf_nics ) / sizeof ( intelxvf_nics[0] ) ),
+	.probe = intelxvf_probe,
+	.remove = intelxvf_remove,
+};
diff --git a/roms/ipxe/src/drivers/net/intelxvf.h b/roms/ipxe/src/drivers/net/intelxvf.h
new file mode 100644
index 000000000..ad046a65c
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/intelxvf.h
@@ -0,0 +1,104 @@
+#ifndef _INTELXVF_H
+#define _INTELXVF_H
+
+/** @file
+ *
+ * Intel 10 Gigabit Ethernet virtual function network card driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include "intelvf.h"
+
+/** Control Register */
+#define INTELXVF_CTRL 0x0000UL
+#define INTELXVF_CTRL_RST	0x04000000UL	/**< Function-level reset */
+
+/** Link Status Register */
+#define INTELXVF_LINKS 0x0010UL
+#define INTELXVF_LINKS_UP	0x40000000UL	/**< Link up */
+
+/** Extended Interrupt Cause Read Register */
+#define INTELXVF_EICR 0x0100UL
+#define INTELXVF_EIRQ_RX0	0x00000001UL	/**< RX queue 0 (via IVAR) */
+#define INTELXVF_EIRQ_TX0	0x00000002UL	/**< TX queue 0 (via IVAR) */
+#define INTELXVF_EIRQ_MBOX	0x00000004UL	/**< Mailbox (via IVARM) */
+
+/** Extended Interrupt Mask Set/Read Register */
+#define INTELXVF_EIMS 0x0108UL
+
+/** Extended Interrupt Mask Clear Register */
+#define INTELXVF_EIMC 0x010cUL
+
+/** Interrupt Vector Allocation Register */
+#define INTELXVF_IVAR 0x0120UL
+#define INTELXVF_IVAR_RX0(bit)	( (bit) << 0 )	/**< RX queue 0 allocation */
+#define INTELXVF_IVAR_RX0_DEFAULT INTELXVF_IVAR_RX0 ( 0x00 )
+#define INTELXVF_IVAR_RX0_MASK	INTELXVF_IVAR_RX0 ( 0x01 )
+#define INTELXVF_IVAR_RX0_VALID	0x00000080UL	/**< RX queue 0 valid */
+#define INTELXVF_IVAR_TX0(bit)	( (bit) << 8 )	/**< TX queue 0 allocation */
+#define INTELXVF_IVAR_TX0_DEFAULT INTELXVF_IVAR_TX0 ( 0x01 )
+#define INTELXVF_IVAR_TX0_MASK	INTELXVF_IVAR_TX0 ( 0x01 )
+#define INTELXVF_IVAR_TX0_VALID	0x00008000UL	/**< TX queue 0 valid */
+
+/** Interrupt Vector Allocation Miscellaneous Register */
+#define INTELXVF_IVARM 0x0140UL
+#define INTELXVF_IVARM_MBOX(bit) ( (bit) << 0 )	/**< Mailbox allocation */
+#define INTELXVF_IVARM_MBOX_DEFAULT INTELXVF_IVARM_MBOX ( 0x02 )
+#define INTELXVF_IVARM_MBOX_MASK INTELXVF_IVARM_MBOX ( 0x03 )
+#define INTELXVF_IVARM_MBOX_VALID 0x00000080UL	/**< Mailbox valid */
+
+/** Mailbox Memory Register Base */
+#define INTELXVF_MBMEM 0x0200UL
+
+/** Mailbox Control Register */
+#define INTELXVF_MBCTRL 0x02fcUL
+
+/** Receive Descriptor register block */
+#define INTELXVF_RD 0x1000UL
+
+/** RX DCA Control Register */
+#define INTELXVF_DCA_RXCTRL 0x100cUL
+#define INTELXVF_DCA_RXCTRL_MUST_BE_ZERO 0x00001000UL /**< Must be zero */
+
+/** Split Receive Control Register */
+#define INTELXVF_SRRCTL 0x1014UL
+#define INTELXVF_SRRCTL_BSIZE(kb) ( (kb) << 0 )	/**< Receive buffer size */
+#define INTELXVF_SRRCTL_BSIZE_DEFAULT INTELXVF_SRRCTL_BSIZE ( 0x02 )
+#define INTELXVF_SRRCTL_BSIZE_MASK INTELXVF_SRRCTL_BSIZE ( 0x1f )
+#define INTELXVF_SRRCTL_DESCTYPE(typ) ( (typ) << 25 ) /**< Descriptor type */
+#define INTELXVF_SRRCTL_DESCTYPE_DEFAULT INTELXVF_SRRCTL_DESCTYPE ( 0x00 )
+#define INTELXVF_SRRCTL_DESCTYPE_MASK INTELXVF_SRRCTL_DESCTYPE ( 0x07 )
+
+/** Good Packets Received Count */
+#define INTELXVF_GPRC 0x101c
+
+/** Good Packets Received Count Low */
+#define INTELXVF_GORCL 0x1020
+
+/** Good Packets Received Count High */
+#define INTELXVF_GORCH 0x1024
+
+/* Multicast Packets Received Count */
+#define INTELXVF_MPRC 0x1034
+
+/** Transmit Descriptor register block */
+#define INTELXVF_TD 0x2000UL
+
+/** Good Packets Transmitted Count */
+#define INTELXVF_GPTC 0x201c
+
+/** Good Packets Transmitted Count Low */
+#define INTELXVF_GOTCL 0x2020
+
+/** Good Packets Transmitted Count High */
+#define INTELXVF_GOTCH 0x2024
+
+/** Negotiate API version mailbox message */
+#define INTELXVF_MSG_TYPE_VERSION 0x00000008UL
+
+/** API version 1.1 */
+#define INTELXVF_MSG_VERSION_1_1 0x00000002UL
+
+#endif /* _INTELXVF_H */
diff --git a/roms/ipxe/src/drivers/net/ipoib.c b/roms/ipxe/src/drivers/net/ipoib.c
index 1b5391776..6552d764e 100644
--- a/roms/ipxe/src/drivers/net/ipoib.c
+++ b/roms/ipxe/src/drivers/net/ipoib.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -29,8 +33,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/errortab.h>
 #include <ipxe/malloc.h>
 #include <ipxe/if_arp.h>
+#include <ipxe/arp.h>
 #include <ipxe/if_ether.h>
 #include <ipxe/ethernet.h>
+#include <ipxe/ip.h>
 #include <ipxe/iobuf.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/infiniband.h>
@@ -44,6 +50,20 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * IP over Infiniband
  */
 
+/* Disambiguate the various error causes */
+#define ENXIO_ARP_REPLY __einfo_error ( EINFO_ENXIO_ARP_REPLY )
+#define EINFO_ENXIO_ARP_REPLY						\
+	__einfo_uniqify ( EINFO_ENXIO, 0x01,				\
+			  "Missing REMAC for ARP reply target address" )
+#define ENXIO_NON_IPV4 __einfo_error ( EINFO_ENXIO_NON_IPV4 )
+#define EINFO_ENXIO_NON_IPV4						\
+	__einfo_uniqify ( EINFO_ENXIO, 0x02,				\
+			  "Missing REMAC for non-IPv4 packet" )
+#define ENXIO_ARP_SENT __einfo_error ( EINFO_ENXIO_ARP_SENT )
+#define EINFO_ENXIO_ARP_SENT						\
+	__einfo_uniqify ( EINFO_ENXIO, 0x03,				\
+			  "Missing REMAC for IPv4 packet (ARP sent)" )
+
 /** Number of IPoIB send work queue entries */
 #define IPOIB_NUM_SEND_WQES 2
 
@@ -96,6 +116,8 @@ struct errortab ipoib_errors[] __errortab = {
 	__einfo_errortab ( EINFO_EINPROGRESS_JOINING ),
 };
 
+static struct net_device_operations ipoib_operations;
+
 /****************************************************************************
  *
  * IPoIB REMAC cache
@@ -124,8 +146,10 @@ static struct ipoib_mac * ipoib_find_remac ( struct ipoib_device *ipoib,
 					     const struct ipoib_remac *remac ) {
 	struct ipoib_peer *peer;
 
-	/* Check for broadcast REMAC */
-	if ( is_broadcast_ether_addr ( remac ) )
+	/* Check for broadcast or multicast REMAC.  We transmit
+	 * multicasts as broadcasts for simplicity.
+	 */
+	if ( is_multicast_ether_addr ( remac ) )
 		return &ipoib->broadcast;
 
 	/* Try to find via REMAC cache */
@@ -202,14 +226,20 @@ static void ipoib_flush_remac ( struct ipoib_device *ipoib ) {
  * @ret discarded	Number of cached items discarded
  */
 static unsigned int ipoib_discard_remac ( void ) {
-	struct ib_device *ibdev;
+	struct net_device *netdev;
 	struct ipoib_device *ipoib;
 	struct ipoib_peer *peer;
 	unsigned int discarded = 0;
 
 	/* Try to discard one cache entry for each IPoIB device */
-	for_each_ibdev ( ibdev ) {
-		ipoib = ib_get_ownerdata ( ibdev );
+	for_each_netdev ( netdev ) {
+
+		/* Skip non-IPoIB devices */
+		if ( netdev->op != &ipoib_operations )
+			continue;
+		ipoib = netdev->priv;
+
+		/* Discard least recently used cache entry (if any) */
 		list_for_each_entry_reverse ( peer, &ipoib->peers, list ) {
 			list_del ( &peer->list );
 			free ( peer );
@@ -222,7 +252,7 @@ static unsigned int ipoib_discard_remac ( void ) {
 }
 
 /** IPoIB cache discarder */
-struct cache_discarder ipoib_discarder __cache_discarder ( CACHE_NORMAL ) = {
+struct cache_discarder ipoib_discarder __cache_discarder ( CACHE_EXPENSIVE ) = {
 	.discard = ipoib_discard_remac,
 };
 
@@ -324,8 +354,11 @@ static int ipoib_translate_tx_arp ( struct net_device *netdev,
 	/* Look up REMAC, if applicable */
 	if ( arphdr->ar_op == ARPOP_REPLY ) {
 		target_ha = ipoib_find_remac ( ipoib, arp_target_pa ( arphdr ));
-		if ( ! target_ha )
-			return -ENXIO;
+		if ( ! target_ha ) {
+			DBGC ( ipoib, "IPoIB %p no REMAC for %s ARP reply\n",
+			       ipoib, eth_ntoa ( arp_target_pa ( arphdr ) ) );
+			return -ENXIO_ARP_REPLY;
+		}
 	}
 
 	/* Construct new packet */
@@ -461,6 +494,7 @@ static int ipoib_transmit ( struct net_device *netdev,
 	struct ipoib_device *ipoib = netdev->priv;
 	struct ib_device *ibdev = ipoib->ibdev;
 	struct ethhdr *ethhdr;
+	struct iphdr *iphdr;
 	struct ipoib_hdr *ipoib_hdr;
 	struct ipoib_mac *mac;
 	struct ib_address_vector dest;
@@ -485,9 +519,34 @@ static int ipoib_transmit ( struct net_device *netdev,
 	iob_pull ( iobuf, sizeof ( *ethhdr ) );
 
 	/* Identify destination address */
-	mac = ipoib_find_remac ( ipoib, ( ( void *) ethhdr->h_dest ) );
-	if ( ! mac )
-		return -ENXIO;
+	mac = ipoib_find_remac ( ipoib, ( ( void * ) ethhdr->h_dest ) );
+	if ( ! mac ) {
+		/* Generate a new ARP request (if possible) to trigger
+		 * population of the REMAC cache entry.
+		 */
+		if ( ( net_proto != htons ( ETH_P_IP ) ) ||
+		     ( iob_len ( iobuf ) < sizeof ( *iphdr ) ) ) {
+			DBGC ( ipoib, "IPoIB %p no REMAC for %s non-IPv4 "
+			       "packet type %04x\n", ipoib,
+			       eth_ntoa ( ethhdr->h_dest ),
+			       ntohs ( net_proto ) );
+			return -ENXIO_NON_IPV4;
+		}
+		iphdr = iobuf->data;
+		if ( ( rc = arp_tx_request ( netdev, &ipv4_protocol,
+					     &iphdr->dest, &iphdr->src ) ) !=0){
+			DBGC ( ipoib, "IPoIB %p could not ARP for %s/%s/",
+			       ipoib, eth_ntoa ( ethhdr->h_dest ),
+			       inet_ntoa ( iphdr->dest ) );
+			DBGC ( ipoib, "%s: %s\n", inet_ntoa ( iphdr->src ),
+			       strerror ( rc ) );
+			return rc;
+		}
+		DBGC ( ipoib, "IPoIB %p no REMAC for %s/%s/", ipoib,
+		       eth_ntoa ( ethhdr->h_dest ), inet_ntoa ( iphdr->dest ) );
+		DBGC  ( ipoib, "%s\n", inet_ntoa ( iphdr->src ) );
+		return -ENXIO_ARP_SENT;
+	}
 
 	/* Translate packet if applicable */
 	if ( ( rc = ipoib_translate_tx ( netdev, iobuf, net_proto ) ) != 0 )
@@ -732,7 +791,8 @@ static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
 	int rc;
 
 	/* Leave existing broadcast group */
-	ipoib_leave_broadcast_group ( ipoib );
+	if ( ipoib->qp )
+		ipoib_leave_broadcast_group ( ipoib );
 
 	/* Update MAC address based on potentially-new GID prefix */
 	memcpy ( &ipoib->mac.gid.s.prefix, &ibdev->gid.s.prefix,
@@ -747,7 +807,7 @@ static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
 	netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
 
 	/* Join new broadcast group */
-	if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) &&
+	if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) && ipoib->qp &&
 	     ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
 		DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
 		       "%s\n", ipoib, strerror ( rc ) );
@@ -835,7 +895,9 @@ static void ipoib_close ( struct net_device *netdev ) {
 
 	/* Tear down the queues */
 	ib_destroy_qp ( ibdev, ipoib->qp );
+	ipoib->qp = NULL;
 	ib_destroy_cq ( ibdev, ipoib->cq );
+	ipoib->cq = NULL;
 
 	/* Close IB device */
 	ib_close ( ibdev );
diff --git a/roms/ipxe/src/drivers/net/legacy.c b/roms/ipxe/src/drivers/net/legacy.c
index 4edbef162..73a80194f 100644
--- a/roms/ipxe/src/drivers/net/legacy.c
+++ b/roms/ipxe/src/drivers/net/legacy.c
@@ -17,7 +17,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct nic nic;
 
diff --git a/roms/ipxe/src/drivers/net/mii.c b/roms/ipxe/src/drivers/net/mii.c
index c4d32514d..9b297029a 100644
--- a/roms/ipxe/src/drivers/net/mii.c
+++ b/roms/ipxe/src/drivers/net/mii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <unistd.h>
@@ -111,3 +115,35 @@ int mii_reset ( struct mii_interface *mii ) {
 	DBGC ( mii, "MII %p timed out waiting for reset\n", mii );
 	return -ETIMEDOUT;
 }
+
+/**
+ * Update link status via MII
+ *
+ * @v mii		MII interface
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+int mii_check_link ( struct mii_interface *mii, struct net_device *netdev ) {
+	int bmsr;
+	int link;
+	int rc;
+
+	/* Read BMSR */
+	bmsr = mii_read ( mii, MII_BMSR );
+	if ( bmsr < 0 ) {
+		rc = bmsr;
+		return rc;
+	}
+
+	/* Report link status */
+	link = ( bmsr & BMSR_LSTATUS );
+	DBGC ( mii, "MII %p link %s (BMSR %#04x)\n",
+	       mii, ( link ? "up" : "down" ), bmsr );
+	if ( link ) {
+		netdev_link_up ( netdev );
+	} else {
+		netdev_link_down ( netdev );
+	}
+
+	return 0;
+}
diff --git a/roms/ipxe/src/drivers/net/myson.c b/roms/ipxe/src/drivers/net/myson.c
index 6abb55660..84a550596 100644
--- a/roms/ipxe/src/drivers/net/myson.c
+++ b/roms/ipxe/src/drivers/net/myson.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/drivers/net/myson.h b/roms/ipxe/src/drivers/net/myson.h
index 8d7cc5855..05a6b8a58 100644
--- a/roms/ipxe/src/drivers/net/myson.h
+++ b/roms/ipxe/src/drivers/net/myson.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/if_ether.h>
diff --git a/roms/ipxe/src/drivers/net/ncm.c b/roms/ipxe/src/drivers/net/ncm.c
new file mode 100644
index 000000000..10728d2a1
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/ncm.c
@@ -0,0 +1,672 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/profile.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include "ecm.h"
+#include "ncm.h"
+
+/** @file
+ *
+ * CDC-NCM USB Ethernet driver
+ *
+ */
+
+/** Interrupt completion profiler */
+static struct profiler ncm_intr_profiler __profiler =
+	{ .name = "ncm.intr" };
+
+/** Bulk IN completion profiler */
+static struct profiler ncm_in_profiler __profiler =
+	{ .name = "ncm.in" };
+
+/** Bulk IN per-datagram profiler */
+static struct profiler ncm_in_datagram_profiler __profiler =
+	{ .name = "ncm.in_dgram" };
+
+/** Bulk OUT profiler */
+static struct profiler ncm_out_profiler __profiler =
+	{ .name = "ncm.out" };
+
+/******************************************************************************
+ *
+ * CDC-NCM communications interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ncm_intr_complete ( struct usb_endpoint *ep,
+				struct io_buffer *iobuf, int rc ) {
+	struct ncm_device *ncm = container_of ( ep, struct ncm_device,
+						usbnet.intr );
+	struct net_device *netdev = ncm->netdev;
+	struct usb_setup_packet *message;
+	size_t len = iob_len ( iobuf );
+
+	/* Profile completions */
+	profile_start ( &ncm_intr_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Ignore packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( ncm, "NCM %p interrupt failed: %s\n",
+		       ncm, strerror ( rc ) );
+		DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+		goto error;
+	}
+
+	/* Extract message header */
+	if ( len < sizeof ( *message ) ) {
+		DBGC ( ncm, "NCM %p underlength interrupt:\n", ncm );
+		DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto error;
+	}
+	message = iobuf->data;
+
+	/* Parse message header */
+	switch ( message->request ) {
+
+	case cpu_to_le16 ( CDC_NETWORK_CONNECTION ) :
+		if ( message->value ) {
+			DBGC ( ncm, "NCM %p link up\n", ncm );
+			netdev_link_up ( netdev );
+		} else {
+			DBGC ( ncm, "NCM %p link down\n", ncm );
+			netdev_link_down ( netdev );
+		}
+		break;
+
+	case cpu_to_le16 ( CDC_CONNECTION_SPEED_CHANGE ) :
+		/* Ignore */
+		break;
+
+	default:
+		DBGC ( ncm, "NCM %p unrecognised interrupt:\n", ncm );
+		DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+		goto error;
+	}
+
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+	profile_stop ( &ncm_intr_profiler );
+
+	return;
+
+ error:
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+ ignore:
+	free_iob ( iobuf );
+	return;
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations ncm_intr_operations = {
+	.complete = ncm_intr_complete,
+};
+
+/******************************************************************************
+ *
+ * CDC-NCM data interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Prefill bulk IN endpoint
+ *
+ * @v ncm		CDC-NCM device
+ * @ret rc		Return status code
+ */
+static int ncm_in_prefill ( struct ncm_device *ncm ) {
+	struct usb_bus *bus = ncm->bus;
+	size_t mtu;
+	unsigned int count;
+	int rc;
+
+	/* Some devices have a very small number of internal buffers,
+	 * and rely on being able to pack multiple packets into each
+	 * buffer.  We therefore want to use large buffers if
+	 * possible.  However, large allocations have a reasonable
+	 * chance of failure, especially if this is not the first or
+	 * only device to be opened.
+	 *
+	 * We therefore attempt to find a usable buffer size, starting
+	 * large and working downwards until allocation succeeds.
+	 * Smaller buffers will still work, albeit with a higher
+	 * chance of packet loss and so lower overall throughput.
+	 */
+	for ( mtu = ncm->mtu ; mtu >= NCM_MIN_NTB_INPUT_SIZE ; mtu >>= 1 ) {
+
+		/* Attempt allocation at this MTU */
+		if ( mtu > NCM_MAX_NTB_INPUT_SIZE )
+			continue;
+		if ( mtu > bus->mtu )
+			continue;
+		count = ( NCM_IN_MIN_SIZE / mtu );
+		if ( count < NCM_IN_MIN_COUNT )
+			count = NCM_IN_MIN_COUNT;
+		if ( ( count * mtu ) > NCM_IN_MAX_SIZE )
+			continue;
+		usb_refill_init ( &ncm->usbnet.in, mtu, count );
+		if ( ( rc = usb_prefill ( &ncm->usbnet.in ) ) != 0 ) {
+			DBGC ( ncm, "NCM %p could not prefill %dx %zd-byte "
+			       "buffers for bulk IN\n", ncm, count, mtu );
+			continue;
+		}
+
+		DBGC ( ncm, "NCM %p using %dx %zd-byte buffers for bulk IN\n",
+		       ncm, count, mtu );
+		return 0;
+	}
+
+	DBGC ( ncm, "NCM %p could not prefill bulk IN endpoint\n", ncm );
+	return -ENOMEM;
+}
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ncm_in_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			      int rc ) {
+	struct ncm_device *ncm = container_of ( ep, struct ncm_device,
+						usbnet.in );
+	struct net_device *netdev = ncm->netdev;
+	struct ncm_transfer_header *nth;
+	struct ncm_datagram_pointer *ndp;
+	struct ncm_datagram_descriptor *desc;
+	struct io_buffer *pkt;
+	unsigned int remaining;
+	size_t ndp_offset;
+	size_t ndp_len;
+	size_t pkt_offset;
+	size_t pkt_len;
+	size_t headroom;
+	size_t len;
+
+	/* Profile overall bulk IN completion */
+	profile_start ( &ncm_in_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( ncm, "NCM %p bulk IN failed: %s\n",
+		       ncm, strerror ( rc ) );
+		goto error;
+	}
+
+	/* Locate transfer header */
+	len = iob_len ( iobuf );
+	if ( sizeof ( *nth ) > len ) {
+		DBGC ( ncm, "NCM %p packet too short for NTH:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+	nth = iobuf->data;
+
+	/* Locate datagram pointer */
+	ndp_offset = le16_to_cpu ( nth->offset );
+	if ( ( ndp_offset + sizeof ( *ndp ) ) > len ) {
+		DBGC ( ncm, "NCM %p packet too short for NDP:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+	ndp = ( iobuf->data + ndp_offset );
+	ndp_len = le16_to_cpu ( ndp->header_len );
+	if ( ndp_len < offsetof ( typeof ( *ndp ), desc ) ) {
+		DBGC ( ncm, "NCM %p NDP header length too short:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+	if ( ( ndp_offset + ndp_len ) > len ) {
+		DBGC ( ncm, "NCM %p packet too short for NDP:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+
+	/* Process datagrams */
+	remaining = ( ( ndp_len - offsetof ( typeof ( *ndp ), desc ) ) /
+		      sizeof ( ndp->desc[0] ) );
+	for ( desc = ndp->desc ; remaining && desc->offset ; remaining-- ) {
+
+		/* Profile individual datagrams */
+		profile_start ( &ncm_in_datagram_profiler );
+
+		/* Locate datagram */
+		pkt_offset = le16_to_cpu ( desc->offset );
+		pkt_len = le16_to_cpu ( desc->len );
+		if ( pkt_len < ETH_HLEN ) {
+			DBGC ( ncm, "NCM %p underlength datagram:\n", ncm );
+			rc = -EINVAL;
+			goto error;
+		}
+		if ( ( pkt_offset + pkt_len ) > len ) {
+			DBGC ( ncm, "NCM %p datagram exceeds packet:\n", ncm );
+			rc = -EINVAL;
+			goto error;
+		}
+
+		/* Move to next descriptor */
+		desc++;
+
+		/* Copy data to a new I/O buffer.  Our USB buffers may
+		 * be very large and so we choose to recycle the
+		 * buffers directly rather than attempt reallocation
+		 * while the device is running.  We therefore copy the
+		 * data to a new I/O buffer even if this is the only
+		 * (or last) packet within the buffer.
+		 *
+		 * We reserve enough space at the start of each buffer
+		 * to allow for our own transmission header, to
+		 * support protocols such as ARP which may modify the
+		 * received packet and reuse the same I/O buffer for
+		 * transmission.
+		 */
+		headroom = ( sizeof ( struct ncm_ntb_header ) + ncm->padding );
+		pkt = alloc_iob ( headroom + pkt_len );
+		if ( ! pkt ) {
+			/* Record error and continue */
+			netdev_rx_err ( netdev, NULL, -ENOMEM );
+			continue;
+		}
+		iob_reserve ( pkt, headroom );
+		memcpy ( iob_put ( pkt, pkt_len ),
+			 ( iobuf->data + pkt_offset ), pkt_len );
+
+		/* Strip CRC, if present */
+		if ( ndp->magic & cpu_to_le32 ( NCM_DATAGRAM_POINTER_MAGIC_CRC))
+			iob_unput ( pkt, 4 /* CRC32 */ );
+
+		/* Hand off to network stack */
+		netdev_rx ( netdev, pkt );
+		profile_stop ( &ncm_in_datagram_profiler );
+	}
+
+	/* Recycle I/O buffer */
+	usb_recycle ( &ncm->usbnet.in, iobuf );
+	profile_stop ( &ncm_in_profiler );
+
+	return;
+
+ error:
+	/* Record error against network device */
+	DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+	netdev_rx_err ( netdev, NULL, rc );
+ ignore:
+	usb_recycle ( &ncm->usbnet.in, iobuf );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations ncm_in_operations = {
+	.complete = ncm_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v ncm		CDC-NCM device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ncm_out_transmit ( struct ncm_device *ncm,
+			      struct io_buffer *iobuf ) {
+	struct ncm_ntb_header *header;
+	size_t len = iob_len ( iobuf );
+	size_t header_len = ( sizeof ( *header ) + ncm->padding );
+	int rc;
+
+	/* Profile transmissions */
+	profile_start ( &ncm_out_profiler );
+
+	/* Prepend header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, header_len ) ) != 0 )
+		return rc;
+	header = iob_push ( iobuf, header_len );
+
+	/* Populate header */
+	header->nth.magic = cpu_to_le32 ( NCM_TRANSFER_HEADER_MAGIC );
+	header->nth.header_len = cpu_to_le16 ( sizeof ( header->nth ) );
+	header->nth.sequence = cpu_to_le16 ( ncm->sequence );
+	header->nth.len = cpu_to_le16 ( iob_len ( iobuf ) );
+	header->nth.offset =
+		cpu_to_le16 ( offsetof ( typeof ( *header ), ndp ) );
+	header->ndp.magic = cpu_to_le32 ( NCM_DATAGRAM_POINTER_MAGIC );
+	header->ndp.header_len = cpu_to_le16 ( sizeof ( header->ndp ) +
+					       sizeof ( header->desc ) );
+	header->ndp.offset = cpu_to_le16 ( 0 );
+	header->desc[0].offset = cpu_to_le16 ( header_len );
+	header->desc[0].len = cpu_to_le16 ( len );
+	memset ( &header->desc[1], 0, sizeof ( header->desc[1] ) );
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &ncm->usbnet.out, iobuf, 0 ) ) != 0 )
+		return rc;
+
+	/* Increment sequence number */
+	ncm->sequence++;
+
+	profile_stop ( &ncm_out_profiler );
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ncm_out_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			       int rc ) {
+	struct ncm_device *ncm = container_of ( ep, struct ncm_device,
+						usbnet.out );
+	struct net_device *netdev = ncm->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations ncm_out_operations = {
+	.complete = ncm_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int ncm_open ( struct net_device *netdev ) {
+	struct ncm_device *ncm = netdev->priv;
+	struct usb_device *usb = ncm->usb;
+	struct ncm_set_ntb_input_size size;
+	int rc;
+
+	/* Reset sequence number */
+	ncm->sequence = 0;
+
+	/* Prefill I/O buffers */
+	if ( ( rc = ncm_in_prefill ( ncm ) ) != 0 )
+		goto err_prefill;
+
+	/* Set maximum input size */
+	memset ( &size, 0, sizeof ( size ) );
+	size.mtu = cpu_to_le32 ( ncm->usbnet.in.len );
+	if ( ( rc = usb_control ( usb, NCM_SET_NTB_INPUT_SIZE, 0,
+				  ncm->usbnet.comms, &size,
+				  sizeof ( size ) ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not set input size to %zd: %s\n",
+		       ncm, ncm->usbnet.in.len, strerror ( rc ) );
+		goto err_set_ntb_input_size;
+	}
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &ncm->usbnet ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not open: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_open;
+	}
+
+	return 0;
+
+	usbnet_close ( &ncm->usbnet );
+ err_open:
+ err_set_ntb_input_size:
+	usb_flush ( &ncm->usbnet.in );
+ err_prefill:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void ncm_close ( struct net_device *netdev ) {
+	struct ncm_device *ncm = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &ncm->usbnet );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ncm_transmit ( struct net_device *netdev,
+			  struct io_buffer *iobuf ) {
+	struct ncm_device *ncm = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = ncm_out_transmit ( ncm, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void ncm_poll ( struct net_device *netdev ) {
+	struct ncm_device *ncm = netdev->priv;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( ncm->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &ncm->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+
+}
+
+/** CDC-NCM network device operations */
+static struct net_device_operations ncm_operations = {
+	.open		= ncm_open,
+	.close		= ncm_close,
+	.transmit	= ncm_transmit,
+	.poll		= ncm_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int ncm_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct ncm_device *ncm;
+	struct usb_interface_descriptor *comms;
+	struct ecm_ethernet_descriptor *ethernet;
+	struct ncm_ntb_parameters params;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *ncm ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &ncm_operations );
+	netdev->dev = &func->dev;
+	ncm = netdev->priv;
+	memset ( ncm, 0, sizeof ( *ncm ) );
+	ncm->usb = usb;
+	ncm->bus = usb->port->hub->bus;
+	ncm->netdev = netdev;
+	usbnet_init ( &ncm->usbnet, func, &ncm_intr_operations,
+		      &ncm_in_operations, &ncm_out_operations );
+	usb_refill_init ( &ncm->usbnet.intr, 0, NCM_INTR_COUNT );
+	DBGC ( ncm, "NCM %p on %s\n", ncm, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &ncm->usbnet, config ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not describe: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Locate Ethernet descriptor */
+	comms = usb_interface_descriptor ( config, ncm->usbnet.comms, 0 );
+	assert ( comms != NULL );
+	ethernet = ecm_ethernet_descriptor ( config, comms );
+	if ( ! ethernet ) {
+		DBGC ( ncm, "NCM %p has no Ethernet descriptor\n", ncm );
+		rc = -EINVAL;
+		goto err_ethernet;
+	}
+
+	/* Fetch MAC address */
+	if ( ( rc = ecm_fetch_mac ( usb, ethernet, netdev->hw_addr ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not fetch MAC address: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_fetch_mac;
+	}
+
+	/* Get NTB parameters */
+	if ( ( rc = usb_control ( usb, NCM_GET_NTB_PARAMETERS, 0,
+				  ncm->usbnet.comms, &params,
+				  sizeof ( params ) ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not get NTB parameters: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_ntb_parameters;
+	}
+
+	/* Get maximum supported input size */
+	ncm->mtu = le32_to_cpu ( params.in.mtu );
+	DBGC2 ( ncm, "NCM %p maximum IN size is %zd bytes\n", ncm, ncm->mtu );
+
+	/* Calculate transmit padding */
+	ncm->padding = ( ( le16_to_cpu ( params.out.remainder ) -
+			   sizeof ( struct ncm_ntb_header ) - ETH_HLEN ) &
+			 ( le16_to_cpu ( params.out.divisor ) - 1 ) );
+	DBGC2 ( ncm, "NCM %p using %zd-byte transmit padding\n",
+		ncm, ncm->padding );
+	assert ( ( ( sizeof ( struct ncm_ntb_header ) + ncm->padding +
+		     ETH_HLEN ) % le16_to_cpu ( params.out.divisor ) ) ==
+		 le16_to_cpu ( params.out.remainder ) );
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, ncm );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_ntb_parameters:
+ err_fetch_mac:
+ err_ethernet:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void ncm_remove ( struct usb_function *func ) {
+	struct ncm_device *ncm = usb_func_get_drvdata ( func );
+	struct net_device *netdev = ncm->netdev;
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** CDC-NCM device IDs */
+static struct usb_device_id ncm_ids[] = {
+	{
+		.name = "cdc-ncm",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_CDC,
+			.subclass = USB_SUBCLASS_CDC_NCM,
+			.protocol = 0,
+		},
+	},
+};
+
+/** CDC-NCM driver */
+struct usb_driver ncm_driver __usb_driver = {
+	.ids = ncm_ids,
+	.id_count = ( sizeof ( ncm_ids ) / sizeof ( ncm_ids[0] ) ),
+	.probe = ncm_probe,
+	.remove = ncm_remove,
+};
diff --git a/roms/ipxe/src/drivers/net/ncm.h b/roms/ipxe/src/drivers/net/ncm.h
new file mode 100644
index 000000000..a9565a56b
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/ncm.h
@@ -0,0 +1,173 @@
+#ifndef _NCM_H
+#define _NCM_H
+
+/** @file
+ *
+ * CDC-NCM USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/usb.h>
+#include <ipxe/cdc.h>
+#include <byteswap.h>
+#include "ecm.h"
+
+/** CDC-NCM subclass */
+#define USB_SUBCLASS_CDC_NCM 0x0d
+
+/** Get NTB parameters */
+#define NCM_GET_NTB_PARAMETERS						\
+	( USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x80 ) )
+
+/** NTB datagram parameters */
+struct ncm_ntb_datagram_parameters {
+	/** Maximum size */
+	uint32_t mtu;
+	/** Alignment divisor */
+	uint16_t divisor;
+	/** Alignment remainder */
+	uint16_t remainder;
+	/** Alignment modulus */
+	uint16_t modulus;
+} __attribute__ (( packed ));
+
+/** NTB parameters */
+struct ncm_ntb_parameters {
+	/** Length */
+	uint16_t len;
+	/** Supported formats */
+	uint16_t formats;
+	/** IN datagram parameters */
+	struct ncm_ntb_datagram_parameters in;
+	/** Reserved */
+	uint16_t reserved;
+	/** OUT datagram parameters */
+	struct ncm_ntb_datagram_parameters out;
+	/** Maximum number of datagrams per OUT NTB */
+	uint16_t max;
+} __attribute__ (( packed ));
+
+/** Set NTB input size */
+#define NCM_SET_NTB_INPUT_SIZE						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x86 ) )
+
+/** Set NTB input size */
+struct ncm_set_ntb_input_size {
+	/** Maximum size */
+	uint32_t mtu;
+} __attribute__ (( packed ));
+
+/** Minimum allowed NTB input size */
+#define NCM_MIN_NTB_INPUT_SIZE 2048
+
+/** Maximum allowed NTB input size (16-bit) */
+#define NCM_MAX_NTB_INPUT_SIZE 65536
+
+/** CDC-NCM transfer header (16-bit) */
+struct ncm_transfer_header {
+	/** Signature */
+	uint32_t magic;
+	/** Header length */
+	uint16_t header_len;
+	/** Sequence number */
+	uint16_t sequence;
+	/** Total length */
+	uint16_t len;
+	/** Offset of first datagram pointer */
+	uint16_t offset;
+} __attribute__ (( packed ));
+
+/** CDC-NCM transfer header magic */
+#define NCM_TRANSFER_HEADER_MAGIC 0x484d434eUL
+
+/** CDC-NCM datagram descriptor (16-bit) */
+struct ncm_datagram_descriptor {
+	/** Starting offset */
+	uint16_t offset;
+	/** Length */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** CDC-NCM datagram pointer (16-bit) */
+struct ncm_datagram_pointer {
+	/** Signature */
+	uint32_t magic;
+	/** Header length */
+	uint16_t header_len;
+	/** Offset of next datagram pointer */
+	uint16_t offset;
+	/** Datagram descriptors
+	 *
+	 * Must be terminated by an empty descriptor.
+	 */
+	struct ncm_datagram_descriptor desc[0];
+} __attribute__ (( packed ));
+
+/** CDC-NCM datagram pointer magic */
+#define NCM_DATAGRAM_POINTER_MAGIC 0x304d434eUL
+
+/** CDC-NCM datagram pointer CRC present flag */
+#define NCM_DATAGRAM_POINTER_MAGIC_CRC 0x01000000UL
+
+/** NTB constructed for transmitted packets (excluding padding)
+ *
+ * This is a policy decision.
+ */
+struct ncm_ntb_header {
+	/** Transfer header */
+	struct ncm_transfer_header nth;
+	/** Datagram pointer */
+	struct ncm_datagram_pointer ndp;
+	/** Datagram descriptors */
+	struct ncm_datagram_descriptor desc[2];
+} __attribute__ (( packed ));
+
+/** A CDC-NCM network device */
+struct ncm_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+
+	/** Maximum supported NTB input size */
+	size_t mtu;
+	/** Transmitted packet sequence number */
+	uint16_t sequence;
+	/** Alignment padding required on transmitted packets */
+	size_t padding;
+};
+
+/** Bulk IN ring minimum buffer count
+ *
+ * This is a policy decision.
+ */
+#define NCM_IN_MIN_COUNT 3
+
+/** Bulk IN ring minimum total buffer size
+ *
+ * This is a policy decision.
+ */
+#define NCM_IN_MIN_SIZE 16384
+
+/** Bulk IN ring maximum total buffer size
+ *
+ * This is a policy decision.
+ */
+#define NCM_IN_MAX_SIZE 131072
+
+/** Interrupt ring buffer count
+ *
+ * This is a policy decision.
+ */
+#define NCM_INTR_COUNT 2
+
+#endif /* _NCM_H */
diff --git a/roms/ipxe/src/drivers/net/netfront.c b/roms/ipxe/src/drivers/net/netfront.c
index 4b816329e..2f4bbf2a0 100644
--- a/roms/ipxe/src/drivers/net/netfront.c
+++ b/roms/ipxe/src/drivers/net/netfront.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -135,7 +139,7 @@ static int netfront_read_mac ( struct netfront_nic *netfront, void *hw_addr ) {
 		xendev->key, mac );
 
 	/* Decode MAC address */
-	len = hex_decode ( mac, ':', hw_addr, ETH_ALEN );
+	len = hex_decode ( ':', mac, hw_addr, ETH_ALEN );
 	if ( len < 0 ) {
 		rc = len;
 		DBGC ( netfront, "NETFRONT %s could not decode MAC address "
@@ -593,6 +597,11 @@ static int netfront_open ( struct net_device *netdev ) {
 					  "feature-no-csum-offload" ) ) != 0 )
 		goto err_feature_no_csum_offload;
 
+	/* Inform backend that we will send notifications for RX requests */
+	if ( ( rc = netfront_write_flag ( netfront,
+					  "feature-rx-notify" ) ) != 0 )
+		goto err_feature_rx_notify;
+
 	/* Set state to Connected */
 	if ( ( rc = xenbus_set_state ( xendev, XenbusStateConnected ) ) != 0 ) {
 		DBGC ( netfront, "NETFRONT %s could not set state=\"%d\": %s\n",
@@ -618,6 +627,8 @@ static int netfront_open ( struct net_device *netdev ) {
  err_backend_wait:
 	netfront_reset ( netfront );
  err_set_state:
+	netfront_rm ( netfront, "feature-rx-notify" );
+ err_feature_rx_notify:
 	netfront_rm ( netfront, "feature-no-csum-offload" );
  err_feature_no_csum_offload:
 	netfront_rm ( netfront, "request-rx-copy" );
@@ -661,6 +672,7 @@ static void netfront_close ( struct net_device *netdev ) {
 	}
 
 	/* Delete flags */
+	netfront_rm ( netfront, "feature-rx-notify" );
 	netfront_rm ( netfront, "feature-no-csum-offload" );
 	netfront_rm ( netfront, "request-rx-copy" );
 
diff --git a/roms/ipxe/src/drivers/net/netfront.h b/roms/ipxe/src/drivers/net/netfront.h
index b3f899f3c..38fd0a77e 100644
--- a/roms/ipxe/src/drivers/net/netfront.h
+++ b/roms/ipxe/src/drivers/net/netfront.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/io/netif.h>
diff --git a/roms/ipxe/src/drivers/net/netvsc.c b/roms/ipxe/src/drivers/net/netvsc.c
new file mode 100644
index 000000000..d269cd63e
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/netvsc.c
@@ -0,0 +1,848 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Hyper-V network virtual service client
+ *
+ * The network virtual service client (NetVSC) connects to the network
+ * virtual service provider (NetVSP) via the Hyper-V virtual machine
+ * bus (VMBus).  It provides a transport layer for RNDIS packets.
+ */
+
+#include <errno.h>
+#include <unistd.h>
+#include <byteswap.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/rndis.h>
+#include <ipxe/vmbus.h>
+#include "netvsc.h"
+
+/**
+ * Send control message and wait for completion
+ *
+ * @v netvsc		NetVSC device
+ * @v xrid		Relative transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_control ( struct netvsc_device *netvsc, unsigned int xrid,
+			    const void *data, size_t len ) {
+	uint64_t xid = ( NETVSC_BASE_XID + xrid );
+	unsigned int i;
+	int rc;
+
+	/* Send control message */
+	if ( ( rc = vmbus_send_control ( netvsc->vmdev, xid, data, len ) ) !=0){
+		DBGC ( netvsc, "NETVSC %s could not send control message: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Record transaction ID */
+	netvsc->wait_xrid = xrid;
+
+	/* Wait for operation to complete */
+	for ( i = 0 ; i < NETVSC_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for completion */
+		if ( ! netvsc->wait_xrid )
+			return netvsc->wait_rc;
+
+		/* Poll VMBus device */
+		vmbus_poll ( netvsc->vmdev );
+
+		/* Delay for 1ms */
+		mdelay ( 1 );
+	}
+
+	DBGC ( netvsc, "NETVSC %s timed out waiting for XRID %d\n",
+	       netvsc->name, xrid );
+	vmbus_dump_channel ( netvsc->vmdev );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Handle generic completion
+ *
+ * @v netvsc		NetVSC device
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_completed ( struct netvsc_device *netvsc __unused,
+			      const void *data __unused, size_t len __unused ) {
+	return 0;
+}
+
+/**
+ * Initialise communication
+ *
+ * @v netvsc		NetVSC device
+ * @ret rc		Return status code
+ */
+static int netvsc_initialise ( struct netvsc_device *netvsc ) {
+	struct netvsc_init_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( NETVSC_INIT_MSG );
+	msg.min = cpu_to_le32 ( NETVSC_VERSION_1 );
+	msg.max = cpu_to_le32 ( NETVSC_VERSION_1 );
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, NETVSC_INIT_XRID, &msg,
+				     sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not initialise: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle initialisation completion
+ *
+ * @v netvsc		NetVSC device
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int
+netvsc_initialised ( struct netvsc_device *netvsc, const void *data,
+		     size_t len ) {
+	const struct netvsc_init_completion *cmplt = data;
+
+	/* Check completion */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( netvsc, "NETVSC %s underlength initialisation "
+		       "completion (%zd bytes)\n", netvsc->name, len );
+		return -EINVAL;
+	}
+	if ( cmplt->header.type != cpu_to_le32 ( NETVSC_INIT_CMPLT ) ) {
+		DBGC ( netvsc, "NETVSC %s unexpected initialisation completion "
+		       "type %d\n", netvsc->name,
+		       le32_to_cpu ( cmplt->header.type ) );
+		return -EPROTO;
+	}
+	if ( cmplt->status != cpu_to_le32 ( NETVSC_OK ) ) {
+		DBGC ( netvsc, "NETVSC %s initialisation failure status %d\n",
+		       netvsc->name, le32_to_cpu ( cmplt->status ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Set NDIS version
+ *
+ * @v netvsc		NetVSC device
+ * @ret rc		Return status code
+ */
+static int netvsc_ndis_version ( struct netvsc_device *netvsc ) {
+	struct netvsc_ndis_version_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( NETVSC_NDIS_VERSION_MSG );
+	msg.major = cpu_to_le32 ( NETVSC_NDIS_MAJOR );
+	msg.minor = cpu_to_le32 ( NETVSC_NDIS_MINOR );
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, NETVSC_NDIS_VERSION_XRID,
+				     &msg, sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not set NDIS version: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Establish data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ * @ret rc		Return status code
+ */
+static int netvsc_establish_buffer ( struct netvsc_device *netvsc,
+				     struct netvsc_buffer *buffer ) {
+	struct netvsc_establish_buffer_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( buffer->establish_type );
+	msg.gpadl = cpu_to_le32 ( buffer->gpadl );
+	msg.pageset = buffer->pages.pageset; /* Already protocol-endian */
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, buffer->establish_xrid, &msg,
+				     sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not establish buffer: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle establish receive data buffer completion
+ *
+ * @v netvsc		NetVSC device
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_rx_established_buffer ( struct netvsc_device *netvsc,
+					  const void *data, size_t len ) {
+	const struct netvsc_rx_establish_buffer_completion *cmplt = data;
+
+	/* Check completion */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( netvsc, "NETVSC %s underlength buffer completion (%zd "
+		       "bytes)\n", netvsc->name, len );
+		return -EINVAL;
+	}
+	if ( cmplt->header.type != cpu_to_le32 ( NETVSC_RX_ESTABLISH_CMPLT ) ) {
+		DBGC ( netvsc, "NETVSC %s unexpected buffer completion type "
+		       "%d\n", netvsc->name, le32_to_cpu ( cmplt->header.type));
+		return -EPROTO;
+	}
+	if ( cmplt->status != cpu_to_le32 ( NETVSC_OK ) ) {
+		DBGC ( netvsc, "NETVSC %s buffer failure status %d\n",
+		       netvsc->name, le32_to_cpu ( cmplt->status ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Revoke data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ * @ret rc		Return status code
+ */
+static int netvsc_revoke_buffer ( struct netvsc_device *netvsc,
+				  struct netvsc_buffer *buffer ) {
+	struct netvsc_revoke_buffer_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( buffer->revoke_type );
+	msg.pageset = buffer->pages.pageset; /* Already protocol-endian */
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, buffer->revoke_xrid,
+				     &msg, sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not revoke buffer: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle received control packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_control ( struct vmbus_device *vmdev, uint64_t xid,
+				 const void *data, size_t len ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+
+	DBGC ( netvsc, "NETVSC %s received unsupported control packet "
+	       "(%08llx):\n", netvsc->name, xid );
+	DBGC_HDA ( netvsc, 0, data, len );
+	return -ENOTSUP;
+}
+
+/**
+ * Handle received data packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @v list		List of I/O buffers
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_data ( struct vmbus_device *vmdev, uint64_t xid,
+			      const void *data, size_t len,
+			      struct list_head *list ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+	const struct netvsc_rndis_message *msg = data;
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( netvsc, "NETVSC %s received underlength RNDIS packet "
+		       "(%zd bytes)\n", netvsc->name, len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+	if ( msg->header.type != cpu_to_le32 ( NETVSC_RNDIS_MSG ) ) {
+		DBGC ( netvsc, "NETVSC %s received unexpected RNDIS packet "
+		       "type %d\n", netvsc->name,
+		       le32_to_cpu ( msg->header.type ) );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Send completion back to host */
+	if ( ( rc = vmbus_send_completion ( vmdev, xid, NULL, 0 ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not send completion: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_completion;
+	}
+
+	/* Hand off to RNDIS */
+	list_for_each_entry_safe ( iobuf, tmp, list, list ) {
+		list_del ( &iobuf->list );
+		rndis_rx ( rndis, iob_disown ( iobuf ) );
+	}
+
+	return 0;
+
+ err_completion:
+ err_sanity:
+	list_for_each_entry_safe ( iobuf, tmp, list, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+	return rc;
+}
+
+/**
+ * Handle received completion packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_completion ( struct vmbus_device *vmdev, uint64_t xid,
+				    const void *data, size_t len ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+	struct io_buffer *iobuf;
+	int ( * completion ) ( struct netvsc_device *netvsc,
+			       const void *data, size_t len );
+	unsigned int xrid = ( xid - NETVSC_BASE_XID );
+	unsigned int tx_id;
+	int rc;
+
+	/* Handle transmit completion, if applicable */
+	tx_id = ( xrid - NETVSC_TX_BASE_XRID );
+	if ( ( tx_id < NETVSC_TX_NUM_DESC ) &&
+	     ( ( iobuf = netvsc->tx.iobufs[tx_id] ) != NULL ) ) {
+
+		/* Free buffer ID */
+		netvsc->tx.iobufs[tx_id] = NULL;
+		netvsc->tx.ids[ ( netvsc->tx.id_cons++ ) &
+				( netvsc->tx.count - 1 ) ] = tx_id;
+
+		/* Hand back to RNDIS */
+		rndis_tx_complete ( rndis, iobuf );
+		return 0;
+	}
+
+	/* Otherwise determine completion handler */
+	if ( xrid == NETVSC_INIT_XRID ) {
+		completion = netvsc_initialised;
+	} else if ( xrid == NETVSC_RX_ESTABLISH_XRID ) {
+		completion = netvsc_rx_established_buffer;
+	} else if ( ( netvsc->wait_xrid != 0 ) &&
+		    ( xrid == netvsc->wait_xrid ) ) {
+		completion = netvsc_completed;
+	} else {
+		DBGC ( netvsc, "NETVSC %s received unexpected completion "
+		       "(%08llx)\n", netvsc->name, xid );
+		return -EPIPE;
+	}
+
+	/* Hand off to completion handler */
+	rc = completion ( netvsc, data, len );
+
+	/* Record completion handler result if applicable */
+	if ( xrid == netvsc->wait_xrid ) {
+		netvsc->wait_xrid = 0;
+		netvsc->wait_rc = rc;
+	}
+
+	return rc;
+}
+
+/**
+ * Handle received cancellation packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_cancellation ( struct vmbus_device *vmdev,
+				      uint64_t xid ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+
+	DBGC ( netvsc, "NETVSC %s received unsupported cancellation packet "
+	       "(%08llx):\n", netvsc->name, xid );
+	return -ENOTSUP;
+}
+
+/** VMBus channel operations */
+static struct vmbus_channel_operations netvsc_channel_operations = {
+	.recv_control = netvsc_recv_control,
+	.recv_data = netvsc_recv_data,
+	.recv_completion = netvsc_recv_completion,
+	.recv_cancellation = netvsc_recv_cancellation,
+};
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v rndis		RNDIS device
+ */
+static void netvsc_poll ( struct rndis_device *rndis ) {
+	struct netvsc_device *netvsc = rndis->priv;
+	struct vmbus_device *vmdev = netvsc->vmdev;
+
+	/* Poll VMBus device */
+	while ( vmbus_has_data ( vmdev ) )
+		vmbus_poll ( vmdev );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * If this method returns success then the RNDIS device must
+ * eventually report completion via rndis_tx_complete().
+ */
+static int netvsc_transmit ( struct rndis_device *rndis,
+			     struct io_buffer *iobuf ) {
+	struct netvsc_device *netvsc = rndis->priv;
+	struct rndis_header *header = iobuf->data;
+	struct netvsc_rndis_message msg;
+	unsigned int tx_id;
+	unsigned int xrid;
+	uint64_t xid;
+	int rc;
+
+	/* Sanity check */
+	assert ( iob_len ( iobuf ) >= sizeof ( *header ) );
+	assert ( iob_len ( iobuf ) == le32_to_cpu ( header->len ) );
+
+	/* Check that we have space in the transmit ring */
+	if ( netvsc_ring_is_full ( &netvsc->tx ) )
+		return rndis_tx_defer ( rndis, iobuf );
+
+	/* Allocate buffer ID and calculate transaction ID */
+	tx_id = netvsc->tx.ids[ netvsc->tx.id_prod & ( netvsc->tx.count - 1 ) ];
+	assert ( netvsc->tx.iobufs[tx_id] == NULL );
+	xrid = ( NETVSC_TX_BASE_XRID + tx_id );
+	xid = ( NETVSC_BASE_XID + xrid );
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( NETVSC_RNDIS_MSG );
+	msg.channel = ( ( header->type == cpu_to_le32 ( RNDIS_PACKET_MSG ) ) ?
+			NETVSC_RNDIS_DATA : NETVSC_RNDIS_CONTROL );
+	msg.buffer = cpu_to_le32 ( NETVSC_RNDIS_NO_BUFFER );
+
+	/* Send message */
+	if ( ( rc = vmbus_send_data ( netvsc->vmdev, xid, &msg, sizeof ( msg ),
+				      iobuf ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not send RNDIS message: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Store I/O buffer and consume buffer ID */
+	netvsc->tx.iobufs[tx_id] = iobuf;
+	netvsc->tx.id_prod++;
+
+	return 0;
+}
+
+/**
+ * Cancel transmission
+ *
+ * @v netvsc		NetVSC device
+ * @v iobuf		I/O buffer
+ * @v tx_id		Transmission ID
+ */
+static void netvsc_cancel_transmit ( struct netvsc_device *netvsc,
+				     struct io_buffer *iobuf,
+				     unsigned int tx_id ) {
+	unsigned int xrid;
+	uint64_t xid;
+
+	/* Send cancellation */
+	xrid = ( NETVSC_TX_BASE_XRID + tx_id );
+	xid = ( NETVSC_BASE_XID + xrid );
+	DBGC ( netvsc, "NETVSC %s cancelling transmission %#x\n",
+	       netvsc->name, tx_id );
+	vmbus_send_cancellation ( netvsc->vmdev, xid );
+
+	/* Report back to RNDIS */
+	rndis_tx_complete_err ( netvsc->rndis, iobuf, -ECANCELED );
+}
+
+/**
+ * Create descriptor ring
+ *
+ * @v netvsc		NetVSC device
+ * @v ring		Descriptor ring
+ * @ret rc		Return status code
+ */
+static int netvsc_create_ring ( struct netvsc_device *netvsc __unused,
+				struct netvsc_ring *ring ) {
+	unsigned int i;
+
+	/* Initialise buffer ID ring */
+	for ( i = 0 ; i < ring->count ; i++ ) {
+		ring->ids[i] = i;
+		assert ( ring->iobufs[i] == NULL );
+	}
+	ring->id_prod = 0;
+	ring->id_cons = 0;
+
+	return 0;
+}
+
+/**
+ * Destroy descriptor ring
+ *
+ * @v netvsc		NetVSC device
+ * @v ring		Descriptor ring
+ * @v discard		Method used to discard outstanding buffer, or NULL
+ */
+static void netvsc_destroy_ring ( struct netvsc_device *netvsc,
+				  struct netvsc_ring *ring,
+				  void ( * discard ) ( struct netvsc_device *,
+						       struct io_buffer *,
+						       unsigned int ) ) {
+	struct io_buffer *iobuf;
+	unsigned int i;
+
+	/* Flush any outstanding buffers */
+	for ( i = 0 ; i < ring->count ; i++ ) {
+		iobuf = ring->iobufs[i];
+		if ( ! iobuf )
+			continue;
+		ring->iobufs[i] = NULL;
+		ring->ids[ ( ring->id_cons++ ) & ( ring->count - 1 ) ] = i;
+		if ( discard )
+			discard ( netvsc, iobuf, i );
+	}
+
+	/* Sanity check */
+	assert ( netvsc_ring_is_empty ( ring ) );
+}
+
+/**
+ * Copy data from data buffer
+ *
+ * @v pages		Transfer page set
+ * @v data		Data buffer
+ * @v offset		Offset within page set
+ * @v len		Length within page set
+ * @ret rc		Return status code
+ */
+static int netvsc_buffer_copy ( struct vmbus_xfer_pages *pages, void *data,
+				size_t offset, size_t len ) {
+	struct netvsc_buffer *buffer =
+		container_of ( pages, struct netvsc_buffer, pages );
+
+	/* Sanity check */
+	if ( ( offset > buffer->len ) || ( len > ( buffer->len - offset ) ) )
+		return -ERANGE;
+
+	/* Copy data from buffer */
+	copy_from_user ( data, buffer->data, offset, len );
+
+	return 0;
+}
+
+/** Transfer page set operations */
+static struct vmbus_xfer_pages_operations netvsc_xfer_pages_operations = {
+	.copy = netvsc_buffer_copy,
+};
+
+/**
+ * Create data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ * @ret rc		Return status code
+ */
+static int netvsc_create_buffer ( struct netvsc_device *netvsc,
+				  struct netvsc_buffer *buffer ) {
+	struct vmbus_device *vmdev = netvsc->vmdev;
+	int gpadl;
+	int rc;
+
+	/* Allocate receive buffer */
+	buffer->data = umalloc ( buffer->len );
+	if ( ! buffer->data ) {
+		DBGC ( netvsc, "NETVSC %s could not allocate %zd-byte buffer\n",
+		       netvsc->name, buffer->len );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Establish GPA descriptor list */
+	gpadl = vmbus_establish_gpadl ( vmdev, buffer->data, buffer->len );
+	if ( gpadl < 0 ) {
+		rc = gpadl;
+		DBGC ( netvsc, "NETVSC %s could not establish GPADL: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_establish_gpadl;
+	}
+	buffer->gpadl = gpadl;
+
+	/* Register transfer page set */
+	if ( ( rc = vmbus_register_pages ( vmdev, &buffer->pages ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not register transfer pages: "
+		       "%s\n", netvsc->name, strerror ( rc ) );
+		goto err_register_pages;
+	}
+
+	return 0;
+
+	vmbus_unregister_pages ( vmdev, &buffer->pages );
+ err_register_pages:
+	vmbus_gpadl_teardown ( vmdev, gpadl );
+ err_establish_gpadl:
+	ufree ( buffer->data );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Destroy data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ */
+static void netvsc_destroy_buffer ( struct netvsc_device *netvsc,
+				    struct netvsc_buffer *buffer ) {
+	struct vmbus_device *vmdev = netvsc->vmdev;
+	int rc;
+
+	/* Unregister transfer pages */
+	vmbus_unregister_pages ( vmdev, &buffer->pages );
+
+	/* Tear down GPA descriptor list */
+	if ( ( rc = vmbus_gpadl_teardown ( vmdev, buffer->gpadl ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not tear down GPADL: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		/* Death is imminent.  The host may well continue to
+		 * write to the data buffer.  The best we can do is
+		 * leak memory for now and hope that the host doesn't
+		 * write to this region after we load an OS.
+		 */
+		return;
+	}
+
+	/* Free buffer */
+	ufree ( buffer->data );
+}
+
+/**
+ * Open device
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int netvsc_open ( struct rndis_device *rndis ) {
+	struct netvsc_device *netvsc = rndis->priv;
+	int rc;
+
+	/* Initialise receive buffer */
+	if ( ( rc = netvsc_create_buffer ( netvsc, &netvsc->rx ) ) != 0 )
+		goto err_create_rx;
+
+	/* Open channel */
+	if ( ( rc = vmbus_open ( netvsc->vmdev, &netvsc_channel_operations,
+				 PAGE_SIZE, PAGE_SIZE, NETVSC_MTU ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not open VMBus: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_vmbus_open;
+	}
+
+	/* Initialise communication with NetVSP */
+	if ( ( rc = netvsc_initialise ( netvsc ) ) != 0 )
+		goto err_initialise;
+	if ( ( rc = netvsc_ndis_version ( netvsc ) ) != 0 )
+		goto err_ndis_version;
+
+	/* Initialise transmit ring */
+	if ( ( rc = netvsc_create_ring ( netvsc, &netvsc->tx ) ) != 0 )
+		goto err_create_tx;
+
+	/* Establish receive buffer */
+	if ( ( rc = netvsc_establish_buffer ( netvsc, &netvsc->rx ) ) != 0 )
+		goto err_establish_rx;
+
+	return 0;
+
+	netvsc_revoke_buffer ( netvsc, &netvsc->rx );
+ err_establish_rx:
+	netvsc_destroy_ring ( netvsc, &netvsc->tx, NULL );
+ err_create_tx:
+ err_ndis_version:
+ err_initialise:
+	vmbus_close ( netvsc->vmdev );
+ err_vmbus_open:
+	netvsc_destroy_buffer ( netvsc, &netvsc->rx );
+ err_create_rx:
+	return rc;
+}
+
+/**
+ * Close device
+ *
+ * @v rndis		RNDIS device
+ */
+static void netvsc_close ( struct rndis_device *rndis ) {
+	struct netvsc_device *netvsc = rndis->priv;
+
+	/* Revoke receive buffer */
+	netvsc_revoke_buffer ( netvsc, &netvsc->rx );
+
+	/* Destroy transmit ring */
+	netvsc_destroy_ring ( netvsc, &netvsc->tx, netvsc_cancel_transmit );
+
+	/* Close channel */
+	vmbus_close ( netvsc->vmdev );
+
+	/* Destroy receive buffer */
+	netvsc_destroy_buffer ( netvsc, &netvsc->rx );
+}
+
+/** RNDIS operations */
+static struct rndis_operations netvsc_operations = {
+	.open = netvsc_open,
+	.close = netvsc_close,
+	.transmit = netvsc_transmit,
+	.poll = netvsc_poll,
+};
+
+/**
+ * Probe device
+ *
+ * @v vmdev		VMBus device
+ * @ret rc		Return status code
+ */
+static int netvsc_probe ( struct vmbus_device *vmdev ) {
+	struct netvsc_device *netvsc;
+	struct rndis_device *rndis;
+	int rc;
+
+	/* Allocate and initialise structure */
+	rndis = alloc_rndis ( sizeof ( *netvsc ) );
+	if ( ! rndis ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	rndis_init ( rndis, &netvsc_operations );
+	rndis->netdev->dev = &vmdev->dev;
+	netvsc = rndis->priv;
+	netvsc->vmdev = vmdev;
+	netvsc->rndis = rndis;
+	netvsc->name = vmdev->dev.name;
+	netvsc_init_ring ( &netvsc->tx, NETVSC_TX_NUM_DESC,
+			   netvsc->tx_iobufs, netvsc->tx_ids );
+	netvsc_init_buffer ( &netvsc->rx, NETVSC_RX_BUF_PAGESET,
+			     &netvsc_xfer_pages_operations,
+			     NETVSC_RX_ESTABLISH_MSG, NETVSC_RX_ESTABLISH_XRID,
+			     NETVSC_RX_REVOKE_MSG, NETVSC_RX_REVOKE_XRID,
+			     NETVSC_RX_BUF_LEN );
+	vmbus_set_drvdata ( vmdev, rndis );
+
+	/* Register RNDIS device */
+	if ( ( rc = register_rndis ( rndis ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not register: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_register;
+	}
+
+	return 0;
+
+	unregister_rndis ( rndis );
+ err_register:
+	free_rndis ( rndis );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v vmdev		VMBus device
+ */
+static void netvsc_remove ( struct vmbus_device *vmdev ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+
+	/* Unregister RNDIS device */
+	unregister_rndis ( rndis );
+
+	/* Free RNDIS device */
+	free_rndis ( rndis );
+}
+
+/** NetVSC driver */
+struct vmbus_driver netvsc_driver __vmbus_driver = {
+	.name = "netvsc",
+	.type = VMBUS_TYPE ( 0xf8615163, 0xdf3e, 0x46c5, 0x913f,
+			     0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e ),
+	.probe = netvsc_probe,
+	.remove = netvsc_remove,
+};
diff --git a/roms/ipxe/src/drivers/net/netvsc.h b/roms/ipxe/src/drivers/net/netvsc.h
new file mode 100644
index 000000000..39eeb891c
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/netvsc.h
@@ -0,0 +1,365 @@
+#ifndef _NETVSC_H
+#define _NETVSC_H
+
+/** @file
+ *
+ * Hyper-V network virtual service client
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Maximum supported NetVSC message length */
+#define NETVSC_MTU 512
+
+/** Maximum time to wait for a transaction to complete
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_MAX_WAIT_MS 1000
+
+/** Number of transmit ring entries
+ *
+ * Must be a power of two.  This is a policy decision.  This value
+ * must be sufficiently small to guarantee that we never run out of
+ * space in the VMBus outbound ring buffer.
+ */
+#define NETVSC_TX_NUM_DESC 32
+
+/** RX data buffer page set ID
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_RX_BUF_PAGESET 0xbead
+
+/** RX data buffer length
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_RX_BUF_LEN ( 16 * PAGE_SIZE )
+
+/** Base transaction ID
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_BASE_XID 0x18ae0000UL
+
+/** Relative transaction IDs */
+enum netvsc_xrid {
+	/** Transmit descriptors (one per transmit buffer ID) */
+	NETVSC_TX_BASE_XRID = 0,
+	/** Initialisation */
+	NETVSC_INIT_XRID = ( NETVSC_TX_BASE_XRID + NETVSC_TX_NUM_DESC ),
+	/** NDIS version */
+	NETVSC_NDIS_VERSION_XRID,
+	/** Establish receive buffer */
+	NETVSC_RX_ESTABLISH_XRID,
+	/** Revoke receive buffer */
+	NETVSC_RX_REVOKE_XRID,
+};
+
+/** NetVSC status codes */
+enum netvsc_status {
+	NETVSC_NONE = 0,
+	NETVSC_OK = 1,
+	NETVSC_FAIL = 2,
+	NETVSC_TOO_NEW = 3,
+	NETVSC_TOO_OLD = 4,
+	NETVSC_BAD_PACKET = 5,
+	NETVSC_BUSY = 6,
+	NETVSC_UNSUPPORTED = 7,
+};
+
+/** NetVSC message header */
+struct netvsc_header {
+	/** Type */
+	uint32_t type;
+} __attribute__ (( packed ));
+
+/** NetVSC initialisation message */
+#define NETVSC_INIT_MSG 1
+
+/** NetVSC initialisation message */
+struct netvsc_init_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** Minimum supported protocol version */
+	uint32_t min;
+	/** Maximum supported protocol version */
+	uint32_t max;
+	/** Reserved */
+	uint8_t reserved[20];
+} __attribute__ (( packed ));
+
+/** Oldest known NetVSC protocol version */
+#define NETVSC_VERSION_1 2 /* sic */
+
+/** NetVSC initialisation completion */
+#define NETVSC_INIT_CMPLT 2
+
+/** NetVSC initialisation completion */
+struct netvsc_init_completion {
+	/** Message header */
+	struct netvsc_header header;
+	/** Protocol version */
+	uint32_t version;
+	/** Maximum memory descriptor list length */
+	uint32_t max_mdl_len;
+	/** Status */
+	uint32_t status;
+	/** Reserved */
+	uint8_t reserved[16];
+} __attribute__ (( packed ));
+
+/** NetVSC NDIS version message */
+#define NETVSC_NDIS_VERSION_MSG 100
+
+/** NetVSC NDIS version message */
+struct netvsc_ndis_version_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** Major version */
+	uint32_t major;
+	/** Minor version */
+	uint32_t minor;
+	/** Reserved */
+	uint8_t reserved[20];
+} __attribute__ (( packed ));
+
+/** NetVSC NDIS major version */
+#define NETVSC_NDIS_MAJOR 6
+
+/** NetVSC NDIS minor version */
+#define NETVSC_NDIS_MINOR 1
+
+/** NetVSC establish receive data buffer message */
+#define NETVSC_RX_ESTABLISH_MSG 101
+
+/** NetVSC establish receive data buffer completion */
+#define NETVSC_RX_ESTABLISH_CMPLT 102
+
+/** NetVSC revoke receive data buffer message */
+#define NETVSC_RX_REVOKE_MSG 103
+
+/** NetVSC establish transmit data buffer message */
+#define NETVSC_TX_ESTABLISH_MSG 104
+
+/** NetVSC establish transmit data buffer completion */
+#define NETVSC_TX_ESTABLISH_CMPLT 105
+
+/** NetVSC revoke transmit data buffer message */
+#define NETVSC_TX_REVOKE_MSG 106
+
+/** NetVSC establish data buffer message */
+struct netvsc_establish_buffer_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** GPADL ID */
+	uint32_t gpadl;
+	/** Page set ID */
+	uint16_t pageset;
+	/** Reserved */
+	uint8_t reserved[22];
+} __attribute__ (( packed ));
+
+/** NetVSC receive data buffer section */
+struct netvsc_rx_buffer_section {
+	/** Starting offset */
+	uint32_t start;
+	/** Subsection length */
+	uint32_t len;
+	/** Number of subsections */
+	uint32_t count;
+	/** Ending offset */
+	uint32_t end;
+} __attribute__ (( packed ));
+
+/** NetVSC establish receive data buffer completion */
+struct netvsc_rx_establish_buffer_completion {
+	/** Message header */
+	struct netvsc_header header;
+	/** Status */
+	uint32_t status;
+	/** Number of sections (must be 1) */
+	uint32_t count;
+	/** Section descriptors */
+	struct netvsc_rx_buffer_section section[1];
+} __attribute__ (( packed ));
+
+/** NetVSC establish transmit data buffer completion */
+struct netvsc_tx_establish_buffer_completion {
+	/** Message header */
+	struct netvsc_header header;
+	/** Status */
+	uint32_t status;
+	/** Section length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** NetVSC revoke data buffer message */
+struct netvsc_revoke_buffer_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** Page set ID */
+	uint16_t pageset;
+	/** Reserved */
+	uint8_t reserved[26];
+} __attribute__ (( packed ));
+
+/** NetVSC RNDIS message */
+#define NETVSC_RNDIS_MSG 107
+
+/** NetVSC RNDIS message */
+struct netvsc_rndis_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** RNDIS channel */
+	uint32_t channel;
+	/** Buffer index (or NETVSC_RNDIS_NO_BUFFER) */
+	uint32_t buffer;
+	/** Buffer length */
+	uint32_t len;
+	/** Reserved */
+	uint8_t reserved[16];
+} __attribute__ (( packed ));
+
+/** RNDIS data channel (for RNDIS_PACKET_MSG only) */
+#define NETVSC_RNDIS_DATA 0
+
+/** RNDIS control channel (for all other RNDIS messages) */
+#define NETVSC_RNDIS_CONTROL 1
+
+/** "No buffer used" index */
+#define NETVSC_RNDIS_NO_BUFFER 0xffffffffUL
+
+/** A NetVSC descriptor ring */
+struct netvsc_ring {
+	/** Number of descriptors */
+	unsigned int count;
+	/** I/O buffers, indexed by buffer ID */
+	struct io_buffer **iobufs;
+	/** Buffer ID ring */
+	uint8_t *ids;
+	/** Buffer ID producer counter */
+	unsigned int id_prod;
+	/** Buffer ID consumer counter */
+	unsigned int id_cons;
+};
+
+/**
+ * Initialise descriptor ring
+ *
+ * @v ring		Descriptor ring
+ * @v count		Maximum number of used descriptors
+ * @v iobufs		I/O buffers
+ * @v ids		Buffer IDs
+ */
+static inline __attribute__ (( always_inline )) void
+netvsc_init_ring ( struct netvsc_ring *ring, unsigned int count,
+		   struct io_buffer **iobufs, uint8_t *ids ) {
+
+	ring->count = count;
+	ring->iobufs = iobufs;
+	ring->ids = ids;
+}
+
+/**
+ * Check whether or not descriptor ring is full
+ *
+ * @v ring		Descriptor ring
+ * @v is_full		Ring is full
+ */
+static inline __attribute__ (( always_inline )) int
+netvsc_ring_is_full ( struct netvsc_ring *ring ) {
+	unsigned int fill_level;
+
+	fill_level = ( ring->id_prod - ring->id_cons );
+	assert ( fill_level <= ring->count );
+	return ( fill_level >= ring->count );
+}
+
+/**
+ * Check whether or not descriptor ring is empty
+ *
+ * @v ring		Descriptor ring
+ * @v is_empty		Ring is empty
+ */
+static inline __attribute__ (( always_inline )) int
+netvsc_ring_is_empty ( struct netvsc_ring *ring ) {
+
+	return ( ring->id_prod == ring->id_cons );
+}
+
+/** A NetVSC data buffer */
+struct netvsc_buffer {
+	/** Transfer page set */
+	struct vmbus_xfer_pages pages;
+	/** Establish data buffer message type */
+	uint8_t establish_type;
+	/** Establish data buffer relative transaction ID */
+	uint8_t establish_xrid;
+	/** Revoke data buffer message type */
+	uint8_t revoke_type;
+	/** Revoke data buffer relative transaction ID */
+	uint8_t revoke_xrid;
+	/** Buffer length */
+	size_t len;
+	/** Buffer */
+	userptr_t data;
+	/** GPADL ID */
+	unsigned int gpadl;
+};
+
+/**
+ * Initialise data buffer
+ *
+ * @v buffer		Data buffer
+ * @v pageset		Page set ID
+ * @v op		Page set operations
+ * @v establish_type	Establish data buffer message type
+ * @v establish_xrid	Establish data buffer relative transaction ID
+ * @v revoke_type	Revoke data buffer message type
+ * @v revoke_type	Revoke data buffer relative transaction ID
+ * @v len		Required length
+ */
+static inline __attribute__ (( always_inline )) void
+netvsc_init_buffer ( struct netvsc_buffer *buffer, uint16_t pageset,
+		     struct vmbus_xfer_pages_operations *op,
+		     uint8_t establish_type, uint8_t establish_xrid,
+		     uint8_t revoke_type, uint8_t revoke_xrid, size_t len ) {
+
+	buffer->pages.pageset = cpu_to_le16 ( pageset );
+	buffer->pages.op = op;
+	buffer->establish_type = establish_type;
+	buffer->establish_xrid = establish_xrid;
+	buffer->revoke_type = revoke_type;
+	buffer->revoke_xrid = revoke_xrid;
+	buffer->len = len;
+}
+
+/** A NetVSC device */
+struct netvsc_device {
+	/** VMBus device */
+	struct vmbus_device *vmdev;
+	/** RNDIS device */
+	struct rndis_device *rndis;
+	/** Name */
+	const char *name;
+
+	/** Transmit ring */
+	struct netvsc_ring tx;
+	/** Transmit buffer IDs */
+	uint8_t tx_ids[NETVSC_TX_NUM_DESC];
+	/** Transmit I/O buffers */
+	struct io_buffer *tx_iobufs[NETVSC_TX_NUM_DESC];
+
+	/** Receive buffer */
+	struct netvsc_buffer rx;
+
+	/** Relative transaction ID for current blocking transaction */
+	unsigned int wait_xrid;
+	/** Return status code for current blocking transaction */
+	int wait_rc;
+};
+
+#endif /* _NETVSC_H */
diff --git a/roms/ipxe/src/drivers/net/phantom/nx_bitops.h b/roms/ipxe/src/drivers/net/phantom/nx_bitops.h
index 15f3d3767..1687b6952 100644
--- a/roms/ipxe/src/drivers/net/phantom/nx_bitops.h
+++ b/roms/ipxe/src/drivers/net/phantom/nx_bitops.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/drivers/net/phantom/phantom.c b/roms/ipxe/src/drivers/net/phantom/phantom.c
index e70ded08c..38b66743c 100644
--- a/roms/ipxe/src/drivers/net/phantom/phantom.c
+++ b/roms/ipxe/src/drivers/net/phantom/phantom.c
@@ -16,9 +16,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/drivers/net/phantom/phantom.h b/roms/ipxe/src/drivers/net/phantom/phantom.h
index 1647168ba..967603409 100644
--- a/roms/ipxe/src/drivers/net/phantom/phantom.h
+++ b/roms/ipxe/src/drivers/net/phantom/phantom.h
@@ -19,9 +19,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/drivers/net/phantom/phantom_hw.h b/roms/ipxe/src/drivers/net/phantom/phantom_hw.h
index 7dfff52b2..016730de3 100644
--- a/roms/ipxe/src/drivers/net/phantom/phantom_hw.h
+++ b/roms/ipxe/src/drivers/net/phantom/phantom_hw.h
@@ -19,9 +19,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/drivers/net/pnic.c b/roms/ipxe/src/drivers/net/pnic.c
index 4170cc640..ca64299ea 100644
--- a/roms/ipxe/src/drivers/net/pnic.c
+++ b/roms/ipxe/src/drivers/net/pnic.c
@@ -6,8 +6,18 @@ Bochs Pseudo NIC driver for Etherboot
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  *
  * See pnic_api.h for an explanation of the Bochs Pseudo NIC.
  */
diff --git a/roms/ipxe/src/drivers/net/prism2.c b/roms/ipxe/src/drivers/net/prism2.c
index ab974264c..4331f2cd0 100644
--- a/roms/ipxe/src/drivers/net/prism2.c
+++ b/roms/ipxe/src/drivers/net/prism2.c
@@ -9,8 +9,18 @@ $Id$
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
diff --git a/roms/ipxe/src/drivers/net/prism2_pci.c b/roms/ipxe/src/drivers/net/prism2_pci.c
index 72549babf..69ddf0fb0 100644
--- a/roms/ipxe/src/drivers/net/prism2_pci.c
+++ b/roms/ipxe/src/drivers/net/prism2_pci.c
@@ -10,8 +10,18 @@ $Id$
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -40,8 +50,6 @@ static void prism2_pci_disable ( struct nic *nic ) {
 
 static struct pci_device_id prism2_pci_nics[] = {
 PCI_ROM(0x1260, 0x3873, "prism2_pci",	"Harris Semiconductor Prism2.5 clone", 0),
-PCI_ROM(0x1260, 0x3873, "hwp01170",	"ActionTec HWP01170", 0),
-PCI_ROM(0x1260, 0x3873, "dwl520",	"DLink DWL-520", 0),
 };
 
 PCI_DRIVER ( prism2_pci_driver, prism2_pci_nics, PCI_NO_CLASS );
diff --git a/roms/ipxe/src/drivers/net/prism2_plx.c b/roms/ipxe/src/drivers/net/prism2_plx.c
index 2098f7f09..a73b0e087 100644
--- a/roms/ipxe/src/drivers/net/prism2_plx.c
+++ b/roms/ipxe/src/drivers/net/prism2_plx.c
@@ -10,8 +10,18 @@ $Id$
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -44,10 +54,10 @@ static int prism2_find_plx ( hfa384x_t *hw, struct pci_device *p )
   
   /* Obtain all memory and IO base addresses */
   pci_read_config_dword( p, PLX_LOCAL_CONFIG_REGISTER_BASE, &plx_lcr);
-  plx_lcr &= PCI_BASE_ADDRESS_IO_MASK;
+  plx_lcr &= ~PCI_BASE_ADDRESS_IO_MASK;
   pci_read_config_dword( p, PRISM2_PLX_ATTR_MEM_BASE, &attr_mem);
   pci_read_config_dword( p, PRISM2_PLX_IO_BASE, &iobase);
-  iobase &= PCI_BASE_ADDRESS_IO_MASK;
+  iobase &= ~PCI_BASE_ADDRESS_IO_MASK;
 
   /* Fill out hw structure */
   hw->iobase = iobase;
diff --git a/roms/ipxe/src/drivers/net/realtek.c b/roms/ipxe/src/drivers/net/realtek.c
index 0aca8c77f..022b59324 100644
--- a/roms/ipxe/src/drivers/net/realtek.c
+++ b/roms/ipxe/src/drivers/net/realtek.c
@@ -17,9 +17,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -194,7 +198,6 @@ static int realtek_init_eeprom ( struct net_device *netdev ) {
 		DBGC ( rtl, "REALTEK %p EEPROM is a 93C46\n", rtl );
 		init_at93c46 ( &rtl->eeprom, 16 );
 	}
-	rtl->eeprom.bus = &rtl->spibit.bus;
 
 	/* Check for EEPROM presence.  Some onboard NICs will have no
 	 * EEPROM connected, with the BIOS being responsible for
@@ -1085,6 +1088,7 @@ static void realtek_detect ( struct realtek_nic *rtl ) {
 			       rtl );
 			rtl->legacy = 1;
 		}
+		rtl->eeprom.bus = &rtl->spibit.bus;
 	}
 }
 
@@ -1132,7 +1136,8 @@ static int realtek_probe ( struct pci_device *pci ) {
 	realtek_detect ( rtl );
 
 	/* Initialise EEPROM */
-	if ( ( rc = realtek_init_eeprom ( netdev ) ) == 0 ) {
+	if ( rtl->eeprom.bus &&
+	     ( ( rc = realtek_init_eeprom ( netdev ) ) == 0 ) ) {
 
 		/* Read MAC address from EEPROM */
 		if ( ( rc = nvs_read ( &rtl->eeprom.nvs, RTL_EEPROM_MAC,
diff --git a/roms/ipxe/src/drivers/net/realtek.h b/roms/ipxe/src/drivers/net/realtek.h
index ac33405e8..b1ce7f98f 100644
--- a/roms/ipxe/src/drivers/net/realtek.h
+++ b/roms/ipxe/src/drivers/net/realtek.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/spi.h>
 #include <ipxe/spi_bit.h>
diff --git a/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c b/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c
index 8851d1bfb..5f97480fa 100644
--- a/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c
+++ b/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c
@@ -3,15 +3,23 @@
 FILE_LICENCE(GPL2_OR_LATER);
 
 #include <ipxe/pci.h>
+#include "rtl818x.h"
 
-REQUIRE_OBJECT(rtl818x);
-REQUIRE_OBJECT(rtl8180_grf5101);
-REQUIRE_OBJECT(rtl8180_max2820);
-REQUIRE_OBJECT(rtl8180_sa2400);
-
-static struct pci_device_id rtl8180_nics[] __unused = {
+static struct pci_device_id rtl8180_nics[] = {
 	PCI_ROM(0x10ec, 0x8180, "rtl8180", "Realtek 8180", 0),
 	PCI_ROM(0x1799, 0x6001, "f5d6001", "Belkin F5D6001", 0),
 	PCI_ROM(0x1799, 0x6020, "f5d6020", "Belkin F5D6020", 0),
 	PCI_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
 };
+
+struct pci_driver rtl8180_driver __pci_driver = {
+	.ids            = rtl8180_nics,
+	.id_count       = sizeof(rtl8180_nics) / sizeof(rtl8180_nics[0]),
+	.probe		= rtl818x_probe,
+	.remove		= rtl818x_remove,
+};
+
+REQUIRING_SYMBOL(rtl8180_driver);
+REQUIRE_OBJECT(rtl8180_grf5101);
+REQUIRE_OBJECT(rtl8180_max2820);
+REQUIRE_OBJECT(rtl8180_sa2400);
diff --git a/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c b/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c
index fd27e5c8c..234978cea 100644
--- a/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c
+++ b/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c
@@ -3,12 +3,20 @@
 FILE_LICENCE(GPL2_OR_LATER);
 
 #include <ipxe/pci.h>
-
-REQUIRE_OBJECT(rtl818x);
-REQUIRE_OBJECT(rtl8185_rtl8225);
+#include "rtl818x.h"
 
 static struct pci_device_id rtl8185_nics[] __unused = {
 	PCI_ROM(0x10ec, 0x8185, "rtl8185", "Realtek 8185", 0),
 	PCI_ROM(0x1799, 0x700f, "f5d7000", "Belkin F5D7000", 0),
 	PCI_ROM(0x1799, 0x701f, "f5d7010", "Belkin F5D7010", 0),
 };
+
+struct pci_driver rtl8185_driver __pci_driver = {
+	.ids            = rtl8185_nics,
+	.id_count       = sizeof(rtl8185_nics) / sizeof(rtl8185_nics[0]),
+	.probe		= rtl818x_probe,
+	.remove		= rtl818x_remove,
+};
+
+REQUIRING_SYMBOL(rtl8185_driver);
+REQUIRE_OBJECT(rtl8185_rtl8225);
diff --git a/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c b/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c
index cf4c7556f..8b3c206d4 100644
--- a/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c
+++ b/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c
@@ -649,7 +649,7 @@ struct net80211_device_operations rtl818x_operations = {
 	.config = rtl818x_config,
 };
 
-static int rtl818x_probe(struct pci_device *pdev )
+int rtl818x_probe(struct pci_device *pdev )
 {
 	struct net80211_device *dev;
 	struct rtl818x_priv *priv;
@@ -820,7 +820,7 @@ static int rtl818x_probe(struct pci_device *pdev )
 	return err;
 }
 
-static void rtl818x_remove(struct pci_device *pdev)
+void rtl818x_remove(struct pci_device *pdev)
 {
 	struct net80211_device *dev = pci_get_drvdata(pdev);
 
@@ -830,25 +830,3 @@ static void rtl818x_remove(struct pci_device *pdev)
 	net80211_unregister(dev);
 	net80211_free(dev);
 }
-
-/* Hide PCI_ROM definitions in here from parserom.pl; the definitions
-   that should be used are in rtl8180.c and rtl8185.c. */
-#define RTL_ROM PCI_ROM
-
-static struct pci_device_id rtl818x_nics[] = {
-	RTL_ROM(0x10ec, 0x8185, "rtl8185", "Realtek 8185", 0),
-	RTL_ROM(0x1799, 0x700f, "f5d7000", "Belkin F5D7000", 0),
-	RTL_ROM(0x1799, 0x701f, "f5d7010", "Belkin F5D7010", 0),
-
-	RTL_ROM(0x10ec, 0x8180, "rtl8180", "Realtek 8180", 0),
-	RTL_ROM(0x1799, 0x6001, "f5d6001", "Belkin F5D6001", 0),
-	RTL_ROM(0x1799, 0x6020, "f5d6020", "Belkin F5D6020", 0),
-	RTL_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
-};
-
-struct pci_driver rtl818x_driver __pci_driver = {
-	.ids            = rtl818x_nics,
-	.id_count       = sizeof(rtl818x_nics) / sizeof(rtl818x_nics[0]),
-	.probe		= rtl818x_probe,
-	.remove		= rtl818x_remove,
-};
diff --git a/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h b/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h
index 4e57d0bd3..ae4b8a96f 100644
--- a/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h
+++ b/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h
@@ -19,6 +19,7 @@
 
 #include <ipxe/spi_bit.h>
 #include <ipxe/tables.h>
+#include <ipxe/net80211.h>
 
 FILE_LICENCE(GPL2_ONLY);
 
@@ -356,4 +357,7 @@ struct rtl818x_rf_ops {
 	void (*conf_erp)(struct net80211_device *dev); /* set based on dev->erp_flags */
 };
 
+extern int rtl818x_probe(struct pci_device *pdev );
+extern void rtl818x_remove(struct pci_device *pdev);
+
 #endif /* RTL818X_H */
diff --git a/roms/ipxe/src/drivers/net/skeleton.c b/roms/ipxe/src/drivers/net/skeleton.c
index 365111b8d..0435b9d0e 100644
--- a/roms/ipxe/src/drivers/net/skeleton.c
+++ b/roms/ipxe/src/drivers/net/skeleton.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/drivers/net/skeleton.h b/roms/ipxe/src/drivers/net/skeleton.h
index 3de2afa5b..2ab01bd56 100644
--- a/roms/ipxe/src/drivers/net/skeleton.h
+++ b/roms/ipxe/src/drivers/net/skeleton.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Skeleton BAR size */
 #define SKELETON_BAR_SIZE 256
diff --git a/roms/ipxe/src/drivers/net/smsc75xx.c b/roms/ipxe/src/drivers/net/smsc75xx.c
new file mode 100644
index 000000000..017e02a59
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/smsc75xx.c
@@ -0,0 +1,1057 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/profile.h>
+#include "smsc75xx.h"
+
+/** @file
+ *
+ * SMSC LAN75xx USB Ethernet driver
+ *
+ */
+
+/** Interrupt completion profiler */
+static struct profiler smsc75xx_intr_profiler __profiler =
+	{ .name = "smsc75xx.intr" };
+
+/** Bulk IN completion profiler */
+static struct profiler smsc75xx_in_profiler __profiler =
+	{ .name = "smsc75xx.in" };
+
+/** Bulk OUT profiler */
+static struct profiler smsc75xx_out_profiler __profiler =
+	{ .name = "smsc75xx.out" };
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Write register (without byte-swapping)
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @v value		Register value
+ * @ret rc		Return status code
+ */
+static int smsc75xx_raw_writel ( struct smsc75xx_device *smsc75xx,
+				 unsigned int address, uint32_t value ) {
+	int rc;
+
+	/* Write register */
+	if ( ( rc = usb_control ( smsc75xx->usb, SMSC75XX_REGISTER_WRITE, 0,
+				  address, &value, sizeof ( value ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not write %03x: %s\n",
+		       smsc75xx, address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Write register
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @v value		Register value
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+smsc75xx_writel ( struct smsc75xx_device *smsc75xx, unsigned int address,
+		  uint32_t value ) {
+	int rc;
+
+	/* Write register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, address,
+					  cpu_to_le32 ( value ) ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Read register (without byte-swapping)
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @ret value		Register value
+ * @ret rc		Return status code
+ */
+static int smsc75xx_raw_readl ( struct smsc75xx_device *smsc75xx,
+				unsigned int address, uint32_t *value ) {
+	int rc;
+
+	/* Read register */
+	if ( ( rc = usb_control ( smsc75xx->usb, SMSC75XX_REGISTER_READ, 0,
+				  address, value, sizeof ( *value ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not read %03x: %s\n",
+		       smsc75xx, address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Read register
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @ret value		Register value
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+smsc75xx_readl ( struct smsc75xx_device *smsc75xx, unsigned int address,
+		 uint32_t *value ) {
+	int rc;
+
+	/* Read register */
+	if ( ( rc = smsc75xx_raw_readl ( smsc75xx, address, value ) ) != 0 )
+		return rc;
+	le32_to_cpus ( value );
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * EEPROM access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Wait for EEPROM to become idle
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_eeprom_wait ( struct smsc75xx_device *smsc75xx ) {
+	uint32_t e2p_cmd;
+	unsigned int i;
+	int rc;
+
+	/* Wait for EPC_BSY to become clear */
+	for ( i = 0 ; i < SMSC75XX_EEPROM_MAX_WAIT_MS ; i++ ) {
+
+		/* Read E2P_CMD and check EPC_BSY */
+		if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_E2P_CMD,
+					     &e2p_cmd ) ) != 0 )
+			return rc;
+		if ( ! ( e2p_cmd & SMSC75XX_E2P_CMD_EPC_BSY ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( smsc75xx, "SMSC75XX %p timed out waiting for EEPROM\n",
+	       smsc75xx );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Read byte from EEPROM
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		EEPROM address
+ * @ret byte		Byte read, or negative error
+ */
+static int smsc75xx_eeprom_read_byte ( struct smsc75xx_device *smsc75xx,
+				       unsigned int address ) {
+	uint32_t e2p_cmd;
+	uint32_t e2p_data;
+	int rc;
+
+	/* Wait for EEPROM to become idle */
+	if ( ( rc = smsc75xx_eeprom_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Initiate read command */
+	e2p_cmd = ( SMSC75XX_E2P_CMD_EPC_BSY | SMSC75XX_E2P_CMD_EPC_CMD_READ |
+		    SMSC75XX_E2P_CMD_EPC_ADDR ( address ) );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_E2P_CMD,
+				      e2p_cmd ) ) != 0 )
+		return rc;
+
+	/* Wait for command to complete */
+	if ( ( rc = smsc75xx_eeprom_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Read EEPROM data */
+	if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_E2P_DATA,
+				     &e2p_data ) ) != 0 )
+		return rc;
+
+	return SMSC75XX_E2P_DATA_GET ( e2p_data );
+}
+
+/**
+ * Read data from EEPROM
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		EEPROM address
+ * @v data		Data buffer
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int smsc75xx_eeprom_read ( struct smsc75xx_device *smsc75xx,
+				  unsigned int address, void *data,
+				  size_t len ) {
+	uint8_t *bytes;
+	int byte;
+
+	/* Read bytes */
+	for ( bytes = data ; len-- ; address++, bytes++ ) {
+		byte = smsc75xx_eeprom_read_byte ( smsc75xx, address );
+		if ( byte < 0 )
+			return byte;
+		*bytes = byte;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * MII access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Wait for MII to become idle
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_mii_wait ( struct smsc75xx_device *smsc75xx ) {
+	uint32_t mii_access;
+	unsigned int i;
+	int rc;
+
+	/* Wait for MIIBZY to become clear */
+	for ( i = 0 ; i < SMSC75XX_MII_MAX_WAIT_MS ; i++ ) {
+
+		/* Read MII_ACCESS and check MIIBZY */
+		if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_MII_ACCESS,
+					     &mii_access ) ) != 0 )
+			return rc;
+		if ( ! ( mii_access & SMSC75XX_MII_ACCESS_MIIBZY ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( smsc75xx, "SMSC75XX %p timed out waiting for MII\n",
+	       smsc75xx );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Read from MII register
+ *
+ * @v mii		MII interface
+ * @v reg		Register address
+ * @ret value		Data read, or negative error
+ */
+static int smsc75xx_mii_read ( struct mii_interface *mii, unsigned int reg ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( mii, struct smsc75xx_device, mii );
+	uint32_t mii_access;
+	uint32_t mii_data;
+	int rc;
+
+	/* Wait for MII to become idle */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Initiate read command */
+	mii_access = ( SMSC75XX_MII_ACCESS_PHY_ADDRESS |
+		       SMSC75XX_MII_ACCESS_MIIRINDA ( reg ) |
+		       SMSC75XX_MII_ACCESS_MIIBZY );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MII_ACCESS,
+				      mii_access ) ) != 0 )
+		return rc;
+
+	/* Wait for command to complete */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Read MII data */
+	if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_MII_DATA,
+				     &mii_data ) ) != 0 )
+		return rc;
+
+	return SMSC75XX_MII_DATA_GET ( mii_data );
+}
+
+/**
+ * Write to MII register
+ *
+ * @v mii		MII interface
+ * @v reg		Register address
+ * @v data		Data to write
+ * @ret rc		Return status code
+ */
+static int smsc75xx_mii_write ( struct mii_interface *mii, unsigned int reg,
+				unsigned int data ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( mii, struct smsc75xx_device, mii );
+	uint32_t mii_access;
+	uint32_t mii_data;
+	int rc;
+
+	/* Wait for MII to become idle */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Write MII data */
+	mii_data = SMSC75XX_MII_DATA_SET ( data );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MII_DATA,
+				      mii_data ) ) != 0 )
+		return rc;
+
+	/* Initiate write command */
+	mii_access = ( SMSC75XX_MII_ACCESS_PHY_ADDRESS |
+		       SMSC75XX_MII_ACCESS_MIIRINDA ( reg ) |
+		       SMSC75XX_MII_ACCESS_MIIWNR |
+		       SMSC75XX_MII_ACCESS_MIIBZY );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MII_ACCESS,
+				      mii_access ) ) != 0 )
+		return rc;
+
+	/* Wait for command to complete */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/** MII operations */
+static struct mii_operations smsc75xx_mii_operations = {
+	.read = smsc75xx_mii_read,
+	.write = smsc75xx_mii_write,
+};
+
+/**
+ * Check link status
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_check_link ( struct smsc75xx_device *smsc75xx ) {
+	struct net_device *netdev = smsc75xx->netdev;
+	int intr;
+	int rc;
+
+	/* Read PHY interrupt source */
+	intr = mii_read ( &smsc75xx->mii, SMSC75XX_MII_PHY_INTR_SOURCE );
+	if ( intr < 0 ) {
+		rc = intr;
+		DBGC ( smsc75xx, "SMSC75XX %p could not get PHY interrupt "
+		       "source: %s\n", smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Acknowledge PHY interrupt */
+	if ( ( rc = mii_write ( &smsc75xx->mii, SMSC75XX_MII_PHY_INTR_SOURCE,
+				intr ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not acknowledge PHY "
+		       "interrupt: %s\n", smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check link status */
+	if ( ( rc = mii_check_link ( &smsc75xx->mii, netdev ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not check link: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	DBGC ( smsc75xx, "SMSC75XX %p link %s (intr %#04x)\n",
+	       smsc75xx, ( netdev_link_ok ( netdev ) ? "up" : "down" ), intr );
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Statistics (for debugging)
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get statistics
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v stats		Statistics to fill in
+ * @ret rc		Return status code
+ */
+static int smsc75xx_get_statistics ( struct smsc75xx_device *smsc75xx,
+				     struct smsc75xx_statistics *stats ) {
+	int rc;
+
+	/* Get statistics */
+	if ( ( rc = usb_control ( smsc75xx->usb, SMSC75XX_GET_STATISTICS, 0, 0,
+				  stats, sizeof ( *stats ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not get statistics: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Dump statistics (for debugging)
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_dump_statistics ( struct smsc75xx_device *smsc75xx ) {
+	struct smsc75xx_statistics stats;
+	int rc;
+
+	/* Do nothing unless debugging is enabled */
+	if ( ! DBG_LOG )
+		return 0;
+
+	/* Get statistics */
+	if ( ( rc = smsc75xx_get_statistics ( smsc75xx, &stats ) ) != 0 )
+		return rc;
+
+	/* Dump statistics */
+	DBGC ( smsc75xx, "SMSC75XX %p RXE fcs %d aln %d frg %d jab %d und %d "
+	       "ovr %d drp %d\n", smsc75xx, le32_to_cpu ( stats.rx.err.fcs ),
+	       le32_to_cpu ( stats.rx.err.alignment ),
+	       le32_to_cpu ( stats.rx.err.fragment ),
+	       le32_to_cpu ( stats.rx.err.jabber ),
+	       le32_to_cpu ( stats.rx.err.undersize ),
+	       le32_to_cpu ( stats.rx.err.oversize ),
+	       le32_to_cpu ( stats.rx.err.dropped ) );
+	DBGC ( smsc75xx, "SMSC75XX %p RXB ucast %d bcast %d mcast %d\n",
+	       smsc75xx, le32_to_cpu ( stats.rx.byte.unicast ),
+	       le32_to_cpu ( stats.rx.byte.broadcast ),
+	       le32_to_cpu ( stats.rx.byte.multicast ) );
+	DBGC ( smsc75xx, "SMSC75XX %p RXF ucast %d bcast %d mcast %d pause "
+	       "%d\n", smsc75xx, le32_to_cpu ( stats.rx.frame.unicast ),
+	       le32_to_cpu ( stats.rx.frame.broadcast ),
+	       le32_to_cpu ( stats.rx.frame.multicast ),
+	       le32_to_cpu ( stats.rx.frame.pause ) );
+	DBGC ( smsc75xx, "SMSC75XX %p TXE fcs %d def %d car %d cnt %d sgl %d "
+	       "mul %d exc %d lat %d\n", smsc75xx,
+	       le32_to_cpu ( stats.tx.err.fcs ),
+	       le32_to_cpu ( stats.tx.err.deferral ),
+	       le32_to_cpu ( stats.tx.err.carrier ),
+	       le32_to_cpu ( stats.tx.err.count ),
+	       le32_to_cpu ( stats.tx.err.single ),
+	       le32_to_cpu ( stats.tx.err.multiple ),
+	       le32_to_cpu ( stats.tx.err.excessive ),
+	       le32_to_cpu ( stats.tx.err.late ) );
+	DBGC ( smsc75xx, "SMSC75XX %p TXB ucast %d bcast %d mcast %d\n",
+	       smsc75xx, le32_to_cpu ( stats.tx.byte.unicast ),
+	       le32_to_cpu ( stats.tx.byte.broadcast ),
+	       le32_to_cpu ( stats.tx.byte.multicast ) );
+	DBGC ( smsc75xx, "SMSC75XX %p TXF ucast %d bcast %d mcast %d pause "
+	       "%d\n", smsc75xx, le32_to_cpu ( stats.tx.frame.unicast ),
+	       le32_to_cpu ( stats.tx.frame.broadcast ),
+	       le32_to_cpu ( stats.tx.frame.multicast ),
+	       le32_to_cpu ( stats.tx.frame.pause ) );
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Device reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Reset device
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_reset ( struct smsc75xx_device *smsc75xx ) {
+	uint32_t hw_cfg;
+	int rc;
+
+	/* Reset device */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_HW_CFG,
+				      SMSC75XX_HW_CFG_LRST ) ) != 0 )
+		return rc;
+
+	/* Wait for reset to complete */
+	udelay ( SMSC75XX_RESET_DELAY_US );
+
+	/* Check that reset has completed */
+	if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_HW_CFG,
+				     &hw_cfg ) ) != 0 )
+		return rc;
+	if ( hw_cfg & SMSC75XX_HW_CFG_LRST ) {
+		DBGC ( smsc75xx, "SMSC75XX %p failed to reset\n", smsc75xx );
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void smsc75xx_intr_complete ( struct usb_endpoint *ep,
+				     struct io_buffer *iobuf, int rc ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( ep, struct smsc75xx_device, usbnet.intr );
+	struct net_device *netdev = smsc75xx->netdev;
+	struct smsc75xx_interrupt *intr;
+
+	/* Profile completions */
+	profile_start ( &smsc75xx_intr_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto done;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p interrupt failed: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, rc );
+		goto done;
+	}
+
+	/* Extract interrupt data */
+	if ( iob_len ( iobuf ) != sizeof ( *intr ) ) {
+		DBGC ( smsc75xx, "SMSC75XX %p malformed interrupt\n",
+		       smsc75xx );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, rc );
+		goto done;
+	}
+	intr = iobuf->data;
+
+	/* Record interrupt status */
+	smsc75xx->int_sts = le32_to_cpu ( intr->int_sts );
+	profile_stop ( &smsc75xx_intr_profiler );
+
+ done:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations smsc75xx_intr_operations = {
+	.complete = smsc75xx_intr_complete,
+};
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void smsc75xx_in_complete ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf, int rc ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( ep, struct smsc75xx_device, usbnet.in );
+	struct net_device *netdev = smsc75xx->netdev;
+	struct smsc75xx_rx_header *header;
+
+	/* Profile completions */
+	profile_start ( &smsc75xx_in_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open ) {
+		free_iob ( iobuf );
+		return;
+	}
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p bulk IN failed: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < ( sizeof ( *header ) ) ) {
+		DBGC ( smsc75xx, "SMSC75XX %p underlength bulk IN\n",
+		       smsc75xx );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto err;
+	}
+
+	/* Strip header */
+	header = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *header ) );
+
+	/* Check for errors */
+	if ( header->command & cpu_to_le32 ( SMSC75XX_RX_RED ) ) {
+		DBGC ( smsc75xx, "SMSC75XX %p receive error (%08x):\n",
+		       smsc75xx, le32_to_cpu ( header->command ) );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EIO;
+		goto err;
+	}
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+
+	profile_stop ( &smsc75xx_in_profiler );
+	return;
+
+ err:
+	/* Hand off to network stack */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations smsc75xx_in_operations = {
+	.complete = smsc75xx_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int smsc75xx_out_transmit ( struct smsc75xx_device *smsc75xx,
+				   struct io_buffer *iobuf ) {
+	struct smsc75xx_tx_header *header;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Profile transmissions */
+	profile_start ( &smsc75xx_out_profiler );
+
+	/* Prepend header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, sizeof ( *header ) ) ) != 0 )
+		return rc;
+	header = iob_push ( iobuf, sizeof ( *header ) );
+	header->command = cpu_to_le32 ( SMSC75XX_TX_FCS | len );
+	header->tag = 0;
+	header->mss = 0;
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &smsc75xx->usbnet.out, iobuf, 0 ) ) != 0 )
+		return rc;
+
+	profile_stop ( &smsc75xx_out_profiler );
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void smsc75xx_out_complete ( struct usb_endpoint *ep,
+				    struct io_buffer *iobuf, int rc ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( ep, struct smsc75xx_device, usbnet.out );
+	struct net_device *netdev = smsc75xx->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations smsc75xx_out_operations = {
+	.complete = smsc75xx_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_open ( struct net_device *netdev ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+	union smsc75xx_mac mac;
+	int rc;
+
+	/* Clear stored interrupt status */
+	smsc75xx->int_sts = 0;
+
+	/* Copy MAC address */
+	memset ( &mac, 0, sizeof ( mac ) );
+	memcpy ( mac.raw, netdev->ll_addr, ETH_ALEN );
+
+	/* Configure bulk IN empty response */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_HW_CFG,
+				      SMSC75XX_HW_CFG_BIR ) ) != 0 )
+		goto err_hw_cfg;
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &smsc75xx->usbnet ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not open: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Configure interrupt endpoint */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_INT_EP_CTL,
+				      ( SMSC75XX_INT_EP_CTL_RDFO_EN |
+					SMSC75XX_INT_EP_CTL_PHY_EN ) ) ) != 0 )
+		goto err_int_ep_ctl;
+
+	/* Configure bulk IN delay */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_BULK_IN_DLY,
+				      SMSC75XX_BULK_IN_DLY_SET ( 0 ) ) ) != 0 )
+		goto err_bulk_in_dly;
+
+	/* Configure receive filters */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_RFE_CTL,
+				      ( SMSC75XX_RFE_CTL_AB |
+					SMSC75XX_RFE_CTL_AM |
+					SMSC75XX_RFE_CTL_AU ) ) ) != 0 )
+		goto err_rfe_ctl;
+
+	/* Configure receive FIFO */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_FCT_RX_CTL,
+				      ( SMSC75XX_FCT_RX_CTL_EN |
+					SMSC75XX_FCT_RX_CTL_BAD ) ) ) != 0 )
+		goto err_fct_rx_ctl;
+
+	/* Configure transmit FIFO */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_FCT_TX_CTL,
+				      SMSC75XX_FCT_TX_CTL_EN ) ) != 0 )
+		goto err_fct_tx_ctl;
+
+	/* Configure receive datapath */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MAC_RX,
+				      ( SMSC75XX_MAC_RX_MAX_SIZE_DEFAULT |
+					SMSC75XX_MAC_RX_FCS |
+					SMSC75XX_MAC_RX_EN ) ) ) != 0 )
+		goto err_mac_rx;
+
+	/* Configure transmit datapath */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MAC_TX,
+				      SMSC75XX_MAC_TX_EN ) ) != 0 )
+		goto err_mac_tx;
+
+	/* Write MAC address high register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_RX_ADDRH,
+					  mac.addr.h ) ) != 0 )
+		goto err_rx_addrh;
+
+	/* Write MAC address low register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_RX_ADDRL,
+					  mac.addr.l ) ) != 0 )
+		goto err_rx_addrl;
+
+	/* Write MAC address perfect filter high register */
+	mac.addr.h |= cpu_to_le32 ( SMSC75XX_ADDR_FILTH_VALID );
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_ADDR_FILTH ( 0 ),
+					  mac.addr.h ) ) != 0 )
+		goto err_addr_filth;
+
+	/* Write MAC address perfect filter low register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_ADDR_FILTL ( 0 ),
+					  mac.addr.l ) ) != 0 )
+		goto err_addr_filtl;
+
+	/* Enable PHY interrupts */
+	if ( ( rc = mii_write ( &smsc75xx->mii, SMSC75XX_MII_PHY_INTR_MASK,
+				( SMSC75XX_PHY_INTR_ANEG_DONE |
+				  SMSC75XX_PHY_INTR_LINK_DOWN ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not set PHY interrupt "
+		       "mask: %s\n", smsc75xx, strerror ( rc ) );
+		goto err_phy_intr_mask;
+	}
+
+	/* Update link status */
+	smsc75xx_check_link ( smsc75xx );
+
+	return 0;
+
+ err_phy_intr_mask:
+ err_addr_filtl:
+ err_addr_filth:
+ err_rx_addrl:
+ err_rx_addrh:
+ err_mac_tx:
+ err_mac_rx:
+ err_fct_tx_ctl:
+ err_fct_rx_ctl:
+ err_rfe_ctl:
+ err_bulk_in_dly:
+ err_int_ep_ctl:
+	usbnet_close ( &smsc75xx->usbnet );
+ err_open:
+ err_hw_cfg:
+	smsc75xx_reset ( smsc75xx );
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void smsc75xx_close ( struct net_device *netdev ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &smsc75xx->usbnet );
+
+	/* Dump statistics (for debugging) */
+	smsc75xx_dump_statistics ( smsc75xx );
+
+	/* Reset device */
+	smsc75xx_reset ( smsc75xx );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int smsc75xx_transmit ( struct net_device *netdev,
+			       struct io_buffer *iobuf ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = smsc75xx_out_transmit ( smsc75xx, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void smsc75xx_poll ( struct net_device *netdev ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+	uint32_t int_sts;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( smsc75xx->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &smsc75xx->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+
+	/* Do nothing more unless there are interrupts to handle */
+	int_sts = smsc75xx->int_sts;
+	if ( ! int_sts )
+		return;
+
+	/* Check link status if applicable */
+	if ( int_sts & SMSC75XX_INT_STS_PHY_INT ) {
+		smsc75xx_check_link ( smsc75xx );
+		int_sts &= ~SMSC75XX_INT_STS_PHY_INT;
+	}
+
+	/* Record RX FIFO overflow if applicable */
+	if ( int_sts & SMSC75XX_INT_STS_RDFO_INT ) {
+		DBGC2 ( smsc75xx, "SMSC75XX %p RX FIFO overflowed\n",
+			smsc75xx );
+		netdev_rx_err ( netdev, NULL, -ENOBUFS );
+		int_sts &= ~SMSC75XX_INT_STS_RDFO_INT;
+	}
+
+	/* Check for unexpected interrupts */
+	if ( int_sts ) {
+		DBGC ( smsc75xx, "SMSC75XX %p unexpected interrupt %#08x\n",
+		       smsc75xx, int_sts );
+		netdev_rx_err ( netdev, NULL, -ENOTTY );
+	}
+
+	/* Clear interrupts */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_INT_STS,
+				      smsc75xx->int_sts ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+	smsc75xx->int_sts = 0;
+}
+
+/** SMSC75xx network device operations */
+static struct net_device_operations smsc75xx_operations = {
+	.open		= smsc75xx_open,
+	.close		= smsc75xx_close,
+	.transmit	= smsc75xx_transmit,
+	.poll		= smsc75xx_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int smsc75xx_probe ( struct usb_function *func,
+			    struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct smsc75xx_device *smsc75xx;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *smsc75xx ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &smsc75xx_operations );
+	netdev->dev = &func->dev;
+	smsc75xx = netdev->priv;
+	memset ( smsc75xx, 0, sizeof ( *smsc75xx ) );
+	smsc75xx->usb = usb;
+	smsc75xx->bus = usb->port->hub->bus;
+	smsc75xx->netdev = netdev;
+	usbnet_init ( &smsc75xx->usbnet, func, &smsc75xx_intr_operations,
+		      &smsc75xx_in_operations, &smsc75xx_out_operations );
+	usb_refill_init ( &smsc75xx->usbnet.intr, 0, SMSC75XX_INTR_MAX_FILL );
+	usb_refill_init ( &smsc75xx->usbnet.in, SMSC75XX_IN_MTU,
+			  SMSC75XX_IN_MAX_FILL );
+	mii_init ( &smsc75xx->mii, &smsc75xx_mii_operations );
+	DBGC ( smsc75xx, "SMSC75XX %p on %s\n", smsc75xx, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &smsc75xx->usbnet, config ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not describe: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Reset device */
+	if ( ( rc = smsc75xx_reset ( smsc75xx ) ) != 0 )
+		goto err_reset;
+
+	/* Read MAC address */
+	if ( ( rc = smsc75xx_eeprom_read ( smsc75xx, SMSC75XX_EEPROM_MAC,
+					   netdev->hw_addr, ETH_ALEN ) ) != 0 )
+		goto err_eeprom_read;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, netdev );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_eeprom_read:
+ err_reset:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void smsc75xx_remove ( struct usb_function *func ) {
+	struct net_device *netdev = usb_func_get_drvdata ( func );
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** SMSC75xx device IDs */
+static struct usb_device_id smsc75xx_ids[] = {
+	{
+		.name = "smsc7500",
+		.vendor = 0x0424,
+		.product = 0x7500,
+		.class = { 0xff, 0x00, 0xff },
+	},
+	{
+		.name = "smsc7505",
+		.vendor = 0x0424,
+		.product = 0x7505,
+		.class = { 0xff, 0x00, 0xff },
+	},
+};
+
+/** SMSC LAN75xx driver */
+struct usb_driver smsc75xx_driver __usb_driver = {
+	.ids = smsc75xx_ids,
+	.id_count = ( sizeof ( smsc75xx_ids ) / sizeof ( smsc75xx_ids[0] ) ),
+	.probe = smsc75xx_probe,
+	.remove = smsc75xx_remove,
+};
diff --git a/roms/ipxe/src/drivers/net/smsc75xx.h b/roms/ipxe/src/drivers/net/smsc75xx.h
new file mode 100644
index 000000000..2463b72a1
--- /dev/null
+++ b/roms/ipxe/src/drivers/net/smsc75xx.h
@@ -0,0 +1,309 @@
+#ifndef _SMSC75XX_H
+#define _SMSC75XX_H
+
+/** @file
+ *
+ * SMSC LAN75xx USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/mii.h>
+
+/** Register write command */
+#define SMSC75XX_REGISTER_WRITE					\
+	( USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0xa0 ) )
+
+/** Register read command */
+#define SMSC75XX_REGISTER_READ					\
+	( USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0xa1 ) )
+
+/** Get statistics command */
+#define SMSC75XX_GET_STATISTICS					\
+	( USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0xa2 ) )
+
+/** Interrupt status register */
+#define SMSC75XX_INT_STS 0x00c
+#define SMSC75XX_INT_STS_RDFO_INT	0x00400000UL	/**< RX FIFO overflow */
+#define SMSC75XX_INT_STS_PHY_INT	0x00020000UL	/**< PHY interrupt */
+
+/** Hardware configuration register */
+#define SMSC75XX_HW_CFG 0x010
+#define SMSC75XX_HW_CFG_BIR		0x00000080UL	/**< Bulk IN use NAK */
+#define SMSC75XX_HW_CFG_LRST		0x00000002UL	/**< Soft lite reset */
+
+/** Interrupt endpoint control register */
+#define SMSC75XX_INT_EP_CTL 0x038
+#define SMSC75XX_INT_EP_CTL_RDFO_EN	0x00400000UL	/**< RX FIFO overflow */
+#define SMSC75XX_INT_EP_CTL_PHY_EN	0x00020000UL	/**< PHY interrupt */
+
+/** Bulk IN delay register */
+#define SMSC75XX_BULK_IN_DLY 0x03c
+#define SMSC75XX_BULK_IN_DLY_SET(ticks)	( (ticks) << 0 ) /**< Delay / 16.7ns */
+
+/** EEPROM command register */
+#define SMSC75XX_E2P_CMD 0x040
+#define SMSC75XX_E2P_CMD_EPC_BSY	0x80000000UL	/**< EPC busy */
+#define SMSC75XX_E2P_CMD_EPC_CMD_READ	0x00000000UL	/**< READ command */
+#define SMSC75XX_E2P_CMD_EPC_ADDR(addr) ( (addr) << 0 )	/**< EPC address */
+
+/** EEPROM data register */
+#define SMSC75XX_E2P_DATA 0x044
+#define SMSC75XX_E2P_DATA_GET(e2p_data) \
+	( ( (e2p_data) >> 0 ) & 0xff )			/**< EEPROM data */
+
+/** MAC address EEPROM address */
+#define SMSC75XX_EEPROM_MAC 0x01
+
+/** Receive filtering engine control register */
+#define SMSC75XX_RFE_CTL 0x060
+#define SMSC75XX_RFE_CTL_AB		0x00000400UL	/**< Accept broadcast */
+#define SMSC75XX_RFE_CTL_AM		0x00000200UL	/**< Accept multicast */
+#define SMSC75XX_RFE_CTL_AU		0x00000100UL	/**< Accept unicast */
+
+/** FIFO controller RX FIFO control register */
+#define SMSC75XX_FCT_RX_CTL 0x090
+#define SMSC75XX_FCT_RX_CTL_EN		0x80000000UL	/**< FCT RX enable */
+#define SMSC75XX_FCT_RX_CTL_BAD		0x02000000UL	/**< Store bad frames */
+
+/** FIFO controller TX FIFO control register */
+#define SMSC75XX_FCT_TX_CTL 0x094
+#define SMSC75XX_FCT_TX_CTL_EN		0x80000000UL	/**< FCT TX enable */
+
+/** MAC receive register */
+#define SMSC75XX_MAC_RX 0x104
+#define SMSC75XX_MAC_RX_MAX_SIZE(mtu)	( (mtu) << 16 )	/**< Max frame size */
+#define SMSC75XX_MAC_RX_MAX_SIZE_DEFAULT \
+	SMSC75XX_MAC_RX_MAX_SIZE ( ETH_FRAME_LEN + 4 /* VLAN */ + 4 /* CRC */ )
+#define SMSC75XX_MAC_RX_FCS		0x00000010UL	/**< FCS stripping */
+#define SMSC75XX_MAC_RX_EN		0x00000001UL	/**< RX enable */
+
+/** MAC transmit register */
+#define SMSC75XX_MAC_TX 0x108
+#define SMSC75XX_MAC_TX_EN		0x00000001UL	/**< TX enable */
+
+/** MAC receive address high register */
+#define SMSC75XX_RX_ADDRH 0x118
+
+/** MAC receive address low register */
+#define SMSC75XX_RX_ADDRL 0x11c
+
+/** MII access register */
+#define SMSC75XX_MII_ACCESS 0x120
+#define SMSC75XX_MII_ACCESS_PHY_ADDRESS	0x00000800UL	/**< PHY address */
+#define SMSC75XX_MII_ACCESS_MIIRINDA(addr) ( (addr) << 6 ) /**< MII register */
+#define SMSC75XX_MII_ACCESS_MIIWNR	0x00000002UL	/**< MII write */
+#define SMSC75XX_MII_ACCESS_MIIBZY	0x00000001UL	/**< MII busy */
+
+/** MII data register */
+#define SMSC75XX_MII_DATA 0x124
+#define SMSC75XX_MII_DATA_SET(data)	( (data) << 0 )	/**< Set data */
+#define SMSC75XX_MII_DATA_GET(mii_data) \
+	( ( (mii_data) >> 0 ) & 0xffff )		/**< Get data */
+
+/** PHY interrupt source MII register */
+#define SMSC75XX_MII_PHY_INTR_SOURCE 29
+
+/** PHY interrupt mask MII register */
+#define SMSC75XX_MII_PHY_INTR_MASK 30
+
+/** PHY interrupt: auto-negotiation complete */
+#define SMSC75XX_PHY_INTR_ANEG_DONE	0x0040
+
+/** PHY interrupt: link down */
+#define SMSC75XX_PHY_INTR_LINK_DOWN	0x0010
+
+/** MAC address perfect filter N high register */
+#define SMSC75XX_ADDR_FILTH(n) ( 0x300 + ( 8 * (n) ) )
+#define SMSC75XX_ADDR_FILTH_VALID	0x80000000UL	/**< Address valid */
+
+/** MAC address perfect filter N low register */
+#define SMSC75XX_ADDR_FILTL(n) ( 0x304 + ( 8 * (n) ) )
+
+/** MAC address */
+union smsc75xx_mac {
+	/** MAC receive address registers */
+	struct {
+		/** MAC receive address low register */
+		uint32_t l;
+		/** MAC receive address high register */
+		uint32_t h;
+	} __attribute__ (( packed )) addr;
+	/** Raw MAC address */
+	uint8_t raw[ETH_ALEN];
+};
+
+/** Receive packet header */
+struct smsc75xx_rx_header {
+	/** RX command word */
+	uint32_t command;
+	/** VLAN tag */
+	uint16_t vtag;
+	/** Checksum */
+	uint16_t csum;
+	/** Two-byte padding used to align Ethernet payload */
+	uint16_t pad;
+} __attribute__ (( packed ));
+
+/** Receive error detected */
+#define SMSC75XX_RX_RED 0x00400000UL
+
+/** Transmit packet header */
+struct smsc75xx_tx_header {
+	/** TX command word */
+	uint32_t command;
+	/** VLAN tag */
+	uint16_t tag;
+	/** Maximum segment size */
+	uint16_t mss;
+} __attribute__ (( packed ));
+
+/** Insert frame checksum and pad */
+#define SMSC75XX_TX_FCS 0x00400000UL
+
+/** Interrupt packet format */
+struct smsc75xx_interrupt {
+	/** Current value of INT_STS register */
+	uint32_t int_sts;
+} __attribute__ (( packed ));
+
+/** Byte count statistics */
+struct smsc75xx_byte_statistics {
+	/** Unicast byte count */
+	uint32_t unicast;
+	/** Broadcast byte count */
+	uint32_t broadcast;
+	/** Multicast byte count */
+	uint32_t multicast;
+} __attribute__ (( packed ));
+
+/** Frame count statistics */
+struct smsc75xx_frame_statistics {
+	/** Unicast frames */
+	uint32_t unicast;
+	/** Broadcast frames */
+	uint32_t broadcast;
+	/** Multicast frames */
+	uint32_t multicast;
+	/** Pause frames */
+	uint32_t pause;
+	/** Frames by length category */
+	uint32_t len[7];
+} __attribute__ (( packed ));
+
+/** Receive error statistics */
+struct smsc75xx_rx_error_statistics {
+	/** FCS errors */
+	uint32_t fcs;
+	/** Alignment errors */
+	uint32_t alignment;
+	/** Fragment errors */
+	uint32_t fragment;
+	/** Jabber errors */
+	uint32_t jabber;
+	/** Undersize frame errors */
+	uint32_t undersize;
+	/** Oversize frame errors */
+	uint32_t oversize;
+	/** Dropped frame errors */
+	uint32_t dropped;
+} __attribute__ (( packed ));
+
+/** Receive statistics */
+struct smsc75xx_rx_statistics {
+	/** Error statistics */
+	struct smsc75xx_rx_error_statistics err;
+	/** Byte count statistics */
+	struct smsc75xx_byte_statistics byte;
+	/** Frame count statistics */
+	struct smsc75xx_frame_statistics frame;
+} __attribute__ (( packed ));
+
+/** Transmit error statistics */
+struct smsc75xx_tx_error_statistics {
+	/** FCS errors */
+	uint32_t fcs;
+	/** Excess deferral errors */
+	uint32_t deferral;
+	/** Carrier errors */
+	uint32_t carrier;
+	/** Bad byte count */
+	uint32_t count;
+	/** Single collisions */
+	uint32_t single;
+	/** Multiple collisions */
+	uint32_t multiple;
+	/** Excession collisions */
+	uint32_t excessive;
+	/** Late collisions */
+	uint32_t late;
+} __attribute__ (( packed ));
+
+/** Transmit statistics */
+struct smsc75xx_tx_statistics {
+	/** Error statistics */
+	struct smsc75xx_tx_error_statistics err;
+	/** Byte count statistics */
+	struct smsc75xx_byte_statistics byte;
+	/** Frame count statistics */
+	struct smsc75xx_frame_statistics frame;
+} __attribute__ (( packed ));
+
+/** Statistics */
+struct smsc75xx_statistics {
+	/** Receive statistics */
+	struct smsc75xx_rx_statistics rx;
+	/** Transmit statistics */
+	struct smsc75xx_tx_statistics tx;
+} __attribute__ (( packed ));
+
+/** A SMSC75xx network device */
+struct smsc75xx_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+	/** MII interface */
+	struct mii_interface mii;
+	/** Interrupt status */
+	uint32_t int_sts;
+};
+
+/** Reset delay (in microseconds) */
+#define SMSC75XX_RESET_DELAY_US 2
+
+/** Maximum time to wait for EEPROM (in milliseconds) */
+#define SMSC75XX_EEPROM_MAX_WAIT_MS 100
+
+/** Maximum time to wait for MII (in milliseconds) */
+#define SMSC75XX_MII_MAX_WAIT_MS 100
+
+/** Interrupt maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define SMSC75XX_INTR_MAX_FILL 2
+
+/** Bulk IN maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define SMSC75XX_IN_MAX_FILL 8
+
+/** Bulk IN buffer size */
+#define SMSC75XX_IN_MTU						\
+	( sizeof ( struct smsc75xx_rx_header ) +		\
+	  ETH_FRAME_LEN + 4 /* possible VLAN header */ )
+
+#endif /* _SMSC75XX_H */
diff --git a/roms/ipxe/src/drivers/net/sundance.c b/roms/ipxe/src/drivers/net/sundance.c
index eef7c9c7c..9127fa2cd 100644
--- a/roms/ipxe/src/drivers/net/sundance.c
+++ b/roms/ipxe/src/drivers/net/sundance.c
@@ -601,7 +601,7 @@ static int sundance_probe ( struct nic *nic, struct pci_device *pci ) {
 	sdc->nic_name = pci->id->name;
 	sdc->mtu = mtu;
 
-	pci_read_config_byte(pci, PCI_REVISION_ID, &sdc->pci_rev_id);
+	pci_read_config_byte(pci, PCI_REVISION, &sdc->pci_rev_id);
 
 	DBG ( "Device revision id: %hx\n", sdc->pci_rev_id );
 
diff --git a/roms/ipxe/src/drivers/net/tg3/tg3.c b/roms/ipxe/src/drivers/net/tg3/tg3.c
index 32ca1609c..42bfa2d99 100644
--- a/roms/ipxe/src/drivers/net/tg3/tg3.c
+++ b/roms/ipxe/src/drivers/net/tg3/tg3.c
@@ -928,6 +928,7 @@ static struct pci_device_id tg3_nics[] = {
 	PCI_ROM(0x14e4, 0x16b6, "14e4-16b6", "14e4-16b6", 0),
 	PCI_ROM(0x14e4, 0x1657, "14e4-1657", "14e4-1657", 0),
 	PCI_ROM(0x14e4, 0x165f, "14e4-165f", "14e4-165f", 0),
+	PCI_ROM(0x14e4, 0x1686, "14e4-1686", "14e4-1686", 0),
 	PCI_ROM(0x1148, 0x4400, "1148-4400", "1148-4400", 0),
 	PCI_ROM(0x1148, 0x4500, "1148-4500", "1148-4500", 0),
 	PCI_ROM(0x173b, 0x03e8, "173b-03e8", "173b-03e8", 0),
diff --git a/roms/ipxe/src/drivers/net/tg3/tg3.h b/roms/ipxe/src/drivers/net/tg3/tg3.h
index 660368394..2b85b065b 100644
--- a/roms/ipxe/src/drivers/net/tg3/tg3.h
+++ b/roms/ipxe/src/drivers/net/tg3/tg3.h
@@ -131,6 +131,10 @@
 #define PCI_DEVICE_ID_TIGON3_5901_2	0x170e
 #define PCI_DEVICE_ID_TIGON3_5906	0x1712
 #define PCI_DEVICE_ID_TIGON3_5906M	0x1713
+#define PCI_VENDOR_ID_COMPAQ		0x0e11
+#define PCI_VENDOR_ID_IBM		0x1014
+#define PCI_VENDOR_ID_DELL		0x1028
+#define PCI_VENDOR_ID_3COM		0x10b7
 /* </pci_ids.h> */
 
 #define SPEED_10			10
@@ -185,6 +189,7 @@
 #define  TG3PCI_DEVICE_TIGON3_57761	 0x16b0
 #define  TG3PCI_DEVICE_TIGON3_57762	 0x1682
 #define  TG3PCI_DEVICE_TIGON3_57765	 0x16b4
+#define  TG3PCI_DEVICE_TIGON3_57766	 0x1686
 #define  TG3PCI_DEVICE_TIGON3_57791	 0x16b2
 #define  TG3PCI_DEVICE_TIGON3_57795	 0x16b6
 #define  TG3PCI_DEVICE_TIGON3_5719	 0x1657
diff --git a/roms/ipxe/src/drivers/net/tg3/tg3_hw.c b/roms/ipxe/src/drivers/net/tg3/tg3_hw.c
index 3a481aba3..50353cf36 100644
--- a/roms/ipxe/src/drivers/net/tg3/tg3_hw.c
+++ b/roms/ipxe/src/drivers/net/tg3/tg3_hw.c
@@ -436,6 +436,7 @@ int tg3_get_invariants(struct tg3 *tp)
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765 ||
+			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795)
 			pci_read_config_dword(tp->pdev,
diff --git a/roms/ipxe/src/drivers/net/virtio-net.c b/roms/ipxe/src/drivers/net/virtio-net.c
index d5fd81979..533ccb0c6 100644
--- a/roms/ipxe/src/drivers/net/virtio-net.c
+++ b/roms/ipxe/src/drivers/net/virtio-net.c
@@ -20,7 +20,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <stdlib.h>
@@ -131,8 +131,8 @@ static void virtnet_enqueue_iob ( struct net_device *netdev,
 		},
 	};
 
-	DBGC ( virtnet, "VIRTIO-NET %p enqueuing iobuf %p on vq %d\n",
-	       virtnet, iobuf, vq_idx );
+	DBGC2 ( virtnet, "VIRTIO-NET %p enqueuing iobuf %p on vq %d\n",
+		virtnet, iobuf, vq_idx );
 
 	vring_add_buf ( vq, list, out, in, iobuf, 0 );
 	vring_kick ( virtnet->ioaddr, vq, 1 );
@@ -256,8 +256,8 @@ static void virtnet_process_tx_packets ( struct net_device *netdev ) {
 	while ( vring_more_used ( tx_vq ) ) {
 		struct io_buffer *iobuf = vring_get_buf ( tx_vq, NULL );
 
-		DBGC ( virtnet, "VIRTIO-NET %p tx complete iobuf %p\n",
-		       virtnet, iobuf );
+		DBGC2 ( virtnet, "VIRTIO-NET %p tx complete iobuf %p\n",
+			virtnet, iobuf );
 
 		netdev_tx_complete ( netdev, iobuf );
 	}
@@ -283,8 +283,8 @@ static void virtnet_process_rx_packets ( struct net_device *netdev ) {
 		iob_unput ( iobuf, RX_BUF_SIZE );
 		iob_put ( iobuf, len - sizeof ( struct virtio_net_hdr ) );
 
-		DBGC ( virtnet, "VIRTIO-NET %p rx complete iobuf %p len %zd\n",
-		       virtnet, iobuf, iob_len ( iobuf ) );
+		DBGC2 ( virtnet, "VIRTIO-NET %p rx complete iobuf %p len %zd\n",
+			virtnet, iobuf, iob_len ( iobuf ) );
 
 		/* Pass completed packet to the network stack */
 		netdev_rx ( netdev, iobuf );
diff --git a/roms/ipxe/src/drivers/net/vmxnet3.c b/roms/ipxe/src/drivers/net/vmxnet3.c
index 31082bf6f..8d4f4b843 100644
--- a/roms/ipxe/src/drivers/net/vmxnet3.c
+++ b/roms/ipxe/src/drivers/net/vmxnet3.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/drivers/net/vmxnet3.h b/roms/ipxe/src/drivers/net/vmxnet3.h
index db313d4b8..a1671d9dd 100644
--- a/roms/ipxe/src/drivers/net/vmxnet3.h
+++ b/roms/ipxe/src/drivers/net/vmxnet3.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/drivers/net/vxge/vxge.c b/roms/ipxe/src/drivers/net/vxge/vxge.c
index bf20ec43c..d50ac05b5 100644
--- a/roms/ipxe/src/drivers/net/vxge/vxge.c
+++ b/roms/ipxe/src/drivers/net/vxge/vxge.c
@@ -5,10 +5,11 @@
  * as "vxge" even though the code is in vxge_* named files.
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE(GPL2_OR_LATER_OR_UBDL);
 
 #include <ipxe/pci.h>
 
+PROVIDE_REQUIRING_SYMBOL();
 REQUIRE_OBJECT(vxge_main);
 
 /** vxge PCI IDs for util/parserom.pl which are put into bin/NIC */
diff --git a/roms/ipxe/src/drivers/net/vxge/vxge_main.c b/roms/ipxe/src/drivers/net/vxge/vxge_main.c
index 130eab617..8b099c0e2 100644
--- a/roms/ipxe/src/drivers/net/vxge/vxge_main.c
+++ b/roms/ipxe/src/drivers/net/vxge/vxge_main.c
@@ -509,7 +509,7 @@ vxge_probe(struct pci_device *pdev)
 	vxge_debug(VXGE_INFO, "vxge_probe for device " PCI_FMT "\n",
 			PCI_ARGS(pdev));
 
-	pci_read_config_byte(pdev, PCI_REVISION_ID, &revision);
+	pci_read_config_byte(pdev, PCI_REVISION, &revision);
 	titan1 = is_titan1(pdev->device, revision);
 
 	mmio_start = pci_bar_start(pdev, PCI_BASE_ADDRESS_0);
diff --git a/roms/ipxe/src/drivers/net/w89c840.c b/roms/ipxe/src/drivers/net/w89c840.c
index ce638ab99..d8144a8ce 100644
--- a/roms/ipxe/src/drivers/net/w89c840.c
+++ b/roms/ipxe/src/drivers/net/w89c840.c
@@ -641,7 +641,9 @@ static int w89c840_probe ( struct nic *nic, struct pci_device *p ) {
 
     ioaddr = ioaddr & ~3; /* Mask the bit that says "this is an io addr" */
 
+#define PCI_VENDOR_ID_WINBOND2		0x1050
 #define PCI_DEVICE_ID_WINBOND2_89C840   0x0840
+#define PCI_VENDOR_ID_COMPEX		0x11f6
 #define PCI_DEVICE_ID_COMPEX_RL100ATX   0x2011
 
     /* From Matt Hortman <mbhortman@acpthinclient.com> */
diff --git a/roms/ipxe/src/drivers/nvs/nvs.c b/roms/ipxe/src/drivers/nvs/nvs.c
index ccb2145bd..af7c466c4 100644
--- a/roms/ipxe/src/drivers/nvs/nvs.c
+++ b/roms/ipxe/src/drivers/nvs/nvs.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/drivers/nvs/nvsvpd.c b/roms/ipxe/src/drivers/nvs/nvsvpd.c
index 33148d5b9..3e88531c7 100644
--- a/roms/ipxe/src/drivers/nvs/nvsvpd.c
+++ b/roms/ipxe/src/drivers/nvs/nvsvpd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/drivers/nvs/spi.c b/roms/ipxe/src/drivers/nvs/spi.c
index 84613b9dd..dcfe1af91 100644
--- a/roms/ipxe/src/drivers/nvs/spi.c
+++ b/roms/ipxe/src/drivers/nvs/spi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/drivers/nvs/threewire.c b/roms/ipxe/src/drivers/nvs/threewire.c
index 53f1ad8de..547f35382 100644
--- a/roms/ipxe/src/drivers/nvs/threewire.c
+++ b/roms/ipxe/src/drivers/nvs/threewire.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
diff --git a/roms/ipxe/src/drivers/usb/ehci.c b/roms/ipxe/src/drivers/usb/ehci.c
new file mode 100644
index 000000000..4124692a6
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/ehci.c
@@ -0,0 +1,1994 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+#include <ipxe/init.h>
+#include "ehci.h"
+
+/** @file
+ *
+ * USB Enhanced Host Controller Interface (EHCI) driver
+ *
+ */
+
+/**
+ * Construct error code from transfer descriptor status
+ *
+ * @v status		Transfer descriptor status
+ * @ret rc		Error code
+ *
+ * Bits 2-5 of the status code provide some indication as to the root
+ * cause of the error.  We incorporate these into the error code as
+ * reported to usb_complete_err().
+ */
+#define EIO_STATUS( status ) EUNIQ ( EINFO_EIO, ( ( (status) >> 2 ) & 0xf ) )
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Initialise device
+ *
+ * @v ehci		EHCI device
+ * @v regs		MMIO registers
+ */
+static void ehci_init ( struct ehci_device *ehci, void *regs ) {
+	uint32_t hcsparams;
+	uint32_t hccparams;
+	size_t caplength;
+
+	/* Locate capability and operational registers */
+	ehci->cap = regs;
+	caplength = readb ( ehci->cap + EHCI_CAP_CAPLENGTH );
+	ehci->op = ( ehci->cap + caplength );
+	DBGC2 ( ehci, "EHCI %s cap %08lx op %08lx\n", ehci->name,
+		virt_to_phys ( ehci->cap ), virt_to_phys ( ehci->op ) );
+
+	/* Read structural parameters */
+	hcsparams = readl ( ehci->cap + EHCI_CAP_HCSPARAMS );
+	ehci->ports = EHCI_HCSPARAMS_PORTS ( hcsparams );
+	DBGC ( ehci, "EHCI %s has %d ports\n", ehci->name, ehci->ports );
+
+	/* Read capability parameters 1 */
+	hccparams = readl ( ehci->cap + EHCI_CAP_HCCPARAMS );
+	ehci->addr64 = EHCI_HCCPARAMS_ADDR64 ( hccparams );
+	ehci->flsize = ( EHCI_HCCPARAMS_FLSIZE ( hccparams ) ?
+			 EHCI_FLSIZE_SMALL : EHCI_FLSIZE_DEFAULT );
+	ehci->eecp = EHCI_HCCPARAMS_EECP ( hccparams );
+	DBGC2 ( ehci, "EHCI %s %d-bit flsize %d\n", ehci->name,
+		( ehci->addr64 ? 64 : 32 ), ehci->flsize );
+}
+
+/**
+ * Find extended capability
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ * @v id		Capability ID
+ * @v offset		Offset to previous extended capability instance, or zero
+ * @ret offset		Offset to extended capability, or zero if not found
+ */
+static unsigned int ehci_extended_capability ( struct ehci_device *ehci,
+					       struct pci_device *pci,
+					       unsigned int id,
+					       unsigned int offset ) {
+	uint32_t eecp;
+
+	/* Locate the extended capability */
+	while ( 1 ) {
+
+		/* Locate first or next capability as applicable */
+		if ( offset ) {
+			pci_read_config_dword ( pci, offset, &eecp );
+			offset = EHCI_EECP_NEXT ( eecp );
+		} else {
+			offset = ehci->eecp;
+		}
+		if ( ! offset )
+			return 0;
+
+		/* Check if this is the requested capability */
+		pci_read_config_dword ( pci, offset, &eecp );
+		if ( EHCI_EECP_ID ( eecp ) == id )
+			return offset;
+	}
+}
+
+/**
+ * Calculate buffer alignment
+ *
+ * @v len		Length
+ * @ret align		Buffer alignment
+ *
+ * Determine alignment required for a buffer which must be aligned to
+ * at least EHCI_MIN_ALIGN and which must not cross a page boundary.
+ */
+static inline size_t ehci_align ( size_t len ) {
+	size_t align;
+
+	/* Align to own length (rounded up to a power of two) */
+	align = ( 1 << fls ( len - 1 ) );
+
+	/* Round up to EHCI_MIN_ALIGN if needed */
+	if ( align < EHCI_MIN_ALIGN )
+		align = EHCI_MIN_ALIGN;
+
+	return align;
+}
+
+/**
+ * Check control data structure reachability
+ *
+ * @v ehci		EHCI device
+ * @v ptr		Data structure pointer
+ * @ret rc		Return status code
+ */
+static int ehci_ctrl_reachable ( struct ehci_device *ehci, void *ptr ) {
+	physaddr_t phys = virt_to_phys ( ptr );
+	uint32_t segment;
+
+	/* Always reachable in a 32-bit build */
+	if ( sizeof ( physaddr_t ) <= sizeof ( uint32_t ) )
+		return 0;
+
+	/* Reachable only if control segment matches in a 64-bit build */
+	segment = ( ( ( uint64_t ) phys ) >> 32 );
+	if ( segment == ehci->ctrldssegment )
+		return 0;
+
+	return -ENOTSUP;
+}
+
+/******************************************************************************
+ *
+ * USB legacy support
+ *
+ ******************************************************************************
+ */
+
+/** Prevent the release of ownership back to BIOS */
+static int ehci_legacy_prevent_release;
+
+/**
+ * Initialise USB legacy support
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ */
+static void ehci_legacy_init ( struct ehci_device *ehci,
+			       struct pci_device *pci ) {
+	unsigned int legacy;
+	uint8_t bios;
+
+	/* Locate USB legacy support capability (if present) */
+	legacy = ehci_extended_capability ( ehci, pci, EHCI_EECP_ID_LEGACY, 0 );
+	if ( ! legacy ) {
+		/* Not an error; capability may not be present */
+		DBGC ( ehci, "EHCI %s has no USB legacy support capability\n",
+		       ehci->name );
+		return;
+	}
+
+	/* Check if legacy USB support is enabled */
+	pci_read_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_BIOS ), &bios );
+	if ( ! ( bios & EHCI_USBLEGSUP_BIOS_OWNED ) ) {
+		/* Not an error; already owned by OS */
+		DBGC ( ehci, "EHCI %s USB legacy support already disabled\n",
+		       ehci->name );
+		return;
+	}
+
+	/* Record presence of USB legacy support capability */
+	ehci->legacy = legacy;
+}
+
+/**
+ * Claim ownership from BIOS
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ */
+static void ehci_legacy_claim ( struct ehci_device *ehci,
+				struct pci_device *pci ) {
+	unsigned int legacy = ehci->legacy;
+	uint32_t ctlsts;
+	uint8_t bios;
+	unsigned int i;
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! legacy )
+		return;
+
+	/* Claim ownership */
+	pci_write_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_OS ),
+				EHCI_USBLEGSUP_OS_OWNED );
+
+	/* Wait for BIOS to release ownership */
+	for ( i = 0 ; i < EHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if BIOS has released ownership */
+		pci_read_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_BIOS ),
+				       &bios );
+		if ( ! ( bios & EHCI_USBLEGSUP_BIOS_OWNED ) ) {
+			DBGC ( ehci, "EHCI %s claimed ownership from BIOS\n",
+			       ehci->name );
+			pci_read_config_dword ( pci, ( legacy +
+						       EHCI_USBLEGSUP_CTLSTS ),
+						&ctlsts );
+			if ( ctlsts ) {
+				DBGC ( ehci, "EHCI %s warning: BIOS retained "
+				       "SMIs: %08x\n", ehci->name, ctlsts );
+			}
+			return;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* BIOS did not release ownership.  Claim it forcibly by
+	 * disabling all SMIs.
+	 */
+	DBGC ( ehci, "EHCI %s could not claim ownership from BIOS: forcibly "
+	       "disabling SMIs\n", ehci->name );
+	pci_write_config_dword ( pci, ( legacy + EHCI_USBLEGSUP_CTLSTS ), 0 );
+}
+
+/**
+ * Release ownership back to BIOS
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ */
+static void ehci_legacy_release ( struct ehci_device *ehci,
+				  struct pci_device *pci ) {
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! ehci->legacy )
+		return;
+
+	/* Do nothing if releasing ownership is prevented */
+	if ( ehci_legacy_prevent_release ) {
+		DBGC ( ehci, "EHCI %s not releasing ownership to BIOS\n",
+		       ehci->name );
+		return;
+	}
+
+	/* Release ownership */
+	pci_write_config_byte ( pci, ( ehci->legacy + EHCI_USBLEGSUP_OS ), 0 );
+	DBGC ( ehci, "EHCI %s released ownership to BIOS\n", ehci->name );
+}
+
+/******************************************************************************
+ *
+ * Companion controllers
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Poll child companion controllers
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_poll_companions ( struct ehci_device *ehci ) {
+	struct usb_bus *bus;
+	struct device_description *desc;
+
+	/* Poll any USB buses belonging to child companion controllers */
+	for_each_usb_bus ( bus ) {
+
+		/* Get underlying devices description */
+		desc = &bus->dev->desc;
+
+		/* Skip buses that are not PCI devices */
+		if ( desc->bus_type != BUS_TYPE_PCI )
+			continue;
+
+		/* Skip buses that are not part of the same PCI device */
+		if ( PCI_FIRST_FUNC ( desc->location ) !=
+		     PCI_FIRST_FUNC ( ehci->bus->dev->desc.location ) )
+			continue;
+
+		/* Skip buses that are not UHCI or OHCI PCI devices */
+		if ( ( desc->class != PCI_CLASS ( PCI_CLASS_SERIAL,
+						  PCI_CLASS_SERIAL_USB,
+						  PCI_CLASS_SERIAL_USB_UHCI ))&&
+		     ( desc->class != PCI_CLASS ( PCI_CLASS_SERIAL,
+						  PCI_CLASS_SERIAL_USB,
+						  PCI_CLASS_SERIAL_USB_OHCI ) ))
+			continue;
+
+		/* Poll child companion controller bus */
+		DBGC2 ( ehci, "EHCI %s polling companion %s\n",
+			ehci->name, bus->name );
+		usb_poll ( bus );
+	}
+}
+
+/**
+ * Locate EHCI companion controller
+ *
+ * @v pci		PCI device
+ * @ret busdevfn	EHCI companion controller bus:dev.fn (if any)
+ */
+unsigned int ehci_companion ( struct pci_device *pci ) {
+	struct pci_device tmp;
+	unsigned int busdevfn;
+	int rc;
+
+	/* Look for an EHCI function on the same PCI device */
+	busdevfn = pci->busdevfn;
+	while ( ++busdevfn <= PCI_LAST_FUNC ( pci->busdevfn ) ) {
+		pci_init ( &tmp, busdevfn );
+		if ( ( rc = pci_read_config ( &tmp ) ) != 0 )
+			continue;
+		if ( tmp.class == PCI_CLASS ( PCI_CLASS_SERIAL,
+					      PCI_CLASS_SERIAL_USB,
+					      PCI_CLASS_SERIAL_USB_EHCI ) )
+			return busdevfn;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Run / stop / reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Start EHCI device
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_run ( struct ehci_device *ehci ) {
+	uint32_t usbcmd;
+
+	/* Set run/stop bit */
+	usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+	usbcmd &= ~EHCI_USBCMD_FLSIZE_MASK;
+	usbcmd |= ( EHCI_USBCMD_RUN | EHCI_USBCMD_FLSIZE ( ehci->flsize ) |
+		    EHCI_USBCMD_PERIODIC | EHCI_USBCMD_ASYNC );
+	writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
+}
+
+/**
+ * Stop EHCI device
+ *
+ * @v ehci		EHCI device
+ * @ret rc		Return status code
+ */
+static int ehci_stop ( struct ehci_device *ehci ) {
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	unsigned int i;
+
+	/* Clear run/stop bit */
+	usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+	usbcmd &= ~( EHCI_USBCMD_RUN | EHCI_USBCMD_PERIODIC |
+		     EHCI_USBCMD_ASYNC );
+	writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
+
+	/* Wait for device to stop */
+	for ( i = 0 ; i < EHCI_STOP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if device is stopped */
+		usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
+		if ( usbsts & EHCI_USBSTS_HCH )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( ehci, "EHCI %s timed out waiting for stop\n", ehci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Reset EHCI device
+ *
+ * @v ehci		EHCI device
+ * @ret rc		Return status code
+ */
+static int ehci_reset ( struct ehci_device *ehci ) {
+	uint32_t usbcmd;
+	unsigned int i;
+	int rc;
+
+	/* The EHCI specification states that resetting a running
+	 * device may result in undefined behaviour, so try stopping
+	 * it first.
+	 */
+	if ( ( rc = ehci_stop ( ehci ) ) != 0 ) {
+		/* Ignore errors and attempt to reset the device anyway */
+	}
+
+	/* Reset device */
+	writel ( EHCI_USBCMD_HCRST, ehci->op + EHCI_OP_USBCMD );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < EHCI_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if reset is complete */
+		usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+		if ( ! ( usbcmd & EHCI_USBCMD_HCRST ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( ehci, "EHCI %s timed out waiting for reset\n", ehci->name );
+	return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Transfer descriptor rings
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate transfer descriptor ring
+ *
+ * @v ehci		EHCI device
+ * @v ring		Transfer descriptor ring
+ * @ret rc		Return status code
+ */
+static int ehci_ring_alloc ( struct ehci_device *ehci,
+			     struct ehci_ring *ring ) {
+	struct ehci_transfer_descriptor *desc;
+	struct ehci_transfer_descriptor *next;
+	unsigned int i;
+	size_t len;
+	uint32_t link;
+	int rc;
+
+	/* Initialise structure */
+	memset ( ring, 0, sizeof ( *ring ) );
+
+	/* Allocate I/O buffers */
+	ring->iobuf = zalloc ( EHCI_RING_COUNT * sizeof ( ring->iobuf[0] ) );
+	if ( ! ring->iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc_iobuf;
+	}
+
+	/* Allocate queue head */
+	ring->head = malloc_dma ( sizeof ( *ring->head ),
+				  ehci_align ( sizeof ( *ring->head ) ) );
+	if ( ! ring->head ) {
+		rc = -ENOMEM;
+		goto err_alloc_queue;
+	}
+	if ( ( rc = ehci_ctrl_reachable ( ehci, ring->head ) ) != 0 ) {
+		DBGC ( ehci, "EHCI %s queue head unreachable\n", ehci->name );
+		goto err_unreachable_queue;
+	}
+	memset ( ring->head, 0, sizeof ( *ring->head ) );
+
+	/* Allocate transfer descriptors */
+	len = ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) );
+	ring->desc = malloc_dma ( len, sizeof ( ring->desc[0] ) );
+	if ( ! ring->desc ) {
+		rc = -ENOMEM;
+		goto err_alloc_desc;
+	}
+	memset ( ring->desc, 0, len );
+
+	/* Initialise transfer descriptors */
+	for ( i = 0 ; i < EHCI_RING_COUNT ; i++ ) {
+		desc = &ring->desc[i];
+		if ( ( rc = ehci_ctrl_reachable ( ehci, desc ) ) != 0 ) {
+			DBGC ( ehci, "EHCI %s descriptor unreachable\n",
+			       ehci->name );
+			goto err_unreachable_desc;
+		}
+		next = &ring->desc[ ( i + 1 ) % EHCI_RING_COUNT ];
+		link = virt_to_phys ( next );
+		desc->next = cpu_to_le32 ( link );
+		desc->alt = cpu_to_le32 ( link );
+	}
+
+	/* Initialise queue head */
+	link = virt_to_phys ( &ring->desc[0] );
+	ring->head->cache.next = cpu_to_le32 ( link );
+
+	return 0;
+
+ err_unreachable_desc:
+	free_dma ( ring->desc, len );
+ err_alloc_desc:
+ err_unreachable_queue:
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+ err_alloc_queue:
+	free ( ring->iobuf );
+ err_alloc_iobuf:
+	return rc;
+}
+
+/**
+ * Free transfer descriptor ring
+ *
+ * @v ring		Transfer descriptor ring
+ */
+static void ehci_ring_free ( struct ehci_ring *ring ) {
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( ehci_ring_fill ( ring ) == 0 );
+	for ( i = 0 ; i < EHCI_RING_COUNT ; i++ )
+		assert ( ring->iobuf[i] == NULL );
+
+	/* Free transfer descriptors */
+	free_dma ( ring->desc, ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) ) );
+
+	/* Free queue head */
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+
+	/* Free I/O buffers */
+	free ( ring->iobuf );
+}
+
+/**
+ * Enqueue transfer descriptors
+ *
+ * @v ehci		EHCI device
+ * @v ring		Transfer descriptor ring
+ * @v iobuf		I/O buffer
+ * @v xfers		Transfers
+ * @v count		Number of transfers
+ * @ret rc		Return status code
+ */
+static int ehci_enqueue ( struct ehci_device *ehci, struct ehci_ring *ring,
+			  struct io_buffer *iobuf,
+			  const struct ehci_transfer *xfer,
+			  unsigned int count ) {
+	struct ehci_transfer_descriptor *desc;
+	physaddr_t phys;
+	void *data;
+	size_t len;
+	size_t offset;
+	size_t frag_len;
+	unsigned int toggle;
+	unsigned int index;
+	unsigned int i;
+
+	/* Sanity check */
+	assert ( iobuf != NULL );
+	assert ( count > 0 );
+
+	/* Fail if ring does not have sufficient space */
+	if ( ehci_ring_remaining ( ring ) < count )
+		return -ENOBUFS;
+
+	/* Fail if any portion is unreachable */
+	for ( i = 0 ; i < count ; i++ ) {
+		phys = ( virt_to_phys ( xfer[i].data ) + xfer[i].len - 1 );
+		if ( ( phys > 0xffffffffUL ) && ( ! ehci->addr64 ) )
+			return -ENOTSUP;
+	}
+
+	/* Enqueue each transfer, recording the I/O buffer with the last */
+	for ( ; count ; ring->prod++, xfer++ ) {
+
+		/* Populate descriptor header */
+		index = ( ring->prod % EHCI_RING_COUNT );
+		desc = &ring->desc[index];
+		toggle = ( xfer->flags & EHCI_FL_TOGGLE );
+		assert ( xfer->len <= EHCI_LEN_MASK );
+		assert ( EHCI_FL_TOGGLE == EHCI_LEN_TOGGLE );
+		desc->len = cpu_to_le16 ( xfer->len | toggle );
+		desc->flags = ( xfer->flags | EHCI_FL_CERR_MAX );
+
+		/* Populate buffer pointers */
+		data = xfer->data;
+		len = xfer->len;
+		for ( i = 0 ; len ; i++ ) {
+
+			/* Calculate length of this fragment */
+			phys = virt_to_phys ( data );
+			offset = ( phys & ( EHCI_PAGE_ALIGN - 1 ) );
+			frag_len = ( EHCI_PAGE_ALIGN - offset );
+			if ( frag_len > len )
+				frag_len = len;
+
+			/* Sanity checks */
+			assert ( ( i == 0 ) || ( offset == 0 ) );
+			assert ( i < ( sizeof ( desc->low ) /
+				       sizeof ( desc->low[0] ) ) );
+
+			/* Populate buffer pointer */
+			desc->low[i] = cpu_to_le32 ( phys );
+			if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) {
+				desc->high[i] =
+					cpu_to_le32 ( ((uint64_t) phys) >> 32 );
+			}
+
+			/* Move to next fragment */
+			data += frag_len;
+			len -= frag_len;
+		}
+
+		/* Ensure everything is valid before activating descriptor */
+		wmb();
+		desc->status = EHCI_STATUS_ACTIVE;
+
+		/* Record I/O buffer against last ring index */
+		if ( --count == 0 )
+			ring->iobuf[index] = iobuf;
+	}
+
+	return 0;
+}
+
+/**
+ * Dequeue a transfer descriptor
+ *
+ * @v ring		Transfer descriptor ring
+ * @ret iobuf		I/O buffer (or NULL)
+ */
+static struct io_buffer * ehci_dequeue ( struct ehci_ring *ring ) {
+	struct ehci_transfer_descriptor *desc;
+	struct io_buffer *iobuf;
+	unsigned int index = ( ring->cons % EHCI_RING_COUNT );
+
+	/* Sanity check */
+	assert ( ehci_ring_fill ( ring ) > 0 );
+
+	/* Mark descriptor as inactive (and not halted) */
+	desc = &ring->desc[index];
+	desc->status = 0;
+
+	/* Retrieve I/O buffer */
+	iobuf = ring->iobuf[index];
+	ring->iobuf[index] = NULL;
+
+	/* Update consumer counter */
+	ring->cons++;
+
+	return iobuf;
+}
+
+/******************************************************************************
+ *
+ * Schedule management
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get link value for a queue head
+ *
+ * @v queue		Queue head
+ * @ret link		Link value
+ */
+static inline uint32_t ehci_link_qh ( struct ehci_queue_head *queue ) {
+
+	return ( virt_to_phys ( queue ) | EHCI_LINK_TYPE_QH );
+}
+
+/**
+ * (Re)build asynchronous schedule
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_async_schedule ( struct ehci_device *ehci ) {
+	struct ehci_endpoint *endpoint;
+	struct ehci_queue_head *queue;
+	uint32_t link;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	link = ehci_link_qh ( ehci->head );
+	list_for_each_entry_reverse ( endpoint, &ehci->async, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		link = ehci_link_qh ( queue );
+	}
+	ehci->head->link = cpu_to_le32 ( link );
+	wmb();
+}
+
+/**
+ * Add endpoint to asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_async_add ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+
+	/* Add to end of schedule */
+	list_add_tail ( &endpoint->schedule, &ehci->async );
+
+	/* Rebuild schedule */
+	ehci_async_schedule ( ehci );
+}
+
+/**
+ * Remove endpoint from asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_async_del ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	unsigned int i;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &ehci->async, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	ehci_async_schedule ( ehci );
+
+	/* Request notification when asynchronous schedule advances */
+	usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+	usbcmd |= EHCI_USBCMD_ASYNC_ADVANCE;
+	writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
+
+	/* Wait for asynchronous schedule to advance */
+	for ( i = 0 ; i < EHCI_ASYNC_ADVANCE_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for asynchronous schedule advancing */
+		usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
+		if ( usbsts & EHCI_USBSTS_ASYNC_ADVANCE ) {
+			usbsts &= ~EHCI_USBSTS_CHANGE;
+			usbsts |= EHCI_USBSTS_ASYNC_ADVANCE;
+			writel ( usbsts, ehci->op + EHCI_OP_USBSTS );
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* Bad things will probably happen now */
+	DBGC ( ehci, "EHCI %s timed out waiting for asynchronous schedule "
+	       "to advance\n", ehci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * (Re)build periodic schedule
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_periodic_schedule ( struct ehci_device *ehci ) {
+	struct ehci_endpoint *endpoint;
+	struct ehci_queue_head *queue;
+	uint32_t link;
+	unsigned int frames;
+	unsigned int max_interval;
+	unsigned int i;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	DBGCP ( ehci, "EHCI %s periodic schedule: ", ehci->name );
+	link = EHCI_LINK_TERMINATE;
+	list_for_each_entry_reverse ( endpoint, &ehci->periodic, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		DBGCP ( ehci, "%s%d",
+			( ( link == EHCI_LINK_TERMINATE ) ? "" : "<-" ),
+			endpoint->ep->interval );
+		link = ehci_link_qh ( queue );
+	}
+	DBGCP ( ehci, "\n" );
+
+	/* Populate periodic frame list */
+	DBGCP ( ehci, "EHCI %s periodic frame list:", ehci->name );
+	frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
+	for ( i = 0 ; i < frames ; i++ ) {
+
+		/* Calculate maximum interval (in microframes) which
+		 * may appear as part of this frame list.
+		 */
+		if ( i == 0 ) {
+			/* Start of list: include all endpoints */
+			max_interval = -1U;
+		} else {
+			/* Calculate highest power-of-two frame interval */
+			max_interval = ( 1 << ( ffs ( i ) - 1 ) );
+			/* Convert to microframes */
+			max_interval <<= 3;
+			/* Round up to nearest 2^n-1 */
+			max_interval = ( ( max_interval << 1 ) - 1 );
+		}
+
+		/* Find first endpoint in schedule satisfying this
+		 * maximum interval constraint.
+		 */
+		link = EHCI_LINK_TERMINATE;
+		list_for_each_entry ( endpoint, &ehci->periodic, schedule ) {
+			if ( endpoint->ep->interval <= max_interval ) {
+				queue = endpoint->ring.head;
+				link = ehci_link_qh ( queue );
+				DBGCP ( ehci, " %d:%d",
+					i, endpoint->ep->interval );
+				break;
+			}
+		}
+		ehci->frame[i].link = cpu_to_le32 ( link );
+	}
+	wmb();
+	DBGCP ( ehci, "\n" );
+}
+
+/**
+ * Add endpoint to periodic schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_periodic_add ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+	struct ehci_endpoint *before;
+	unsigned int interval = endpoint->ep->interval;
+
+	/* Find first endpoint with a smaller interval */
+	list_for_each_entry ( before, &ehci->periodic, schedule ) {
+		if ( before->ep->interval < interval )
+			break;
+	}
+	list_add_tail ( &endpoint->schedule, &before->schedule );
+
+	/* Rebuild schedule */
+	ehci_periodic_schedule ( ehci );
+}
+
+/**
+ * Remove endpoint from periodic schedule
+ *
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_periodic_del ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &ehci->periodic, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	ehci_periodic_schedule ( ehci );
+
+	/* Delay for a whole USB frame (with a 100% safety margin) */
+	mdelay ( 2 );
+
+	return 0;
+}
+
+/**
+ * Add endpoint to appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_schedule_add ( struct ehci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		ehci_periodic_add ( endpoint );
+	} else {
+		ehci_async_add ( endpoint );
+	}
+}
+
+/**
+ * Remove endpoint from appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_schedule_del ( struct ehci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		return ehci_periodic_del ( endpoint );
+	} else {
+		return ehci_async_del ( endpoint );
+	}
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Determine endpoint characteristics
+ *
+ * @v ep		USB endpoint
+ * @ret chr		Endpoint characteristics
+ */
+static uint32_t ehci_endpoint_characteristics ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	uint32_t chr;
+
+	/* Determine basic characteristics */
+	chr = ( EHCI_CHR_ADDRESS ( usb->address ) |
+		EHCI_CHR_ENDPOINT ( ep->address ) |
+		EHCI_CHR_MAX_LEN ( ep->mtu ) );
+
+	/* Control endpoints require manual control of the data toggle */
+	if ( attr == USB_ENDPOINT_ATTR_CONTROL )
+		chr |= EHCI_CHR_TOGGLE;
+
+	/* Determine endpoint speed */
+	if ( usb->port->speed == USB_SPEED_HIGH ) {
+		chr |= EHCI_CHR_EPS_HIGH;
+	} else {
+		if ( usb->port->speed == USB_SPEED_FULL ) {
+			chr |= EHCI_CHR_EPS_FULL;
+		} else {
+			chr |= EHCI_CHR_EPS_LOW;
+		}
+		if ( attr == USB_ENDPOINT_ATTR_CONTROL )
+			chr |= EHCI_CHR_CONTROL;
+	}
+
+	return chr;
+}
+
+/**
+ * Determine endpoint capabilities
+ *
+ * @v ep		USB endpoint
+ * @ret cap		Endpoint capabilities
+ */
+static uint32_t ehci_endpoint_capabilities ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *tt = usb_transaction_translator ( usb );
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	uint32_t cap;
+	unsigned int i;
+
+	/* Determine basic capabilities */
+	cap = EHCI_CAP_MULT ( ep->burst + 1 );
+
+	/* Determine interrupt schedule mask, if applicable */
+	if ( ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) &&
+	     ( ( ep->interval != 0 ) /* avoid infinite loop */ ) ) {
+		for ( i = 0 ; i < 8 /* microframes per frame */ ;
+		      i += ep->interval ) {
+			cap |= EHCI_CAP_INTR_SCHED ( i );
+		}
+	}
+
+	/* Set transaction translator hub address and port, if applicable */
+	if ( tt ) {
+		assert ( tt->hub->usb );
+		cap |= ( EHCI_CAP_TT_HUB ( tt->hub->usb->address ) |
+			 EHCI_CAP_TT_PORT ( tt->address ) );
+		if ( attr == USB_ENDPOINT_ATTR_INTERRUPT )
+			cap |= EHCI_CAP_SPLIT_SCHED_DEFAULT;
+	}
+
+	return cap;
+}
+
+/**
+ * Update endpoint characteristics and capabilities
+ *
+ * @v ep		USB endpoint
+ */
+static void ehci_endpoint_update ( struct usb_endpoint *ep ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_queue_head *head;
+
+	/* Update queue characteristics and capabilities */
+	head = endpoint->ring.head;
+	head->chr = cpu_to_le32 ( ehci_endpoint_characteristics ( ep ) );
+	head->cap = cpu_to_le32 ( ehci_endpoint_capabilities ( ep ) );
+}
+
+/**
+ * Open endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct ehci_device *ehci = usb_get_hostdata ( usb );
+	struct ehci_endpoint *endpoint;
+	int rc;
+
+	/* Allocate and initialise structure */
+	endpoint = zalloc ( sizeof ( *endpoint ) );
+	if ( ! endpoint ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	endpoint->ehci = ehci;
+	endpoint->ep = ep;
+	usb_endpoint_set_hostdata ( ep, endpoint );
+
+	/* Initialise descriptor ring */
+	if ( ( rc = ehci_ring_alloc ( ehci, &endpoint->ring ) ) != 0 )
+		goto err_ring_alloc;
+
+	/* Update queue characteristics and capabilities */
+	ehci_endpoint_update ( ep );
+
+	/* Add to list of endpoints */
+	list_add_tail ( &endpoint->list, &ehci->endpoints );
+
+	/* Add to schedule */
+	ehci_schedule_add ( endpoint );
+
+	return 0;
+
+	ehci_ring_free ( &endpoint->ring );
+ err_ring_alloc:
+	free ( endpoint );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Close endpoint
+ *
+ * @v ep		USB endpoint
+ */
+static void ehci_endpoint_close ( struct usb_endpoint *ep ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_device *ehci = endpoint->ehci;
+	struct usb_device *usb = ep->usb;
+	struct io_buffer *iobuf;
+	int rc;
+
+	/* Remove from schedule */
+	if ( ( rc = ehci_schedule_del ( endpoint ) ) != 0 ) {
+		/* No way to prevent hardware from continuing to
+		 * access the memory, so leak it.
+		 */
+		DBGC ( ehci, "EHCI %s %s could not unschedule: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return;
+	}
+
+	/* Cancel any incomplete transfers */
+	while ( ehci_ring_fill ( &endpoint->ring ) ) {
+		iobuf = ehci_dequeue ( &endpoint->ring );
+		if ( iobuf )
+			usb_complete_err ( ep, iobuf, -ECANCELED );
+	}
+
+	/* Remove from list of endpoints */
+	list_del ( &endpoint->list );
+
+	/* Free descriptor ring */
+	ehci_ring_free ( &endpoint->ring );
+
+	/* Free endpoint */
+	free ( endpoint );
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_ring *ring = &endpoint->ring;
+	struct ehci_transfer_descriptor *cache = &ring->head->cache;
+	uint32_t link;
+
+	/* Sanity checks */
+	assert ( ! ( cache->status & EHCI_STATUS_ACTIVE ) );
+	assert ( cache->status & EHCI_STATUS_HALTED );
+
+	/* Reset residual count */
+	ring->residual = 0;
+
+	/* Reset data toggle */
+	cache->len = 0;
+
+	/* Prepare to restart at next unconsumed descriptor */
+	link = virt_to_phys ( &ring->desc[ ring->cons % EHCI_RING_COUNT ] );
+	cache->next = cpu_to_le32 ( link );
+
+	/* Restart ring */
+	wmb();
+	cache->status = 0;
+
+	return 0;
+}
+
+/**
+ * Update MTU
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_mtu ( struct usb_endpoint *ep ) {
+
+	/* Update endpoint characteristics and capabilities */
+	ehci_endpoint_update ( ep );
+
+	return 0;
+}
+
+/**
+ * Enqueue message transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_message ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_device *ehci = endpoint->ehci;
+	struct usb_setup_packet *packet;
+	unsigned int input;
+	struct ehci_transfer xfers[3];
+	struct ehci_transfer *xfer = xfers;
+	size_t len;
+	int rc;
+
+	/* Construct setup stage */
+	assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
+	packet = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *packet ) );
+	xfer->data = packet;
+	xfer->len = sizeof ( *packet );
+	xfer->flags = EHCI_FL_PID_SETUP;
+	xfer++;
+
+	/* Construct data stage, if applicable */
+	len = iob_len ( iobuf );
+	input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
+	if ( len ) {
+		xfer->data = iobuf->data;
+		xfer->len = len;
+		xfer->flags = ( EHCI_FL_TOGGLE |
+				( input ? EHCI_FL_PID_IN : EHCI_FL_PID_OUT ) );
+		xfer++;
+	}
+
+	/* Construct status stage */
+	xfer->data = NULL;
+	xfer->len = 0;
+	xfer->flags = ( EHCI_FL_TOGGLE | EHCI_FL_IOC |
+			( ( len && input ) ? EHCI_FL_PID_OUT : EHCI_FL_PID_IN));
+	xfer++;
+
+	/* Enqueue transfer */
+	if ( ( rc = ehci_enqueue ( ehci, &endpoint->ring, iobuf, xfers,
+				   ( xfer - xfers ) ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Enqueue stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_stream ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int terminate ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_device *ehci = endpoint->ehci;
+	unsigned int input = ( ep->address & USB_DIR_IN );
+	struct ehci_transfer xfers[2];
+	struct ehci_transfer *xfer = xfers;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Create transfer */
+	xfer->data = iobuf->data;
+	xfer->len = len;
+	xfer->flags = ( EHCI_FL_IOC |
+			( input ? EHCI_FL_PID_IN : EHCI_FL_PID_OUT ) );
+	xfer++;
+	if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) {
+		xfer->data = NULL;
+		xfer->len = 0;
+		assert ( ! input );
+		xfer->flags = ( EHCI_FL_IOC | EHCI_FL_PID_OUT );
+		xfer++;
+	}
+
+	/* Enqueue transfer */
+	if ( ( rc = ehci_enqueue ( ehci, &endpoint->ring, iobuf, xfers,
+				   ( xfer - xfers ) ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completions
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_endpoint_poll ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+	struct ehci_ring *ring = &endpoint->ring;
+	struct ehci_transfer_descriptor *desc;
+	struct usb_endpoint *ep = endpoint->ep;
+	struct usb_device *usb = ep->usb;
+	struct io_buffer *iobuf;
+	unsigned int index;
+	unsigned int status;
+	int rc;
+
+	/* Consume all completed descriptors */
+	while ( ehci_ring_fill ( &endpoint->ring ) ) {
+
+		/* Stop if we reach an uncompleted descriptor */
+		rmb();
+		index = ( ring->cons % EHCI_RING_COUNT );
+		desc = &ring->desc[index];
+		status = desc->status;
+		if ( status & EHCI_STATUS_ACTIVE )
+			break;
+
+		/* Consume this descriptor */
+		iobuf = ehci_dequeue ( ring );
+
+		/* If we have encountered an error, then consume all
+		 * remaining descriptors in this transaction, report
+		 * the error to the USB core, and stop further
+		 * processing.
+		 */
+		if ( status & EHCI_STATUS_HALTED ) {
+			rc = -EIO_STATUS ( status );
+			DBGC ( ehci, "EHCI %s %s completion %d failed (status "
+			       "%02x): %s\n", usb->name,
+			       usb_endpoint_name ( ep ), index, status,
+			       strerror ( rc ) );
+			while ( ! iobuf )
+				iobuf = ehci_dequeue ( ring );
+			usb_complete_err ( endpoint->ep, iobuf, rc );
+			return;
+		}
+
+		/* Accumulate residual data count */
+		ring->residual += ( le16_to_cpu ( desc->len ) & EHCI_LEN_MASK );
+
+		/* If this is not the end of a transaction (i.e. has
+		 * no I/O buffer), then continue to next descriptor.
+		 */
+		if ( ! iobuf )
+			continue;
+
+		/* Update I/O buffer length */
+		iob_unput ( iobuf, ring->residual );
+		ring->residual = 0;
+
+		/* Report completion to USB core */
+		usb_complete ( endpoint->ep, iobuf );
+	}
+}
+
+/******************************************************************************
+ *
+ * Device operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int ehci_device_open ( struct usb_device *usb ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( usb->port->hub->bus );
+
+	usb_set_hostdata ( usb, ehci );
+	return 0;
+}
+
+/**
+ * Close device
+ *
+ * @v usb		USB device
+ */
+static void ehci_device_close ( struct usb_device *usb ) {
+	struct ehci_device *ehci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = ehci->bus;
+
+	/* Free device address, if assigned */
+	if ( usb->address )
+		usb_free_address ( bus, usb->address );
+}
+
+/**
+ * Assign device address
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int ehci_device_address ( struct usb_device *usb ) {
+	struct ehci_device *ehci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = ehci->bus;
+	struct usb_endpoint *ep0 = usb_endpoint ( usb, USB_EP0_ADDRESS );
+	int address;
+	int rc;
+
+	/* Sanity checks */
+	assert ( usb->address == 0 );
+	assert ( ep0 != NULL );
+
+	/* Allocate device address */
+	address = usb_alloc_address ( bus );
+	if ( address < 0 ) {
+		rc = address;
+		DBGC ( ehci, "EHCI %s could not allocate address: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_alloc_address;
+	}
+
+	/* Set address */
+	if ( ( rc = usb_set_address ( usb, address ) ) != 0 )
+		goto err_set_address;
+
+	/* Update device address */
+	usb->address = address;
+
+	/* Update control endpoint characteristics and capabilities */
+	ehci_endpoint_update ( ep0 );
+
+	return 0;
+
+ err_set_address:
+	usb_free_address ( bus, address );
+ err_alloc_address:
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * Hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int ehci_hub_open ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+	return 0;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void ehci_hub_close ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+}
+
+/******************************************************************************
+ *
+ * Root hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open root hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int ehci_root_open ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	uint32_t portsc;
+	unsigned int i;
+
+	/* Route all ports to EHCI controller */
+	writel ( EHCI_CONFIGFLAG_CF, ehci->op + EHCI_OP_CONFIGFLAG );
+
+	/* Enable power to all ports */
+	for ( i = 1 ; i <= ehci->ports ; i++ ) {
+		portsc = readl ( ehci->op + EHCI_OP_PORTSC ( i ) );
+		portsc &= ~EHCI_PORTSC_CHANGE;
+		portsc |= EHCI_PORTSC_PP;
+		writel ( portsc, ehci->op + EHCI_OP_PORTSC ( i ) );
+	}
+
+	/* Wait 20ms after potentially enabling power to a port */
+	mdelay ( EHCI_PORT_POWER_DELAY_MS );
+
+	/* Record hub driver private data */
+	usb_hub_set_drvdata ( hub, ehci );
+
+	return 0;
+}
+
+/**
+ * Close root hub
+ *
+ * @v hub		USB hub
+ */
+static void ehci_root_close ( struct usb_hub *hub ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+
+	/* Route all ports back to companion controllers */
+	writel ( 0, ehci->op + EHCI_OP_CONFIGFLAG );
+
+	/* Clear hub driver private data */
+	usb_hub_set_drvdata ( hub, NULL );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int ehci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int line;
+	unsigned int i;
+
+	/* Check for a low-speed device */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	line = EHCI_PORTSC_LINE_STATUS ( portsc );
+	if ( line == EHCI_PORTSC_LINE_STATUS_LOW ) {
+		DBGC ( ehci, "EHCI %s-%d detected low-speed device: "
+		       "disowning\n", ehci->name, port->address );
+		goto disown;
+	}
+
+	/* Reset port */
+	portsc &= ~( EHCI_PORTSC_PED | EHCI_PORTSC_CHANGE );
+	portsc |= EHCI_PORTSC_PR;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_DELAY_MS );
+	portsc &= ~EHCI_PORTSC_PR;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < EHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check port status */
+		portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+		if ( ! ( portsc & EHCI_PORTSC_PR ) ) {
+			if ( portsc & EHCI_PORTSC_PED )
+				return 0;
+			DBGC ( ehci, "EHCI %s-%d not enabled after reset: "
+			       "disowning\n", ehci->name, port->address );
+			goto disown;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( ehci, "EHCI %s-%d timed out waiting for port to reset\n",
+	       ehci->name, port->address );
+	return -ETIMEDOUT;
+
+ disown:
+	/* Disown port */
+	portsc &= ~EHCI_PORTSC_CHANGE;
+	portsc |= EHCI_PORTSC_OWNER;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Delay to allow child companion controllers to settle */
+	mdelay ( EHCI_DISOWN_DELAY_MS );
+
+	/* Poll child companion controllers */
+	ehci_poll_companions ( ehci );
+
+	return -ENODEV;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int ehci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+
+	/* Disable port */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	portsc &= ~( EHCI_PORTSC_PED | EHCI_PORTSC_CHANGE );
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Update root hub port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int ehci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int speed;
+	unsigned int line;
+	int ccs;
+	int csc;
+	int ped;
+
+	/* Read port status */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	DBGC2 ( ehci, "EHCI %s-%d status is %08x\n",
+		ehci->name, port->address, portsc );
+	ccs = ( portsc & EHCI_PORTSC_CCS );
+	csc = ( portsc & EHCI_PORTSC_CSC );
+	ped = ( portsc & EHCI_PORTSC_PED );
+	line = EHCI_PORTSC_LINE_STATUS ( portsc );
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= csc;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Determine port speed */
+	if ( ! ccs ) {
+		/* Port not connected */
+		speed = USB_SPEED_NONE;
+	} else if ( line == EHCI_PORTSC_LINE_STATUS_LOW ) {
+		/* Detected as low-speed */
+		speed = USB_SPEED_LOW;
+	} else if ( ped ) {
+		/* Port already enabled: must be high-speed */
+		speed = USB_SPEED_HIGH;
+	} else {
+		/* Not low-speed and not yet enabled.  Could be either
+		 * full-speed or high-speed; we can't yet tell.
+		 */
+		speed = USB_SPEED_FULL;
+	}
+	port->speed = speed;
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+				struct usb_endpoint *ep ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+
+	/* Should never be called; this is a root hub */
+	DBGC ( ehci, "EHCI %s-%d nonsensical CLEAR_TT for %s %s\n", ehci->name,
+	       port->address, ep->usb->name, usb_endpoint_name ( ep ) );
+
+	return -ENOTSUP;
+}
+
+/**
+ * Poll for port status changes
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ */
+static void ehci_root_poll ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	uint32_t change;
+
+	/* Do nothing unless something has changed */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	change = ( portsc & EHCI_PORTSC_CHANGE );
+	if ( ! change )
+		return;
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= ( portsc & EHCI_PORTSC_CSC );
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Report port status change */
+	usb_port_changed ( port );
+}
+
+/******************************************************************************
+ *
+ * Bus operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+static int ehci_bus_open ( struct usb_bus *bus ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	unsigned int frames;
+	size_t len;
+	int rc;
+
+	/* Sanity checks */
+	assert ( list_empty ( &ehci->async ) );
+	assert ( list_empty ( &ehci->periodic ) );
+
+	/* Allocate and initialise asynchronous queue head */
+	ehci->head = malloc_dma ( sizeof ( *ehci->head ),
+				  ehci_align ( sizeof ( *ehci->head ) ) );
+	if ( ! ehci->head ) {
+		rc = -ENOMEM;
+		goto err_alloc_head;
+	}
+	memset ( ehci->head, 0, sizeof ( *ehci->head ) );
+	ehci->head->chr = cpu_to_le32 ( EHCI_CHR_HEAD );
+	ehci->head->cache.next = cpu_to_le32 ( EHCI_LINK_TERMINATE );
+	ehci->head->cache.status = EHCI_STATUS_HALTED;
+	ehci_async_schedule ( ehci );
+	writel ( virt_to_phys ( ehci->head ),
+		 ehci->op + EHCI_OP_ASYNCLISTADDR );
+
+	/* Use async queue head to determine control data structure segment */
+	ehci->ctrldssegment =
+		( ( ( uint64_t ) virt_to_phys ( ehci->head ) ) >> 32 );
+	if ( ehci->addr64 ) {
+		writel ( ehci->ctrldssegment, ehci->op + EHCI_OP_CTRLDSSEGMENT);
+	} else if ( ehci->ctrldssegment ) {
+		DBGC ( ehci, "EHCI %s CTRLDSSEGMENT not supported\n",
+		       ehci->name );
+		rc = -ENOTSUP;
+		goto err_ctrldssegment;
+	}
+
+	/* Allocate periodic frame list */
+	frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
+	len = ( frames * sizeof ( ehci->frame[0] ) );
+	ehci->frame = malloc_dma ( len, EHCI_PAGE_ALIGN );
+	if ( ! ehci->frame ) {
+		rc = -ENOMEM;
+		goto err_alloc_frame;
+	}
+	if ( ( rc = ehci_ctrl_reachable ( ehci, ehci->frame ) ) != 0 ) {
+		DBGC ( ehci, "EHCI %s frame list unreachable\n", ehci->name );
+		goto err_unreachable_frame;
+	}
+	ehci_periodic_schedule ( ehci );
+	writel ( virt_to_phys ( ehci->frame ),
+		 ehci->op + EHCI_OP_PERIODICLISTBASE );
+
+	/* Start controller */
+	ehci_run ( ehci );
+
+	return 0;
+
+	ehci_stop ( ehci );
+ err_unreachable_frame:
+	free_dma ( ehci->frame, len );
+ err_alloc_frame:
+ err_ctrldssegment:
+	free_dma ( ehci->head, sizeof ( *ehci->head ) );
+ err_alloc_head:
+	return rc;
+}
+
+/**
+ * Close USB bus
+ *
+ * @v bus		USB bus
+ */
+static void ehci_bus_close ( struct usb_bus *bus ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	unsigned int frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
+
+	/* Sanity checks */
+	assert ( list_empty ( &ehci->async ) );
+	assert ( list_empty ( &ehci->periodic ) );
+
+	/* Stop controller */
+	ehci_stop ( ehci );
+
+	/* Free periodic frame list */
+	free_dma ( ehci->frame, ( frames * sizeof ( ehci->frame[0] ) ) );
+
+	/* Free asynchronous schedule */
+	free_dma ( ehci->head, sizeof ( *ehci->head ) );
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static void ehci_bus_poll ( struct usb_bus *bus ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	struct usb_hub *hub = bus->hub;
+	struct ehci_endpoint *endpoint;
+	unsigned int i;
+	uint32_t usbsts;
+	uint32_t change;
+
+	/* Do nothing unless something has changed */
+	usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
+	assert ( usbsts & EHCI_USBSTS_ASYNC );
+	assert ( usbsts & EHCI_USBSTS_PERIODIC );
+	assert ( ! ( usbsts & EHCI_USBSTS_HCH ) );
+	change = ( usbsts & EHCI_USBSTS_CHANGE );
+	if ( ! change )
+		return;
+
+	/* Acknowledge changes */
+	writel ( usbsts, ehci->op + EHCI_OP_USBSTS );
+
+	/* Process completions, if applicable */
+	if ( change & ( EHCI_USBSTS_USBINT | EHCI_USBSTS_USBERRINT ) ) {
+
+		/* Iterate over all endpoints looking for completed
+		 * descriptors.  We trust that completion handlers are
+		 * minimal and will not do anything that could
+		 * plausibly affect the endpoint list itself.
+		 */
+		list_for_each_entry ( endpoint, &ehci->endpoints, list )
+			ehci_endpoint_poll ( endpoint );
+	}
+
+	/* Process port status changes, if applicable */
+	if ( change & EHCI_USBSTS_PORT ) {
+
+		/* Iterate over all ports looking for status changes */
+		for ( i = 1 ; i <= ehci->ports ; i++ )
+			ehci_root_poll ( hub, usb_port ( hub, i ) );
+	}
+
+	/* Report fatal errors */
+	if ( change & EHCI_USBSTS_SYSERR )
+		DBGC ( ehci, "EHCI %s host system error\n", ehci->name );
+}
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** USB host controller operations */
+static struct usb_host_operations ehci_operations = {
+	.endpoint = {
+		.open = ehci_endpoint_open,
+		.close = ehci_endpoint_close,
+		.reset = ehci_endpoint_reset,
+		.mtu = ehci_endpoint_mtu,
+		.message = ehci_endpoint_message,
+		.stream = ehci_endpoint_stream,
+	},
+	.device = {
+		.open = ehci_device_open,
+		.close = ehci_device_close,
+		.address = ehci_device_address,
+	},
+	.bus = {
+		.open = ehci_bus_open,
+		.close = ehci_bus_close,
+		.poll = ehci_bus_poll,
+	},
+	.hub = {
+		.open = ehci_hub_open,
+		.close = ehci_hub_close,
+	},
+	.root = {
+		.open = ehci_root_open,
+		.close = ehci_root_close,
+		.enable = ehci_root_enable,
+		.disable = ehci_root_disable,
+		.speed = ehci_root_speed,
+		.clear_tt = ehci_root_clear_tt,
+	},
+};
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int ehci_probe ( struct pci_device *pci ) {
+	struct ehci_device *ehci;
+	struct usb_port *port;
+	unsigned long bar_start;
+	size_t bar_size;
+	unsigned int i;
+	int rc;
+
+	/* Allocate and initialise structure */
+	ehci = zalloc ( sizeof ( *ehci ) );
+	if ( ! ehci ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	ehci->name = pci->dev.name;
+	INIT_LIST_HEAD ( &ehci->endpoints );
+	INIT_LIST_HEAD ( &ehci->async );
+	INIT_LIST_HEAD ( &ehci->periodic );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Map registers */
+	bar_start = pci_bar_start ( pci, EHCI_BAR );
+	bar_size = pci_bar_size ( pci, EHCI_BAR );
+	ehci->regs = ioremap ( bar_start, bar_size );
+	if ( ! ehci->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Initialise EHCI device */
+	ehci_init ( ehci, ehci->regs );
+
+	/* Initialise USB legacy support and claim ownership */
+	ehci_legacy_init ( ehci, pci );
+	ehci_legacy_claim ( ehci, pci );
+
+	/* Reset device */
+	if ( ( rc = ehci_reset ( ehci ) ) != 0 )
+		goto err_reset;
+
+	/* Allocate USB bus */
+	ehci->bus = alloc_usb_bus ( &pci->dev, ehci->ports, EHCI_MTU,
+				    &ehci_operations );
+	if ( ! ehci->bus ) {
+		rc = -ENOMEM;
+		goto err_alloc_bus;
+	}
+	usb_bus_set_hostdata ( ehci->bus, ehci );
+	usb_hub_set_drvdata ( ehci->bus->hub, ehci );
+
+	/* Set port protocols */
+	for ( i = 1 ; i <= ehci->ports ; i++ ) {
+		port = usb_port ( ehci->bus->hub, i );
+		port->protocol = USB_PROTO_2_0;
+	}
+
+	/* Register USB bus */
+	if ( ( rc = register_usb_bus ( ehci->bus ) ) != 0 )
+		goto err_register;
+
+	pci_set_drvdata ( pci, ehci );
+	return 0;
+
+	unregister_usb_bus ( ehci->bus );
+ err_register:
+	free_usb_bus ( ehci->bus );
+ err_alloc_bus:
+	ehci_reset ( ehci );
+ err_reset:
+	ehci_legacy_release ( ehci, pci );
+	iounmap ( ehci->regs );
+ err_ioremap:
+	free ( ehci );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void ehci_remove ( struct pci_device *pci ) {
+	struct ehci_device *ehci = pci_get_drvdata ( pci );
+	struct usb_bus *bus = ehci->bus;
+
+	unregister_usb_bus ( bus );
+	assert ( list_empty ( &ehci->async ) );
+	assert ( list_empty ( &ehci->periodic ) );
+	free_usb_bus ( bus );
+	ehci_reset ( ehci );
+	ehci_legacy_release ( ehci, pci );
+	iounmap ( ehci->regs );
+	free ( ehci );
+}
+
+/** EHCI PCI device IDs */
+static struct pci_device_id ehci_ids[] = {
+	PCI_ROM ( 0xffff, 0xffff, "ehci", "EHCI", 0 ),
+};
+
+/** EHCI PCI driver */
+struct pci_driver ehci_driver __pci_driver = {
+	.ids = ehci_ids,
+	.id_count = ( sizeof ( ehci_ids ) / sizeof ( ehci_ids[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
+				PCI_CLASS_SERIAL_USB_EHCI ),
+	.probe = ehci_probe,
+	.remove = ehci_remove,
+};
+
+/**
+ * Prepare for exit
+ *
+ * @v booting		System is shutting down for OS boot
+ */
+static void ehci_shutdown ( int booting ) {
+	/* If we are shutting down to boot an OS, then prevent the
+	 * release of ownership back to BIOS.
+	 */
+	ehci_legacy_prevent_release = booting;
+}
+
+/** Startup/shutdown function */
+struct startup_fn ehci_startup __startup_fn ( STARTUP_LATE ) = {
+	.shutdown = ehci_shutdown,
+};
diff --git a/roms/ipxe/src/drivers/usb/ehci.h b/roms/ipxe/src/drivers/usb/ehci.h
new file mode 100644
index 000000000..42e282e92
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/ehci.h
@@ -0,0 +1,544 @@
+#ifndef _IPXE_EHCI_H
+#define _IPXE_EHCI_H
+
+/** @file
+ *
+ * USB Enhanced Host Controller Interface (EHCI) driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+
+/** Minimum alignment required for data structures
+ *
+ * With the exception of the periodic frame list (which is
+ * page-aligned), data structures used by EHCI generally require
+ * 32-byte alignment and must not cross a 4kB page boundary.  We
+ * simplify this requirement by aligning each structure on its own
+ * size, with a minimum of a 32 byte alignment.
+ */
+#define EHCI_MIN_ALIGN 32
+
+/** Maximum transfer size
+ *
+ * EHCI allows for transfers of up to 20kB with page-alignment, or
+ * 16kB with arbitrary alignment.
+ */
+#define EHCI_MTU 16384
+
+/** Page-alignment required for some data structures */
+#define EHCI_PAGE_ALIGN 4096
+
+/** EHCI PCI BAR */
+#define EHCI_BAR PCI_BASE_ADDRESS_0
+
+/** Capability register length */
+#define EHCI_CAP_CAPLENGTH 0x00
+
+/** Host controller interface version number */
+#define EHCI_CAP_HCIVERSION 0x02
+
+/** Structural parameters */
+#define EHCI_CAP_HCSPARAMS 0x04
+
+/** Number of ports */
+#define EHCI_HCSPARAMS_PORTS(params) ( ( (params) >> 0 ) & 0x0f )
+
+/** Capability parameters */
+#define EHCI_CAP_HCCPARAMS 0x08
+
+/** 64-bit addressing capability */
+#define EHCI_HCCPARAMS_ADDR64(params) ( ( (params) >> 0 ) & 0x1 )
+
+/** Programmable frame list flag */
+#define EHCI_HCCPARAMS_FLSIZE(params) ( ( (params) >> 1 ) & 0x1 )
+
+/** EHCI extended capabilities pointer */
+#define EHCI_HCCPARAMS_EECP(params) ( ( ( (params) >> 8 ) & 0xff ) )
+
+/** EHCI extended capability ID */
+#define EHCI_EECP_ID(eecp) ( ( (eecp) >> 0 ) & 0xff )
+
+/** Next EHCI extended capability pointer */
+#define EHCI_EECP_NEXT(eecp) ( ( ( (eecp) >> 8 ) & 0xff ) )
+
+/** USB legacy support extended capability */
+#define EHCI_EECP_ID_LEGACY 1
+
+/** USB legacy support BIOS owned semaphore */
+#define EHCI_USBLEGSUP_BIOS 0x02
+
+/** USB legacy support BIOS ownership flag */
+#define EHCI_USBLEGSUP_BIOS_OWNED 0x01
+
+/** USB legacy support OS owned semaphore */
+#define EHCI_USBLEGSUP_OS 0x03
+
+/** USB legacy support OS ownership flag */
+#define EHCI_USBLEGSUP_OS_OWNED 0x01
+
+/** USB legacy support control/status */
+#define EHCI_USBLEGSUP_CTLSTS 0x04
+
+/** USB command register */
+#define EHCI_OP_USBCMD 0x00
+
+/** Run/stop */
+#define EHCI_USBCMD_RUN 0x00000001UL
+
+/** Host controller reset */
+#define EHCI_USBCMD_HCRST 0x00000002UL
+
+/** Frame list size */
+#define EHCI_USBCMD_FLSIZE(flsize) ( (flsize) << 2 )
+
+/** Frame list size mask */
+#define EHCI_USBCMD_FLSIZE_MASK EHCI_USBCMD_FLSIZE ( 3 )
+
+/** Default frame list size */
+#define EHCI_FLSIZE_DEFAULT 0
+
+/** Smallest allowed frame list size */
+#define EHCI_FLSIZE_SMALL 2
+
+/** Number of elements in frame list */
+#define EHCI_PERIODIC_FRAMES(flsize) ( 1024 >> (flsize) )
+
+/** Periodic schedule enable */
+#define EHCI_USBCMD_PERIODIC 0x00000010UL
+
+/** Asynchronous schedule enable */
+#define EHCI_USBCMD_ASYNC 0x00000020UL
+
+/** Asyncchronous schedule advance doorbell */
+#define EHCI_USBCMD_ASYNC_ADVANCE 0x000040UL
+
+/** USB status register */
+#define EHCI_OP_USBSTS 0x04
+
+/** USB interrupt */
+#define EHCI_USBSTS_USBINT 0x00000001UL
+
+/** USB error interrupt */
+#define EHCI_USBSTS_USBERRINT 0x00000002UL
+
+/** Port change detect */
+#define EHCI_USBSTS_PORT 0x00000004UL
+
+/** Frame list rollover */
+#define EHCI_USBSTS_ROLLOVER 0x00000008UL
+
+/** Host system error */
+#define EHCI_USBSTS_SYSERR 0x00000010UL
+
+/** Asynchronous schedule advanced */
+#define EHCI_USBSTS_ASYNC_ADVANCE 0x00000020UL
+
+/** Periodic schedule enabled */
+#define EHCI_USBSTS_PERIODIC 0x00004000UL
+
+/** Asynchronous schedule enabled */
+#define EHCI_USBSTS_ASYNC 0x00008000UL
+
+/** Host controller halted */
+#define EHCI_USBSTS_HCH 0x00001000UL
+
+/** USB status change mask */
+#define EHCI_USBSTS_CHANGE						\
+	( EHCI_USBSTS_USBINT | EHCI_USBSTS_USBERRINT |			\
+	  EHCI_USBSTS_PORT | EHCI_USBSTS_ROLLOVER |			\
+	  EHCI_USBSTS_SYSERR | EHCI_USBSTS_ASYNC_ADVANCE )
+
+/** USB interrupt enable register */
+#define EHCI_OP_USBINTR 0x08
+
+/** Frame index register */
+#define EHCI_OP_FRINDEX 0x0c
+
+/** Control data structure segment register */
+#define EHCI_OP_CTRLDSSEGMENT 0x10
+
+/** Periodic frame list base address register */
+#define EHCI_OP_PERIODICLISTBASE 0x14
+
+/** Current asynchronous list address register */
+#define EHCI_OP_ASYNCLISTADDR 0x18
+
+/** Configure flag register */
+#define EHCI_OP_CONFIGFLAG 0x40
+
+/** Configure flag */
+#define EHCI_CONFIGFLAG_CF 0x00000001UL
+
+/** Port status and control register */
+#define EHCI_OP_PORTSC(port) ( 0x40 + ( (port) << 2 ) )
+
+/** Current connect status */
+#define EHCI_PORTSC_CCS 0x00000001UL
+
+/** Connect status change */
+#define EHCI_PORTSC_CSC 0x00000002UL
+
+/** Port enabled */
+#define EHCI_PORTSC_PED 0x00000004UL
+
+/** Port enabled/disabled change */
+#define EHCI_PORTSC_PEC 0x00000008UL
+
+/** Over-current change */
+#define EHCI_PORTSC_OCC 0x00000020UL
+
+/** Port reset */
+#define EHCI_PORTSC_PR 0x00000100UL
+
+/** Line status */
+#define EHCI_PORTSC_LINE_STATUS(portsc) ( ( (portsc) >> 10 ) & 0x3 )
+
+/** Line status: low-speed device */
+#define EHCI_PORTSC_LINE_STATUS_LOW 0x1
+
+/** Port power */
+#define EHCI_PORTSC_PP 0x00001000UL
+
+/** Port owner */
+#define EHCI_PORTSC_OWNER 0x00002000UL
+
+/** Port status change mask */
+#define EHCI_PORTSC_CHANGE \
+	( EHCI_PORTSC_CSC | EHCI_PORTSC_PEC | EHCI_PORTSC_OCC )
+
+/** List terminator */
+#define EHCI_LINK_TERMINATE 0x00000001UL
+
+/** Frame list type */
+#define EHCI_LINK_TYPE(type) ( (type) << 1 )
+
+/** Queue head type */
+#define EHCI_LINK_TYPE_QH EHCI_LINK_TYPE ( 1 )
+
+/** A periodic frame list entry */
+struct ehci_periodic_frame {
+	/** First queue head */
+	uint32_t link;
+} __attribute__ (( packed ));
+
+/** A transfer descriptor */
+struct ehci_transfer_descriptor {
+	/** Next transfer descriptor */
+	uint32_t next;
+	/** Alternate next transfer descriptor */
+	uint32_t alt;
+	/** Status */
+	uint8_t status;
+	/** Flags */
+	uint8_t flags;
+	/** Transfer length */
+	uint16_t len;
+	/** Buffer pointers (low 32 bits) */
+	uint32_t low[5];
+	/** Extended buffer pointers (high 32 bits) */
+	uint32_t high[5];
+	/** Reserved */
+	uint8_t reserved[12];
+} __attribute__ (( packed ));
+
+/** Transaction error */
+#define EHCI_STATUS_XACT_ERR 0x08
+
+/** Babble detected */
+#define EHCI_STATUS_BABBLE 0x10
+
+/** Data buffer error */
+#define EHCI_STATUS_BUFFER 0x20
+
+/** Halted */
+#define EHCI_STATUS_HALTED 0x40
+
+/** Active */
+#define EHCI_STATUS_ACTIVE 0x80
+
+/** PID code */
+#define EHCI_FL_PID(code) ( (code) << 0 )
+
+/** OUT token */
+#define EHCI_FL_PID_OUT EHCI_FL_PID ( 0 )
+
+/** IN token */
+#define EHCI_FL_PID_IN EHCI_FL_PID ( 1 )
+
+/** SETUP token */
+#define EHCI_FL_PID_SETUP EHCI_FL_PID ( 2 )
+
+/** Error counter */
+#define EHCI_FL_CERR( count ) ( (count) << 2 )
+
+/** Error counter maximum value */
+#define EHCI_FL_CERR_MAX EHCI_FL_CERR ( 3 )
+
+/** Interrupt on completion */
+#define EHCI_FL_IOC 0x80
+
+/** Length mask */
+#define EHCI_LEN_MASK 0x7fff
+
+/** Data toggle */
+#define EHCI_LEN_TOGGLE 0x8000
+
+/** A queue head */
+struct ehci_queue_head {
+	/** Horizontal link pointer */
+	uint32_t link;
+	/** Endpoint characteristics */
+	uint32_t chr;
+	/** Endpoint capabilities */
+	uint32_t cap;
+	/** Current transfer descriptor */
+	uint32_t current;
+	/** Transfer descriptor cache */
+	struct ehci_transfer_descriptor cache;
+} __attribute__ (( packed ));
+
+/** Device address */
+#define EHCI_CHR_ADDRESS( address ) ( (address) << 0 )
+
+/** Endpoint number */
+#define EHCI_CHR_ENDPOINT( address ) ( ( (address) & 0xf ) << 8 )
+
+/** Endpoint speed */
+#define EHCI_CHR_EPS( eps ) ( (eps) << 12 )
+
+/** Full-speed endpoint */
+#define EHCI_CHR_EPS_FULL EHCI_CHR_EPS ( 0 )
+
+/** Low-speed endpoint */
+#define EHCI_CHR_EPS_LOW EHCI_CHR_EPS ( 1 )
+
+/** High-speed endpoint */
+#define EHCI_CHR_EPS_HIGH EHCI_CHR_EPS ( 2 )
+
+/** Explicit data toggles */
+#define EHCI_CHR_TOGGLE 0x00004000UL
+
+/** Head of reclamation list flag */
+#define EHCI_CHR_HEAD 0x00008000UL
+
+/** Maximum packet length */
+#define EHCI_CHR_MAX_LEN( len ) ( (len) << 16 )
+
+/** Control endpoint flag */
+#define EHCI_CHR_CONTROL 0x08000000UL
+
+/** Interrupt schedule mask */
+#define EHCI_CAP_INTR_SCHED( uframe ) ( 1 << ( (uframe) + 0 ) )
+
+/** Split completion schedule mask */
+#define EHCI_CAP_SPLIT_SCHED( uframe ) ( 1 << ( (uframe) + 8 ) )
+
+/** Default split completion schedule mask
+ *
+ * We schedule all split starts in microframe 0, on the assumption
+ * that we will never have to deal with more than sixteen actively
+ * interrupting devices via the same transaction translator.  We
+ * schedule split completions for all remaining microframes after
+ * microframe 1 (in which the low-speed or full-speed transaction is
+ * assumed to execute).  This is a very crude approximation designed
+ * to avoid the need for calculating exactly when low-speed and
+ * full-speed transactions will execute.  Since we only ever deal with
+ * interrupt endpoints (rather than isochronous endpoints), the volume
+ * of periodic traffic is extremely low, and this approximation should
+ * remain valid.
+ */
+#define EHCI_CAP_SPLIT_SCHED_DEFAULT					\
+	( EHCI_CAP_SPLIT_SCHED ( 2 ) | EHCI_CAP_SPLIT_SCHED ( 3 ) |	\
+	  EHCI_CAP_SPLIT_SCHED ( 4 ) | EHCI_CAP_SPLIT_SCHED ( 5 ) |	\
+	  EHCI_CAP_SPLIT_SCHED ( 6 ) | EHCI_CAP_SPLIT_SCHED ( 7 ) )
+
+/** Transaction translator hub address */
+#define EHCI_CAP_TT_HUB( address ) ( (address) << 16 )
+
+/** Transaction translator port number */
+#define EHCI_CAP_TT_PORT( port ) ( (port) << 23 )
+
+/** High-bandwidth pipe multiplier */
+#define EHCI_CAP_MULT( mult ) ( (mult) << 30 )
+
+/** A transfer descriptor ring */
+struct ehci_ring {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+
+	/** Residual untransferred data */
+	size_t residual;
+
+	/** I/O buffers */
+	struct io_buffer **iobuf;
+
+	/** Queue head */
+	struct ehci_queue_head *head;
+	/** Transfer descriptors */
+	struct ehci_transfer_descriptor *desc;
+};
+
+/** Number of transfer descriptors in a ring
+ *
+ * This is a policy decision.
+ */
+#define EHCI_RING_COUNT 64
+
+/**
+ * Calculate space used in transfer descriptor ring
+ *
+ * @v ring		Transfer descriptor ring
+ * @ret fill		Number of entries used
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+ehci_ring_fill ( struct ehci_ring *ring ) {
+	unsigned int fill;
+
+	fill = ( ring->prod - ring->cons );
+	assert ( fill <= EHCI_RING_COUNT );
+	return fill;
+}
+
+/**
+ * Calculate space remaining in transfer descriptor ring
+ *
+ * @v ring		Transfer descriptor ring
+ * @ret remaining	Number of entries remaining
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+ehci_ring_remaining ( struct ehci_ring *ring ) {
+	unsigned int fill = ehci_ring_fill ( ring );
+
+	return ( EHCI_RING_COUNT - fill );
+}
+
+/** Time to delay after enabling power to a port
+ *
+ * This is not mandated by EHCI; we use the value given for xHCI.
+ */
+#define EHCI_PORT_POWER_DELAY_MS 20
+
+/** Time to delay after releasing ownership of a port
+ *
+ * This is a policy decision.
+ */
+#define EHCI_DISOWN_DELAY_MS 100
+
+/** Maximum time to wait for BIOS to release ownership
+ *
+ * This is a policy decision.
+ */
+#define EHCI_USBLEGSUP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for asynchronous schedule to advance
+ *
+ * This is a policy decision.
+ */
+#define EHCI_ASYNC_ADVANCE_MAX_WAIT_MS 100
+
+/** Maximum time to wait for host controller to stop
+ *
+ * This is a policy decision.
+ */
+#define EHCI_STOP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for reset to complete
+ *
+ * This is a policy decision.
+ */
+#define EHCI_RESET_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a port reset to complete
+ *
+ * This is a policy decision.
+ */
+#define EHCI_PORT_RESET_MAX_WAIT_MS 500
+
+/** An EHCI transfer */
+struct ehci_transfer {
+	/** Data buffer */
+	void *data;
+	/** Length */
+	size_t len;
+	/** Flags
+	 *
+	 * This is the bitwise OR of zero or more EHCI_FL_XXX values.
+	 * The low 8 bits are copied to the flags byte within the
+	 * transfer descriptor; the remaining bits hold flags
+	 * meaningful only to our driver code.
+	 */
+	unsigned int flags;
+};
+
+/** Set initial data toggle */
+#define EHCI_FL_TOGGLE 0x8000
+
+/** An EHCI device */
+struct ehci_device {
+	/** Registers */
+	void *regs;
+	/** Name */
+	const char *name;
+
+	/** Capability registers */
+	void *cap;
+	/** Operational registers */
+	void *op;
+
+	/** Number of ports */
+	unsigned int ports;
+	/** 64-bit addressing capability */
+	int addr64;
+	/** Frame list size */
+	unsigned int flsize;
+	/** EHCI extended capabilities offset */
+	unsigned int eecp;
+
+	/** USB legacy support capability (if present and enabled) */
+	unsigned int legacy;
+
+	/** Control data structure segment */
+	uint32_t ctrldssegment;
+	/** Asynchronous queue head */
+	struct ehci_queue_head *head;
+	/** Periodic frame list */
+	struct ehci_periodic_frame *frame;
+
+	/** List of all endpoints */
+	struct list_head endpoints;
+	/** Asynchronous schedule */
+	struct list_head async;
+	/** Periodic schedule
+	 *
+	 * Listed in decreasing order of endpoint interval.
+	 */
+	struct list_head periodic;
+
+	/** USB bus */
+	struct usb_bus *bus;
+};
+
+/** An EHCI endpoint */
+struct ehci_endpoint {
+	/** EHCI device */
+	struct ehci_device *ehci;
+	/** USB endpoint */
+	struct usb_endpoint *ep;
+	/** List of all endpoints */
+	struct list_head list;
+	/** Endpoint schedule */
+	struct list_head schedule;
+
+	/** Transfer descriptor ring */
+	struct ehci_ring ring;
+};
+
+extern unsigned int ehci_companion ( struct pci_device *pci );
+
+#endif /* _IPXE_EHCI_H */
diff --git a/roms/ipxe/src/drivers/usb/uhci.c b/roms/ipxe/src/drivers/usb/uhci.c
new file mode 100644
index 000000000..b6bb92560
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/uhci.c
@@ -0,0 +1,1577 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <strings.h>
+#include <unistd.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+#include "ehci.h"
+#include "uhci.h"
+
+/** @file
+ *
+ * USB Universal Host Controller Interface (UHCI) driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Check that address is reachable
+ *
+ * @v addr		Address
+ * @v len		Length
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline)) int
+uhci_reachable ( void *addr, size_t len ) {
+	physaddr_t phys = virt_to_phys ( addr );
+
+	/* Always reachable in a 32-bit build */
+	if ( sizeof ( physaddr_t ) <= sizeof ( uint32_t ) )
+		return 0;
+
+	/* Reachable if below 4GB */
+	if ( ( ( phys + len - 1 ) & ~0xffffffffULL ) == 0 )
+		return 0;
+
+	return -ENOTSUP;
+}
+
+/******************************************************************************
+ *
+ * Run / stop / reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Start UHCI device
+ *
+ * @v uhci		UHCI device
+ */
+static void uhci_run ( struct uhci_device *uhci ) {
+	uint16_t usbcmd;
+
+	/* Set run/stop bit */
+	usbcmd = inw ( uhci->regs + UHCI_USBCMD );
+	usbcmd |= ( UHCI_USBCMD_RUN | UHCI_USBCMD_MAX64 );
+	outw ( usbcmd, uhci->regs + UHCI_USBCMD );
+}
+
+/**
+ * Stop UHCI device
+ *
+ * @v uhci		UHCI device
+ * @ret rc		Return status code
+ */
+static int uhci_stop ( struct uhci_device *uhci ) {
+	uint16_t usbcmd;
+	uint16_t usbsts;
+	unsigned int i;
+
+	/* Clear run/stop bit */
+	usbcmd = inw ( uhci->regs + UHCI_USBCMD );
+	usbcmd &= ~UHCI_USBCMD_RUN;
+	outw ( usbcmd, uhci->regs + UHCI_USBCMD );
+
+	/* Wait for device to stop */
+	for ( i = 0 ; i < UHCI_STOP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if device is stopped */
+		usbsts = inw ( uhci->regs + UHCI_USBSTS );
+		if ( usbsts & UHCI_USBSTS_HCHALTED )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( uhci, "UHCI %s timed out waiting for stop\n", uhci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Reset UHCI device
+ *
+ * @v uhci		UHCI device
+ * @ret rc		Return status code
+ */
+static int uhci_reset ( struct uhci_device *uhci ) {
+	uint16_t usbcmd;
+	unsigned int i;
+	int rc;
+
+	/* The UHCI specification states that resetting a running
+	 * device may result in undefined behaviour, so try stopping
+	 * it first.
+	 */
+	if ( ( rc = uhci_stop ( uhci ) ) != 0 ) {
+		/* Ignore errors and attempt to reset the device anyway */
+	}
+
+	/* Reset device */
+	outw ( UHCI_USBCMD_HCRESET, uhci->regs + UHCI_USBCMD );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < UHCI_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if reset is complete */
+		usbcmd = inw ( uhci->regs + UHCI_USBCMD );
+		if ( ! ( usbcmd & UHCI_USBCMD_HCRESET ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( uhci, "UHCI %s timed out waiting for reset\n", uhci->name );
+	return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Transfer descriptor rings
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate transfer ring
+ *
+ * @v ring		Transfer ring
+ * @ret rc		Return status code
+ */
+static int uhci_ring_alloc ( struct uhci_ring *ring ) {
+	int rc;
+
+	/* Initialise structure */
+	memset ( ring, 0, sizeof ( *ring ) );
+
+	/* Allocate queue head */
+	ring->head = malloc_dma ( sizeof ( *ring->head ), UHCI_ALIGN );
+	if ( ! ring->head ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	if ( ( rc = uhci_reachable ( ring->head,
+				     sizeof ( *ring->head ) ) ) != 0 )
+		goto err_unreachable;
+
+	/* Initialise queue head */
+	ring->head->current = cpu_to_le32 ( UHCI_LINK_TERMINATE );
+
+	return 0;
+
+ err_unreachable:
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Free transfer ring
+ *
+ * @v ring		Transfer ring
+ */
+static void uhci_ring_free ( struct uhci_ring *ring ) {
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( uhci_ring_fill ( ring ) == 0 );
+	for ( i = 0 ; i < UHCI_RING_COUNT ; i++ )
+		assert ( ring->xfer[i] == NULL );
+
+	/* Free queue head */
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+}
+
+/**
+ * Enqueue new transfer
+ *
+ * @v ring		Transfer ring
+ * @v iobuf		I/O buffer
+ * @v count		Number of descriptors
+ * @ret rc		Return status code
+ */
+static int uhci_enqueue ( struct uhci_ring *ring, struct io_buffer *iobuf,
+			  unsigned int count ) {
+	struct uhci_transfer *xfer;
+	struct uhci_transfer *end;
+	struct uhci_transfer_descriptor *desc;
+	unsigned int index = ( ring->prod % UHCI_RING_COUNT );
+	uint32_t link;
+	size_t len;
+	int rc;
+
+	/* Sanity check */
+	assert ( count > 0 );
+	assert ( iobuf != NULL );
+
+	/* Check for space in ring */
+	if ( ! uhci_ring_remaining ( ring ) ) {
+		rc = -ENOBUFS;
+		goto err_ring_full;
+	}
+
+	/* Check for reachability of I/O buffer */
+	if ( ( rc = uhci_reachable ( iobuf->data, iob_len ( iobuf ) ) ) != 0 )
+		goto err_unreachable_iobuf;
+
+	/* Allocate transfer */
+	xfer = malloc ( sizeof ( *xfer ) );
+	if ( ! xfer ) {
+		rc = -ENOMEM;
+		goto err_alloc_xfer;
+	}
+
+	/* Initialise transfer */
+	xfer->prod = 0;
+	xfer->cons = 0;
+	xfer->len = 0;
+	xfer->iobuf = iobuf;
+
+	/* Allocate transfer descriptors */
+	len = ( count * sizeof ( xfer->desc[0] ) );
+	xfer->desc = malloc_dma ( len, UHCI_ALIGN );
+	if ( ! xfer->desc ) {
+		rc = -ENOMEM;
+		goto err_alloc_desc;
+	}
+	if ( ( rc = uhci_reachable ( xfer->desc, len ) ) != 0 )
+		goto err_unreachable_desc;
+
+	/* Initialise transfer descriptors */
+	memset ( xfer->desc, 0, len );
+	desc = xfer->desc;
+	for ( ; --count ; desc++ ) {
+		link = ( virt_to_phys ( desc + 1 ) | UHCI_LINK_DEPTH_FIRST );
+		desc->link = cpu_to_le32 ( link );
+		desc->flags = ring->flags;
+	}
+	desc->link = cpu_to_le32 ( UHCI_LINK_TERMINATE );
+	desc->flags = ( ring->flags | UHCI_FL_IOC );
+
+	/* Add to ring */
+	wmb();
+	link = virt_to_phys ( xfer->desc );
+	if ( uhci_ring_fill ( ring ) > 0 ) {
+		end = ring->end;
+		end->desc[ end->prod - 1 ].link = cpu_to_le32 ( link );
+	} else {
+		ring->head->current = cpu_to_le32 ( link );
+	}
+	assert ( ring->xfer[index] == NULL );
+	ring->xfer[index] = xfer;
+	ring->end = xfer;
+	ring->prod++;
+
+	return 0;
+
+ err_unreachable_desc:
+	free_dma ( xfer->desc, len );
+ err_alloc_desc:
+	free ( xfer );
+ err_alloc_xfer:
+ err_unreachable_iobuf:
+ err_ring_full:
+	return rc;
+}
+
+/**
+ * Describe transfer
+ *
+ * @v ring		Transfer ring
+ * @v data		Data
+ * @v len		Length of data
+ * @v pid		Packet ID
+ */
+static void uhci_describe ( struct uhci_ring *ring, void *data,
+			    size_t len, uint8_t pid ) {
+	struct uhci_transfer *xfer = ring->end;
+	struct uhci_transfer_descriptor *desc;
+	size_t frag_len;
+	uint32_t control;
+
+	do {
+		/* Calculate fragment length */
+		frag_len = len;
+		if ( frag_len > ring->mtu )
+			frag_len = ring->mtu;
+
+		/* Populate descriptor */
+		desc = &xfer->desc[xfer->prod++];
+		if ( pid == USB_PID_IN )
+			desc->flags |= UHCI_FL_SPD;
+		control = ( ring->control | UHCI_CONTROL_PID ( pid ) |
+			    UHCI_CONTROL_LEN ( frag_len ) );
+		desc->control = cpu_to_le32 ( control );
+		if ( data )
+			desc->data = virt_to_phys ( data );
+		wmb();
+		desc->status = UHCI_STATUS_ACTIVE;
+
+		/* Update data toggle */
+		ring->control ^= UHCI_CONTROL_TOGGLE;
+
+		/* Move to next descriptor */
+		data += frag_len;
+		len -= frag_len;
+
+	} while ( len );
+}
+
+/**
+ * Dequeue transfer
+ *
+ * @v ring		Transfer ring
+ * @ret iobuf		I/O buffer
+ */
+static struct io_buffer * uhci_dequeue ( struct uhci_ring *ring ) {
+	unsigned int index = ( ring->cons % UHCI_RING_COUNT );
+	struct io_buffer *iobuf;
+	struct uhci_transfer *xfer;
+	size_t len;
+
+	/* Sanity checks */
+	assert ( uhci_ring_fill ( ring ) > 0 );
+
+	/* Consume transfer */
+	xfer = ring->xfer[index];
+	assert ( xfer != NULL );
+	assert ( xfer->desc != NULL );
+	iobuf = xfer->iobuf;
+	assert ( iobuf != NULL );
+	ring->xfer[index] = NULL;
+	ring->cons++;
+
+	/* Free transfer descriptors */
+	len = ( xfer->prod * sizeof ( xfer->desc[0] ) );
+	free_dma ( xfer->desc, len );
+
+	/* Free transfer */
+	free ( xfer );
+
+	return iobuf;
+}
+
+/**
+ * Restart ring
+ *
+ * @v ring		Transfer ring
+ * @v toggle		Expected data toggle for next descriptor
+ */
+static void uhci_restart ( struct uhci_ring *ring, uint32_t toggle ) {
+	struct uhci_transfer *xfer;
+	struct uhci_transfer_descriptor *desc;
+	struct uhci_transfer_descriptor *first;
+	uint32_t link;
+	unsigned int i;
+	unsigned int j;
+
+	/* Sanity check */
+	assert ( ring->head->current == cpu_to_le32 ( UHCI_LINK_TERMINATE ) );
+
+	/* If ring is empty, then just update the data toggle for the
+	 * next descriptor.
+	 */
+	if ( uhci_ring_fill ( ring ) == 0 ) {
+		ring->control &= ~UHCI_CONTROL_TOGGLE;
+		ring->control |= toggle;
+		return;
+	}
+
+	/* If expected toggle does not match the toggle in the first
+	 * unconsumed descriptor, then invert all toggles.
+	 */
+	xfer = ring->xfer[ ring->cons % UHCI_RING_COUNT ];
+	assert ( xfer != NULL );
+	assert ( xfer->cons == 0 );
+	first = &xfer->desc[0];
+	if ( ( le32_to_cpu ( first->control ) ^ toggle ) & UHCI_CONTROL_TOGGLE){
+
+		/* Invert toggle on all unconsumed transfer descriptors */
+		for ( i = ring->cons ; i != ring->prod ; i++ ) {
+			xfer = ring->xfer[ i % UHCI_RING_COUNT ];
+			assert ( xfer != NULL );
+			assert ( xfer->cons == 0 );
+			for ( j = 0 ; j < xfer->prod ; j++ ) {
+				desc = &xfer->desc[j];
+				desc->control ^=
+					cpu_to_le32 ( UHCI_CONTROL_TOGGLE );
+			}
+		}
+
+		/* Invert toggle for next descriptor to be enqueued */
+		ring->control ^= UHCI_CONTROL_TOGGLE;
+	}
+
+	/* Restart ring at first unconsumed transfer */
+	link = virt_to_phys ( first );
+	wmb();
+	ring->head->current = cpu_to_le32 ( link );
+}
+
+/******************************************************************************
+ *
+ * Schedule management
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get link value for a queue head
+ *
+ * @v queue		Queue head
+ * @ret link		Link value
+ */
+static inline uint32_t uhci_link_qh ( struct uhci_queue_head *queue ) {
+
+	return ( virt_to_phys ( queue ) | UHCI_LINK_TYPE_QH );
+}
+
+/**
+ * (Re)build asynchronous schedule
+ *
+ * @v uhci		UHCI device
+ */
+static void uhci_async_schedule ( struct uhci_device *uhci ) {
+	struct uhci_endpoint *endpoint;
+	struct uhci_queue_head *queue;
+	uint32_t end;
+	uint32_t link;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	link = end = uhci_link_qh ( uhci->head );
+	list_for_each_entry_reverse ( endpoint, &uhci->async, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		link = uhci_link_qh ( queue );
+	}
+	if ( link == end )
+		link = UHCI_LINK_TERMINATE;
+	uhci->head->link = cpu_to_le32 ( link );
+	wmb();
+}
+
+/**
+ * Add endpoint to asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_async_add ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+
+	/* Add to end of schedule */
+	list_add_tail ( &endpoint->schedule, &uhci->async );
+
+	/* Rebuild schedule */
+	uhci_async_schedule ( uhci );
+}
+
+/**
+ * Remove endpoint from asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_async_del ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &uhci->async, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	uhci_async_schedule ( uhci );
+
+	/* Delay for a whole USB frame (with a 100% safety margin) */
+	mdelay ( 2 );
+}
+
+/**
+ * (Re)build periodic schedule
+ *
+ * @v uhci		UHCI device
+ */
+static void uhci_periodic_schedule ( struct uhci_device *uhci ) {
+	struct uhci_endpoint *endpoint;
+	struct uhci_queue_head *queue;
+	uint32_t link;
+	uint32_t end;
+	unsigned int max_interval;
+	unsigned int i;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	DBGCP ( uhci, "UHCI %s periodic schedule: ", uhci->name );
+	link = end = uhci_link_qh ( uhci->head );
+	list_for_each_entry_reverse ( endpoint, &uhci->periodic, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		DBGCP ( uhci, "%s%d", ( ( link == end ) ? "" : "<-" ),
+			endpoint->ep->interval );
+		link = uhci_link_qh ( queue );
+	}
+	DBGCP ( uhci, "\n" );
+
+	/* Populate periodic frame list */
+	DBGCP ( uhci, "UHCI %s periodic frame list:", uhci->name );
+	for ( i = 0 ; i < UHCI_FRAMES ; i++ ) {
+
+		/* Calculate maximum interval (in microframes) which
+		 * may appear as part of this frame list.
+		 */
+		if ( i == 0 ) {
+			/* Start of list: include all endpoints */
+			max_interval = -1U;
+		} else {
+			/* Calculate highest power-of-two frame interval */
+			max_interval = ( 1 << ( ffs ( i ) - 1 ) );
+			/* Convert to microframes */
+			max_interval <<= 3;
+			/* Round up to nearest 2^n-1 */
+			max_interval = ( ( max_interval << 1 ) - 1 );
+		}
+
+		/* Find first endpoint in schedule satisfying this
+		 * maximum interval constraint.
+		 */
+		link = uhci_link_qh ( uhci->head );
+		list_for_each_entry ( endpoint, &uhci->periodic, schedule ) {
+			if ( endpoint->ep->interval <= max_interval ) {
+				queue = endpoint->ring.head;
+				link = uhci_link_qh ( queue );
+				DBGCP ( uhci, " %d:%d",
+					i, endpoint->ep->interval );
+				break;
+			}
+		}
+		uhci->frame->link[i] = cpu_to_le32 ( link );
+	}
+	wmb();
+	DBGCP ( uhci, "\n" );
+}
+
+/**
+ * Add endpoint to periodic schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_periodic_add ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+	struct uhci_endpoint *before;
+	unsigned int interval = endpoint->ep->interval;
+
+	/* Find first endpoint with a smaller interval */
+	list_for_each_entry ( before, &uhci->periodic, schedule ) {
+		if ( before->ep->interval < interval )
+			break;
+	}
+	list_add_tail ( &endpoint->schedule, &before->schedule );
+
+	/* Rebuild schedule */
+	uhci_periodic_schedule ( uhci );
+}
+
+/**
+ * Remove endpoint from periodic schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_periodic_del ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &uhci->periodic, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	uhci_periodic_schedule ( uhci );
+
+	/* Delay for a whole USB frame (with a 100% safety margin) */
+	mdelay ( 2 );
+}
+
+/**
+ * Add endpoint to appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_schedule_add ( struct uhci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		uhci_periodic_add ( endpoint );
+	} else {
+		uhci_async_add ( endpoint );
+	}
+}
+
+/**
+ * Remove endpoint from appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_schedule_del ( struct uhci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		uhci_periodic_del ( endpoint );
+	} else {
+		uhci_async_del ( endpoint );
+	}
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct uhci_device *uhci = usb_get_hostdata ( usb );
+	struct uhci_endpoint *endpoint;
+	int rc;
+
+	/* Allocate and initialise structure */
+	endpoint = zalloc ( sizeof ( *endpoint ) );
+	if ( ! endpoint ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	endpoint->uhci = uhci;
+	endpoint->ep = ep;
+	usb_endpoint_set_hostdata ( ep, endpoint );
+
+	/* Initialise descriptor ring */
+	if ( ( rc = uhci_ring_alloc ( &endpoint->ring ) ) != 0 )
+		goto err_ring_alloc;
+	endpoint->ring.mtu = ep->mtu;
+	endpoint->ring.flags = UHCI_FL_CERR_MAX;
+	if ( usb->port->speed < USB_SPEED_FULL )
+		endpoint->ring.flags |= UHCI_FL_LS;
+	endpoint->ring.control = ( UHCI_CONTROL_DEVICE ( usb->address ) |
+				   UHCI_CONTROL_ENDPOINT ( ep->address ) );
+
+	/* Add to list of endpoints */
+	list_add_tail ( &endpoint->list, &uhci->endpoints );
+
+	/* Add to schedule */
+	uhci_schedule_add ( endpoint );
+
+	return 0;
+
+	uhci_ring_free ( &endpoint->ring );
+ err_ring_alloc:
+	free ( endpoint );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Close endpoint
+ *
+ * @v ep		USB endpoint
+ */
+static void uhci_endpoint_close ( struct usb_endpoint *ep ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct io_buffer *iobuf;
+
+	/* Remove from schedule */
+	uhci_schedule_del ( endpoint );
+
+	/* Cancel any incomplete transfers */
+	while ( uhci_ring_fill ( &endpoint->ring ) ) {
+		iobuf = uhci_dequeue ( &endpoint->ring );
+		if ( iobuf )
+			usb_complete_err ( ep, iobuf, -ECANCELED );
+	}
+
+	/* Remove from list of endpoints */
+	list_del ( &endpoint->list );
+
+	/* Free descriptor ring */
+	uhci_ring_free ( &endpoint->ring );
+
+	/* Free endpoint */
+	free ( endpoint );
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct uhci_ring *ring = &endpoint->ring;
+
+	/* Restart ring */
+	uhci_restart ( ring, 0 );
+
+	return 0;
+}
+
+/**
+ * Update MTU
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_mtu ( struct usb_endpoint *ep ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+
+	/* Update endpoint MTU */
+	endpoint->ring.mtu = ep->mtu;
+
+	return 0;
+}
+
+/**
+ * Enqueue message transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_message ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct uhci_ring *ring = &endpoint->ring;
+	struct usb_setup_packet *packet;
+	unsigned int count;
+	size_t len;
+	int input;
+	int rc;
+
+	/* Calculate number of descriptors */
+	assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
+	len = ( iob_len ( iobuf ) - sizeof ( *packet ) );
+	count = ( 1 /* setup stage */ +
+		  ( ( len + ring->mtu - 1 ) / ring->mtu ) /* data stage */ +
+		  1 /* status stage */ );
+
+	/* Enqueue transfer */
+	if ( ( rc = uhci_enqueue ( ring, iobuf, count ) ) != 0 )
+		return rc;
+
+	/* Describe setup stage */
+	packet = iobuf->data;
+	ring->control &= ~UHCI_CONTROL_TOGGLE;
+	uhci_describe ( ring, packet, sizeof ( *packet ), USB_PID_SETUP );
+	iob_pull ( iobuf, sizeof ( *packet ) );
+
+	/* Describe data stage, if applicable */
+	assert ( ring->control & UHCI_CONTROL_TOGGLE );
+	input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
+	if ( len ) {
+		uhci_describe ( ring, iobuf->data, len,
+				( input ? USB_PID_IN : USB_PID_OUT ) );
+	}
+
+	/* Describe status stage */
+	ring->control |= UHCI_CONTROL_TOGGLE;
+	uhci_describe ( ring, NULL, 0,
+			( ( len && input ) ? USB_PID_OUT : USB_PID_IN ) );
+
+	/* Sanity check */
+	assert ( ring->end->prod == count );
+
+	return 0;
+}
+
+/**
+ * Enqueue stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_stream ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int terminate ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct uhci_ring *ring = &endpoint->ring;
+	unsigned int count;
+	size_t len;
+	int input;
+	int zlp;
+	int rc;
+
+	/* Calculate number of descriptors */
+	len = iob_len ( iobuf );
+	zlp = ( terminate && ( ( len & ( ring->mtu - 1 ) ) == 0 ) );
+	count = ( ( ( len + ring->mtu - 1 ) / ring->mtu ) + ( zlp ? 1 : 0 ) );
+
+	/* Enqueue transfer */
+	if ( ( rc = uhci_enqueue ( ring, iobuf, count ) ) != 0 )
+		return rc;
+
+	/* Describe data packet */
+	input = ( ep->address & USB_DIR_IN );
+	uhci_describe ( ring, iobuf->data, len,
+			( input ? USB_PID_IN : USB_PID_OUT ) );
+
+	/* Describe zero-length packet, if applicable */
+	if ( zlp )
+		uhci_describe ( ring, NULL, 0, USB_PID_OUT );
+
+	/* Sanity check */
+	assert ( ring->end->prod == count );
+
+	return 0;
+}
+
+/**
+ * Check if transfer is a message transfer
+ *
+ * @v xfer		UHCI transfer
+ * @ret is_message	Transfer is a message transfer
+ */
+static inline int uhci_is_message ( struct uhci_transfer *xfer ) {
+	struct uhci_transfer_descriptor *desc = &xfer->desc[0];
+
+	return ( ( desc->control & cpu_to_le32 ( UHCI_CONTROL_PID_MASK ) ) ==
+		 cpu_to_le32 ( UHCI_CONTROL_PID ( USB_PID_SETUP ) ) );
+}
+
+/**
+ * Poll for completions
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_endpoint_poll ( struct uhci_endpoint *endpoint ) {
+	struct uhci_ring *ring = &endpoint->ring;
+	struct uhci_device *uhci = endpoint->uhci;
+	struct usb_endpoint *ep = endpoint->ep;
+	struct usb_device *usb = ep->usb;
+	struct uhci_transfer *xfer;
+	struct uhci_transfer_descriptor *desc;
+	struct io_buffer *iobuf;
+	unsigned int index;
+	uint32_t link;
+	uint32_t toggle;
+	uint32_t control;
+	uint16_t actual;
+	size_t len;
+
+	/* Consume all completed descriptors */
+	while ( uhci_ring_fill ( ring ) ) {
+
+		/* Stop if we reach an uncompleted descriptor */
+		index = ( ring->cons % UHCI_RING_COUNT );
+		xfer = ring->xfer[index];
+		assert ( xfer != NULL );
+		assert ( xfer->cons < xfer->prod );
+		desc = &xfer->desc[xfer->cons];
+		rmb();
+		if ( desc->status & UHCI_STATUS_ACTIVE )
+			break;
+		control = le32_to_cpu ( desc->control );
+		actual = le16_to_cpu ( desc->actual );
+
+		/* Update data length, if applicable */
+		if ( UHCI_DATA_PACKET ( control ) )
+			xfer->len += UHCI_ACTUAL_LEN ( actual );
+
+		/* If we have encountered an error, then deactivate
+		 * the queue head (to prevent further hardware
+		 * accesses to this transfer), consume the transfer,
+		 * and report the error to the USB core.
+		 */
+		if ( desc->status & UHCI_STATUS_STALLED ) {
+			DBGC ( uhci, "UHCI %s %s completion %d.%d failed "
+			       "(status %02x)\n", usb->name,
+			       usb_endpoint_name ( ep ), index,
+			       xfer->cons, desc->status );
+			link = UHCI_LINK_TERMINATE;
+			ring->head->current = cpu_to_le32 ( link );
+			wmb();
+			iobuf = uhci_dequeue ( ring );
+			usb_complete_err ( ep, iobuf, -EIO );
+			break;
+		}
+
+		/* Consume this descriptor */
+		xfer->cons++;
+
+		/* Check for short packets */
+		if ( UHCI_SHORT_PACKET ( control, actual ) ) {
+
+			/* Sanity checks */
+			assert ( desc->flags & UHCI_FL_SPD );
+			link = virt_to_phys ( desc );
+			assert ( ( le32_to_cpu ( ring->head->current ) &
+				   ~( UHCI_ALIGN - 1 ) ) == link );
+
+			/* If this is a message transfer, then restart
+			 * at the status stage.
+			 */
+			if ( uhci_is_message ( xfer ) ) {
+				xfer->cons = ( xfer->prod - 1 );
+				link = virt_to_phys ( &xfer->desc[xfer->cons] );
+				ring->head->current = cpu_to_le32 ( link );
+				break;
+			}
+
+			/* Otherwise, this is a stream transfer.
+			 * First, prevent further hardware access to
+			 * this transfer.
+			 */
+			link = UHCI_LINK_TERMINATE;
+			ring->head->current = cpu_to_le32 ( link );
+			wmb();
+
+			/* Determine expected data toggle for next descriptor */
+			toggle = ( ( control ^ UHCI_CONTROL_TOGGLE ) &
+				   UHCI_CONTROL_TOGGLE );
+
+			/* Consume this transfer */
+			len = xfer->len;
+			iobuf = uhci_dequeue ( ring );
+
+			/* Update packet length */
+			assert ( len <= iob_len ( iobuf ) );
+			iob_unput ( iobuf, ( iob_len ( iobuf ) - len ) );
+
+			/* Restart ring */
+			uhci_restart ( ring, toggle );
+
+		} else if ( xfer->cons == xfer->prod ) {
+
+			/* Completed a transfer: consume it */
+			len = xfer->len;
+			iobuf = uhci_dequeue ( ring );
+			assert ( len == iob_len ( iobuf ) );
+
+		} else {
+
+			/* Not a short packet and not yet complete:
+			 * continue processing.
+			 */
+			continue;
+		}
+
+		/* Report completion to USB core */
+		usb_complete ( ep, iobuf );
+	}
+}
+
+/******************************************************************************
+ *
+ * Device operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int uhci_device_open ( struct usb_device *usb ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( usb->port->hub->bus );
+
+	usb_set_hostdata ( usb, uhci );
+	return 0;
+}
+
+/**
+ * Close device
+ *
+ * @v usb		USB device
+ */
+static void uhci_device_close ( struct usb_device *usb ) {
+	struct uhci_device *uhci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = uhci->bus;
+
+	/* Free device address, if assigned */
+	if ( usb->address )
+		usb_free_address ( bus, usb->address );
+}
+
+/**
+ * Assign device address
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int uhci_device_address ( struct usb_device *usb ) {
+	struct uhci_device *uhci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = uhci->bus;
+	struct usb_endpoint *ep0 = usb_endpoint ( usb, USB_EP0_ADDRESS );
+	struct uhci_endpoint *endpoint0 = usb_endpoint_get_hostdata ( ep0 );
+	int address;
+	int rc;
+
+	/* Sanity checks */
+	assert ( usb->address == 0 );
+	assert ( ep0 != NULL );
+
+	/* Allocate device address */
+	address = usb_alloc_address ( bus );
+	if ( address < 0 ) {
+		rc = address;
+		DBGC ( uhci, "UHCI %s could not allocate address: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_alloc_address;
+	}
+
+	/* Set address */
+	if ( ( rc = usb_set_address ( usb, address ) ) != 0 )
+		goto err_set_address;
+
+	/* Update device address */
+	usb->address = address;
+	endpoint0->ring.control |= UHCI_CONTROL_DEVICE ( address );
+
+	return 0;
+
+ err_set_address:
+	usb_free_address ( bus, address );
+ err_alloc_address:
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * Hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int uhci_hub_open ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+	return 0;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void uhci_hub_close ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+}
+
+/******************************************************************************
+ *
+ * Root hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open root hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int uhci_root_open ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+
+	/* Record hub driver private data */
+	usb_hub_set_drvdata ( hub, uhci );
+
+	return 0;
+}
+
+/**
+ * Close root hub
+ *
+ * @v hub		USB hub
+ */
+static void uhci_root_close ( struct usb_hub *hub ) {
+
+	/* Clear hub driver private data */
+	usb_hub_set_drvdata ( hub, NULL );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int uhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	uint16_t portsc;
+	unsigned int i;
+
+	/* Reset port */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	portsc |= UHCI_PORTSC_PR;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_DELAY_MS );
+	portsc &= ~UHCI_PORTSC_PR;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_RECOVER_DELAY_MS );
+
+	/* Enable port */
+	portsc |= UHCI_PORTSC_PED;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_RECOVER_DELAY_MS );
+
+	/* Wait for port to become enabled */
+	for ( i = 0 ; i < UHCI_PORT_ENABLE_MAX_WAIT_MS ; i++ ) {
+
+		/* Check port status */
+		portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+		if ( portsc & UHCI_PORTSC_PED )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( uhci, "UHCI %s-%d timed out waiting for port to enable "
+	       "(status %04x)\n",  uhci->name, port->address, portsc );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int uhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	uint16_t portsc;
+
+	/* Disable port */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	portsc &= ~UHCI_PORTSC_PED;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Update root hub port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int uhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	struct pci_device pci;
+	uint16_t portsc;
+	unsigned int speed;
+
+	/* Read port status */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	if ( ! ( portsc & UHCI_PORTSC_CCS ) ) {
+		/* Port not connected */
+		speed = USB_SPEED_NONE;
+	} else if ( uhci->companion &&
+		    ! find_usb_bus_by_location ( BUS_TYPE_PCI,
+						 uhci->companion ) ) {
+		/* Defer connection detection until companion
+		 * controller has been enumerated.
+		 */
+		pci_init ( &pci, uhci->companion );
+		DBGC ( uhci, "UHCI %s-%d deferring for companion " PCI_FMT "\n",
+		       uhci->name, port->address, PCI_ARGS ( &pci ) );
+		speed = USB_SPEED_NONE;
+	} else if ( portsc & UHCI_PORTSC_LS ) {
+		/* Low-speed device */
+		speed = USB_SPEED_LOW;
+	} else {
+		/* Full-speed device */
+		speed = USB_SPEED_FULL;
+	}
+	port->speed = speed;
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= ( portsc & UHCI_PORTSC_CSC );
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+				struct usb_endpoint *ep ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+
+	/* Should never be called; this is a root hub */
+	DBGC ( uhci, "UHCI %s-%d nonsensical CLEAR_TT for %s %s\n", uhci->name,
+	       port->address, ep->usb->name, usb_endpoint_name ( ep ) );
+
+	return -ENOTSUP;
+}
+
+/**
+ * Poll for port status changes
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ */
+static void uhci_root_poll ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	uint16_t portsc;
+	uint16_t change;
+
+	/* Do nothing unless something has changed */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	change = ( portsc & UHCI_PORTSC_CHANGE );
+	if ( ! change )
+		return;
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= ( portsc & UHCI_PORTSC_CSC );
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+
+	/* Report port status change */
+	usb_port_changed ( port );
+}
+
+/******************************************************************************
+ *
+ * Bus operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+static int uhci_bus_open ( struct usb_bus *bus ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+	int rc;
+
+	/* Sanity checks */
+	assert ( list_empty ( &uhci->async ) );
+	assert ( list_empty ( &uhci->periodic ) );
+
+	/* Allocate and initialise asynchronous queue head */
+	uhci->head = malloc_dma ( sizeof ( *uhci->head ), UHCI_ALIGN );
+	if ( ! uhci->head ) {
+		rc = -ENOMEM;
+		goto err_alloc_head;
+	}
+	if ( ( rc = uhci_reachable ( uhci->head, sizeof ( *uhci->head ) ) ) !=0)
+		goto err_unreachable_head;
+	memset ( uhci->head, 0, sizeof ( *uhci->head ) );
+	uhci->head->current = cpu_to_le32 ( UHCI_LINK_TERMINATE );
+	uhci_async_schedule ( uhci );
+
+	/* Allocate periodic frame list */
+	uhci->frame = malloc_dma ( sizeof ( *uhci->frame ),
+				   sizeof ( *uhci->frame ) );
+	if ( ! uhci->frame ) {
+		rc = -ENOMEM;
+		goto err_alloc_frame;
+	}
+	if ( ( rc = uhci_reachable ( uhci->frame,
+				     sizeof ( *uhci->frame ) ) ) != 0 )
+		goto err_unreachable_frame;
+	uhci_periodic_schedule ( uhci );
+	outl ( virt_to_phys ( uhci->frame ), uhci->regs + UHCI_FLBASEADD );
+
+	/* Start controller */
+	uhci_run ( uhci );
+
+	return 0;
+
+	uhci_stop ( uhci );
+ err_unreachable_frame:
+	free_dma ( uhci->frame, sizeof ( *uhci->frame ) );
+ err_alloc_frame:
+ err_unreachable_head:
+	free_dma ( uhci->head, sizeof ( *uhci->head ) );
+ err_alloc_head:
+	return rc;
+}
+
+/**
+ * Close USB bus
+ *
+ * @v bus		USB bus
+ */
+static void uhci_bus_close ( struct usb_bus *bus ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+
+	/* Sanity checks */
+	assert ( list_empty ( &uhci->async ) );
+	assert ( list_empty ( &uhci->periodic ) );
+
+	/* Stop controller */
+	uhci_stop ( uhci );
+
+	/* Free periodic frame list */
+	free_dma ( uhci->frame, sizeof ( *uhci->frame ) );
+
+	/* Free asynchronous schedule */
+	free_dma ( uhci->head, sizeof ( *uhci->head ) );
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static void uhci_bus_poll ( struct usb_bus *bus ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+	struct usb_hub *hub = bus->hub;
+	struct uhci_endpoint *endpoint;
+	unsigned int i;
+
+	/* UHCI defers interrupts (including short packet detection)
+	 * until the end of the frame.  This can result in bulk IN
+	 * endpoints remaining halted for much of the time, waiting
+	 * for software action to reset the data toggles.  We
+	 * therefore ignore USBSTS and unconditionally poll all
+	 * endpoints for completed transfer descriptors.
+	 *
+	 * As with EHCI, we trust that completion handlers are minimal
+	 * and will not do anything that could plausibly affect the
+	 * endpoint list itself.
+	 */
+	list_for_each_entry ( endpoint, &uhci->endpoints, list )
+		uhci_endpoint_poll ( endpoint );
+
+	/* UHCI provides no single bit to indicate that a port status
+	 * change has occurred.  We therefore unconditionally iterate
+	 * over all ports looking for status changes.
+	 */
+	for ( i = 1 ; i <= UHCI_PORTS ; i++ )
+		uhci_root_poll ( hub, usb_port ( hub, i ) );
+}
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** USB host controller operations */
+static struct usb_host_operations uhci_operations = {
+	.endpoint = {
+		.open = uhci_endpoint_open,
+		.close = uhci_endpoint_close,
+		.reset = uhci_endpoint_reset,
+		.mtu = uhci_endpoint_mtu,
+		.message = uhci_endpoint_message,
+		.stream = uhci_endpoint_stream,
+	},
+	.device = {
+		.open = uhci_device_open,
+		.close = uhci_device_close,
+		.address = uhci_device_address,
+	},
+	.bus = {
+		.open = uhci_bus_open,
+		.close = uhci_bus_close,
+		.poll = uhci_bus_poll,
+	},
+	.hub = {
+		.open = uhci_hub_open,
+		.close = uhci_hub_close,
+	},
+	.root = {
+		.open = uhci_root_open,
+		.close = uhci_root_close,
+		.enable = uhci_root_enable,
+		.disable = uhci_root_disable,
+		.speed = uhci_root_speed,
+		.clear_tt = uhci_root_clear_tt,
+	},
+};
+
+/**
+ * Locate EHCI companion controller (when no EHCI support is present)
+ *
+ * @v pci		PCI device
+ * @ret busdevfn	EHCI companion controller bus:dev.fn (if any)
+ */
+__weak unsigned int ehci_companion ( struct pci_device *pci __unused ) {
+	return 0;
+}
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int uhci_probe ( struct pci_device *pci ) {
+	struct uhci_device *uhci;
+	struct usb_port *port;
+	unsigned int i;
+	int rc;
+
+	/* Allocate and initialise structure */
+	uhci = zalloc ( sizeof ( *uhci ) );
+	if ( ! uhci ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	uhci->name = pci->dev.name;
+	INIT_LIST_HEAD ( &uhci->endpoints );
+	INIT_LIST_HEAD ( &uhci->async );
+	INIT_LIST_HEAD ( &uhci->periodic );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Identify EHCI companion controller, if any */
+	uhci->companion = ehci_companion ( pci );
+
+	/* Claim ownership from BIOS.  (There is no release mechanism
+	 * for UHCI.)
+	 */
+	pci_write_config_word ( pci, UHCI_USBLEGSUP, UHCI_USBLEGSUP_DEFAULT );
+
+	/* Map registers */
+	uhci->regs = pci->ioaddr;
+	if ( ! uhci->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Reset device */
+	if ( ( rc = uhci_reset ( uhci ) ) != 0 )
+		goto err_reset;
+
+	/* Allocate USB bus */
+	uhci->bus = alloc_usb_bus ( &pci->dev, UHCI_PORTS, UHCI_MTU,
+				    &uhci_operations );
+	if ( ! uhci->bus ) {
+		rc = -ENOMEM;
+		goto err_alloc_bus;
+	}
+	usb_bus_set_hostdata ( uhci->bus, uhci );
+	usb_hub_set_drvdata ( uhci->bus->hub, uhci );
+
+	/* Set port protocols */
+	for ( i = 1 ; i <= UHCI_PORTS ; i++ ) {
+		port = usb_port ( uhci->bus->hub, i );
+		port->protocol = USB_PROTO_2_0;
+	}
+
+	/* Register USB bus */
+	if ( ( rc = register_usb_bus ( uhci->bus ) ) != 0 )
+		goto err_register;
+
+	pci_set_drvdata ( pci, uhci );
+	return 0;
+
+	unregister_usb_bus ( uhci->bus );
+ err_register:
+	free_usb_bus ( uhci->bus );
+ err_alloc_bus:
+	uhci_reset ( uhci );
+ err_reset:
+ err_ioremap:
+	free ( uhci );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void uhci_remove ( struct pci_device *pci ) {
+	struct uhci_device *uhci = pci_get_drvdata ( pci );
+	struct usb_bus *bus = uhci->bus;
+
+	unregister_usb_bus ( bus );
+	assert ( list_empty ( &uhci->async ) );
+	assert ( list_empty ( &uhci->periodic ) );
+	free_usb_bus ( bus );
+	uhci_reset ( uhci );
+	free ( uhci );
+}
+
+/** UHCI PCI device IDs */
+static struct pci_device_id uhci_ids[] = {
+	PCI_ROM ( 0xffff, 0xffff, "uhci", "UHCI", 0 ),
+};
+
+/** UHCI PCI driver */
+struct pci_driver uhci_driver __pci_driver = {
+	.ids = uhci_ids,
+	.id_count = ( sizeof ( uhci_ids ) / sizeof ( uhci_ids[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
+				PCI_CLASS_SERIAL_USB_UHCI ),
+	.probe = uhci_probe,
+	.remove = uhci_remove,
+};
diff --git a/roms/ipxe/src/drivers/usb/uhci.h b/roms/ipxe/src/drivers/usb/uhci.h
new file mode 100644
index 000000000..ba4c28f7e
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/uhci.h
@@ -0,0 +1,350 @@
+#ifndef _IPXE_UHCI_H
+#define _IPXE_UHCI_H
+
+/** @file
+ *
+ * USB Universal Host Controller Interface (UHCI) driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+
+/** Minimum alignment required for data structures
+ *
+ * With the exception of the frame list (which is page-aligned), data
+ * structures used by UHCI generally require 16-byte alignment.
+ */
+#define UHCI_ALIGN 16
+
+/** Number of ports */
+#define UHCI_PORTS 2
+
+/** Maximum transfer size */
+#define UHCI_MTU 1280
+
+/** I/O BAR size */
+#define UHCI_BAR_SIZE 0x14
+
+/** USB command register */
+#define UHCI_USBCMD 0x00
+
+/** Max packet is 64 bytes */
+#define UHCI_USBCMD_MAX64 0x0080
+
+/** Host controller reset */
+#define UHCI_USBCMD_HCRESET 0x0002
+
+/** Run/stop */
+#define UHCI_USBCMD_RUN 0x0001
+
+/** USB status register */
+#define UHCI_USBSTS 0x02
+
+/** Host controller halted */
+#define UHCI_USBSTS_HCHALTED 0x0020
+
+/** USB interrupt */
+#define UHCI_USBSTS_USBINT 0x0001
+
+/** Frame list base address register */
+#define UHCI_FLBASEADD 0x08
+
+/** Port status and control register */
+#define UHCI_PORTSC(port) ( 0x0e + ( (port) << 1 ) )
+
+/** Port reset */
+#define UHCI_PORTSC_PR 0x0200
+
+/** Low-speed device attached */
+#define UHCI_PORTSC_LS 0x0100
+
+/** Port enabled/disabled change */
+#define UHCI_PORTSC_PEC 0x0008
+
+/** Port enabled */
+#define UHCI_PORTSC_PED 0x0004
+
+/** Connect status change */
+#define UHCI_PORTSC_CSC 0x0002
+
+/** Current connect status */
+#define UHCI_PORTSC_CCS 0x0001
+
+/** Port status change mask */
+#define UHCI_PORTSC_CHANGE ( UHCI_PORTSC_CSC | UHCI_PORTSC_PEC )
+
+/** Depth-first processing */
+#define UHCI_LINK_DEPTH_FIRST 0x00000004UL
+
+/** Queue head type */
+#define UHCI_LINK_TYPE_QH 0x00000002UL
+
+/** List terminator */
+#define UHCI_LINK_TERMINATE 0x00000001UL
+
+/** Number of frames in frame list */
+#define UHCI_FRAMES 1024
+
+/** A frame list */
+struct uhci_frame_list {
+	/** Link pointer */
+	uint32_t link[UHCI_FRAMES];
+} __attribute__ (( packed ));
+
+/** A transfer descriptor */
+struct uhci_transfer_descriptor {
+	/** Link pointer */
+	uint32_t link;
+	/** Actual length */
+	uint16_t actual;
+	/** Status */
+	uint8_t status;
+	/** Flags */
+	uint8_t flags;
+	/** Control */
+	uint32_t control;
+	/** Buffer pointer */
+	uint32_t data;
+} __attribute__ (( packed ));
+
+/** Length mask */
+#define UHCI_LEN_MASK 0x7ff
+
+/** Actual length */
+#define UHCI_ACTUAL_LEN( actual ) ( ( (actual) + 1 ) & UHCI_LEN_MASK )
+
+/** Active */
+#define UHCI_STATUS_ACTIVE 0x80
+
+/** Stalled */
+#define UHCI_STATUS_STALLED 0x40
+
+/** Data buffer error */
+#define UHCI_STATUS_BUFFER 0x20
+
+/** Babble detected */
+#define UHCI_STATUS_BABBLE 0x10
+
+/** NAK received */
+#define UHCI_STATUS_NAK 0x08
+
+/** CRC/timeout error */
+#define UHCI_STATUS_CRC_TIMEOUT 0x04
+
+/** Bitstuff error */
+#define UHCI_STATUS_BITSTUFF 0x02
+
+/** Short packet detect */
+#define UHCI_FL_SPD 0x20
+
+/** Error counter */
+#define UHCI_FL_CERR( count ) ( (count) << 3 )
+
+/** Error counter maximum value */
+#define UHCI_FL_CERR_MAX UHCI_FL_CERR ( 3 )
+
+/** Low speed device */
+#define UHCI_FL_LS 0x04
+
+/** Interrupt on completion */
+#define UHCI_FL_IOC 0x01
+
+/** Packet ID */
+#define UHCI_CONTROL_PID( pid ) ( (pid) << 0 )
+
+/** Packet ID mask */
+#define UHCI_CONTROL_PID_MASK UHCI_CONTROL_PID ( 0xff )
+
+/** Device address */
+#define UHCI_CONTROL_DEVICE( address ) ( (address) << 8 )
+
+/** Endpoint address */
+#define UHCI_CONTROL_ENDPOINT( address ) ( (address) << 15 )
+
+/** Data toggle */
+#define UHCI_CONTROL_TOGGLE ( 1 << 19 )
+
+/** Data length */
+#define UHCI_CONTROL_LEN( len ) ( ( ( (len) - 1 ) & UHCI_LEN_MASK ) << 21 )
+
+/** Check for data packet
+ *
+ * This check is based on the fact that only USB_PID_SETUP has bit 2
+ * set.
+ */
+#define UHCI_DATA_PACKET( control ) ( ! ( control & 0x04 ) )
+
+/** Check for short packet */
+#define UHCI_SHORT_PACKET( control, actual ) \
+	( ( ( (control) >> 21 ) ^ (actual) ) & UHCI_LEN_MASK )
+
+/** USB legacy support register (in PCI configuration space) */
+#define UHCI_USBLEGSUP 0xc0
+
+/** USB legacy support default value */
+#define UHCI_USBLEGSUP_DEFAULT 0x2000
+
+/** A queue head */
+struct uhci_queue_head {
+	/** Horizontal link pointer */
+	uint32_t link;
+	/** Current transfer descriptor */
+	uint32_t current;
+} __attribute__ (( packed ));
+
+/** A single UHCI transfer
+ *
+ * UHCI hardware is extremely simple, and requires software to build
+ * the entire packet schedule (including manually handling all of the
+ * data toggles).  The hardware requires at least 16 bytes of transfer
+ * descriptors per 64 bytes of transmitted/received data.  We allocate
+ * the transfer descriptors at the time that the transfer is enqueued,
+ * to avoid the need to allocate unreasonably large blocks when the
+ * endpoint is opened.
+ */
+struct uhci_transfer {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+	/** Completed data length */
+	size_t len;
+
+	/** Transfer descriptors */
+	struct uhci_transfer_descriptor *desc;
+
+	/** I/O buffer */
+	struct io_buffer *iobuf;
+};
+
+/** Number of transfer descriptors in a ring
+ *
+ * This is a policy decision.
+ */
+#define UHCI_RING_COUNT 16
+
+/** A transfer ring */
+struct uhci_ring {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+
+	/** Maximum packet length */
+	size_t mtu;
+	/** Base flags
+	 *
+	 * This incorporates the CERR and LS bits
+	 */
+	uint8_t flags;
+	/** Base control word
+	 *
+	 * This incorporates the device address, the endpoint address,
+	 * and the data toggle for the next descriptor to be enqueued.
+	 */
+	uint32_t control;
+
+	/** Transfers */
+	struct uhci_transfer *xfer[UHCI_RING_COUNT];
+	/** End of transfer ring (if non-empty) */
+	struct uhci_transfer *end;
+
+	/** Queue head */
+	struct uhci_queue_head *head;
+};
+
+/**
+ * Calculate space used in transfer ring
+ *
+ * @v ring		Transfer ring
+ * @ret fill		Number of entries used
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+uhci_ring_fill ( struct uhci_ring *ring ) {
+	unsigned int fill;
+
+	fill = ( ring->prod - ring->cons );
+	assert ( fill <= UHCI_RING_COUNT );
+	return fill;
+}
+
+/**
+ * Calculate space remaining in transfer ring
+ *
+ * @v ring		Transfer ring
+ * @ret remaining	Number of entries remaining
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+uhci_ring_remaining ( struct uhci_ring *ring ) {
+	unsigned int fill = uhci_ring_fill ( ring );
+
+	return ( UHCI_RING_COUNT - fill );
+}
+
+/** Maximum time to wait for host controller to stop
+ *
+ * This is a policy decision.
+ */
+#define UHCI_STOP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for reset to complete
+ *
+ * This is a policy decision.
+ */
+#define UHCI_RESET_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a port to be enabled
+ *
+ * This is a policy decision.
+ */
+#define UHCI_PORT_ENABLE_MAX_WAIT_MS 500
+
+/** A UHCI device */
+struct uhci_device {
+	/** Registers */
+	unsigned long regs;
+	/** Name */
+	const char *name;
+
+	/** EHCI companion controller bus:dev.fn address (if any) */
+	unsigned int companion;
+
+	/** Asynchronous queue head */
+	struct uhci_queue_head *head;
+	/** Frame list */
+	struct uhci_frame_list *frame;
+
+	/** List of all endpoints */
+	struct list_head endpoints;
+	/** Asynchronous schedule */
+	struct list_head async;
+	/** Periodic schedule
+	 *
+	 * Listed in decreasing order of endpoint interval.
+	 */
+	struct list_head periodic;
+
+	/** USB bus */
+	struct usb_bus *bus;
+};
+
+/** A UHCI endpoint */
+struct uhci_endpoint {
+	/** UHCI device */
+	struct uhci_device *uhci;
+	/** USB endpoint */
+	struct usb_endpoint *ep;
+	/** List of all endpoints */
+	struct list_head list;
+	/** Endpoint schedule */
+	struct list_head schedule;
+
+	/** Transfer ring */
+	struct uhci_ring ring;
+};
+
+#endif /* _IPXE_UHCI_H */
diff --git a/roms/ipxe/src/drivers/usb/usbhid.c b/roms/ipxe/src/drivers/usb/usbhid.c
new file mode 100644
index 000000000..c74535a05
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/usbhid.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbhid.h>
+
+/** @file
+ *
+ * USB human interface devices (HID)
+ *
+ */
+
+/**
+ * Open USB human interface device
+ *
+ * @v hid		USB human interface device
+ * @ret rc		Return status code
+ */
+int usbhid_open ( struct usb_hid *hid ) {
+	int rc;
+
+	/* Open interrupt IN endpoint */
+	if ( ( rc = usb_endpoint_open ( &hid->in ) ) != 0 ) {
+		DBGC ( hid, "HID %s could not open interrupt IN: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		goto err_open_in;
+	}
+
+	/* Refill interrupt IN endpoint */
+	if ( ( rc = usb_refill ( &hid->in ) ) != 0 ) {
+		DBGC ( hid, "HID %s could not refill interrupt IN: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		goto err_refill_in;
+	}
+
+	/* Open interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb &&
+	     ( ( rc = usb_endpoint_open ( &hid->out ) ) != 0 ) ) {
+		DBGC ( hid, "HID %s could not open interrupt OUT: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		goto err_open_out;
+	}
+
+	return 0;
+
+	usb_endpoint_close ( &hid->out );
+ err_open_out:
+ err_refill_in:
+	usb_endpoint_close ( &hid->in );
+ err_open_in:
+	return rc;
+}
+
+/**
+ * Close USB human interface device
+ *
+ * @v hid		USB human interface device
+ */
+void usbhid_close ( struct usb_hid *hid ) {
+
+	/* Close interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb )
+		usb_endpoint_close ( &hid->out );
+
+	/* Close interrupt IN endpoint */
+	usb_endpoint_close ( &hid->in );
+}
+
+/**
+ * Refill USB human interface device endpoints
+ *
+ * @v hid		USB human interface device
+ * @ret rc		Return status code
+ */
+int usbhid_refill ( struct usb_hid *hid ) {
+	int rc;
+
+	/* Refill interrupt IN endpoint */
+	if ( ( rc = usb_refill ( &hid->in ) ) != 0 )
+		return rc;
+
+	/* Refill interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb && ( ( rc = usb_refill ( &hid->out ) ) != 0 ) )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Describe USB human interface device
+ *
+ * @v hid		USB human interface device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+int usbhid_describe ( struct usb_hid *hid,
+		      struct usb_configuration_descriptor *config ) {
+	struct usb_interface_descriptor *desc;
+	int rc;
+
+	/* Locate interface descriptor */
+	desc = usb_interface_descriptor ( config, hid->func->interface[0], 0 );
+	if ( ! desc ) {
+		DBGC ( hid, "HID %s has no interface descriptor\n",
+		       hid->func->name );
+		return -EINVAL;
+	}
+
+	/* Describe interrupt IN endpoint */
+	if ( ( rc = usb_endpoint_described ( &hid->in, config, desc,
+					     USB_INTERRUPT_IN, 0 ) ) != 0 ) {
+		DBGC ( hid, "HID %s could not describe interrupt IN: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Describe interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb &&
+	     ( ( rc = usb_endpoint_described ( &hid->out, config, desc,
+					       USB_INTERRUPT_OUT, 0 ) ) != 0 )){
+		DBGC ( hid, "HID %s could not describe interrupt OUT: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
diff --git a/roms/ipxe/src/drivers/usb/usbhub.c b/roms/ipxe/src/drivers/usb/usbhub.c
new file mode 100644
index 000000000..bf2a20005
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/usbhub.c
@@ -0,0 +1,547 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/usb.h>
+#include "usbhub.h"
+
+/** @file
+ *
+ * USB hub driver
+ *
+ */
+
+/**
+ * Refill interrupt ring
+ *
+ * @v hubdev		Hub device
+ */
+static void hub_refill ( struct usb_hub_device *hubdev ) {
+	int rc;
+
+	/* Refill interrupt endpoint */
+	if ( ( rc = usb_refill ( &hubdev->intr ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not refill interrupt: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		/* Continue attempting to refill */
+		return;
+	}
+
+	/* Stop refill process */
+	process_del ( &hubdev->refill );
+}
+
+/** Refill process descriptor */
+static struct process_descriptor hub_refill_desc =
+	PROC_DESC ( struct usb_hub_device, refill, hub_refill );
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void hub_complete ( struct usb_endpoint *ep,
+			   struct io_buffer *iobuf, int rc ) {
+	struct usb_hub_device *hubdev =
+		container_of ( ep, struct usb_hub_device, intr );
+	struct usb_hub *hub = hubdev->hub;
+	uint8_t *data = iobuf->data;
+	unsigned int bits = ( 8 * iob_len ( iobuf ) );
+	unsigned int i;
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto done;
+
+	/* Ignore packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( hubdev, "HUB %s interrupt failed: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		DBGC_HDA ( hubdev, 0, iobuf->data, iob_len ( iobuf ) );
+		goto done;
+	}
+
+	/* Report any port status changes */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+
+		/* Sanity check */
+		if ( i > bits ) {
+			DBGC ( hubdev, "HUB %s underlength interrupt:\n",
+			       hubdev->name );
+			DBGC_HDA ( hubdev, 0, iobuf->data, iob_len ( iobuf ) );
+			goto done;
+		}
+
+		/* Report port status change if applicable */
+		if ( data[ i / 8 ] & ( 1 << ( i % 8 ) ) ) {
+			DBGC2 ( hubdev, "HUB %s port %d status changed\n",
+				hubdev->name, i );
+			usb_port_changed ( usb_port ( hub, i ) );
+		}
+	}
+
+ done:
+	/* Start refill process */
+	process_add ( &hubdev->refill );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations usb_hub_intr_operations = {
+	.complete = hub_complete,
+};
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int hub_open ( struct usb_hub *hub ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	unsigned int i;
+	int rc;
+
+	/* Ensure ports are powered */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		if ( ( rc = usb_hub_set_port_feature ( usb, i,
+						       USB_HUB_PORT_POWER,
+						       0 ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s port %d could not apply power: "
+			       "%s\n", hubdev->name, i, strerror ( rc ) );
+			goto err_power;
+		}
+	}
+
+	/* Open interrupt endpoint */
+	if ( ( rc = usb_endpoint_open ( &hubdev->intr ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not register interrupt: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Start refill process */
+	process_add ( &hubdev->refill );
+
+	/* Refill interrupt ring */
+	hub_refill ( hubdev );
+
+	return 0;
+
+	usb_endpoint_close ( &hubdev->intr );
+ err_open:
+ err_power:
+	return rc;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void hub_close ( struct usb_hub *hub ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+
+	/* Close interrupt endpoint */
+	usb_endpoint_close ( &hubdev->intr );
+
+	/* Stop refill process */
+	process_del ( &hubdev->refill );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int hub_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	struct usb_hub_port_status status;
+	unsigned int current;
+	unsigned int i;
+	int rc;
+
+	/* Initiate reset if applicable */
+	if ( ( hub->protocol < USB_PROTO_3_0 ) &&
+	     ( ( rc = usb_hub_set_port_feature ( usb, port->address,
+						 USB_HUB_PORT_RESET, 0 ) )!=0)){
+		DBGC ( hubdev, "HUB %s port %d could not initiate reset: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Wait for port to become enabled */
+	for ( i = 0 ; i < USB_HUB_ENABLE_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for port being enabled */
+		if ( ( rc = usb_hub_get_port_status ( usb, port->address,
+						      &status ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s port %d could not get status: "
+			       "%s\n", hubdev->name, port->address,
+			       strerror ( rc ) );
+			return rc;
+		}
+		current = le16_to_cpu ( status.current );
+		if ( current & ( 1 << USB_HUB_PORT_ENABLE ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( hubdev, "HUB %s port %d timed out waiting for enable\n",
+	       hubdev->name, port->address );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int hub_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	int rc;
+
+	/* Disable port */
+	if ( ( rc = usb_hub_clear_port_feature ( usb, port->address,
+						 USB_HUB_PORT_ENABLE, 0 ) )!=0){
+		DBGC ( hubdev, "HUB %s port %d could not disable: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Clear port status change bits
+ *
+ * @v hubdev		USB hub device
+ * @v port		Port number
+ * @v changed		Port status change bits
+ * @ret rc		Return status code
+ */
+static int hub_clear_changes ( struct usb_hub_device *hubdev,
+			       unsigned int port, uint16_t changed ) {
+	struct usb_device *usb = hubdev->usb;
+	unsigned int bit;
+	unsigned int feature;
+	int rc;
+
+	/* Clear each set bit */
+	for ( bit = 0 ; bit < 16 ; bit++ ) {
+
+		/* Skip unset bits */
+		if ( ! ( changed & ( 1 << bit ) ) )
+			continue;
+
+		/* Skip unused features */
+		feature = USB_HUB_C_FEATURE ( bit );
+		if ( ! ( hubdev->features & ( 1 << feature ) ) )
+			continue;
+
+		/* Clear bit */
+		if ( ( rc = usb_hub_clear_port_feature ( usb, port,
+							 feature, 0 ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s port %d could not clear feature "
+			       "%d: %s\n", hubdev->name, port, feature,
+			       strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Update port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int hub_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	struct usb_hub_port_status status;
+	unsigned int current;
+	unsigned int changed;
+	int rc;
+
+	/* Get port status */
+	if ( ( rc = usb_hub_get_port_status ( usb, port->address,
+					      &status ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s port %d could not get status: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+	current = le16_to_cpu ( status.current );
+	changed = le16_to_cpu ( status.changed );
+	DBGC2 ( hubdev, "HUB %s port %d status is %04x:%04x\n",
+		hubdev->name, port->address, changed, current );
+
+	/* Update port speed */
+	if ( current & ( 1 << USB_HUB_PORT_CONNECTION ) ) {
+		if ( hub->protocol >= USB_PROTO_3_0 ) {
+			port->speed = USB_SPEED_SUPER;
+		} else if ( current & ( 1 << USB_HUB_PORT_LOW_SPEED ) ) {
+			port->speed = USB_SPEED_LOW;
+		} else if ( current & ( 1 << USB_HUB_PORT_HIGH_SPEED ) ) {
+			port->speed = USB_SPEED_HIGH;
+		} else {
+			port->speed = USB_SPEED_FULL;
+		}
+	} else {
+		port->speed = USB_SPEED_NONE;
+	}
+
+	/* Record disconnections */
+	port->disconnected |= ( changed & ( 1 << USB_HUB_PORT_CONNECTION ) );
+
+	/* Clear port status change bits */
+	if ( ( rc = hub_clear_changes ( hubdev, port->address, changed ) ) != 0)
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int hub_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+			  struct usb_endpoint *ep ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	int rc;
+
+	/* Clear transaction translator buffer.  All hubs must support
+	 * single-TT operation; we simplify our code by supporting
+	 * only this configuration.
+	 */
+	if ( ( rc = usb_hub_clear_tt_buffer ( usb, ep->usb->address,
+					      ep->address, ep->attributes,
+					      USB_HUB_TT_SINGLE ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s port %d could not clear TT buffer: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/** USB hub operations */
+static struct usb_hub_driver_operations hub_operations = {
+	.open = hub_open,
+	.close = hub_close,
+	.enable = hub_enable,
+	.disable = hub_disable,
+	.speed = hub_speed,
+	.clear_tt = hub_clear_tt,
+};
+
+/**
+ * Probe USB hub
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int hub_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct usb_bus *bus = usb->port->hub->bus;
+	struct usb_hub_device *hubdev;
+	struct usb_interface_descriptor *interface;
+	union usb_hub_descriptor desc;
+	unsigned int depth;
+	unsigned int ports;
+	int enhanced;
+	int rc;
+
+	/* Allocate and initialise structure */
+	hubdev = zalloc ( sizeof ( *hubdev ) );
+	if ( ! hubdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	enhanced = ( usb->port->protocol >= USB_PROTO_3_0 );
+	hubdev->name = func->name;
+	hubdev->usb = usb;
+	hubdev->features =
+		( enhanced ? USB_HUB_FEATURES_ENHANCED : USB_HUB_FEATURES );
+	usb_endpoint_init ( &hubdev->intr, usb, &usb_hub_intr_operations );
+	usb_refill_init ( &hubdev->intr, 0, USB_HUB_INTR_FILL );
+	process_init_stopped ( &hubdev->refill, &hub_refill_desc, NULL );
+
+	/* Locate hub interface descriptor */
+	interface = usb_interface_descriptor ( config, func->interface[0], 0 );
+	if ( ! interface ) {
+		DBGC ( hubdev, "HUB %s has no interface descriptor\n",
+		       hubdev->name );
+		rc = -EINVAL;
+		goto err_interface;
+	}
+
+	/* Locate interrupt endpoint descriptor */
+	if ( ( rc = usb_endpoint_described ( &hubdev->intr, config, interface,
+					     USB_INTERRUPT_IN, 0 ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not describe interrupt endpoint: "
+		       "%s\n", hubdev->name, strerror ( rc ) );
+		goto err_endpoint;
+	}
+
+	/* Set hub depth */
+	depth = usb_depth ( usb );
+	if ( enhanced ) {
+		if ( ( rc = usb_hub_set_hub_depth ( usb, depth ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s could not set hub depth to %d: "
+			       "%s\n", hubdev->name, depth, strerror ( rc ) );
+			goto err_set_hub_depth;
+		}
+	}
+
+	/* Get hub descriptor */
+	if ( ( rc = usb_hub_get_descriptor ( usb, enhanced, &desc ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not get hub descriptor: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		goto err_hub_descriptor;
+	}
+	ports = desc.basic.ports;
+	DBGC ( hubdev, "HUB %s has %d ports at depth %d%s\n", hubdev->name,
+	       ports, depth, ( enhanced ? " (enhanced)" : "" ) );
+
+	/* Allocate hub */
+	hubdev->hub = alloc_usb_hub ( bus, usb, ports, &hub_operations );
+	if ( ! hubdev->hub ) {
+		rc = -ENOMEM;
+		goto err_alloc_hub;
+	}
+	usb_hub_set_drvdata ( hubdev->hub, hubdev );
+
+	/* Register hub */
+	if ( ( rc = register_usb_hub ( hubdev->hub ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not register: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		goto err_register_hub;
+	}
+
+	usb_func_set_drvdata ( func, hubdev );
+	return 0;
+
+	unregister_usb_hub ( hubdev->hub );
+ err_register_hub:
+	free_usb_hub ( hubdev->hub );
+ err_alloc_hub:
+ err_hub_descriptor:
+ err_set_hub_depth:
+ err_endpoint:
+ err_interface:
+	free ( hubdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove USB hub
+ *
+ * @v func		USB function
+ * @ret rc		Return status code
+ */
+static void hub_remove ( struct usb_function *func ) {
+	struct usb_hub_device *hubdev = usb_func_get_drvdata ( func );
+	struct usb_hub *hub = hubdev->hub;
+	struct usb_device *usb = hubdev->usb;
+	struct usb_port *port;
+	unsigned int i;
+
+	/* If hub has been unplugged, mark all ports as unplugged */
+	if ( usb->port->speed == USB_SPEED_NONE ) {
+		for ( i = 1 ; i <= hub->ports ; i++ ) {
+			port = usb_port ( hub, i );
+			port->speed = USB_SPEED_NONE;
+		}
+	}
+
+	/* Unregister hub */
+	unregister_usb_hub ( hubdev->hub );
+	assert ( ! process_running ( &hubdev->refill ) );
+
+	/* Free hub */
+	free_usb_hub ( hubdev->hub );
+
+	/* Free hub device */
+	free ( hubdev );
+}
+
+/** USB hub device IDs */
+static struct usb_device_id hub_ids[] = {
+	{
+		.name = "hub-1",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_HUB,
+			.subclass = 0,
+			.protocol = 0,
+		},
+	},
+	{
+		.name = "hub-2",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_HUB,
+			.subclass = 0,
+			.protocol = 1,
+		},
+	},
+};
+
+/** USB hub driver */
+struct usb_driver usb_hub_driver __usb_driver = {
+	.ids = hub_ids,
+	.id_count = ( sizeof ( hub_ids ) / sizeof ( hub_ids[0] ) ),
+	.probe = hub_probe,
+	.remove = hub_remove,
+};
diff --git a/roms/ipxe/src/drivers/usb/usbhub.h b/roms/ipxe/src/drivers/usb/usbhub.h
new file mode 100644
index 000000000..d7d8f9610
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/usbhub.h
@@ -0,0 +1,279 @@
+#ifndef _USBHUB_H
+#define _USBHUB_H
+
+/** @file
+ *
+ * USB hubs
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/list.h>
+#include <ipxe/process.h>
+
+/** Request recipient is a port */
+#define USB_HUB_RECIP_PORT ( 3 << 0 )
+
+/** A basic USB hub descriptor */
+struct usb_hub_descriptor_basic {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Number of ports */
+	uint8_t ports;
+	/** Characteristics */
+	uint16_t characteristics;
+	/** Power-on delay (in 2ms intervals */
+	uint8_t delay;
+	/** Controller current (in mA) */
+	uint8_t current;
+} __attribute__ (( packed ));
+
+/** A basic USB hub descriptor */
+#define USB_HUB_DESCRIPTOR 41
+
+/** An enhanced USB hub descriptor */
+struct usb_hub_descriptor_enhanced {
+	/** Basic USB hub descriptor */
+	struct usb_hub_descriptor_basic basic;
+	/** Header decode latency */
+	uint8_t latency;
+	/** Maximum delay */
+	uint16_t delay;
+	/** Removable device bitmask */
+	uint16_t removable;
+} __attribute__ (( packed ));
+
+/** An enhanced USB hub descriptor */
+#define USB_HUB_DESCRIPTOR_ENHANCED 42
+
+/** A USB hub descriptor */
+union usb_hub_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Basic hub descriptor */
+	struct usb_hub_descriptor_basic basic;
+	/** Enhanced hub descriptor */
+	struct usb_hub_descriptor_enhanced enhanced;
+} __attribute__ (( packed ));
+
+/** Port status */
+struct usb_hub_port_status {
+	/** Current status */
+	uint16_t current;
+	/** Changed status */
+	uint16_t changed;
+} __attribute__ (( packed ));
+
+/** Current connect status feature */
+#define USB_HUB_PORT_CONNECTION 0
+
+/** Port enabled/disabled feature */
+#define USB_HUB_PORT_ENABLE 1
+
+/** Port reset feature */
+#define USB_HUB_PORT_RESET 4
+
+/** Port power feature */
+#define USB_HUB_PORT_POWER 8
+
+/** Low-speed device attached */
+#define USB_HUB_PORT_LOW_SPEED 9
+
+/** High-speed device attached */
+#define USB_HUB_PORT_HIGH_SPEED 10
+
+/** Connect status changed */
+#define USB_HUB_C_PORT_CONNECTION 16
+
+/** Port enable/disable changed */
+#define	USB_HUB_C_PORT_ENABLE 17
+
+/** Suspend changed */
+#define USB_HUB_C_PORT_SUSPEND 18
+
+/** Over-current indicator changed */
+#define USB_HUB_C_PORT_OVER_CURRENT 19
+
+/** Reset changed */
+#define USB_HUB_C_PORT_RESET 20
+
+/** Link state changed */
+#define USB_HUB_C_PORT_LINK_STATE 25
+
+/** Configuration error */
+#define USB_HUB_C_PORT_CONFIG_ERROR 26
+
+/** Calculate feature from change bit number */
+#define USB_HUB_C_FEATURE( bit ) ( 16 + (bit) )
+
+/** USB features */
+#define USB_HUB_FEATURES						\
+	( ( 1 << USB_HUB_C_PORT_CONNECTION ) |				\
+	  ( 1 << USB_HUB_C_PORT_ENABLE ) |				\
+	  ( 1 << USB_HUB_C_PORT_SUSPEND ) |				\
+	  ( 1 << USB_HUB_C_PORT_OVER_CURRENT ) |			\
+	  ( 1 << USB_HUB_C_PORT_RESET ) )
+
+/** USB features for enhanced hubs */
+#define USB_HUB_FEATURES_ENHANCED					\
+	( ( 1 << USB_HUB_C_PORT_CONNECTION ) |				\
+	  ( 1 << USB_HUB_C_PORT_OVER_CURRENT ) |			\
+	  ( 1 << USB_HUB_C_PORT_RESET ) |				\
+	  ( 1 << USB_HUB_C_PORT_LINK_STATE ) |				\
+	  ( 1 << USB_HUB_C_PORT_CONFIG_ERROR ) )
+
+/** Set hub depth */
+#define USB_HUB_SET_HUB_DEPTH						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_DEVICE |		\
+	  USB_REQUEST_TYPE ( 12 ) )
+
+/** Clear transaction translator buffer */
+#define USB_HUB_CLEAR_TT_BUFFER						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_HUB_RECIP_PORT |		\
+	  USB_REQUEST_TYPE ( 8 ) )
+
+/**
+ * Get hub descriptor
+ *
+ * @v usb		USB device
+ * @v enhanced		Hub is an enhanced hub
+ * @v data		Hub descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_get_descriptor ( struct usb_device *usb, int enhanced,
+			 union usb_hub_descriptor *data ) {
+	unsigned int desc;
+	size_t len;
+
+	/* Determine descriptor type and length */
+	desc = ( enhanced ? USB_HUB_DESCRIPTOR_ENHANCED : USB_HUB_DESCRIPTOR );
+	len = ( enhanced ? sizeof ( data->enhanced ) : sizeof ( data->basic ) );
+
+	return usb_get_descriptor ( usb, USB_TYPE_CLASS, desc, 0, 0,
+				    &data->header, len );
+}
+
+/**
+ * Get port status
+ *
+ * @v usb		USB device
+ * @v port		Port address
+ * @v status		Port status descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_get_port_status ( struct usb_device *usb, unsigned int port,
+			  struct usb_hub_port_status *status ) {
+
+	return usb_get_status ( usb, ( USB_TYPE_CLASS | USB_HUB_RECIP_PORT ),
+				port, status, sizeof ( *status ) );
+}
+
+/**
+ * Clear port feature
+ *
+ * @v usb		USB device
+ * @v port		Port address
+ * @v feature		Feature to clear
+ * @v index		Index (when clearing a port indicator)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_clear_port_feature ( struct usb_device *usb, unsigned int port,
+			     unsigned int feature, unsigned int index ) {
+
+	return usb_clear_feature ( usb, ( USB_TYPE_CLASS | USB_HUB_RECIP_PORT ),
+				   feature, ( ( index << 8 ) | port ) );
+}
+
+/**
+ * Set port feature
+ *
+ * @v usb		USB device
+ * @v port		Port address
+ * @v feature		Feature to clear
+ * @v index		Index (when clearing a port indicator)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_set_port_feature ( struct usb_device *usb, unsigned int port,
+			   unsigned int feature, unsigned int index ) {
+
+	return usb_set_feature ( usb, ( USB_TYPE_CLASS | USB_HUB_RECIP_PORT ),
+				 feature, ( ( index << 8 ) | port ) );
+}
+
+/**
+ * Set hub depth
+ *
+ * @v usb		USB device
+ * @v depth		Hub depth
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_set_hub_depth ( struct usb_device *usb, unsigned int depth ) {
+
+	return usb_control ( usb, USB_HUB_SET_HUB_DEPTH, depth, 0, NULL, 0 );
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v usb		USB device
+ * @v device		Device address
+ * @v endpoint		Endpoint address
+ * @v attributes	Endpoint attributes
+ * @v tt_port		Transaction translator port (or 1 for single-TT hubs)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_clear_tt_buffer ( struct usb_device *usb, unsigned int device,
+			  unsigned int endpoint, unsigned int attributes,
+			  unsigned int tt_port ) {
+	unsigned int value;
+
+	/* Calculate value */
+	value = ( ( ( endpoint & USB_ENDPOINT_MAX ) << 0 ) | ( device << 4 ) |
+		  ( ( attributes & USB_ENDPOINT_ATTR_TYPE_MASK ) << 11 ) |
+		  ( ( endpoint & USB_ENDPOINT_IN ) << 8 ) );
+
+	return usb_control ( usb, USB_HUB_CLEAR_TT_BUFFER, value,
+			     tt_port, NULL, 0 );
+}
+
+/** Transaction translator port value for single-TT hubs */
+#define USB_HUB_TT_SINGLE 1
+
+/** A USB hub device */
+struct usb_hub_device {
+	/** Name */
+	const char *name;
+	/** USB device */
+	struct usb_device *usb;
+	/** USB hub */
+	struct usb_hub *hub;
+	/** Features */
+	unsigned int features;
+
+	/** Interrupt endpoint */
+	struct usb_endpoint intr;
+	/** Interrupt endpoint refill process */
+	struct process refill;
+};
+
+/** Interrupt ring fill level
+ *
+ * This is a policy decision.
+ */
+#define USB_HUB_INTR_FILL 4
+
+/** Maximum time to wait for port to become enabled
+ *
+ * This is a policy decision.
+ */
+#define USB_HUB_ENABLE_MAX_WAIT_MS 100
+
+#endif /* _USBHUB_H */
diff --git a/roms/ipxe/src/drivers/usb/usbkbd.c b/roms/ipxe/src/drivers/usb/usbkbd.c
new file mode 100644
index 000000000..ea94f2e63
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/usbkbd.c
@@ -0,0 +1,509 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/console.h>
+#include <ipxe/keys.h>
+#include <ipxe/usb.h>
+#include "usbkbd.h"
+
+/** @file
+ *
+ * USB keyboard driver
+ *
+ */
+
+/** List of USB keyboards */
+static LIST_HEAD ( usb_keyboards );
+
+/******************************************************************************
+ *
+ * Keyboard map
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Map USB keycode to iPXE key
+ *
+ * @v keycode		Keycode
+ * @v modifiers		Modifiers
+ * @ret key		iPXE key
+ *
+ * Key codes are defined in the USB HID Usage Tables Keyboard/Keypad
+ * page.
+ */
+static unsigned int usbkbd_map ( unsigned int keycode,
+				 unsigned int modifiers ) {
+	unsigned int key;
+
+	if ( keycode < USBKBD_KEY_A ) {
+		/* Not keys */
+		key = 0;
+	} else if ( keycode <= USBKBD_KEY_Z ) {
+		/* Alphabetic keys */
+		key = ( keycode - USBKBD_KEY_A + 'a' );
+		if ( modifiers & USBKBD_CTRL ) {
+			key -= ( 'a' - CTRL_A );
+		} else if ( modifiers & USBKBD_SHIFT ) {
+			key -= ( 'a' - 'A' );
+		}
+	} else if ( keycode <= USBKBD_KEY_0 ) {
+		/* Numeric key row */
+		if ( modifiers & USBKBD_SHIFT ) {
+			key = "!@#$%^&*()" [ keycode - USBKBD_KEY_1 ];
+		} else {
+			key = ( ( ( keycode - USBKBD_KEY_1 + 1 ) % 10 ) + '0' );
+		}
+	} else if ( keycode <= USBKBD_KEY_SPACE ) {
+		/* Unmodifiable keys */
+		static const uint8_t unmodifable[] =
+			{ LF, ESC, BACKSPACE, TAB, ' ' };
+		key = unmodifable[ keycode - USBKBD_KEY_ENTER ];
+	} else if ( keycode <= USBKBD_KEY_SLASH ) {
+		/* Punctuation keys */
+		if ( modifiers & USBKBD_SHIFT ) {
+			key = "_+{}|~:\"~<>?" [ keycode - USBKBD_KEY_MINUS ];
+		} else {
+			key = "-=[]\\#;'`,./" [ keycode - USBKBD_KEY_MINUS ];
+		}
+	} else if ( keycode <= USBKBD_KEY_UP ) {
+		/* Special keys */
+		static const uint16_t special[] = {
+			0, 0, 0, 0, 0, KEY_F5, KEY_F6, KEY_F7, KEY_F8, KEY_F9,
+			KEY_F10, KEY_F11, KEY_F12, 0, 0, 0, KEY_IC, KEY_HOME,
+			KEY_PPAGE, KEY_DC, KEY_END, KEY_NPAGE, KEY_RIGHT,
+			KEY_LEFT, KEY_DOWN, KEY_UP
+		};
+		key = special[ keycode - USBKBD_KEY_CAPSLOCK ];
+	} else {
+		key = 0;
+	}
+
+	return key;
+}
+
+/******************************************************************************
+ *
+ * Keyboard buffer
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Insert keypress into keyboard buffer
+ *
+ * @v kbd		USB keyboard
+ * @v keycode		Keycode
+ * @v modifiers		Modifiers
+ */
+static void usbkbd_produce ( struct usb_keyboard *kbd, unsigned int keycode,
+			     unsigned int modifiers ) {
+	unsigned int key;
+
+	/* Map to iPXE key */
+	key = usbkbd_map ( keycode, modifiers );
+
+	/* Do nothing if this keycode has no corresponding iPXE key */
+	if ( ! key ) {
+		DBGC ( kbd, "KBD %s has no key for keycode %#02x:%#02x\n",
+		       kbd->name, modifiers, keycode );
+		return;
+	}
+
+	/* Check for buffer overrun */
+	if ( usbkbd_fill ( kbd ) >= USBKBD_BUFSIZE ) {
+		DBGC ( kbd, "KBD %s buffer overrun (key %#02x)\n",
+		       kbd->name, key );
+		return;
+	}
+
+	/* Insert into buffer */
+	kbd->key[ ( kbd->prod++ ) % USBKBD_BUFSIZE ] = key;
+	DBGC2 ( kbd, "KBD %s key %#02x produced\n", kbd->name, key );
+}
+
+/**
+ * Consume character from keyboard buffer
+ *
+ * @v kbd		USB keyboard
+ * @ret character	Character
+ */
+static unsigned int usbkbd_consume ( struct usb_keyboard *kbd ) {
+	static char buf[] = "\x1b[xx~";
+	char *tmp = &buf[2];
+	unsigned int key;
+	unsigned int character;
+	unsigned int ansi_n;
+	unsigned int len;
+
+	/* Sanity check */
+	assert ( usbkbd_fill ( kbd ) > 0 );
+
+	/* Get current keypress */
+	key = kbd->key[ kbd->cons % USBKBD_BUFSIZE ];
+
+	/* If this is a straightforward key, just consume and return it */
+	if ( key < KEY_MIN ) {
+		kbd->cons++;
+		DBGC2 ( kbd, "KBD %s key %#02x consumed\n", kbd->name, key );
+		return key;
+	}
+
+	/* Construct ANSI sequence */
+	ansi_n = KEY_ANSI_N ( key );
+	if ( ansi_n )
+		tmp += sprintf ( tmp, "%d", ansi_n );
+	*(tmp++) = KEY_ANSI_TERMINATOR ( key );
+	*tmp = '\0';
+	len = ( tmp - buf );
+	assert ( len < sizeof ( buf ) );
+	if ( kbd->subcons == 0 ) {
+		DBGC2 ( kbd, "KBD %s key %#02x consumed as ^[%s\n",
+			kbd->name, key, &buf[1] );
+	}
+
+	/* Extract character from ANSI sequence */
+	assert ( kbd->subcons < len );
+	character = buf[ kbd->subcons++ ];
+
+	/* Consume key if applicable */
+	if ( kbd->subcons == len ) {
+		kbd->cons++;
+		kbd->subcons = 0;
+	}
+
+	return character;
+}
+
+/******************************************************************************
+ *
+ * Keyboard report
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Check for presence of keycode in report
+ *
+ * @v report		Keyboard report
+ * @v keycode		Keycode (must be non-zero)
+ * @ret has_keycode	Keycode is present in report
+ */
+static int usbkbd_has_keycode ( struct usb_keyboard_report *report,
+				unsigned int keycode ) {
+	unsigned int i;
+
+	/* Check for keycode */
+	for ( i = 0 ; i < ( sizeof ( report->keycode ) /
+			    sizeof ( report->keycode[0] ) ) ; i++ ) {
+		if ( report->keycode[i] == keycode )
+			return keycode;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle keyboard report
+ *
+ * @v kbd		USB keyboard
+ * @v new		New keyboard report
+ */
+static void usbkbd_report ( struct usb_keyboard *kbd,
+			    struct usb_keyboard_report *new ) {
+	struct usb_keyboard_report *old = &kbd->report;
+	unsigned int keycode;
+	unsigned int i;
+
+	/* Check if current key has been released */
+	if ( kbd->keycode && ! usbkbd_has_keycode ( new, kbd->keycode ) ) {
+		DBGC2 ( kbd, "KBD %s keycode %#02x released\n",
+			kbd->name, kbd->keycode );
+		kbd->keycode = 0;
+	}
+
+	/* Decrement auto-repeat hold-off timer, if applicable */
+	if ( kbd->holdoff )
+		kbd->holdoff--;
+
+	/* Check if a new key has been pressed */
+	for ( i = 0 ; i < ( sizeof ( new->keycode ) /
+			    sizeof ( new->keycode[0] ) ) ; i++ ) {
+
+		/* Ignore keys present in the previous report */
+		keycode = new->keycode[i];
+		if ( ( keycode == 0 ) || usbkbd_has_keycode ( old, keycode ) )
+			continue;
+		DBGC2 ( kbd, "KBD %s keycode %#02x pressed\n",
+			kbd->name, keycode );
+
+		/* Insert keypress into keyboard buffer */
+		usbkbd_produce ( kbd, keycode, new->modifiers );
+
+		/* Record as most recent keycode */
+		kbd->keycode = keycode;
+
+		/* Start auto-repeat hold-off timer */
+		kbd->holdoff = USBKBD_HOLDOFF;
+	}
+
+	/* Insert auto-repeated keypress into keyboard buffer, if applicable */
+	if ( kbd->keycode && ! kbd->holdoff )
+		usbkbd_produce ( kbd, kbd->keycode, new->modifiers );
+
+	/* Record report */
+	memcpy ( old, new, sizeof ( *old ) );
+}
+
+/******************************************************************************
+ *
+ * Interrupt endpoint
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void usbkbd_complete ( struct usb_endpoint *ep,
+			      struct io_buffer *iobuf, int rc ) {
+	struct usb_keyboard *kbd = container_of ( ep, struct usb_keyboard,
+						  hid.in );
+	struct usb_keyboard_report *report;
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto drop;
+
+	/* Ignore packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( kbd, "KBD %s interrupt IN failed: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto drop;
+	}
+
+	/* Ignore underlength packets */
+	if ( iob_len ( iobuf ) < sizeof ( *report ) ) {
+		DBGC ( kbd, "KBD %s underlength report:\n", kbd->name );
+		DBGC_HDA ( kbd, 0, iobuf->data, iob_len ( iobuf ) );
+		goto drop;
+	}
+	report = iobuf->data;
+
+	/* Handle keyboard report */
+	usbkbd_report ( kbd, report );
+
+ drop:
+	/* Recycle I/O buffer */
+	usb_recycle ( &kbd->hid.in, iobuf );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations usbkbd_operations = {
+	.complete = usbkbd_complete,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usbkbd_probe ( struct usb_function *func,
+			  struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct usb_keyboard *kbd;
+	int rc;
+
+	/* Allocate and initialise structure */
+	kbd = zalloc ( sizeof ( *kbd ) );
+	if ( ! kbd ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	kbd->name = func->name;
+	kbd->bus = usb->port->hub->bus;
+	usbhid_init ( &kbd->hid, func, &usbkbd_operations, NULL );
+	usb_refill_init ( &kbd->hid.in, sizeof ( kbd->report ),
+			  USBKBD_INTR_MAX_FILL );
+
+	/* Describe USB human interface device */
+	if ( ( rc = usbhid_describe ( &kbd->hid, config ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not describe: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_describe;
+	}
+	DBGC ( kbd, "KBD %s using %s (len %zd)\n",
+	       kbd->name, usb_endpoint_name ( &kbd->hid.in ), kbd->hid.in.mtu );
+
+	/* Set boot protocol */
+	if ( ( rc = usbhid_set_protocol ( usb, func->interface[0],
+					  USBHID_PROTOCOL_BOOT ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not set boot protocol: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_set_protocol;
+	}
+
+	/* Set idle time */
+	if ( ( rc = usbhid_set_idle ( usb, func->interface[0], 0,
+				      USBKBD_IDLE_DURATION ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not set idle time: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_set_idle;
+	}
+
+	/* Open USB human interface device */
+	if ( ( rc = usbhid_open ( &kbd->hid ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not open: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Add to list of USB keyboards */
+	list_add_tail ( &kbd->list, &usb_keyboards );
+
+	usb_func_set_drvdata ( func, kbd );
+	return 0;
+
+	usbhid_close ( &kbd->hid );
+ err_open:
+ err_set_idle:
+ err_set_protocol:
+ err_describe:
+	free ( kbd );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void usbkbd_remove ( struct usb_function *func ) {
+	struct usb_keyboard *kbd = usb_func_get_drvdata ( func );
+
+	/* Remove from list of USB keyboards */
+	list_del ( &kbd->list );
+
+	/* Close USB human interface device */
+	usbhid_close ( &kbd->hid );
+
+	/* Free device */
+	free ( kbd );
+}
+
+/** USB keyboard device IDs */
+static struct usb_device_id usbkbd_ids[] = {
+	{
+		.name = "kbd",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_HID,
+			.subclass = USB_SUBCLASS_HID_BOOT,
+			.protocol = USBKBD_PROTOCOL,
+		},
+	},
+};
+
+/** USB keyboard driver */
+struct usb_driver usbkbd_driver __usb_driver = {
+	.ids = usbkbd_ids,
+	.id_count = ( sizeof ( usbkbd_ids ) / sizeof ( usbkbd_ids[0] ) ),
+	.probe = usbkbd_probe,
+	.remove = usbkbd_remove,
+};
+
+/******************************************************************************
+ *
+ * Console interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Read a character from the console
+ *
+ * @ret character	Character read
+ */
+static int usbkbd_getchar ( void ) {
+	struct usb_keyboard *kbd;
+
+	/* Consume first available key */
+	list_for_each_entry ( kbd, &usb_keyboards, list ) {
+		if ( usbkbd_fill ( kbd ) )
+			return usbkbd_consume ( kbd );
+	}
+
+	return 0;
+}
+
+/**
+ * Check for available input
+ *
+ * @ret is_available	Input is available
+ */
+static int usbkbd_iskey ( void ) {
+	struct usb_keyboard *kbd;
+	unsigned int fill;
+
+	/* Poll all USB keyboards and refill endpoints */
+	list_for_each_entry ( kbd, &usb_keyboards, list ) {
+		usb_poll ( kbd->bus );
+		usb_refill ( &kbd->hid.in );
+	}
+
+	/* Check for a non-empty keyboard buffer */
+	list_for_each_entry ( kbd, &usb_keyboards, list ) {
+		fill = usbkbd_fill ( kbd );
+		if ( fill )
+			return fill;
+	}
+
+	return 0;
+}
+
+/** USB keyboard console */
+struct console_driver usbkbd_console __console_driver = {
+	.getchar = usbkbd_getchar,
+	.iskey = usbkbd_iskey,
+};
diff --git a/roms/ipxe/src/drivers/usb/usbkbd.h b/roms/ipxe/src/drivers/usb/usbkbd.h
new file mode 100644
index 000000000..7eab24e46
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/usbkbd.h
@@ -0,0 +1,154 @@
+#ifndef _USBKBD_H
+#define _USBKBD_H
+
+/** @file
+ *
+ * USB keyboard driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbhid.h>
+
+/** Keyboard protocol */
+#define USBKBD_PROTOCOL 1
+
+/** A USB keyboard report */
+struct usb_keyboard_report {
+	/** Modifier keys */
+	uint8_t modifiers;
+	/** Reserved */
+	uint8_t reserved;
+	/** Keycodes */
+	uint8_t keycode[6];
+} __attribute__ (( packed ));
+
+/** USB modifier keys */
+enum usb_keyboard_modifier {
+	/** Left Ctrl key */
+	USBKBD_CTRL_LEFT = 0x01,
+	/** Left Shift key */
+	USBKBD_SHIFT_LEFT = 0x02,
+	/** Left Alt key */
+	USBKBD_ALT_LEFT = 0x04,
+	/** Left GUI key */
+	USBKBD_GUI_LEFT = 0x08,
+	/** Right Ctrl key */
+	USBKBD_CTRL_RIGHT = 0x10,
+	/** Right Shift key */
+	USBKBD_SHIFT_RIGHT = 0x20,
+	/** Right Alt key */
+	USBKBD_ALT_RIGHT = 0x40,
+	/** Right GUI key */
+	USBKBD_GUI_RIGHT = 0x80,
+};
+
+/** Either Ctrl key */
+#define USBKBD_CTRL ( USBKBD_CTRL_LEFT | USBKBD_CTRL_RIGHT )
+
+/** Either Shift key */
+#define USBKBD_SHIFT ( USBKBD_SHIFT_LEFT | USBKBD_SHIFT_RIGHT )
+
+/** Either Alt key */
+#define USBKBD_ALT ( USBKBD_ALT_LEFT | USBKBD_ALT_RIGHT )
+
+/** Either GUI key */
+#define USBKBD_GUI ( USBKBD_GUI_LEFT | USBKBD_GUI_RIGHT )
+
+/** USB keycodes */
+enum usb_keycode {
+	USBKBD_KEY_A = 0x04,
+	USBKBD_KEY_Z = 0x1d,
+	USBKBD_KEY_1 = 0x1e,
+	USBKBD_KEY_0 = 0x27,
+	USBKBD_KEY_ENTER = 0x28,
+	USBKBD_KEY_SPACE = 0x2c,
+	USBKBD_KEY_MINUS = 0x2d,
+	USBKBD_KEY_SLASH = 0x38,
+	USBKBD_KEY_CAPSLOCK = 0x39,
+	USBKBD_KEY_UP = 0x52,
+};
+
+/** Keyboard idle duration (in 4ms units)
+ *
+ * This is a policy decision.  We choose to use an autorepeat rate of
+ * approximately 40ms.
+ */
+#define USBKBD_IDLE_DURATION 10 /* 10 x 4ms = 40ms */
+
+/** Keyboard auto-repeat hold-off (in units of USBKBD_IDLE_DURATION)
+ *
+ * This is a policy decision.  We choose to use an autorepeat delay of
+ * approximately 500ms.
+ */
+#define USBKBD_HOLDOFF 12 /* 12 x 40ms = 480ms */
+
+/** Interrupt endpoint maximum fill level
+ *
+ * When idling, we are likely to poll the USB endpoint at only the
+ * 18.2Hz system timer tick rate.  With a typical observed bInterval
+ * of 10ms (which will be rounded down to 8ms by the HCI drivers),
+ * this gives approximately 7 completions per poll.
+ */
+#define USBKBD_INTR_MAX_FILL 8
+
+/** Keyboard buffer size
+ *
+ * Must be a power of two.
+ */
+#define USBKBD_BUFSIZE 8
+
+/** A USB keyboard device */
+struct usb_keyboard {
+	/** Name */
+	const char *name;
+	/** List of all USB keyboards */
+	struct list_head list;
+
+	/** USB bus */
+	struct usb_bus *bus;
+	/** USB human interface device */
+	struct usb_hid hid;
+
+	/** Most recent keyboard report */
+	struct usb_keyboard_report report;
+	/** Most recently pressed non-modifier key (if any) */
+	unsigned int keycode;
+	/** Autorepeat hold-off time (in number of completions reported) */
+	unsigned int holdoff;
+
+	/** Keyboard buffer
+	 *
+	 * This stores iPXE key values.
+	 */
+	unsigned int key[USBKBD_BUFSIZE];
+	/** Keyboard buffer producer counter */
+	unsigned int prod;
+	/** Keyboard buffer consumer counter */
+	unsigned int cons;
+	/** Keyboard buffer sub-consumer counter
+	 *
+	 * This represents the index within the ANSI escape sequence
+	 * corresponding to an iPXE key value.
+	 */
+	unsigned int subcons;
+};
+
+/**
+ * Calculate keyboard buffer fill level
+ *
+ * @v kbd		USB keyboard
+ * @ret fill		Keyboard buffer fill level
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+usbkbd_fill ( struct usb_keyboard *kbd ) {
+	unsigned int fill = ( kbd->prod - kbd->cons );
+
+	assert ( fill <= USBKBD_BUFSIZE );
+	return fill;
+}
+
+#endif /* _USBKBD_H */
diff --git a/roms/ipxe/src/drivers/usb/usbnet.c b/roms/ipxe/src/drivers/usb/usbnet.c
new file mode 100644
index 000000000..b92336d05
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/usbnet.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+
+/** @file
+ *
+ * USB network devices
+ *
+ * USB network devices use a variety of packet formats and interface
+ * descriptors, but tend to have several features in common:
+ *
+ *  - a single interrupt endpoint using the generic refill mechanism
+ *
+ *  - a single bulk IN endpoint using the generic refill mechanism
+ *
+ *  - a single bulk OUT endpoint
+ *
+ *  - optional use of an alternate setting to enable the data interface
+ *
+ */
+
+/**
+ * Open USB network device
+ *
+ * @v usbnet		USB network device
+ * @ret rc		Return status code
+ */
+int usbnet_open ( struct usbnet_device *usbnet ) {
+	struct usb_device *usb = usbnet->func->usb;
+	int rc;
+
+	/* Open interrupt endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbnet->intr ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not open interrupt: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_open_intr;
+	}
+
+	/* Refill interrupt endpoint */
+	if ( ( rc = usb_refill ( &usbnet->intr ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not refill interrupt: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_refill_intr;
+	}
+
+	/* Select alternate setting for data interface, if applicable */
+	if ( usbnet->alternate &&
+	     ( ( rc = usb_set_interface ( usb, usbnet->data,
+					  usbnet->alternate ) ) != 0 ) ) {
+		DBGC ( usbnet, "USBNET %s could not set alternate interface "
+		       "%d: %s\n", usbnet->func->name, usbnet->alternate,
+		       strerror ( rc ) );
+		goto err_set_interface;
+	}
+
+	/* Open bulk IN endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbnet->in ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not open bulk IN: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_open_in;
+	}
+
+	/* Open bulk OUT endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbnet->out ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not open bulk OUT: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_open_out;
+	}
+
+	/* Refill bulk IN endpoint */
+	if ( ( rc = usb_refill ( &usbnet->in ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not refill bulk IN: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_refill_in;
+	}
+
+	return 0;
+
+ err_refill_in:
+	usb_endpoint_close ( &usbnet->out );
+ err_open_out:
+	usb_endpoint_close ( &usbnet->in );
+ err_open_in:
+	if ( usbnet->alternate )
+		usb_set_interface ( usb, usbnet->data, 0 );
+ err_set_interface:
+ err_refill_intr:
+	usb_endpoint_close ( &usbnet->intr );
+ err_open_intr:
+	return rc;
+}
+
+/**
+ * Close USB network device
+ *
+ * @v usbnet		USB network device
+ */
+void usbnet_close ( struct usbnet_device *usbnet ) {
+	struct usb_device *usb = usbnet->func->usb;
+
+	/* Close bulk OUT endpoint */
+	usb_endpoint_close ( &usbnet->out );
+
+	/* Close bulk IN endpoint */
+	usb_endpoint_close ( &usbnet->in );
+
+	/* Reset alternate setting for data interface, if applicable */
+	if ( usbnet->alternate )
+		usb_set_interface ( usb, usbnet->data, 0 );
+
+	/* Close interrupt endpoint */
+	usb_endpoint_close ( &usbnet->intr );
+}
+
+/**
+ * Refill USB network device bulk IN and interrupt endpoints
+ *
+ * @v usbnet		USB network device
+ * @ret rc		Return status code
+ */
+int usbnet_refill ( struct usbnet_device *usbnet ) {
+	int rc;
+
+	/* Refill bulk IN endpoint */
+	if ( ( rc = usb_refill ( &usbnet->in ) ) != 0 )
+		return rc;
+
+	/* Refill interrupt endpoint */
+	if ( ( rc = usb_refill ( &usbnet->intr ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Describe communications interface and interrupt endpoint
+ *
+ * @v usbnet		USB network device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usbnet_comms_describe ( struct usbnet_device *usbnet,
+				   struct usb_configuration_descriptor *config){
+	struct usb_interface_descriptor *desc;
+	unsigned int comms;
+	unsigned int i;
+	int rc;
+
+	/* Iterate over all available interfaces */
+	for ( i = 0 ; i < usbnet->func->count ; i++ ) {
+
+		/* Get interface number */
+		comms = usbnet->func->interface[i];
+
+		/* Locate interface descriptor */
+		desc = usb_interface_descriptor ( config, comms, 0 );
+		if ( ! desc )
+			continue;
+
+		/* Describe interrupt endpoint */
+		if ( ( rc = usb_endpoint_described ( &usbnet->intr, config,
+						     desc, USB_INTERRUPT_IN,
+						     0 ) ) != 0 )
+			continue;
+
+		/* Record communications interface */
+		usbnet->comms = comms;
+		DBGC ( usbnet, "USBNET %s found communications interface %d\n",
+		       usbnet->func->name, comms );
+		return 0;
+	}
+
+	DBGC ( usbnet, "USBNET %s found no communications interface\n",
+	       usbnet->func->name );
+	return -ENOENT;
+}
+
+/**
+ * Describe data interface and bulk endpoints
+ *
+ * @v usbnet		USB network device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usbnet_data_describe ( struct usbnet_device *usbnet,
+				  struct usb_configuration_descriptor *config ){
+	struct usb_interface_descriptor *desc;
+	unsigned int data;
+	unsigned int alt;
+	unsigned int i;
+	int rc;
+
+	/* Iterate over all available interfaces */
+	for ( i = 0 ; i < usbnet->func->count ; i++ ) {
+
+		/* Get interface number */
+		data = usbnet->func->interface[i];
+
+		/* Iterate over all existent alternate settings */
+		for ( alt = 0 ; ; alt++ ) {
+
+			/* Locate interface descriptor */
+			desc = usb_interface_descriptor ( config, data, alt );
+			if ( ! desc )
+				break;
+
+			/* Describe bulk IN endpoint */
+			if ( ( rc = usb_endpoint_described ( &usbnet->in,
+							     config, desc,
+							     USB_BULK_IN,
+							     0 ) ) != 0 )
+				continue;
+
+			/* Describe bulk OUT endpoint */
+			if ( ( rc = usb_endpoint_described ( &usbnet->out,
+							     config, desc,
+							     USB_BULK_OUT,
+							     0 ) ) != 0 )
+				continue;
+
+			/* Record data interface and alternate setting */
+			usbnet->data = data;
+			usbnet->alternate = alt;
+			DBGC ( usbnet, "USBNET %s found data interface %d",
+			       usbnet->func->name, data );
+			if ( alt )
+				DBGC ( usbnet, " using alternate %d", alt );
+			DBGC ( usbnet, "\n" );
+			return 0;
+		}
+	}
+
+	DBGC ( usbnet, "USBNET %s found no data interface\n",
+	       usbnet->func->name );
+	return -ENOENT;
+}
+
+/**
+ * Describe USB network device interfaces
+ *
+ * @v usbnet		USB network device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+int usbnet_describe ( struct usbnet_device *usbnet,
+		      struct usb_configuration_descriptor *config ) {
+	int rc;
+
+	/* Describe communications interface */
+	if ( ( rc = usbnet_comms_describe ( usbnet, config ) ) != 0 )
+		return rc;
+
+	/* Describe data interface */
+	if ( ( rc = usbnet_data_describe ( usbnet, config ) ) != 0 )
+		return rc;
+
+	return 0;
+}
diff --git a/roms/ipxe/src/drivers/usb/xhci.c b/roms/ipxe/src/drivers/usb/xhci.c
new file mode 100644
index 000000000..49e67316b
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/xhci.c
@@ -0,0 +1,3321 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+#include <ipxe/init.h>
+#include <ipxe/profile.h>
+#include "xhci.h"
+
+/** @file
+ *
+ * USB eXtensible Host Controller Interface (xHCI) driver
+ *
+ */
+
+/** Message transfer profiler */
+static struct profiler xhci_message_profiler __profiler =
+	{ .name = "xhci.message" };
+
+/** Stream transfer profiler */
+static struct profiler xhci_stream_profiler __profiler =
+	{ .name = "xhci.stream" };
+
+/** Event ring profiler */
+static struct profiler xhci_event_profiler __profiler =
+	{ .name = "xhci.event" };
+
+/** Transfer event profiler */
+static struct profiler xhci_transfer_profiler __profiler =
+	{ .name = "xhci.transfer" };
+
+/* Disambiguate the various error causes */
+#define EIO_DATA							\
+	__einfo_error ( EINFO_EIO_DATA )
+#define EINFO_EIO_DATA							\
+	__einfo_uniqify ( EINFO_EIO, ( 2 - 0 ),				\
+			  "Data buffer error" )
+#define EIO_BABBLE							\
+	__einfo_error ( EINFO_EIO_BABBLE )
+#define EINFO_EIO_BABBLE						\
+	__einfo_uniqify ( EINFO_EIO, ( 3 - 0 ),				\
+			  "Babble detected" )
+#define EIO_USB								\
+	__einfo_error ( EINFO_EIO_USB )
+#define EINFO_EIO_USB							\
+	__einfo_uniqify ( EINFO_EIO, ( 4 - 0 ),				\
+			  "USB transaction error" )
+#define EIO_TRB								\
+	__einfo_error ( EINFO_EIO_TRB )
+#define EINFO_EIO_TRB							\
+	__einfo_uniqify ( EINFO_EIO, ( 5 - 0 ),				\
+			  "TRB error" )
+#define EIO_STALL							\
+	__einfo_error ( EINFO_EIO_STALL )
+#define EINFO_EIO_STALL							\
+	__einfo_uniqify ( EINFO_EIO, ( 6 - 0 ),				\
+			  "Stall error" )
+#define EIO_RESOURCE							\
+	__einfo_error ( EINFO_EIO_RESOURCE )
+#define EINFO_EIO_RESOURCE						\
+	__einfo_uniqify ( EINFO_EIO, ( 7 - 0 ),				\
+			  "Resource error" )
+#define EIO_BANDWIDTH							\
+	__einfo_error ( EINFO_EIO_BANDWIDTH )
+#define EINFO_EIO_BANDWIDTH						\
+	__einfo_uniqify ( EINFO_EIO, ( 8 - 0 ),				\
+			  "Bandwidth error" )
+#define EIO_NO_SLOTS							\
+	__einfo_error ( EINFO_EIO_NO_SLOTS )
+#define EINFO_EIO_NO_SLOTS						\
+	__einfo_uniqify ( EINFO_EIO, ( 9 - 0 ),				\
+			  "No slots available" )
+#define EIO_STREAM_TYPE							\
+	__einfo_error ( EINFO_EIO_STREAM_TYPE )
+#define EINFO_EIO_STREAM_TYPE						\
+	__einfo_uniqify ( EINFO_EIO, ( 10 - 0 ),			\
+			  "Invalid stream type" )
+#define EIO_SLOT							\
+	__einfo_error ( EINFO_EIO_SLOT )
+#define EINFO_EIO_SLOT							\
+	__einfo_uniqify ( EINFO_EIO, ( 11 - 0 ),			\
+			  "Slot not enabled" )
+#define EIO_ENDPOINT							\
+	__einfo_error ( EINFO_EIO_ENDPOINT )
+#define EINFO_EIO_ENDPOINT						\
+	__einfo_uniqify ( EINFO_EIO, ( 12 - 0 ),			\
+			  "Endpoint not enabled" )
+#define EIO_SHORT							\
+	__einfo_error ( EINFO_EIO_SHORT )
+#define EINFO_EIO_SHORT							\
+	__einfo_uniqify ( EINFO_EIO, ( 13 - 0 ),			\
+			  "Short packet" )
+#define EIO_UNDERRUN							\
+	__einfo_error ( EINFO_EIO_UNDERRUN )
+#define EINFO_EIO_UNDERRUN						\
+	__einfo_uniqify ( EINFO_EIO, ( 14 - 0 ),			\
+			  "Ring underrun" )
+#define EIO_OVERRUN							\
+	__einfo_error ( EINFO_EIO_OVERRUN )
+#define EINFO_EIO_OVERRUN						\
+	__einfo_uniqify ( EINFO_EIO, ( 15 - 0 ),			\
+			  "Ring overrun" )
+#define EIO_VF_RING_FULL						\
+	__einfo_error ( EINFO_EIO_VF_RING_FULL )
+#define EINFO_EIO_VF_RING_FULL						\
+	__einfo_uniqify ( EINFO_EIO, ( 16 - 0 ),			\
+			  "Virtual function event ring full" )
+#define EIO_PARAMETER							\
+	__einfo_error ( EINFO_EIO_PARAMETER )
+#define EINFO_EIO_PARAMETER						\
+	__einfo_uniqify ( EINFO_EIO, ( 17 - 0 ),			\
+			  "Parameter error" )
+#define EIO_BANDWIDTH_OVERRUN						\
+	__einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
+#define EINFO_EIO_BANDWIDTH_OVERRUN					\
+	__einfo_uniqify ( EINFO_EIO, ( 18 - 0 ),			\
+			  "Bandwidth overrun" )
+#define EIO_CONTEXT							\
+	__einfo_error ( EINFO_EIO_CONTEXT )
+#define EINFO_EIO_CONTEXT						\
+	__einfo_uniqify ( EINFO_EIO, ( 19 - 0 ),			\
+			  "Context state error" )
+#define EIO_NO_PING							\
+	__einfo_error ( EINFO_EIO_NO_PING )
+#define EINFO_EIO_NO_PING						\
+	__einfo_uniqify ( EINFO_EIO, ( 20 - 0 ),			\
+			  "No ping response" )
+#define EIO_RING_FULL							\
+	__einfo_error ( EINFO_EIO_RING_FULL )
+#define EINFO_EIO_RING_FULL						\
+	__einfo_uniqify ( EINFO_EIO, ( 21 - 0 ),			\
+			  "Event ring full" )
+#define EIO_INCOMPATIBLE						\
+	__einfo_error ( EINFO_EIO_INCOMPATIBLE )
+#define EINFO_EIO_INCOMPATIBLE						\
+	__einfo_uniqify ( EINFO_EIO, ( 22 - 0 ),			\
+			  "Incompatible device" )
+#define EIO_MISSED							\
+	__einfo_error ( EINFO_EIO_MISSED )
+#define EINFO_EIO_MISSED						\
+	__einfo_uniqify ( EINFO_EIO, ( 23 - 0 ),			\
+			  "Missed service error" )
+#define EIO_CMD_STOPPED							\
+	__einfo_error ( EINFO_EIO_CMD_STOPPED )
+#define EINFO_EIO_CMD_STOPPED						\
+	__einfo_uniqify ( EINFO_EIO, ( 24 - 0 ),			\
+			  "Command ring stopped" )
+#define EIO_CMD_ABORTED							\
+	__einfo_error ( EINFO_EIO_CMD_ABORTED )
+#define EINFO_EIO_CMD_ABORTED						\
+	__einfo_uniqify ( EINFO_EIO, ( 25 - 0 ),			\
+			  "Command aborted" )
+#define EIO_STOP							\
+	__einfo_error ( EINFO_EIO_STOP )
+#define EINFO_EIO_STOP							\
+	__einfo_uniqify ( EINFO_EIO, ( 26 - 0 ),			\
+			  "Stopped" )
+#define EIO_STOP_LEN							\
+	__einfo_error ( EINFO_EIO_STOP_LEN )
+#define EINFO_EIO_STOP_LEN						\
+	__einfo_uniqify ( EINFO_EIO, ( 27 - 0 ),			\
+			  "Stopped - length invalid" )
+#define EIO_STOP_SHORT							\
+	__einfo_error ( EINFO_EIO_STOP_SHORT )
+#define EINFO_EIO_STOP_SHORT						\
+	__einfo_uniqify ( EINFO_EIO, ( 28 - 0 ),			\
+			  "Stopped - short packet" )
+#define EIO_LATENCY							\
+	__einfo_error ( EINFO_EIO_LATENCY )
+#define EINFO_EIO_LATENCY						\
+	__einfo_uniqify ( EINFO_EIO, ( 29 - 0 ),			\
+			  "Maximum exit latency too large" )
+#define EIO_ISOCH							\
+	__einfo_error ( EINFO_EIO_ISOCH )
+#define EINFO_EIO_ISOCH							\
+	__einfo_uniqify ( EINFO_EIO, ( 31 - 0 ),			\
+			  "Isochronous buffer overrun" )
+#define EPROTO_LOST							\
+	__einfo_error ( EINFO_EPROTO_LOST )
+#define EINFO_EPROTO_LOST						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ),			\
+			  "Event lost" )
+#define EPROTO_UNDEFINED						\
+	__einfo_error ( EINFO_EPROTO_UNDEFINED )
+#define EINFO_EPROTO_UNDEFINED						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ),			\
+			  "Undefined error" )
+#define EPROTO_STREAM_ID						\
+	__einfo_error ( EINFO_EPROTO_STREAM_ID )
+#define EINFO_EPROTO_STREAM_ID						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ),			\
+			  "Invalid stream ID" )
+#define EPROTO_SECONDARY						\
+	__einfo_error ( EINFO_EPROTO_SECONDARY )
+#define EINFO_EPROTO_SECONDARY						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ),			\
+			  "Secondary bandwidth error" )
+#define EPROTO_SPLIT							\
+	__einfo_error ( EINFO_EPROTO_SPLIT )
+#define EINFO_EPROTO_SPLIT						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ),			\
+			  "Split transaction error" )
+#define ECODE(code)							\
+	( ( (code) < 32 ) ?						\
+	  EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE,	\
+		  EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE,		\
+		  EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE,		\
+		  EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN,	\
+		  EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER,		\
+		  EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING,	\
+		  EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED,		\
+		  EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP,		\
+		  EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY,		\
+		  EIO_ISOCH ) :						\
+	  ( (code) < 64 ) ?						\
+	  EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST,		\
+		  EPROTO_UNDEFINED, EPROTO_STREAM_ID,			\
+		  EPROTO_SECONDARY, EPROTO_SPLIT ) :			\
+	  EFAULT )
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Initialise device
+ *
+ * @v xhci		xHCI device
+ * @v regs		MMIO registers
+ */
+static void xhci_init ( struct xhci_device *xhci, void *regs ) {
+	uint32_t hcsparams1;
+	uint32_t hcsparams2;
+	uint32_t hccparams1;
+	uint32_t pagesize;
+	size_t caplength;
+	size_t rtsoff;
+	size_t dboff;
+
+	/* Locate capability, operational, runtime, and doorbell registers */
+	xhci->cap = regs;
+	caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
+	rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
+	dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
+	xhci->op = ( xhci->cap + caplength );
+	xhci->run = ( xhci->cap + rtsoff );
+	xhci->db = ( xhci->cap + dboff );
+	DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n",
+		xhci->name, virt_to_phys ( xhci->cap ),
+		virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ),
+		virt_to_phys ( xhci->db ) );
+
+	/* Read structural parameters 1 */
+	hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
+	xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
+	xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
+	xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
+	DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n",
+	       xhci->name, xhci->slots, xhci->intrs, xhci->ports );
+
+	/* Read structural parameters 2 */
+	hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
+	xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
+	DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n",
+		xhci->name, xhci->scratchpads );
+
+	/* Read capability parameters 1 */
+	hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
+	xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
+	xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
+	xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
+
+	/* Read page size */
+	pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
+	xhci->pagesize = XHCI_PAGESIZE ( pagesize );
+	assert ( xhci->pagesize != 0 );
+	assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
+	DBGC2 ( xhci, "XHCI %s page size %zd bytes\n",
+		xhci->name, xhci->pagesize );
+}
+
+/**
+ * Find extended capability
+ *
+ * @v xhci		xHCI device
+ * @v id		Capability ID
+ * @v offset		Offset to previous extended capability instance, or zero
+ * @ret offset		Offset to extended capability, or zero if not found
+ */
+static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
+					       unsigned int id,
+					       unsigned int offset ) {
+	uint32_t xecp;
+	unsigned int next;
+
+	/* Locate the extended capability */
+	while ( 1 ) {
+
+		/* Locate first or next capability as applicable */
+		if ( offset ) {
+			xecp = readl ( xhci->cap + offset );
+			next = XHCI_XECP_NEXT ( xecp );
+		} else {
+			next = xhci->xecp;
+		}
+		if ( ! next )
+			return 0;
+		offset += next;
+
+		/* Check if this is the requested capability */
+		xecp = readl ( xhci->cap + offset );
+		if ( XHCI_XECP_ID ( xecp ) == id )
+			return offset;
+	}
+}
+
+/**
+ * Write potentially 64-bit register
+ *
+ * @v xhci		xHCI device
+ * @v value		Value
+ * @v reg		Register address
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
+
+	/* If this is a 32-bit build, then this can never fail
+	 * (allowing the compiler to optimise out the error path).
+	 */
+	if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
+		writel ( value, reg );
+		writel ( 0, ( reg + sizeof ( uint32_t ) ) );
+		return 0;
+	}
+
+	/* If the device does not support 64-bit addresses and this
+	 * address is outside the 32-bit address space, then fail.
+	 */
+	if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
+		DBGC ( xhci, "XHCI %s cannot access address %lx\n",
+		       xhci->name, value );
+		return -ENOTSUP;
+	}
+
+	/* If this is a 64-bit build, then writeq() is available */
+	writeq ( value, reg );
+	return 0;
+}
+
+/**
+ * Calculate buffer alignment
+ *
+ * @v len		Length
+ * @ret align		Buffer alignment
+ *
+ * Determine alignment required for a buffer which must be aligned to
+ * at least XHCI_MIN_ALIGN and which must not cross a page boundary.
+ */
+static inline size_t xhci_align ( size_t len ) {
+	size_t align;
+
+	/* Align to own length (rounded up to a power of two) */
+	align = ( 1 << fls ( len - 1 ) );
+
+	/* Round up to XHCI_MIN_ALIGN if needed */
+	if ( align < XHCI_MIN_ALIGN )
+		align = XHCI_MIN_ALIGN;
+
+	return align;
+}
+
+/**
+ * Calculate device context offset
+ *
+ * @v xhci		xHCI device
+ * @v ctx		Context index
+ */
+static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
+						  unsigned int ctx ) {
+
+	return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
+}
+
+/**
+ * Calculate input context offset
+ *
+ * @v xhci		xHCI device
+ * @v ctx		Context index
+ */
+static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
+						 unsigned int ctx ) {
+
+	return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
+}
+
+/******************************************************************************
+ *
+ * Diagnostics
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Dump host controller registers
+ *
+ * @v xhci		xHCI device
+ */
+static inline void xhci_dump ( struct xhci_device *xhci ) {
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	uint32_t pagesize;
+	uint32_t dnctrl;
+	uint32_t config;
+
+	/* Do nothing unless debugging is enabled */
+	if ( ! DBG_LOG )
+		return;
+
+	/* Dump USBCMD */
+	usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+	DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd,
+	       ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
+	       ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
+
+	/* Dump USBSTS */
+	usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
+	DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts,
+	       ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
+
+	/* Dump PAGESIZE */
+	pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
+	DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize );
+
+	/* Dump DNCTRL */
+	dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
+	DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl );
+
+	/* Dump CONFIG */
+	config = readl ( xhci->op + XHCI_OP_CONFIG );
+	DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config );
+}
+
+/**
+ * Dump port registers
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ */
+static inline void xhci_dump_port ( struct xhci_device *xhci,
+				    unsigned int port ) {
+	uint32_t portsc;
+	uint32_t portpmsc;
+	uint32_t portli;
+	uint32_t porthlpmc;
+
+	/* Do nothing unless debugging is enabled */
+	if ( ! DBG_LOG )
+		return;
+
+	/* Dump PORTSC */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n",
+	       xhci->name, port, portsc,
+	       ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
+	       ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
+	       ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
+	       ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
+	       XHCI_PORTSC_PSIV ( portsc ) );
+
+	/* Dump PORTPMSC */
+	portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc );
+
+	/* Dump PORTLI */
+	portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli );
+
+	/* Dump PORTHLPMC */
+	porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n",
+	       xhci->name, port, porthlpmc );
+}
+
+/******************************************************************************
+ *
+ * USB legacy support
+ *
+ ******************************************************************************
+ */
+
+/** Prevent the release of ownership back to BIOS */
+static int xhci_legacy_prevent_release;
+
+/**
+ * Initialise USB legacy support
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_legacy_init ( struct xhci_device *xhci ) {
+	unsigned int legacy;
+	uint8_t bios;
+
+	/* Locate USB legacy support capability (if present) */
+	legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
+	if ( ! legacy ) {
+		/* Not an error; capability may not be present */
+		DBGC ( xhci, "XHCI %s has no USB legacy support capability\n",
+		       xhci->name );
+		return;
+	}
+
+	/* Check if legacy USB support is enabled */
+	bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
+	if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
+		/* Not an error; already owned by OS */
+		DBGC ( xhci, "XHCI %s USB legacy support already disabled\n",
+		       xhci->name );
+		return;
+	}
+
+	/* Record presence of USB legacy support capability */
+	xhci->legacy = legacy;
+}
+
+/**
+ * Claim ownership from BIOS
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_legacy_claim ( struct xhci_device *xhci ) {
+	uint32_t ctlsts;
+	uint8_t bios;
+	unsigned int i;
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! xhci->legacy )
+		return;
+
+	/* Claim ownership */
+	writeb ( XHCI_USBLEGSUP_OS_OWNED,
+		 xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
+
+	/* Wait for BIOS to release ownership */
+	for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if BIOS has released ownership */
+		bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
+		if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
+			DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n",
+			       xhci->name );
+			ctlsts = readl ( xhci->cap + xhci->legacy +
+					 XHCI_USBLEGSUP_CTLSTS );
+			if ( ctlsts ) {
+				DBGC ( xhci, "XHCI %s warning: BIOS retained "
+				       "SMIs: %08x\n", xhci->name, ctlsts );
+			}
+			return;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* BIOS did not release ownership.  Claim it forcibly by
+	 * disabling all SMIs.
+	 */
+	DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly "
+	       "disabling SMIs\n", xhci->name );
+	writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS );
+}
+
+/**
+ * Release ownership back to BIOS
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_legacy_release ( struct xhci_device *xhci ) {
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! xhci->legacy )
+		return;
+
+	/* Do nothing if releasing ownership is prevented */
+	if ( xhci_legacy_prevent_release ) {
+		DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n",
+		       xhci->name );
+		return;
+	}
+
+	/* Release ownership */
+	writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
+	DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name );
+}
+
+/******************************************************************************
+ *
+ * Supported protocols
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Transcribe port speed (for debugging)
+ *
+ * @v psi		Protocol speed ID
+ * @ret speed		Transcribed speed
+ */
+static inline const char * xhci_speed_name ( uint32_t psi ) {
+	static const char *exponents[4] = { "", "k", "M", "G" };
+	static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
+	unsigned int mantissa;
+	unsigned int exponent;
+
+	/* Extract mantissa and exponent */
+	mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+	exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+
+	/* Transcribe speed */
+	snprintf ( buf, sizeof ( buf ), "%d%sbps",
+		   mantissa, exponents[exponent] );
+	return buf;
+}
+
+/**
+ * Find supported protocol extended capability for a port
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @ret supported	Offset to extended capability, or zero if not found
+ */
+static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
+					      unsigned int port ) {
+	unsigned int supported = 0;
+	unsigned int offset;
+	unsigned int count;
+	uint32_t ports;
+
+	/* Iterate over all supported protocol structures */
+	while ( ( supported = xhci_extended_capability ( xhci,
+							 XHCI_XECP_ID_SUPPORTED,
+							 supported ) ) ) {
+
+		/* Determine port range */
+		ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+		offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
+		count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
+
+		/* Check if port lies within this range */
+		if ( ( port - offset ) < count )
+			return supported;
+	}
+
+	DBGC ( xhci, "XHCI %s-%d has no supported protocol\n",
+	       xhci->name, port );
+	return 0;
+}
+
+/**
+ * Find port protocol
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @ret protocol	USB protocol, or zero if not found
+ */
+static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
+					 unsigned int port ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	union {
+		uint32_t raw;
+		char text[5];
+	} name;
+	unsigned int protocol;
+	unsigned int type;
+	unsigned int psic;
+	unsigned int psiv;
+	unsigned int i;
+	uint32_t revision;
+	uint32_t ports;
+	uint32_t slot;
+	uint32_t psi;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return 0;
+
+	/* Determine protocol version */
+	revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
+	protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
+
+	/* Describe port protocol */
+	if ( DBG_EXTRA ) {
+		name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
+						 XHCI_SUPPORTED_NAME ) );
+		name.text[4] = '\0';
+		slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
+		type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
+		DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d",
+			xhci->name, port, name.text, protocol, type );
+		ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+		psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+		if ( psic ) {
+			DBGC2 ( xhci, " speeds" );
+			for ( i = 0 ; i < psic ; i++ ) {
+				psi = readl ( xhci->cap + supported +
+					      XHCI_SUPPORTED_PSI ( i ) );
+				psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
+				DBGC2 ( xhci, " %d:%s", psiv,
+					xhci_speed_name ( psi ) );
+			}
+		}
+		if ( xhci->quirks & XHCI_BAD_PSIV )
+			DBGC2 ( xhci, " (ignored)" );
+		DBGC2 ( xhci, "\n" );
+	}
+
+	return protocol;
+}
+
+/**
+ * Find port slot type
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @ret type		Slot type, or negative error
+ */
+static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	unsigned int type;
+	uint32_t slot;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return -ENOTSUP;
+
+	/* Get slot type */
+	slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
+	type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
+
+	return type;
+}
+
+/**
+ * Find port speed
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @v psiv		Protocol speed ID value
+ * @ret speed		Port speed, or negative error
+ */
+static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
+			     unsigned int psiv ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	unsigned int psic;
+	unsigned int mantissa;
+	unsigned int exponent;
+	unsigned int speed;
+	unsigned int i;
+	uint32_t ports;
+	uint32_t psi;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return -ENOTSUP;
+
+	/* Get protocol speed ID count */
+	ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+	psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+
+	/* Use the default mappings if applicable */
+	if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
+		switch ( psiv ) {
+		case XHCI_SPEED_LOW :	return USB_SPEED_LOW;
+		case XHCI_SPEED_FULL :	return USB_SPEED_FULL;
+		case XHCI_SPEED_HIGH :	return USB_SPEED_HIGH;
+		case XHCI_SPEED_SUPER :	return USB_SPEED_SUPER;
+		default:
+			DBGC ( xhci, "XHCI %s-%d non-standard PSI value %d\n",
+			       xhci->name, port, psiv );
+			return -ENOTSUP;
+		}
+	}
+
+	/* Iterate over PSI dwords looking for a match */
+	for ( i = 0 ; i < psic ; i++ ) {
+		psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
+		if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
+			mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+			exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+			speed = USB_SPEED ( mantissa, exponent );
+			return speed;
+		}
+	}
+
+	DBGC ( xhci, "XHCI %s-%d spurious PSI value %d\n",
+	       xhci->name, port, psiv );
+	return -ENOENT;
+}
+
+/**
+ * Find protocol speed ID value
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @v speed		USB speed
+ * @ret psiv		Protocol speed ID value, or negative error
+ */
+static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
+			    unsigned int speed ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	unsigned int psic;
+	unsigned int mantissa;
+	unsigned int exponent;
+	unsigned int psiv;
+	unsigned int i;
+	uint32_t ports;
+	uint32_t psi;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return -ENOTSUP;
+
+	/* Get protocol speed ID count */
+	ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+	psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+
+	/* Use the default mappings if applicable */
+	if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
+		switch ( speed ) {
+		case USB_SPEED_LOW :	return XHCI_SPEED_LOW;
+		case USB_SPEED_FULL :	return XHCI_SPEED_FULL;
+		case USB_SPEED_HIGH :	return XHCI_SPEED_HIGH;
+		case USB_SPEED_SUPER :	return XHCI_SPEED_SUPER;
+		default:
+			DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n",
+			       xhci->name, port, speed );
+			return -ENOTSUP;
+		}
+	}
+
+	/* Iterate over PSI dwords looking for a match */
+	for ( i = 0 ; i < psic ; i++ ) {
+		psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
+		mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+		exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+		if ( speed == USB_SPEED ( mantissa, exponent ) ) {
+			psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
+			return psiv;
+		}
+	}
+
+	DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n",
+	       xhci->name, port, speed );
+	return -ENOENT;
+}
+
+/******************************************************************************
+ *
+ * Device context base address array
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate device context base address array
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
+	size_t len;
+	physaddr_t dcbaap;
+	int rc;
+
+	/* Allocate and initialise structure.  Must be at least
+	 * 64-byte aligned and must not cross a page boundary, so
+	 * align on its own size (rounded up to a power of two and
+	 * with a minimum of 64 bytes).
+	 */
+	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
+	xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! xhci->dcbaa ) {
+		DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memset ( xhci->dcbaa, 0, len );
+
+	/* Program DCBAA pointer */
+	dcbaap = virt_to_phys ( xhci->dcbaa );
+	if ( ( rc = xhci_writeq ( xhci, dcbaap,
+				  xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
+		goto err_writeq;
+
+	DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n",
+		xhci->name, dcbaap, ( dcbaap + len ) );
+	return 0;
+
+ err_writeq:
+	free_dma ( xhci->dcbaa, len );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Free device context base address array
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
+	size_t len;
+	unsigned int i;
+
+	/* Sanity check */
+	for ( i = 0 ; i <= xhci->slots ; i++ )
+		assert ( xhci->dcbaa[i] == 0 );
+
+	/* Clear DCBAA pointer */
+	xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
+
+	/* Free DCBAA */
+	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
+	free_dma ( xhci->dcbaa, len );
+}
+
+/******************************************************************************
+ *
+ * Scratchpad buffers
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate scratchpad buffers
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
+	size_t array_len;
+	size_t len;
+	physaddr_t phys;
+	unsigned int i;
+	int rc;
+
+	/* Do nothing if no scratchpad buffers are used */
+	if ( ! xhci->scratchpads )
+		return 0;
+
+	/* Allocate scratchpads */
+	len = ( xhci->scratchpads * xhci->pagesize );
+	xhci->scratchpad = umalloc ( len );
+	if ( ! xhci->scratchpad ) {
+		DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n",
+		       xhci->name );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memset_user ( xhci->scratchpad, 0, 0, len );
+
+	/* Allocate scratchpad array */
+	array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
+	xhci->scratchpad_array =
+		malloc_dma ( array_len, xhci_align ( array_len ) );
+	if ( ! xhci->scratchpad_array ) {
+		DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer "
+		       "array\n", xhci->name );
+		rc = -ENOMEM;
+		goto err_alloc_array;
+	}
+
+	/* Populate scratchpad array */
+	for ( i = 0 ; i < xhci->scratchpads ; i++ ) {
+		phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize ));
+		xhci->scratchpad_array[i] = phys;
+	}
+
+	/* Set scratchpad array pointer */
+	assert ( xhci->dcbaa != NULL );
+	xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array ));
+
+	DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
+		xhci->name, user_to_phys ( xhci->scratchpad, 0 ),
+		user_to_phys ( xhci->scratchpad, len ),
+		virt_to_phys ( xhci->scratchpad_array ),
+		( virt_to_phys ( xhci->scratchpad_array ) + array_len ) );
+	return 0;
+
+	free_dma ( xhci->scratchpad_array, array_len );
+ err_alloc_array:
+	ufree ( xhci->scratchpad );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Free scratchpad buffers
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
+	size_t array_len;
+
+	/* Do nothing if no scratchpad buffers are used */
+	if ( ! xhci->scratchpads )
+		return;
+
+	/* Clear scratchpad array pointer */
+	assert ( xhci->dcbaa != NULL );
+	xhci->dcbaa[0] = 0;
+
+	/* Free scratchpad array */
+	array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
+	free_dma ( xhci->scratchpad_array, array_len );
+
+	/* Free scratchpads */
+	ufree ( xhci->scratchpad );
+}
+
+/******************************************************************************
+ *
+ * Run / stop / reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Start xHCI device
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_run ( struct xhci_device *xhci ) {
+	uint32_t config;
+	uint32_t usbcmd;
+
+	/* Configure number of device slots */
+	config = readl ( xhci->op + XHCI_OP_CONFIG );
+	config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
+	config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
+	writel ( config, xhci->op + XHCI_OP_CONFIG );
+
+	/* Set run/stop bit */
+	usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+	usbcmd |= XHCI_USBCMD_RUN;
+	writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
+}
+
+/**
+ * Stop xHCI device
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_stop ( struct xhci_device *xhci ) {
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	unsigned int i;
+
+	/* Clear run/stop bit */
+	usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+	usbcmd &= ~XHCI_USBCMD_RUN;
+	writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
+
+	/* Wait for device to stop */
+	for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if device is stopped */
+		usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
+		if ( usbsts & XHCI_USBSTS_HCH )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Reset xHCI device
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_reset ( struct xhci_device *xhci ) {
+	uint32_t usbcmd;
+	unsigned int i;
+	int rc;
+
+	/* The xHCI specification states that resetting a running
+	 * device may result in undefined behaviour, so try stopping
+	 * it first.
+	 */
+	if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
+		/* Ignore errors and attempt to reset the device anyway */
+	}
+
+	/* Reset device */
+	writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if reset is complete */
+		usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+		if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name );
+	return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Transfer request blocks
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate transfer request block ring
+ *
+ * @v xhci		xHCI device
+ * @v ring		TRB ring
+ * @v shift		Ring size (log2)
+ * @v slot		Device slot
+ * @v target		Doorbell target
+ * @v stream		Doorbell stream ID
+ * @ret rc		Return status code
+ */
+static int xhci_ring_alloc ( struct xhci_device *xhci,
+			     struct xhci_trb_ring *ring,
+			     unsigned int shift, unsigned int slot,
+			     unsigned int target, unsigned int stream ) {
+	struct xhci_trb_link *link;
+	unsigned int count;
+	int rc;
+
+	/* Sanity check */
+	assert ( shift > 0 );
+
+	/* Initialise structure */
+	memset ( ring, 0, sizeof ( *ring ) );
+	ring->shift = shift;
+	count = ( 1U << shift );
+	ring->mask = ( count - 1 );
+	ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
+	ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
+	ring->dbval = XHCI_DBVAL ( target, stream );
+
+	/* Allocate I/O buffers */
+	ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
+	if ( ! ring->iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc_iobuf;
+	}
+
+	/* Allocate TRBs */
+	ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) );
+	if ( ! ring->trb ) {
+		rc = -ENOMEM;
+		goto err_alloc_trb;
+	}
+	memset ( ring->trb, 0, ring->len );
+
+	/* Initialise Link TRB */
+	link = &ring->trb[count].link;
+	link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) );
+	link->flags = XHCI_TRB_TC;
+	link->type = XHCI_TRB_LINK;
+	ring->link = link;
+
+	return 0;
+
+	free_dma ( ring->trb, ring->len );
+ err_alloc_trb:
+	free ( ring->iobuf );
+ err_alloc_iobuf:
+	return rc;
+}
+
+/**
+ * Reset transfer request block ring
+ *
+ * @v ring		TRB ring
+ */
+static void xhci_ring_reset ( struct xhci_trb_ring *ring ) {
+	unsigned int count = ( 1U << ring->shift );
+
+	/* Reset producer and consumer counters */
+	ring->prod = 0;
+	ring->cons = 0;
+
+	/* Reset TRBs (except Link TRB) */
+	memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) );
+}
+
+/**
+ * Free transfer request block ring
+ *
+ * @v ring		TRB ring
+ */
+static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
+	unsigned int count = ( 1U << ring->shift );
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( ring->cons == ring->prod );
+	for ( i = 0 ; i < count ; i++ )
+		assert ( ring->iobuf[i] == NULL );
+
+	/* Free TRBs */
+	free_dma ( ring->trb, ring->len );
+
+	/* Free I/O buffers */
+	free ( ring->iobuf );
+}
+
+/**
+ * Enqueue a transfer request block
+ *
+ * @v ring		TRB ring
+ * @v iobuf		I/O buffer (if any)
+ * @v trb		Transfer request block (with empty Cycle flag)
+ * @ret rc		Return status code
+ *
+ * This operation does not implicitly ring the doorbell register.
+ */
+static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
+			  const union xhci_trb *trb ) {
+	union xhci_trb *dest;
+	unsigned int prod;
+	unsigned int mask;
+	unsigned int index;
+	unsigned int cycle;
+
+	/* Sanity check */
+	assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
+
+	/* Fail if ring is full */
+	if ( ! xhci_ring_remaining ( ring ) )
+		return -ENOBUFS;
+
+	/* Update producer counter (and link TRB, if applicable) */
+	prod = ring->prod++;
+	mask = ring->mask;
+	cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
+	index = ( prod & mask );
+	if ( index == 0 )
+		ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
+
+	/* Record I/O buffer */
+	ring->iobuf[index] = iobuf;
+
+	/* Enqueue TRB */
+	dest = &ring->trb[index];
+	dest->template.parameter = trb->template.parameter;
+	dest->template.status = trb->template.status;
+	wmb();
+	dest->template.control = ( trb->template.control |
+				   cpu_to_le32 ( cycle ) );
+
+	return 0;
+}
+
+/**
+ * Dequeue a transfer request block
+ *
+ * @v ring		TRB ring
+ * @ret iobuf		I/O buffer
+ */
+static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
+	struct io_buffer *iobuf;
+	unsigned int cons;
+	unsigned int mask;
+	unsigned int index;
+
+	/* Sanity check */
+	assert ( xhci_ring_fill ( ring ) != 0 );
+
+	/* Update consumer counter */
+	cons = ring->cons++;
+	mask = ring->mask;
+	index = ( cons & mask );
+
+	/* Retrieve I/O buffer */
+	iobuf = ring->iobuf[index];
+	ring->iobuf[index] = NULL;
+
+	return iobuf;
+}
+
+/**
+ * Enqueue multiple transfer request blocks
+ *
+ * @v ring		TRB ring
+ * @v iobuf		I/O buffer
+ * @v trbs		Transfer request blocks (with empty Cycle flag)
+ * @v count		Number of transfer request blocks
+ * @ret rc		Return status code
+ *
+ * This operation does not implicitly ring the doorbell register.
+ */
+static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
+				struct io_buffer *iobuf,
+				const union xhci_trb *trbs,
+				unsigned int count ) {
+	const union xhci_trb *trb = trbs;
+	int rc;
+
+	/* Sanity check */
+	assert ( iobuf != NULL );
+
+	/* Fail if ring does not have sufficient space */
+	if ( xhci_ring_remaining ( ring ) < count )
+		return -ENOBUFS;
+
+	/* Enqueue each TRB, recording the I/O buffer with the final TRB */
+	while ( count-- ) {
+		rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
+		assert ( rc == 0 ); /* Should never be able to fail */
+	}
+
+	return 0;
+}
+
+/**
+ * Dequeue multiple transfer request blocks
+ *
+ * @v ring		TRB ring
+ * @ret iobuf		I/O buffer
+ */
+static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
+	struct io_buffer *iobuf;
+
+	/* Dequeue TRBs until we reach the final TRB for an I/O buffer */
+	do {
+		iobuf = xhci_dequeue ( ring );
+	} while ( iobuf == NULL );
+
+	return iobuf;
+}
+
+/**
+ * Ring doorbell register
+ *
+ * @v ring		TRB ring
+ */
+static inline __attribute__ (( always_inline )) void
+xhci_doorbell ( struct xhci_trb_ring *ring ) {
+
+	wmb();
+	writel ( ring->dbval, ring->db );
+}
+
+/******************************************************************************
+ *
+ * Command and event rings
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate command ring
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_command_alloc ( struct xhci_device *xhci ) {
+	physaddr_t crp;
+	int rc;
+
+	/* Allocate TRB ring */
+	if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
+				      0, 0, 0 ) ) != 0 )
+		goto err_ring_alloc;
+
+	/* Program command ring control register */
+	crp = virt_to_phys ( xhci->command.trb );
+	if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
+				  xhci->op + XHCI_OP_CRCR ) ) != 0 )
+		goto err_writeq;
+
+	DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n",
+		xhci->name, crp, ( crp + xhci->command.len ) );
+	return 0;
+
+ err_writeq:
+	xhci_ring_free ( &xhci->command );
+ err_ring_alloc:
+	return rc;
+}
+
+/**
+ * Free command ring
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_command_free ( struct xhci_device *xhci ) {
+
+	/* Sanity check */
+	assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
+
+	/* Clear command ring control register */
+	xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
+
+	/* Free TRB ring */
+	xhci_ring_free ( &xhci->command );
+}
+
+/**
+ * Allocate event ring
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_event_alloc ( struct xhci_device *xhci ) {
+	struct xhci_event_ring *event = &xhci->event;
+	unsigned int count;
+	size_t len;
+	int rc;
+
+	/* Allocate event ring */
+	count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
+	len = ( count * sizeof ( event->trb[0] ) );
+	event->trb = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! event->trb ) {
+		rc = -ENOMEM;
+		goto err_alloc_trb;
+	}
+	memset ( event->trb, 0, len );
+
+	/* Allocate event ring segment table */
+	event->segment = malloc_dma ( sizeof ( event->segment[0] ),
+				      xhci_align ( sizeof (event->segment[0])));
+	if ( ! event->segment ) {
+		rc = -ENOMEM;
+		goto err_alloc_segment;
+	}
+	memset ( event->segment, 0, sizeof ( event->segment[0] ) );
+	event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) );
+	event->segment[0].count = cpu_to_le32 ( count );
+
+	/* Program event ring registers */
+	writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
+	if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ),
+				  xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
+		goto err_writeq_erdp;
+	if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ),
+				  xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
+		goto err_writeq_erstba;
+
+	DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
+		xhci->name, virt_to_phys ( event->trb ),
+		( virt_to_phys ( event->trb ) + len ),
+		virt_to_phys ( event->segment ),
+		( virt_to_phys ( event->segment ) +
+		  sizeof (event->segment[0] ) ) );
+	return 0;
+
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
+ err_writeq_erstba:
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
+ err_writeq_erdp:
+	free_dma ( event->trb, len );
+ err_alloc_segment:
+	free_dma ( event->segment, sizeof ( event->segment[0] ) );
+ err_alloc_trb:
+	return rc;
+}
+
+/**
+ * Free event ring
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_event_free ( struct xhci_device *xhci ) {
+	struct xhci_event_ring *event = &xhci->event;
+	unsigned int count;
+	size_t len;
+
+	/* Clear event ring registers */
+	writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
+
+	/* Free event ring segment table */
+	free_dma ( event->segment, sizeof ( event->segment[0] ) );
+
+	/* Free event ring */
+	count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
+	len = ( count * sizeof ( event->trb[0] ) );
+	free_dma ( event->trb, len );
+}
+
+/**
+ * Handle transfer event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Transfer event TRB
+ */
+static void xhci_transfer ( struct xhci_device *xhci,
+			    struct xhci_trb_transfer *trb ) {
+	struct xhci_slot *slot;
+	struct xhci_endpoint *endpoint;
+	struct io_buffer *iobuf;
+	int rc;
+
+	/* Profile transfer events */
+	profile_start ( &xhci_transfer_profiler );
+
+	/* Identify slot */
+	if ( ( trb->slot > xhci->slots ) ||
+	     ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) {
+		DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n",
+		       xhci->name, trb->slot );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+		return;
+	}
+
+	/* Identify endpoint */
+	if ( ( trb->endpoint > XHCI_CTX_END ) ||
+	     ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) {
+		DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid "
+		       "%d:\n", xhci->name, slot->id, trb->endpoint );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+		return;
+	}
+
+	/* Dequeue TRB(s) */
+	iobuf = xhci_dequeue_multi ( &endpoint->ring );
+	assert ( iobuf != NULL );
+
+	/* Check for errors */
+	if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) ||
+		 ( trb->code == XHCI_CMPLT_SHORT ) ) ) {
+
+		/* Construct error */
+		rc = -ECODE ( trb->code );
+		DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n",
+		       xhci->name, slot->id, endpoint->ctx, trb->code,
+		       strerror ( rc ) );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+
+		/* Sanity check */
+		assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
+			 != XHCI_ENDPOINT_RUNNING );
+
+		/* Report failure to USB core */
+		usb_complete_err ( endpoint->ep, iobuf, rc );
+		return;
+	}
+
+	/* Record actual transfer size */
+	iob_unput ( iobuf, le16_to_cpu ( trb->residual ) );
+
+	/* Sanity check (for successful completions only) */
+	assert ( xhci_ring_consumed ( &endpoint->ring ) ==
+		 le64_to_cpu ( trb->transfer ) );
+
+	/* Report completion to USB core */
+	usb_complete ( endpoint->ep, iobuf );
+	profile_stop ( &xhci_transfer_profiler );
+}
+
+/**
+ * Handle command completion event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Command completion event
+ */
+static void xhci_complete ( struct xhci_device *xhci,
+			    struct xhci_trb_complete *trb ) {
+	int rc;
+
+	/* Ignore "command ring stopped" notifications */
+	if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) {
+		DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name );
+		return;
+	}
+
+	/* Ignore unexpected completions */
+	if ( ! xhci->pending ) {
+		rc = -ECODE ( trb->code );
+		DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n",
+		       xhci->name, trb->code, strerror ( rc ) );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+		return;
+	}
+
+	/* Dequeue command TRB */
+	xhci_dequeue ( &xhci->command );
+
+	/* Sanity check */
+	assert ( xhci_ring_consumed ( &xhci->command ) ==
+		 le64_to_cpu ( trb->command ) );
+
+	/* Record completion */
+	memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) );
+	xhci->pending = NULL;
+}
+
+/**
+ * Handle port status event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Port status event
+ */
+static void xhci_port_status ( struct xhci_device *xhci,
+			       struct xhci_trb_port_status *trb ) {
+	struct usb_port *port = usb_port ( xhci->bus->hub, trb->port );
+	uint32_t portsc;
+
+	/* Sanity check */
+	assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) );
+
+	/* Record disconnections and clear changes */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) );
+	port->disconnected |= ( portsc & XHCI_PORTSC_CSC );
+	portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) );
+
+	/* Report port status change */
+	usb_port_changed ( port );
+}
+
+/**
+ * Handle host controller event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Host controller event
+ */
+static void xhci_host_controller ( struct xhci_device *xhci,
+				   struct xhci_trb_host_controller *trb ) {
+	int rc;
+
+	/* Construct error */
+	rc = -ECODE ( trb->code );
+	DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n",
+	       xhci->name, trb->code, strerror ( rc ) );
+}
+
+/**
+ * Poll event ring
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_event_poll ( struct xhci_device *xhci ) {
+	struct xhci_event_ring *event = &xhci->event;
+	union xhci_trb *trb;
+	unsigned int shift = XHCI_EVENT_TRBS_LOG2;
+	unsigned int count = ( 1 << shift );
+	unsigned int mask = ( count - 1 );
+	unsigned int consumed;
+	unsigned int type;
+
+	/* Poll for events */
+	profile_start ( &xhci_event_profiler );
+	for ( consumed = 0 ; ; consumed++ ) {
+
+		/* Stop if we reach an empty TRB */
+		rmb();
+		trb = &event->trb[ event->cons & mask ];
+		if ( ! ( ( trb->common.flags ^
+			   ( event->cons >> shift ) ) & XHCI_TRB_C ) )
+			break;
+
+		/* Handle TRB */
+		type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
+		switch ( type ) {
+
+		case XHCI_TRB_TRANSFER :
+			xhci_transfer ( xhci, &trb->transfer );
+			break;
+
+		case XHCI_TRB_COMPLETE :
+			xhci_complete ( xhci, &trb->complete );
+			break;
+
+		case XHCI_TRB_PORT_STATUS:
+			xhci_port_status ( xhci, &trb->port );
+			break;
+
+		case XHCI_TRB_HOST_CONTROLLER:
+			xhci_host_controller ( xhci, &trb->host );
+			break;
+
+		default:
+			DBGC ( xhci, "XHCI %s unrecognised event %#x\n:",
+			       xhci->name, event->cons );
+			DBGC_HDA ( xhci, virt_to_phys ( trb ),
+				   trb, sizeof ( *trb ) );
+			break;
+		}
+
+		/* Consume this TRB */
+		event->cons++;
+	}
+
+	/* Update dequeue pointer if applicable */
+	if ( consumed ) {
+		xhci_writeq ( xhci, virt_to_phys ( trb ),
+			      xhci->run + XHCI_RUN_ERDP ( 0 ) );
+		profile_stop ( &xhci_event_profiler );
+	}
+}
+
+/**
+ * Abort command
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_abort ( struct xhci_device *xhci ) {
+	physaddr_t crp;
+
+	/* Abort the command */
+	DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name );
+	xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR );
+
+	/* Allow time for command to abort */
+	mdelay ( XHCI_COMMAND_ABORT_DELAY_MS );
+
+	/* Sanity check */
+	assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
+
+	/* Consume (and ignore) any final command status */
+	xhci_event_poll ( xhci );
+
+	/* Reset the command ring control register */
+	xhci_ring_reset ( &xhci->command );
+	crp = virt_to_phys ( xhci->command.trb );
+	xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR );
+}
+
+/**
+ * Issue command and wait for completion
+ *
+ * @v xhci		xHCI device
+ * @v trb		Transfer request block (with empty Cycle flag)
+ * @ret rc		Return status code
+ *
+ * On a successful completion, the TRB will be overwritten with the
+ * completion.
+ */
+static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
+	struct xhci_trb_complete *complete = &trb->complete;
+	unsigned int i;
+	int rc;
+
+	/* Record the pending command */
+	xhci->pending = trb;
+
+	/* Enqueue the command */
+	if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
+		goto err_enqueue;
+
+	/* Ring the command doorbell */
+	xhci_doorbell ( &xhci->command );
+
+	/* Wait for the command to complete */
+	for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
+
+		/* Poll event ring */
+		xhci_event_poll ( xhci );
+
+		/* Check for completion */
+		if ( ! xhci->pending ) {
+			if ( complete->code != XHCI_CMPLT_SUCCESS ) {
+				rc = -ECODE ( complete->code );
+				DBGC ( xhci, "XHCI %s command failed (code "
+				       "%d): %s\n", xhci->name, complete->code,
+				       strerror ( rc ) );
+				DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+				return rc;
+			}
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* Timeout */
+	DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name );
+	rc = -ETIMEDOUT;
+
+	/* Abort command */
+	xhci_abort ( xhci );
+
+ err_enqueue:
+	xhci->pending = NULL;
+	return rc;
+}
+
+/**
+ * Issue NOP and wait for completion
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static inline int xhci_nop ( struct xhci_device *xhci ) {
+	union xhci_trb trb;
+	struct xhci_trb_common *nop = &trb.common;
+	int rc;
+
+	/* Construct command */
+	memset ( nop, 0, sizeof ( *nop ) );
+	nop->flags = XHCI_TRB_IOC;
+	nop->type = XHCI_TRB_NOP_CMD;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Enable slot
+ *
+ * @v xhci		xHCI device
+ * @v type		Slot type
+ * @ret slot		Device slot ID, or negative error
+ */
+static inline int xhci_enable_slot ( struct xhci_device *xhci,
+				     unsigned int type ) {
+	union xhci_trb trb;
+	struct xhci_trb_enable_slot *enable = &trb.enable;
+	struct xhci_trb_complete *enabled = &trb.complete;
+	unsigned int slot;
+	int rc;
+
+	/* Construct command */
+	memset ( enable, 0, sizeof ( *enable ) );
+	enable->slot = type;
+	enable->type = XHCI_TRB_ENABLE_SLOT;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s could not enable new slot: %s\n",
+		       xhci->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Extract slot number */
+	slot = enabled->slot;
+
+	DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot );
+	return slot;
+}
+
+/**
+ * Disable slot
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @ret rc		Return status code
+ */
+static inline int xhci_disable_slot ( struct xhci_device *xhci,
+				      unsigned int slot ) {
+	union xhci_trb trb;
+	struct xhci_trb_disable_slot *disable = &trb.disable;
+	int rc;
+
+	/* Construct command */
+	memset ( disable, 0, sizeof ( *disable ) );
+	disable->type = XHCI_TRB_DISABLE_SLOT;
+	disable->slot = slot;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n",
+		       xhci->name, slot, strerror ( rc ) );
+		return rc;
+	}
+
+	DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot );
+	return 0;
+}
+
+/**
+ * Issue context-based command and wait for completion
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v type		TRB type
+ * @v populate		Input context populater
+ * @ret rc		Return status code
+ */
+static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
+			  struct xhci_endpoint *endpoint, unsigned int type,
+			  void ( * populate ) ( struct xhci_device *xhci,
+						struct xhci_slot *slot,
+						struct xhci_endpoint *endpoint,
+						void *input ) ) {
+	union xhci_trb trb;
+	struct xhci_trb_context *context = &trb.context;
+	size_t len;
+	void *input;
+	int rc;
+
+	/* Allocate an input context */
+	len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
+	input = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! input ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memset ( input, 0, len );
+
+	/* Populate input context */
+	populate ( xhci, slot, endpoint, input );
+
+	/* Construct command */
+	memset ( context, 0, sizeof ( *context ) );
+	context->type = type;
+	context->input = cpu_to_le64 ( virt_to_phys ( input ) );
+	context->slot = slot->id;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
+		goto err_command;
+
+ err_command:
+	free_dma ( input, len );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Populate address device input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void xhci_address_device_input ( struct xhci_device *xhci,
+					struct xhci_slot *slot,
+					struct xhci_endpoint *endpoint,
+					void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+	struct xhci_endpoint_context *ep_ctx;
+
+	/* Sanity checks */
+	assert ( endpoint->ctx == XHCI_CTX_EP0 );
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+					 ( 1 << XHCI_CTX_EP0 ) );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
+							slot->route ) );
+	slot_ctx->port = slot->port;
+	slot_ctx->tt_id = slot->tt_id;
+	slot_ctx->tt_port = slot->tt_port;
+
+	/* Populate control endpoint context */
+	ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
+	ep_ctx->type = XHCI_EP_TYPE_CONTROL;
+	ep_ctx->burst = endpoint->ep->burst;
+	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+	ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+					XHCI_EP_DCS );
+	ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
+}
+
+/**
+ * Address device
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @ret rc		Return status code
+ */
+static inline int xhci_address_device ( struct xhci_device *xhci,
+					struct xhci_slot *slot ) {
+	struct usb_device *usb = slot->usb;
+	struct xhci_slot_context *slot_ctx;
+	int rc;
+
+	/* Assign device address */
+	if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
+				   XHCI_TRB_ADDRESS_DEVICE,
+				   xhci_address_device_input ) ) != 0 )
+		return rc;
+
+	/* Get assigned address */
+	slot_ctx = ( slot->context +
+		     xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
+	usb->address = slot_ctx->address;
+	DBGC2 ( xhci, "XHCI %s assigned address %d to %s\n",
+		xhci->name, usb->address, usb->name );
+
+	return 0;
+}
+
+/**
+ * Populate configure endpoint input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
+					    struct xhci_slot *slot,
+					    struct xhci_endpoint *endpoint,
+					    void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+	struct xhci_endpoint_context *ep_ctx;
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+					 ( 1 << endpoint->ctx ) );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+							( slot->ports ? 1 : 0 ),
+							slot->psiv, 0 ) );
+	slot_ctx->ports = slot->ports;
+
+	/* Populate endpoint context */
+	ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
+	ep_ctx->interval = endpoint->interval;
+	ep_ctx->type = endpoint->type;
+	ep_ctx->burst = endpoint->ep->burst;
+	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+	ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+					XHCI_EP_DCS );
+	ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
+}
+
+/**
+ * Configure endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
+					    struct xhci_slot *slot,
+					    struct xhci_endpoint *endpoint ) {
+	int rc;
+
+	/* Configure endpoint */
+	if ( ( rc = xhci_context ( xhci, slot, endpoint,
+				   XHCI_TRB_CONFIGURE_ENDPOINT,
+				   xhci_configure_endpoint_input ) ) != 0 )
+		return rc;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n",
+		xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Populate deconfigure endpoint input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void
+xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
+				  struct xhci_slot *slot __unused,
+				  struct xhci_endpoint *endpoint,
+				  void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
+	control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+							0, 0, 0 ) );
+}
+
+/**
+ * Deconfigure endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
+					      struct xhci_slot *slot,
+					      struct xhci_endpoint *endpoint ) {
+	int rc;
+
+	/* Deconfigure endpoint */
+	if ( ( rc = xhci_context ( xhci, slot, endpoint,
+				   XHCI_TRB_CONFIGURE_ENDPOINT,
+				   xhci_deconfigure_endpoint_input ) ) != 0 )
+		return rc;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n",
+		xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Populate evaluate context input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void xhci_evaluate_context_input ( struct xhci_device *xhci,
+					  struct xhci_slot *slot __unused,
+					  struct xhci_endpoint *endpoint,
+					  void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+	struct xhci_endpoint_context *ep_ctx;
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+					 ( 1 << endpoint->ctx ) );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+							0, 0, 0 ) );
+
+	/* Populate endpoint context */
+	ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
+	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+}
+
+/**
+ * Evaluate context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_evaluate_context ( struct xhci_device *xhci,
+					  struct xhci_slot *slot,
+					  struct xhci_endpoint *endpoint ) {
+	int rc;
+
+	/* Configure endpoint */
+	if ( ( rc = xhci_context ( xhci, slot, endpoint,
+				   XHCI_TRB_EVALUATE_CONTEXT,
+				   xhci_evaluate_context_input ) ) != 0 )
+		return rc;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n",
+		xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
+					struct xhci_slot *slot,
+					struct xhci_endpoint *endpoint ) {
+	union xhci_trb trb;
+	struct xhci_trb_reset_endpoint *reset = &trb.reset;
+	int rc;
+
+	/* Construct command */
+	memset ( reset, 0, sizeof ( *reset ) );
+	reset->slot = slot->id;
+	reset->endpoint = endpoint->ctx;
+	reset->type = XHCI_TRB_RESET_ENDPOINT;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint "
+		       "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
+		       endpoint->context->state, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Stop endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
+				       struct xhci_slot *slot,
+				       struct xhci_endpoint *endpoint ) {
+	union xhci_trb trb;
+	struct xhci_trb_stop_endpoint *stop = &trb.stop;
+	int rc;
+
+	/* Construct command */
+	memset ( stop, 0, sizeof ( *stop ) );
+	stop->slot = slot->id;
+	stop->endpoint = endpoint->ctx;
+	stop->type = XHCI_TRB_STOP_ENDPOINT;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint "
+		       "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
+		       endpoint->context->state, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Set transfer ring dequeue pointer
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int
+xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
+			      struct xhci_slot *slot,
+			      struct xhci_endpoint *endpoint ) {
+	union xhci_trb trb;
+	struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
+	struct xhci_trb_ring *ring = &endpoint->ring;
+	unsigned int cons;
+	unsigned int mask;
+	unsigned int index;
+	unsigned int dcs;
+	int rc;
+
+	/* Construct command */
+	memset ( dequeue, 0, sizeof ( *dequeue ) );
+	cons = ring->cons;
+	mask = ring->mask;
+	dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
+	index = ( cons & mask );
+	dequeue->dequeue =
+		cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs );
+	dequeue->slot = slot->id;
+	dequeue->endpoint = endpoint->ctx;
+	dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue "
+		       "pointer in state %d: %s\n", xhci->name, slot->id,
+		       endpoint->ctx, endpoint->context->state, strerror ( rc));
+		return rc;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct xhci_slot *slot = usb_get_hostdata ( usb );
+	struct xhci_device *xhci = slot->xhci;
+	struct xhci_endpoint *endpoint;
+	unsigned int ctx;
+	unsigned int type;
+	unsigned int interval;
+	int rc;
+
+	/* Calculate context index */
+	ctx = XHCI_CTX ( ep->address );
+	assert ( slot->endpoint[ctx] == NULL );
+
+	/* Calculate endpoint type */
+	type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
+		type = XHCI_EP_TYPE_CONTROL;
+	if ( ep->address & USB_DIR_IN )
+		type |= XHCI_EP_TYPE_IN;
+
+	/* Calculate interval */
+	if ( type & XHCI_EP_TYPE_PERIODIC ) {
+		interval = ( fls ( ep->interval ) - 1 );
+	} else {
+		interval = ep->interval;
+	}
+
+	/* Allocate and initialise structure */
+	endpoint = zalloc ( sizeof ( *endpoint ) );
+	if ( ! endpoint ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	usb_endpoint_set_hostdata ( ep, endpoint );
+	slot->endpoint[ctx] = endpoint;
+	endpoint->xhci = xhci;
+	endpoint->slot = slot;
+	endpoint->ep = ep;
+	endpoint->ctx = ctx;
+	endpoint->type = type;
+	endpoint->interval = interval;
+	endpoint->context = ( ( ( void * ) slot->context ) +
+			      xhci_device_context_offset ( xhci, ctx ) );
+
+	/* Allocate transfer ring */
+	if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
+				      XHCI_TRANSFER_TRBS_LOG2,
+				      slot->id, ctx, 0 ) ) != 0 )
+		goto err_ring_alloc;
+
+	/* Configure endpoint, if applicable */
+	if ( ( ctx != XHCI_CTX_EP0 ) &&
+	     ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
+		goto err_configure_endpoint;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n",
+		xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
+		( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
+	return 0;
+
+	xhci_deconfigure_endpoint ( xhci, slot, endpoint );
+ err_configure_endpoint:
+	xhci_ring_free ( &endpoint->ring );
+ err_ring_alloc:
+	slot->endpoint[ctx] = NULL;
+	free ( endpoint );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Close endpoint
+ *
+ * @v ep		USB endpoint
+ */
+static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct xhci_slot *slot = endpoint->slot;
+	struct xhci_device *xhci = slot->xhci;
+	struct io_buffer *iobuf;
+	unsigned int ctx = endpoint->ctx;
+
+	/* Deconfigure endpoint, if applicable */
+	if ( ctx != XHCI_CTX_EP0 )
+		xhci_deconfigure_endpoint ( xhci, slot, endpoint );
+
+	/* Cancel any incomplete transfers */
+	while ( xhci_ring_fill ( &endpoint->ring ) ) {
+		iobuf = xhci_dequeue_multi ( &endpoint->ring );
+		usb_complete_err ( ep, iobuf, -ECANCELED );
+	}
+
+	/* Free endpoint */
+	xhci_ring_free ( &endpoint->ring );
+	slot->endpoint[ctx] = NULL;
+	free ( endpoint );
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct xhci_slot *slot = endpoint->slot;
+	struct xhci_device *xhci = slot->xhci;
+	int rc;
+
+	/* Reset endpoint context */
+	if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
+		return rc;
+
+	/* Set transfer ring dequeue pointer */
+	if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
+		return rc;
+
+	/* Ring doorbell to resume processing */
+	xhci_doorbell ( &endpoint->ring );
+
+	DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n",
+	       xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Update MTU
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct xhci_slot *slot = endpoint->slot;
+	struct xhci_device *xhci = slot->xhci;
+	int rc;
+
+	/* Evalulate context */
+	if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Enqueue message transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_message ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct usb_setup_packet *packet;
+	unsigned int input;
+	size_t len;
+	union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
+			     1 /* status */ ];
+	union xhci_trb *trb = trbs;
+	struct xhci_trb_setup *setup;
+	struct xhci_trb_data *data;
+	struct xhci_trb_status *status;
+	int rc;
+
+	/* Profile message transfers */
+	profile_start ( &xhci_message_profiler );
+
+	/* Construct setup stage TRB */
+	memset ( trbs, 0, sizeof ( trbs ) );
+	assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
+	packet = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *packet ) );
+	setup = &(trb++)->setup;
+	memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
+	setup->len = cpu_to_le32 ( sizeof ( *packet ) );
+	setup->flags = XHCI_TRB_IDT;
+	setup->type = XHCI_TRB_SETUP;
+	len = iob_len ( iobuf );
+	input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
+	if ( len )
+		setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
+
+	/* Construct data stage TRB, if applicable */
+	if ( len ) {
+		data = &(trb++)->data;
+		data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
+		data->len = cpu_to_le32 ( len );
+		data->type = XHCI_TRB_DATA;
+		data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
+	}
+
+	/* Construct status stage TRB */
+	status = &(trb++)->status;
+	status->flags = XHCI_TRB_IOC;
+	status->type = XHCI_TRB_STATUS;
+	status->direction =
+		( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
+
+	/* Enqueue TRBs */
+	if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
+					 ( trb - trbs ) ) ) != 0 )
+		return rc;
+
+	/* Ring the doorbell */
+	xhci_doorbell ( &endpoint->ring );
+
+	profile_stop ( &xhci_message_profiler );
+	return 0;
+}
+
+/**
+ * Enqueue stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_stream ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int terminate ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	union xhci_trb trbs[ 1 /* Normal */ + 1 /* Possible zero-length */ ];
+	union xhci_trb *trb = trbs;
+	struct xhci_trb_normal *normal;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Profile stream transfers */
+	profile_start ( &xhci_stream_profiler );
+
+	/* Construct normal TRBs */
+	memset ( &trbs, 0, sizeof ( trbs ) );
+	normal = &(trb++)->normal;
+	normal->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
+	normal->len = cpu_to_le32 ( len );
+	normal->type = XHCI_TRB_NORMAL;
+	if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) {
+		normal->flags = XHCI_TRB_CH;
+		normal = &(trb++)->normal;
+		normal->type = XHCI_TRB_NORMAL;
+	}
+	normal->flags = XHCI_TRB_IOC;
+
+	/* Enqueue TRBs */
+	if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
+					 ( trb - trbs ) ) ) != 0 )
+		return rc;
+
+	/* Ring the doorbell */
+	xhci_doorbell ( &endpoint->ring );
+
+	profile_stop ( &xhci_stream_profiler );
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Device operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int xhci_device_open ( struct usb_device *usb ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
+	struct usb_port *tt = usb_transaction_translator ( usb );
+	struct xhci_slot *slot;
+	struct xhci_slot *tt_slot;
+	size_t len;
+	int type;
+	int id;
+	int rc;
+
+	/* Determine applicable slot type */
+	type = xhci_port_slot_type ( xhci, usb->port->address );
+	if ( type < 0 ) {
+		rc = type;
+		DBGC ( xhci, "XHCI %s-%d has no slot type\n",
+		       xhci->name, usb->port->address );
+		goto err_type;
+	}
+
+	/* Allocate a device slot number */
+	id = xhci_enable_slot ( xhci, type );
+	if ( id < 0 ) {
+		rc = id;
+		goto err_enable_slot;
+	}
+	assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) );
+	assert ( xhci->slot[id] == NULL );
+
+	/* Allocate and initialise structure */
+	slot = zalloc ( sizeof ( *slot ) );
+	if ( ! slot ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	usb_set_hostdata ( usb, slot );
+	xhci->slot[id] = slot;
+	slot->xhci = xhci;
+	slot->usb = usb;
+	slot->id = id;
+	if ( tt ) {
+		tt_slot = usb_get_hostdata ( tt->hub->usb );
+		slot->tt_id = tt_slot->id;
+		slot->tt_port = tt->address;
+	}
+
+	/* Allocate a device context */
+	len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
+	slot->context = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! slot->context ) {
+		rc = -ENOMEM;
+		goto err_alloc_context;
+	}
+	memset ( slot->context, 0, len );
+
+	/* Set device context base address */
+	assert ( xhci->dcbaa[id] == 0 );
+	xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) );
+
+	DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n",
+		xhci->name, slot->id, virt_to_phys ( slot->context ),
+		( virt_to_phys ( slot->context ) + len ), usb->name );
+	return 0;
+
+	xhci->dcbaa[id] = 0;
+	free_dma ( slot->context, len );
+ err_alloc_context:
+	xhci->slot[id] = NULL;
+	free ( slot );
+ err_alloc:
+	xhci_disable_slot ( xhci, id );
+ err_enable_slot:
+ err_type:
+	return rc;
+}
+
+/**
+ * Close device
+ *
+ * @v usb		USB device
+ */
+static void xhci_device_close ( struct usb_device *usb ) {
+	struct xhci_slot *slot = usb_get_hostdata ( usb );
+	struct xhci_device *xhci = slot->xhci;
+	size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
+	unsigned int id = slot->id;
+	int rc;
+
+	/* Disable slot */
+	if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) {
+		/* Slot is still enabled.  Leak the slot context,
+		 * since the controller may still write to this
+		 * memory, and leave the DCBAA entry intact.
+		 *
+		 * If the controller later reports that this same slot
+		 * has been re-enabled, then some assertions will be
+		 * triggered.
+		 */
+		DBGC ( xhci, "XHCI %s slot %d leaking context memory\n",
+		       xhci->name, slot->id );
+		slot->context = NULL;
+	}
+
+	/* Free slot */
+	if ( slot->context ) {
+		free_dma ( slot->context, len );
+		xhci->dcbaa[id] = 0;
+	}
+	xhci->slot[id] = NULL;
+	free ( slot );
+}
+
+/**
+ * Assign device address
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int xhci_device_address ( struct usb_device *usb ) {
+	struct xhci_slot *slot = usb_get_hostdata ( usb );
+	struct xhci_device *xhci = slot->xhci;
+	struct usb_port *port = usb->port;
+	struct usb_port *root_port;
+	int psiv;
+	int rc;
+
+	/* Calculate route string */
+	slot->route = usb_route_string ( usb );
+
+	/* Calculate root hub port number */
+	root_port = usb_root_hub_port ( usb );
+	slot->port = root_port->address;
+
+	/* Calculate protocol speed ID */
+	psiv = xhci_port_psiv ( xhci, slot->port, port->speed );
+	if ( psiv < 0 ) {
+		rc = psiv;
+		return rc;
+	}
+	slot->psiv = psiv;
+
+	/* Address device */
+	if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Bus operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+static int xhci_bus_open ( struct usb_bus *bus ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+	int rc;
+
+	/* Allocate device slot array */
+	xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) );
+	if ( ! xhci->slot ) {
+		rc = -ENOMEM;
+		goto err_slot_alloc;
+	}
+
+	/* Allocate device context base address array */
+	if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
+		goto err_dcbaa_alloc;
+
+	/* Allocate scratchpad buffers */
+	if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
+		goto err_scratchpad_alloc;
+
+	/* Allocate command ring */
+	if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
+		goto err_command_alloc;
+
+	/* Allocate event ring */
+	if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
+		goto err_event_alloc;
+
+	/* Start controller */
+	xhci_run ( xhci );
+
+	return 0;
+
+	xhci_stop ( xhci );
+	xhci_event_free ( xhci );
+ err_event_alloc:
+	xhci_command_free ( xhci );
+ err_command_alloc:
+	xhci_scratchpad_free ( xhci );
+ err_scratchpad_alloc:
+	xhci_dcbaa_free ( xhci );
+ err_dcbaa_alloc:
+	free ( xhci->slot );
+ err_slot_alloc:
+	return rc;
+}
+
+/**
+ * Close USB bus
+ *
+ * @v bus		USB bus
+ */
+static void xhci_bus_close ( struct usb_bus *bus ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( xhci->slot != NULL );
+	for ( i = 0 ; i <= xhci->slots ; i++ )
+		assert ( xhci->slot[i] == NULL );
+
+	xhci_stop ( xhci );
+	xhci_event_free ( xhci );
+	xhci_command_free ( xhci );
+	xhci_scratchpad_free ( xhci );
+	xhci_dcbaa_free ( xhci );
+	free ( xhci->slot );
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static void xhci_bus_poll ( struct usb_bus *bus ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+
+	/* Poll event ring */
+	xhci_event_poll ( xhci );
+}
+
+/******************************************************************************
+ *
+ * Hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int xhci_hub_open ( struct usb_hub *hub ) {
+	struct xhci_slot *slot;
+
+	/* Do nothing if this is the root hub */
+	if ( ! hub->usb )
+		return 0;
+
+	/* Get device slot */
+	slot = usb_get_hostdata ( hub->usb );
+
+	/* Update device slot hub parameters.  We don't inform the
+	 * hardware of this information until the hub's interrupt
+	 * endpoint is opened, since the only mechanism for so doing
+	 * provided by the xHCI specification is a Configure Endpoint
+	 * command, and we can't issue that command until we have a
+	 * non-EP0 endpoint to configure.
+	 */
+	slot->ports = hub->ports;
+
+	return 0;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void xhci_hub_close ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+}
+
+/******************************************************************************
+ *
+ * Root hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open root hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int xhci_root_open ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+	struct usb_port *port;
+	uint32_t portsc;
+	unsigned int i;
+
+	/* Enable power to all ports */
+	for ( i = 1 ; i <= xhci->ports ; i++ ) {
+		portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
+		portsc &= XHCI_PORTSC_PRESERVE;
+		portsc |= XHCI_PORTSC_PP;
+		writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
+	}
+
+	/* xHCI spec requires us to potentially wait 20ms after
+	 * enabling power to a port.
+	 */
+	mdelay ( XHCI_PORT_POWER_DELAY_MS );
+
+	/* USB3 ports may power up as Disabled */
+	for ( i = 1 ; i <= xhci->ports ; i++ ) {
+		portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
+		port = usb_port ( hub, i );
+		if ( ( port->protocol >= USB_PROTO_3_0 ) &&
+		     ( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
+		       XHCI_PORTSC_PLS_DISABLED ) ) {
+			/* Force link state to RxDetect */
+			portsc &= XHCI_PORTSC_PRESERVE;
+			portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
+			writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
+		}
+	}
+
+	/* Some xHCI cards seem to require an additional delay after
+	 * setting the link state to RxDetect.
+	 */
+	mdelay ( XHCI_LINK_STATE_DELAY_MS );
+
+	/* Record hub driver private data */
+	usb_hub_set_drvdata ( hub, xhci );
+
+	return 0;
+}
+
+/**
+ * Close root hub
+ *
+ * @v hub		USB hub
+ */
+static void xhci_root_close ( struct usb_hub *hub ) {
+
+	/* Clear hub driver private data */
+	usb_hub_set_drvdata ( hub, NULL );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int i;
+
+	/* Reset port */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+	portsc &= XHCI_PORTSC_PRESERVE;
+	portsc |= XHCI_PORTSC_PR;
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+
+	/* Wait for port to become enabled */
+	for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check port status */
+		portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+		if ( portsc & XHCI_PORTSC_PED )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n",
+	       xhci->name, port->address );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+
+	/* Disable port */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+	portsc &= XHCI_PORTSC_PRESERVE;
+	portsc |= XHCI_PORTSC_PED;
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Update root hub port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int psiv;
+	int ccs;
+	int ped;
+	int csc;
+	int speed;
+	int rc;
+
+	/* Read port status */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+	DBGC2 ( xhci, "XHCI %s-%d status is %08x\n",
+		xhci->name, port->address, portsc );
+	ccs = ( portsc & XHCI_PORTSC_CCS );
+	ped = ( portsc & XHCI_PORTSC_PED );
+	csc = ( portsc & XHCI_PORTSC_CSC );
+	psiv = XHCI_PORTSC_PSIV ( portsc );
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= csc;
+	portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+
+	/* Port speed is not valid unless port is connected */
+	if ( ! ccs ) {
+		port->speed = USB_SPEED_NONE;
+		return 0;
+	}
+
+	/* For USB2 ports, the PSIV field is not valid until the port
+	 * completes reset and becomes enabled.
+	 */
+	if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
+		port->speed = USB_SPEED_FULL;
+		return 0;
+	}
+
+	/* Get port speed and map to generic USB speed */
+	speed = xhci_port_speed ( xhci, port->address, psiv );
+	if ( speed < 0 ) {
+		rc = speed;
+		return rc;
+	}
+
+	port->speed = speed;
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+				struct usb_endpoint *ep ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+
+	/* Should never be called; this is a root hub */
+	DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name,
+	       port->address, ep->usb->name, usb_endpoint_name ( ep ) );
+
+	return -ENOTSUP;
+}
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** USB host controller operations */
+static struct usb_host_operations xhci_operations = {
+	.endpoint = {
+		.open = xhci_endpoint_open,
+		.close = xhci_endpoint_close,
+		.reset = xhci_endpoint_reset,
+		.mtu = xhci_endpoint_mtu,
+		.message = xhci_endpoint_message,
+		.stream = xhci_endpoint_stream,
+	},
+	.device = {
+		.open = xhci_device_open,
+		.close = xhci_device_close,
+		.address = xhci_device_address,
+	},
+	.bus = {
+		.open = xhci_bus_open,
+		.close = xhci_bus_close,
+		.poll = xhci_bus_poll,
+	},
+	.hub = {
+		.open = xhci_hub_open,
+		.close = xhci_hub_close,
+	},
+	.root = {
+		.open = xhci_root_open,
+		.close = xhci_root_close,
+		.enable = xhci_root_enable,
+		.disable = xhci_root_disable,
+		.speed = xhci_root_speed,
+		.clear_tt = xhci_root_clear_tt,
+	},
+};
+
+/**
+ * Fix Intel PCH-specific quirks
+ *
+ * @v xhci		xHCI device
+ * @v pci		PCI device
+ */
+static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) {
+	struct xhci_pch *pch = &xhci->pch;
+	uint32_t xusb2pr;
+	uint32_t xusb2prm;
+	uint32_t usb3pssen;
+	uint32_t usb3prm;
+
+	/* Enable SuperSpeed capability.  Do this before rerouting
+	 * USB2 ports, so that USB3 devices connect at SuperSpeed.
+	 */
+	pci_read_config_dword ( pci, XHCI_PCH_USB3PSSEN, &usb3pssen );
+	pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm );
+	if ( usb3prm & ~usb3pssen ) {
+		DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n",
+		       xhci->name, ( usb3prm & ~usb3pssen ) );
+	}
+	pch->usb3pssen = usb3pssen;
+	usb3pssen |= usb3prm;
+	pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, usb3pssen );
+
+	/* Route USB2 ports from EHCI to xHCI */
+	pci_read_config_dword ( pci, XHCI_PCH_XUSB2PR, &xusb2pr );
+	pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm );
+	if ( xusb2prm & ~xusb2pr ) {
+		DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n",
+		       xhci->name, ( xusb2prm & ~xusb2pr ) );
+	}
+	pch->xusb2pr = xusb2pr;
+	xusb2pr |= xusb2prm;
+	pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, xusb2pr );
+}
+
+/**
+ * Undo Intel PCH-specific quirk fixes
+ *
+ * @v xhci		xHCI device
+ * @v pci		PCI device
+ */
+static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) {
+	struct xhci_pch *pch = &xhci->pch;
+
+	/* Restore USB2 port routing to original state */
+	pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, pch->xusb2pr );
+
+	/* Restore SuperSpeed capability to original state */
+	pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, pch->usb3pssen );
+}
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int xhci_probe ( struct pci_device *pci ) {
+	struct xhci_device *xhci;
+	struct usb_port *port;
+	unsigned long bar_start;
+	size_t bar_size;
+	unsigned int i;
+	int rc;
+
+	/* Allocate and initialise structure */
+	xhci = zalloc ( sizeof ( *xhci ) );
+	if ( ! xhci ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	xhci->name = pci->dev.name;
+	xhci->quirks = pci->id->driver_data;
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Map registers */
+	bar_start = pci_bar_start ( pci, XHCI_BAR );
+	bar_size = pci_bar_size ( pci, XHCI_BAR );
+	xhci->regs = ioremap ( bar_start, bar_size );
+	if ( ! xhci->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Initialise xHCI device */
+	xhci_init ( xhci, xhci->regs );
+
+	/* Initialise USB legacy support and claim ownership */
+	xhci_legacy_init ( xhci );
+	xhci_legacy_claim ( xhci );
+
+	/* Fix Intel PCH-specific quirks, if applicable */
+	if ( xhci->quirks & XHCI_PCH )
+		xhci_pch_fix ( xhci, pci );
+
+	/* Reset device */
+	if ( ( rc = xhci_reset ( xhci ) ) != 0 )
+		goto err_reset;
+
+	/* Allocate USB bus */
+	xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports, XHCI_MTU,
+				    &xhci_operations );
+	if ( ! xhci->bus ) {
+		rc = -ENOMEM;
+		goto err_alloc_bus;
+	}
+	usb_bus_set_hostdata ( xhci->bus, xhci );
+	usb_hub_set_drvdata ( xhci->bus->hub, xhci );
+
+	/* Set port protocols */
+	for ( i = 1 ; i <= xhci->ports ; i++ ) {
+		port = usb_port ( xhci->bus->hub, i );
+		port->protocol = xhci_port_protocol ( xhci, i );
+	}
+
+	/* Register USB bus */
+	if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
+		goto err_register;
+
+	pci_set_drvdata ( pci, xhci );
+	return 0;
+
+	unregister_usb_bus ( xhci->bus );
+ err_register:
+	free_usb_bus ( xhci->bus );
+ err_alloc_bus:
+	xhci_reset ( xhci );
+ err_reset:
+	if ( xhci->quirks & XHCI_PCH )
+		xhci_pch_undo ( xhci, pci );
+	xhci_legacy_release ( xhci );
+	iounmap ( xhci->regs );
+ err_ioremap:
+	free ( xhci );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void xhci_remove ( struct pci_device *pci ) {
+	struct xhci_device *xhci = pci_get_drvdata ( pci );
+	struct usb_bus *bus = xhci->bus;
+
+	unregister_usb_bus ( bus );
+	free_usb_bus ( bus );
+	xhci_reset ( xhci );
+	if ( xhci->quirks & XHCI_PCH )
+		xhci_pch_undo ( xhci, pci );
+	xhci_legacy_release ( xhci );
+	iounmap ( xhci->regs );
+	free ( xhci );
+}
+
+/** XHCI PCI device IDs */
+static struct pci_device_id xhci_ids[] = {
+	PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ),
+	PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ),
+	PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
+};
+
+/** XHCI PCI driver */
+struct pci_driver xhci_driver __pci_driver = {
+	.ids = xhci_ids,
+	.id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
+				PCI_CLASS_SERIAL_USB_XHCI ),
+	.probe = xhci_probe,
+	.remove = xhci_remove,
+};
+
+/**
+ * Prepare for exit
+ *
+ * @v booting		System is shutting down for OS boot
+ */
+static void xhci_shutdown ( int booting ) {
+	/* If we are shutting down to boot an OS, then prevent the
+	 * release of ownership back to BIOS.
+	 */
+	xhci_legacy_prevent_release = booting;
+}
+
+/** Startup/shutdown function */
+struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = {
+	.shutdown = xhci_shutdown,
+};
diff --git a/roms/ipxe/src/drivers/usb/xhci.h b/roms/ipxe/src/drivers/usb/xhci.h
new file mode 100644
index 000000000..83bf71e7e
--- /dev/null
+++ b/roms/ipxe/src/drivers/usb/xhci.h
@@ -0,0 +1,1150 @@
+#ifndef _IPXE_XHCI_H
+#define _IPXE_XHCI_H
+
+/** @file
+ *
+ * USB eXtensible Host Controller Interface (xHCI) driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/pci.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/usb.h>
+
+/** Minimum alignment required for data structures
+ *
+ * With the exception of the scratchpad buffer pages (which are
+ * page-aligned), data structures used by xHCI generally require from
+ * 16 to 64 byte alignment and must not cross an (xHCI) page boundary.
+ * We simplify this requirement by aligning each structure on its own
+ * size, with a minimum of a 64 byte alignment.
+ */
+#define XHCI_MIN_ALIGN 64
+
+/** Maximum transfer size */
+#define XHCI_MTU 65536
+
+/** xHCI PCI BAR */
+#define XHCI_BAR PCI_BASE_ADDRESS_0
+
+/** Capability register length */
+#define XHCI_CAP_CAPLENGTH 0x00
+
+/** Host controller interface version number */
+#define XHCI_CAP_HCIVERSION 0x02
+
+/** Structural parameters 1 */
+#define XHCI_CAP_HCSPARAMS1 0x04
+
+/** Number of device slots */
+#define XHCI_HCSPARAMS1_SLOTS(params) ( ( (params) >> 0 ) & 0xff )
+
+/** Number of interrupters */
+#define XHCI_HCSPARAMS1_INTRS(params) ( ( (params) >> 8 ) & 0x3ff )
+
+/** Number of ports */
+#define XHCI_HCSPARAMS1_PORTS(params) ( ( (params) >> 24 ) & 0xff )
+
+/** Structural parameters 2 */
+#define XHCI_CAP_HCSPARAMS2 0x08
+
+/** Number of page-sized scratchpad buffers */
+#define XHCI_HCSPARAMS2_SCRATCHPADS(params) \
+	( ( ( (params) >> 16 ) & 0x3e0 ) | ( ( (params) >> 27 ) & 0x1f ) )
+
+/** Capability parameters */
+#define XHCI_CAP_HCCPARAMS1 0x10
+
+/** 64-bit addressing capability */
+#define XHCI_HCCPARAMS1_ADDR64(params) ( ( (params) >> 0 ) & 0x1 )
+
+/** Context size shift */
+#define XHCI_HCCPARAMS1_CSZ_SHIFT(params) ( 5 + ( ( (params) >> 2 ) & 0x1 ) )
+
+/** xHCI extended capabilities pointer */
+#define XHCI_HCCPARAMS1_XECP(params) ( ( ( (params) >> 16 ) & 0xffff ) << 2 )
+
+/** Doorbell offset */
+#define XHCI_CAP_DBOFF 0x14
+
+/** Runtime register space offset */
+#define XHCI_CAP_RTSOFF 0x18
+
+/** xHCI extended capability ID */
+#define XHCI_XECP_ID(xecp) ( ( (xecp) >> 0 ) & 0xff )
+
+/** Next xHCI extended capability pointer */
+#define XHCI_XECP_NEXT(xecp) ( ( ( (xecp) >> 8 ) & 0xff ) << 2 )
+
+/** USB legacy support extended capability */
+#define XHCI_XECP_ID_LEGACY 1
+
+/** USB legacy support BIOS owned semaphore */
+#define XHCI_USBLEGSUP_BIOS 0x02
+
+/** USB legacy support BIOS ownership flag */
+#define XHCI_USBLEGSUP_BIOS_OWNED 0x01
+
+/** USB legacy support OS owned semaphore */
+#define XHCI_USBLEGSUP_OS 0x03
+
+/** USB legacy support OS ownership flag */
+#define XHCI_USBLEGSUP_OS_OWNED 0x01
+
+/** USB legacy support control/status */
+#define XHCI_USBLEGSUP_CTLSTS 0x04
+
+/** Supported protocol extended capability */
+#define XHCI_XECP_ID_SUPPORTED 2
+
+/** Supported protocol revision */
+#define XHCI_SUPPORTED_REVISION 0x00
+
+/** Supported protocol minor revision */
+#define XHCI_SUPPORTED_REVISION_VER(revision) ( ( (revision) >> 16 ) & 0xffff )
+
+/** Supported protocol name */
+#define XHCI_SUPPORTED_NAME 0x04
+
+/** Supported protocol ports */
+#define XHCI_SUPPORTED_PORTS 0x08
+
+/** Supported protocol port offset */
+#define XHCI_SUPPORTED_PORTS_OFFSET(ports) ( ( (ports) >> 0 ) & 0xff )
+
+/** Supported protocol port count */
+#define XHCI_SUPPORTED_PORTS_COUNT(ports) ( ( (ports) >> 8 ) & 0xff )
+
+/** Supported protocol PSI count */
+#define XHCI_SUPPORTED_PORTS_PSIC(ports) ( ( (ports) >> 28 ) & 0x0f )
+
+/** Supported protocol slot */
+#define XHCI_SUPPORTED_SLOT 0x0c
+
+/** Supported protocol slot type */
+#define XHCI_SUPPORTED_SLOT_TYPE(slot) ( ( (slot) >> 0 ) & 0x1f )
+
+/** Supported protocol PSI */
+#define XHCI_SUPPORTED_PSI(index) ( 0x10 + ( (index) * 4 ) )
+
+/** Supported protocol PSI value */
+#define XHCI_SUPPORTED_PSI_VALUE(psi) ( ( (psi) >> 0 ) & 0x0f )
+
+/** Supported protocol PSI mantissa */
+#define XHCI_SUPPORTED_PSI_MANTISSA(psi) ( ( (psi) >> 16 ) & 0xffff )
+
+/** Supported protocol PSI exponent */
+#define XHCI_SUPPORTED_PSI_EXPONENT(psi) ( ( (psi) >> 4 ) & 0x03 )
+
+/** Default PSI values */
+enum xhci_default_psi_value {
+	/** Full speed (12Mbps) */
+	XHCI_SPEED_FULL = 1,
+	/** Low speed (1.5Mbps) */
+	XHCI_SPEED_LOW = 2,
+	/** High speed (480Mbps) */
+	XHCI_SPEED_HIGH = 3,
+	/** Super speed */
+	XHCI_SPEED_SUPER = 4,
+};
+
+/** USB command register */
+#define XHCI_OP_USBCMD 0x00
+
+/** Run/stop */
+#define XHCI_USBCMD_RUN 0x00000001UL
+
+/** Host controller reset */
+#define XHCI_USBCMD_HCRST 0x00000002UL
+
+/** USB status register */
+#define XHCI_OP_USBSTS 0x04
+
+/** Host controller halted */
+#define XHCI_USBSTS_HCH 0x00000001UL
+
+/** Page size register */
+#define XHCI_OP_PAGESIZE 0x08
+
+/** Page size */
+#define XHCI_PAGESIZE(pagesize) ( (pagesize) << 12 )
+
+/** Device notifcation control register */
+#define XHCI_OP_DNCTRL 0x14
+
+/** Command ring control register */
+#define XHCI_OP_CRCR 0x18
+
+/** Command ring cycle state */
+#define XHCI_CRCR_RCS 0x00000001UL
+
+/** Command abort */
+#define XHCI_CRCR_CA 0x00000004UL
+
+/** Command ring running */
+#define XHCI_CRCR_CRR 0x00000008UL
+
+/** Device context base address array pointer */
+#define XHCI_OP_DCBAAP 0x30
+
+/** Configure register */
+#define XHCI_OP_CONFIG 0x38
+
+/** Maximum device slots enabled */
+#define XHCI_CONFIG_MAX_SLOTS_EN(slots) ( (slots) << 0 )
+
+/** Maximum device slots enabled mask */
+#define XHCI_CONFIG_MAX_SLOTS_EN_MASK \
+	XHCI_CONFIG_MAX_SLOTS_EN ( 0xff )
+
+/** Port status and control register */
+#define XHCI_OP_PORTSC(port) ( 0x400 - 0x10 + ( (port) << 4 ) )
+
+/** Current connect status */
+#define XHCI_PORTSC_CCS 0x00000001UL
+
+/** Port enabled */
+#define XHCI_PORTSC_PED 0x00000002UL
+
+/** Port reset */
+#define XHCI_PORTSC_PR 0x00000010UL
+
+/** Port link state */
+#define XHCI_PORTSC_PLS(pls) ( (pls) << 5 )
+
+/** Disabled port link state */
+#define XHCI_PORTSC_PLS_DISABLED XHCI_PORTSC_PLS ( 4 )
+
+/** RxDetect port link state */
+#define XHCI_PORTSC_PLS_RXDETECT XHCI_PORTSC_PLS ( 5 )
+
+/** Port link state mask */
+#define XHCI_PORTSC_PLS_MASK XHCI_PORTSC_PLS ( 0xf )
+
+/** Port power */
+#define XHCI_PORTSC_PP 0x00000200UL
+
+/** Time to delay after enabling power to a port */
+#define XHCI_PORT_POWER_DELAY_MS 20
+
+/** Port speed ID value */
+#define XHCI_PORTSC_PSIV(portsc) ( ( (portsc) >> 10 ) & 0xf )
+
+/** Port indicator control */
+#define XHCI_PORTSC_PIC(indicators) ( (indicators) << 14 )
+
+/** Port indicator control mask */
+#define XHCI_PORTSC_PIC_MASK XHCI_PORTSC_PIC ( 3 )
+
+/** Port link state write strobe */
+#define XHCI_PORTSC_LWS 0x00010000UL
+
+/** Time to delay after writing the port link state */
+#define XHCI_LINK_STATE_DELAY_MS 20
+
+/** Connect status change */
+#define XHCI_PORTSC_CSC 0x00020000UL
+
+/** Port enabled/disabled change */
+#define XHCI_PORTSC_PEC 0x00040000UL
+
+/** Warm port reset change */
+#define XHCI_PORTSC_WRC 0x00080000UL
+
+/** Over-current change */
+#define XHCI_PORTSC_OCC 0x00100000UL
+
+/** Port reset change */
+#define XHCI_PORTSC_PRC 0x00200000UL
+
+/** Port link state change */
+#define XHCI_PORTSC_PLC 0x00400000UL
+
+/** Port config error change */
+#define XHCI_PORTSC_CEC 0x00800000UL
+
+/** Port status change mask */
+#define XHCI_PORTSC_CHANGE					\
+	( XHCI_PORTSC_CSC | XHCI_PORTSC_PEC | XHCI_PORTSC_WRC |	\
+	  XHCI_PORTSC_OCC | XHCI_PORTSC_PRC | XHCI_PORTSC_PLC |	\
+	  XHCI_PORTSC_CEC )
+
+/** Port status and control bits which should be preserved
+ *
+ * The port status and control register is a horrendous mix of
+ * differing semantics.  Some bits are written to only when a separate
+ * write strobe bit is set.  Some bits should be preserved when
+ * modifying other bits.  Some bits will be cleared if written back as
+ * a one.  Most excitingly, the "port enabled" bit has the semantics
+ * that 1=enabled, 0=disabled, yet writing a 1 will disable the port.
+ */
+#define XHCI_PORTSC_PRESERVE ( XHCI_PORTSC_PP | XHCI_PORTSC_PIC_MASK )
+
+/** Port power management status and control register */
+#define XHCI_OP_PORTPMSC(port) ( 0x404 - 0x10 + ( (port) << 4 ) )
+
+/** Port link info register */
+#define XHCI_OP_PORTLI(port) ( 0x408 - 0x10 + ( (port) << 4 ) )
+
+/** Port hardware link power management control register */
+#define XHCI_OP_PORTHLPMC(port) ( 0x40c - 0x10 + ( (port) << 4 ) )
+
+/** Event ring segment table size register */
+#define XHCI_RUN_ERSTSZ(intr) ( 0x28 + ( (intr) << 5 ) )
+
+/** Event ring segment table base address register */
+#define XHCI_RUN_ERSTBA(intr) ( 0x30 + ( (intr) << 5 ) )
+
+/** Event ring dequeue pointer register */
+#define XHCI_RUN_ERDP(intr) ( 0x38 + ( (intr) << 5 ) )
+
+/** A transfer request block template */
+struct xhci_trb_template {
+	/** Parameter */
+	uint64_t parameter;
+	/** Status */
+	uint32_t status;
+	/** Control */
+	uint32_t control;
+};
+
+/** A transfer request block */
+struct xhci_trb_common {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** Transfer request block cycle bit flag */
+#define XHCI_TRB_C 0x01
+
+/** Transfer request block toggle cycle bit flag */
+#define XHCI_TRB_TC 0x02
+
+/** Transfer request block chain flag */
+#define XHCI_TRB_CH 0x10
+
+/** Transfer request block interrupt on completion flag */
+#define XHCI_TRB_IOC 0x20
+
+/** Transfer request block immediate data flag */
+#define XHCI_TRB_IDT 0x40
+
+/** Transfer request block type */
+#define XHCI_TRB_TYPE(type) ( (type) << 2 )
+
+/** Transfer request block type mask */
+#define XHCI_TRB_TYPE_MASK XHCI_TRB_TYPE ( 0x3f )
+
+/** A normal transfer request block */
+struct xhci_trb_normal {
+	/** Data buffer */
+	uint64_t data;
+	/** Length */
+	uint32_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved;
+} __attribute__ (( packed ));
+
+/** A normal transfer request block */
+#define XHCI_TRB_NORMAL XHCI_TRB_TYPE ( 1 )
+
+/** Construct TD size field */
+#define XHCI_TD_SIZE(remaining) \
+	( ( ( (remaining) <= 0xf ) ? remaining : 0xf ) << 17 )
+
+/** A setup stage transfer request block */
+struct xhci_trb_setup {
+	/** Setup packet */
+	struct usb_setup_packet packet;
+	/** Length */
+	uint32_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Transfer direction */
+	uint8_t direction;
+	/** Reserved */
+	uint8_t reserved;
+} __attribute__ (( packed ));
+
+/** A setup stage transfer request block */
+#define XHCI_TRB_SETUP XHCI_TRB_TYPE ( 2 )
+
+/** Setup stage input data direction */
+#define XHCI_SETUP_IN 3
+
+/** Setup stage output data direction */
+#define XHCI_SETUP_OUT 2
+
+/** A data stage transfer request block */
+struct xhci_trb_data {
+	/** Data buffer */
+	uint64_t data;
+	/** Length */
+	uint32_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Transfer direction */
+	uint8_t direction;
+	/** Reserved */
+	uint8_t reserved;
+} __attribute__ (( packed ));
+
+/** A data stage transfer request block */
+#define XHCI_TRB_DATA XHCI_TRB_TYPE ( 3 )
+
+/** Input data direction */
+#define XHCI_DATA_IN 0x01
+
+/** Output data direction */
+#define XHCI_DATA_OUT 0x00
+
+/** A status stage transfer request block */
+struct xhci_trb_status {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Direction */
+	uint8_t direction;
+	/** Reserved */
+	uint8_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A status stage transfer request block */
+#define XHCI_TRB_STATUS XHCI_TRB_TYPE ( 4 )
+
+/** Input status direction */
+#define XHCI_STATUS_IN 0x01
+
+/** Output status direction */
+#define XHCI_STATUS_OUT 0x00
+
+/** A link transfer request block */
+struct xhci_trb_link {
+	/** Next ring segment */
+	uint64_t next;
+	/** Reserved */
+	uint32_t reserved_a;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A link transfer request block */
+#define XHCI_TRB_LINK XHCI_TRB_TYPE ( 6 )
+
+/** A no-op transfer request block */
+#define XHCI_TRB_NOP XHCI_TRB_TYPE ( 8 )
+
+/** An enable slot transfer request block */
+struct xhci_trb_enable_slot {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Slot type */
+	uint8_t slot;
+	/** Reserved */
+	uint8_t reserved_c;
+} __attribute__ (( packed ));
+
+/** An enable slot transfer request block */
+#define XHCI_TRB_ENABLE_SLOT XHCI_TRB_TYPE ( 9 )
+
+/** A disable slot transfer request block */
+struct xhci_trb_disable_slot {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint8_t reserved_c;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A disable slot transfer request block */
+#define XHCI_TRB_DISABLE_SLOT XHCI_TRB_TYPE ( 10 )
+
+/** A context transfer request block */
+struct xhci_trb_context {
+	/** Input context */
+	uint64_t input;
+	/** Reserved */
+	uint32_t reserved_a;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint8_t reserved_b;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** An address device transfer request block */
+#define XHCI_TRB_ADDRESS_DEVICE XHCI_TRB_TYPE ( 11 )
+
+/** A configure endpoint transfer request block */
+#define XHCI_TRB_CONFIGURE_ENDPOINT XHCI_TRB_TYPE ( 12 )
+
+/** An evaluate context transfer request block */
+#define XHCI_TRB_EVALUATE_CONTEXT XHCI_TRB_TYPE ( 13 )
+
+/** A reset endpoint transfer request block */
+struct xhci_trb_reset_endpoint {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A reset endpoint transfer request block */
+#define XHCI_TRB_RESET_ENDPOINT XHCI_TRB_TYPE ( 14 )
+
+/** A stop endpoint transfer request block */
+struct xhci_trb_stop_endpoint {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A stop endpoint transfer request block */
+#define XHCI_TRB_STOP_ENDPOINT XHCI_TRB_TYPE ( 15 )
+
+/** A set transfer ring dequeue pointer transfer request block */
+struct xhci_trb_set_tr_dequeue_pointer {
+	/** Dequeue pointer */
+	uint64_t dequeue;
+	/** Reserved */
+	uint32_t reserved;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A set transfer ring dequeue pointer transfer request block */
+#define XHCI_TRB_SET_TR_DEQUEUE_POINTER XHCI_TRB_TYPE ( 16 )
+
+/** A no-op command transfer request block */
+#define XHCI_TRB_NOP_CMD XHCI_TRB_TYPE ( 23 )
+
+/** A transfer event transfer request block */
+struct xhci_trb_transfer {
+	/** Transfer TRB pointer */
+	uint64_t transfer;
+	/** Residual transfer length */
+	uint16_t residual;
+	/** Reserved */
+	uint8_t reserved;
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A transfer event transfer request block */
+#define XHCI_TRB_TRANSFER XHCI_TRB_TYPE ( 32 )
+
+/** A command completion event transfer request block */
+struct xhci_trb_complete {
+	/** Command TRB pointer */
+	uint64_t command;
+	/** Parameter */
+	uint8_t parameter[3];
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Virtual function ID */
+	uint8_t vf;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A command completion event transfer request block */
+#define XHCI_TRB_COMPLETE XHCI_TRB_TYPE ( 33 )
+
+/** xHCI completion codes */
+enum xhci_completion_code {
+	/** Success */
+	XHCI_CMPLT_SUCCESS = 1,
+	/** Short packet */
+	XHCI_CMPLT_SHORT = 13,
+	/** Command ring stopped */
+	XHCI_CMPLT_CMD_STOPPED = 24,
+};
+
+/** A port status change transfer request block */
+struct xhci_trb_port_status {
+	/** Reserved */
+	uint8_t reserved_a[3];
+	/** Port ID */
+	uint8_t port;
+	/** Reserved */
+	uint8_t reserved_b[7];
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A port status change transfer request block */
+#define XHCI_TRB_PORT_STATUS XHCI_TRB_TYPE ( 34 )
+
+/** A port status change transfer request block */
+struct xhci_trb_host_controller {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint8_t reserved_b[3];
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A port status change transfer request block */
+#define XHCI_TRB_HOST_CONTROLLER XHCI_TRB_TYPE ( 37 )
+
+/** A transfer request block */
+union xhci_trb {
+	/** Template */
+	struct xhci_trb_template template;
+	/** Common fields */
+	struct xhci_trb_common common;
+	/** Normal TRB */
+	struct xhci_trb_normal normal;
+	/** Setup stage TRB */
+	struct xhci_trb_setup setup;
+	/** Data stage TRB */
+	struct xhci_trb_data data;
+	/** Status stage TRB */
+	struct xhci_trb_status status;
+	/** Link TRB */
+	struct xhci_trb_link link;
+	/** Enable slot TRB */
+	struct xhci_trb_enable_slot enable;
+	/** Disable slot TRB */
+	struct xhci_trb_disable_slot disable;
+	/** Input context TRB */
+	struct xhci_trb_context context;
+	/** Reset endpoint TRB */
+	struct xhci_trb_reset_endpoint reset;
+	/** Stop endpoint TRB */
+	struct xhci_trb_stop_endpoint stop;
+	/** Set transfer ring dequeue pointer TRB */
+	struct xhci_trb_set_tr_dequeue_pointer dequeue;
+	/** Transfer event */
+	struct xhci_trb_transfer transfer;
+	/** Command completion event */
+	struct xhci_trb_complete complete;
+	/** Port status changed event */
+	struct xhci_trb_port_status port;
+	/** Host controller event */
+	struct xhci_trb_host_controller host;
+} __attribute__ (( packed ));
+
+/** An input control context */
+struct xhci_control_context {
+	/** Drop context flags */
+	uint32_t drop;
+	/** Add context flags */
+	uint32_t add;
+	/** Reserved */
+	uint32_t reserved_a[5];
+	/** Configuration value */
+	uint8_t config;
+	/** Interface number */
+	uint8_t intf;
+	/** Alternate setting */
+	uint8_t alt;
+	/** Reserved */
+	uint8_t reserved_b;
+} __attribute__ (( packed ));
+
+/** A slot context */
+struct xhci_slot_context {
+	/** Device info */
+	uint32_t info;
+	/** Maximum exit latency */
+	uint16_t latency;
+	/** Root hub port number */
+	uint8_t port;
+	/** Number of downstream ports */
+	uint8_t ports;
+	/** TT hub slot ID */
+	uint8_t tt_id;
+	/** TT port number */
+	uint8_t tt_port;
+	/** Interrupter target */
+	uint16_t intr;
+	/** USB address */
+	uint8_t address;
+	/** Reserved */
+	uint16_t reserved_a;
+	/** Slot state */
+	uint8_t state;
+	/** Reserved */
+	uint32_t reserved_b[4];
+} __attribute__ (( packed ));
+
+/** Construct slot context device info */
+#define XHCI_SLOT_INFO( entries, hub, speed, route ) \
+	( ( (entries) << 27 ) | ( (hub) << 26 ) | ( (speed) << 20 ) | (route) )
+
+/** An endpoint context */
+struct xhci_endpoint_context {
+	/** Endpoint state */
+	uint8_t state;
+	/** Stream configuration */
+	uint8_t stream;
+	/** Polling interval */
+	uint8_t interval;
+	/** Max ESIT payload high */
+	uint8_t esit_high;
+	/** Endpoint type */
+	uint8_t type;
+	/** Maximum burst size */
+	uint8_t burst;
+	/** Maximum packet size */
+	uint16_t mtu;
+	/** Transfer ring dequeue pointer */
+	uint64_t dequeue;
+	/** Average TRB length */
+	uint16_t trb_len;
+	/** Max ESIT payload low */
+	uint16_t esit_low;
+	/** Reserved */
+	uint32_t reserved[3];
+} __attribute__ (( packed ));
+
+/** Endpoint states */
+enum xhci_endpoint_state {
+	/** Endpoint is disabled */
+	XHCI_ENDPOINT_DISABLED = 0,
+	/** Endpoint is running */
+	XHCI_ENDPOINT_RUNNING = 1,
+	/** Endpoint is halted due to a USB Halt condition */
+	XHCI_ENDPOINT_HALTED = 2,
+	/** Endpoint is stopped */
+	XHCI_ENDPOINT_STOPPED = 3,
+	/** Endpoint is halted due to a TRB error */
+	XHCI_ENDPOINT_ERROR = 4,
+};
+
+/** Endpoint state mask */
+#define XHCI_ENDPOINT_STATE_MASK 0x07
+
+/** Endpoint type */
+#define XHCI_EP_TYPE(type) ( (type) << 3 )
+
+/** Control endpoint type */
+#define XHCI_EP_TYPE_CONTROL XHCI_EP_TYPE ( 4 )
+
+/** Input endpoint type */
+#define XHCI_EP_TYPE_IN XHCI_EP_TYPE ( 4 )
+
+/** Periodic endpoint type */
+#define XHCI_EP_TYPE_PERIODIC XHCI_EP_TYPE ( 1 )
+
+/** Endpoint dequeue cycle state */
+#define XHCI_EP_DCS 0x00000001UL
+
+/** Control endpoint average TRB length */
+#define XHCI_EP0_TRB_LEN 8
+
+/** An event ring segment */
+struct xhci_event_ring_segment {
+	/** Base address */
+	uint64_t base;
+	/** Number of TRBs */
+	uint32_t count;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** A transfer request block command/transfer ring */
+struct xhci_trb_ring {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+	/** Ring size (log2) */
+	unsigned int shift;
+	/** Ring counter mask */
+	unsigned int mask;
+
+	/** I/O buffers */
+	struct io_buffer **iobuf;
+
+	/** Transfer request blocks */
+	union xhci_trb *trb;
+	/** Length of transfer request blocks */
+	size_t len;
+	/** Link TRB (if applicable) */
+	struct xhci_trb_link *link;
+
+	/** Doorbell register */
+	void *db;
+	/** Doorbell register value */
+	uint32_t dbval;
+};
+
+/** An event ring */
+struct xhci_event_ring {
+	/** Consumer counter */
+	unsigned int cons;
+	/** Event ring segment table */
+	struct xhci_event_ring_segment *segment;
+	/** Transfer request blocks */
+	union xhci_trb *trb;
+};
+
+/**
+ * Calculate doorbell register value
+ *
+ * @v target		Doorbell target
+ * @v stream		Doorbell stream ID
+ * @ret dbval		Doorbell register value
+ */
+#define XHCI_DBVAL( target, stream ) ( (target) | ( (stream) << 16 ) )
+
+/**
+ * Calculate space used in TRB ring
+ *
+ * @v ring		TRB ring
+ * @ret fill		Number of entries used
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+xhci_ring_fill ( struct xhci_trb_ring *ring ) {
+
+	return ( ring->prod - ring->cons );
+}
+
+/**
+ * Calculate space remaining in TRB ring
+ *
+ * @v ring		TRB ring
+ * @ret remaining	Number of entries remaining
+ *
+ * xHCI does not allow us to completely fill a ring; there must be at
+ * least one free entry (excluding the Link TRB).
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+xhci_ring_remaining ( struct xhci_trb_ring *ring ) {
+	unsigned int fill = xhci_ring_fill ( ring );
+
+	/* We choose to utilise rings with ( 2^n + 1 ) entries, with
+	 * the final entry being a Link TRB.  The maximum fill level
+	 * is therefore
+	 *
+	 *   ( ( 2^n + 1 ) - 1 (Link TRB) - 1 (one slot always empty)
+	 *       == ( 2^n - 1 )
+	 *
+	 * which is therefore equal to the ring mask.
+	 */
+	assert ( fill <= ring->mask );
+	return ( ring->mask - fill );
+}
+
+/**
+ * Calculate physical address of most recently consumed TRB
+ *
+ * @v ring		TRB ring
+ * @ret trb		TRB physical address
+ */
+static inline __attribute__ (( always_inline )) physaddr_t
+xhci_ring_consumed ( struct xhci_trb_ring *ring ) {
+	unsigned int index = ( ( ring->cons - 1 ) & ring->mask );
+
+	return virt_to_phys ( &ring->trb[index] );
+}
+
+/** Slot context index */
+#define XHCI_CTX_SLOT 0
+
+/** Calculate context index from USB endpoint address */
+#define XHCI_CTX(address)						\
+	( (address) ? ( ( ( (address) & 0x0f ) << 1 ) |			\
+			( ( (address) & 0x80 ) >> 7 ) ) : 1 )
+
+/** Endpoint zero context index */
+#define XHCI_CTX_EP0 XHCI_CTX ( 0x00 )
+
+/** End of contexts */
+#define XHCI_CTX_END 32
+
+/** Device context index */
+#define XHCI_DCI(ctx) ( (ctx) + 0 )
+
+/** Input context index */
+#define XHCI_ICI(ctx) ( (ctx) + 1 )
+
+/** Number of TRBs (excluding Link TRB) in the command ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_CMD_TRBS_LOG2 2
+
+/** Number of TRBs in the event ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_EVENT_TRBS_LOG2 6
+
+/** Number of TRBs in a transfer ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_TRANSFER_TRBS_LOG2 6
+
+/** Maximum time to wait for BIOS to release ownership
+ *
+ * This is a policy decision.
+ */
+#define XHCI_USBLEGSUP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for host controller to stop
+ *
+ * This is a policy decision.
+ */
+#define XHCI_STOP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for reset to complete
+ *
+ * This is a policy decision.
+ */
+#define XHCI_RESET_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a command to complete
+ *
+ * The "address device" command involves waiting for a response to a
+ * USB control transaction, and so we must wait for up to the 5000ms
+ * that USB allows for devices to respond to control transactions.
+ */
+#define XHCI_COMMAND_MAX_WAIT_MS USB_CONTROL_MAX_WAIT_MS
+
+/** Time to delay after aborting a command
+ *
+ * This is a policy decision
+ */
+#define XHCI_COMMAND_ABORT_DELAY_MS 500
+
+/** Maximum time to wait for a port reset to complete
+ *
+ * This is a policy decision.
+ */
+#define XHCI_PORT_RESET_MAX_WAIT_MS 500
+
+/** Intel PCH quirk */
+struct xhci_pch {
+	/** USB2 port routing register original value */
+	uint32_t xusb2pr;
+	/** USB3 port SuperSpeed enable register original value */
+	uint32_t usb3pssen;
+};
+
+/** Intel PCH quirk flag */
+#define XHCI_PCH 0x0001
+
+/** Intel PCH USB2 port routing register */
+#define XHCI_PCH_XUSB2PR 0xd0
+
+/** Intel PCH USB2 port routing mask register */
+#define XHCI_PCH_XUSB2PRM 0xd4
+
+/** Intel PCH SuperSpeed enable register */
+#define XHCI_PCH_USB3PSSEN 0xd8
+
+/** Intel PCH USB3 port routing mask register */
+#define XHCI_PCH_USB3PRM 0xdc
+
+/** Invalid protocol speed ID values quirk */
+#define XHCI_BAD_PSIV 0x0002
+
+/** An xHCI device */
+struct xhci_device {
+	/** Registers */
+	void *regs;
+	/** Name */
+	const char *name;
+	/** Quirks */
+	unsigned int quirks;
+
+	/** Capability registers */
+	void *cap;
+	/** Operational registers */
+	void *op;
+	/** Runtime registers */
+	void *run;
+	/** Doorbell registers */
+	void *db;
+
+	/** Number of device slots */
+	unsigned int slots;
+	/** Number of interrupters */
+	unsigned int intrs;
+	/** Number of ports */
+	unsigned int ports;
+
+	/** Number of page-sized scratchpad buffers */
+	unsigned int scratchpads;
+
+	/** 64-bit addressing capability */
+	int addr64;
+	/** Context size shift */
+	unsigned int csz_shift;
+	/** xHCI extended capabilities offset */
+	unsigned int xecp;
+
+	/** Page size */
+	size_t pagesize;
+
+	/** USB legacy support capability (if present and enabled) */
+	unsigned int legacy;
+
+	/** Device context base address array */
+	uint64_t *dcbaa;
+
+	/** Scratchpad buffer area */
+	userptr_t scratchpad;
+	/** Scratchpad buffer array */
+	uint64_t *scratchpad_array;
+
+	/** Command ring */
+	struct xhci_trb_ring command;
+	/** Event ring */
+	struct xhci_event_ring event;
+	/** Current command (if any) */
+	union xhci_trb *pending;
+
+	/** Device slots, indexed by slot ID */
+	struct xhci_slot **slot;
+
+	/** USB bus */
+	struct usb_bus *bus;
+
+	/** Intel PCH quirk */
+	struct xhci_pch pch;
+};
+
+/** An xHCI device slot */
+struct xhci_slot {
+	/** xHCI device */
+	struct xhci_device *xhci;
+	/** USB device */
+	struct usb_device *usb;
+	/** Slot ID */
+	unsigned int id;
+	/** Slot context */
+	struct xhci_slot_context *context;
+	/** Route string */
+	unsigned int route;
+	/** Root hub port number */
+	unsigned int port;
+	/** Protocol speed ID */
+	unsigned int psiv;
+	/** Number of ports (if this device is a hub) */
+	unsigned int ports;
+	/** Transaction translator slot ID */
+	unsigned int tt_id;
+	/** Transaction translator port */
+	unsigned int tt_port;
+	/** Endpoints, indexed by context ID */
+	struct xhci_endpoint *endpoint[XHCI_CTX_END];
+};
+
+/** An xHCI endpoint */
+struct xhci_endpoint {
+	/** xHCI device */
+	struct xhci_device *xhci;
+	/** xHCI slot */
+	struct xhci_slot *slot;
+	/** USB endpoint */
+	struct usb_endpoint *ep;
+	/** Context index */
+	unsigned int ctx;
+	/** Endpoint type */
+	unsigned int type;
+	/** Endpoint interval */
+	unsigned int interval;
+	/** Endpoint context */
+	struct xhci_endpoint_context *context;
+	/** Transfer ring */
+	struct xhci_trb_ring ring;
+};
+
+#endif /* _IPXE_XHCI_H */
diff --git a/roms/ipxe/src/hci/commands/autoboot_cmd.c b/roms/ipxe/src/hci/commands/autoboot_cmd.c
index 62235a278..56f39a1ce 100644
--- a/roms/ipxe/src/hci/commands/autoboot_cmd.c
+++ b/roms/ipxe/src/hci/commands/autoboot_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -25,7 +29,7 @@
 #include <hci/ifmgmt_cmd.h>
 #include <usr/autoboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/config_cmd.c b/roms/ipxe/src/hci/commands/config_cmd.c
index b81c866ff..ad415e045 100644
--- a/roms/ipxe/src/hci/commands/config_cmd.c
+++ b/roms/ipxe/src/hci/commands/config_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <string.h>
@@ -26,7 +30,7 @@
 #include <ipxe/settings.h>
 #include <ipxe/settings_ui.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/console_cmd.c b/roms/ipxe/src/hci/commands/console_cmd.c
index d2eae59f0..ba472b9f6 100644
--- a/roms/ipxe/src/hci/commands/console_cmd.c
+++ b/roms/ipxe/src/hci/commands/console_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/dhcp_cmd.c b/roms/ipxe/src/hci/commands/dhcp_cmd.c
index feeb55ee5..45a922b51 100644
--- a/roms/ipxe/src/hci/commands/dhcp_cmd.c
+++ b/roms/ipxe/src/hci/commands/dhcp_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdint.h>
diff --git a/roms/ipxe/src/hci/commands/fcmgmt_cmd.c b/roms/ipxe/src/hci/commands/fcmgmt_cmd.c
index 1c199b5dc..97f10f4dd 100644
--- a/roms/ipxe/src/hci/commands/fcmgmt_cmd.c
+++ b/roms/ipxe/src/hci/commands/fcmgmt_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/hci/commands/gdbstub_cmd.c b/roms/ipxe/src/hci/commands/gdbstub_cmd.c
index 33890aebc..c4a831e7a 100644
--- a/roms/ipxe/src/hci/commands/gdbstub_cmd.c
+++ b/roms/ipxe/src/hci/commands/gdbstub_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/hci/commands/ifmgmt_cmd.c b/roms/ipxe/src/hci/commands/ifmgmt_cmd.c
index 5307c9423..c89af2e81 100644
--- a/roms/ipxe/src/hci/commands/ifmgmt_cmd.c
+++ b/roms/ipxe/src/hci/commands/ifmgmt_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/hci/commands/image_cmd.c b/roms/ipxe/src/hci/commands/image_cmd.c
index a9e831bf5..4a7c500a4 100644
--- a/roms/ipxe/src/hci/commands/image_cmd.c
+++ b/roms/ipxe/src/hci/commands/image_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/hci/commands/image_trust_cmd.c b/roms/ipxe/src/hci/commands/image_trust_cmd.c
index ca59a858a..f9d6b5b3e 100644
--- a/roms/ipxe/src/hci/commands/image_trust_cmd.c
+++ b/roms/ipxe/src/hci/commands/image_trust_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
@@ -169,6 +173,9 @@ struct command image_trust_commands[] __command = {
 	},
 };
 
+/* Drag in objects via command list */
+REQUIRING_SYMBOL ( image_trust_commands );
+
 /* Drag in objects typically required for signature verification */
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( md5 );
diff --git a/roms/ipxe/src/hci/commands/ipstat_cmd.c b/roms/ipxe/src/hci/commands/ipstat_cmd.c
index d565dc0ae..763e4dfd6 100644
--- a/roms/ipxe/src/hci/commands/ipstat_cmd.c
+++ b/roms/ipxe/src/hci/commands/ipstat_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <getopt.h>
diff --git a/roms/ipxe/src/hci/commands/login_cmd.c b/roms/ipxe/src/hci/commands/login_cmd.c
index f5db427d5..c9e196437 100644
--- a/roms/ipxe/src/hci/commands/login_cmd.c
+++ b/roms/ipxe/src/hci/commands/login_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <string.h>
@@ -23,7 +27,7 @@
 #include <ipxe/parseopt.h>
 #include <ipxe/login_ui.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/lotest_cmd.c b/roms/ipxe/src/hci/commands/lotest_cmd.c
index 0fa031bcb..a989932d4 100644
--- a/roms/ipxe/src/hci/commands/lotest_cmd.c
+++ b/roms/ipxe/src/hci/commands/lotest_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/hci/commands/menu_cmd.c b/roms/ipxe/src/hci/commands/menu_cmd.c
index 66a6262e6..76bce8695 100644
--- a/roms/ipxe/src/hci/commands/menu_cmd.c
+++ b/roms/ipxe/src/hci/commands/menu_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/neighbour_cmd.c b/roms/ipxe/src/hci/commands/neighbour_cmd.c
index a1e052439..816e87357 100644
--- a/roms/ipxe/src/hci/commands/neighbour_cmd.c
+++ b/roms/ipxe/src/hci/commands/neighbour_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/nvo_cmd.c b/roms/ipxe/src/hci/commands/nvo_cmd.c
index e63dab08e..ac0d60651 100644
--- a/roms/ipxe/src/hci/commands/nvo_cmd.c
+++ b/roms/ipxe/src/hci/commands/nvo_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdint.h>
@@ -29,7 +33,7 @@
 #include <ipxe/parseopt.h>
 #include <readline/readline.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/param_cmd.c b/roms/ipxe/src/hci/commands/param_cmd.c
index 6cf096d00..bff04f2ff 100644
--- a/roms/ipxe/src/hci/commands/param_cmd.c
+++ b/roms/ipxe/src/hci/commands/param_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/pci_cmd.c b/roms/ipxe/src/hci/commands/pci_cmd.c
index f5145fb35..a2a811aa0 100644
--- a/roms/ipxe/src/hci/commands/pci_cmd.c
+++ b/roms/ipxe/src/hci/commands/pci_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -23,7 +27,7 @@
 #include <ipxe/command.h>
 #include <ipxe/parseopt.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/ping_cmd.c b/roms/ipxe/src/hci/commands/ping_cmd.c
index 34807696f..ab271e75a 100644
--- a/roms/ipxe/src/hci/commands/ping_cmd.c
+++ b/roms/ipxe/src/hci/commands/ping_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/hci/commands/poweroff_cmd.c b/roms/ipxe/src/hci/commands/poweroff_cmd.c
index 9d487d330..afdf12dde 100644
--- a/roms/ipxe/src/hci/commands/poweroff_cmd.c
+++ b/roms/ipxe/src/hci/commands/poweroff_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -24,7 +28,7 @@
 #include <ipxe/parseopt.h>
 #include <ipxe/reboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/profstat_cmd.c b/roms/ipxe/src/hci/commands/profstat_cmd.c
index e4c9e5a24..dc6f649e3 100644
--- a/roms/ipxe/src/hci/commands/profstat_cmd.c
+++ b/roms/ipxe/src/hci/commands/profstat_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <getopt.h>
diff --git a/roms/ipxe/src/hci/commands/reboot_cmd.c b/roms/ipxe/src/hci/commands/reboot_cmd.c
index 485939e42..45d54cc2c 100644
--- a/roms/ipxe/src/hci/commands/reboot_cmd.c
+++ b/roms/ipxe/src/hci/commands/reboot_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <getopt.h>
@@ -22,7 +26,7 @@
 #include <ipxe/parseopt.h>
 #include <ipxe/reboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/route_cmd.c b/roms/ipxe/src/hci/commands/route_cmd.c
index cc5ffc2f2..8aa535363 100644
--- a/roms/ipxe/src/hci/commands/route_cmd.c
+++ b/roms/ipxe/src/hci/commands/route_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <getopt.h>
diff --git a/roms/ipxe/src/hci/commands/sanboot_cmd.c b/roms/ipxe/src/hci/commands/sanboot_cmd.c
index 5954b6326..24ec8bc4e 100644
--- a/roms/ipxe/src/hci/commands/sanboot_cmd.c
+++ b/roms/ipxe/src/hci/commands/sanboot_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -27,7 +31,7 @@
 #include <ipxe/sanboot.h>
 #include <usr/autoboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/commands/sync_cmd.c b/roms/ipxe/src/hci/commands/sync_cmd.c
index adf7e3cc6..54799d422 100644
--- a/roms/ipxe/src/hci/commands/sync_cmd.c
+++ b/roms/ipxe/src/hci/commands/sync_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/hci/commands/vlan_cmd.c b/roms/ipxe/src/hci/commands/vlan_cmd.c
index 5d7298220..8a2f0c749 100644
--- a/roms/ipxe/src/hci/commands/vlan_cmd.c
+++ b/roms/ipxe/src/hci/commands/vlan_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/hci/editstring.c b/roms/ipxe/src/hci/editstring.c
index 5f6f04d51..8cbce0767 100644
--- a/roms/ipxe/src/hci/editstring.c
+++ b/roms/ipxe/src/hci/editstring.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <assert.h>
 #include <string.h>
diff --git a/roms/ipxe/src/hci/jumpscroll.c b/roms/ipxe/src/hci/jumpscroll.c
new file mode 100644
index 000000000..dd6bcac2b
--- /dev/null
+++ b/roms/ipxe/src/hci/jumpscroll.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Jump scrolling
+ *
+ */
+
+#include <assert.h>
+#include <ipxe/keys.h>
+#include <ipxe/jumpscroll.h>
+
+/**
+ * Handle keypress
+ *
+ * @v scroll		Jump scroller
+ * @v key		Key pressed by user
+ * @ret move		Scroller movement, or zero
+ */
+int jump_scroll_key ( struct jump_scroller *scroll, int key ) {
+
+	/* Sanity checks */
+	assert ( scroll->rows != 0 );
+	assert ( scroll->count != 0 );
+	assert ( scroll->current < scroll->count );
+	assert ( scroll->first < scroll->count );
+	assert ( scroll->first <= scroll->current );
+	assert ( scroll->current < ( scroll->first + scroll->rows ) );
+
+	/* Handle key, if applicable */
+	switch ( key ) {
+	case KEY_UP:
+		return -1;
+	case KEY_DOWN:
+		return +1;
+	case KEY_PPAGE:
+		return ( scroll->first - scroll->current - 1 );
+	case KEY_NPAGE:
+		return ( scroll->first - scroll->current + scroll->rows );
+	case KEY_HOME:
+		return -( scroll->count );
+	case KEY_END:
+		return +( scroll->count );
+	default:
+		return 0;
+	}
+}
+
+/**
+ * Move scroller
+ *
+ * @v scroll		Jump scroller
+ * @v move		Scroller movement
+ * @ret move		Continuing scroller movement (if applicable)
+ */
+int jump_scroll_move ( struct jump_scroller *scroll, int move ) {
+	int current = scroll->current;
+	int last = ( scroll->count - 1 );
+
+	/* Sanity checks */
+	assert ( move != 0 );
+	assert ( scroll->count != 0 );
+
+	/* Move to the new current item */
+	current += move;
+
+	/* Check for start/end of list */
+	if ( current < 0 ) {
+		/* We have attempted to move before the start of the
+		 * list.  Move to the start of the list and continue
+		 * moving forwards (if applicable).
+		 */
+		scroll->current = 0;
+		return +1;
+	} else if ( current > last ) {
+		/* We have attempted to move after the end of the
+		 * list.  Move to the end of the list and continue
+		 * moving backwards (if applicable).
+		 */
+		scroll->current = last;
+		return -1;
+	} else {
+		/* Update the current item and continue moving in the
+		 * same direction (if applicable).
+		 */
+		scroll->current = current;
+		return ( ( move > 0 ) ? +1 : -1 );
+	}
+}
+
+/**
+ * Jump scroll to new page (if applicable)
+ *
+ * @v scroll		Jump scroller
+ * @ret jumped		Jumped to a new page
+ */
+int jump_scroll ( struct jump_scroller *scroll ) {
+	unsigned int index;
+
+	/* Sanity checks */
+	assert ( scroll->rows != 0 );
+	assert ( scroll->count != 0 );
+	assert ( scroll->current < scroll->count );
+	assert ( scroll->first < scroll->count );
+
+	/* Do nothing if we are already on the correct page */
+	index = ( scroll->current - scroll->first );
+	if ( index < scroll->rows )
+		return 0;
+
+	/* Move to required page */
+	while ( scroll->first < scroll->current )
+		scroll->first += scroll->rows;
+	while ( scroll->first > scroll->current )
+		scroll->first -= scroll->rows;
+
+	return 1;
+}
diff --git a/roms/ipxe/src/hci/mucurses/alert.c b/roms/ipxe/src/hci/mucurses/alert.c
index 00e959a89..7dc61c222 100644
--- a/roms/ipxe/src/hci/mucurses/alert.c
+++ b/roms/ipxe/src/hci/mucurses/alert.c
@@ -7,6 +7,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Audible signal
  *
diff --git a/roms/ipxe/src/hci/mucurses/ansi_screen.c b/roms/ipxe/src/hci/mucurses/ansi_screen.c
index 1d3143f89..1cf3309dd 100644
--- a/roms/ipxe/src/hci/mucurses/ansi_screen.c
+++ b/roms/ipxe/src/hci/mucurses/ansi_screen.c
@@ -3,7 +3,7 @@
 #include <ipxe/ansicol.h>
 #include <ipxe/console.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static void ansiscr_reset(struct _curses_screen *scr) __nonnull;
 static void ansiscr_movetoyx(struct _curses_screen *scr,
diff --git a/roms/ipxe/src/hci/mucurses/clear.c b/roms/ipxe/src/hci/mucurses/clear.c
index f5e52ca20..2054f72cc 100644
--- a/roms/ipxe/src/hci/mucurses/clear.c
+++ b/roms/ipxe/src/hci/mucurses/clear.c
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Clear a window to the bottom from current cursor position
diff --git a/roms/ipxe/src/hci/mucurses/colour.c b/roms/ipxe/src/hci/mucurses/colour.c
index c1359c868..b0c480b1f 100644
--- a/roms/ipxe/src/hci/mucurses/colour.c
+++ b/roms/ipxe/src/hci/mucurses/colour.c
@@ -1,6 +1,6 @@
 #include <curses.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct colour_pair {
 	short fcol;
diff --git a/roms/ipxe/src/hci/mucurses/cursor.h b/roms/ipxe/src/hci/mucurses/cursor.h
index 16b7d27c2..2e0c896a6 100644
--- a/roms/ipxe/src/hci/mucurses/cursor.h
+++ b/roms/ipxe/src/hci/mucurses/cursor.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct cursor_pos {
 	unsigned int y, x;
diff --git a/roms/ipxe/src/hci/mucurses/edging.c b/roms/ipxe/src/hci/mucurses/edging.c
index eccd32422..e938d338b 100644
--- a/roms/ipxe/src/hci/mucurses/edging.c
+++ b/roms/ipxe/src/hci/mucurses/edging.c
@@ -8,6 +8,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Draw borders from single-byte characters and renditions around a
  * window
diff --git a/roms/ipxe/src/hci/mucurses/kb.c b/roms/ipxe/src/hci/mucurses/kb.c
index b38c8c146..8face14d8 100644
--- a/roms/ipxe/src/hci/mucurses/kb.c
+++ b/roms/ipxe/src/hci/mucurses/kb.c
@@ -8,6 +8,8 @@
  * MuCurses keyboard input handling functions
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #define INPUT_DELAY 		200 // half-blocking delay timer resolution (ms)
 #define INPUT_DELAY_TIMEOUT 	1000 // half-blocking delay timeout
 
diff --git a/roms/ipxe/src/hci/mucurses/mucurses.c b/roms/ipxe/src/hci/mucurses/mucurses.c
index b67445baf..98a8a2c59 100644
--- a/roms/ipxe/src/hci/mucurses/mucurses.c
+++ b/roms/ipxe/src/hci/mucurses/mucurses.c
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static void _wupdcurs ( WINDOW *win ) __nonnull;
 void _wputch ( WINDOW *win, chtype ch, int wrap ) __nonnull;
diff --git a/roms/ipxe/src/hci/mucurses/mucurses.h b/roms/ipxe/src/hci/mucurses/mucurses.h
index 7ac1086ac..270394787 100644
--- a/roms/ipxe/src/hci/mucurses/mucurses.h
+++ b/roms/ipxe/src/hci/mucurses/mucurses.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define WRAP 0
 #define NOWRAP 1
diff --git a/roms/ipxe/src/hci/mucurses/print.c b/roms/ipxe/src/hci/mucurses/print.c
index 9c682588b..e8831c58f 100644
--- a/roms/ipxe/src/hci/mucurses/print.c
+++ b/roms/ipxe/src/hci/mucurses/print.c
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Add a single-byte character and rendition to a window and advance
diff --git a/roms/ipxe/src/hci/mucurses/print_nadv.c b/roms/ipxe/src/hci/mucurses/print_nadv.c
index ee472e685..3a44e5bd2 100644
--- a/roms/ipxe/src/hci/mucurses/print_nadv.c
+++ b/roms/ipxe/src/hci/mucurses/print_nadv.c
@@ -8,6 +8,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Add string of single-byte characters and renditions to a window
  *
diff --git a/roms/ipxe/src/hci/mucurses/slk.c b/roms/ipxe/src/hci/mucurses/slk.c
index 600658e75..660eb65c0 100644
--- a/roms/ipxe/src/hci/mucurses/slk.c
+++ b/roms/ipxe/src/hci/mucurses/slk.c
@@ -11,6 +11,8 @@
  * Soft label key functions
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #define MIN_SPACE_SIZE 2
 
 #define SLK_MAX_LABEL_LEN 8
diff --git a/roms/ipxe/src/hci/mucurses/widgets/editbox.c b/roms/ipxe/src/hci/mucurses/widgets/editbox.c
index 630a66e0b..210de4481 100644
--- a/roms/ipxe/src/hci/mucurses/widgets/editbox.c
+++ b/roms/ipxe/src/hci/mucurses/widgets/editbox.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <assert.h>
diff --git a/roms/ipxe/src/hci/mucurses/winattrs.c b/roms/ipxe/src/hci/mucurses/winattrs.c
index f549d7519..97a5a18b3 100644
--- a/roms/ipxe/src/hci/mucurses/winattrs.c
+++ b/roms/ipxe/src/hci/mucurses/winattrs.c
@@ -6,7 +6,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Get the background rendition attributes for a window
diff --git a/roms/ipxe/src/hci/mucurses/windows.c b/roms/ipxe/src/hci/mucurses/windows.c
index 63d0af08c..7f39bdea2 100644
--- a/roms/ipxe/src/hci/mucurses/windows.c
+++ b/roms/ipxe/src/hci/mucurses/windows.c
@@ -9,6 +9,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Delete a window
  *
diff --git a/roms/ipxe/src/hci/mucurses/wininit.c b/roms/ipxe/src/hci/mucurses/wininit.c
index b75abba44..dd84d2f1d 100644
--- a/roms/ipxe/src/hci/mucurses/wininit.c
+++ b/roms/ipxe/src/hci/mucurses/wininit.c
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Initialise console environment
diff --git a/roms/ipxe/src/hci/readline.c b/roms/ipxe/src/hci/readline.c
index 40aa59787..83a2e0b90 100644
--- a/roms/ipxe/src/hci/readline.c
+++ b/roms/ipxe/src/hci/readline.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <string.h>
diff --git a/roms/ipxe/src/hci/shell.c b/roms/ipxe/src/hci/shell.c
index c1a543849..276eb3527 100644
--- a/roms/ipxe/src/hci/shell.c
+++ b/roms/ipxe/src/hci/shell.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -28,6 +32,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/command.h>
 #include <ipxe/parseopt.h>
 #include <ipxe/shell.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -36,7 +41,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /** The shell prompt string */
-static const char shell_prompt[] = "iPXE> ";
+static const char shell_prompt[] = PRODUCT_SHORT_NAME "> ";
 
 /**
  * "help" command
diff --git a/roms/ipxe/src/hci/strerror.c b/roms/ipxe/src/hci/strerror.c
index 9356e9e0a..1bba8c620 100644
--- a/roms/ipxe/src/hci/strerror.c
+++ b/roms/ipxe/src/hci/strerror.c
@@ -2,6 +2,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <ipxe/errortab.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -18,7 +19,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Find error description
@@ -74,7 +75,7 @@ static struct errortab * find_closest_error ( int errno ) {
  * call to strerror().
  *
  */
-const char * strerror ( int errno ) {
+char * strerror ( int errno ) {
 	static char errbuf[64];
 	struct errortab *errortab;
 
@@ -88,11 +89,11 @@ const char * strerror ( int errno ) {
 	/* Construct the error message */
 	if ( errortab ) {
 		snprintf ( errbuf, sizeof ( errbuf ),
-			   "%s (http://ipxe.org/%08x)",
+			   "%s (" PRODUCT_ERROR_URI ")",
 			   errortab->text, errno );
 	} else {
 		snprintf ( errbuf, sizeof ( errbuf ),
-			   "Error %#08x (http://ipxe.org/%08x)",
+			   "Error %#08x (" PRODUCT_ERROR_URI ")",
 			   errno, errno );
 	}
 
diff --git a/roms/ipxe/src/hci/tui/login_ui.c b/roms/ipxe/src/hci/tui/login_ui.c
index 996b68a0a..3c55325d5 100644
--- a/roms/ipxe/src/hci/tui/login_ui.c
+++ b/roms/ipxe/src/hci/tui/login_ui.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/hci/tui/menu_ui.c b/roms/ipxe/src/hci/tui/menu_ui.c
index 0a9566def..f9dd9d100 100644
--- a/roms/ipxe/src/hci/tui/menu_ui.c
+++ b/roms/ipxe/src/hci/tui/menu_ui.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -32,6 +36,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/timer.h>
 #include <ipxe/console.h>
 #include <ipxe/ansicol.h>
+#include <ipxe/jumpscroll.h>
 #include <ipxe/menu.h>
 
 /* Screen layout */
@@ -46,12 +51,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 struct menu_ui {
 	/** Menu */
 	struct menu *menu;
-	/** Number of menu items */
-	unsigned int count;
-	/** Currently selected item */
-	int selected;
-	/** First visible item */
-	int first_visible;
+	/** Jump scroller */
+	struct jump_scroller scroll;
 	/** Timeout (0=indefinite) */
 	unsigned long timeout;
 };
@@ -80,7 +81,7 @@ static struct menu_item * menu_item ( struct menu *menu, unsigned int index ) {
  * @v ui		Menu user interface
  * @v index		Index
  */
-static void draw_menu_item ( struct menu_ui *ui, int index ) {
+static void draw_menu_item ( struct menu_ui *ui, unsigned int index ) {
 	struct menu_item *item;
 	unsigned int row_offset;
 	char buf[ MENU_COLS + 1 /* NUL */ ];
@@ -90,7 +91,7 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 	size_t len;
 
 	/* Move to start of row */
-	row_offset = ( index - ui->first_visible );
+	row_offset = ( index - ui->scroll.first );
 	move ( ( MENU_ROW + row_offset ), MENU_COL );
 
 	/* Get menu item */
@@ -102,7 +103,7 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 			color_set ( CPAIR_SEPARATOR, NULL );
 
 		/* Highlight if this is the selected item */
-		if ( index == ui->selected ) {
+		if ( index == ui->scroll.current ) {
 			color_set ( CPAIR_SELECT, NULL );
 			attron ( A_BOLD );
 		}
@@ -121,7 +122,7 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 			snprintf ( timeout_buf, sizeof ( timeout_buf ), "(%ld)",
 				   ( ( ui->timeout + TICKS_PER_SEC - 1 ) /
 				     TICKS_PER_SEC ) );
-		if ( ( index == ui->selected ) && ( ui->timeout != 0 ) ) {
+		if ( ( index == ui->scroll.current ) && ( ui->timeout != 0 ) ) {
 			memcpy ( ( buf + MENU_COLS - MENU_PAD - timeout_len ),
 				 timeout_buf, timeout_len );
 		}
@@ -150,24 +151,17 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 static void draw_menu_items ( struct menu_ui *ui ) {
 	unsigned int i;
 
-	/* Jump scroll to correct point in list */
-	while ( ui->first_visible < ui->selected )
-		ui->first_visible += MENU_ROWS;
-	while ( ui->first_visible > ui->selected )
-		ui->first_visible -= MENU_ROWS;
-
 	/* Draw ellipses before and/or after the list as necessary */
 	color_set ( CPAIR_SEPARATOR, NULL );
 	mvaddstr ( ( MENU_ROW - 1 ), ( MENU_COL + MENU_PAD ),
-		   ( ( ui->first_visible > 0 ) ? "..." : "   " ) );
+		   ( jump_scroll_is_first ( &ui->scroll ) ? "   " : "..." ) );
 	mvaddstr ( ( MENU_ROW + MENU_ROWS ), ( MENU_COL + MENU_PAD ),
-		   ( ( ( ui->first_visible + MENU_ROWS ) < ui->count ) ?
-		     "..." : "   " ) );
+		   ( jump_scroll_is_last ( &ui->scroll ) ? "   " : "..." ) );
 	color_set ( CPAIR_NORMAL, NULL );
 
 	/* Draw visible items */
 	for ( i = 0 ; i < MENU_ROWS ; i++ )
-		draw_menu_item ( ui, ( ui->first_visible + i ) );
+		draw_menu_item ( ui, ( ui->scroll.first + i ) );
 }
 
 /**
@@ -180,8 +174,7 @@ static void draw_menu_items ( struct menu_ui *ui ) {
 static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 	struct menu_item *item;
 	unsigned long timeout;
-	unsigned int delta;
-	int current;
+	unsigned int previous;
 	int key;
 	int i;
 	int move;
@@ -190,7 +183,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 
 	do {
 		/* Record current selection */
-		current = ui->selected;
+		previous = ui->scroll.current;
 
 		/* Calculate timeout as remainder of current second */
 		timeout = ( ui->timeout % TICKS_PER_SEC );
@@ -209,27 +202,11 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 			/* Cancel any timeout */
 			ui->timeout = 0;
 
-			/* Handle key */
+			/* Handle scroll keys */
+			move = jump_scroll_key ( &ui->scroll, key );
+
+			/* Handle other keys */
 			switch ( key ) {
-			case KEY_UP:
-				move = -1;
-				break;
-			case KEY_DOWN:
-				move = +1;
-				break;
-			case KEY_PPAGE:
-				move = ( ui->first_visible - ui->selected - 1 );
-				break;
-			case KEY_NPAGE:
-				move = ( ui->first_visible - ui->selected
-					 + MENU_ROWS );
-				break;
-			case KEY_HOME:
-				move = -ui->count;
-				break;
-			case KEY_END:
-				move = +ui->count;
-				break;
 			case ESC:
 			case CTRL_C:
 				rc = -ECANCELED;
@@ -247,7 +224,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 						i++;
 						continue;
 					}
-					ui->selected = i;
+					ui->scroll.current = i;
 					if ( item->label ) {
 						chosen = 1;
 					} else {
@@ -260,31 +237,22 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 
 		/* Move selection, if applicable */
 		while ( move ) {
-			ui->selected += move;
-			if ( ui->selected < 0 ) {
-				ui->selected = 0;
-				move = +1;
-			} else if ( ui->selected >= ( int ) ui->count ) {
-				ui->selected = ( ui->count - 1 );
-				move = -1;
-			}
-			item = menu_item ( ui->menu, ui->selected );
+			move = jump_scroll_move ( &ui->scroll, move );
+			item = menu_item ( ui->menu, ui->scroll.current );
 			if ( item->label )
 				break;
-			move = ( ( move > 0 ) ? +1 : -1 );
 		}
 
 		/* Redraw selection if necessary */
-		if ( ( ui->selected != current ) || ( timeout != 0 ) ) {
-			draw_menu_item ( ui, current );
-			delta = ( ui->selected - ui->first_visible );
-			if ( delta >= MENU_ROWS )
+		if ( ( ui->scroll.current != previous ) || ( timeout != 0 ) ) {
+			draw_menu_item ( ui, previous );
+			if ( jump_scroll ( &ui->scroll ) )
 				draw_menu_items ( ui );
-			draw_menu_item ( ui, ui->selected );
+			draw_menu_item ( ui, ui->scroll.current );
 		}
 
 		/* Record selection */
-		item = menu_item ( ui->menu, ui->selected );
+		item = menu_item ( ui->menu, ui->scroll.current );
 		assert ( item != NULL );
 		assert ( item->label != NULL );
 		*selected = item;
@@ -313,21 +281,22 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 	/* Initialise UI */
 	memset ( &ui, 0, sizeof ( ui ) );
 	ui.menu = menu;
+	ui.scroll.rows = MENU_ROWS;
 	ui.timeout = timeout;
 	list_for_each_entry ( item, &menu->items, list ) {
 		if ( item->label ) {
 			if ( ! labelled_count )
-				ui.selected = ui.count;
+				ui.scroll.current = ui.scroll.count;
 			labelled_count++;
 			if ( select ) {
 				if ( strcmp ( select, item->label ) == 0 )
-					ui.selected = ui.count;
+					ui.scroll.current = ui.scroll.count;
 			} else {
 				if ( item->is_default )
-					ui.selected = ui.count;
+					ui.scroll.current = ui.scroll.count;
 			}
 		}
-		ui.count++;
+		ui.scroll.count++;
 	}
 	if ( ! labelled_count ) {
 		/* Menus with no labelled items cannot be selected
@@ -349,8 +318,9 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 	snprintf ( buf, sizeof ( buf ), "%s", ui.menu->title );
 	mvprintw ( TITLE_ROW, ( ( COLS - strlen ( buf ) ) / 2 ), "%s", buf );
 	attroff ( A_BOLD );
+	jump_scroll ( &ui.scroll );
 	draw_menu_items ( &ui );
-	draw_menu_item ( &ui, ui.selected );
+	draw_menu_item ( &ui, ui.scroll.current );
 
 	/* Enter main loop */
 	rc = menu_loop ( &ui, selected );
diff --git a/roms/ipxe/src/hci/tui/settings_ui.c b/roms/ipxe/src/hci/tui/settings_ui.c
index 221839730..be421cc0a 100644
--- a/roms/ipxe/src/hci/tui/settings_ui.c
+++ b/roms/ipxe/src/hci/tui/settings_ui.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdarg.h>
@@ -29,7 +33,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/editbox.h>
 #include <ipxe/keys.h>
 #include <ipxe/ansicol.h>
+#include <ipxe/jumpscroll.h>
 #include <ipxe/settings_ui.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -47,7 +53,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define INSTRUCTION_ROW		( LINES - 2U )
 #define INSTRUCTION_PAD "     "
 
-/** Layout of text within a setting widget */
+/** Layout of text within a setting row */
 #define SETTING_ROW_TEXT( cols ) struct {				\
 	char start[0];							\
 	char pad1[1];							\
@@ -63,8 +69,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 	char nul;							\
 } __attribute__ (( packed ))
 
-/** A setting row widget */
-struct setting_row_widget {
+/** A settings user interface row */
+struct settings_ui_row {
 	/** Target configuration settings block
 	 *
 	 * Valid only for rows that lead to new settings blocks.
@@ -82,8 +88,6 @@ struct setting_row_widget {
 	struct setting setting;
 	/** Screen row */
 	unsigned int row;
-	/** Screen column */
-	unsigned int col;
 	/** Edit box widget used for editing setting */
 	struct edit_box editbox;
 	/** Editing in progress flag */
@@ -92,28 +96,24 @@ struct setting_row_widget {
 	char value[256]; /* enough size for a DHCP string */
 };
 
-/** A settings widget */
-struct setting_widget {
+/** A settings user interface */
+struct settings_ui {
 	/** Settings block */
 	struct settings *settings;
-	/** Number of rows */
-	unsigned int num_rows;
-	/** Current row index */
-	unsigned int current;
-        /** Index of the first visible row, for scrolling. */
-	unsigned int first_visible;
-	/** Active row */
-	struct setting_row_widget row;
+	/** Jump scroller */
+	struct jump_scroller scroll;
+	/** Current row */
+	struct settings_ui_row row;
 };
 
 /**
- * Select a setting row
+ * Select a setting
  *
- * @v widget		Setting widget
+ * @v ui		Settings user interface
  * @v index		Index of setting row
- * @ret count		Number of settings rows
+ * @ret count		Number of setting rows
  */
-static unsigned int select_setting_row ( struct setting_widget *widget,
+static unsigned int select_setting_row ( struct settings_ui *ui,
 					 unsigned int index ) {
 	SETTING_ROW_TEXT ( COLS ) *text;
 	struct settings *settings;
@@ -122,25 +122,22 @@ static unsigned int select_setting_row ( struct setting_widget *widget,
 	unsigned int count = 0;
 
 	/* Initialise structure */
-	memset ( &widget->row, 0, sizeof ( widget->row ) );
-	widget->current = index;
-	widget->row.row = ( SETTINGS_LIST_ROW + index - widget->first_visible );
-	widget->row.col = SETTINGS_LIST_COL;
+	memset ( &ui->row, 0, sizeof ( ui->row ) );
+	ui->row.row = ( SETTINGS_LIST_ROW + index - ui->scroll.first );
 
 	/* Include parent settings block, if applicable */
-	if ( widget->settings->parent && ( count++ == index ) ) {
-		widget->row.settings = widget->settings->parent;
-		snprintf ( widget->row.value, sizeof ( widget->row.value ),
+	if ( ui->settings->parent && ( count++ == index ) ) {
+		ui->row.settings = ui->settings->parent;
+		snprintf ( ui->row.value, sizeof ( ui->row.value ),
 			   "../" );
 	}
 
 	/* Include any child settings blocks, if applicable */
-	list_for_each_entry ( settings, &widget->settings->children, siblings ){
+	list_for_each_entry ( settings, &ui->settings->children, siblings ) {
 		if ( count++ == index ) {
-			widget->row.settings = settings;
-			snprintf ( widget->row.value,
-				   sizeof ( widget->row.value ), "%s/",
-				   settings->name );
+			ui->row.settings = settings;
+			snprintf ( ui->row.value, sizeof ( ui->row.value ),
+				   "%s/", settings->name );
 		}
 	}
 
@@ -148,7 +145,7 @@ static unsigned int select_setting_row ( struct setting_widget *widget,
 	for_each_table_entry ( setting, SETTINGS ) {
 
 		/* Skip inapplicable settings */
-		if ( ! setting_applies ( widget->settings, setting ) )
+		if ( ! setting_applies ( ui->settings, setting ) )
 			continue;
 
 		/* Skip duplicate settings */
@@ -158,18 +155,16 @@ static unsigned int select_setting_row ( struct setting_widget *widget,
 
 		/* Read current setting value and origin */
 		if ( count++ == index ) {
-			fetchf_setting ( widget->settings, setting,
-					 &widget->row.origin,
-					 &widget->row.setting,
-					 widget->row.value,
-					 sizeof ( widget->row.value ) );
+			fetchf_setting ( ui->settings, setting, &ui->row.origin,
+					 &ui->row.setting, ui->row.value,
+					 sizeof ( ui->row.value ) );
 		}
 	}
 
 	/* Initialise edit box */
-	init_editbox ( &widget->row.editbox, widget->row.value,
-		       sizeof ( widget->row.value ), NULL, widget->row.row,
-		       ( widget->row.col +
+	init_editbox ( &ui->row.editbox, ui->row.value,
+		       sizeof ( ui->row.value ), NULL, ui->row.row,
+		       ( SETTINGS_LIST_COL +
 			 offsetof ( typeof ( *text ), u.setting.value ) ),
 		       sizeof ( text->u.setting.value ), 0 );
 
@@ -197,9 +192,9 @@ static size_t string_copy ( char *dest, const char *src, size_t len ) {
 /**
  * Draw setting row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_setting_row ( struct setting_widget *widget ) {
+static void draw_setting_row ( struct settings_ui *ui ) {
 	SETTING_ROW_TEXT ( COLS ) text;
 	unsigned int curs_offset;
 	char *value;
@@ -209,12 +204,12 @@ static void draw_setting_row ( struct setting_widget *widget ) {
 	text.nul = '\0';
 
 	/* Construct row content */
-	if ( widget->row.settings ) {
+	if ( ui->row.settings ) {
 
 		/* Construct space-padded name */
 		curs_offset = ( offsetof ( typeof ( text ), u.settings ) +
 				string_copy ( text.u.settings,
-					      widget->row.value,
+					      ui->row.value,
 					      sizeof ( text.u.settings ) ) );
 
 	} else {
@@ -222,11 +217,11 @@ static void draw_setting_row ( struct setting_widget *widget ) {
 		/* Construct dot-padded name */
 		memset ( text.u.setting.name, '.',
 			 sizeof ( text.u.setting.name ) );
-		string_copy ( text.u.setting.name, widget->row.setting.name,
+		string_copy ( text.u.setting.name, ui->row.setting.name,
 			      sizeof ( text.u.setting.name ) );
 
 		/* Construct space-padded value */
-		value = widget->row.value;
+		value = ui->row.value;
 		if ( ! *value )
 			value = "<not specified>";
 		curs_offset = ( offsetof ( typeof ( text ), u.setting.value ) +
@@ -235,37 +230,34 @@ static void draw_setting_row ( struct setting_widget *widget ) {
 	}
 
 	/* Print row */
-	if ( ( widget->row.origin == widget->settings ) ||
-	     ( widget->row.settings != NULL ) ) {
+	if ( ( ui->row.origin == ui->settings ) || ( ui->row.settings != NULL ))
 		attron ( A_BOLD );
-	}
-	mvprintw ( widget->row.row, widget->row.col, "%s", text.start );
+	mvprintw ( ui->row.row, SETTINGS_LIST_COL, "%s", text.start );
 	attroff ( A_BOLD );
-	move ( widget->row.row, widget->row.col + curs_offset );
+	move ( ui->row.row, ( SETTINGS_LIST_COL + curs_offset ) );
 }
 
 /**
- * Edit setting widget
+ * Edit setting ui
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  * @v key		Key pressed by user
  * @ret key		Key returned to application, or zero
  */
-static int edit_setting ( struct setting_widget *widget, int key ) {
-	assert ( widget->row.setting.name != NULL );
-	widget->row.editing = 1;
-	return edit_editbox ( &widget->row.editbox, key );
+static int edit_setting ( struct settings_ui *ui, int key ) {
+	assert ( ui->row.setting.name != NULL );
+	ui->row.editing = 1;
+	return edit_editbox ( &ui->row.editbox, key );
 }
 
 /**
- * Save setting widget value back to configuration settings
+ * Save setting ui value back to configuration settings
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static int save_setting ( struct setting_widget *widget ) {
-	assert ( widget->row.setting.name != NULL );
-	return storef_setting ( widget->settings, &widget->row.setting,
-				widget->row.value );
+static int save_setting ( struct settings_ui *ui ) {
+	assert ( ui->row.setting.name != NULL );
+	return storef_setting ( ui->settings, &ui->row.setting, ui->row.value );
 }
 
 /**
@@ -340,15 +332,15 @@ static void alert ( const char *fmt, ... ) {
 /**
  * Draw title row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_title_row ( struct setting_widget *widget ) {
+static void draw_title_row ( struct settings_ui *ui ) {
 	const char *name;
 
 	clearmsg ( TITLE_ROW );
-	name = settings_name ( widget->settings );
+	name = settings_name ( ui->settings );
 	attron ( A_BOLD );
-	msg ( TITLE_ROW, "iPXE configuration settings%s%s",
+	msg ( TITLE_ROW, PRODUCT_SHORT_NAME " configuration settings%s%s",
 	      ( name[0] ? " - " : "" ), name );
 	attroff ( A_BOLD );
 }
@@ -356,89 +348,73 @@ static void draw_title_row ( struct setting_widget *widget ) {
 /**
  * Draw information row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_info_row ( struct setting_widget *widget ) {
+static void draw_info_row ( struct settings_ui *ui ) {
 	char buf[32];
 
 	/* Draw nothing unless this row represents a setting */
 	clearmsg ( INFO_ROW );
 	clearmsg ( INFO_ROW + 1 );
-	if ( ! widget->row.setting.name )
+	if ( ! ui->row.setting.name )
 		return;
 
 	/* Determine a suitable setting name */
-	setting_name ( ( widget->row.origin ?
-			 widget->row.origin : widget->settings ),
-		       &widget->row.setting, buf, sizeof ( buf ) );
+	setting_name ( ( ui->row.origin ?
+			 ui->row.origin : ui->settings ),
+		       &ui->row.setting, buf, sizeof ( buf ) );
 
 	/* Draw row */
 	attron ( A_BOLD );
-	msg ( INFO_ROW, "%s - %s", buf, widget->row.setting.description );
+	msg ( INFO_ROW, "%s - %s", buf, ui->row.setting.description );
 	attroff ( A_BOLD );
 	color_set ( CPAIR_URL, NULL );
-	msg ( ( INFO_ROW + 1 ), "http://ipxe.org/cfg/%s",
-	      widget->row.setting.name );
+	msg ( ( INFO_ROW + 1 ), PRODUCT_SETTING_URI, ui->row.setting.name );
 	color_set ( CPAIR_NORMAL, NULL );
 }
 
 /**
  * Draw instruction row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_instruction_row ( struct setting_widget *widget ) {
+static void draw_instruction_row ( struct settings_ui *ui ) {
 
 	clearmsg ( INSTRUCTION_ROW );
-	if ( widget->row.editing ) {
+	if ( ui->row.editing ) {
 		msg ( INSTRUCTION_ROW,
 		      "Enter - accept changes" INSTRUCTION_PAD
 		      "Ctrl-C - discard changes" );
 	} else {
 		msg ( INSTRUCTION_ROW,
 		      "%sCtrl-X - exit configuration utility",
-		      ( ( widget->row.origin == widget->settings ) ?
+		      ( ( ui->row.origin == ui->settings ) ?
 			"Ctrl-D - delete setting" INSTRUCTION_PAD : "" ) );
 	}
 }
 
 /**
- * Reveal setting row
+ * Draw the current block of setting rows
  *
- * @v widget		Setting widget
- * @v index		Index of setting row
+ * @v ui		Settings UI
  */
-static void reveal_setting_row ( struct setting_widget *widget,
-				 unsigned int index ) {
+static void draw_setting_rows ( struct settings_ui *ui ) {
 	unsigned int i;
 
-	/* Simply return if setting N is already on-screen. */
-	if ( index - widget->first_visible < SETTINGS_LIST_ROWS )
-		return;
-
-	/* Jump scroll to make the specified setting row visible. */
-	while ( widget->first_visible < index )
-		widget->first_visible += SETTINGS_LIST_ROWS;
-	while ( widget->first_visible > index )
-		widget->first_visible -= SETTINGS_LIST_ROWS;
-
-	/* Draw ellipses before and/or after the settings list to
-	 * represent any invisible settings.
-	 */
-	mvaddstr ( SETTINGS_LIST_ROW - 1,
-		   SETTINGS_LIST_COL + 1,
-		   widget->first_visible > 0 ? "..." : "   " );
-	mvaddstr ( SETTINGS_LIST_ROW + SETTINGS_LIST_ROWS,
-		   SETTINGS_LIST_COL + 1,
-		   ( ( widget->first_visible + SETTINGS_LIST_ROWS )
-		     < widget->num_rows ? "..." : "   " ) );
+	/* Draw ellipses before and/or after the list as necessary */
+	color_set ( CPAIR_SEPARATOR, NULL );
+	mvaddstr ( ( SETTINGS_LIST_ROW - 1 ), ( SETTINGS_LIST_COL + 1 ),
+		   jump_scroll_is_first ( &ui->scroll ) ? "   " : "..." );
+	mvaddstr ( ( SETTINGS_LIST_ROW + SETTINGS_LIST_ROWS ),
+		   ( SETTINGS_LIST_COL + 1 ),
+		   jump_scroll_is_last ( &ui->scroll ) ? "   " : "..." );
+	color_set ( CPAIR_NORMAL, NULL );
 
 	/* Draw visible settings. */
-	for ( i = 0; i < SETTINGS_LIST_ROWS; i++ ) {
-		if ( ( widget->first_visible + i ) < widget->num_rows ) {
-			select_setting_row ( widget,
-					     widget->first_visible + i );
-			draw_setting_row ( widget );
+	for ( i = 0 ; i < SETTINGS_LIST_ROWS ; i++ ) {
+		if ( ( ui->scroll.first + i ) < ui->scroll.count ) {
+			select_setting_row ( ui, ( ui->scroll.first + i ) );
+			draw_setting_row ( ui );
 		} else {
 			clearmsg ( SETTINGS_LIST_ROW + i );
 		}
@@ -446,69 +422,72 @@ static void reveal_setting_row ( struct setting_widget *widget,
 }
 
 /**
- * Reveal setting row
+ * Select settings block
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  * @v settings		Settings block
  */
-static void init_widget ( struct setting_widget *widget,
-			  struct settings *settings ) {
-
-	widget->settings = settings_target ( settings );
-	widget->num_rows = select_setting_row ( widget, 0 );
-	widget->first_visible = SETTINGS_LIST_ROWS;
-	draw_title_row ( widget );
-	reveal_setting_row ( widget, 0 );
-	select_setting_row ( widget, 0 );
+static void select_settings ( struct settings_ui *ui,
+			      struct settings *settings ) {
+
+	ui->settings = settings_target ( settings );
+	ui->scroll.count = select_setting_row ( ui, 0 );
+	ui->scroll.rows = SETTINGS_LIST_ROWS;
+	ui->scroll.current = 0;
+	ui->scroll.first = 0;
+	draw_title_row ( ui );
+	draw_setting_rows ( ui );
+	select_setting_row ( ui, 0 );
 }
 
 static int main_loop ( struct settings *settings ) {
-	struct setting_widget widget;
+	struct settings_ui ui;
+	unsigned int previous;
 	int redraw = 1;
 	int move;
-	unsigned int next;
 	int key;
 	int rc;
 
 	/* Print initial screen content */
 	color_set ( CPAIR_NORMAL, NULL );
-	memset ( &widget, 0, sizeof ( widget ) );
-	init_widget ( &widget, settings );
+	memset ( &ui, 0, sizeof ( ui ) );
+	select_settings ( &ui, settings );
 
 	while ( 1 ) {
 
 		/* Redraw rows if necessary */
 		if ( redraw ) {
-			draw_info_row ( &widget );
-			draw_instruction_row ( &widget );
-			color_set ( ( widget.row.editing ?
+			draw_info_row ( &ui );
+			draw_instruction_row ( &ui );
+			color_set ( ( ui.row.editing ?
 				      CPAIR_EDIT : CPAIR_SELECT ), NULL );
-			draw_setting_row ( &widget );
+			draw_setting_row ( &ui );
 			color_set ( CPAIR_NORMAL, NULL );
-			curs_set ( widget.row.editing );
+			curs_set ( ui.row.editing );
 			redraw = 0;
 		}
 
-		if ( widget.row.editing ) {
+		/* Edit setting, if we are currently editing */
+		if ( ui.row.editing ) {
 
 			/* Sanity check */
-			assert ( widget.row.setting.name != NULL );
+			assert ( ui.row.setting.name != NULL );
 
 			/* Redraw edit box */
 			color_set ( CPAIR_EDIT, NULL );
-			draw_editbox ( &widget.row.editbox );
+			draw_editbox ( &ui.row.editbox );
 			color_set ( CPAIR_NORMAL, NULL );
 
 			/* Process keypress */
-			key = edit_setting ( &widget, getkey ( 0 ) );
+			key = edit_setting ( &ui, getkey ( 0 ) );
 			switch ( key ) {
 			case CR:
 			case LF:
-				if ( ( rc = save_setting ( &widget ) ) != 0 )
+				if ( ( rc = save_setting ( &ui ) ) != 0 )
 					alert ( " %s ", strerror ( rc ) );
 				/* Fall through */
 			case CTRL_C:
-				select_setting_row ( &widget, widget.current );
+				select_setting_row ( &ui, ui.scroll.current );
 				redraw = 1;
 				break;
 			default:
@@ -516,72 +495,52 @@ static int main_loop ( struct settings *settings ) {
 				break;
 			}
 
-		} else {
+			continue;
+		}
 
-			/* Process keypress */
-			key = getkey ( 0 );
-			move = 0;
-			switch ( key ) {
-			case KEY_UP:
-				move = -1;
-				break;
-			case KEY_DOWN:
-				move = +1;
-				break;
-			case KEY_PPAGE:
-				move = ( widget.first_visible -
-					 widget.current - 1 );
-				break;
-			case KEY_NPAGE:
-				move = ( widget.first_visible - widget.current
-					 + SETTINGS_LIST_ROWS );
-				break;
-			case KEY_HOME:
-				move = -widget.num_rows;
-				break;
-			case KEY_END:
-				move = +widget.num_rows;
-				break;
-			case CTRL_D:
-				if ( ! widget.row.setting.name )
-					break;
-				if ( ( rc = delete_setting ( widget.settings,
-						&widget.row.setting ) ) != 0 ) {
-					alert ( " %s ", strerror ( rc ) );
-				}
-				select_setting_row ( &widget, widget.current );
+		/* Otherwise, navigate through settings */
+		key = getkey ( 0 );
+		move = jump_scroll_key ( &ui.scroll, key );
+		if ( move ) {
+			previous = ui.scroll.current;
+			jump_scroll_move ( &ui.scroll, move );
+			if ( ui.scroll.current != previous ) {
+				draw_setting_row ( &ui );
 				redraw = 1;
+				if ( jump_scroll ( &ui.scroll ) )
+					draw_setting_rows ( &ui );
+				select_setting_row ( &ui, ui.scroll.current );
+			}
+			continue;
+		}
+
+		/* Handle non-navigation keys */
+		switch ( key ) {
+		case CTRL_D:
+			if ( ! ui.row.setting.name )
 				break;
-			case CTRL_X:
-				return 0;
-			case CR:
-			case LF:
-				if ( widget.row.settings ) {
-					init_widget ( &widget,
-						      widget.row.settings );
-					redraw = 1;
-				}
-				/* Fall through */
-			default:
-				if ( widget.row.setting.name ) {
-					edit_setting ( &widget, key );
-					redraw = 1;
-				}
-				break;
+			if ( ( rc = delete_setting ( ui.settings,
+						     &ui.row.setting ) ) != 0 ){
+				alert ( " %s ", strerror ( rc ) );
+			}
+			select_setting_row ( &ui, ui.scroll.current );
+			redraw = 1;
+			break;
+		case CTRL_X:
+			return 0;
+		case CR:
+		case LF:
+			if ( ui.row.settings ) {
+				select_settings ( &ui, ui.row.settings );
+				redraw = 1;
 			}
-			if ( move ) {
-				next = ( widget.current + move );
-				if ( ( int ) next < 0 )
-					next = 0;
-				if ( next >= widget.num_rows )
-					next = ( widget.num_rows - 1 );
-				if ( next != widget.current ) {
-					draw_setting_row ( &widget );
-					redraw = 1;
-					reveal_setting_row ( &widget, next );
-					select_setting_row ( &widget, next );
-				}
+			/* Fall through */
+		default:
+			if ( ui.row.setting.name ) {
+				edit_setting ( &ui, key );
+				redraw = 1;
 			}
+			break;
 		}
 	}
 }
diff --git a/roms/ipxe/src/image/elf.c b/roms/ipxe/src/image/elf.c
index 51636a8e9..5c2f9db25 100644
--- a/roms/ipxe/src/image/elf.c
+++ b/roms/ipxe/src/image/elf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -36,27 +40,54 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/image.h>
 #include <ipxe/elf.h>
 
-typedef Elf32_Ehdr	Elf_Ehdr;
-typedef Elf32_Phdr	Elf_Phdr;
-typedef Elf32_Off	Elf_Off;
-#define ELFCLASS	ELFCLASS32
-
 /**
  * Load ELF segment into memory
  *
  * @v image		ELF file
  * @v phdr		ELF program header
+ * @v dest		Destination address
+ * @ret rc		Return status code
+ */
+static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
+			      physaddr_t dest ) {
+	userptr_t buffer = phys_to_user ( dest );
+	int rc;
+
+	DBGC ( image, "ELF %p loading segment [%x,%x) to [%lx,%lx,%lx)\n",
+	       image, phdr->p_offset, ( phdr->p_offset + phdr->p_filesz ),
+	       dest, ( dest + phdr->p_filesz ), ( dest + phdr->p_memsz ) );
+
+	/* Verify and prepare segment */
+	if ( ( rc = prep_segment ( buffer, phdr->p_filesz,
+				   phdr->p_memsz ) ) != 0 ) {
+		DBGC ( image, "ELF %p could not prepare segment: %s\n",
+		       image, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Copy image to segment */
+	memcpy_user ( buffer, 0, image->data, phdr->p_offset, phdr->p_filesz );
+
+	return 0;
+}
+
+/**
+ * Process ELF segment
+ *
+ * @v image		ELF file
  * @v ehdr		ELF executable header
+ * @v phdr		ELF program header
+ * @v process		Segment processor
  * @ret entry		Entry point, if found
  * @ret max		Maximum used address
  * @ret rc		Return status code
  */
-static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
-			      Elf_Ehdr *ehdr, physaddr_t *entry,
-			      physaddr_t *max ) {
+static int elf_segment ( struct image *image, Elf_Ehdr *ehdr, Elf_Phdr *phdr,
+			 int ( * process ) ( struct image *image,
+					     Elf_Phdr *phdr, physaddr_t dest ),
+			 physaddr_t *entry, physaddr_t *max ) {
 	physaddr_t dest;
 	physaddr_t end;
-	userptr_t buffer;
 	unsigned long e_offset;
 	int rc;
 
@@ -82,28 +113,15 @@ static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
 		       image );
 		return -ENOEXEC;
 	}
-	buffer = phys_to_user ( dest );
 	end = ( dest + phdr->p_memsz );
 
-	DBGC ( image, "ELF %p loading segment [%x,%x) to [%x,%x,%x)\n", image,
-	       phdr->p_offset, ( phdr->p_offset + phdr->p_filesz ),
-	       phdr->p_paddr, ( phdr->p_paddr + phdr->p_filesz ),
-	       ( phdr->p_paddr + phdr->p_memsz ) );
-
-	/* Verify and prepare segment */
-	if ( ( rc = prep_segment ( buffer, phdr->p_filesz,
-				   phdr->p_memsz ) ) != 0 ) {
-		DBGC ( image, "ELF %p could not prepare segment: %s\n",
-		       image, strerror ( rc ) );
-		return rc;
-	}
-
 	/* Update maximum used address, if applicable */
 	if ( end > *max )
 		*max = end;
 
-	/* Copy image to segment */
-	memcpy_user ( buffer, 0, image->data, phdr->p_offset, phdr->p_filesz );
+	/* Process segment */
+	if ( ( rc = process ( image, phdr, dest ) ) != 0 )
+		return rc;
 
 	/* Set execution address, if it lies within this segment */
 	if ( ( e_offset = ( ehdr->e_entry - dest ) ) < phdr->p_filesz ) {
@@ -124,62 +142,85 @@ static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
 }
 
 /**
- * Load ELF image into memory
+ * Process ELF segments
  *
  * @v image		ELF file
- * @ret entry		Entry point
+ * @v ehdr		ELF executable header
+ * @v process		Segment processor
+ * @ret entry		Entry point, if found
  * @ret max		Maximum used address
  * @ret rc		Return status code
  */
-int elf_load ( struct image *image, physaddr_t *entry, physaddr_t *max ) {
-	static const uint8_t e_ident[] = {
-		[EI_MAG0]	= ELFMAG0,
-		[EI_MAG1]	= ELFMAG1,
-		[EI_MAG2]	= ELFMAG2,
-		[EI_MAG3]	= ELFMAG3,
-		[EI_CLASS]	= ELFCLASS,
-	};
-	Elf_Ehdr ehdr;
+int elf_segments ( struct image *image, Elf_Ehdr *ehdr,
+		   int ( * process ) ( struct image *image, Elf_Phdr *phdr,
+				       physaddr_t dest ),
+		   physaddr_t *entry, physaddr_t *max ) {
 	Elf_Phdr phdr;
 	Elf_Off phoff;
 	unsigned int phnum;
 	int rc;
 
-	/* Read ELF header */
-	copy_from_user ( &ehdr, image->data, 0, sizeof ( ehdr ) );
-	if ( memcmp ( &ehdr.e_ident[EI_MAG0], e_ident,
-		      sizeof ( e_ident ) ) != 0 ) {
-		DBGC ( image, "ELF %p has invalid signature\n", image );
-		return -ENOEXEC;
-	}
-
 	/* Initialise maximum used address */
 	*max = 0;
 
 	/* Invalidate entry point */
 	*entry = 0;
 
-	/* Read ELF program headers */
-	for ( phoff = ehdr.e_phoff , phnum = ehdr.e_phnum ; phnum ;
-	      phoff += ehdr.e_phentsize, phnum-- ) {
+	/* Read and process ELF program headers */
+	for ( phoff = ehdr->e_phoff , phnum = ehdr->e_phnum ; phnum ;
+	      phoff += ehdr->e_phentsize, phnum-- ) {
 		if ( phoff > image->len ) {
 			DBGC ( image, "ELF %p program header %d outside "
 			       "image\n", image, phnum );
 			return -ENOEXEC;
 		}
 		copy_from_user ( &phdr, image->data, phoff, sizeof ( phdr ) );
-		if ( ( rc = elf_load_segment ( image, &phdr, &ehdr,
-					       entry, max ) ) != 0 ) {
+		if ( ( rc = elf_segment ( image, ehdr, &phdr, process,
+					  entry, max ) ) != 0 )
 			return rc;
-		}
 	}
 
 	/* Check for a valid execution address */
 	if ( ! *entry ) {
 		DBGC ( image, "ELF %p entry point %lx outside image\n",
-		       image, ( ( unsigned long ) ehdr.e_entry ) );
+		       image, ( ( unsigned long ) ehdr->e_entry ) );
+		return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+/**
+ * Load ELF image into memory
+ *
+ * @v image		ELF file
+ * @ret entry		Entry point
+ * @ret max		Maximum used address
+ * @ret rc		Return status code
+ */
+int elf_load ( struct image *image, physaddr_t *entry, physaddr_t *max ) {
+	static const uint8_t e_ident[] = {
+		[EI_MAG0]	= ELFMAG0,
+		[EI_MAG1]	= ELFMAG1,
+		[EI_MAG2]	= ELFMAG2,
+		[EI_MAG3]	= ELFMAG3,
+		[EI_CLASS]	= ELFCLASS,
+	};
+	Elf_Ehdr ehdr;
+	int rc;
+
+	/* Read ELF header */
+	copy_from_user ( &ehdr, image->data, 0, sizeof ( ehdr ) );
+	if ( memcmp ( &ehdr.e_ident[EI_MAG0], e_ident,
+		      sizeof ( e_ident ) ) != 0 ) {
+		DBGC ( image, "ELF %p has invalid signature\n", image );
 		return -ENOEXEC;
 	}
 
+	/* Load ELF segments into memory */
+	if ( ( rc = elf_segments ( image, &ehdr, elf_load_segment,
+				   entry, max ) ) != 0 )
+		return rc;
+
 	return 0;
 }
diff --git a/roms/ipxe/src/image/embedded.c b/roms/ipxe/src/image/embedded.c
index 6358378fb..48dd86851 100644
--- a/roms/ipxe/src/image/embedded.c
+++ b/roms/ipxe/src/image/embedded.c
@@ -6,7 +6,7 @@
  * fetching over the network.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/image.h>
diff --git a/roms/ipxe/src/image/png.c b/roms/ipxe/src/image/png.c
index c14608553..5c4bcb3a0 100644
--- a/roms/ipxe/src/image/png.c
+++ b/roms/ipxe/src/image/png.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/image/pnm.c b/roms/ipxe/src/image/pnm.c
index af9e571a2..f24b28841 100644
--- a/roms/ipxe/src/image/pnm.c
+++ b/roms/ipxe/src/image/pnm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/image/script.c b/roms/ipxe/src/image/script.c
index 5328da8b4..28050868a 100644
--- a/roms/ipxe/src/image/script.c
+++ b/roms/ipxe/src/image/script.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/image/segment.c b/roms/ipxe/src/image/segment.c
index 86fe42662..2d0f2f0fc 100644
--- a/roms/ipxe/src/image/segment.c
+++ b/roms/ipxe/src/image/segment.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/include/.gitignore b/roms/ipxe/src/include/.gitignore
deleted file mode 100644
index de1598ef3..000000000
--- a/roms/ipxe/src/include/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-.buildserial.h
diff --git a/roms/ipxe/src/include/assert.h b/roms/ipxe/src/include/assert.h
index a33f6017c..07f3ecb8c 100644
--- a/roms/ipxe/src/include/assert.h
+++ b/roms/ipxe/src/include/assert.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NDEBUG
 #define ASSERTING 0
diff --git a/roms/ipxe/src/include/big_bswap.h b/roms/ipxe/src/include/big_bswap.h
deleted file mode 100644
index 6c375a573..000000000
--- a/roms/ipxe/src/include/big_bswap.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#ifndef ETHERBOOT_BIG_BSWAP_H
-#define ETHERBOOT_BIG_BSWAP_H
-
-#define htonll(x)	(x)
-#define ntohll(x)	(x)
-#define ntohl(x) 	(x)
-#define htonl(x) 	(x)
-#define ntohs(x) 	(x)
-#define htons(x) 	(x)
-#define cpu_to_le64(x)	__bswap_64(x)
-#define cpu_to_le32(x)	__bswap_32(x)
-#define cpu_to_le16(x)	__bswap_16(x)
-#define cpu_to_be64(x)	(x)
-#define cpu_to_be32(x)	(x)
-#define cpu_to_be16(x)	(x)
-#define le64_to_cpu(x)	__bswap_64(x)
-#define le32_to_cpu(x)	__bswap_32(x)
-#define le16_to_cpu(x)	__bswap_16(x)
-#define be64_to_cpu(x)	(x)
-#define be32_to_cpu(x)	(x)
-#define be16_to_cpu(x)	(x)
-#define cpu_to_le64s(x) __bswap_64s(x)
-#define cpu_to_le32s(x) __bswap_32s(x)
-#define cpu_to_le16s(x) __bswap_16s(x)
-#define cpu_to_be64s(x) do {} while (0)
-#define cpu_to_be32s(x) do {} while (0)
-#define cpu_to_be16s(x) do {} while (0)
-#define le64_to_cpus(x) __bswap_64s(x)
-#define le32_to_cpus(x) __bswap_32s(x)
-#define le16_to_cpus(x) __bswap_16s(x)
-#define be64_to_cpus(x) do {} while (0)
-#define be32_to_cpus(x) do {} while (0)
-#define be16_to_cpus(x) do {} while (0)
-
-#endif /* ETHERBOOT_BIG_BSWAP_H */
diff --git a/roms/ipxe/src/include/byteswap.h b/roms/ipxe/src/include/byteswap.h
index 466759cf8..d1028c579 100644
--- a/roms/ipxe/src/include/byteswap.h
+++ b/roms/ipxe/src/include/byteswap.h
@@ -1,59 +1,138 @@
-#ifndef ETHERBOOT_BYTESWAP_H
-#define ETHERBOOT_BYTESWAP_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include "endian.h"
-#include "bits/byteswap.h"
-
-#define __bswap_constant_16(x) \
-	((uint16_t)((((uint16_t)(x) & 0x00ff) << 8) | \
-		    (((uint16_t)(x) & 0xff00) >> 8)))
-
-#define __bswap_constant_32(x) \
-	((uint32_t)((((uint32_t)(x) & 0x000000ffU) << 24) | \
-		    (((uint32_t)(x) & 0x0000ff00U) <<  8) | \
-		    (((uint32_t)(x) & 0x00ff0000U) >>  8) | \
-		    (((uint32_t)(x) & 0xff000000U) >> 24)))
-
-#define __bswap_constant_64(x) \
-	((uint64_t)((((uint64_t)(x) & 0x00000000000000ffULL) << 56) | \
-		    (((uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \
-		    (((uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \
-		    (((uint64_t)(x) & 0x00000000ff000000ULL) <<  8) | \
-		    (((uint64_t)(x) & 0x000000ff00000000ULL) >>  8) | \
-		    (((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \
-		    (((uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \
-		    (((uint64_t)(x) & 0xff00000000000000ULL) >> 56)))
-
-#define __bswap_16(x) \
-	((uint16_t)(__builtin_constant_p(x) ? \
-	__bswap_constant_16(x) : \
-	__bswap_variable_16(x)))
-
-#define __bswap_32(x) \
-	((uint32_t)(__builtin_constant_p(x) ? \
-	__bswap_constant_32(x) : \
-	__bswap_variable_32(x)))
-
-#define __bswap_64(x) \
-	((uint64_t)(__builtin_constant_p(x) ? \
-	__bswap_constant_64(x) : \
-	__bswap_variable_64(x)))
+#ifndef BYTESWAP_H
+#define BYTESWAP_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <endian.h>
+#include <bits/byteswap.h>
+
+/**
+ * Byte-swap a 16-bit constant
+ *
+ * @v value		Constant value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_constant_16( value )					\
+	( ( ( (value) & 0x00ff ) << 8 ) |				\
+	  ( ( (value) & 0xff00 ) >> 8 ) )
+
+/**
+ * Byte-swap a 32-bit constant
+ *
+ * @v value		Constant value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_constant_32( value ) \
+	( ( ( (value) & 0x000000ffUL ) << 24 ) |			\
+	  ( ( (value) & 0x0000ff00UL ) <<  8 ) |			\
+	  ( ( (value) & 0x00ff0000UL ) >>  8 ) |			\
+	  ( ( (value) & 0xff000000UL ) >> 24 ) )
+
+/**
+ * Byte-swap a 64-bit constant
+ *
+ * @v value		Constant value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_constant_64( value )					\
+	( ( ( (value) & 0x00000000000000ffULL ) << 56 ) |		\
+	  ( ( (value) & 0x000000000000ff00ULL ) << 40 ) |		\
+	  ( ( (value) & 0x0000000000ff0000ULL ) << 24 ) |		\
+	  ( ( (value) & 0x00000000ff000000ULL ) <<  8 ) |		\
+	  ( ( (value) & 0x000000ff00000000ULL ) >>  8 ) |		\
+	  ( ( (value) & 0x0000ff0000000000ULL ) >> 24 ) |		\
+	  ( ( (value) & 0x00ff000000000000ULL ) >> 40 ) |		\
+	  ( ( (value) & 0xff00000000000000ULL ) >> 56 ) )
+
+/**
+ * Byte-swap a 16-bit value
+ *
+ * @v value		Value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_16( value )						\
+	( __builtin_constant_p (value) ?				\
+	  ( ( uint16_t ) __bswap_constant_16 ( ( uint16_t ) (value) ) ) \
+	  : __bswap_variable_16 (value) )
+#define bswap_16( value ) __bswap_16 (value)
+
+/**
+ * Byte-swap a 32-bit value
+ *
+ * @v value		Value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_32( value )						\
+	( __builtin_constant_p (value) ?				\
+	  ( ( uint32_t ) __bswap_constant_32 ( ( uint32_t ) (value) ) ) \
+	  : __bswap_variable_32 (value) )
+#define bswap_32( value ) __bswap_32 (value)
+
+/**
+ * Byte-swap a 64-bit value
+ *
+ * @v value		Value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_64( value )						\
+	( __builtin_constant_p (value) ?				\
+	  ( ( uint64_t ) __bswap_constant_64 ( ( uint64_t ) (value) ) ) \
+          : __bswap_variable_64 (value) )
+#define bswap_64( value ) __bswap_64 (value)
 
 #if __BYTE_ORDER == __LITTLE_ENDIAN
-#include "little_bswap.h"
+#define __cpu_to_leNN( bits, value ) (value)
+#define __cpu_to_beNN( bits, value ) __bswap_ ## bits (value)
+#define __leNN_to_cpu( bits, value ) (value)
+#define __beNN_to_cpu( bits, value ) __bswap_ ## bits (value)
+#define __cpu_to_leNNs( bits, ptr ) do { } while ( 0 )
+#define __cpu_to_beNNs( bits, ptr ) __bswap_ ## bits ## s (ptr)
+#define __leNN_to_cpus( bits, ptr ) do { } while ( 0 )
+#define __beNN_to_cpus( bits, ptr ) __bswap_ ## bits ## s (ptr)
 #endif
+
 #if __BYTE_ORDER == __BIG_ENDIAN
-#include "big_bswap.h"
+#define __cpu_to_leNN( bits, value ) __bswap_ ## bits (value)
+#define __cpu_to_beNN( bits, value ) (value)
+#define __leNN_to_cpu( bits, value ) __bswap_ ## bits (value)
+#define __beNN_to_cpu( bits, value ) (value)
+#define __cpu_to_leNNs( bits, ptr ) __bswap_ ## bits ## s (ptr)
+#define __cpu_to_beNNs( bits, ptr ) do { } while ( 0 )
+#define __leNN_to_cpus( bits, ptr ) __bswap_ ## bits ## s (ptr)
+#define __beNN_to_cpus( bits, ptr ) do { } while ( 0 )
 #endif
 
-/* Make routines available to all */
-#define swap64(x)	__bswap_64(x)
-#define swap32(x)	__bswap_32(x)
-#define swap16(x)	__bswap_16(x)
-#define bswap_64(x)	__bswap_64(x)
-#define bswap_32(x)	__bswap_32(x)
-#define bswap_16(x)	__bswap_16(x)
-	
-#endif /* ETHERBOOT_BYTESWAP_H */
+#define cpu_to_le16( value ) __cpu_to_leNN ( 16, value )
+#define cpu_to_le32( value ) __cpu_to_leNN ( 32, value )
+#define cpu_to_le64( value ) __cpu_to_leNN ( 64, value )
+#define cpu_to_be16( value ) __cpu_to_beNN ( 16, value )
+#define cpu_to_be32( value ) __cpu_to_beNN ( 32, value )
+#define cpu_to_be64( value ) __cpu_to_beNN ( 64, value )
+#define le16_to_cpu( value ) __leNN_to_cpu ( 16, value )
+#define le32_to_cpu( value ) __leNN_to_cpu ( 32, value )
+#define le64_to_cpu( value ) __leNN_to_cpu ( 64, value )
+#define be16_to_cpu( value ) __beNN_to_cpu ( 16, value )
+#define be32_to_cpu( value ) __beNN_to_cpu ( 32, value )
+#define be64_to_cpu( value ) __beNN_to_cpu ( 64, value )
+#define cpu_to_le16s( ptr ) __cpu_to_leNNs ( 16, ptr )
+#define cpu_to_le32s( ptr ) __cpu_to_leNNs ( 32, ptr )
+#define cpu_to_le64s( ptr ) __cpu_to_leNNs ( 64, ptr )
+#define cpu_to_be16s( ptr ) __cpu_to_beNNs ( 16, ptr )
+#define cpu_to_be32s( ptr ) __cpu_to_beNNs ( 32, ptr )
+#define cpu_to_be64s( ptr ) __cpu_to_beNNs ( 64, ptr )
+#define le16_to_cpus( ptr ) __leNN_to_cpus ( 16, ptr )
+#define le32_to_cpus( ptr ) __leNN_to_cpus ( 32, ptr )
+#define le64_to_cpus( ptr ) __leNN_to_cpus ( 64, ptr )
+#define be16_to_cpus( ptr ) __beNN_to_cpus ( 16, ptr )
+#define be32_to_cpus( ptr ) __beNN_to_cpus ( 32, ptr )
+#define be64_to_cpus( ptr ) __beNN_to_cpus ( 64, ptr )
+
+#define htonll( value ) cpu_to_be64 (value)
+#define ntohll( value ) be64_to_cpu (value)
+#define htonl( value ) cpu_to_be32 (value)
+#define ntohl( value ) be32_to_cpu (value)
+#define htons( value ) cpu_to_be16 (value)
+#define ntohs( value ) be16_to_cpu (value)
+
+#endif /* BYTESWAP_H */
diff --git a/roms/ipxe/src/include/compiler.h b/roms/ipxe/src/include/compiler.h
index 3f5c913a0..ca82f9523 100644
--- a/roms/ipxe/src/include/compiler.h
+++ b/roms/ipxe/src/include/compiler.h
@@ -57,101 +57,100 @@
  * @{
  */
 
-/** Provide a symbol within this object file */
+/**
+ * Provide a symbol within this object file
+ *
+ * @v symbol		Symbol name
+ */
 #ifdef ASSEMBLY
-#define PROVIDE_SYMBOL( _sym )				\
-	.section ".provided", "a", @nobits ;		\
-	.hidden _sym ;					\
-	.globl	_sym ;					\
-	_sym: ;						\
+#define PROVIDE_SYMBOL( symbol )				\
+	.section ".provided", "a", @nobits ;			\
+	.hidden symbol ;					\
+	.globl	symbol ;					\
+	symbol: ;						\
 	.previous
-#else /* ASSEMBLY */
-#define PROVIDE_SYMBOL( _sym )				\
-	char _sym[0]					\
+#else
+#define PROVIDE_SYMBOL( symbol )				\
+	char symbol[0]						\
 	  __attribute__ (( section ( ".provided" ) ))
-#endif /* ASSEMBLY */
+#endif
 
-/** Require a symbol within this object file
+/**
+ * Request a symbol
+ *
+ * @v symbol		Symbol name
  *
- * The symbol is referenced by a relocation in a discarded section, so
- * if it is not available at link time the link will fail.
+ * Request a symbol to be included within the link.  If the symbol
+ * cannot be found, the link will succeed anyway.
  */
 #ifdef ASSEMBLY
-#define REQUIRE_SYMBOL( _sym )				\
-	.section ".discard", "a", @progbits ;		\
-	.extern	_sym ;					\
-	.long	_sym ;					\
-	.previous
-#else /* ASSEMBLY */
-#define REQUIRE_SYMBOL( _sym )				\
-	extern char _sym;				\
-	static char * _C2 ( _C2 ( __require_, _sym ), _C2 ( _, __LINE__ ) ) \
-		__attribute__ (( section ( ".discard" ), used )) \
-		= &_sym
+#define REQUEST_SYMBOL( symbol )				\
+	.equ __request_ ## symbol, symbol
+#else
+#define REQUEST_SYMBOL( symbol )				\
+	__asm__ ( ".equ __request_" #symbol ", " #symbol )
 #endif
 
-/** Request that a symbol be available at runtime
+/**
+ * Require a symbol
+ *
+ * @v symbol		Symbol name
  *
- * The requested symbol is entered as undefined into the symbol table
- * for this object, so the linker will pull in other object files as
- * necessary to satisfy the reference. However, the undefined symbol
- * is not referenced in any relocations, so the link can still succeed
- * if no file contains it.
+ * Require a symbol to be included within the link.  If the symbol
+ * cannot be found, the link will fail.
  *
- * A symbol passed to this macro may not be referenced anywhere
- * else in the file. If you want to do that, see IMPORT_SYMBOL().
+ * To use this macro within a file, you must also specify the file's
+ * "requiring symbol" using the REQUIRING_SYMBOL() or
+ * PROVIDE_REQUIRING_SYMBOL() macros.
  */
 #ifdef ASSEMBLY
-#define REQUEST_SYMBOL( _sym )				\
-	.equ	__need_ ## _sym, _sym
-#else /* ASSEMBLY */
-#define REQUEST_SYMBOL( _sym )				\
-	__asm__ ( ".equ\t__need_" #_sym ", " #_sym )
-#endif /* ASSEMBLY */
+#define REQUIRE_SYMBOL( symbol )				\
+	.reloc __requiring_symbol__, RELOC_TYPE_NONE, symbol
+#else
+#define REQUIRE_SYMBOL( symbol )				\
+	__asm__ ( ".reloc __requiring_symbol__, "		\
+		  _S2 ( RELOC_TYPE_NONE ) ", " #symbol )
+#endif
 
-/** Set up a symbol to be usable in another file by IMPORT_SYMBOL()
+/**
+ * Specify the file's requiring symbol
+ *
+ * @v symbol		Symbol name
  *
- * The symbol must already be marked as global.
+ * REQUIRE_SYMBOL() works by defining a dummy relocation record
+ * against a nominated "requiring symbol".  The presence of the
+ * nominated requiring symbol will drag in all of the symbols
+ * specified using REQUIRE_SYMBOL().
  */
-#define EXPORT_SYMBOL( _sym )	PROVIDE_SYMBOL ( __export_ ## _sym )
+#ifdef ASSEMBLY
+#define REQUIRING_SYMBOL( symbol )				\
+	.equ __requiring_symbol__, symbol
+#else
+#define REQUIRING_SYMBOL( symbol )				\
+	__asm__ ( ".equ __requiring_symbol__, " #symbol )
+#endif
 
-/** Make a symbol usable to this file if available at link time
- *
- * If no file passed to the linker contains the symbol, it will have
- * @c NULL value to future uses. Keep in mind that the symbol value is
- * really the @e address of a variable or function; see the code
- * snippet below.
- *
- * In C using IMPORT_SYMBOL, you must specify the declaration as the
- * second argument, for instance
- *
- * @code
- *   IMPORT_SYMBOL ( my_func, int my_func ( int arg ) );
- *   IMPORT_SYMBOL ( my_var, int my_var );
- *
- *   void use_imports ( void ) {
- * 	if ( my_func && &my_var )
- * 	   my_var = my_func ( my_var );
- *   }
- * @endcode
- *
- * GCC considers a weak declaration to override a strong one no matter
- * which comes first, so it is safe to include a header file declaring
- * the imported symbol normally, but providing the declaration to
- * IMPORT_SYMBOL is still required.
+/**
+ * Provide a file's requiring symbol
  *
- * If no EXPORT_SYMBOL declaration exists for the imported symbol in
- * another file, the behavior will be most likely be identical to that
- * for an unavailable symbol.
+ * If the file contains no symbols that can be used as the requiring
+ * symbol, you can provide a dummy one-byte-long symbol using
+ * PROVIDE_REQUIRING_SYMBOL().
  */
 #ifdef ASSEMBLY
-#define IMPORT_SYMBOL( _sym )				\
-	REQUEST_SYMBOL ( __export_ ## _sym ) ;		\
-	.weak	_sym
-#else /* ASSEMBLY */
-#define IMPORT_SYMBOL( _sym, _decl )			\
-	REQUEST_SYMBOL ( __export_ ## _sym ) ;		\
-	extern _decl __attribute__ (( weak ))
+#define PROVIDE_REQUIRING_SYMBOL()				\
+	.section ".tbl.requiring_symbols", "a", @progbits ;	\
+	__requiring_symbol__:	.byte 0 ;			\
+	.size __requiring_symbol__, . - __requiring_symbol__ ;	\
+	.previous
+#else
+#define PROVIDE_REQUIRING_SYMBOL()				\
+	__asm__ ( ".section \".tbl.requiring_symbols\", "	\
+		  "         \"a\", @progbits\n"			\
+		  "__requiring_symbol__:\t.byte 0\n"		\
+		  ".size __requiring_symbol__, "		\
+		  "      . - __requiring_symbol__\n"		\
+		  ".previous" )
 #endif
 
 /** @} */
@@ -163,20 +162,33 @@
 
 #define PREFIX_OBJECT( _prefix ) _C2 ( _prefix, OBJECT )
 #define OBJECT_SYMBOL PREFIX_OBJECT ( obj_ )
-#define REQUEST_EXPANDED( _sym ) REQUEST_SYMBOL ( _sym )
-#define CONFIG_SYMBOL PREFIX_OBJECT ( obj_config_ )
 
 /** Always provide the symbol for the current object (defined by -DOBJECT) */
 PROVIDE_SYMBOL ( OBJECT_SYMBOL );
 
-/** Pull in an object-specific configuration file if available */
-REQUEST_EXPANDED ( CONFIG_SYMBOL );
-
-/** Explicitly require another object */
-#define REQUIRE_OBJECT( _obj ) REQUIRE_SYMBOL ( obj_ ## _obj )
+/**
+ * Request an object
+ *
+ * @v object		Object name
+ *
+ * Request an object to be included within the link.  If the object
+ * cannot be found, the link will succeed anyway.
+ */
+#define REQUEST_OBJECT( object ) REQUEST_SYMBOL ( obj_ ## object )
 
-/** Pull in another object if it exists */
-#define REQUEST_OBJECT( _obj ) REQUEST_SYMBOL ( obj_ ## _obj )
+/**
+ * Require an object
+ *
+ * @v object		Object name
+ *
+ * Require an object to be included within the link.  If the object
+ * cannot be found, the link will fail.
+ *
+ * To use this macro within a file, you must also specify the file's
+ * "requiring symbol" using the REQUIRING_SYMBOL() or
+ * PROVIDE_REQUIRING_SYMBOL() macros.
+ */
+#define REQUIRE_OBJECT( object ) REQUIRE_SYMBOL ( obj_ ## object )
 
 /** @} */
 
@@ -195,14 +207,6 @@ REQUEST_EXPANDED ( CONFIG_SYMBOL );
  */
 #define __weak		__attribute__ (( weak, noinline ))
 
-/** Prevent a function from being optimized away without inlining
- *
- * Calls to functions with void return type that contain no code in their body
- * may be removed by gcc's optimizer even when inlining is inhibited. Placing
- * this macro in the body of the function prevents that from occurring.
- */
-#define __keepme	asm("");
-
 #endif
 
 /** @defgroup dbg Debugging infrastructure
@@ -730,13 +734,24 @@ int __debug_disable;
 #define FILE_LICENCE_MIT \
 	PROVIDE_SYMBOL ( PREFIX_OBJECT ( __licence__mit__ ) )
 
+/** Declare a file as being under GPLv2+ or UBDL
+ *
+ * This licence declaration is applicable when a file states itself to
+ * be licensed under the GNU GPL; "either version 2 of the License, or
+ * (at your option) any later version" and also states that it may be
+ * distributed under the terms of the Unmodified Binary Distribution
+ * Licence (as given in the file COPYING.UBDL).
+ */
+#define FILE_LICENCE_GPL2_OR_LATER_OR_UBDL \
+	PROVIDE_SYMBOL ( PREFIX_OBJECT ( __licence__gpl2_or_later_or_ubdl__ ) )
+
 /** Declare a particular licence as applying to a file */
 #define FILE_LICENCE( _licence ) FILE_LICENCE_ ## _licence
 
 /** @} */
 
-/* This file itself is under GPLv2-or-later */
-FILE_LICENCE ( GPL2_OR_LATER );
+/* This file itself is under GPLv2+/UBDL */
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <bits/compiler.h>
 
diff --git a/roms/ipxe/src/include/ctype.h b/roms/ipxe/src/include/ctype.h
index e92ecb1c0..0d79ecd19 100644
--- a/roms/ipxe/src/include/ctype.h
+++ b/roms/ipxe/src/include/ctype.h
@@ -4,30 +4,114 @@
 /** @file
  *
  * Character types
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#define isdigit(c)	((c) >= '0' && (c) <= '9')
-#define islower(c)	((c) >= 'a' && (c) <= 'z')
-#define isupper(c)	((c) >= 'A' && (c) <= 'Z')
-#define isxdigit(c)	(isdigit(c) || ((c) >= 'A' && (c) <= 'F') || ((c) >= 'a' && (c) <= 'f'))
-#define isprint(c)	((c) >= ' ' && (c) <= '~' )
+/**
+ * Check if character is a decimal digit
+ *
+ * @v character		ASCII character
+ * @ret is_digit	Character is a decimal digit
+ */
+static inline int isdigit ( int character ) {
 
-static inline unsigned char tolower(unsigned char c)
-{
-	if (isupper(c))
-		c -= 'A'-'a';
-	return c;
+	return ( ( character >= '0' ) && ( character <= '9' ) );
 }
 
-static inline unsigned char toupper(unsigned char c)
-{
-	if (islower(c))
-		c -= 'a'-'A';
-	return c;
+/**
+ * Check if character is a hexadecimal digit
+ *
+ * @v character		ASCII character
+ * @ret is_xdigit	Character is a hexadecimal digit
+ */
+static inline int isxdigit ( int character ) {
+
+	return ( ( ( character >= '0' ) && ( character <= '9' ) ) ||
+		 ( ( character >= 'A' ) && ( character <= 'F' ) ) ||
+		 ( ( character >= 'a' ) && ( character <= 'f' ) ) );
+}
+
+/**
+ * Check if character is an upper-case letter
+ *
+ * @v character		ASCII character
+ * @ret is_upper	Character is an upper-case letter
+ */
+static inline int isupper ( int character ) {
+
+	return ( ( character >= 'A' ) && ( character <= 'Z' ) );
+}
+
+/**
+ * Check if character is a lower-case letter
+ *
+ * @v character		ASCII character
+ * @ret is_lower	Character is a lower-case letter
+ */
+static inline int islower ( int character ) {
+
+	return ( ( character >= 'a' ) && ( character <= 'z' ) );
+}
+
+/**
+ * Check if character is alphabetic
+ *
+ * @v character		ASCII character
+ * @ret is_alpha	Character is alphabetic
+ */
+static inline int isalpha ( int character ) {
+
+	return ( isupper ( character ) || islower ( character ) );
+}
+
+/**
+ * Check if character is alphanumeric
+ *
+ * @v character		ASCII character
+ * @ret is_alnum	Character is alphanumeric
+ */
+static inline int isalnum ( int character ) {
+
+	return ( isalpha ( character ) || isdigit ( character ) );
+}
+
+/**
+ * Check if character is printable
+ *
+ * @v character		ASCII character
+ * @ret is_print	Character is printable
+ */
+static inline int isprint ( int character ) {
+
+	return ( ( character >= ' ' ) && ( character <= '~' ) );
+}
+
+/**
+ * Convert character to lower case
+ *
+ * @v character		Character
+ * @v character		Lower-case character
+ */
+static inline int tolower ( int character ) {
+
+	return ( isupper ( character ) ?
+		 ( character - 'A' + 'a' ) : character );
+}
+
+/**
+ * Convert character to upper case
+ *
+ * @v character		Character
+ * @v character		Upper-case character
+ */
+static inline int toupper ( int character ) {
+
+	return ( islower ( character ) ?
+		 ( character - 'a' + 'A' ) : character );
 }
 
-extern int isspace ( int c );
+extern int isspace ( int character );
 
 #endif /* _CTYPE_H */
diff --git a/roms/ipxe/src/include/curses.h b/roms/ipxe/src/include/curses.h
index f16f9d7d0..04060fe27 100644
--- a/roms/ipxe/src/include/curses.h
+++ b/roms/ipxe/src/include/curses.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #undef  ERR
 #define ERR	(-1)
diff --git a/roms/ipxe/src/include/elf.h b/roms/ipxe/src/include/elf.h
index 04022b687..18f755a21 100644
--- a/roms/ipxe/src/include/elf.h
+++ b/roms/ipxe/src/include/elf.h
@@ -1,234 +1,81 @@
 #ifndef ELF_H
 #define ELF_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#define EI_NIDENT	16	/* Size of e_ident array. */
-
-/* Values for e_type. */
-#define ET_NONE		0	/* No file type */
-#define ET_REL		1	/* Relocatable file */
-#define ET_EXEC		2	/* Executable file */
-#define ET_DYN		3	/* Shared object file */
-#define ET_CORE		4	/* Core file */
-
-/* Values for e_machine (architecute). */
-#define EM_NONE		 0		/* No machine */
-#define EM_M32		 1		/* AT&T WE 32100 */
-#define EM_SPARC	 2		/* SUN SPARC */
-#define EM_386		 3		/* Intel 80386+ */
-#define EM_68K		 4		/* Motorola m68k family */
-#define EM_88K		 5		/* Motorola m88k family */
-#define EM_486		 6		/* Perhaps disused */
-#define EM_860		 7		/* Intel 80860 */
-#define EM_MIPS		 8		/* MIPS R3000 big-endian */
-#define EM_S370		 9		/* IBM System/370 */
-#define EM_MIPS_RS3_LE	10		/* MIPS R3000 little-endian */
-
-#define EM_PARISC	15		/* HPPA */
-#define EM_VPP500	17		/* Fujitsu VPP500 */
-#define EM_SPARC32PLUS	18		/* Sun's "v8plus" */
-#define EM_960		19		/* Intel 80960 */
-#define EM_PPC		20		/* PowerPC */
-#define EM_PPC64	21		/* PowerPC 64-bit */
-#define EM_S390		22		/* IBM S390 */
-
-#define EM_V800		36		/* NEC V800 series */
-#define EM_FR20		37		/* Fujitsu FR20 */
-#define EM_RH32		38		/* TRW RH-32 */
-#define EM_RCE		39		/* Motorola RCE */
-#define EM_ARM		40		/* ARM */
-#define EM_FAKE_ALPHA	41		/* Digital Alpha */
-#define EM_SH		42		/* Hitachi SH */
-#define EM_SPARCV9	43		/* SPARC v9 64-bit */
-#define EM_TRICORE	44		/* Siemens Tricore */
-#define EM_ARC		45		/* Argonaut RISC Core */
-#define EM_H8_300	46		/* Hitachi H8/300 */
-#define EM_H8_300H	47		/* Hitachi H8/300H */
-#define EM_H8S		48		/* Hitachi H8S */
-#define EM_H8_500	49		/* Hitachi H8/500 */
-#define EM_IA_64	50		/* Intel Merced */
-#define EM_MIPS_X	51		/* Stanford MIPS-X */
-#define EM_COLDFIRE	52		/* Motorola Coldfire */
-#define EM_68HC12	53		/* Motorola M68HC12 */
-#define EM_MMA		54		/* Fujitsu MMA Multimedia Accelerator*/
-#define EM_PCP		55		/* Siemens PCP */
-#define EM_NCPU		56		/* Sony nCPU embeeded RISC */
-#define EM_NDR1		57		/* Denso NDR1 microprocessor */
-#define EM_STARCORE	58		/* Motorola Start*Core processor */
-#define EM_ME16		59		/* Toyota ME16 processor */
-#define EM_ST100	60		/* STMicroelectronic ST100 processor */
-#define EM_TINYJ	61		/* Advanced Logic Corp. Tinyj emb.fam*/
-#define EM_X86_64	62		/* AMD x86-64 architecture */
-#define EM_PDSP		63		/* Sony DSP Processor */
-
-#define EM_FX66		66		/* Siemens FX66 microcontroller */
-#define EM_ST9PLUS	67		/* STMicroelectronics ST9+ 8/16 mc */
-#define EM_ST7		68		/* STmicroelectronics ST7 8 bit mc */
-#define EM_68HC16	69		/* Motorola MC68HC16 microcontroller */
-#define EM_68HC11	70		/* Motorola MC68HC11 microcontroller */
-#define EM_68HC08	71		/* Motorola MC68HC08 microcontroller */
-#define EM_68HC05	72		/* Motorola MC68HC05 microcontroller */
-#define EM_SVX		73		/* Silicon Graphics SVx */
-#define EM_AT19		74		/* STMicroelectronics ST19 8 bit mc */
-#define EM_VAX		75		/* Digital VAX */
-#define EM_CRIS		76		/* Axis Communications 32-bit embedded processor */
-#define EM_JAVELIN	77		/* Infineon Technologies 32-bit embedded processor */
-#define EM_FIREPATH	78		/* Element 14 64-bit DSP Processor */
-#define EM_ZSP		79		/* LSI Logic 16-bit DSP Processor */
-#define EM_MMIX		80		/* Donald Knuth's educational 64-bit processor */
-#define EM_HUANY	81		/* Harvard University machine-independent object files */
-#define EM_PRISM	82		/* SiTera Prism */
-#define EM_AVR		83		/* Atmel AVR 8-bit microcontroller */
-#define EM_FR30		84		/* Fujitsu FR30 */
-#define EM_D10V		85		/* Mitsubishi D10V */
-#define EM_D30V		86		/* Mitsubishi D30V */
-#define EM_V850		87		/* NEC v850 */
-#define EM_M32R		88		/* Mitsubishi M32R */
-#define EM_MN10300	89		/* Matsushita MN10300 */
-#define EM_MN10200	90		/* Matsushita MN10200 */
-#define EM_PJ		91		/* picoJava */
-#define EM_OPENRISC	92		/* OpenRISC 32-bit embedded processor */
-#define EM_ARC_A5	93		/* ARC Cores Tangent-A5 */
-#define EM_XTENSA	94		/* Tensilica Xtensa Architecture */
-#define EM_NUM		95
-
-/* Values for p_type. */
-#define PT_NULL		0	/* Unused entry. */
-#define PT_LOAD		1	/* Loadable segment. */
-#define PT_DYNAMIC	2	/* Dynamic linking information segment. */
-#define PT_INTERP	3	/* Pathname of interpreter. */
-#define PT_NOTE		4	/* Auxiliary information. */
-#define PT_SHLIB	5	/* Reserved (not used). */
-#define PT_PHDR		6	/* Location of program header itself. */
-
-/* Values for p_flags. */
-#define PF_X		0x1	/* Executable. */
-#define PF_W		0x2	/* Writable. */
-#define PF_R		0x4	/* Readable. */
-
-
-#define	ELF_PROGRAM_RETURNS_BIT	0x8000000	/* e_flags bit 31 */
-
-#define EI_MAG0		0
-#define ELFMAG0		0x7f
-
-#define EI_MAG1		1
-#define ELFMAG1		'E'
-
-#define EI_MAG2		2
-#define ELFMAG2		'L'
-
-#define EI_MAG3		3
-#define ELFMAG3		'F'
-
-#define ELFMAG		"\177ELF"
-#define	SELFMAG		4
-
-#define EI_CLASS	4	/* File class byte index */
-#define ELFCLASSNONE	0	/* Invalid class */
-#define ELFCLASS32	1	/* 32-bit objects */
-#define ELFCLASS64	2	/* 64-bit objects */
+/**
+ * @file
+ *
+ * ELF headers
+ *
+ */
 
-#define EI_DATA		5	/* Data encodeing byte index */
-#define ELFDATANONE	0	/* Invalid data encoding */
-#define ELFDATA2LSB	1	/* 2's complement little endian */
-#define ELFDATA2MSB	2	/* 2's complement big endian */
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#define EI_VERSION	6	/* File version byte index */
-				/* Value must be EV_CURRENT */
+#include <stdint.h>
 
-#define EV_NONE		0	/* Invalid ELF Version */
-#define EV_CURRENT	1	/* Current version */
+typedef uint32_t Elf32_Addr;
+typedef uint16_t Elf32_Half;
+typedef uint32_t Elf32_Off;
+typedef int32_t Elf32_Sword;
+typedef uint32_t Elf32_Word;
 
-#define ELF32_PHDR_SIZE (8*4)	/* Size of an elf program header */
+/** Length of ELF identifier */
+#define EI_NIDENT 16
 
-#ifndef ASSEMBLY
+/** ELF header */
+typedef struct {
+	unsigned char e_ident[EI_NIDENT];
+	Elf32_Half e_type;
+	Elf32_Half e_machine;
+	Elf32_Word e_version;
+	Elf32_Addr e_entry;
+	Elf32_Off e_phoff;
+	Elf32_Off e_shoff;
+	Elf32_Word e_flags;
+	Elf32_Half e_ehsize;
+	Elf32_Half e_phentsize;
+	Elf32_Half e_phnum;
+	Elf32_Half e_shentsize;
+	Elf32_Half e_shnum;
+	Elf32_Half e_shstrndx;
+} Elf32_Ehdr;
 
-#include <stdint.h>
+/* ELF identifier indexes */
+#define EI_MAG0 0
+#define EI_MAG1 1
+#define EI_MAG2 2
+#define EI_MAG3 3
+#define EI_CLASS 4
+#define EI_DATA 5
+#define EI_VERSION 6
 
-/*
- * ELF definitions common to all 32-bit architectures.
- */
+/* ELF magic signature bytes */
+#define ELFMAG0 0x7f
+#define ELFMAG1 'E'
+#define ELFMAG2 'L'
+#define ELFMAG3 'F'
 
-typedef uint32_t	Elf32_Addr;
-typedef uint16_t	Elf32_Half;
-typedef uint32_t	Elf32_Off;
-typedef int32_t		Elf32_Sword;
-typedef uint32_t	Elf32_Word;
-typedef uint32_t	Elf32_Size;
+/* ELF classes */
+#define ELFCLASS32 1
 
-typedef uint64_t	Elf64_Addr;
-typedef uint16_t	Elf64_Half;
-typedef uint64_t	Elf64_Off;
-typedef int32_t		Elf64_Sword;
-typedef uint32_t	Elf64_Word;
-typedef uint64_t	Elf64_Size;
+/* ELF data encodings */
+#define ELFDATA2LSB 1
 
-/*
- * ELF header.
- */
-typedef struct {
-	unsigned char	e_ident[EI_NIDENT];	/* File identification. */
-	Elf32_Half	e_type;		/* File type. */
-	Elf32_Half	e_machine;	/* Machine architecture. */
-	Elf32_Word	e_version;	/* ELF format version. */
-	Elf32_Addr	e_entry;	/* Entry point. */
-	Elf32_Off	e_phoff;	/* Program header file offset. */
-	Elf32_Off	e_shoff;	/* Section header file offset. */
-	Elf32_Word	e_flags;	/* Architecture-specific flags. */
-	Elf32_Half	e_ehsize;	/* Size of ELF header in bytes. */
-	Elf32_Half	e_phentsize;	/* Size of program header entry. */
-	Elf32_Half	e_phnum;	/* Number of program header entries. */
-	Elf32_Half	e_shentsize;	/* Size of section header entry. */
-	Elf32_Half	e_shnum;	/* Number of section header entries. */
-	Elf32_Half	e_shstrndx;	/* Section name strings section. */
-} Elf32_Ehdr;
-
-typedef struct {
-	unsigned char	e_ident[EI_NIDENT];	/* File identification. */
-	Elf64_Half	e_type;		/* File type. */
-	Elf64_Half	e_machine;	/* Machine architecture. */
-	Elf64_Word	e_version;	/* ELF format version. */
-	Elf64_Addr	e_entry;	/* Entry point. */
-	Elf64_Off	e_phoff;	/* Program header file offset. */
-	Elf64_Off	e_shoff;	/* Section header file offset. */
-	Elf64_Word	e_flags;	/* Architecture-specific flags. */
-	Elf64_Half	e_ehsize;	/* Size of ELF header in bytes. */
-	Elf64_Half	e_phentsize;	/* Size of program header entry. */
-	Elf64_Half	e_phnum;	/* Number of program header entries. */
-	Elf64_Half	e_shentsize;	/* Size of section header entry. */
-	Elf64_Half	e_shnum;	/* Number of section header entries. */
-	Elf64_Half	e_shstrndx;	/* Section name strings section. */
-} Elf64_Ehdr;
+/* ELF versions */
+#define EV_CURRENT 1
 
-/*
- * Program header.
- */
+/** ELF program header */
 typedef struct {
-	Elf32_Word	p_type;		/* Entry type. */
-	Elf32_Off	p_offset;	/* File offset of contents. */
-	Elf32_Addr	p_vaddr;	/* Virtual address (not used). */
-	Elf32_Addr	p_paddr;	/* Physical address. */
-	Elf32_Size	p_filesz;	/* Size of contents in file. */
-	Elf32_Size	p_memsz;	/* Size of contents in memory. */
-	Elf32_Word	p_flags;	/* Access permission flags. */
-	Elf32_Size	p_align;	/* Alignment in memory and file. */
+	Elf32_Word p_type;
+	Elf32_Off p_offset;
+	Elf32_Addr p_vaddr;
+	Elf32_Addr p_paddr;
+	Elf32_Word p_filesz;
+	Elf32_Word p_memsz;
+	Elf32_Word p_flags;
+	Elf32_Word p_align;
 } Elf32_Phdr;
 
-typedef struct {
-	Elf64_Word	p_type;		/* Entry type. */
-	Elf64_Word	p_flags;	/* Access permission flags. */
-	Elf64_Off	p_offset;	/* File offset of contents. */
-	Elf64_Addr	p_vaddr;	/* Virtual address (not used). */
-	Elf64_Addr	p_paddr;	/* Physical address. */
-	Elf64_Size	p_filesz;	/* Size of contents in file. */
-	Elf64_Size	p_memsz;	/* Size of contents in memory. */
-	Elf64_Size	p_align;	/* Alignment in memory and file. */
-} Elf64_Phdr;
-
-/* Standardized Elf image notes for booting... The name for all of these is ELFBoot */
-
-#endif /* ASSEMBLY */
+/* ELF segment types */
+#define PT_LOAD 1
 
 #endif /* ELF_H */
diff --git a/roms/ipxe/src/include/endian.h b/roms/ipxe/src/include/endian.h
index 9682cf9b4..79c3163ee 100644
--- a/roms/ipxe/src/include/endian.h
+++ b/roms/ipxe/src/include/endian.h
@@ -1,21 +1,22 @@
-#ifndef ETHERBOOT_ENDIAN_H
-#define ETHERBOOT_ENDIAN_H
+#ifndef _ENDIAN_H
+#define _ENDIAN_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-/* Definitions for byte order, according to significance of bytes,
-   from low addresses to high addresses.  The value is what you get by
-   putting '4' in the most significant byte, '3' in the second most
-   significant byte, '2' in the second least significant byte, and '1'
-   in the least significant byte, and then writing down one digit for
-   each byte, starting with the byte at the lowest address at the left,
-   and proceeding to the byte with the highest address at the right.  */
+/** Constant representing little-endian byte order
+ *
+ * Little-endian systems should define BYTE_ORDER as LITTLE_ENDIAN.
+ * This constant is intended to be used only at compile time.
+ */
+#define __LITTLE_ENDIAN 0x44332211UL
 
-#define __LITTLE_ENDIAN 1234
-#define __BIG_ENDIAN    4321
-#define __PDP_ENDIAN    3412
+/** Constant representing big-endian byte order
+ *
+ * Big-endian systems should define BYTE_ORDER as BIG_ENDIAN.
+ * This constant is intended to be used only at compile time.
+ */
+#define __BIG_ENDIAN 0x11223344UL
 
 #include "bits/endian.h"
 
-
-#endif /* ETHERBOOT_ENDIAN_H */
+#endif /* _ENDIAN_H */
diff --git a/roms/ipxe/src/include/errno.h b/roms/ipxe/src/include/errno.h
index bcc4a8816..036479aff 100644
--- a/roms/ipxe/src/include/errno.h
+++ b/roms/ipxe/src/include/errno.h
@@ -15,12 +15,16 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef ERRNO_H
 #define ERRNO_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/include/getopt.h b/roms/ipxe/src/include/getopt.h
index 0fe43567e..db3de1786 100644
--- a/roms/ipxe/src/include/getopt.h
+++ b/roms/ipxe/src/include/getopt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 
diff --git a/roms/ipxe/src/include/hci/ifmgmt_cmd.h b/roms/ipxe/src/include/hci/ifmgmt_cmd.h
index 913b911d8..5debf85c2 100644
--- a/roms/ipxe/src/include/hci/ifmgmt_cmd.h
+++ b/roms/ipxe/src/include/hci/ifmgmt_cmd.h
@@ -15,12 +15,16 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _IFMGMT_CMD_H
 #define _IFMGMT_CMD_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/parseopt.h>
 
diff --git a/roms/ipxe/src/include/ipxe/acpi.h b/roms/ipxe/src/include/ipxe/acpi.h
index 282b6d92d..2ccd691ed 100644
--- a/roms/ipxe/src/include/ipxe/acpi.h
+++ b/roms/ipxe/src/include/ipxe/acpi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/interface.h>
diff --git a/roms/ipxe/src/include/ipxe/aes.h b/roms/ipxe/src/include/ipxe/aes.h
index 4e44f9853..0432e43ee 100644
--- a/roms/ipxe/src/include/ipxe/aes.h
+++ b/roms/ipxe/src/include/ipxe/aes.h
@@ -1,31 +1,51 @@
 #ifndef _IPXE_AES_H
 #define _IPXE_AES_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+/** @file
+ *
+ * AES algorithm
+ *
+ */
 
-struct cipher_algorithm;
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-/** Basic AES blocksize */
+#include <ipxe/crypto.h>
+
+/** AES blocksize */
 #define AES_BLOCKSIZE 16
 
-#include "crypto/axtls/crypto.h"
+/** Maximum number of AES rounds */
+#define AES_MAX_ROUNDS 15
+
+/** AES matrix */
+union aes_matrix {
+	/** Viewed as an array of bytes */
+	uint8_t byte[16];
+	/** Viewed as an array of four-byte columns */
+	uint32_t column[4];
+} __attribute__ (( packed ));
+
+/** AES round keys */
+struct aes_round_keys {
+	/** Round keys */
+	union aes_matrix key[AES_MAX_ROUNDS];
+};
 
 /** AES context */
 struct aes_context {
-	/** AES context for AXTLS */
-	AES_CTX axtls_ctx;
-	/** Cipher is being used for decrypting */
-	int decrypting;
+	/** Encryption keys */
+	struct aes_round_keys encrypt;
+	/** Decryption keys */
+	struct aes_round_keys decrypt;
+	/** Number of rounds */
+	unsigned int rounds;
 };
 
 /** AES context size */
 #define AES_CTX_SIZE sizeof ( struct aes_context )
 
-/* AXTLS functions */
-extern void axtls_aes_encrypt ( const AES_CTX *ctx, uint32_t *data );
-extern void axtls_aes_decrypt ( const AES_CTX *ctx, uint32_t *data );
-
 extern struct cipher_algorithm aes_algorithm;
+extern struct cipher_algorithm aes_ecb_algorithm;
 extern struct cipher_algorithm aes_cbc_algorithm;
 
 int aes_wrap ( const void *kek, const void *src, void *dest, int nblk );
diff --git a/roms/ipxe/src/include/ipxe/ansicol.h b/roms/ipxe/src/include/ipxe/ansicol.h
index 707d1599d..2b54ecaca 100644
--- a/roms/ipxe/src/include/ipxe/ansicol.h
+++ b/roms/ipxe/src/include/ipxe/ansicol.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <curses.h> /* For COLOR_RED etc. */
diff --git a/roms/ipxe/src/include/ipxe/ansiesc.h b/roms/ipxe/src/include/ipxe/ansiesc.h
index c1c74481d..80bc83308 100644
--- a/roms/ipxe/src/include/ipxe/ansiesc.h
+++ b/roms/ipxe/src/include/ipxe/ansiesc.h
@@ -26,7 +26,7 @@
  * 
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct ansiesc_context;
 
diff --git a/roms/ipxe/src/include/ipxe/aoe.h b/roms/ipxe/src/include/ipxe/aoe.h
index 60f3bd959..0c656e7c2 100644
--- a/roms/ipxe/src/include/ipxe/aoe.h
+++ b/roms/ipxe/src/include/ipxe/aoe.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/roms/ipxe/src/include/ipxe/api.h b/roms/ipxe/src/include/ipxe/api.h
index 838b8936e..d05d3b07a 100644
--- a/roms/ipxe/src/include/ipxe/api.h
+++ b/roms/ipxe/src/include/ipxe/api.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @defgroup Single-implementation APIs
  *
diff --git a/roms/ipxe/src/include/ipxe/arp.h b/roms/ipxe/src/include/ipxe/arp.h
index e30ae6b76..5822fa095 100644
--- a/roms/ipxe/src/include/ipxe/arp.h
+++ b/roms/ipxe/src/include/ipxe/arp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 #include <ipxe/netdevice.h>
@@ -57,4 +57,8 @@ static inline int arp_tx ( struct io_buffer *iobuf, struct net_device *netdev,
 			      &arp_discovery, net_source, ll_source );
 }
 
+extern int arp_tx_request ( struct net_device *netdev,
+			    struct net_protocol *net_protocol,
+			    const void *net_dest, const void *net_source );
+
 #endif /* _IPXE_ARP_H */
diff --git a/roms/ipxe/src/include/ipxe/asn1.h b/roms/ipxe/src/include/ipxe/asn1.h
index d12524ddb..5fbd58281 100644
--- a/roms/ipxe/src/include/ipxe/asn1.h
+++ b/roms/ipxe/src/include/ipxe/asn1.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <time.h>
@@ -141,6 +141,24 @@ struct asn1_builder_header {
 	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
 	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 11 )
 
+/** ASN.1 OID for sha384WithRSAEncryption (1.2.840.113549.1.1.12) */
+#define ASN1_OID_SHA384WITHRSAENCRYPTION			\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 12 )
+
+/** ASN.1 OID for sha512WithRSAEncryption (1.2.840.113549.1.1.13) */
+#define ASN1_OID_SHA512WITHRSAENCRYPTION			\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 13 )
+
+/** ASN.1 OID for sha224WithRSAEncryption (1.2.840.113549.1.1.14) */
+#define ASN1_OID_SHA224WITHRSAENCRYPTION			\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 14 )
+
 /** ASN.1 OID for id-md5 (1.2.840.113549.2.5) */
 #define ASN1_OID_MD5						\
 	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
@@ -160,6 +178,41 @@ struct asn1_builder_header {
 	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
 	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 1 )
 
+/** ASN.1 OID for id-sha384 (2.16.840.1.101.3.4.2.2) */
+#define ASN1_OID_SHA384						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 2 )
+
+/** ASN.1 OID for id-sha512 (2.16.840.1.101.3.4.2.3) */
+#define ASN1_OID_SHA512						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 3 )
+
+/** ASN.1 OID for id-sha224 (2.16.840.1.101.3.4.2.4) */
+#define ASN1_OID_SHA224						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 4 )
+
+/** ASN.1 OID for id-sha512-224 (2.16.840.1.101.3.4.2.5) */
+#define ASN1_OID_SHA512_224						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 5 )
+
+/** ASN.1 OID for id-sha512-256 (2.16.840.1.101.3.4.2.6) */
+#define ASN1_OID_SHA512_256						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 6 )
+
 /** ASN.1 OID for commonName (2.5.4.3) */
 #define ASN1_OID_COMMON_NAME					\
 	ASN1_OID_INITIAL ( 2, 5 ), ASN1_OID_SINGLE ( 4 ),	\
diff --git a/roms/ipxe/src/include/ipxe/ata.h b/roms/ipxe/src/include/ipxe/ata.h
index b7f02d655..a10cfafcc 100644
--- a/roms/ipxe/src/include/ipxe/ata.h
+++ b/roms/ipxe/src/include/ipxe/ata.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * An ATA Logical Block Address
diff --git a/roms/ipxe/src/include/ipxe/base16.h b/roms/ipxe/src/include/ipxe/base16.h
index 60e3f2315..8c44da17e 100644
--- a/roms/ipxe/src/include/ipxe/base16.h
+++ b/roms/ipxe/src/include/ipxe/base16.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -32,9 +32,36 @@ static inline size_t base16_decoded_max_len ( const char *encoded ) {
 	return ( ( strlen ( encoded ) + 1 ) / 2 );
 }
 
-extern void base16_encode ( const uint8_t *raw, size_t len, char *encoded );
-extern int hex_decode ( const char *string, char separator, void *data,
+extern size_t hex_encode ( char separator, const void *raw, size_t raw_len,
+			   char *data, size_t len );
+extern int hex_decode ( char separator, const char *encoded, void *data,
 			size_t len );
-extern int base16_decode ( const char *encoded, uint8_t *raw );
+
+/**
+ * Base16-encode data
+ *
+ * @v raw		Raw data
+ * @v raw_len		Length of raw data
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Encoded length
+ */
+static inline __attribute__ (( always_inline )) size_t
+base16_encode ( const void *raw, size_t raw_len, char *data, size_t len ) {
+	return hex_encode ( 0, raw, raw_len, data, len );
+}
+
+/**
+ * Base16-decode data
+ *
+ * @v encoded		Encoded string
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of data, or negative error
+ */
+static inline __attribute__ (( always_inline )) int
+base16_decode ( const char *encoded, void *data, size_t len ) {
+	return hex_decode ( 0, encoded, data, len );
+}
 
 #endif /* _IPXE_BASE16_H */
diff --git a/roms/ipxe/src/include/ipxe/base64.h b/roms/ipxe/src/include/ipxe/base64.h
index 5fe134dc8..0c70d8382 100644
--- a/roms/ipxe/src/include/ipxe/base64.h
+++ b/roms/ipxe/src/include/ipxe/base64.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -35,7 +35,8 @@ static inline size_t base64_decoded_max_len ( const char *encoded ) {
 	return ( ( ( strlen ( encoded ) + 4 - 1 ) / 4 ) * 3 );
 }
 
-extern void base64_encode ( const uint8_t *raw, size_t len, char *encoded );
-extern int base64_decode ( const char *encoded, uint8_t *raw );
+extern size_t base64_encode ( const void *raw, size_t raw_len, char *data,
+			      size_t len );
+extern int base64_decode ( const char *encoded, void *data, size_t len );
 
 #endif /* _IPXE_BASE64_H */
diff --git a/roms/ipxe/src/include/ipxe/bigint.h b/roms/ipxe/src/include/ipxe/bigint.h
index 97fbce245..2f99f8445 100644
--- a/roms/ipxe/src/include/ipxe/bigint.h
+++ b/roms/ipxe/src/include/ipxe/bigint.h
@@ -6,7 +6,7 @@
  * Big integer support
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Define a big-integer type
diff --git a/roms/ipxe/src/include/ipxe/bitbash.h b/roms/ipxe/src/include/ipxe/bitbash.h
index 69d5d9e3e..2a2e475d0 100644
--- a/roms/ipxe/src/include/ipxe/bitbash.h
+++ b/roms/ipxe/src/include/ipxe/bitbash.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct bit_basher;
 
diff --git a/roms/ipxe/src/include/ipxe/bitmap.h b/roms/ipxe/src/include/ipxe/bitmap.h
index b18584c1f..38aca694b 100644
--- a/roms/ipxe/src/include/ipxe/bitmap.h
+++ b/roms/ipxe/src/include/ipxe/bitmap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/roms/ipxe/src/include/ipxe/bitops.h b/roms/ipxe/src/include/ipxe/bitops.h
index 73e859f41..220ab0fe7 100644
--- a/roms/ipxe/src/include/ipxe/bitops.h
+++ b/roms/ipxe/src/include/ipxe/bitops.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/include/ipxe/blockdev.h b/roms/ipxe/src/include/ipxe/blockdev.h
index 9f0a9f787..418c43004 100644
--- a/roms/ipxe/src/include/ipxe/blockdev.h
+++ b/roms/ipxe/src/include/ipxe/blockdev.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/roms/ipxe/src/include/ipxe/blocktrans.h b/roms/ipxe/src/include/ipxe/blocktrans.h
new file mode 100644
index 000000000..fee71b96c
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/blocktrans.h
@@ -0,0 +1,38 @@
+#ifndef _IPXE_BLOCKTRANS_H
+#define _IPXE_BLOCKTRANS_H
+
+/** @file
+ *
+ * Block device translator
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/uaccess.h>
+
+/** A block device translator */
+struct block_translator {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Block device interface */
+	struct interface block;
+	/** Data transfer interface */
+	struct interface xfer;
+
+	/** Data transfer buffer */
+	struct xfer_buffer xferbuf;
+	/** Data buffer */
+	userptr_t buffer;
+	/** Block size */
+	size_t blksize;
+};
+
+extern int block_translate ( struct interface *block,
+			     userptr_t buffer, size_t size );
+
+#endif /* _IPXE_BLOCKTRANS_H */
diff --git a/roms/ipxe/src/include/ipxe/bofm.h b/roms/ipxe/src/include/ipxe/bofm.h
index 1da47f651..bc994ea8b 100644
--- a/roms/ipxe/src/include/ipxe/bofm.h
+++ b/roms/ipxe/src/include/ipxe/bofm.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/roms/ipxe/src/include/ipxe/cbc.h b/roms/ipxe/src/include/ipxe/cbc.h
index fae376577..18a94e144 100644
--- a/roms/ipxe/src/include/ipxe/cbc.h
+++ b/roms/ipxe/src/include/ipxe/cbc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/crypto.h>
 
diff --git a/roms/ipxe/src/include/ipxe/cdc.h b/roms/ipxe/src/include/ipxe/cdc.h
new file mode 100644
index 000000000..f1799cd9a
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/cdc.h
@@ -0,0 +1,55 @@
+#ifndef _IPXE_CDC_H
+#define _IPXE_CDC_H
+
+/** @file
+ *
+ * USB Communications Device Class (CDC)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+
+/** Class code for communications devices */
+#define USB_CLASS_CDC 2
+
+/** Union functional descriptor */
+struct cdc_union_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Descriptor subtype */
+	uint8_t subtype;
+	/** Interfaces (variable-length) */
+	uint8_t interface[1];
+} __attribute__ (( packed ));
+
+/** Union functional descriptor subtype */
+#define CDC_SUBTYPE_UNION 6
+
+/** Ethernet descriptor subtype */
+#define CDC_SUBTYPE_ETHERNET 15
+
+/** Network connection notification */
+#define CDC_NETWORK_CONNECTION						\
+	( USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x00 ) )
+
+/** Connection speed change notification */
+#define CDC_CONNECTION_SPEED_CHANGE					\
+	( USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x2a ) )
+
+/** Connection speed change notification */
+struct cdc_connection_speed_change {
+	/** Downlink bit rate, in bits per second */
+	uint32_t down;
+	/** Uplink bit rate, in bits per second */
+	uint32_t up;
+} __attribute__ (( packed ));
+
+extern struct cdc_union_descriptor *
+cdc_union_descriptor ( struct usb_configuration_descriptor *config,
+		       struct usb_interface_descriptor *interface );
+
+#endif /* _IPXE_CDC_H */
diff --git a/roms/ipxe/src/include/ipxe/certstore.h b/roms/ipxe/src/include/ipxe/certstore.h
index 7456db621..49b3b512c 100644
--- a/roms/ipxe/src/include/ipxe/certstore.h
+++ b/roms/ipxe/src/include/ipxe/certstore.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/asn1.h>
 #include <ipxe/x509.h>
diff --git a/roms/ipxe/src/include/ipxe/chap.h b/roms/ipxe/src/include/ipxe/chap.h
index fce48f3ea..7c693e29d 100644
--- a/roms/ipxe/src/include/ipxe/chap.h
+++ b/roms/ipxe/src/include/ipxe/chap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/md5.h>
diff --git a/roms/ipxe/src/include/ipxe/cms.h b/roms/ipxe/src/include/ipxe/cms.h
index e026ebd2f..7adf724b2 100644
--- a/roms/ipxe/src/include/ipxe/cms.h
+++ b/roms/ipxe/src/include/ipxe/cms.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <time.h>
 #include <ipxe/asn1.h>
diff --git a/roms/ipxe/src/include/ipxe/command.h b/roms/ipxe/src/include/ipxe/command.h
index 432da1abb..a208e7d8f 100644
--- a/roms/ipxe/src/include/ipxe/command.h
+++ b/roms/ipxe/src/include/ipxe/command.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_COMMAND_H
 #define _IPXE_COMMAND_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/roms/ipxe/src/include/ipxe/console.h b/roms/ipxe/src/include/ipxe/console.h
index 4b90c9cec..1b764aaca 100644
--- a/roms/ipxe/src/include/ipxe/console.h
+++ b/roms/ipxe/src/include/ipxe/console.h
@@ -16,7 +16,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct pixel_buffer;
 
diff --git a/roms/ipxe/src/include/ipxe/cpio.h b/roms/ipxe/src/include/ipxe/cpio.h
index 277232808..0637c531d 100644
--- a/roms/ipxe/src/include/ipxe/cpio.h
+++ b/roms/ipxe/src/include/ipxe/cpio.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A CPIO archive header
  *
diff --git a/roms/ipxe/src/include/ipxe/crc32.h b/roms/ipxe/src/include/ipxe/crc32.h
index 38ac1b31f..30d2fe66c 100644
--- a/roms/ipxe/src/include/ipxe/crc32.h
+++ b/roms/ipxe/src/include/ipxe/crc32.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_CRC32_H
 #define _IPXE_CRC32_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/crypto.h b/roms/ipxe/src/include/ipxe/crypto.h
index 3eda5ec6e..fc0d8b22b 100644
--- a/roms/ipxe/src/include/ipxe/crypto.h
+++ b/roms/ipxe/src/include/ipxe/crypto.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/roms/ipxe/src/include/ipxe/deflate.h b/roms/ipxe/src/include/ipxe/deflate.h
index 19c5125eb..b751aa9a3 100644
--- a/roms/ipxe/src/include/ipxe/deflate.h
+++ b/roms/ipxe/src/include/ipxe/deflate.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/include/ipxe/device.h b/roms/ipxe/src/include/ipxe/device.h
index 7202a6966..d81417e8e 100644
--- a/roms/ipxe/src/include/ipxe/device.h
+++ b/roms/ipxe/src/include/ipxe/device.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/tables.h>
@@ -63,10 +63,16 @@ struct device_description {
 /** Xen bus type */
 #define BUS_TYPE_XEN 8
 
+/** Hyper-V bus type */
+#define BUS_TYPE_HV 9
+
+/** USB bus type */
+#define BUS_TYPE_USB 10
+
 /** A hardware device */
 struct device {
 	/** Name */
-	char name[16];
+	char name[32];
 	/** Driver name */
 	const char *driver_name;
 	/** Device description */
@@ -93,6 +99,8 @@ struct root_device {
 	struct device dev;
 	/** Root device driver */
 	struct root_driver *driver;
+	/** Driver-private data */
+	void *priv;
 };
 
 /** A root device driver */
@@ -123,6 +131,27 @@ struct root_driver {
 /** Declare a root device */
 #define __root_device __table_entry ( ROOT_DEVICES, 01 )
 
+/**
+ * Set root device driver-private data
+ *
+ * @v rootdev		Root device
+ * @v priv		Private data
+ */
+static inline void rootdev_set_drvdata ( struct root_device *rootdev,
+					 void *priv ){
+	rootdev->priv = priv;
+}
+
+/**
+ * Get root device driver-private data
+ *
+ * @v rootdev		Root device
+ * @ret priv		Private data
+ */
+static inline void * rootdev_get_drvdata ( struct root_device *rootdev ) {
+	return rootdev->priv;
+}
+
 extern int device_keep_count;
 
 /**
diff --git a/roms/ipxe/src/include/ipxe/dhcp.h b/roms/ipxe/src/include/ipxe/dhcp.h
index bcfb85cc1..a11db3497 100644
--- a/roms/ipxe/src/include/ipxe/dhcp.h
+++ b/roms/ipxe/src/include/ipxe/dhcp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
@@ -639,16 +639,6 @@ struct dhcphdr {
  */
 #define DHCP_MIN_LEN 552
 
-/** Timeouts for sending DHCP packets */
-#define DHCP_MIN_TIMEOUT ( 1 * TICKS_PER_SEC )
-#define DHCP_MAX_TIMEOUT ( 10 * TICKS_PER_SEC )
-
-/** Maximum time that we will wait for ProxyDHCP responses */
-#define PROXYDHCP_MAX_TIMEOUT ( 2 * TICKS_PER_SEC )
-
-/** Maximum time that we will wait for Boot Server responses */
-#define PXEBS_MAX_TIMEOUT ( 3 * TICKS_PER_SEC )
-
 /** Settings block name used for DHCP responses */
 #define DHCP_SETTINGS_NAME "dhcp"
 
diff --git a/roms/ipxe/src/include/ipxe/dhcpopts.h b/roms/ipxe/src/include/ipxe/dhcpopts.h
index c5af5d749..707fda4a8 100644
--- a/roms/ipxe/src/include/ipxe/dhcpopts.h
+++ b/roms/ipxe/src/include/ipxe/dhcpopts.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/dhcppkt.h b/roms/ipxe/src/include/ipxe/dhcppkt.h
index 3179a6bb0..f13dfc93d 100644
--- a/roms/ipxe/src/include/ipxe/dhcppkt.h
+++ b/roms/ipxe/src/include/ipxe/dhcppkt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 #include <ipxe/dhcpopts.h>
diff --git a/roms/ipxe/src/include/ipxe/dhcpv6.h b/roms/ipxe/src/include/ipxe/dhcpv6.h
index 2636b8ab2..9307b6cae 100644
--- a/roms/ipxe/src/include/ipxe/dhcpv6.h
+++ b/roms/ipxe/src/include/ipxe/dhcpv6.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/roms/ipxe/src/include/ipxe/dns.h b/roms/ipxe/src/include/ipxe/dns.h
index 4f6cab3a4..738dea6e4 100644
--- a/roms/ipxe/src/include/ipxe/dns.h
+++ b/roms/ipxe/src/include/ipxe/dns.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/roms/ipxe/src/include/ipxe/downloader.h b/roms/ipxe/src/include/ipxe/downloader.h
index de1a2e75e..ccb1abfef 100644
--- a/roms/ipxe/src/include/ipxe/downloader.h
+++ b/roms/ipxe/src/include/ipxe/downloader.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct interface;
 struct image;
diff --git a/roms/ipxe/src/include/ipxe/drbg.h b/roms/ipxe/src/include/ipxe/drbg.h
index 6374e7787..ed2b3757a 100644
--- a/roms/ipxe/src/include/ipxe/drbg.h
+++ b/roms/ipxe/src/include/ipxe/drbg.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/sha256.h>
diff --git a/roms/ipxe/src/include/ipxe/ecb.h b/roms/ipxe/src/include/ipxe/ecb.h
new file mode 100644
index 000000000..4e6aa3c81
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/ecb.h
@@ -0,0 +1,55 @@
+#ifndef _IPXE_ECB_H
+#define _IPXE_ECB_H
+
+/** @file
+ *
+ * Electronic codebook (ECB)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/crypto.h>
+
+extern void ecb_encrypt ( void *ctx, const void *src, void *dst,
+			  size_t len, struct cipher_algorithm *raw_cipher );
+extern void ecb_decrypt ( void *ctx, const void *src, void *dst,
+			  size_t len, struct cipher_algorithm *raw_cipher );
+
+/**
+ * Create a cipher-block chaining mode of behaviour of an existing cipher
+ *
+ * @v _ecb_name		Name for the new ECB cipher
+ * @v _ecb_cipher	New cipher algorithm
+ * @v _raw_cipher	Underlying cipher algorithm
+ * @v _raw_context	Context structure for the underlying cipher
+ * @v _blocksize	Cipher block size
+ */
+#define ECB_CIPHER( _ecb_name, _ecb_cipher, _raw_cipher, _raw_context,	\
+		    _blocksize )					\
+static int _ecb_name ## _setkey ( void *ctx, const void *key,		\
+				  size_t keylen ) {			\
+	return cipher_setkey ( &_raw_cipher, ctx, key, keylen );	\
+}									\
+static void _ecb_name ## _setiv ( void *ctx, const void *iv ) {		\
+	cipher_setiv ( &_raw_cipher, ctx, iv );				\
+}									\
+static void _ecb_name ## _encrypt ( void *ctx, const void *src,		\
+				    void *dst, size_t len ) {		\
+	ecb_encrypt ( ctx, src, dst, len, &_raw_cipher );		\
+}									\
+static void _ecb_name ## _decrypt ( void *ctx, const void *src,		\
+				    void *dst, size_t len ) {		\
+	ecb_decrypt ( ctx, src, dst, len, &_raw_cipher );		\
+}									\
+struct cipher_algorithm _ecb_cipher = {					\
+	.name		= #_ecb_name,					\
+	.ctxsize	= sizeof ( _raw_context ),			\
+	.blocksize	= _blocksize,					\
+	.setkey		= _ecb_name ## _setkey,				\
+	.setiv		= _ecb_name ## _setiv,				\
+	.encrypt	= _ecb_name ## _encrypt,			\
+	.decrypt	= _ecb_name ## _decrypt,			\
+};
+
+#endif /* _IPXE_ECB_H */
diff --git a/roms/ipxe/src/include/ipxe/edd.h b/roms/ipxe/src/include/ipxe/edd.h
index 0c25593d5..1914fd0b0 100644
--- a/roms/ipxe/src/include/ipxe/edd.h
+++ b/roms/ipxe/src/include/ipxe/edd.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/interface.h>
diff --git a/roms/ipxe/src/include/ipxe/editbox.h b/roms/ipxe/src/include/ipxe/editbox.h
index 9122dbbf3..2c70e0b6b 100644
--- a/roms/ipxe/src/include/ipxe/editbox.h
+++ b/roms/ipxe/src/include/ipxe/editbox.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <curses.h>
 #include <ipxe/editstring.h>
diff --git a/roms/ipxe/src/include/ipxe/editstring.h b/roms/ipxe/src/include/ipxe/editstring.h
index 2ef546a63..a00a8adaa 100644
--- a/roms/ipxe/src/include/ipxe/editstring.h
+++ b/roms/ipxe/src/include/ipxe/editstring.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** An editable string */
 struct edit_string {
diff --git a/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h b/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h
index 1294459f9..7466814fa 100644
--- a/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h
+++ b/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_EFI_PROCESSOR_BIND_H
 #define _IPXE_EFI_PROCESSOR_BIND_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * EFI header files rely on having the CPU architecture directory
diff --git a/roms/ipxe/src/include/ipxe/efi/Protocol/Rng.h b/roms/ipxe/src/include/ipxe/efi/Protocol/Rng.h
new file mode 100644
index 000000000..f04efbb03
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/efi/Protocol/Rng.h
@@ -0,0 +1,158 @@
+/** @file
+  EFI_RNG_PROTOCOL as defined in UEFI 2.4.
+  The UEFI Random Number Generator Protocol is used to provide random bits for use
+  in applications, or entropy for seeding other random number generators.
+
+Copyright (c) 2013, Intel Corporation. All rights reserved.<BR>
+This program and the accompanying materials are licensed and made available under
+the terms and conditions of the BSD License that accompanies this distribution.
+The full text of the license may be found at
+http://opensource.org/licenses/bsd-license.php.
+
+THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+
+**/
+
+#ifndef __EFI_RNG_PROTOCOL_H__
+#define __EFI_RNG_PROTOCOL_H__
+
+FILE_LICENCE ( BSD3 );
+
+///
+/// Global ID for the Random Number Generator Protocol
+///
+#define EFI_RNG_PROTOCOL_GUID \
+  { \
+    0x3152bca5, 0xeade, 0x433d, {0x86, 0x2e, 0xc0, 0x1c, 0xdc, 0x29, 0x1f, 0x44 } \
+  }
+
+typedef struct _EFI_RNG_PROTOCOL EFI_RNG_PROTOCOL;
+
+///
+/// A selection of EFI_RNG_PROTOCOL algorithms.
+/// The algorithms listed are optional, not meant to be exhaustive and be argmented by
+/// vendors or other industry standards.
+///
+
+typedef EFI_GUID EFI_RNG_ALGORITHM;
+
+///
+/// The algorithms corresponds to SP800-90 as defined in
+/// NIST SP 800-90, "Recommendation for Random Number Generation Using Deterministic Random
+/// Bit Generators", March 2007.
+///
+#define EFI_RNG_ALGORITHM_SP800_90_HASH_256_GUID \
+  { \
+    0xa7af67cb, 0x603b, 0x4d42, {0xba, 0x21, 0x70, 0xbf, 0xb6, 0x29, 0x3f, 0x96 } \
+  }
+#define EFI_RNG_ALGORITHM_SP800_90_HMAC_256_GUID \
+  { \
+    0xc5149b43, 0xae85, 0x4f53, {0x99, 0x82, 0xb9, 0x43, 0x35, 0xd3, 0xa9, 0xe7 } \
+  }
+#define EFI_RNG_ALGORITHM_SP800_90_CTR_256_GUID \
+  { \
+    0x44f0de6e, 0x4d8c, 0x4045, {0xa8, 0xc7, 0x4d, 0xd1, 0x68, 0x85, 0x6b, 0x9e } \
+  }
+///
+/// The algorithms correspond to X9.31 as defined in
+/// NIST, "Recommended Random Number Generator Based on ANSI X9.31 Appendix A.2.4 Using
+/// the 3-Key Triple DES and AES Algorithm", January 2005.
+///
+#define EFI_RNG_ALGORITHM_X9_31_3DES_GUID \
+  { \
+    0x63c4785a, 0xca34, 0x4012, {0xa3, 0xc8, 0x0b, 0x6a, 0x32, 0x4f, 0x55, 0x46 } \
+  }
+#define EFI_RNG_ALGORITHM_X9_31_AES_GUID \
+  { \
+    0xacd03321, 0x777e, 0x4d3d, {0xb1, 0xc8, 0x20, 0xcf, 0xd8, 0x88, 0x20, 0xc9 } \
+  }
+///
+/// The "raw" algorithm, when supported, is intended to provide entropy directly from
+/// the source, without it going through some deterministic random bit generator.
+///
+#define EFI_RNG_ALGORITHM_RAW \
+  { \
+    0xe43176d7, 0xb6e8, 0x4827, {0xb7, 0x84, 0x7f, 0xfd, 0xc4, 0xb6, 0x85, 0x61 } \
+  }
+
+/**
+  Returns information about the random number generation implementation.
+
+  @param[in]     This                 A pointer to the EFI_RNG_PROTOCOL instance.
+  @param[in,out] RNGAlgorithmListSize On input, the size in bytes of RNGAlgorithmList.
+                                      On output with a return code of EFI_SUCCESS, the size
+                                      in bytes of the data returned in RNGAlgorithmList. On output
+                                      with a return code of EFI_BUFFER_TOO_SMALL,
+                                      the size of RNGAlgorithmList required to obtain the list.
+  @param[out] RNGAlgorithmList        A caller-allocated memory buffer filled by the driver
+                                      with one EFI_RNG_ALGORITHM element for each supported
+                                      RNG algorithm. The list must not change across multiple
+                                      calls to the same driver. The first algorithm in the list
+                                      is the default algorithm for the driver.
+
+  @retval EFI_SUCCESS                 The RNG algorithm list was returned successfully.
+  @retval EFI_UNSUPPORTED             The services is not supported by this driver.
+  @retval EFI_DEVICE_ERROR            The list of algorithms could not be retrieved due to a
+                                      hardware or firmware error.
+  @retval EFI_INVALID_PARAMETER       One or more of the parameters are incorrect.
+  @retval EFI_BUFFER_TOO_SMALL        The buffer RNGAlgorithmList is too small to hold the result.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_RNG_GET_INFO) (
+  IN EFI_RNG_PROTOCOL             *This,
+  IN OUT UINTN                    *RNGAlgorithmListSize,
+  OUT EFI_RNG_ALGORITHM           *RNGAlgorithmList
+  );
+
+/**
+  Produces and returns an RNG value using either the default or specified RNG algorithm.
+
+  @param[in]  This                    A pointer to the EFI_RNG_PROTOCOL instance.
+  @param[in]  RNGAlgorithm            A pointer to the EFI_RNG_ALGORITHM that identifies the RNG
+                                      algorithm to use. May be NULL in which case the function will
+                                      use its default RNG algorithm.
+  @param[in]  RNGValueLength          The length in bytes of the memory buffer pointed to by
+                                      RNGValue. The driver shall return exactly this numbers of bytes.
+  @param[out] RNGValue                A caller-allocated memory buffer filled by the driver with the
+                                      resulting RNG value.
+
+  @retval EFI_SUCCESS                 The RNG value was returned successfully.
+  @retval EFI_UNSUPPORTED             The algorithm specified by RNGAlgorithm is not supported by
+                                      this driver.
+  @retval EFI_DEVICE_ERROR            An RNG value could not be retrieved due to a hardware or
+                                      firmware error.
+  @retval EFI_NOT_READY               There is not enough random data available to satisfy the length
+                                      requested by RNGValueLength.
+  @retval EFI_INVALID_PARAMETER       RNGValue is NULL or RNGValueLength is zero.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_RNG_GET_RNG) (
+  IN EFI_RNG_PROTOCOL            *This,
+  IN EFI_RNG_ALGORITHM           *RNGAlgorithm, OPTIONAL
+  IN UINTN                       RNGValueLength,
+  OUT UINT8                      *RNGValue
+  );
+
+///
+/// The Random Number Generator (RNG) protocol provides random bits for use in
+/// applications, or entropy for seeding other random number generators.
+///
+struct _EFI_RNG_PROTOCOL {
+  EFI_RNG_GET_INFO                GetInfo;
+  EFI_RNG_GET_RNG                 GetRNG;
+};
+
+extern EFI_GUID gEfiRngProtocolGuid;
+extern EFI_GUID gEfiRngAlgorithmSp80090Hash256Guid;
+extern EFI_GUID gEfiRngAlgorithmSp80090Hmac256Guid;
+extern EFI_GUID gEfiRngAlgorithmSp80090Ctr256Guid;
+extern EFI_GUID gEfiRngAlgorithmX9313DesGuid;
+extern EFI_GUID gEfiRngAlgorithmX931AesGuid;
+extern EFI_GUID gEfiRngAlgorithmRaw;
+
+#endif
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h b/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h
index d4a26850c..1d5ddc8c3 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void efi_set_autoboot ( void );
 
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_driver.h b/roms/ipxe/src/include/ipxe/efi/efi_driver.h
index e16a24daa..f497df3e3 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_driver.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_driver.h
@@ -6,7 +6,7 @@
  * EFI driver interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 #include <ipxe/tables.h>
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_entropy.h b/roms/ipxe/src/include/ipxe/efi/efi_entropy.h
new file mode 100644
index 000000000..39a667355
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/efi/efi_entropy.h
@@ -0,0 +1,35 @@
+#ifndef _IPXE_EFI_ENTROPY_H
+#define _IPXE_EFI_ENTROPY_H
+
+/** @file
+ *
+ * EFI entropy source
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+#ifdef ENTROPY_EFI
+#define ENTROPY_PREFIX_efi
+#else
+#define ENTROPY_PREFIX_efi __efi_
+#endif
+
+/**
+ * min-entropy per sample
+ *
+ * @ret min_entropy	min-entropy of each sample
+ */
+static inline __always_inline double
+ENTROPY_INLINE ( efi, min_entropy_per_sample ) ( void ) {
+
+	/* We use essentially the same mechanism as for the BIOS
+	 * RTC-based entropy source, and so assume the same
+	 * min-entropy per sample.
+	 */
+	return 1.3;
+}
+
+#endif /* _IPXE_EFI_ENTROPY_H */
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_hii.h b/roms/ipxe/src/include/ipxe/efi/efi_hii.h
index 8e94bbe7e..bbec31194 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_hii.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_hii.h
@@ -6,7 +6,7 @@
  * EFI human interface infrastructure
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/efi/Uefi/UefiInternalFormRepresentation.h>
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_pci.h b/roms/ipxe/src/include/ipxe/efi/efi_pci.h
index af36613d9..6dd945f05 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_pci.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_pci.h
@@ -6,7 +6,7 @@
  * EFI driver interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/pci.h>
 #include <ipxe/efi/efi.h>
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h b/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h
index 498a0388b..887d5ee14 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef PCIAPI_EFI
 #define PCIAPI_PREFIX_efi
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_reboot.h b/roms/ipxe/src/include/ipxe/efi/efi_reboot.h
index 33921b913..249cae8c5 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_reboot.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef REBOOT_EFI
 #define REBOOT_PREFIX_efi
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_smbios.h b/roms/ipxe/src/include/ipxe/efi/efi_smbios.h
index 7642e5bc5..d890d5460 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_smbios.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SMBIOS_EFI
 #define SMBIOS_PREFIX_efi
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_snp.h b/roms/ipxe/src/include/ipxe/efi/efi_snp.h
index a18bced5f..1e5c66626 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_snp.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_snp.h
@@ -18,6 +18,9 @@
 #include <ipxe/efi/Protocol/HiiDatabase.h>
 #include <ipxe/efi/Protocol/LoadFile.h>
 
+/** SNP transmit completion ring size */
+#define EFI_SNP_NUM_TX 32
+
 /** An SNP device */
 struct efi_snp_device {
 	/** List of SNP devices */
@@ -34,20 +37,16 @@ struct efi_snp_device {
 	EFI_SIMPLE_NETWORK_MODE mode;
 	/** Started flag */
 	int started;
-	/** Outstanding TX packet count (via "interrupt status")
-	 *
-	 * Used in order to generate TX completions.
-	 */
-	unsigned int tx_count_interrupts;
-	/** Outstanding TX packet count (via "recycled tx buffers")
-	 *
-	 * Used in order to generate TX completions.
-	 */
-	unsigned int tx_count_txbufs;
-	/** Outstanding RX packet count (via "interrupt status") */
-	unsigned int rx_count_interrupts;
-	/** Outstanding RX packet count (via WaitForPacket event) */
-	unsigned int rx_count_events;
+	/** Pending interrupt status */
+	unsigned int interrupts;
+	/** Transmit completion ring */
+	VOID *tx[EFI_SNP_NUM_TX];
+	/** Transmit completion ring producer counter */
+	unsigned int tx_prod;
+	/** Transmit completion ring consumer counter */
+	unsigned int tx_cons;
+	/** Receive queue */
+	struct list_head rx;
 	/** The network interface identifier */
 	EFI_NETWORK_INTERFACE_IDENTIFIER_PROTOCOL nii;
 	/** Component name protocol */
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_strings.h b/roms/ipxe/src/include/ipxe/efi/efi_strings.h
index 023ccda07..2f241537e 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_strings.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_strings.h
@@ -6,7 +6,7 @@
  * EFI strings
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_time.h b/roms/ipxe/src/include/ipxe/efi/efi_time.h
new file mode 100644
index 000000000..099994b57
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/efi/efi_time.h
@@ -0,0 +1,20 @@
+#ifndef _IPXE_EFI_TIME_H
+#define _IPXE_EFI_TIME_H
+
+/** @file
+ *
+ * EFI time source
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+#ifdef TIME_EFI
+#define TIME_PREFIX_efi
+#else
+#define TIME_PREFIX_efi __efi_
+#endif
+
+#endif /* _IPXE_EFI_TIME_H */
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_timer.h b/roms/ipxe/src/include/ipxe/efi/efi_timer.h
index b10543d6c..c03765393 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_timer.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_EFI
 #define TIMER_PREFIX_efi
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h b/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h
index 870a089b2..3cc750405 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h
@@ -10,7 +10,7 @@
  * no-ops.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef UACCESS_EFI
 #define UACCESS_PREFIX_efi
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h b/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h
index 911e69a96..4eb2a5f9b 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef UMALLOC_EFI
 #define UMALLOC_PREFIX_efi
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_utils.h b/roms/ipxe/src/include/ipxe/efi/efi_utils.h
index 9164be190..57268daf7 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_utils.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_utils.h
@@ -6,7 +6,7 @@
  * EFI utilities
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/Protocol/DevicePath.h>
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_watchdog.h b/roms/ipxe/src/include/ipxe/efi/efi_watchdog.h
new file mode 100644
index 000000000..4a56b9a29
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/efi/efi_watchdog.h
@@ -0,0 +1,31 @@
+#ifndef _IPXE_EFI_WATCHDOG_H
+#define _IPXE_EFI_WATCHDOG_H
+
+/** @file
+ *
+ * EFI watchdog holdoff timer
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern struct retry_timer efi_watchdog;
+
+/**
+ * Start EFI watchdog holdoff timer
+ *
+ */
+static inline void efi_watchdog_start ( void ) {
+
+	start_timer_nodelay ( &efi_watchdog );
+}
+
+/**
+ * Stop EFI watchdog holdoff timer
+ *
+ */
+static inline void efi_watchdog_stop ( void ) {
+
+	stop_timer ( &efi_watchdog );
+}
+
+#endif /* _IPXE_EFI_WATCHDOG_H */
diff --git a/roms/ipxe/src/include/ipxe/efi/efi_wrap.h b/roms/ipxe/src/include/ipxe/efi/efi_wrap.h
index 7579e0fe9..d8ed1a5cc 100644
--- a/roms/ipxe/src/include/ipxe/efi/efi_wrap.h
+++ b/roms/ipxe/src/include/ipxe/efi/efi_wrap.h
@@ -6,7 +6,7 @@
  * EFI driver interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 
diff --git a/roms/ipxe/src/include/ipxe/eisa.h b/roms/ipxe/src/include/ipxe/eisa.h
index 22a1ed94e..e7dac1f39 100644
--- a/roms/ipxe/src/include/ipxe/eisa.h
+++ b/roms/ipxe/src/include/ipxe/eisa.h
@@ -1,7 +1,7 @@
 #ifndef EISA_H
 #define EISA_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/isa_ids.h>
diff --git a/roms/ipxe/src/include/ipxe/elf.h b/roms/ipxe/src/include/ipxe/elf.h
index ec675c047..033c3f7a8 100644
--- a/roms/ipxe/src/include/ipxe/elf.h
+++ b/roms/ipxe/src/include/ipxe/elf.h
@@ -8,10 +8,21 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stdint.h>
+#include <ipxe/image.h>
 #include <elf.h>
 
+typedef Elf32_Ehdr	Elf_Ehdr;
+typedef Elf32_Phdr	Elf_Phdr;
+typedef Elf32_Off	Elf_Off;
+#define ELFCLASS	ELFCLASS32
+
+extern int elf_segments ( struct image *image, Elf_Ehdr *ehdr,
+			  int ( * process ) ( struct image *image,
+					      Elf_Phdr *phdr, physaddr_t dest ),
+			  physaddr_t *entry, physaddr_t *max );
 extern int elf_load ( struct image *image, physaddr_t *entry, physaddr_t *max );
 
 #endif /* _IPXE_ELF_H */
diff --git a/roms/ipxe/src/include/ipxe/eltorito.h b/roms/ipxe/src/include/ipxe/eltorito.h
index 3302b38b6..27e361b16 100644
--- a/roms/ipxe/src/include/ipxe/eltorito.h
+++ b/roms/ipxe/src/include/ipxe/eltorito.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/iso9660.h>
diff --git a/roms/ipxe/src/include/ipxe/entropy.h b/roms/ipxe/src/include/ipxe/entropy.h
index adf325e79..beeb3abfa 100644
--- a/roms/ipxe/src/include/ipxe/entropy.h
+++ b/roms/ipxe/src/include/ipxe/entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -54,6 +54,7 @@ typedef uint8_t entropy_sample_t;
 
 /* Include all architecture-independent entropy API headers */
 #include <ipxe/null_entropy.h>
+#include <ipxe/efi/efi_entropy.h>
 #include <ipxe/linux/linux_entropy.h>
 
 /* Include all architecture-dependent entropy API headers */
diff --git a/roms/ipxe/src/include/ipxe/errfile.h b/roms/ipxe/src/include/ipxe/errfile.h
index f809337ff..e21c95938 100644
--- a/roms/ipxe/src/include/ipxe/errfile.h
+++ b/roms/ipxe/src/include/ipxe/errfile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <bits/errfile.h>
 
@@ -68,6 +68,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_fbcon		       ( ERRFILE_CORE | 0x001c0000 )
 #define ERRFILE_ansicol		       ( ERRFILE_CORE | 0x001d0000 )
 #define ERRFILE_ansicoldef	       ( ERRFILE_CORE | 0x001e0000 )
+#define ERRFILE_fault		       ( ERRFILE_CORE | 0x001f0000 )
+#define ERRFILE_blocktrans	       ( ERRFILE_CORE | 0x00200000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
@@ -76,12 +78,20 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_pci		     ( ERRFILE_DRIVER | 0x00040000 )
 #define ERRFILE_linux		     ( ERRFILE_DRIVER | 0x00050000 )
 #define ERRFILE_pcivpd		     ( ERRFILE_DRIVER | 0x00060000 )
+#define ERRFILE_usb		     ( ERRFILE_DRIVER | 0x00070000 )
+#define ERRFILE_usbhub		     ( ERRFILE_DRIVER | 0x00080000 )
+#define ERRFILE_xhci		     ( ERRFILE_DRIVER | 0x00090000 )
+#define ERRFILE_ehci		     ( ERRFILE_DRIVER | 0x000a0000 )
+#define ERRFILE_uhci		     ( ERRFILE_DRIVER | 0x000b0000 )
+#define ERRFILE_usbhid		     ( ERRFILE_DRIVER | 0x000c0000 )
+#define ERRFILE_usbkbd		     ( ERRFILE_DRIVER | 0x000d0000 )
 
 #define ERRFILE_nvs		     ( ERRFILE_DRIVER | 0x00100000 )
 #define ERRFILE_spi		     ( ERRFILE_DRIVER | 0x00110000 )
 #define ERRFILE_i2c_bit		     ( ERRFILE_DRIVER | 0x00120000 )
 #define ERRFILE_spi_bit		     ( ERRFILE_DRIVER | 0x00130000 )
 #define ERRFILE_nvsvpd		     ( ERRFILE_DRIVER | 0x00140000 )
+#define ERRFILE_uart		     ( ERRFILE_DRIVER | 0x00150000 )
 
 #define ERRFILE_3c509		     ( ERRFILE_DRIVER | 0x00200000 )
 #define ERRFILE_bnx2		     ( ERRFILE_DRIVER | 0x00210000 )
@@ -157,7 +167,11 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_snp		     ( ERRFILE_DRIVER | 0x00680000 )
 #define ERRFILE_netfront	     ( ERRFILE_DRIVER | 0x00690000 )
 #define ERRFILE_nii		     ( ERRFILE_DRIVER | 0x006a0000 )
-
+#define ERRFILE_netvsc		     ( ERRFILE_DRIVER | 0x006b0000 )
+#define ERRFILE_ecm		     ( ERRFILE_DRIVER | 0x006c0000 )
+#define ERRFILE_ncm		     ( ERRFILE_DRIVER | 0x006d0000 )
+#define ERRFILE_usbnet		     ( ERRFILE_DRIVER | 0x006e0000 )
+#define ERRFILE_dm96xx		     ( ERRFILE_DRIVER | 0x006f0000 )
 #define ERRFILE_scsi		     ( ERRFILE_DRIVER | 0x00700000 )
 #define ERRFILE_arbel		     ( ERRFILE_DRIVER | 0x00710000 )
 #define ERRFILE_hermon		     ( ERRFILE_DRIVER | 0x00720000 )
@@ -165,6 +179,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_ata		     ( ERRFILE_DRIVER | 0x00740000 )
 #define ERRFILE_srp		     ( ERRFILE_DRIVER | 0x00750000 )
 #define ERRFILE_qib7322		     ( ERRFILE_DRIVER | 0x00760000 )
+#define ERRFILE_smsc75xx	     ( ERRFILE_DRIVER | 0x00770000 )
+#define ERRFILE_intelvf		     ( ERRFILE_DRIVER | 0x00780000 )
+#define ERRFILE_intelxvf	     ( ERRFILE_DRIVER | 0x00790000 )
 
 #define ERRFILE_aoe			( ERRFILE_NET | 0x00000000 )
 #define ERRFILE_arp			( ERRFILE_NET | 0x00010000 )
@@ -227,6 +244,17 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_ping			( ERRFILE_NET | 0x003a0000 )
 #define ERRFILE_dhcpv6			( ERRFILE_NET | 0x003b0000 )
 #define ERRFILE_nfs_uri			( ERRFILE_NET | 0x003c0000 )
+#define ERRFILE_rndis			( ERRFILE_NET | 0x003d0000 )
+#define ERRFILE_pccrc			( ERRFILE_NET | 0x003e0000 )
+#define ERRFILE_stp			( ERRFILE_NET | 0x003f0000 )
+#define ERRFILE_pccrd			( ERRFILE_NET | 0x00400000 )
+#define ERRFILE_httpconn		( ERRFILE_NET | 0x00410000 )
+#define ERRFILE_httpauth		( ERRFILE_NET | 0x00420000 )
+#define ERRFILE_httpbasic		( ERRFILE_NET | 0x00430000 )
+#define ERRFILE_httpdigest		( ERRFILE_NET | 0x00440000 )
+#define ERRFILE_peerdisc		( ERRFILE_NET | 0x00450000 )
+#define ERRFILE_peerblk			( ERRFILE_NET | 0x00460000 )
+#define ERRFILE_peermux			( ERRFILE_NET | 0x00470000 )
 
 #define ERRFILE_image		      ( ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_elf		      ( ERRFILE_IMAGE | 0x00010000 )
@@ -245,7 +273,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_imgmgmt		      ( ERRFILE_OTHER | 0x00050000 )
 #define ERRFILE_pxe_tftp	      ( ERRFILE_OTHER | 0x00060000 )
 #define ERRFILE_pxe_udp		      ( ERRFILE_OTHER | 0x00070000 )
-#define ERRFILE_axtls_aes	      ( ERRFILE_OTHER | 0x00080000 )
+#define ERRFILE_aes	 	      ( ERRFILE_OTHER | 0x00080000 )
 #define ERRFILE_cipher		      ( ERRFILE_OTHER | 0x00090000 )
 #define ERRFILE_image_cmd	      ( ERRFILE_OTHER | 0x000a0000 )
 #define ERRFILE_uri_test	      ( ERRFILE_OTHER | 0x000b0000 )
@@ -308,6 +336,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_xengrant	      ( ERRFILE_OTHER | 0x00440000 )
 #define ERRFILE_efi_utils	      ( ERRFILE_OTHER | 0x00450000 )
 #define ERRFILE_efi_wrap	      ( ERRFILE_OTHER | 0x00460000 )
+#define ERRFILE_vmbus		      ( ERRFILE_OTHER | 0x00470000 )
+#define ERRFILE_efi_time	      ( ERRFILE_OTHER | 0x00480000 )
+#define ERRFILE_efi_watchdog	      ( ERRFILE_OTHER | 0x00490000 )
 
 /** @} */
 
diff --git a/roms/ipxe/src/include/ipxe/errno/efi.h b/roms/ipxe/src/include/ipxe/errno/efi.h
index 2d2c50176..9f010f5fb 100644
--- a/roms/ipxe/src/include/ipxe/errno/efi.h
+++ b/roms/ipxe/src/include/ipxe/errno/efi.h
@@ -21,7 +21,7 @@
  * as-is.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/Uefi/UefiBaseType.h>
diff --git a/roms/ipxe/src/include/ipxe/errno/linux.h b/roms/ipxe/src/include/ipxe/errno/linux.h
index 11309b4ad..99133c816 100644
--- a/roms/ipxe/src/include/ipxe/errno/linux.h
+++ b/roms/ipxe/src/include/ipxe/errno/linux.h
@@ -10,7 +10,7 @@
  * directly as our platform error codes.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Convert platform error code to platform component of iPXE error code
diff --git a/roms/ipxe/src/include/ipxe/errortab.h b/roms/ipxe/src/include/ipxe/errortab.h
index a2f6a70f5..4fe81a6be 100644
--- a/roms/ipxe/src/include/ipxe/errortab.h
+++ b/roms/ipxe/src/include/ipxe/errortab.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/roms/ipxe/src/include/ipxe/eth_slow.h b/roms/ipxe/src/include/ipxe/eth_slow.h
index 00509197d..f6d731b3b 100644
--- a/roms/ipxe/src/include/ipxe/eth_slow.h
+++ b/roms/ipxe/src/include/ipxe/eth_slow.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Slow protocols header */
 struct eth_slow_header {
diff --git a/roms/ipxe/src/include/ipxe/ethernet.h b/roms/ipxe/src/include/ipxe/ethernet.h
index d1263d7c3..dd04e00ce 100644
--- a/roms/ipxe/src/include/ipxe/ethernet.h
+++ b/roms/ipxe/src/include/ipxe/ethernet.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/netdevice.h>
diff --git a/roms/ipxe/src/include/ipxe/fakedhcp.h b/roms/ipxe/src/include/ipxe/fakedhcp.h
index ea06b06dc..d016b5237 100644
--- a/roms/ipxe/src/include/ipxe/fakedhcp.h
+++ b/roms/ipxe/src/include/ipxe/fakedhcp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/fault.h b/roms/ipxe/src/include/ipxe/fault.h
new file mode 100644
index 000000000..356296c35
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/fault.h
@@ -0,0 +1,53 @@
+#ifndef _IPXE_FAULT_H
+#define _IPXE_FAULT_H
+
+/** @file
+ *
+ * Fault injection
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <config/fault.h>
+
+extern int inject_fault_nonzero ( unsigned int rate );
+extern void inject_corruption_nonzero ( unsigned int rate, const void *data,
+					size_t len );
+
+/**
+ * Inject fault with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (zero for no faults)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+inject_fault ( unsigned int rate ) {
+
+	/* Force dead code elimination in non-fault-injecting builds */
+	if ( rate == 0 )
+		return 0;
+
+	return inject_fault_nonzero ( rate );
+}
+
+/**
+ * Corrupt data with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (zero for no faults)
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) void
+inject_corruption ( unsigned int rate, const void *data, size_t len ) {
+
+	/* Force dead code elimination in non-fault-injecting builds */
+	if ( rate == 0 )
+		return;
+
+	return inject_corruption_nonzero ( rate, data, len );
+}
+
+#endif /* _IPXE_FAULT_H */
diff --git a/roms/ipxe/src/include/ipxe/fbcon.h b/roms/ipxe/src/include/ipxe/fbcon.h
index 0538449ac..d442bb918 100644
--- a/roms/ipxe/src/include/ipxe/fbcon.h
+++ b/roms/ipxe/src/include/ipxe/fbcon.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/ansiesc.h>
diff --git a/roms/ipxe/src/include/ipxe/fc.h b/roms/ipxe/src/include/ipxe/fc.h
index 6fdef092d..840d11f62 100644
--- a/roms/ipxe/src/include/ipxe/fc.h
+++ b/roms/ipxe/src/include/ipxe/fc.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/include/ipxe/fcels.h b/roms/ipxe/src/include/ipxe/fcels.h
index 45fa69a4a..02f755115 100644
--- a/roms/ipxe/src/include/ipxe/fcels.h
+++ b/roms/ipxe/src/include/ipxe/fcels.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/roms/ipxe/src/include/ipxe/fcns.h b/roms/ipxe/src/include/ipxe/fcns.h
index e25d9b9d5..9011a7be7 100644
--- a/roms/ipxe/src/include/ipxe/fcns.h
+++ b/roms/ipxe/src/include/ipxe/fcns.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/roms/ipxe/src/include/ipxe/fcoe.h b/roms/ipxe/src/include/ipxe/fcoe.h
index 6ba5b406a..b61e82fea 100644
--- a/roms/ipxe/src/include/ipxe/fcoe.h
+++ b/roms/ipxe/src/include/ipxe/fcoe.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/roms/ipxe/src/include/ipxe/fcp.h b/roms/ipxe/src/include/ipxe/fcp.h
index f6922bc7c..853ca13f6 100644
--- a/roms/ipxe/src/include/ipxe/fcp.h
+++ b/roms/ipxe/src/include/ipxe/fcp.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/roms/ipxe/src/include/ipxe/features.h b/roms/ipxe/src/include/ipxe/features.h
index d8b8b2184..e86a2d226 100644
--- a/roms/ipxe/src/include/ipxe/features.h
+++ b/roms/ipxe/src/include/ipxe/features.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @defgroup featurecat Feature categories
diff --git a/roms/ipxe/src/include/ipxe/fragment.h b/roms/ipxe/src/include/ipxe/fragment.h
index e311ad1e4..0069e5e08 100644
--- a/roms/ipxe/src/include/ipxe/fragment.h
+++ b/roms/ipxe/src/include/ipxe/fragment.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/roms/ipxe/src/include/ipxe/ftp.h b/roms/ipxe/src/include/ipxe/ftp.h
index cbab12d2c..3180f1631 100644
--- a/roms/ipxe/src/include/ipxe/ftp.h
+++ b/roms/ipxe/src/include/ipxe/ftp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** FTP default port */
 #define FTP_PORT 21
diff --git a/roms/ipxe/src/include/ipxe/gdbserial.h b/roms/ipxe/src/include/ipxe/gdbserial.h
index a3b56173c..e1040c94e 100644
--- a/roms/ipxe/src/include/ipxe/gdbserial.h
+++ b/roms/ipxe/src/include/ipxe/gdbserial.h
@@ -7,15 +7,14 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
 
 struct gdb_transport;
 
-/**
- * Set up the serial transport
- *
- * @ret transport suitable for starting the GDB stub or NULL on error
- */
-struct gdb_transport *gdbserial_configure ( void );
+extern struct gdb_transport * gdbserial_configure ( unsigned int port,
+						    unsigned int baud,
+						    uint8_t lcr );
 
 #endif /* _IPXE_GDBSERIAL_H */
diff --git a/roms/ipxe/src/include/ipxe/gdbstub.h b/roms/ipxe/src/include/ipxe/gdbstub.h
index 319606747..13ca33ddb 100644
--- a/roms/ipxe/src/include/ipxe/gdbstub.h
+++ b/roms/ipxe/src/include/ipxe/gdbstub.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
diff --git a/roms/ipxe/src/include/ipxe/gdbudp.h b/roms/ipxe/src/include/ipxe/gdbudp.h
index db7a451c9..a1c091522 100644
--- a/roms/ipxe/src/include/ipxe/gdbudp.h
+++ b/roms/ipxe/src/include/ipxe/gdbudp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct sockaddr_in;
 struct gdb_transport;
diff --git a/roms/ipxe/src/include/ipxe/hash_df.h b/roms/ipxe/src/include/ipxe/hash_df.h
index 607a4a610..e57682446 100644
--- a/roms/ipxe/src/include/ipxe/hash_df.h
+++ b/roms/ipxe/src/include/ipxe/hash_df.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/roms/ipxe/src/include/ipxe/hidemem.h b/roms/ipxe/src/include/ipxe/hidemem.h
index ddc9cd8b3..cc8d5ee37 100644
--- a/roms/ipxe/src/include/ipxe/hidemem.h
+++ b/roms/ipxe/src/include/ipxe/hidemem.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/hmac.h b/roms/ipxe/src/include/ipxe/hmac.h
index d5ec0868d..09d3e273d 100644
--- a/roms/ipxe/src/include/ipxe/hmac.h
+++ b/roms/ipxe/src/include/ipxe/hmac.h
@@ -6,7 +6,7 @@
  * Keyed-Hashing for Message Authentication
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/crypto.h>
 
diff --git a/roms/ipxe/src/include/ipxe/hmac_drbg.h b/roms/ipxe/src/include/ipxe/hmac_drbg.h
index 8dfd2924f..a0f22da75 100644
--- a/roms/ipxe/src/include/ipxe/hmac_drbg.h
+++ b/roms/ipxe/src/include/ipxe/hmac_drbg.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/roms/ipxe/src/include/ipxe/http.h b/roms/ipxe/src/include/ipxe/http.h
index cf8c0c7fa..a0dff7d00 100644
--- a/roms/ipxe/src/include/ipxe/http.h
+++ b/roms/ipxe/src/include/ipxe/http.h
@@ -7,7 +7,26 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/process.h>
+#include <ipxe/retry.h>
+#include <ipxe/linebuf.h>
+#include <ipxe/pool.h>
+#include <ipxe/tables.h>
+
+struct http_transaction;
+
+/******************************************************************************
+ *
+ * HTTP URI schemes
+ *
+ ******************************************************************************
+ */
 
 /** HTTP default port */
 #define HTTP_PORT 80
@@ -15,10 +34,469 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /** HTTPS default port */
 #define HTTPS_PORT 443
 
-extern int http_open_filter ( struct interface *xfer, struct uri *uri,
-			      unsigned int default_port,
-			      int ( * filter ) ( struct interface *,
-						 const char *,
-						 struct interface ** ) );
+/** An HTTP URI scheme */
+struct http_scheme {
+	/** Scheme name (e.g. "http" or "https") */
+	const char *name;
+	/** Default port */
+	unsigned int port;
+	/** Transport-layer filter (if any)
+	 *
+	 * @v xfer		Data transfer interface
+	 * @v name		Host name
+	 * @v next		Next interface
+	 * @ret rc		Return status code
+	 */
+	int ( * filter ) ( struct interface *xfer, const char *name,
+			   struct interface **next );
+};
+
+/** HTTP scheme table */
+#define HTTP_SCHEMES __table ( struct http_scheme, "http_schemes" )
+
+/** Declare an HTTP scheme */
+#define __http_scheme __table_entry ( HTTP_SCHEMES, 01 )
+
+/******************************************************************************
+ *
+ * Connections
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP connection
+ *
+ * This represents a potentially reusable connection to an HTTP
+ * server.
+ */
+struct http_connection {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Connection URI
+	 *
+	 * This encapsulates the server (and protocol) used for the
+	 * connection.  This may be the origin server or a proxy
+	 * server.
+	 */
+	struct uri *uri;
+	/** HTTP scheme */
+	struct http_scheme *scheme;
+	/** Transport layer interface */
+	struct interface socket;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Pooled connection */
+	struct pooled_connection pool;
+};
+
+/******************************************************************************
+ *
+ * HTTP methods
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP method */
+struct http_method {
+	/** Method name (e.g. "GET" or "POST") */
+	const char *name;
+};
+
+extern struct http_method http_head;
+extern struct http_method http_get;
+extern struct http_method http_post;
+
+/******************************************************************************
+ *
+ * Requests
+ *
+ ******************************************************************************
+ */
+
+/** HTTP Digest authentication client nonce count
+ *
+ * We choose to generate a new client nonce each time.
+ */
+#define HTTP_DIGEST_NC "00000001"
+
+/** HTTP Digest authentication client nonce length
+ *
+ * We choose to use a 32-bit hex client nonce.
+ */
+#define HTTP_DIGEST_CNONCE_LEN 8
+
+/** HTTP Digest authentication response length
+ *
+ * The Digest authentication response is a Base16-encoded 16-byte MD5
+ * checksum.
+ */
+#define HTTP_DIGEST_RESPONSE_LEN 32
+
+/** HTTP request range descriptor */
+struct http_request_range {
+	/** Range start */
+	size_t start;
+	/** Range length, or zero for no range request */
+	size_t len;
+};
+
+/** HTTP request content descriptor */
+struct http_request_content {
+	/** Content type (if any) */
+	const char *type;
+	/** Content data (if any) */
+	const void *data;
+	/** Content length */
+	size_t len;
+};
+
+/** HTTP request authentication descriptor */
+struct http_request_auth {
+	/** Authentication scheme (if any) */
+	struct http_authentication *auth;
+	/** Username */
+	const char *username;
+	/** Password */
+	const char *password;
+	/** Quality of protection */
+	const char *qop;
+	/** Algorithm */
+	const char *algorithm;
+	/** Client nonce */
+	char cnonce[ HTTP_DIGEST_CNONCE_LEN + 1 /* NUL */ ];
+	/** Response */
+	char response[ HTTP_DIGEST_RESPONSE_LEN + 1 /* NUL */ ];
+};
+
+/** An HTTP request
+ *
+ * This represents a single request to be sent to a server, including
+ * the values required to construct all headers.
+ *
+ * Pointers within this structure must point to storage which is
+ * guaranteed to remain valid for the lifetime of the containing HTTP
+ * transaction.
+ */
+struct http_request {
+	/** Method */
+	struct http_method *method;
+	/** Request URI string */
+	const char *uri;
+	/** Server host name */
+	const char *host;
+	/** Range descriptor */
+	struct http_request_range range;
+	/** Content descriptor */
+	struct http_request_content content;
+	/** Authentication descriptor */
+	struct http_request_auth auth;
+};
+
+/** An HTTP request header */
+struct http_request_header {
+	/** Header name (e.g. "User-Agent") */
+	const char *name;
+	/** Construct remaining header line
+	 *
+	 * @v http		HTTP transaction
+	 * @v buf		Buffer
+	 * @v len		Length of buffer
+	 * @ret len		Header length if present, or negative error
+	 */
+	int ( * format ) ( struct http_transaction *http, char *buf,
+			   size_t len );
+};
+
+/** HTTP request header table */
+#define HTTP_REQUEST_HEADERS \
+	__table ( struct http_request_header, "http_request_headers" )
+
+/** Declare an HTTP request header */
+#define __http_request_header __table_entry ( HTTP_REQUEST_HEADERS, 01 )
+
+/******************************************************************************
+ *
+ * Responses
+ *
+ ******************************************************************************
+ */
+
+/** HTTP response transfer descriptor */
+struct http_response_transfer {
+	/** Transfer encoding */
+	struct http_transfer_encoding *encoding;
+};
+
+/** HTTP response content descriptor */
+struct http_response_content {
+	/** Content length (may be zero) */
+	size_t len;
+	/** Content encoding */
+	struct http_content_encoding *encoding;
+};
+
+/** HTTP response authorization descriptor */
+struct http_response_auth {
+	/** Authentication scheme (if any) */
+	struct http_authentication *auth;
+	/** Realm */
+	const char *realm;
+	/** Quality of protection */
+	const char *qop;
+	/** Algorithm */
+	const char *algorithm;
+	/** Nonce */
+	const char *nonce;
+	/** Opaque */
+	const char *opaque;
+};
+
+/** An HTTP response
+ *
+ * This represents a single response received from the server,
+ * including all values parsed from headers.
+ *
+ * Pointers within this structure may point into the raw response
+ * buffer, and so should be invalidated when the response buffer is
+ * modified or discarded.
+ */
+struct http_response {
+	/** Raw response header lines
+	 *
+	 * This is the raw response data received from the server, up
+	 * to and including the terminating empty line.  String
+	 * pointers within the response may point into this data
+	 * buffer; NUL terminators will be added (overwriting the
+	 * original terminating characters) as needed.
+	 */
+	struct line_buffer headers;
+	/** Status code
+	 *
+	 * This is the raw HTTP numeric status code (e.g. 404).
+	 */
+	unsigned int status;
+	/** Return status code
+	 *
+	 * This is the iPXE return status code corresponding to the
+	 * HTTP status code (e.g. -ENOENT).
+	 */
+	int rc;
+	/** Redirection location */
+	const char *location;
+	/** Transfer descriptor */
+	struct http_response_transfer transfer;
+	/** Content descriptor */
+	struct http_response_content content;
+	/** Authorization descriptor */
+	struct http_response_auth auth;
+	/** Retry delay (in seconds) */
+	unsigned int retry_after;
+	/** Flags */
+	unsigned int flags;
+};
+
+/** HTTP response flags */
+enum http_response_flags {
+	/** Keep connection alive after close */
+	HTTP_RESPONSE_KEEPALIVE = 0x0001,
+	/** Content length specified */
+	HTTP_RESPONSE_CONTENT_LEN = 0x0002,
+	/** Transaction may be retried on failure */
+	HTTP_RESPONSE_RETRY = 0x0004,
+};
+
+/** An HTTP response header */
+struct http_response_header {
+	/** Header name (e.g. "Transfer-Encoding") */
+	const char *name;
+	/** Parse header line
+	 *
+	 * @v http		HTTP transaction
+	 * @v line		Remaining header line
+	 * @ret rc		Return status code
+	 */
+	int ( * parse ) ( struct http_transaction *http, char *line );
+};
+
+/** HTTP response header table */
+#define HTTP_RESPONSE_HEADERS \
+	__table ( struct http_response_header, "http_response_headers" )
+
+/** Declare an HTTP response header */
+#define __http_response_header __table_entry ( HTTP_RESPONSE_HEADERS, 01 )
+
+/******************************************************************************
+ *
+ * Transactions
+ *
+ ******************************************************************************
+ */
+
+/** HTTP transaction state */
+struct http_state {
+	/** Transmit data
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * tx ) ( struct http_transaction *http );
+	/** Receive data
+	 *
+	 * @v http		HTTP transaction
+	 * @v iobuf		I/O buffer (may be claimed)
+	 * @ret rc		Return status code
+	 */
+	int ( * rx ) ( struct http_transaction *http,
+		       struct io_buffer **iobuf );
+	/** Server connection closed
+	 *
+	 * @v http		HTTP transaction
+	 * @v rc		Reason for close
+	 */
+	void ( * close ) ( struct http_transaction *http, int rc );
+};
+
+/** An HTTP transaction */
+struct http_transaction {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Content-decoded interface */
+	struct interface content;
+	/** Transfer-decoded interface */
+	struct interface transfer;
+	/** Server connection */
+	struct interface conn;
+	/** Transmit process */
+	struct process process;
+	/** Reconnection timer */
+	struct retry_timer timer;
+
+	/** Request URI */
+	struct uri *uri;
+	/** Request */
+	struct http_request request;
+	/** Response */
+	struct http_response response;
+	/** Temporary line buffer */
+	struct line_buffer linebuf;
+
+	/** Transaction state */
+	struct http_state *state;
+	/** Accumulated transfer-decoded length */
+	size_t len;
+	/** Chunk length remaining */
+	size_t remaining;
+};
+
+/******************************************************************************
+ *
+ * Transfer encoding
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP transfer encoding */
+struct http_transfer_encoding {
+	/** Name */
+	const char *name;
+	/** Initialise transfer encoding
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * init ) ( struct http_transaction *http );
+	/** Receive data state */
+	struct http_state state;
+};
+
+/** HTTP transfer encoding table */
+#define HTTP_TRANSFER_ENCODINGS \
+	__table ( struct http_transfer_encoding, "http_transfer_encodings" )
+
+/** Declare an HTTP transfer encoding */
+#define __http_transfer_encoding __table_entry ( HTTP_TRANSFER_ENCODINGS, 01 )
+
+/******************************************************************************
+ *
+ * Content encoding
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP content encoding */
+struct http_content_encoding {
+	/** Name */
+	const char *name;
+	/** Check if content encoding is supported for this request
+	 *
+	 * @v http		HTTP transaction
+	 * @ret supported	Content encoding is supported for this request
+	 */
+	int ( * supported ) ( struct http_transaction *http );
+	/** Initialise content encoding
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * init ) ( struct http_transaction *http );
+};
+
+/** HTTP content encoding table */
+#define HTTP_CONTENT_ENCODINGS \
+	__table ( struct http_content_encoding, "http_content_encodings" )
+
+/** Declare an HTTP content encoding */
+#define __http_content_encoding __table_entry ( HTTP_CONTENT_ENCODINGS, 01 )
+
+/******************************************************************************
+ *
+ * Authentication
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP authentication scheme */
+struct http_authentication {
+	/** Name (e.g. "Digest") */
+	const char *name;
+	/** Perform authentication
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * authenticate ) ( struct http_transaction *http );
+	/** Construct remaining "Authorization" header line
+	 *
+	 * @v http		HTTP transaction
+	 * @v buf		Buffer
+	 * @v len		Length of buffer
+	 * @ret len		Header length if present, or negative error
+	 */
+	int ( * format ) ( struct http_transaction *http, char *buf,
+			   size_t len );
+};
+
+/** HTTP authentication scheme table */
+#define HTTP_AUTHENTICATIONS \
+	__table ( struct http_authentication, "http_authentications" )
+
+/** Declare an HTTP authentication scheme */
+#define __http_authentication __table_entry ( HTTP_AUTHENTICATIONS, 01 )
+
+/******************************************************************************
+ *
+ * General
+ *
+ ******************************************************************************
+ */
+
+extern char * http_token ( char **line, char **value );
+extern int http_connect ( struct interface *xfer, struct uri *uri );
+extern int http_open ( struct interface *xfer, struct http_method *method,
+		       struct uri *uri, struct http_request_range *range,
+		       struct http_request_content *content );
+extern int http_open_uri ( struct interface *xfer, struct uri *uri );
 
 #endif /* _IPXE_HTTP_H */
diff --git a/roms/ipxe/src/include/ipxe/hyperv.h b/roms/ipxe/src/include/ipxe/hyperv.h
new file mode 100644
index 000000000..c61e2a083
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/hyperv.h
@@ -0,0 +1,232 @@
+#ifndef _IPXE_HYPERV_H
+#define _IPXE_HYPERV_H
+
+/** @file
+ *
+ * Hyper-V interface
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/** Hyper-V interface identification */
+#define HV_INTERFACE_ID 0x31237648 /* "Hv#1" */
+
+/** Guest OS identity for iPXE
+ *
+ * This field comprises:
+ *
+ * Bit  63    : set to 1 to indicate an open source OS
+ * Bits 62:56 : OS Type
+ * Bits 55:48 : OS ID
+ * Bits 47:16 : Version
+ * Bits 15:0  : Build number
+ *
+ * There appears to be no central registry for the "OS Type".  The
+ * specification states that "Linux is 0x100", and the FreeBSD source
+ * states that "FreeBSD is 0x200".  Both of these statements are
+ * actually referring to the combined "OS Type" and "OS ID" field.
+ *
+ * We choose to use 0x98ae: this is generated by setting bit 63 (to
+ * indicate an open source OS) and setting the OS Type+ID equal to the
+ * PnP vendor ID used in romprefix.S.  No version information or build
+ * number is included.
+ */
+#define HV_GUEST_OS_ID_IPXE ( ( 1ULL << 63 ) | ( 0x18aeULL << 48 ) )
+
+/** Enable hypercall page */
+#define HV_HYPERCALL_ENABLE 0x00000001UL
+
+/** Enable SynIC */
+#define HV_SCONTROL_ENABLE 0x00000001UL
+
+/** Enable SynIC event flags */
+#define HV_SIEFP_ENABLE 0x00000001UL
+
+/** Enable SynIC messages */
+#define HV_SIMP_ENABLE 0x00000001UL
+
+/** Perform implicit EOI upon synthetic interrupt delivery */
+#define HV_SINT_AUTO_EOI 0x00020000UL
+
+/** Mask synthetic interrupt */
+#define HV_SINT_MASKED 0x00010000UL
+
+/** Synthetic interrupt vector */
+#define HV_SINT_VECTOR(x) ( (x) << 0 )
+
+/** Synthetic interrupt vector mask */
+#define HV_SINT_VECTOR_MASK HV_SINT_VECTOR ( 0xff )
+
+/** Post message */
+#define HV_POST_MESSAGE 0x005c
+
+/** A posted message
+ *
+ * This is the input parameter list for the HvPostMessage hypercall.
+ */
+struct hv_post_message {
+	/** Connection ID */
+	uint32_t id;
+	/** Padding */
+	uint32_t reserved;
+	/** Type */
+	uint32_t type;
+	/** Length of message */
+	uint32_t len;
+	/** Message */
+	uint8_t data[240];
+} __attribute__ (( packed ));
+
+/** A received message
+ *
+ * This is the HV_MESSAGE structure from the Hypervisor Top-Level
+ * Functional Specification.  The field order given in the
+ * documentation is incorrect.
+ */
+struct hv_message {
+	/** Type */
+	uint32_t type;
+	/** Length of message */
+	uint8_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Padding */
+	uint16_t reserved;
+	/** Origin */
+	uint64_t origin;
+	/** Message */
+	uint8_t data[240];
+} __attribute__ (( packed ));
+
+/** Signal event */
+#define HV_SIGNAL_EVENT 0x005d
+
+/** A signalled event */
+struct hv_signal_event {
+	/** Connection ID */
+	uint32_t id;
+	/** Flag number */
+	uint16_t flag;
+	/** Reserved */
+	uint16_t reserved;
+} __attribute__ (( packed ));
+
+/** A received event */
+struct hv_event {
+	/** Event flags */
+	uint8_t flags[256];
+} __attribute__ (( packed ));
+
+/** A monitor trigger group
+ *
+ * This is the HV_MONITOR_TRIGGER_GROUP structure from the Hypervisor
+ * Top-Level Functional Specification.
+ */
+struct hv_monitor_trigger {
+	/** Pending events */
+	uint32_t pending;
+	/** Armed events */
+	uint32_t armed;
+} __attribute__ (( packed ));
+
+/** A monitor parameter set
+ *
+ * This is the HV_MONITOR_PARAMETER structure from the Hypervisor
+ * Top-Level Functional Specification.
+ */
+struct hv_monitor_parameter {
+	/** Connection ID */
+	uint32_t id;
+	/** Flag number */
+	uint16_t flag;
+	/** Reserved */
+	uint16_t reserved;
+} __attribute__ (( packed ));
+
+/** A monitor page
+ *
+ * This is the HV_MONITOR_PAGE structure from the Hypervisor Top-Level
+ * Functional Specification.
+ */
+struct hv_monitor {
+	/** Flags */
+	uint32_t flags;
+	/** Reserved */
+	uint8_t reserved_a[4];
+	/** Trigger groups */
+	struct hv_monitor_trigger trigger[4];
+	/** Reserved */
+	uint8_t reserved_b[536];
+	/** Latencies */
+	uint16 latency[4][32];
+	/** Reserved */
+	uint8_t reserved_c[256];
+	/** Parameters */
+	struct hv_monitor_parameter param[4][32];
+	/** Reserved */
+	uint8_t reserved_d[1984];
+} __attribute__ (( packed ));
+
+/** A synthetic interrupt controller */
+struct hv_synic {
+	/** Message page */
+	struct hv_message *message;
+	/** Event flag page */
+	struct hv_event *event;
+};
+
+/** A message buffer */
+union hv_message_buffer {
+	/** Posted message */
+	struct hv_post_message posted;
+	/** Received message */
+	struct hv_message received;
+	/** Signalled event */
+	struct hv_signal_event signalled;
+};
+
+/** A Hyper-V hypervisor */
+struct hv_hypervisor {
+	/** Hypercall page */
+	void *hypercall;
+	/** Synthetic interrupt controller (SynIC) */
+	struct hv_synic synic;
+	/** Message buffer */
+	union hv_message_buffer *message;
+	/** Virtual machine bus */
+	struct vmbus *vmbus;
+};
+
+#include <bits/hyperv.h>
+
+/**
+ * Calculate the number of pages covering an address range
+ *
+ * @v data		Start of data
+ * @v len		Length of data (must be non-zero)
+ * @ret pfn_count	Number of pages covered
+ */
+static inline unsigned int hv_pfn_count ( physaddr_t data, size_t len ) {
+	unsigned int first_pfn = ( data / PAGE_SIZE );
+	unsigned int last_pfn = ( ( data + len - 1 ) / PAGE_SIZE );
+
+	return ( last_pfn - first_pfn + 1 );
+}
+
+extern __attribute__ (( sentinel )) int
+hv_alloc_pages ( struct hv_hypervisor *hv, ... );
+extern __attribute__ (( sentinel )) void
+hv_free_pages ( struct hv_hypervisor *hv, ... );
+extern void hv_enable_sint ( struct hv_hypervisor *hv, unsigned int sintx );
+extern void hv_disable_sint ( struct hv_hypervisor *hv, unsigned int sintx );
+extern int hv_post_message ( struct hv_hypervisor *hv, unsigned int id,
+			     unsigned int type, const void *data, size_t len );
+extern int hv_wait_for_message ( struct hv_hypervisor *hv, unsigned int sintx );
+extern int hv_signal_event ( struct hv_hypervisor *hv, unsigned int id,
+			     unsigned int flag );
+
+#endif /* _IPXE_HYPERV_H */
diff --git a/roms/ipxe/src/include/ipxe/i2c.h b/roms/ipxe/src/include/ipxe/i2c.h
index c1f5a9bbd..46970515c 100644
--- a/roms/ipxe/src/include/ipxe/i2c.h
+++ b/roms/ipxe/src/include/ipxe/i2c.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/bitbash.h>
diff --git a/roms/ipxe/src/include/ipxe/ib_cm.h b/roms/ipxe/src/include/ipxe/ib_cm.h
index 7d08cd9b1..4913eebae 100644
--- a/roms/ipxe/src/include/ipxe/ib_cm.h
+++ b/roms/ipxe/src/include/ipxe/ib_cm.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 #include <ipxe/retry.h>
diff --git a/roms/ipxe/src/include/ipxe/ib_mad.h b/roms/ipxe/src/include/ipxe/ib_mad.h
index b8694833e..ae1eea7e4 100644
--- a/roms/ipxe/src/include/ipxe/ib_mad.h
+++ b/roms/ipxe/src/include/ipxe/ib_mad.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/ib_packet.h>
diff --git a/roms/ipxe/src/include/ipxe/ib_mcast.h b/roms/ipxe/src/include/ipxe/ib_mcast.h
index a5c22a03e..564066975 100644
--- a/roms/ipxe/src/include/ipxe/ib_mcast.h
+++ b/roms/ipxe/src/include/ipxe/ib_mcast.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 
diff --git a/roms/ipxe/src/include/ipxe/ib_mi.h b/roms/ipxe/src/include/ipxe/ib_mi.h
index 5c5415b71..c7c8143ba 100644
--- a/roms/ipxe/src/include/ipxe/ib_mi.h
+++ b/roms/ipxe/src/include/ipxe/ib_mi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/retry.h>
diff --git a/roms/ipxe/src/include/ipxe/ib_packet.h b/roms/ipxe/src/include/ipxe/ib_packet.h
index a959967cb..f275fcb09 100644
--- a/roms/ipxe/src/include/ipxe/ib_packet.h
+++ b/roms/ipxe/src/include/ipxe/ib_packet.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct ib_device;
 struct ib_queue_pair;
diff --git a/roms/ipxe/src/include/ipxe/ib_pathrec.h b/roms/ipxe/src/include/ipxe/ib_pathrec.h
index 1fe67f87d..a4e11ebe3 100644
--- a/roms/ipxe/src/include/ipxe/ib_pathrec.h
+++ b/roms/ipxe/src/include/ipxe/ib_pathrec.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 
diff --git a/roms/ipxe/src/include/ipxe/ib_sma.h b/roms/ipxe/src/include/ipxe/ib_sma.h
index fa355c652..74003d045 100644
--- a/roms/ipxe/src/include/ipxe/ib_sma.h
+++ b/roms/ipxe/src/include/ipxe/ib_sma.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct ib_device;
 struct ib_mad_interface;
diff --git a/roms/ipxe/src/include/ipxe/ib_smc.h b/roms/ipxe/src/include/ipxe/ib_smc.h
index 259d2cde1..f9b96b1bd 100644
--- a/roms/ipxe/src/include/ipxe/ib_smc.h
+++ b/roms/ipxe/src/include/ipxe/ib_smc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 
diff --git a/roms/ipxe/src/include/ipxe/icmp.h b/roms/ipxe/src/include/ipxe/icmp.h
index 0480ddfaf..803f8e019 100644
--- a/roms/ipxe/src/include/ipxe/icmp.h
+++ b/roms/ipxe/src/include/ipxe/icmp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/iobuf.h>
diff --git a/roms/ipxe/src/include/ipxe/icmpv6.h b/roms/ipxe/src/include/ipxe/icmpv6.h
index b5ea54eab..0474ddca8 100644
--- a/roms/ipxe/src/include/ipxe/icmpv6.h
+++ b/roms/ipxe/src/include/ipxe/icmpv6.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
@@ -40,6 +40,18 @@ struct icmpv6_handler {
 /** Declare an ICMPv6 handler */
 #define __icmpv6_handler __table_entry ( ICMPV6_HANDLERS, 01 )
 
+/** ICMPv6 destination unreachable */
+#define ICMPV6_DESTINATION_UNREACHABLE 1
+
+/** ICMPv6 packet too big */
+#define ICMPV6_PACKET_TOO_BIG 2
+
+/** ICMPv6 time exceeded */
+#define ICMPV6_TIME_EXCEEDED 3
+
+/** ICMPv6 parameter problem */
+#define ICMPV6_PARAMETER_PROBLEM 4
+
 /** ICMPv6 echo request */
 #define ICMPV6_ECHO_REQUEST 128
 
diff --git a/roms/ipxe/src/include/ipxe/if_arp.h b/roms/ipxe/src/include/ipxe/if_arp.h
index fd36e9c67..4eb1f80b7 100644
--- a/roms/ipxe/src/include/ipxe/if_arp.h
+++ b/roms/ipxe/src/include/ipxe/if_arp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/if_ether.h b/roms/ipxe/src/include/ipxe/if_ether.h
index a7e237349..58d91b976 100644
--- a/roms/ipxe/src/include/ipxe/if_ether.h
+++ b/roms/ipxe/src/include/ipxe/if_ether.h
@@ -1,7 +1,7 @@
 #ifndef	_IPXE_IF_ETHER_H
 #define	_IPXE_IF_ETHER_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/image.h b/roms/ipxe/src/include/ipxe/image.h
index 5d7080a75..6abd7a2d2 100644
--- a/roms/ipxe/src/include/ipxe/image.h
+++ b/roms/ipxe/src/include/ipxe/image.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 #include <ipxe/list.h>
@@ -163,7 +163,6 @@ extern int image_set_cmdline ( struct image *image, const char *cmdline );
 extern int register_image ( struct image *image );
 extern void unregister_image ( struct image *image );
 struct image * find_image ( const char *name );
-extern int image_probe ( struct image *image );
 extern int image_exec ( struct image *image );
 extern int image_replace ( struct image *replacement );
 extern int image_select ( struct image *image );
diff --git a/roms/ipxe/src/include/ipxe/in.h b/roms/ipxe/src/include/ipxe/in.h
index de96ca22a..0ebf441c2 100644
--- a/roms/ipxe/src/include/ipxe/in.h
+++ b/roms/ipxe/src/include/ipxe/in.h
@@ -1,9 +1,10 @@
 #ifndef	_IPXE_IN_H
 #define	_IPXE_IN_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
+#include <byteswap.h>
 #include <ipxe/socket.h>
 
 /* Protocol numbers */
@@ -15,17 +16,22 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 /* IP address constants */
 
-#define INADDR_NONE 0xffffffff
+#define INADDR_NONE		htonl ( 0xffffffff )
 
-#define INADDR_BROADCAST 0xffffffff
+#define INADDR_BROADCAST	htonl ( 0xffffffff )
 
-#define	IN_CLASSA(addr)		( ( (addr) & 0x80000000 ) == 0x00000000 )
-#define	IN_CLASSA_NET		0xff000000
-#define	IN_CLASSB(addr)		( ( (addr) & 0xc0000000 ) == 0x80000000 )
-#define	IN_CLASSB_NET		0xffff0000
-#define	IN_CLASSC(addr)		( ( (addr) & 0xe0000000 ) == 0xc0000000 )
-#define	IN_CLASSC_NET		0xffffff00
-#define IN_MULTICAST(addr)	( ( (addr) & 0xf0000000 ) == 0xe0000000 )
+#define	INADDR_NET_CLASSA	htonl ( 0xff000000 )
+#define	INADDR_NET_CLASSB	htonl ( 0xffff0000 )
+#define	INADDR_NET_CLASSC	htonl ( 0xffffff00 )
+
+#define	IN_IS_CLASSA( addr ) \
+	( ( (addr) & htonl ( 0x80000000 ) ) == htonl ( 0x00000000 ) )
+#define	IN_IS_CLASSB( addr ) \
+	( ( (addr) & htonl ( 0xc0000000 ) ) == htonl ( 0x80000000 ) )
+#define	IN_IS_CLASSC( addr ) \
+	( ( (addr) & htonl ( 0xe0000000 ) ) == htonl ( 0xc0000000 ) )
+#define IN_IS_MULTICAST( addr ) \
+	( ( (addr) & htonl ( 0xf0000000 ) ) == htonl ( 0xe0000000 ) )
 
 /**
  * IP address structure
@@ -63,6 +69,9 @@ struct in6_addr {
 	( ( *( ( const uint16_t * ) (addr) ) & htons ( 0xffc0 ) ) ==	\
 	  htons ( 0xfe80 ) )
 
+#define IN6_IS_ADDR_NONGLOBAL( addr )					\
+	( IN6_IS_ADDR_LINKLOCAL (addr) || IN6_IS_ADDR_MULTICAST (addr) )
+
 /**
  * IPv4 socket address
  */
@@ -76,6 +85,11 @@ struct sockaddr_in {
 	uint16_t sin_flags;
 	/** TCP/IP port (part of struct @c sockaddr_tcpip) */
 	uint16_t sin_port;
+	/** Scope ID (part of struct @c sockaddr_tcpip)
+	 *
+	 * For multicast addresses, this is the network device index.
+	 */
+        uint16_t sin_scope_id;
 	/** IPv4 address */
 	struct in_addr sin_addr;
 	/** Padding
@@ -87,6 +101,7 @@ struct sockaddr_in {
 		  ( sizeof ( sa_family_t ) /* sin_family */ +
 		    sizeof ( uint16_t ) /* sin_flags */ +
 		    sizeof ( uint16_t ) /* sin_port */ +
+		    sizeof ( uint16_t ) /* sin_scope_id */ +
 		    sizeof ( struct in_addr ) /* sin_addr */ ) ];
 } __attribute__ (( packed, may_alias ));
 
@@ -103,9 +118,10 @@ struct sockaddr_in6 {
 	uint16_t sin6_flags;
 	/** TCP/IP port (part of struct @c sockaddr_tcpip) */
 	uint16_t sin6_port;
-	/** Scope ID
+	/** Scope ID (part of struct @c sockaddr_tcpip)
 	 *
-	 * For link-local addresses, this is the network device index.
+	 * For link-local or multicast addresses, this is the network
+	 * device index.
 	 */
         uint16_t sin6_scope_id;
 	/** IPv6 address */
diff --git a/roms/ipxe/src/include/ipxe/infiniband.h b/roms/ipxe/src/include/ipxe/infiniband.h
index f546ea61b..87cfe5082 100644
--- a/roms/ipxe/src/include/ipxe/infiniband.h
+++ b/roms/ipxe/src/include/ipxe/infiniband.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/include/ipxe/init.h b/roms/ipxe/src/include/ipxe/init.h
index 19c5925bf..025cfaf37 100644
--- a/roms/ipxe/src/include/ipxe/init.h
+++ b/roms/ipxe/src/include/ipxe/init.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_INIT_H
 #define _IPXE_INIT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
@@ -26,10 +26,9 @@ struct init_fn {
  */
 
 #define INIT_EARLY	01	/**< Early initialisation */
-#define INIT_SERIAL	02	/**< Serial driver initialisation */
-#define	INIT_CONSOLE	03	/**< Console initialisation */
-#define INIT_NORMAL	04	/**< Normal initialisation */
-#define INIT_LATE	05	/**< Late initialisation */
+#define	INIT_CONSOLE	02	/**< Console initialisation */
+#define INIT_NORMAL	03	/**< Normal initialisation */
+#define INIT_LATE	04	/**< Late initialisation */
 
 /** @} */
 
diff --git a/roms/ipxe/src/include/ipxe/interface.h b/roms/ipxe/src/include/ipxe/interface.h
index a474aaad0..a8d823775 100644
--- a/roms/ipxe/src/include/ipxe/interface.h
+++ b/roms/ipxe/src/include/ipxe/interface.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/refcnt.h>
@@ -145,6 +145,11 @@ extern void intf_close ( struct interface *intf, int rc );
 extern void intf_shutdown ( struct interface *intf, int rc );
 extern void intf_restart ( struct interface *intf, int rc );
 
+extern void intf_poke ( struct interface *intf,
+			void ( type ) ( struct interface *intf ) );
+#define intf_poke_TYPE( object_type ) \
+	typeof ( void ( object_type ) )
+
 extern struct interface_descriptor null_intf_desc;
 extern struct interface null_intf;
 
diff --git a/roms/ipxe/src/include/ipxe/io.h b/roms/ipxe/src/include/ipxe/io.h
index 29ccfd1fa..af767915d 100644
--- a/roms/ipxe/src/include/ipxe/io.h
+++ b/roms/ipxe/src/include/ipxe/io.h
@@ -16,7 +16,7 @@
  * the address parameter.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/api.h>
diff --git a/roms/ipxe/src/include/ipxe/iobuf.h b/roms/ipxe/src/include/ipxe/iobuf.h
index b2b0cb440..27d285d44 100644
--- a/roms/ipxe/src/include/ipxe/iobuf.h
+++ b/roms/ipxe/src/include/ipxe/iobuf.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <assert.h>
@@ -217,5 +217,6 @@ extern void free_iob ( struct io_buffer *iobuf );
 extern void iob_pad ( struct io_buffer *iobuf, size_t min_len );
 extern int iob_ensure_headroom ( struct io_buffer *iobuf, size_t len );
 extern struct io_buffer * iob_concatenate ( struct list_head *list );
+extern struct io_buffer * iob_split ( struct io_buffer *iobuf, size_t len );
 
 #endif /* _IPXE_IOBUF_H */
diff --git a/roms/ipxe/src/include/ipxe/ip.h b/roms/ipxe/src/include/ipxe/ip.h
index 1a93a552e..285be6dcd 100644
--- a/roms/ipxe/src/include/ipxe/ip.h
+++ b/roms/ipxe/src/include/ipxe/ip.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/roms/ipxe/src/include/ipxe/ipoib.h b/roms/ipxe/src/include/ipxe/ipoib.h
index 68ff8df49..b34dd32d0 100644
--- a/roms/ipxe/src/include/ipxe/ipoib.h
+++ b/roms/ipxe/src/include/ipxe/ipoib.h
@@ -6,7 +6,7 @@
  * IP over Infiniband
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/if_arp.h>
 #include <ipxe/infiniband.h>
diff --git a/roms/ipxe/src/include/ipxe/ipstat.h b/roms/ipxe/src/include/ipxe/ipstat.h
index c554c1859..b34ed5fcf 100644
--- a/roms/ipxe/src/include/ipxe/ipstat.h
+++ b/roms/ipxe/src/include/ipxe/ipstat.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/roms/ipxe/src/include/ipxe/ipv6.h b/roms/ipxe/src/include/ipxe/ipv6.h
index 48aaf677e..b500382c1 100644
--- a/roms/ipxe/src/include/ipxe/ipv6.h
+++ b/roms/ipxe/src/include/ipxe/ipv6.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/include/ipxe/isa_ids.h b/roms/ipxe/src/include/ipxe/isa_ids.h
index 1faf1148d..d815bda34 100644
--- a/roms/ipxe/src/include/ipxe/isa_ids.h
+++ b/roms/ipxe/src/include/ipxe/isa_ids.h
@@ -19,7 +19,7 @@
  * the underlying "meaning" is big-endian.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <byteswap.h>
 
diff --git a/roms/ipxe/src/include/ipxe/isapnp.h b/roms/ipxe/src/include/ipxe/isapnp.h
index b0b0e98d6..59beac986 100644
--- a/roms/ipxe/src/include/ipxe/isapnp.h
+++ b/roms/ipxe/src/include/ipxe/isapnp.h
@@ -17,6 +17,10 @@
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 *    02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
 *
 *    Portions of this code:
 *		Copyright (C) 2001  P.J.H.Fox (fox@roestock.demon.co.uk)
@@ -34,7 +38,7 @@
 *
 ***************************************************************************/
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef ISAPNP_H
 #define ISAPNP_H
diff --git a/roms/ipxe/src/include/ipxe/iscsi.h b/roms/ipxe/src/include/ipxe/iscsi.h
index be71360a0..c75ff4188 100644
--- a/roms/ipxe/src/include/ipxe/iscsi.h
+++ b/roms/ipxe/src/include/ipxe/iscsi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/socket.h>
diff --git a/roms/ipxe/src/include/ipxe/iso9660.h b/roms/ipxe/src/include/ipxe/iso9660.h
index 02c2ae377..34cb8f0a1 100644
--- a/roms/ipxe/src/include/ipxe/iso9660.h
+++ b/roms/ipxe/src/include/ipxe/iso9660.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/isqrt.h b/roms/ipxe/src/include/ipxe/isqrt.h
index 58ed42f0c..68255d1bc 100644
--- a/roms/ipxe/src/include/ipxe/isqrt.h
+++ b/roms/ipxe/src/include/ipxe/isqrt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern unsigned long isqrt ( unsigned long value );
 
diff --git a/roms/ipxe/src/include/ipxe/job.h b/roms/ipxe/src/include/ipxe/job.h
index a2369f7c2..7e1bd8109 100644
--- a/roms/ipxe/src/include/ipxe/job.h
+++ b/roms/ipxe/src/include/ipxe/job.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/interface.h>
 
diff --git a/roms/ipxe/src/include/ipxe/jumpscroll.h b/roms/ipxe/src/include/ipxe/jumpscroll.h
new file mode 100644
index 000000000..7a5b111c1
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/jumpscroll.h
@@ -0,0 +1,50 @@
+#ifndef _IPXE_JUMPSCROLL_H
+#define _IPXE_JUMPSCROLL_H
+
+/** @file
+ *
+ * Jump scrolling
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** A jump scroller */
+struct jump_scroller {
+	/** Maximum number of visible rows */
+	unsigned int rows;
+	/** Total number of items */
+	unsigned int count;
+	/** Currently selected item */
+	unsigned int current;
+	/** First visible item */
+	unsigned int first;
+};
+
+/**
+ * Check if jump scroller is currently on first page
+ *
+ * @v scroll		Jump scroller
+ * @ret is_first	Scroller is currently on first page
+ */
+static inline int jump_scroll_is_first ( struct jump_scroller *scroll ) {
+
+	return ( scroll->first == 0 );
+}
+
+/**
+ * Check if jump scroller is currently on last page
+ *
+ * @v scroll		Jump scroller
+ * @ret is_last		Scroller is currently on last page
+ */
+static inline int jump_scroll_is_last ( struct jump_scroller *scroll ) {
+
+	return ( ( scroll->first + scroll->rows ) >= scroll->count );
+}
+
+extern int jump_scroll_key ( struct jump_scroller *scroll, int key );
+extern int jump_scroll_move ( struct jump_scroller *scroll, int move );
+extern int jump_scroll ( struct jump_scroller *scroll );
+
+#endif /* _IPXE_JUMPSCROLL_H */
diff --git a/roms/ipxe/src/include/ipxe/keymap.h b/roms/ipxe/src/include/ipxe/keymap.h
index 9ac42a6b1..0f1b0c656 100644
--- a/roms/ipxe/src/include/ipxe/keymap.h
+++ b/roms/ipxe/src/include/ipxe/keymap.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
diff --git a/roms/ipxe/src/include/ipxe/keys.h b/roms/ipxe/src/include/ipxe/keys.h
index 8b13550b9..d15267a1f 100644
--- a/roms/ipxe/src/include/ipxe/keys.h
+++ b/roms/ipxe/src/include/ipxe/keys.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * Symbolic names for some standard ASCII characters
@@ -58,6 +58,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 #define KEY_ANSI( n, terminator ) ( 0x100 * ( (n) + 1 ) + (terminator) )
+#define KEY_ANSI_N( key ) ( ( (key) / 0x100 ) - 1 )
+#define KEY_ANSI_TERMINATOR( key ) ( (key) & 0xff )
 
 #define KEY_MIN		0x101
 #define KEY_UP		KEY_ANSI ( 0, 'A' )	/**< Up arrow */
diff --git a/roms/ipxe/src/include/ipxe/linebuf.h b/roms/ipxe/src/include/ipxe/linebuf.h
index 706ef2554..630278a04 100644
--- a/roms/ipxe/src/include/ipxe/linebuf.h
+++ b/roms/ipxe/src/include/ipxe/linebuf.h
@@ -7,24 +7,24 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
 
 /** A line buffer */
 struct line_buffer {
-	/** Current string in the buffer */
+	/** Data buffer */
 	char *data;
-	/** Length of current string, excluding the terminating NUL */
+	/** Length of buffered data */
 	size_t len;
-	/** String is ready to read */
-	int ready;
+	/** Most recently consumed length */
+	size_t consumed;
 };
 
 extern char * buffered_line ( struct line_buffer *linebuf );
-extern ssize_t line_buffer ( struct line_buffer *linebuf,
-			     const char *data, size_t len );
+extern int line_buffer ( struct line_buffer *linebuf,
+			 const char *data, size_t len );
 extern void empty_line_buffer ( struct line_buffer *linebuf );
 
 #endif /* _IPXE_LINEBUF_H */
diff --git a/roms/ipxe/src/include/ipxe/lineconsole.h b/roms/ipxe/src/include/ipxe/lineconsole.h
index 925c0accc..31117e73c 100644
--- a/roms/ipxe/src/include/ipxe/lineconsole.h
+++ b/roms/ipxe/src/include/ipxe/lineconsole.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/ansiesc.h>
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_entropy.h b/roms/ipxe/src/include/ipxe/linux/linux_entropy.h
index bd89bd52f..afef6fe19 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_entropy.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_entropy.h
@@ -3,11 +3,11 @@
 
 /** @file
  *
- * iPXE entropy API for linux
+ * /dev/random-based entropy source
  *
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef ENTROPY_LINUX
 #define ENTROPY_PREFIX_linux
@@ -23,10 +23,12 @@ FILE_LICENCE(GPL2_OR_LATER);
 static inline __always_inline double
 ENTROPY_INLINE ( linux, min_entropy_per_sample ) ( void ) {
 
-	/* We read single bytes from /dev/random and assume that each
-	 * contains full entropy.
+	/* linux_get_noise() reads a single byte from /dev/random,
+	 * which is supposed to block until a sufficient amount of
+	 * entropy is available.  We therefore assume that each sample
+	 * contains exactly 8 bits of entropy.
 	 */
-	return 8;
+	return 8.0;
 }
 
 #endif /* _IPXE_LINUX_ENTROPY_H */
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_nap.h b/roms/ipxe/src/include/ipxe/linux/linux_nap.h
index 5bac7242f..d072886c7 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_nap.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_LINUX
 #define NAP_PREFIX_linux
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_pci.h b/roms/ipxe/src/include/ipxe/linux/linux_pci.h
index 439166733..22ae7f1bc 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_pci.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_pci.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef PCIAPI_LINUX
 #define PCIAPI_PREFIX_linux
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_smbios.h b/roms/ipxe/src/include/ipxe/linux/linux_smbios.h
index 6d51e13ba..16c6d8acd 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_smbios.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_smbios.h
@@ -3,11 +3,11 @@
 
 /** @file
  *
- * iPXE SMBIOS API for linux
+ * iPXE SMBIOS API for Linux
  *
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SMBIOS_LINUX
 #define SMBIOS_PREFIX_linux
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_time.h b/roms/ipxe/src/include/ipxe/linux/linux_time.h
index 93a257730..872ef5ade 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_time.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIME_LINUX
 #define TIME_PREFIX_linux
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_timer.h b/roms/ipxe/src/include/ipxe/linux/linux_timer.h
index 379507417..7f46e36b2 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_timer.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_LINUX
 #define TIMER_PREFIX_linux
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h b/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h
index e4d16d9e0..acd919a85 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h
@@ -1,116 +1,108 @@
-/*
- * Copyright (C) 2010 Piotr Jaroszyński <p.jaroszynski@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
 #ifndef _IPXE_LINUX_UACCESS_H
 #define _IPXE_LINUX_UACCESS_H
 
-FILE_LICENCE(GPL2_OR_LATER);
-
 /** @file
  *
- * iPXE user access API for linux
+ * iPXE user access API for Linux
+ *
+ * We run with no distinction between internal and external addresses,
+ * so can use trivial_virt_to_user() et al.
  *
- * In linux userspace virtual == user == phys addresses.
- * Physical addresses also being the same is wrong, but there is no general way
- * of converting userspace addresses to physical as what appears to be
- * contiguous in userspace is physically fragmented.
- * Currently only the DMA memory is special-cased, but its conversion to bus
- * addresses is done in phys_to_bus.
- * This is known to break virtio as it is passing phys addresses to the virtual
- * device.
+ * We have no concept of the underlying physical addresses, since
+ * these are not exposed to userspace.  We provide a stub
+ * implementation of user_to_phys() since this is required by
+ * alloc_memblock().  We provide no implementation of phys_to_user();
+ * any code attempting to access physical addresses will therefore
+ * (correctly) fail to link.
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #ifdef UACCESS_LINUX
 #define UACCESS_PREFIX_linux
 #else
 #define UACCESS_PREFIX_linux __linux_
 #endif
 
-static inline __always_inline userptr_t
-UACCESS_INLINE(linux, phys_to_user)(unsigned long phys_addr)
-{
-	return phys_addr;
-}
-
+/**
+ * Convert user buffer to physical address
+ *
+ * @v userptr		User pointer
+ * @v offset		Offset from user pointer
+ * @ret phys_addr	Physical address
+ */
 static inline __always_inline unsigned long
-UACCESS_INLINE(linux, user_to_phys)(userptr_t userptr, off_t offset)
-{
-	return userptr + offset;
+UACCESS_INLINE ( linux, user_to_phys ) ( userptr_t userptr, off_t offset ) {
+
+	/* We do not know the real underlying physical address.  We
+	 * provide this stub implementation only because it is
+	 * required by alloc_memblock() (which allocates memory with
+	 * specified physical address alignment).  We assume that the
+	 * low-order bits of virtual addresses match the low-order
+	 * bits of physical addresses, and so simply returning the
+	 * virtual address will suffice for the purpose of determining
+	 * alignment.
+	 */
+	return ( userptr + offset );
 }
 
 static inline __always_inline userptr_t
-UACCESS_INLINE(linux, virt_to_user)(volatile const void *addr)
-{
-	return trivial_virt_to_user(addr);
+UACCESS_INLINE ( linux, virt_to_user ) ( volatile const void *addr ) {
+	return trivial_virt_to_user ( addr );
 }
 
 static inline __always_inline void *
-UACCESS_INLINE(linux, user_to_virt)(userptr_t userptr, off_t offset)
-{
-	return trivial_user_to_virt(userptr, offset);
+UACCESS_INLINE ( linux, user_to_virt ) ( userptr_t userptr, off_t offset ) {
+	return trivial_user_to_virt ( userptr, offset );
 }
 
 static inline __always_inline userptr_t
-UACCESS_INLINE(linux, userptr_add)(userptr_t userptr, off_t offset)
-{
-	return trivial_userptr_add(userptr, offset);
+UACCESS_INLINE ( linux, userptr_add ) ( userptr_t userptr, off_t offset ) {
+	return trivial_userptr_add ( userptr, offset );
 }
 
 static inline __always_inline off_t
-UACCESS_INLINE(linux, userptr_sub)(userptr_t userptr, userptr_t subtrahend)
-{
+UACCESS_INLINE ( linux, userptr_sub ) ( userptr_t userptr,
+					userptr_t subtrahend ) {
 	return trivial_userptr_sub ( userptr, subtrahend );
 }
 
 static inline __always_inline void
-UACCESS_INLINE(linux, memcpy_user)(userptr_t dest, off_t dest_off, userptr_t src, off_t src_off, size_t len)
-{
-	trivial_memcpy_user(dest, dest_off, src, src_off, len);
+UACCESS_INLINE ( linux, memcpy_user ) ( userptr_t dest, off_t dest_off,
+					userptr_t src, off_t src_off,
+					size_t len ) {
+	trivial_memcpy_user ( dest, dest_off, src, src_off, len );
 }
 
 static inline __always_inline void
-UACCESS_INLINE(linux, memmove_user)(userptr_t dest, off_t dest_off, userptr_t src, off_t src_off, size_t len)
-{
-	trivial_memmove_user(dest, dest_off, src, src_off, len);
+UACCESS_INLINE ( linux, memmove_user ) ( userptr_t dest, off_t dest_off,
+					 userptr_t src, off_t src_off,
+					 size_t len ) {
+	trivial_memmove_user ( dest, dest_off, src, src_off, len );
 }
 
 static inline __always_inline int
-UACCESS_INLINE(linux, memcmp_user)(userptr_t first, off_t first_off, userptr_t second, off_t second_off, size_t len)
-{
-	return trivial_memcmp_user(first, first_off, second, second_off, len);
+UACCESS_INLINE ( linux, memcmp_user ) ( userptr_t first, off_t first_off,
+					userptr_t second, off_t second_off,
+					size_t len ) {
+	return trivial_memcmp_user ( first, first_off, second, second_off, len);
 }
 
 static inline __always_inline void
-UACCESS_INLINE(linux, memset_user)(userptr_t buffer, off_t offset, int c, size_t len)
-{
-	trivial_memset_user(buffer, offset, c, len);
+UACCESS_INLINE ( linux, memset_user ) ( userptr_t buffer, off_t offset,
+					int c, size_t len ) {
+	trivial_memset_user ( buffer, offset, c, len );
 }
 
 static inline __always_inline size_t
-UACCESS_INLINE(linux, strlen_user)(userptr_t buffer, off_t offset)
-{
-	return trivial_strlen_user(buffer, offset);
+UACCESS_INLINE ( linux, strlen_user ) ( userptr_t buffer, off_t offset ) {
+	return trivial_strlen_user ( buffer, offset );
 }
 
 static inline __always_inline off_t
-UACCESS_INLINE(linux, memchr_user)(userptr_t buffer, off_t offset, int c, size_t len)
-{
-	return trivial_memchr_user(buffer, offset, c, len);
+UACCESS_INLINE ( linux, memchr_user ) ( userptr_t buffer, off_t offset,
+					int c, size_t len ) {
+	return trivial_memchr_user ( buffer, offset, c, len );
 }
 
 #endif /* _IPXE_LINUX_UACCESS_H */
diff --git a/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h b/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h
index 4de55ecf3..1811d0bc6 100644
--- a/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h
+++ b/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h
@@ -1,14 +1,14 @@
 #ifndef _IPXE_LINUX_UMALLOC_H
 #define _IPXE_LINUX_UMALLOC_H
 
-FILE_LICENCE(GPL2_OR_LATER);
-
 /** @file
  *
- * iPXE user memory allocation API for linux
+ * iPXE user memory allocation API for Linux
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #ifdef UMALLOC_LINUX
 #define UMALLOC_PREFIX_linux
 #else
diff --git a/roms/ipxe/src/include/ipxe/linux_compat.h b/roms/ipxe/src/include/ipxe/linux_compat.h
index 6f6ed97d7..4704c4817 100644
--- a/roms/ipxe/src/include/ipxe/linux_compat.h
+++ b/roms/ipxe/src/include/ipxe/linux_compat.h
@@ -10,7 +10,7 @@
  * intended to be a substitute for proper porting.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/include/ipxe/list.h b/roms/ipxe/src/include/ipxe/list.h
index 581ec9806..6a9b76f91 100644
--- a/roms/ipxe/src/include/ipxe/list.h
+++ b/roms/ipxe/src/include/ipxe/list.h
@@ -9,7 +9,7 @@
  * list.h.
  */
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <assert.h>
diff --git a/roms/ipxe/src/include/ipxe/login_ui.h b/roms/ipxe/src/include/ipxe/login_ui.h
index 01e5479f7..313e07349 100644
--- a/roms/ipxe/src/include/ipxe/login_ui.h
+++ b/roms/ipxe/src/include/ipxe/login_ui.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int login_ui ( void );
 
diff --git a/roms/ipxe/src/include/ipxe/malloc.h b/roms/ipxe/src/include/ipxe/malloc.h
index bbd6cb898..dd158b8e6 100644
--- a/roms/ipxe/src/include/ipxe/malloc.h
+++ b/roms/ipxe/src/include/ipxe/malloc.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * Prototypes for the standard functions (malloc() et al) are in
@@ -77,8 +77,8 @@ static inline void * __malloc malloc_dma ( size_t size, size_t phys_align ) {
  * If @c ptr is NULL, no action is taken.
  */
 static inline void free_dma ( void *ptr, size_t size ) {
-	free_memblock ( ptr, size );
 	VALGRIND_FREELIKE_BLOCK ( ptr, 0 );
+	free_memblock ( ptr, size );
 }
 
 /** A cache discarder */
diff --git a/roms/ipxe/src/include/ipxe/mca.h b/roms/ipxe/src/include/ipxe/mca.h
index d86dab195..11470ec93 100644
--- a/roms/ipxe/src/include/ipxe/mca.h
+++ b/roms/ipxe/src/include/ipxe/mca.h
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef MCA_H
 #define MCA_H
diff --git a/roms/ipxe/src/include/ipxe/md5.h b/roms/ipxe/src/include/ipxe/md5.h
index 860bc4769..05c3974c8 100644
--- a/roms/ipxe/src/include/ipxe/md5.h
+++ b/roms/ipxe/src/include/ipxe/md5.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/roms/ipxe/src/include/ipxe/memblock.h b/roms/ipxe/src/include/ipxe/memblock.h
index 13af3e433..2bb38c460 100644
--- a/roms/ipxe/src/include/ipxe/memblock.h
+++ b/roms/ipxe/src/include/ipxe/memblock.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/roms/ipxe/src/include/ipxe/menu.h b/roms/ipxe/src/include/ipxe/menu.h
index f2b3caccc..3cc99be48 100644
--- a/roms/ipxe/src/include/ipxe/menu.h
+++ b/roms/ipxe/src/include/ipxe/menu.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 
diff --git a/roms/ipxe/src/include/ipxe/mii.h b/roms/ipxe/src/include/ipxe/mii.h
index f53ad4a62..c2245b49e 100644
--- a/roms/ipxe/src/include/ipxe/mii.h
+++ b/roms/ipxe/src/include/ipxe/mii.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <mii.h>
 #include <ipxe/netdevice.h>
@@ -114,5 +114,7 @@ mii_dump ( struct mii_interface *mii ) {
 
 extern int mii_restart ( struct mii_interface *mii );
 extern int mii_reset ( struct mii_interface *mii );
+extern int mii_check_link ( struct mii_interface *mii,
+			    struct net_device *netdev );
 
 #endif /* _IPXE_MII_H */
diff --git a/roms/ipxe/src/include/ipxe/monojob.h b/roms/ipxe/src/include/ipxe/monojob.h
index aedc37eca..1661d91c2 100644
--- a/roms/ipxe/src/include/ipxe/monojob.h
+++ b/roms/ipxe/src/include/ipxe/monojob.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct interface;
 
diff --git a/roms/ipxe/src/include/ipxe/mount.h b/roms/ipxe/src/include/ipxe/mount.h
index ca958117a..2d42ba080 100644
--- a/roms/ipxe/src/include/ipxe/mount.h
+++ b/roms/ipxe/src/include/ipxe/mount.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** NFS MOUNT protocol number */
 #define ONCRPC_MOUNT 100005
diff --git a/roms/ipxe/src/include/ipxe/nap.h b/roms/ipxe/src/include/ipxe/nap.h
index afc887910..f4de778c4 100644
--- a/roms/ipxe/src/include/ipxe/nap.h
+++ b/roms/ipxe/src/include/ipxe/nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/nap.h>
diff --git a/roms/ipxe/src/include/ipxe/ndp.h b/roms/ipxe/src/include/ipxe/ndp.h
index 7388f938e..1815236f5 100644
--- a/roms/ipxe/src/include/ipxe/ndp.h
+++ b/roms/ipxe/src/include/ipxe/ndp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/roms/ipxe/src/include/ipxe/neighbour.h b/roms/ipxe/src/include/ipxe/neighbour.h
index f2a3946f1..1c1d1b6ca 100644
--- a/roms/ipxe/src/include/ipxe/neighbour.h
+++ b/roms/ipxe/src/include/ipxe/neighbour.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/include/ipxe/net80211_err.h b/roms/ipxe/src/include/ipxe/net80211_err.h
index 7df3d0d85..32ccc257f 100644
--- a/roms/ipxe/src/include/ipxe/net80211_err.h
+++ b/roms/ipxe/src/include/ipxe/net80211_err.h
@@ -10,7 +10,7 @@
  * Copyright (c) 2009 Joshua Oreman <oremanj@rwcr.net>.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/include/ipxe/netdevice.h b/roms/ipxe/src/include/ipxe/netdevice.h
index 95ad1cf1b..a1d207ffc 100644
--- a/roms/ipxe/src/include/ipxe/netdevice.h
+++ b/roms/ipxe/src/include/ipxe/netdevice.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
@@ -15,6 +15,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/refcnt.h>
 #include <ipxe/settings.h>
 #include <ipxe/interface.h>
+#include <ipxe/retry.h>
 
 struct io_buffer;
 struct net_device;
@@ -36,13 +37,12 @@ struct device;
 
 /** Maximum length of a link-layer header
  *
- * The longest currently-supported link-layer header is for 802.11: a
- * 24-byte frame header plus an 8-byte 802.3 LLC/SNAP header, plus a
- * possible 4-byte VLAN header.  (The IPoIB link-layer pseudo-header
- * doesn't actually include link-layer addresses; see ipoib.c for
- * details.)
+ * The longest currently-supported link-layer header is for RNDIS: an
+ * 8-byte RNDIS header, a 32-byte RNDIS packet message header, a
+ * 14-byte Ethernet header and a possible 4-byte VLAN header.  Round
+ * up to 64 bytes.
  */
-#define MAX_LL_HEADER_LEN 36
+#define MAX_LL_HEADER_LEN 64
 
 /** Maximum length of a network-layer address */
 #define MAX_NET_ADDR_LEN 16
@@ -393,6 +393,8 @@ struct net_device {
 	 * indicates the error preventing link-up.
 	 */
 	int link_rc;
+	/** Link block timer */
+	struct retry_timer link_block;
 	/** Maximum packet length
 	 *
 	 * This length includes any link-layer headers.
@@ -428,6 +430,14 @@ struct net_device {
 /** Network device receive queue processing is frozen */
 #define NETDEV_RX_FROZEN 0x0004
 
+/** Network device interrupts are unsupported
+ *
+ * This flag can be used by a network device to indicate that
+ * interrupts are not supported despite the presence of an irq()
+ * method.
+ */
+#define NETDEV_IRQ_UNSUPPORTED 0x0008
+
 /** Link-layer protocol table */
 #define LL_PROTOCOLS __table ( struct ll_protocol, "ll_protocols" )
 
@@ -615,6 +625,17 @@ netdev_link_ok ( struct net_device *netdev ) {
 }
 
 /**
+ * Check link block state of network device
+ *
+ * @v netdev		Network device
+ * @ret link_blocked	Link is blocked
+ */
+static inline __attribute__ (( always_inline )) int
+netdev_link_blocked ( struct net_device *netdev ) {
+	return ( timer_running ( &netdev->link_block ) );
+}
+
+/**
  * Check whether or not network device is open
  *
  * @v netdev		Network device
@@ -633,7 +654,8 @@ netdev_is_open ( struct net_device *netdev ) {
  */
 static inline __attribute__ (( always_inline )) int
 netdev_irq_supported ( struct net_device *netdev ) {
-	return ( netdev->op->irq != NULL );
+	return ( ( netdev->op->irq != NULL ) &&
+		 ! ( netdev->state & NETDEV_IRQ_UNSUPPORTED ) );
 }
 
 /**
@@ -662,6 +684,9 @@ extern void netdev_rx_freeze ( struct net_device *netdev );
 extern void netdev_rx_unfreeze ( struct net_device *netdev );
 extern void netdev_link_err ( struct net_device *netdev, int rc );
 extern void netdev_link_down ( struct net_device *netdev );
+extern void netdev_link_block ( struct net_device *netdev,
+				unsigned long timeout );
+extern void netdev_link_unblock ( struct net_device *netdev );
 extern int netdev_tx ( struct net_device *netdev, struct io_buffer *iobuf );
 extern void netdev_tx_defer ( struct net_device *netdev,
 			      struct io_buffer *iobuf );
diff --git a/roms/ipxe/src/include/ipxe/nfs.h b/roms/ipxe/src/include/ipxe/nfs.h
index 498ed5a27..69b8b5381 100644
--- a/roms/ipxe/src/include/ipxe/nfs.h
+++ b/roms/ipxe/src/include/ipxe/nfs.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** NFS protocol number */
 #define ONCRPC_NFS 100003
diff --git a/roms/ipxe/src/include/ipxe/nfs_open.h b/roms/ipxe/src/include/ipxe/nfs_open.h
index caba977f7..8572c41b3 100644
--- a/roms/ipxe/src/include/ipxe/nfs_open.h
+++ b/roms/ipxe/src/include/ipxe/nfs_open.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _IPXE_NFS_OPEN_H */
diff --git a/roms/ipxe/src/include/ipxe/nfs_uri.h b/roms/ipxe/src/include/ipxe/nfs_uri.h
index d88bd6f65..aaa6d3749 100644
--- a/roms/ipxe/src/include/ipxe/nfs_uri.h
+++ b/roms/ipxe/src/include/ipxe/nfs_uri.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uri.h>
 
diff --git a/roms/ipxe/src/include/ipxe/null_entropy.h b/roms/ipxe/src/include/ipxe/null_entropy.h
index 646d1a17e..91adefa69 100644
--- a/roms/ipxe/src/include/ipxe/null_entropy.h
+++ b/roms/ipxe/src/include/ipxe/null_entropy.h
@@ -9,7 +9,7 @@
  * security-sensitive environment.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/null_nap.h b/roms/ipxe/src/include/ipxe/null_nap.h
index 0c0704bc7..17145b48b 100644
--- a/roms/ipxe/src/include/ipxe/null_nap.h
+++ b/roms/ipxe/src/include/ipxe/null_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_NULL
 #define NAP_PREFIX_null
diff --git a/roms/ipxe/src/include/ipxe/null_reboot.h b/roms/ipxe/src/include/ipxe/null_reboot.h
index 3de36c5b3..5de38afc0 100644
--- a/roms/ipxe/src/include/ipxe/null_reboot.h
+++ b/roms/ipxe/src/include/ipxe/null_reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef REBOOT_NULL
 #define REBOOT_PREFIX_null
diff --git a/roms/ipxe/src/include/ipxe/null_sanboot.h b/roms/ipxe/src/include/ipxe/null_sanboot.h
index 2b3a2c74d..58f03339f 100644
--- a/roms/ipxe/src/include/ipxe/null_sanboot.h
+++ b/roms/ipxe/src/include/ipxe/null_sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SANBOOT_NULL
 #define SANBOOT_PREFIX_null
diff --git a/roms/ipxe/src/include/ipxe/null_time.h b/roms/ipxe/src/include/ipxe/null_time.h
index 2b72cdf50..d2b15194b 100644
--- a/roms/ipxe/src/include/ipxe/null_time.h
+++ b/roms/ipxe/src/include/ipxe/null_time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIME_NULL
 #define TIME_PREFIX_null
diff --git a/roms/ipxe/src/include/ipxe/nvo.h b/roms/ipxe/src/include/ipxe/nvo.h
index 1a629da78..7a3c7a3db 100644
--- a/roms/ipxe/src/include/ipxe/nvo.h
+++ b/roms/ipxe/src/include/ipxe/nvo.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/dhcpopts.h>
diff --git a/roms/ipxe/src/include/ipxe/nvs.h b/roms/ipxe/src/include/ipxe/nvs.h
index 4733123cf..5789f4c0d 100644
--- a/roms/ipxe/src/include/ipxe/nvs.h
+++ b/roms/ipxe/src/include/ipxe/nvs.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/nvsvpd.h b/roms/ipxe/src/include/ipxe/nvsvpd.h
index 3450e5c71..4c50daf85 100644
--- a/roms/ipxe/src/include/ipxe/nvsvpd.h
+++ b/roms/ipxe/src/include/ipxe/nvsvpd.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/nvs.h>
 #include <ipxe/pcivpd.h>
diff --git a/roms/ipxe/src/include/ipxe/ocsp.h b/roms/ipxe/src/include/ipxe/ocsp.h
index 387e28f81..71fa41dc9 100644
--- a/roms/ipxe/src/include/ipxe/ocsp.h
+++ b/roms/ipxe/src/include/ipxe/ocsp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <time.h>
diff --git a/roms/ipxe/src/include/ipxe/oncrpc.h b/roms/ipxe/src/include/ipxe/oncrpc.h
index 76c1260f2..071468711 100644
--- a/roms/ipxe/src/include/ipxe/oncrpc.h
+++ b/roms/ipxe/src/include/ipxe/oncrpc.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** ONC RCP Version */
 #define ONCRPC_VERS 2
diff --git a/roms/ipxe/src/include/ipxe/oncrpc_iob.h b/roms/ipxe/src/include/ipxe/oncrpc_iob.h
index 4858d96b5..b55043770 100644
--- a/roms/ipxe/src/include/ipxe/oncrpc_iob.h
+++ b/roms/ipxe/src/include/ipxe/oncrpc_iob.h
@@ -13,7 +13,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Add a string to the end of an I/O buffer
diff --git a/roms/ipxe/src/include/ipxe/open.h b/roms/ipxe/src/include/ipxe/open.h
index a522f0cd1..43d4cdc66 100644
--- a/roms/ipxe/src/include/ipxe/open.h
+++ b/roms/ipxe/src/include/ipxe/open.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <ipxe/tables.h>
diff --git a/roms/ipxe/src/include/ipxe/params.h b/roms/ipxe/src/include/ipxe/params.h
index c2d82d9cf..dd3292efc 100644
--- a/roms/ipxe/src/include/ipxe/params.h
+++ b/roms/ipxe/src/include/ipxe/params.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/include/ipxe/parseopt.h b/roms/ipxe/src/include/ipxe/parseopt.h
index 840de7497..829b3431c 100644
--- a/roms/ipxe/src/include/ipxe/parseopt.h
+++ b/roms/ipxe/src/include/ipxe/parseopt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/roms/ipxe/src/include/ipxe/pccrc.h b/roms/ipxe/src/include/ipxe/pccrc.h
new file mode 100644
index 000000000..7f0963428
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/pccrc.h
@@ -0,0 +1,447 @@
+#ifndef _IPXE_PCCRC_H
+#define _IPXE_PCCRC_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Content Identification [MS-PCCRC]
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/crypto.h>
+
+/******************************************************************************
+ *
+ * Content Information versioning
+ *
+ ******************************************************************************
+ *
+ * Note that version 1 data structures are little-endian, but version
+ * 2 data structures are big-endian.
+ */
+
+/** Content Information version number */
+union peerdist_info_version {
+	/** Raw version number
+	 *
+	 * Always little-endian, regardless of whether the
+	 * encompassing structure is version 1 (little-endian) or
+	 * version 2 (big-endian).
+	 */
+	uint16_t raw;
+	/** Major:minor version number */
+	struct {
+		/** Minor version number */
+		uint8_t minor;
+		/** Major version number */
+		uint8_t major;
+	} __attribute__ (( packed ));
+} __attribute__ (( packed ));
+
+/** Content Information version 1 */
+#define PEERDIST_INFO_V1 0x0100
+
+/** Content Information version 2 */
+#define PEERDIST_INFO_V2 0x0200
+
+/******************************************************************************
+ *
+ * Content Information version 1
+ *
+ ******************************************************************************
+ */
+
+/** Content Information version 1 data structure header
+ *
+ * All fields are little-endian.
+ */
+struct peerdist_info_v1 {
+	/** Version number */
+	union peerdist_info_version version;
+	/** Hash algorithm
+	 *
+	 * This is a @c PEERDIST_INFO_V1_HASH_XXX constant.
+	 */
+	uint32_t hash;
+	/** Length to skip in first segment
+	 *
+	 * Length at the start of the first segment which is not
+	 * included within the content range.
+	 */
+	uint32_t first;
+	/** Length to read in last segment, or zero
+	 *
+	 * Length within the last segment which is included within the
+	 * content range.  A zero value indicates that the whole of
+	 * the last segment is included within the content range.
+	 */
+	uint32_t last;
+	/** Number of segments within the content information */
+	uint32_t segments;
+	/* Followed by a variable-length array of segment descriptions
+	 * and a list of variable-length block descriptions:
+	 *
+	 * peerdist_info_v1_segment_t(digestsize) segment[segments];
+	 * peerdist_info_v1_block_t(digestsize, block0.blocks) block0;
+	 * peerdist_info_v1_block_t(digestsize, block1.blocks) block1;
+	 * ...
+	 * peerdist_info_v1_block_t(digestsize, blockN.blocks) blockN;
+	 */
+} __attribute__ (( packed ));
+
+/** SHA-256 hash algorithm */
+#define PEERDIST_INFO_V1_HASH_SHA256 0x0000800cUL
+
+/** SHA-384 hash algorithm */
+#define PEERDIST_INFO_V1_HASH_SHA384 0x0000800dUL
+
+/** SHA-512 hash algorithm */
+#define PEERDIST_INFO_V1_HASH_SHA512 0x0000800eUL
+
+/** Content Information version 1 segment description header
+ *
+ * All fields are little-endian.
+ */
+struct peerdist_info_v1_segment {
+	/** Offset of this segment within the content */
+	uint64_t offset;
+	/** Length of this segment
+	 *
+	 * Should always be 32MB, except for the last segment within
+	 * the content.
+	 */
+	uint32_t len;
+	/** Block size for this segment
+	 *
+	 * Should always be 64kB.  Note that the last block within the
+	 * last segment may actually be less than 64kB.
+	 */
+	uint32_t blksize;
+	/* Followed by two variable-length hashes:
+	 *
+	 * uint8_t hash[digestsize];
+	 * uint8_t secret[digestsize];
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm.
+	 *
+	 * Note that the hash is taken over (the hashes of all blocks
+	 * within) the entire segment, even if the blocks do not
+	 * intersect the content range (and so do not appear within
+	 * the block list).  It therefore functions only as a segment
+	 * identifier; it cannot be used to verify the content of the
+	 * segment (since we may not download all blocks within the
+	 * segment).
+	 */
+} __attribute__ (( packed ));
+
+/** Content Information version 1 segment description
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_info_v1_segment_t( digestsize )			\
+	struct {							\
+		struct peerdist_info_v1_segment segment;		\
+		uint8_t hash[digestsize];				\
+		uint8_t secret[digestsize];				\
+	} __attribute__ (( packed ))
+
+/** Content Information version 1 block description header
+ *
+ * All fields are little-endian.
+ */
+struct peerdist_info_v1_block {
+	/** Number of blocks within the block description
+	 *
+	 * This is the number of blocks within the segment which
+	 * overlap the content range.  It may therefore be less than
+	 * the number of blocks within the segment.
+	 */
+	uint32_t blocks;
+	/* Followed by an array of variable-length hashes:
+	 *
+	 * uint8_t hash[blocks][digestsize];
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm.
+	 */
+ } __attribute__ (( packed ));
+
+/** Content Information version 1 block description
+ *
+ * @v digestsize	Digest size
+ * @v blocks		Number of blocks
+ */
+#define peerdist_info_v1_block_t( digestsize, blocks )			\
+	struct {							\
+		struct peerdist_info_v1_block block;			\
+		uint8_t hash[blocks][digestsize];			\
+	} __attribute__ (( packed ))
+
+/******************************************************************************
+ *
+ * Content Information version 2
+ *
+ ******************************************************************************
+ */
+
+/** Content Information version 2 data structure header
+ *
+ * All fields are big-endian.
+ */
+struct peerdist_info_v2 {
+	/** Version number */
+	union peerdist_info_version version;
+	/** Hash algorithm
+	 *
+	 * This is a @c PEERDIST_INFO_V2_HASH_XXX constant.
+	 */
+	uint8_t hash;
+	/** Offset of the first segment within the content */
+	uint64_t offset;
+	/** Index of the first segment within the content */
+	uint64_t index;
+	/** Length to skip in first segment
+	 *
+	 * Length at the start of the first segment which is not
+	 * included within the content range.
+	 */
+	uint32_t first;
+	/** Length of content range, or zero
+	 *
+	 * Length of the content range.  A zero indicates that
+	 * everything up to the end of the last segment is included in
+	 * the content range.
+	 */
+	uint64_t len;
+	/* Followed by a list of chunk descriptions */
+} __attribute__ (( packed ));
+
+/** SHA-512 hash algorithm with output truncated to first 256 bits */
+#define PEERDIST_INFO_V2_HASH_SHA512_TRUNC 0x04
+
+/** Content Information version 2 chunk description header
+ *
+ * All fields are big-endian.
+ */
+struct peerdist_info_v2_chunk {
+	/** Chunk type */
+	uint8_t type;
+	/** Chunk data length */
+	uint32_t len;
+	/* Followed by an array of segment descriptions:
+	 *
+	 * peerdist_info_v2_segment_t(digestsize) segment[segments]
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm, and segments is equal to @c len divided by the
+	 * size of each segment array entry.
+	 */
+} __attribute__ (( packed ));
+
+/** Content Information version 2 chunk description
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_info_v2_chunk_t( digestsize )				\
+	struct {							\
+		struct peerdist_info_v2_chunk chunk;			\
+		peerdist_info_v2_segment_t ( digestsize ) segment[0];	\
+	} __attribute__ (( packed ))
+
+/** Chunk type */
+#define PEERDIST_INFO_V2_CHUNK_TYPE 0x00
+
+/** Content Information version 2 segment description header
+ *
+ * All fields are big-endian.
+ */
+struct peerdist_info_v2_segment {
+	/** Segment length */
+	uint32_t len;
+	/* Followed by two variable-length hashes:
+	 *
+	 * uint8_t hash[digestsize];
+	 * uint8_t secret[digestsize];
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm.
+	 */
+} __attribute__ (( packed ));
+
+/** Content Information version 2 segment description
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_info_v2_segment_t( digestsize )			\
+	struct {							\
+		struct peerdist_info_v2_segment segment;		\
+		uint8_t hash[digestsize];				\
+		uint8_t secret[digestsize];				\
+	} __attribute__ (( packed ))
+
+/******************************************************************************
+ *
+ * Content Information
+ *
+ ******************************************************************************
+ */
+
+/** Maximum digest size for any supported algorithm
+ *
+ * The largest digest size that we support is for SHA-512 at 64 bytes
+ */
+#define PEERDIST_DIGEST_MAX_SIZE 64
+
+/** Raw content information */
+struct peerdist_raw {
+	/** Data buffer */
+	userptr_t data;
+	/** Length of data buffer */
+	size_t len;
+};
+
+/** A content range */
+struct peerdist_range {
+	/** Start offset */
+	size_t start;
+	/** End offset */
+	size_t end;
+};
+
+/** Content information */
+struct peerdist_info {
+	/** Raw content information */
+	struct peerdist_raw raw;
+
+	/** Content information operations */
+	struct peerdist_info_operations *op;
+	/** Digest algorithm */
+	struct digest_algorithm *digest;
+	/** Digest size
+	 *
+	 * Note that this may be shorter than the digest size of the
+	 * digest algorithm.  The truncation does not always take
+	 * place as soon as a digest is calculated.  For example,
+	 * version 2 content information uses SHA-512 with a truncated
+	 * digest size of 32 (256 bits), but the segment identifier
+	 * ("HoHoDk") is calculated by using HMAC with the full
+	 * SHA-512 digest and then truncating the HMAC output, rather
+	 * than by simply using HMAC with the truncated SHA-512
+	 * digest.  This is, of course, totally undocumented.
+	 */
+	size_t digestsize;
+	/** Content range */
+	struct peerdist_range range;
+	/** Trimmed content range */
+	struct peerdist_range trim;
+	/** Number of segments within the content information */
+	unsigned int segments;
+};
+
+/** A content information segment */
+struct peerdist_info_segment {
+	/** Content information */
+	const struct peerdist_info *info;
+	/** Segment index */
+	unsigned int index;
+
+	/** Content range
+	 *
+	 * Note that this range may exceed the overall content range.
+	 */
+	struct peerdist_range range;
+	/** Number of blocks within this segment */
+	unsigned int blocks;
+	/** Block size */
+	size_t blksize;
+	/** Segment hash of data
+	 *
+	 * This is MS-PCCRC's "HoD".
+	 */
+	uint8_t hash[PEERDIST_DIGEST_MAX_SIZE];
+	/** Segment secret
+	 *
+	 * This is MS-PCCRC's "Ke = Kp".
+	 */
+	uint8_t secret[PEERDIST_DIGEST_MAX_SIZE];
+	/** Segment identifier
+	 *
+	 * This is MS-PCCRC's "HoHoDk".
+	 */
+	uint8_t id[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/** Magic string constant used to calculate segment identifier
+ *
+ * Note that the MS-PCCRC specification states that this constant is
+ *
+ *   "the null-terminated ASCII string constant "MS_P2P_CACHING";
+ *    string literals are all ASCII strings with NULL terminators
+ *    unless otherwise noted."
+ *
+ * The specification lies.  This constant is a UTF-16LE string, not an
+ * ASCII string.  The terminating wNUL *is* included within the
+ * constant.
+ */
+#define PEERDIST_SEGMENT_ID_MAGIC L"MS_P2P_CACHING"
+
+/** A content information block */
+struct peerdist_info_block {
+	/** Content information segment */
+	const struct peerdist_info_segment *segment;
+	/** Block index */
+	unsigned int index;
+
+	/** Content range
+	 *
+	 * Note that this range may exceed the overall content range.
+	 */
+	struct peerdist_range range;
+	/** Trimmed content range */
+	struct peerdist_range trim;
+	/** Block hash */
+	uint8_t hash[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/** Content information operations */
+struct peerdist_info_operations {
+	/**
+	 * Populate content information
+	 *
+	 * @v info		Content information to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * info ) ( struct peerdist_info *info );
+	/**
+	 * Populate content information segment
+	 *
+	 * @v segment		Content information segment to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * segment ) ( struct peerdist_info_segment *segment );
+	/**
+	 * Populate content information block
+	 *
+	 * @v block		Content information block to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * block ) ( struct peerdist_info_block *block );
+};
+
+extern struct digest_algorithm sha512_trunc_algorithm;
+
+extern int peerdist_info ( userptr_t data, size_t len,
+			   struct peerdist_info *info );
+extern int peerdist_info_segment ( const struct peerdist_info *info,
+				   struct peerdist_info_segment *segment,
+				   unsigned int index );
+extern int peerdist_info_block ( const struct peerdist_info_segment *segment,
+				 struct peerdist_info_block *block,
+				 unsigned int index );
+
+#endif /* _IPXE_PCCRC_H */
diff --git a/roms/ipxe/src/include/ipxe/pccrd.h b/roms/ipxe/src/include/ipxe/pccrd.h
new file mode 100644
index 000000000..3daa92f29
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/pccrd.h
@@ -0,0 +1,47 @@
+#ifndef _IPXE_PCCRD_H
+#define _IPXE_PCCRD_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Discovery Protocol [MS-PCCRD]
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** PeerDist discovery port */
+#define PEERDIST_DISCOVERY_PORT 3702
+
+/** PeerDist discovery IPv4 address (239.255.255.250) */
+#define PEERDIST_DISCOVERY_IPV4 \
+	( ( 239 << 24 ) | ( 255 << 16 ) | ( 255 << 8 ) | ( 250 << 0 ) )
+
+/** PeerDist discovery IPv6 address (ff02::c) */
+#define PEERDIST_DISCOVERY_IPV6 \
+	{ 0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xc }
+
+/** A PeerDist discovery reply block count */
+struct peerdist_discovery_block_count {
+	/** Count (as an eight-digit hex value) */
+	char hex[8];
+} __attribute__ (( packed ));
+
+/** A PeerDist discovery reply */
+struct peerdist_discovery_reply {
+	/** List of segment ID strings
+	 *
+	 * The list is terminated with a zero-length string.
+	 */
+	char *ids;
+	/** List of peer locations
+	 *
+	 * The list is terminated with a zero-length string.
+	 */
+	char *locations;
+};
+
+extern char * peerdist_discovery_request ( const char *uuid, const char *id );
+extern int peerdist_discovery_reply ( char *data, size_t len,
+				      struct peerdist_discovery_reply *reply );
+
+#endif /* _IPXE_PCCRD_H */
diff --git a/roms/ipxe/src/include/ipxe/pccrr.h b/roms/ipxe/src/include/ipxe/pccrr.h
new file mode 100644
index 000000000..1ea86c40d
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/pccrr.h
@@ -0,0 +1,376 @@
+#ifndef _IPXE_PCCRR_H
+#define _IPXE_PCCRR_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Retrieval Protocol [MS-PCCRR]
+ *
+ * All fields are in network byte order.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/uaccess.h>
+
+/** Magic retrieval URI path */
+#define PEERDIST_MAGIC_PATH "/116B50EB-ECE2-41ac-8429-9F9E963361B7/"
+
+/** Retrieval protocol version */
+union peerdist_msg_version {
+	/** Raw version number */
+	uint32_t raw;
+	/** Major:minor version number */
+	struct {
+		/** Minor version number */
+		uint16_t minor;
+		/** Major version number */
+		uint16_t major;
+	} __attribute__ (( packed ));
+} __attribute__ (( packed ));
+
+/** Retrieval protocol version 1.0 */
+#define PEERDIST_MSG_VERSION_1_0 0x00000001UL
+
+/** Retrieval protocol version 2.0 */
+#define PEERDIST_MSG_VERSION_2_0 0x00000002UL
+
+/** Retrieval protocol supported versions */
+struct peerdist_msg_versions {
+	/** Minimum supported protocol version */
+	union peerdist_msg_version min;
+	/** Maximum supported protocol version */
+	union peerdist_msg_version max;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block range */
+struct peerdist_msg_range {
+	/** First block in range */
+	uint32_t first;
+	/** Number of blocks in range */
+	uint32_t count;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol segment ID header */
+struct peerdist_msg_segment {
+	/** Digest size (i.e. length of segment ID) */
+	uint32_t digestsize;
+	/* Followed by a single variable-length ID and padding:
+	 *
+	 * uint8_t id[digestsize];
+	 * uint8_t pad[ (-digestsize) & 0x3 ];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol segment ID
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_msg_segment_t( digestsize )				\
+	struct {							\
+		struct peerdist_msg_segment segment;			\
+		uint8_t id[digestsize];					\
+		uint8_t pad[ ( -(digestsize) ) & 0x3 ];			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block range list header */
+struct peerdist_msg_ranges {
+	/** Number of ranges */
+	uint32_t count;
+	/* Followed by an array of block ranges:
+	 *
+	 * struct peerdist_msg_range range[count];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block range list
+ *
+ * @v count		Number of ranges
+ */
+#define peerdist_msg_ranges_t( count )					\
+	struct {							\
+		struct peerdist_msg_ranges ranges;			\
+		struct peerdist_msg_range range[count];			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol data block header */
+struct peerdist_msg_block {
+	/** Length of data block */
+	uint32_t len;
+	/* Followed by the (encrypted) data block:
+	 *
+	 * uint8_t data[len];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol data block */
+#define peerdist_msg_block_t( len )					\
+	struct {							\
+		struct peerdist_msg_block block;			\
+		uint8_t data[len];					\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol initialisation vector header */
+struct peerdist_msg_iv {
+	/** Cipher block size */
+	uint32_t blksize;
+	/* Followed by the initialisation vector:
+	 *
+	 * uint8_t data[blksize];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol initialisation vector */
+#define peerdist_msg_iv_t( blksize )					\
+	struct {							\
+		struct peerdist_msg_iv iv;				\
+		uint8_t data[blksize];					\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol useless VRF data header */
+struct peerdist_msg_useless_vrf {
+	/** Length of useless VRF data */
+	uint32_t len;
+	/* Followed by a variable-length useless VRF data block and
+	 * padding:
+	 *
+	 * uint8_t data[len];
+	 * uint8_t pad[ (-len) & 0x3 ];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol useless VRF data */
+#define peerdist_msg_useless_vrf_t( vrf_len )				\
+	struct {							\
+		struct peerdist_msg_useless_vrf vrf;			\
+		uint8_t data[vrf_len];					\
+		uint8_t pad[ ( -(vrf_len) ) & 0x3 ];			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol message header */
+struct peerdist_msg_header {
+	/** Protocol version
+	 *
+	 * This is the protocol version in which the message type was
+	 * first defined.
+	 */
+	union peerdist_msg_version version;
+	/** Message type */
+	uint32_t type;
+	/** Message size (including this header) */
+	uint32_t len;
+	/** Cryptographic algorithm ID */
+	uint32_t algorithm;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol cryptographic algorithm IDs */
+enum peerdist_msg_algorithm {
+	/** No encryption */
+	PEERDIST_MSG_PLAINTEXT = 0x00000000UL,
+	/** AES-128 in CBC mode */
+	PEERDIST_MSG_AES_128_CBC = 0x00000001UL,
+	/** AES-192 in CBC mode */
+	PEERDIST_MSG_AES_192_CBC = 0x00000002UL,
+	/** AES-256 in CBC mode */
+	PEERDIST_MSG_AES_256_CBC = 0x00000003UL,
+};
+
+/** Retrieval protocol transport response header */
+struct peerdist_msg_transport_header {
+	/** Length (excluding this header)
+	 *
+	 * This seems to be identical in both purpose and value to the
+	 * length found within the message header, and therefore
+	 * serves no useful purpose.
+	 */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol negotiation request */
+struct peerdist_msg_nego_req {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/** Supported versions */
+	struct peerdist_msg_versions versions;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol negotiation request version */
+#define PEERDIST_MSG_NEGO_REQ_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol negotiation request type */
+#define PEERDIST_MSG_NEGO_REQ_TYPE 0x00000000UL
+
+/** Retrieval protocol negotiation response */
+struct peerdist_msg_nego_resp {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/** Supported versions */
+	struct peerdist_msg_versions versions;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol negotiation response version */
+#define PEERDIST_MSG_NEGO_RESP_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol negotiation response type */
+#define PEERDIST_MSG_NEGO_RESP_TYPE 0x00000001UL
+
+/** Retrieval protocol block list request header */
+struct peerdist_msg_getblklist {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID and a block range list:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * peerdist_msg_ranges_t(count) ranges;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block list request
+ *
+ * @v digestsize	Digest size
+ * @v count		Block range count
+ */
+#define peerdist_msg_getblklist_t( digestsize, count )			\
+	struct {							\
+		struct peerdist_msg_getblklist getblklist;		\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		peerdist_msg_ranges_t ( count ) ranges;			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block list request version */
+#define PEERDIST_MSG_GETBLKLIST_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block list request type */
+#define PEERDIST_MSG_GETBLKLIST_TYPE 0x00000002UL
+
+/** Retrieval protocol block fetch request header */
+struct peerdist_msg_getblks {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID, a block range list, and a useless
+	 * VRF block:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * peerdist_msg_ranges_t(count) ranges;
+	 * peerdist_msg_vrf_t(vrf_len) vrf;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block fetch request
+ *
+ * @v digestsize	Digest size
+ * @v count		Block range count
+ * @v vrf_len		Length of uselessness
+ */
+#define peerdist_msg_getblks_t( digestsize, count, vrf_len )		\
+	struct {							\
+		struct peerdist_msg_getblks getblks;			\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		peerdist_msg_ranges_t ( count ) ranges;			\
+		peerdist_msg_useless_vrf_t ( vrf_len );			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block fetch request version */
+#define PEERDIST_MSG_GETBLKS_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block fetch request type */
+#define PEERDIST_MSG_GETBLKS_TYPE 0x00000003UL
+
+/** Retrieval protocol block list response header */
+struct peerdist_msg_blklist {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID, a block range list, and a next
+	 * block index:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * peerdist_msg_ranges_t(count) ranges;
+	 * uint32_t next;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block list response
+ *
+ * @v digestsize	Digest size
+ * @v count		Block range count
+ */
+#define peerdist_msg_blklist_t( digestsize, count )			\
+	struct {							\
+		struct peerdist_msg_blklist blklist;			\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		peerdist_msg_ranges_t ( count ) ranges;			\
+		uint32_t next;						\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block list response version */
+#define PEERDIST_MSG_BLKLIST_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block list response type */
+#define PEERDIST_MSG_BLKLIST_TYPE 0x00000004UL
+
+/** Retrieval protocol block fetch response header */
+struct peerdist_msg_blk {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID, a block index, a next block
+	 * index, a data block, a useless VRF block, and an
+	 * initialisation vector:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * uint32_t index;
+	 * uint32_t next;
+	 * peerdist_msg_block_t(len) data;
+	 * peerdist_msg_useless_vrf_t(vrf_len) vrf;
+	 * peerdist_msg_iv_t(blksize) iv;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block fetch response
+ *
+ * @v digestsize	Digest size
+ * @v len		Data block length
+ * @v vrf_len		Length of uselessness
+ * @v blksize		Cipher block size
+ */
+#define peerdist_msg_blk_t( digestsize, len, vrf_len, blksize )		\
+	struct {							\
+		struct peerdist_msg_blk blk;				\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		uint32_t index;						\
+		uint32_t next;						\
+		peerdist_msg_block_t ( len ) block;			\
+		peerdist_msg_useless_vrf_t ( vrf_len ) vrf;		\
+		peerdist_msg_iv_t ( blksize ) iv;			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block fetch response version */
+#define PEERDIST_MSG_BLK_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block fetch response type */
+#define PEERDIST_MSG_BLK_TYPE 0x00000005UL
+
+/**
+ * Parse retrieval protocol block fetch response
+ *
+ * @v raw		Raw data
+ * @v raw_len		Length of raw data
+ * @v digestsize	Digest size
+ * @v blksize		Cipher block size
+ * @v blk		Structure to fill in
+ * @ret rc		Return status code
+ */
+#define peerdist_msg_blk( raw, raw_len, digestsize, blksize, blk ) ( {	\
+	assert ( sizeof ( (blk)->segment.id ) == (digestsize) );	\
+	assert ( sizeof ( (blk)->block.data ) == 0 );			\
+	assert ( sizeof ( (blk)->vrf.data ) == 0 );			\
+	assert ( sizeof ( (blk)->iv.data ) == blksize );		\
+	peerdist_msg_blk_untyped ( (raw), (raw_len), (digestsize),	\
+				   (blksize), blk );			\
+	} )
+
+extern int peerdist_msg_blk_untyped ( userptr_t raw, size_t raw_len,
+				      size_t digestsize, size_t blksize,
+				      void *out );
+
+#endif /* _IPXE_PCCRR_H */
diff --git a/roms/ipxe/src/include/ipxe/pci.h b/roms/ipxe/src/include/ipxe/pci.h
index 692771ebe..a841e00ff 100644
--- a/roms/ipxe/src/include/ipxe/pci.h
+++ b/roms/ipxe/src/include/ipxe/pci.h
@@ -1,268 +1,132 @@
 #ifndef	_IPXE_PCI_H
 #define _IPXE_PCI_H
 
-/*
- * Support for NE2000 PCI clones added David Monro June 1997
- * Generalised for other PCI NICs by Ken Yap July 1997
- * PCI support rewritten by Michael Brown 2006
+/** @file
+ *
+ * PCI bus
  *
- * Most of this is taken from /usr/src/linux/include/linux/pci.h.
- */
-
-/*
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
  */
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/device.h>
 #include <ipxe/tables.h>
 #include <ipxe/pci_io.h>
-#include "pci_ids.h"
 
-/*
- * PCI constants
- *
- */
-
-#define PCI_COMMAND_IO			0x1	/* Enable response in I/O space */
-#define PCI_COMMAND_MEM			0x2	/* Enable response in mem space */
-#define PCI_COMMAND_MASTER		0x4	/* Enable bus mastering */
-
-#define PCI_CACHE_LINE_SIZE		0x0c	/* 8 bits */
-#define PCI_LATENCY_TIMER		0x0d	/* 8 bits */
-
-#define PCI_COMMAND_SPECIAL		0x8	/* Enable response to special cycles */
-#define PCI_COMMAND_INVALIDATE		0x10	/* Use memory write and invalidate */
-#define  PCI_COMMAND_VGA_PALETTE 0x20	/* Enable palette snooping */
-#define  PCI_COMMAND_PARITY	0x40	/* Enable parity checking */
-#define  PCI_COMMAND_WAIT 	0x80	/* Enable address/data stepping */
-#define  PCI_COMMAND_SERR	0x100	/* Enable SERR */
-#define  PCI_COMMAND_FAST_BACK	0x200	/* Enable back-to-back writes */
-#define  PCI_COMMAND_INTX_DISABLE 0x400 /* INTx Emulation Disable */
-
-
-#define PCI_VENDOR_ID           0x00    /* 16 bits */
-#define PCI_DEVICE_ID           0x02    /* 16 bits */
-#define PCI_COMMAND             0x04    /* 16 bits */
-
-#define PCI_STATUS		0x06	/* 16 bits */
-#define  PCI_STATUS_CAP_LIST	0x10	/* Support Capability List */
-#define  PCI_STATUS_66MHZ	0x20	/* Support 66 Mhz PCI 2.1 bus */
-#define  PCI_STATUS_UDF		0x40	/* Support User Definable Features [obsolete] */
-#define  PCI_STATUS_FAST_BACK	0x80	/* Accept fast-back to back */
-#define  PCI_STATUS_PARITY	0x100	/* Detected parity error */
-#define  PCI_STATUS_DEVSEL_MASK	0x600	/* DEVSEL timing */
-#define  PCI_STATUS_DEVSEL_FAST	0x000	
-#define  PCI_STATUS_DEVSEL_MEDIUM 0x200
-#define  PCI_STATUS_DEVSEL_SLOW 0x400
-#define  PCI_STATUS_SIG_TARGET_ABORT 0x800 /* Set on target abort */
-#define  PCI_STATUS_REC_TARGET_ABORT 0x1000 /* Master ack of " */
-#define  PCI_STATUS_REC_MASTER_ABORT 0x2000 /* Set on master abort */
-#define  PCI_STATUS_SIG_SYSTEM_ERROR 0x4000 /* Set when we drive SERR */
-#define  PCI_STATUS_DETECTED_PARITY 0x8000 /* Set on parity error */
-
-#define PCI_REVISION            0x08    /* 8 bits  */
-#define PCI_REVISION_ID         0x08    /* 8 bits  */
-#define PCI_CLASS_REVISION      0x08    /* 32 bits  */
-#define PCI_CLASS_CODE          0x0b    /* 8 bits */
-#define PCI_SUBCLASS_CODE       0x0a    /* 8 bits */
-#define PCI_HEADER_TYPE         0x0e    /* 8 bits */
-#define  PCI_HEADER_TYPE_NORMAL	0
-#define  PCI_HEADER_TYPE_BRIDGE 1
-#define  PCI_HEADER_TYPE_CARDBUS 2
-
-
-/* Header type 0 (normal devices) */
-#define PCI_CARDBUS_CIS		0x28
+/** PCI vendor ID */
+#define PCI_VENDOR_ID		0x00
+
+/** PCI device ID */
+#define PCI_DEVICE_ID		0x02
+
+/** PCI command */
+#define PCI_COMMAND		0x04
+#define PCI_COMMAND_IO			0x0001	/**< I/O space */
+#define PCI_COMMAND_MEM			0x0002	/**< Memory space */
+#define PCI_COMMAND_MASTER		0x0004	/**< Bus master */
+#define PCI_COMMAND_INVALIDATE		0x0010	/**< Mem. write & invalidate */
+#define PCI_COMMAND_PARITY		0x0040	/**< Parity error response */
+#define PCI_COMMAND_SERR		0x0100	/**< SERR# enable */
+#define PCI_COMMAND_INTX_DISABLE	0x0400	/**< Interrupt disable */
+
+/** PCI status */
+#define PCI_STATUS		0x06
+#define PCI_STATUS_CAP_LIST		0x0010	/**< Capabilities list */
+#define PCI_STATUS_PARITY		0x0100	/**< Master data parity error */
+#define PCI_STATUS_REC_TARGET_ABORT	0x1000	/**< Received target abort */
+#define PCI_STATUS_REC_MASTER_ABORT	0x2000	/**< Received master abort */
+#define PCI_STATUS_SIG_SYSTEM_ERROR	0x4000	/**< Signalled system error */
+#define PCI_STATUS_DETECTED_PARITY	0x8000	/**< Detected parity error */
+
+/** PCI revision */
+#define PCI_REVISION		0x08
+
+/** PCI cache line size */
+#define PCI_CACHE_LINE_SIZE	0x0c
+
+/** PCI latency timer */
+#define PCI_LATENCY_TIMER	0x0d
+
+/** PCI header type */
+#define PCI_HEADER_TYPE         0x0e
+#define PCI_HEADER_TYPE_NORMAL		0x00	/**< Normal header */
+#define PCI_HEADER_TYPE_BRIDGE		0x01	/**< PCI-to-PCI bridge header */
+#define PCI_HEADER_TYPE_CARDBUS		0x02	/**< CardBus header */
+#define PCI_HEADER_TYPE_MASK		0x7f	/**< Header type mask */
+#define PCI_HEADER_TYPE_MULTI		0x80	/**< Multi-function device */
+
+/** PCI base address registers */
+#define PCI_BASE_ADDRESS(n)	( 0x10 + ( 4 * (n) ) )
+#define PCI_BASE_ADDRESS_0	PCI_BASE_ADDRESS ( 0 )
+#define PCI_BASE_ADDRESS_1	PCI_BASE_ADDRESS ( 1 )
+#define PCI_BASE_ADDRESS_2	PCI_BASE_ADDRESS ( 2 )
+#define PCI_BASE_ADDRESS_3	PCI_BASE_ADDRESS ( 3 )
+#define PCI_BASE_ADDRESS_4	PCI_BASE_ADDRESS ( 4 )
+#define PCI_BASE_ADDRESS_5	PCI_BASE_ADDRESS ( 5 )
+#define PCI_BASE_ADDRESS_SPACE_IO	0x00000001UL	/**< I/O BAR */
+#define	PCI_BASE_ADDRESS_IO_MASK	0x00000003UL	/**< I/O BAR mask */
+#define PCI_BASE_ADDRESS_MEM_TYPE_64	0x00000004UL	/**< 64-bit memory */
+#define PCI_BASE_ADDRESS_MEM_TYPE_MASK	0x00000006UL	/**< Memory type mask */
+#define	PCI_BASE_ADDRESS_MEM_MASK	0x0000000fUL	/**< Memory BAR mask */
+
+/** PCI subsystem vendor ID */
 #define PCI_SUBSYSTEM_VENDOR_ID	0x2c
+
+/** PCI subsystem ID */
 #define PCI_SUBSYSTEM_ID	0x2e  
 
-#define PCI_BASE_ADDRESS_0      0x10    /* 32 bits */
-#define PCI_BASE_ADDRESS_1      0x14    /* 32 bits */
-#define PCI_BASE_ADDRESS_2      0x18    /* 32 bits */
-#define PCI_BASE_ADDRESS_3      0x1c    /* 32 bits */
-#define PCI_BASE_ADDRESS_4      0x20    /* 32 bits */
-#define PCI_BASE_ADDRESS_5      0x24    /* 32 bits */
-
-#define PCI_BASE_ADDRESS_SPACE		0x01    /* 0 = memory, 1 = I/O */
-#define PCI_BASE_ADDRESS_SPACE_IO	0x01
-#define PCI_BASE_ADDRESS_SPACE_MEMORY	0x00
-
-#define PCI_BASE_ADDRESS_MEM_TYPE_MASK	0x06
-#define PCI_BASE_ADDRESS_MEM_TYPE_32	0x00	/* 32 bit address */
-#define PCI_BASE_ADDRESS_MEM_TYPE_1M	0x02	/* Below 1M [obsolete] */
-#define PCI_BASE_ADDRESS_MEM_TYPE_64	0x04	/* 64 bit address */
-#define	PCI_BASE_ADDRESS_MEM_MASK	(~0x0f)
-#define	PCI_BASE_ADDRESS_IO_MASK	(~0x03)
-#define	PCI_ROM_ADDRESS		0x30	/* 32 bits */
-#define	PCI_ROM_ADDRESS_ENABLE	0x01	/* Write 1 to enable ROM,
-					   bits 31..11 are address,
-					   10..2 are reserved */
-
-#define PCI_CAPABILITY_LIST	0x34	/* Offset of first capability list entry */
-
-#define PCI_INTERRUPT_LINE	0x3c	/* IRQ number (0-15) */
-#define PCI_INTERRUPT_PIN	0x3d	/* IRQ pin on PCI bus (A-D) */
-
-/* Header type 1 (PCI-to-PCI bridges) */
-#define PCI_PRIMARY_BUS		0x18	/* Primary bus number */
-#define PCI_SECONDARY_BUS	0x19	/* Secondary bus number */
-#define PCI_SUBORDINATE_BUS	0x1a	/* Highest bus number behind the bridge */
-#define PCI_SEC_LATENCY_TIMER	0x1b	/* Latency timer for secondary interface */
-#define PCI_IO_BASE		0x1c	/* I/O range behind the bridge */
-#define PCI_IO_LIMIT		0x1d
-#define  PCI_IO_RANGE_TYPE_MASK	0x0f	/* I/O bridging type */
-#define  PCI_IO_RANGE_TYPE_16	0x00
-#define  PCI_IO_RANGE_TYPE_32	0x01
-#define  PCI_IO_RANGE_MASK	~0x0f
-#define PCI_SEC_STATUS		0x1e	/* Secondary status register, only bit 14 used */
-#define PCI_MEMORY_BASE		0x20	/* Memory range behind */
-#define PCI_MEMORY_LIMIT	0x22
-#define  PCI_MEMORY_RANGE_TYPE_MASK 0x0f
-#define  PCI_MEMORY_RANGE_MASK	~0x0f
-#define PCI_PREF_MEMORY_BASE	0x24	/* Prefetchable memory range behind */
-#define PCI_PREF_MEMORY_LIMIT	0x26
-#define  PCI_PREF_RANGE_TYPE_MASK 0x0f
-#define  PCI_PREF_RANGE_TYPE_32	0x00
-#define  PCI_PREF_RANGE_TYPE_64	0x01
-#define  PCI_PREF_RANGE_MASK	~0x0f
-#define PCI_PREF_BASE_UPPER32	0x28	/* Upper half of prefetchable memory range */
-#define PCI_PREF_LIMIT_UPPER32	0x2c
-#define PCI_IO_BASE_UPPER16	0x30	/* Upper half of I/O addresses */
-#define PCI_IO_LIMIT_UPPER16	0x32
-/* 0x34 same as for htype 0 */
-/* 0x35-0x3b is reserved */
-#define PCI_ROM_ADDRESS1	0x38	/* Same as PCI_ROM_ADDRESS, but for htype 1 */
-/* 0x3c-0x3d are same as for htype 0 */
-#define PCI_BRIDGE_CONTROL	0x3e
-#define  PCI_BRIDGE_CTL_PARITY	0x01	/* Enable parity detection on secondary interface */
-#define  PCI_BRIDGE_CTL_SERR	0x02	/* The same for SERR forwarding */
-#define  PCI_BRIDGE_CTL_NO_ISA	0x04	/* Disable bridging of ISA ports */
-#define  PCI_BRIDGE_CTL_VGA	0x08	/* Forward VGA addresses */
-#define  PCI_BRIDGE_CTL_MASTER_ABORT 0x20  /* Report master aborts */
-#define  PCI_BRIDGE_CTL_BUS_RESET 0x40	/* Secondary bus reset */
-#define  PCI_BRIDGE_CTL_FAST_BACK 0x80	/* Fast Back2Back enabled on secondary interface */
+/** PCI expansion ROM base address */
+#define	PCI_ROM_ADDRESS		0x30
 
+/** PCI capabilities pointer */
+#define PCI_CAPABILITY_LIST	0x34
+
+/** CardBus capabilities pointer */
 #define PCI_CB_CAPABILITY_LIST	0x14
 
-/* Capability lists */
-
-#define PCI_CAP_LIST_ID		0	/* Capability ID */
-#define  PCI_CAP_ID_PM		0x01	/* Power Management */
-#define  PCI_CAP_ID_AGP		0x02	/* Accelerated Graphics Port */
-#define  PCI_CAP_ID_VPD		0x03	/* Vital Product Data */
-#define  PCI_CAP_ID_SLOTID	0x04	/* Slot Identification */
-#define  PCI_CAP_ID_MSI		0x05	/* Message Signalled Interrupts */
-#define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
-#define  PCI_CAP_ID_VNDR	0x09	/* Vendor specific */
-#define  PCI_CAP_ID_EXP		0x10	/* PCI Express */
-#define PCI_CAP_LIST_NEXT	1	/* Next capability in the list */
-#define PCI_CAP_FLAGS		2	/* Capability defined flags (16 bits) */
-#define PCI_CAP_SIZEOF		4
-
-/* Power Management Registers */
-
-#define PCI_PM_PMC              2       /* PM Capabilities Register */
-#define  PCI_PM_CAP_VER_MASK	0x0007	/* Version */
-#define  PCI_PM_CAP_PME_CLOCK	0x0008	/* PME clock required */
-#define  PCI_PM_CAP_RESERVED    0x0010  /* Reserved field */
-#define  PCI_PM_CAP_DSI		0x0020	/* Device specific initialization */
-#define  PCI_PM_CAP_AUX_POWER	0x01C0	/* Auxiliary power support mask */
-#define  PCI_PM_CAP_D1		0x0200	/* D1 power state support */
-#define  PCI_PM_CAP_D2		0x0400	/* D2 power state support */
-#define  PCI_PM_CAP_PME		0x0800	/* PME pin supported */
-#define  PCI_PM_CAP_PME_MASK    0xF800  /* PME Mask of all supported states */
-#define  PCI_PM_CAP_PME_D0      0x0800  /* PME# from D0 */
-#define  PCI_PM_CAP_PME_D1      0x1000  /* PME# from D1 */
-#define  PCI_PM_CAP_PME_D2      0x2000  /* PME# from D2 */
-#define  PCI_PM_CAP_PME_D3      0x4000  /* PME# from D3 (hot) */
-#define  PCI_PM_CAP_PME_D3cold  0x8000  /* PME# from D3 (cold) */
-#define PCI_PM_CTRL		4	/* PM control and status register */
-#define  PCI_PM_CTRL_STATE_MASK	0x0003	/* Current power state (D0 to D3) */
-#define  PCI_PM_CTRL_PME_ENABLE	0x0100	/* PME pin enable */
-#define  PCI_PM_CTRL_DATA_SEL_MASK	0x1e00	/* Data select (??) */
-#define  PCI_PM_CTRL_DATA_SCALE_MASK	0x6000	/* Data scale (??) */
-#define  PCI_PM_CTRL_PME_STATUS	0x8000	/* PME pin status */
-#define PCI_PM_PPB_EXTENSIONS	6	/* PPB support extensions (??) */
-#define  PCI_PM_PPB_B2_B3	0x40	/* Stop clock when in D3hot (??) */
-#define  PCI_PM_BPCC_ENABLE	0x80	/* Bus power/clock control enable (??) */
-#define PCI_PM_DATA_REGISTER	7	/* (??) */
-#define PCI_PM_SIZEOF		8
-
-/* AGP registers */
-
-#define PCI_AGP_VERSION		2	/* BCD version number */
-#define PCI_AGP_RFU		3	/* Rest of capability flags */
-#define PCI_AGP_STATUS		4	/* Status register */
-#define  PCI_AGP_STATUS_RQ_MASK	0xff000000	/* Maximum number of requests - 1 */
-#define  PCI_AGP_STATUS_SBA	0x0200	/* Sideband addressing supported */
-#define  PCI_AGP_STATUS_64BIT	0x0020	/* 64-bit addressing supported */
-#define  PCI_AGP_STATUS_FW	0x0010	/* FW transfers supported */
-#define  PCI_AGP_STATUS_RATE4	0x0004	/* 4x transfer rate supported */
-#define  PCI_AGP_STATUS_RATE2	0x0002	/* 2x transfer rate supported */
-#define  PCI_AGP_STATUS_RATE1	0x0001	/* 1x transfer rate supported */
-#define PCI_AGP_COMMAND		8	/* Control register */
-#define  PCI_AGP_COMMAND_RQ_MASK 0xff000000  /* Master: Maximum number of requests */
-#define  PCI_AGP_COMMAND_SBA	0x0200	/* Sideband addressing enabled */
-#define  PCI_AGP_COMMAND_AGP	0x0100	/* Allow processing of AGP transactions */
-#define  PCI_AGP_COMMAND_64BIT	0x0020 	/* Allow processing of 64-bit addresses */
-#define  PCI_AGP_COMMAND_FW	0x0010 	/* Force FW transfers */
-#define  PCI_AGP_COMMAND_RATE4	0x0004	/* Use 4x rate */
-#define  PCI_AGP_COMMAND_RATE2	0x0002	/* Use 2x rate */
-#define  PCI_AGP_COMMAND_RATE1	0x0001	/* Use 1x rate */
-#define PCI_AGP_SIZEOF		12
-
-/* Slot Identification */
-
-#define PCI_SID_ESR		2	/* Expansion Slot Register */
-#define  PCI_SID_ESR_NSLOTS	0x1f	/* Number of expansion slots available */
-#define  PCI_SID_ESR_FIC	0x20	/* First In Chassis Flag */
-#define PCI_SID_CHASSIS_NR	3	/* Chassis Number */
-
-/* Message Signalled Interrupts registers */
-
-#define PCI_MSI_FLAGS		2	/* Various flags */
-#define  PCI_MSI_FLAGS_64BIT	0x80	/* 64-bit addresses allowed */
-#define  PCI_MSI_FLAGS_QSIZE	0x70	/* Message queue size configured */
-#define  PCI_MSI_FLAGS_QMASK	0x0e	/* Maximum queue size available */
-#define  PCI_MSI_FLAGS_ENABLE	0x01	/* MSI feature enabled */
-#define PCI_MSI_RFU		3	/* Rest of capability flags */
-#define PCI_MSI_ADDRESS_LO	4	/* Lower 32 bits */
-#define PCI_MSI_ADDRESS_HI	8	/* Upper 32 bits (if PCI_MSI_FLAGS_64BIT set) */
-#define PCI_MSI_DATA_32		8	/* 16 bits of data for 32-bit devices */
-#define PCI_MSI_DATA_64		12	/* 16 bits of data for 64-bit devices */
-
-/* Advanced Error Reporting */
-
-#define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */
-#define  PCI_ERR_UNC_TRAIN	0x00000001	/* Training */
-#define  PCI_ERR_UNC_DLP	0x00000010	/* Data Link Protocol */
-#define  PCI_ERR_UNC_POISON_TLP	0x00001000	/* Poisoned TLP */
-#define  PCI_ERR_UNC_FCP	0x00002000	/* Flow Control Protocol */
-#define  PCI_ERR_UNC_COMP_TIME	0x00004000	/* Completion Timeout */
-#define  PCI_ERR_UNC_COMP_ABORT	0x00008000	/* Completer Abort */
-#define  PCI_ERR_UNC_UNX_COMP	0x00010000	/* Unexpected Completion */
-#define  PCI_ERR_UNC_RX_OVER	0x00020000	/* Receiver Overflow */
-#define  PCI_ERR_UNC_MALF_TLP	0x00040000	/* Malformed TLP */
-#define  PCI_ERR_UNC_ECRC	0x00080000	/* ECRC Error Status */
-#define  PCI_ERR_UNC_UNSUP	0x00100000	/* Unsupported Request */
-#define PCI_ERR_UNCOR_MASK	8	/* Uncorrectable Error Mask */
-	/* Same bits as above */
-#define PCI_ERR_UNCOR_SEVER	12	/* Uncorrectable Error Severity */
-	/* Same bits as above */
-#define PCI_ERR_COR_STATUS	16	/* Correctable Error Status */
-#define  PCI_ERR_COR_RCVR	0x00000001	/* Receiver Error Status */
-#define  PCI_ERR_COR_BAD_TLP	0x00000040	/* Bad TLP Status */
-#define  PCI_ERR_COR_BAD_DLLP	0x00000080	/* Bad DLLP Status */
-#define  PCI_ERR_COR_REP_ROLL	0x00000100	/* REPLAY_NUM Rollover */
-#define  PCI_ERR_COR_REP_TIMER	0x00001000	/* Replay Timer Timeout */
-#define PCI_ERR_COR_MASK	20	/* Correctable Error Mask */
-	/* Same bits as above */
+/** PCI interrupt line */
+#define PCI_INTERRUPT_LINE	0x3c
+
+/** Capability ID */
+#define PCI_CAP_ID		0x00
+#define PCI_CAP_ID_PM			0x01	/**< Power management */
+#define PCI_CAP_ID_VPD			0x03	/**< Vital product data */
+#define PCI_CAP_ID_VNDR			0x09	/**< Vendor-specific */
+#define PCI_CAP_ID_EXP			0x10	/**< PCI Express */
+
+/** Next capability */
+#define PCI_CAP_NEXT		0x01
+
+/** Power management control and status */
+#define PCI_PM_CTRL		0x04
+#define PCI_PM_CTRL_STATE_MASK		0x0003	/**< Current power state */
+#define PCI_PM_CTRL_PME_ENABLE		0x0100	/**< PME pin enable */
+#define PCI_PM_CTRL_PME_STATUS		0x8000	/**< PME pin status */
+
+/** Uncorrectable error status */
+#define PCI_ERR_UNCOR_STATUS	0x04
+
+/** Network controller */
+#define PCI_CLASS_NETWORK	0x02
+
+/** Serial bus controller */
+#define PCI_CLASS_SERIAL	0x0c
+#define PCI_CLASS_SERIAL_USB		0x03	/**< USB controller */
+#define PCI_CLASS_SERIAL_USB_UHCI	 0x00	/**< UHCI USB controller */
+#define PCI_CLASS_SERIAL_USB_OHCI	 0x10	/**< OHCI USB controller */
+#define PCI_CLASS_SERIAL_USB_EHCI	 0x20	/**< ECHI USB controller */
+#define PCI_CLASS_SERIAL_USB_XHCI	 0x30	/**< xHCI USB controller */
+
+/** Construct PCI class
+ *
+ * @v base		Base class (or PCI_ANY_ID)
+ * @v sub		Subclass (or PCI_ANY_ID)
+ * @v progif		Programming interface (or PCI_ANY_ID)
+ */
+#define PCI_CLASS( base, sub, progif )					\
+	( ( ( (base) & 0xff ) << 16 ) |	( ( (sub) & 0xff ) << 8 ) |	\
+	  ( ( (progif) & 0xff) << 0 ) )
 
 /** A PCI device ID list entry */
 struct pci_device_id {
@@ -279,6 +143,27 @@ struct pci_device_id {
 /** Match-anything ID */
 #define PCI_ANY_ID 0xffff
 
+/** A PCI class ID */
+struct pci_class_id {
+	/** Class */
+	uint32_t class;
+	/** Class mask */
+	uint32_t mask;
+};
+
+/** Construct PCI class ID
+ *
+ * @v base		Base class (or PCI_ANY_ID)
+ * @v sub		Subclass (or PCI_ANY_ID)
+ * @v progif		Programming interface (or PCI_ANY_ID)
+ */
+#define PCI_CLASS_ID( base, sub, progif ) {				   \
+	.class = PCI_CLASS ( base, sub, progif ),			   \
+	.mask = ( ( ( ( (base) == PCI_ANY_ID ) ? 0x00 : 0xff ) << 16 ) |   \
+		  ( ( ( (sub) == PCI_ANY_ID ) ? 0x00 : 0xff ) << 8 ) |	   \
+		  ( ( ( (progif) == PCI_ANY_ID ) ? 0x00 : 0xff ) << 0 ) ), \
+	}
+
 /** A PCI device */
 struct pci_device {
 	/** Generic device */
@@ -322,6 +207,8 @@ struct pci_driver {
 	struct pci_device_id *ids;
 	/** Number of entries in PCI ID table */
 	unsigned int id_count;
+	/** PCI class ID */
+	struct pci_class_id class;
 	/**
 	 * Probe device
 	 *
@@ -352,6 +239,7 @@ struct pci_driver {
 #define PCI_BUSDEVFN( bus, slot, func )	\
 	( ( (bus) << 8 ) | ( (slot) << 3 ) | ( (func) << 0 ) )
 #define PCI_FIRST_FUNC( busdevfn )	( (busdevfn) & ~0x07 )
+#define PCI_LAST_FUNC( busdevfn )	( (busdevfn) | 0x07 )
 
 #define PCI_BASE_CLASS( class )		( (class) >> 16 )
 #define PCI_SUB_CLASS( class )		( ( (class) >> 8 ) & 0xff )
diff --git a/roms/ipxe/src/include/ipxe/pci_ids.h b/roms/ipxe/src/include/ipxe/pci_ids.h
deleted file mode 100644
index 25c7782bc..000000000
--- a/roms/ipxe/src/include/ipxe/pci_ids.h
+++ /dev/null
@@ -1,351 +0,0 @@
-#ifndef _IPXE_PCI_IDS_H
-#define _IPXE_PCI_IDS_H
-
-/*
- *	PCI Class, Vendor and Device IDs
- *
- *	Please keep sorted.
- */
-
-FILE_LICENCE ( GPL2_ONLY );
-
-/* Device classes and subclasses */
-
-#define PCI_CLASS_NOT_DEFINED		0x0000
-#define PCI_CLASS_NOT_DEFINED_VGA	0x0001
-
-#define PCI_BASE_CLASS_STORAGE		0x01
-#define PCI_CLASS_STORAGE_SCSI		0x0100
-#define PCI_CLASS_STORAGE_IDE		0x0101
-#define PCI_CLASS_STORAGE_FLOPPY	0x0102
-#define PCI_CLASS_STORAGE_IPI		0x0103
-#define PCI_CLASS_STORAGE_RAID		0x0104
-#define PCI_CLASS_STORAGE_OTHER		0x0180
-
-#define PCI_BASE_CLASS_NETWORK		0x02
-#define PCI_CLASS_NETWORK_ETHERNET	0x0200
-#define PCI_CLASS_NETWORK_TOKEN_RING	0x0201
-#define PCI_CLASS_NETWORK_FDDI		0x0202
-#define PCI_CLASS_NETWORK_ATM		0x0203
-#define PCI_CLASS_NETWORK_OTHER		0x0280
-
-#define PCI_BASE_CLASS_DISPLAY		0x03
-#define PCI_CLASS_DISPLAY_VGA		0x0300
-#define PCI_CLASS_DISPLAY_XGA		0x0301
-#define PCI_CLASS_DISPLAY_3D		0x0302
-#define PCI_CLASS_DISPLAY_OTHER		0x0380
-
-#define PCI_BASE_CLASS_MULTIMEDIA	0x04
-#define PCI_CLASS_MULTIMEDIA_VIDEO	0x0400
-#define PCI_CLASS_MULTIMEDIA_AUDIO	0x0401
-#define PCI_CLASS_MULTIMEDIA_PHONE	0x0402
-#define PCI_CLASS_MULTIMEDIA_OTHER	0x0480
-
-#define PCI_BASE_CLASS_MEMORY		0x05
-#define PCI_CLASS_MEMORY_RAM		0x0500
-#define PCI_CLASS_MEMORY_FLASH		0x0501
-#define PCI_CLASS_MEMORY_OTHER		0x0580
-
-#define PCI_BASE_CLASS_BRIDGE		0x06
-#define PCI_CLASS_BRIDGE_HOST		0x0600
-#define PCI_CLASS_BRIDGE_ISA		0x0601
-#define PCI_CLASS_BRIDGE_EISA		0x0602
-#define PCI_CLASS_BRIDGE_MC		0x0603
-#define PCI_CLASS_BRIDGE_PCI		0x0604
-#define PCI_CLASS_BRIDGE_PCMCIA		0x0605
-#define PCI_CLASS_BRIDGE_NUBUS		0x0606
-#define PCI_CLASS_BRIDGE_CARDBUS	0x0607
-#define PCI_CLASS_BRIDGE_RACEWAY	0x0608
-#define PCI_CLASS_BRIDGE_OTHER		0x0680
-
-#define PCI_BASE_CLASS_COMMUNICATION	0x07
-#define PCI_CLASS_COMMUNICATION_SERIAL	0x0700
-#define PCI_CLASS_COMMUNICATION_PARALLEL 0x0701
-#define PCI_CLASS_COMMUNICATION_MULTISERIAL 0x0702
-#define PCI_CLASS_COMMUNICATION_MODEM	0x0703
-#define PCI_CLASS_COMMUNICATION_OTHER	0x0780
-
-#define PCI_BASE_CLASS_SYSTEM		0x08
-#define PCI_CLASS_SYSTEM_PIC		0x0800
-#define PCI_CLASS_SYSTEM_DMA		0x0801
-#define PCI_CLASS_SYSTEM_TIMER		0x0802
-#define PCI_CLASS_SYSTEM_RTC		0x0803
-#define PCI_CLASS_SYSTEM_PCI_HOTPLUG	0x0804
-#define PCI_CLASS_SYSTEM_OTHER		0x0880
-
-#define PCI_BASE_CLASS_INPUT		0x09
-#define PCI_CLASS_INPUT_KEYBOARD	0x0900
-#define PCI_CLASS_INPUT_PEN		0x0901
-#define PCI_CLASS_INPUT_MOUSE		0x0902
-#define PCI_CLASS_INPUT_SCANNER		0x0903
-#define PCI_CLASS_INPUT_GAMEPORT	0x0904
-#define PCI_CLASS_INPUT_OTHER		0x0980
-
-#define PCI_BASE_CLASS_DOCKING		0x0a
-#define PCI_CLASS_DOCKING_GENERIC	0x0a00
-#define PCI_CLASS_DOCKING_OTHER		0x0a80
-
-#define PCI_BASE_CLASS_PROCESSOR	0x0b
-#define PCI_CLASS_PROCESSOR_386		0x0b00
-#define PCI_CLASS_PROCESSOR_486		0x0b01
-#define PCI_CLASS_PROCESSOR_PENTIUM	0x0b02
-#define PCI_CLASS_PROCESSOR_ALPHA	0x0b10
-#define PCI_CLASS_PROCESSOR_POWERPC	0x0b20
-#define PCI_CLASS_PROCESSOR_MIPS	0x0b30
-#define PCI_CLASS_PROCESSOR_CO		0x0b40
-
-#define PCI_BASE_CLASS_SERIAL		0x0c
-#define PCI_CLASS_SERIAL_FIREWIRE	0x0c00
-#define PCI_CLASS_SERIAL_ACCESS		0x0c01
-#define PCI_CLASS_SERIAL_SSA		0x0c02
-#define PCI_CLASS_SERIAL_USB		0x0c03
-#define PCI_CLASS_SERIAL_FIBER		0x0c04
-#define PCI_CLASS_SERIAL_SMBUS		0x0c05
-
-#define PCI_BASE_CLASS_INTELLIGENT	0x0e
-#define PCI_CLASS_INTELLIGENT_I2O	0x0e00
-
-#define PCI_BASE_CLASS_SATELLITE	0x0f
-#define PCI_CLASS_SATELLITE_TV		0x0f00
-#define PCI_CLASS_SATELLITE_AUDIO	0x0f01
-#define PCI_CLASS_SATELLITE_VOICE	0x0f03
-#define PCI_CLASS_SATELLITE_DATA	0x0f04
-
-#define PCI_BASE_CLASS_CRYPT		0x10
-#define PCI_CLASS_CRYPT_NETWORK		0x1000
-#define PCI_CLASS_CRYPT_ENTERTAINMENT	0x1001
-#define PCI_CLASS_CRYPT_OTHER		0x1080
-
-#define PCI_BASE_CLASS_SIGNAL_PROCESSING 0x11
-#define PCI_CLASS_SP_DPIO		0x1100
-#define PCI_CLASS_SP_OTHER		0x1180
-
-#define PCI_CLASS_OTHERS		0xff
-
-/* Vendors */
-
-#define PCI_VENDOR_ID_DYNALINK		0x0675
-#define PCI_VENDOR_ID_BERKOM			0x0871
-#define PCI_VENDOR_ID_COMPAQ		0x0e11
-#define PCI_VENDOR_ID_NCR		0x1000
-#define PCI_VENDOR_ID_LSI_LOGIC		0x1000
-#define PCI_VENDOR_ID_ATI		0x1002
-#define PCI_VENDOR_ID_VLSI		0x1004
-#define PCI_VENDOR_ID_ADL		0x1005
-#define PCI_VENDOR_ID_NS		0x100b
-#define PCI_VENDOR_ID_TSENG		0x100c
-#define PCI_VENDOR_ID_WEITEK		0x100e
-#define PCI_VENDOR_ID_DEC		0x1011
-#define PCI_VENDOR_ID_CIRRUS		0x1013
-#define PCI_VENDOR_ID_IBM		0x1014
-#define PCI_VENDOR_ID_COMPEX2		0x101a
-/* pci.ids says "AT&T GIS (NCR)" */
-#define PCI_VENDOR_ID_WD		0x101c
-#define PCI_VENDOR_ID_AMI		0x101e
-#define PCI_VENDOR_ID_AMD		0x1022
-#define PCI_VENDOR_ID_TRIDENT		0x1023
-#define PCI_VENDOR_ID_AI		0x1025
-#define PCI_VENDOR_ID_DELL              0x1028
-#define PCI_VENDOR_ID_MATROX		0x102B
-#define PCI_VENDOR_ID_CT		0x102c
-#define PCI_VENDOR_ID_MIRO		0x1031
-#define PCI_VENDOR_ID_NEC		0x1033
-#define PCI_VENDOR_ID_FD		0x1036
-#define PCI_VENDOR_ID_SIS         	0x1039
-#define PCI_VENDOR_ID_SI		0x1039
-#define PCI_VENDOR_ID_HP		0x103c
-#define PCI_VENDOR_ID_PCTECH		0x1042
-#define PCI_VENDOR_ID_ASUSTEK		0x1043
-#define PCI_VENDOR_ID_DPT		0x1044
-#define PCI_VENDOR_ID_OPTI		0x1045
-#define PCI_VENDOR_ID_ELSA		0x1048
-#define PCI_VENDOR_ID_ELSA		0x1048
-#define PCI_VENDOR_ID_SGS		0x104a
-#define PCI_VENDOR_ID_BUSLOGIC		      0x104B
-#define PCI_VENDOR_ID_TI		0x104c
-#define PCI_VENDOR_ID_SONY		0x104d
-#define PCI_VENDOR_ID_OAK		0x104e
-/* Winbond have two vendor IDs! See 0x10ad as well */
-#define PCI_VENDOR_ID_WINBOND2		0x1050
-#define PCI_VENDOR_ID_ANIGMA		0x1051
-#define PCI_VENDOR_ID_EFAR		0x1055
-#define PCI_VENDOR_ID_MOTOROLA		0x1057
-#define PCI_VENDOR_ID_MOTOROLA_OOPS	0x1507
-#define PCI_VENDOR_ID_PROMISE		0x105a
-#define PCI_VENDOR_ID_N9		0x105d
-#define PCI_VENDOR_ID_UMC		0x1060
-#define PCI_VENDOR_ID_X			0x1061
-#define PCI_VENDOR_ID_MYLEX		0x1069
-#define PCI_VENDOR_ID_PICOP		0x1066
-#define PCI_VENDOR_ID_APPLE		0x106b
-#define PCI_VENDOR_ID_YAMAHA		0x1073
-#define PCI_VENDOR_ID_NEXGEN		0x1074
-#define PCI_VENDOR_ID_QLOGIC		0x1077
-#define PCI_VENDOR_ID_CYRIX		0x1078
-#define PCI_VENDOR_ID_LEADTEK		0x107d
-#define PCI_VENDOR_ID_INTERPHASE	0x107e
-#define PCI_VENDOR_ID_CONTAQ		0x1080
-#define PCI_VENDOR_ID_FOREX		0x1083
-#define PCI_VENDOR_ID_OLICOM		0x108d
-#define PCI_VENDOR_ID_SUN		0x108e
-#define PCI_VENDOR_ID_CMD		0x1095
-#define PCI_VENDOR_ID_VISION		0x1098
-#define PCI_VENDOR_ID_BROOKTREE		0x109e
-#define PCI_VENDOR_ID_SIERRA		0x10a8
-#define PCI_VENDOR_ID_SGI		0x10a9
-#define PCI_VENDOR_ID_ACC		0x10aa
-#define PCI_VENDOR_ID_WINBOND		0x10ad
-#define PCI_VENDOR_ID_DATABOOK		0x10b3
-#define PCI_VENDOR_ID_PLX		0x10b5
-#define PCI_VENDOR_ID_MADGE		0x10b6
-#define PCI_VENDOR_ID_3COM		0x10b7
-#define PCI_VENDOR_ID_SMC		0x10b8
-#define PCI_VENDOR_ID_SUNDANCE		0x13F0
-#define PCI_VENDOR_ID_AL		0x10b9
-#define PCI_VENDOR_ID_MITSUBISHI	0x10ba
-#define PCI_VENDOR_ID_SURECOM		0x10bd
-#define PCI_VENDOR_ID_NEOMAGIC		0x10c8
-#define PCI_VENDOR_ID_ASP		0x10cd
-#define PCI_VENDOR_ID_MACRONIX		0x10d9
-#define PCI_VENDOR_ID_TCONRAD		0x10da
-#define PCI_VENDOR_ID_CERN		0x10dc
-#define PCI_VENDOR_ID_NVIDIA			0x10de
-#define PCI_VENDOR_ID_IMS		0x10e0
-#define PCI_VENDOR_ID_TEKRAM2		0x10e1
-#define PCI_VENDOR_ID_TUNDRA		0x10e3
-#define PCI_VENDOR_ID_AMCC		0x10e8
-#define PCI_VENDOR_ID_INTERG		0x10ea
-#define PCI_VENDOR_ID_REALTEK		0x10ec
-#define PCI_VENDOR_ID_XILINX		0x10ee
-#define PCI_VENDOR_ID_TRUEVISION	0x10fa
-#define PCI_VENDOR_ID_INIT		0x1101
-#define PCI_VENDOR_ID_CREATIVE		0x1102
-/* duplicate: ECTIVA */
-#define PCI_VENDOR_ID_ECTIVA		0x1102
-/* duplicate: CREATIVE */
-#define PCI_VENDOR_ID_TTI		0x1103
-#define PCI_VENDOR_ID_VIA		0x1106
-#define PCI_VENDOR_ID_VIATEC		0x1106
-#define PCI_VENDOR_ID_SIEMENS           0x110A
-#define PCI_VENDOR_ID_SMC2		0x1113
-#define PCI_VENDOR_ID_VORTEX		0x1119
-#define PCI_VENDOR_ID_EF		0x111a
-#define PCI_VENDOR_ID_IDT		0x111d
-#define PCI_VENDOR_ID_FORE		0x1127
-#define PCI_VENDOR_ID_IMAGINGTECH	0x112f
-#define PCI_VENDOR_ID_PHILIPS		0x1131
-#define PCI_VENDOR_ID_EICON		0x1133
-#define PCI_VENDOR_ID_CYCLONE		0x113c
-#define PCI_VENDOR_ID_ALLIANCE		0x1142
-#define PCI_VENDOR_ID_SYSKONNECT	0x1148
-#define PCI_VENDOR_ID_VMIC		0x114a
-#define PCI_VENDOR_ID_DIGI		0x114f
-#define PCI_VENDOR_ID_MUTECH		0x1159
-#define PCI_VENDOR_ID_XIRCOM		0x115d
-#define PCI_VENDOR_ID_RENDITION		0x1163
-#define PCI_VENDOR_ID_SERVERWORKS	  0x1166
-#define PCI_VENDOR_ID_SBE		0x1176
-#define PCI_VENDOR_ID_TOSHIBA		0x1179
-#define PCI_VENDOR_ID_RICOH		0x1180
-#define	PCI_VENDOR_ID_DLINK		0x1186
-#define PCI_VENDOR_ID_ARTOP		0x1191
-#define PCI_VENDOR_ID_ZEITNET		0x1193
-#define PCI_VENDOR_ID_OMEGA		0x119b
-#define PCI_VENDOR_ID_FUJITSU_ME	0x119e
-#define PCI_SUBVENDOR_ID_KEYSPAN	0x11a9
-#define PCI_VENDOR_ID_GALILEO		0x11ab
-#define PCI_VENDOR_ID_LINKSYS		0x11ad
-#define PCI_VENDOR_ID_LITEON		0x11ad
-#define PCI_VENDOR_ID_V3		0x11b0
-#define PCI_VENDOR_ID_NP		0x11bc
-#define PCI_VENDOR_ID_ATT		0x11c1
-#define PCI_VENDOR_ID_SPECIALIX		0x11cb
-#define PCI_VENDOR_ID_AURAVISION	0x11d1
-#define PCI_VENDOR_ID_ANALOG_DEVICES	0x11d4
-#define PCI_VENDOR_ID_IKON		0x11d5
-#define PCI_VENDOR_ID_ZORAN		0x11de
-#define PCI_VENDOR_ID_KINETIC		0x11f4
-#define PCI_VENDOR_ID_COMPEX		0x11f6
-#define PCI_VENDOR_ID_RP		0x11fe
-#define PCI_VENDOR_ID_CYCLADES		0x120e
-#define PCI_VENDOR_ID_ESSENTIAL		0x120f
-#define PCI_VENDOR_ID_O2		0x1217
-#define PCI_VENDOR_ID_3DFX		0x121a
-#define PCI_VENDOR_ID_SIGMADES		0x1236
-#define PCI_VENDOR_ID_CCUBE		0x123f
-#define PCI_VENDOR_ID_AVM		0x1244
-#define PCI_VENDOR_ID_DIPIX		0x1246
-#define PCI_VENDOR_ID_STALLION		0x124d
-#define PCI_VENDOR_ID_OPTIBASE		0x1255
-#define PCI_VENDOR_ID_ESS		0x125d
-#define PCI_VENDOR_ID_HARRIS        	0x1260
-#define PCI_VENDOR_ID_SATSAGEM		0x1267
-#define PCI_VENDOR_ID_HUGHES		0x1273
-#define PCI_VENDOR_ID_ENSONIQ		0x1274
-#define PCI_VENDOR_ID_ROCKWELL		0x127A
-#define PCI_VENDOR_ID_DAVICOM		0x1282
-#define PCI_VENDOR_ID_ITE		0x1283
-/* formerly Platform Tech */
-#define PCI_VENDOR_ID_ESS_OLD		0x1285
-#define PCI_VENDOR_ID_ALTEON		0x12ae
-#define PCI_VENDOR_ID_USR		0x12B9
-#define	PCI_VENDOR_ID_HOLTEK		0x12c3
-#define PCI_SUBVENDOR_ID_CONNECT_TECH			0x12c4
-#define PCI_VENDOR_ID_PICTUREL		0x12c5
-#define PCI_VENDOR_ID_NVIDIA_SGS	0x12d2
-#define PCI_SUBVENDOR_ID_CHASE_PCIFAST		0x12E0
-#define PCI_SUBVENDOR_ID_CHASE_PCIRAS		0x124D
-#define PCI_VENDOR_ID_AUREAL		0x12eb
-#define PCI_VENDOR_ID_CBOARDS		0x1307
-#define PCI_VENDOR_ID_SIIG		0x131f
-#define PCI_VENDOR_ID_ADMTEK            0x1317
-#define PCI_VENDOR_ID_DOMEX		0x134a
-#define PCI_VENDOR_ID_QUATECH		0x135C
-#define PCI_VENDOR_ID_SEALEVEL		0x135e
-#define PCI_VENDOR_ID_HYPERCOPE		0x1365
-#define PCI_VENDOR_ID_KAWASAKI		0x136b
-#define PCI_VENDOR_ID_LMC		0x1376
-#define PCI_VENDOR_ID_NETGEAR		0x1385
-#define PCI_VENDOR_ID_APPLICOM		0x1389
-#define PCI_VENDOR_ID_MOXA		0x1393
-#define PCI_VENDOR_ID_CCD		0x1397
-#define PCI_VENDOR_ID_MICROGATE		0x13c0
-#define PCI_VENDOR_ID_3WARE		0x13C1
-#define PCI_VENDOR_ID_ABOCOM		0x13D1
-#define PCI_VENDOR_ID_CMEDIA		0x13f6
-#define PCI_VENDOR_ID_LAVA		0x1407
-#define PCI_VENDOR_ID_TIMEDIA		0x1409
-#define PCI_VENDOR_ID_OXSEMI		0x1415
-#define PCI_VENDOR_ID_AIRONET		0x14b9
-#define PCI_VENDOR_ID_MYRICOM		0x14c1
-#define PCI_VENDOR_ID_TITAN		0x14D2
-#define PCI_VENDOR_ID_PANACOM		0x14d4
-#define PCI_VENDOR_ID_BROADCOM		0x14e4
-#define PCI_VENDOR_ID_SYBA		0x1592
-#define PCI_VENDOR_ID_MORETON		0x15aa
-#define PCI_VENDOR_ID_ZOLTRIX		0x15b0
-#define PCI_VENDOR_ID_PDC		0x15e9
-#define PCI_VENDOR_ID_FSC		0x1734
-#define PCI_VENDOR_ID_SYMPHONY		0x1c1c
-#define PCI_VENDOR_ID_TEKRAM		0x1de1
-#define PCI_VENDOR_ID_3DLABS		0x3d3d
-#define PCI_VENDOR_ID_AVANCE		0x4005
-#define PCI_VENDOR_ID_AKS		0x416c
-#define PCI_VENDOR_ID_NETVIN		0x4a14
-#define PCI_VENDOR_ID_S3		0x5333
-#define PCI_VENDOR_ID_DCI		0x6666
-#define PCI_VENDOR_ID_GENROCO		0x5555
-#define PCI_VENDOR_ID_INTEL		0x8086
-#define PCI_VENDOR_ID_COMPUTONE		0x8e0e
-#define PCI_SUBVENDOR_ID_COMPUTONE	0x8e0e
-#define PCI_VENDOR_ID_KTI		0x8e2e
-#define PCI_VENDOR_ID_ADAPTEC		0x9004
-#define PCI_VENDOR_ID_ADAPTEC2		0x9005
-#define PCI_VENDOR_ID_ATRONICS		0x907f
-#define PCI_VENDOR_ID_HOLTEK2		0x9412
-#define PCI_VENDOR_ID_NETMOS		0x9710
-#define PCI_SUBVENDOR_ID_EXSYS		0xd84d
-#define PCI_VENDOR_ID_TIGERJET		0xe159
-#define PCI_VENDOR_ID_ARK		0xedd8
-
-#endif /* _IPXE_PCI_IDS_H */
diff --git a/roms/ipxe/src/include/ipxe/pci_io.h b/roms/ipxe/src/include/ipxe/pci_io.h
index 781b77fe1..10e69763e 100644
--- a/roms/ipxe/src/include/ipxe/pci_io.h
+++ b/roms/ipxe/src/include/ipxe/pci_io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/api.h>
diff --git a/roms/ipxe/src/include/ipxe/pcibackup.h b/roms/ipxe/src/include/ipxe/pcibackup.h
index b9f55cf71..159d25392 100644
--- a/roms/ipxe/src/include/ipxe/pcibackup.h
+++ b/roms/ipxe/src/include/ipxe/pcibackup.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/pcivpd.h b/roms/ipxe/src/include/ipxe/pcivpd.h
index 0abf8a956..fefb69740 100644
--- a/roms/ipxe/src/include/ipxe/pcivpd.h
+++ b/roms/ipxe/src/include/ipxe/pcivpd.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/roms/ipxe/src/include/ipxe/peerblk.h b/roms/ipxe/src/include/ipxe/peerblk.h
new file mode 100644
index 000000000..6fc9172f6
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/peerblk.h
@@ -0,0 +1,144 @@
+#ifndef _IPXE_PEERBLK_H
+#define _IPXE_PEERBLK_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol block downloads
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/crypto.h>
+#include <ipxe/aes.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/retry.h>
+#include <ipxe/process.h>
+#include <ipxe/pccrc.h>
+#include <ipxe/peerdisc.h>
+
+/** A PeerDist retrieval protocol decryption buffer descriptor */
+struct peerdist_block_decrypt {
+	/** Data transfer buffer */
+	struct xfer_buffer *xferbuf;
+	/** Offset within data transfer buffer */
+	size_t offset;
+	/** Length to use from data transfer buffer */
+	size_t len;
+};
+
+/** PeerDist retrieval protocol decryption data transfer buffer indices */
+enum peerdist_block_decrypt_index {
+	/** Data before the trimmed content */
+	PEERBLK_BEFORE = 0,
+	/** Data within the trimmed content */
+	PEERBLK_DURING,
+	/** Data after the trimmed content */
+	PEERBLK_AFTER,
+	/** Number of decryption buffers */
+	PEERBLK_NUM_BUFFERS
+};
+
+/** A PeerDist block download */
+struct peerdist_block {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Raw data interface */
+	struct interface raw;
+	/** Retrieval protocol interface */
+	struct interface retrieval;
+
+	/** Original URI */
+	struct uri *uri;
+	/** Content range of this block */
+	struct peerdist_range range;
+	/** Trimmed range of this block */
+	struct peerdist_range trim;
+	/** Offset of first byte in trimmed range within overall download */
+	size_t offset;
+
+	/** Digest algorithm */
+	struct digest_algorithm *digest;
+	/** Digest size
+	 *
+	 * Note that this may be shorter than the digest size of the
+	 * digest algorithm.
+	 */
+	size_t digestsize;
+	/** Digest context (statically allocated at instantiation time) */
+	void *digestctx;
+
+	/** Cipher algorithm */
+	struct cipher_algorithm *cipher;
+	/** Cipher context (dynamically allocated as needed) */
+	void *cipherctx;
+
+	/** Segment index */
+	unsigned int segment;
+	/** Segment identifier */
+	uint8_t id[PEERDIST_DIGEST_MAX_SIZE];
+	/** Segment secret */
+	uint8_t secret[PEERDIST_DIGEST_MAX_SIZE];
+	/** Block index */
+	unsigned int block;
+	/** Block hash */
+	uint8_t hash[PEERDIST_DIGEST_MAX_SIZE];
+
+	/** Current position (relative to incoming data stream) */
+	size_t pos;
+	/** Start of trimmed content (relative to incoming data stream) */
+	size_t start;
+	/** End of trimmed content (relative to incoming data stream) */
+	size_t end;
+	/** Data buffer */
+	struct xfer_buffer buffer;
+
+	/** Decryption process */
+	struct process process;
+	/** Decryption data buffer descriptors */
+	struct peerdist_block_decrypt decrypt[PEERBLK_NUM_BUFFERS];
+	/** Remaining decryption length */
+	size_t cipher_remaining;
+	/** Remaining digest length (excluding AES padding bytes) */
+	size_t digest_remaining;
+
+	/** Discovery client */
+	struct peerdisc_client discovery;
+	/** Current position in discovered peer list */
+	struct peerdisc_peer *peer;
+	/** Retry timer */
+	struct retry_timer timer;
+	/** Number of full attempt cycles completed */
+	unsigned int cycles;
+	/** Most recent attempt failure */
+	int rc;
+
+	/** Time at which block download was started */
+	unsigned long started;
+	/** Time at which most recent attempt was started */
+	unsigned long attempted;
+};
+
+/** Retrieval protocol block fetch response (including transport header)
+ *
+ * @v digestsize	Digest size
+ * @v len		Data block length
+ * @v vrf_len		Length of uselessness
+ * @v blksize		Cipher block size
+ */
+#define peerblk_msg_blk_t( digestsize, len, vrf_len, blksize )		\
+	struct {							\
+		struct peerdist_msg_transport_header hdr;		\
+		peerdist_msg_blk_t ( digestsize, len, vrf_len,		\
+				     blksize ) msg;			\
+	} __attribute__ (( packed ))
+
+extern int peerblk_open ( struct interface *xfer, struct uri *uri,
+			  struct peerdist_info_block *block );
+
+#endif /* _IPXE_PEERBLK_H */
diff --git a/roms/ipxe/src/include/ipxe/peerdisc.h b/roms/ipxe/src/include/ipxe/peerdisc.h
new file mode 100644
index 000000000..f08ccaae2
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/peerdisc.h
@@ -0,0 +1,116 @@
+#ifndef _IPXE_PEERDISC_H
+#define _IPXE_PEERDISC_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol peer discovery
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/list.h>
+#include <ipxe/tables.h>
+#include <ipxe/retry.h>
+#include <ipxe/socket.h>
+#include <ipxe/interface.h>
+#include <ipxe/pccrc.h>
+
+/** A PeerDist discovery socket */
+struct peerdisc_socket {
+	/** Name */
+	const char *name;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Socket address */
+	union {
+		struct sockaddr sa;
+		struct sockaddr_in sin;
+		struct sockaddr_in6 sin6;
+	} address;
+};
+
+/** PeerDist discovery socket table */
+#define PEERDISC_SOCKETS __table ( struct peerdisc_socket, "peerdisc_sockets" )
+
+/** Declare a PeerDist discovery socket */
+#define __peerdisc_socket __table_entry ( PEERDISC_SOCKETS, 01 )
+
+/** A PeerDist discovery segment */
+struct peerdisc_segment {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** List of segments */
+	struct list_head list;
+	/** Segment identifier string
+	 *
+	 * This is MS-PCCRC's "HoHoDk", transcribed as an upper-case
+	 * Base16-encoded string.
+	 */
+	const char *id;
+	/** Message UUID string */
+	const char *uuid;
+	/** List of discovered peers
+	 *
+	 * The list of peers may be appended to during the lifetime of
+	 * the discovery segment.  Discovered peers will not be
+	 * removed from the list until the last discovery has been
+	 * closed; this allows users to safely maintain a pointer to a
+	 * current position within the list.
+	 */
+	struct list_head peers;
+	/** List of active clients */
+	struct list_head clients;
+	/** Transmission timer */
+	struct retry_timer timer;
+};
+
+/** A PeerDist discovery peer */
+struct peerdisc_peer {
+	/** List of peers */
+	struct list_head list;
+	/** Peer location */
+	char location[0];
+};
+
+/** A PeerDist discovery client */
+struct peerdisc_client {
+	/** Discovery segment */
+	struct peerdisc_segment *segment;
+	/** List of clients */
+	struct list_head list;
+	/** Operations */
+	struct peerdisc_client_operations *op;
+};
+
+/** PeerDist discovery client operations */
+struct peerdisc_client_operations {
+	/** New peers have been discovered
+	 *
+	 * @v peerdisc		PeerDist discovery client
+	 */
+	void ( * discovered ) ( struct peerdisc_client *peerdisc );
+};
+
+/**
+ * Initialise PeerDist discovery
+ *
+ * @v peerdisc		PeerDist discovery client
+ * @v op		Discovery operations
+ */
+static inline __attribute__ (( always_inline )) void
+peerdisc_init ( struct peerdisc_client *peerdisc,
+		struct peerdisc_client_operations *op ) {
+
+	peerdisc->op = op;
+}
+
+extern unsigned int peerdisc_timeout_secs;
+
+extern int peerdisc_open ( struct peerdisc_client *peerdisc, const void *id,
+			   size_t len );
+extern void peerdisc_close ( struct peerdisc_client *peerdisc );
+
+#endif /* _IPXE_PEERDISC_H */
diff --git a/roms/ipxe/src/include/ipxe/peermux.h b/roms/ipxe/src/include/ipxe/peermux.h
new file mode 100644
index 000000000..44cbdb9d6
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/peermux.h
@@ -0,0 +1,73 @@
+#ifndef _IPXE_PEERMUX_H
+#define _IPXE_PEERMUX_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol multiplexer
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/list.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/process.h>
+#include <ipxe/uri.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/pccrc.h>
+
+/** Maximum number of concurrent block downloads */
+#define PEERMUX_MAX_BLOCKS 32
+
+/** PeerDist download content information cache */
+struct peerdist_info_cache {
+	/** Content information */
+	struct peerdist_info info;
+	/** Content information segment */
+	struct peerdist_info_segment segment;
+	/** Content information block */
+	struct peerdist_info_block block;
+};
+
+/** A PeerDist multiplexed block download */
+struct peerdist_multiplexed_block {
+	/** PeerDist download multiplexer */
+	struct peerdist_multiplexer *peermux;
+	/** List of multiplexed blocks */
+	struct list_head list;
+	/** Data transfer interface */
+	struct interface xfer;
+};
+
+/** A PeerDist download multiplexer */
+struct peerdist_multiplexer {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Content information interface */
+	struct interface info;
+	/** Original URI */
+	struct uri *uri;
+
+	/** Content information data transfer buffer */
+	struct xfer_buffer buffer;
+	/** Content information cache */
+	struct peerdist_info_cache cache;
+
+	/** Block download initiation process */
+	struct process process;
+	/** List of busy block downloads */
+	struct list_head busy;
+	/** List of idle block downloads */
+	struct list_head idle;
+	/** Block downloads */
+	struct peerdist_multiplexed_block block[PEERMUX_MAX_BLOCKS];
+};
+
+extern int peermux_filter ( struct interface *xfer, struct interface *info,
+			    struct uri *uri );
+
+#endif /* _IPXE_PEERMUX_H */
diff --git a/roms/ipxe/src/include/ipxe/pending.h b/roms/ipxe/src/include/ipxe/pending.h
index e6a369813..be6ed05a1 100644
--- a/roms/ipxe/src/include/ipxe/pending.h
+++ b/roms/ipxe/src/include/ipxe/pending.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A pending operation */
 struct pending_operation {
diff --git a/roms/ipxe/src/include/ipxe/ping.h b/roms/ipxe/src/include/ipxe/ping.h
index 6cd376b6f..c55bd1ab2 100644
--- a/roms/ipxe/src/include/ipxe/ping.h
+++ b/roms/ipxe/src/include/ipxe/ping.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/iobuf.h>
 #include <ipxe/tcpip.h>
diff --git a/roms/ipxe/src/include/ipxe/pinger.h b/roms/ipxe/src/include/ipxe/pinger.h
index 9932df6b0..227f002dc 100644
--- a/roms/ipxe/src/include/ipxe/pinger.h
+++ b/roms/ipxe/src/include/ipxe/pinger.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/interface.h>
diff --git a/roms/ipxe/src/include/ipxe/pixbuf.h b/roms/ipxe/src/include/ipxe/pixbuf.h
index 106b666e6..615744812 100644
--- a/roms/ipxe/src/include/ipxe/pixbuf.h
+++ b/roms/ipxe/src/include/ipxe/pixbuf.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/include/ipxe/png.h b/roms/ipxe/src/include/ipxe/png.h
index f51d1e6fe..3505eefc8 100644
--- a/roms/ipxe/src/include/ipxe/png.h
+++ b/roms/ipxe/src/include/ipxe/png.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/roms/ipxe/src/include/ipxe/pnm.h b/roms/ipxe/src/include/ipxe/pnm.h
index 536c14d5f..860968cbc 100644
--- a/roms/ipxe/src/include/ipxe/pnm.h
+++ b/roms/ipxe/src/include/ipxe/pnm.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/roms/ipxe/src/include/ipxe/pool.h b/roms/ipxe/src/include/ipxe/pool.h
new file mode 100644
index 000000000..27066e9b3
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/pool.h
@@ -0,0 +1,127 @@
+#ifndef _IPXE_POOL_H
+#define _IPXE_POOL_H
+
+/** @file
+ *
+ * Pooled connections
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/interface.h>
+#include <ipxe/list.h>
+#include <ipxe/retry.h>
+
+/** A pooled connection */
+struct pooled_connection {
+	/** List of pooled connections
+	 *
+	 * Note that each connecton in the pool has a running expiry
+	 * timer which holds a reference to the connection.  We
+	 * therefore do not require the connection pool list to hold a
+	 * reference for each pooled connection.
+	 */
+	struct list_head list;
+	/** Expiry timer */
+	struct retry_timer timer;
+	/** Close expired pooled connection
+	 *
+	 * @v pool		Pooled connection
+	 */
+	void ( * expired ) ( struct pooled_connection *pool );
+	/** Flags */
+	unsigned int flags;
+};
+
+/** Pooled connection flags */
+enum pooled_connection_flags {
+	/** Connection should be recycled after closing */
+	POOL_RECYCLABLE = 0x0001,
+	/** Connection has been recycled */
+	POOL_RECYCLED = 0x0002,
+	/** Connection is known to be alive */
+	POOL_ALIVE = 0x0004,
+};
+
+extern void pool_add ( struct pooled_connection *pool, struct list_head *list,
+		       unsigned long expiry );
+extern void pool_del ( struct pooled_connection *pool );
+extern void pool_expired ( struct retry_timer *timer, int over );
+
+/**
+ * Initialise a pooled connection
+ *
+ * @v pool		Pooled connection
+ * @v expired		Close expired pooled connection method
+ * @v refcnt		Containing object reference counter
+ */
+static inline __attribute__ (( always_inline )) void
+pool_init ( struct pooled_connection *pool,
+	    void ( * expired ) ( struct pooled_connection *pool ),
+	    struct refcnt *refcnt ) {
+
+	INIT_LIST_HEAD ( &pool->list );
+	timer_init ( &pool->timer, pool_expired, refcnt );
+	pool->expired = expired;
+}
+
+/**
+ * Mark pooled connection as recyclable
+ *
+ * @v pool		Pooled connection
+ */
+static inline __attribute__ (( always_inline )) void
+pool_recyclable ( struct pooled_connection *pool ) {
+
+	pool->flags |= POOL_RECYCLABLE;
+}
+
+/**
+ * Mark pooled connection as alive
+ *
+ * @v pool		Pooled connection
+ */
+static inline __attribute__ (( always_inline )) void
+pool_alive ( struct pooled_connection *pool ) {
+
+	pool->flags |= POOL_ALIVE;
+}
+
+/**
+ * Check if pooled connection is recyclable
+ *
+ * @v pool		Pooled connection
+ * @ret recyclable	Pooled connection is recyclable
+ */
+static inline __attribute__ (( always_inline )) int
+pool_is_recyclable ( struct pooled_connection *pool ) {
+
+	return ( pool->flags & POOL_RECYCLABLE );
+}
+
+/**
+ * Check if pooled connection is reopenable
+ *
+ * @v pool		Pooled connection
+ * @ret reopenable	Pooled connection is reopenable
+ */
+static inline __attribute__ (( always_inline )) int
+pool_is_reopenable ( struct pooled_connection *pool ) {
+
+	/* A connection is reopenable if it has been recycled but is
+	 * not yet known to be alive.
+	 */
+	return ( ( pool->flags & POOL_RECYCLED ) &
+		 ( ! ( pool->flags & POOL_ALIVE ) ) );
+}
+
+extern void pool_recycle ( struct interface *intf );
+#define pool_recycle_TYPE( object_type ) \
+	typeof ( void ( object_type ) )
+
+extern void pool_reopen ( struct interface *intf );
+#define pool_reopen_TYPE( object_type ) \
+	typeof ( void ( object_type ) )
+
+#endif /* _IPXE_POOL_H */
diff --git a/roms/ipxe/src/include/ipxe/portmap.h b/roms/ipxe/src/include/ipxe/portmap.h
index 9b735bbca..681ca2ec2 100644
--- a/roms/ipxe/src/include/ipxe/portmap.h
+++ b/roms/ipxe/src/include/ipxe/portmap.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** PORTMAP default port */
 #define PORTMAP_PORT   111
diff --git a/roms/ipxe/src/include/ipxe/posix_io.h b/roms/ipxe/src/include/ipxe/posix_io.h
index 11f3bb5c9..1a73b5e86 100644
--- a/roms/ipxe/src/include/ipxe/posix_io.h
+++ b/roms/ipxe/src/include/ipxe/posix_io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/roms/ipxe/src/include/ipxe/privkey.h b/roms/ipxe/src/include/ipxe/privkey.h
index 39049ac9f..81108b6bf 100644
--- a/roms/ipxe/src/include/ipxe/privkey.h
+++ b/roms/ipxe/src/include/ipxe/privkey.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/asn1.h>
 
diff --git a/roms/ipxe/src/include/ipxe/process.h b/roms/ipxe/src/include/ipxe/process.h
index 2c76ff260..d600508e7 100644
--- a/roms/ipxe/src/include/ipxe/process.h
+++ b/roms/ipxe/src/include/ipxe/process.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/include/ipxe/profile.h b/roms/ipxe/src/include/ipxe/profile.h
index 3a745fcfa..b6d2b19e0 100644
--- a/roms/ipxe/src/include/ipxe/profile.h
+++ b/roms/ipxe/src/include/ipxe/profile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <bits/profile.h>
 #include <ipxe/tables.h>
@@ -186,4 +186,18 @@ profile_exclude ( struct profiler *profiler ) {
 		profile_excluded += profile_elapsed ( profiler );
 }
 
+/**
+ * Record profiling sample in custom units
+ *
+ * @v profiler		Profiler
+ * @v sample		Profiling sample
+ */
+static inline __attribute__ (( always_inline )) void
+profile_custom ( struct profiler *profiler, unsigned long sample ) {
+
+	/* If profiling is active then update stats */
+	if ( PROFILING )
+		profile_update ( profiler, sample );
+}
+
 #endif /* _IPXE_PROFILE_H */
diff --git a/roms/ipxe/src/include/ipxe/random_nz.h b/roms/ipxe/src/include/ipxe/random_nz.h
index 6bb80d2ab..4c433fa38 100644
--- a/roms/ipxe/src/include/ipxe/random_nz.h
+++ b/roms/ipxe/src/include/ipxe/random_nz.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/rarp.h b/roms/ipxe/src/include/ipxe/rarp.h
index f84301a43..9054db21a 100644
--- a/roms/ipxe/src/include/ipxe/rarp.h
+++ b/roms/ipxe/src/include/ipxe/rarp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/netdevice.h>
 
diff --git a/roms/ipxe/src/include/ipxe/rbg.h b/roms/ipxe/src/include/ipxe/rbg.h
index 9689142f8..758238a65 100644
--- a/roms/ipxe/src/include/ipxe/rbg.h
+++ b/roms/ipxe/src/include/ipxe/rbg.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/drbg.h>
diff --git a/roms/ipxe/src/include/ipxe/reboot.h b/roms/ipxe/src/include/ipxe/reboot.h
index 97e0d5fb6..33606d9d5 100644
--- a/roms/ipxe/src/include/ipxe/reboot.h
+++ b/roms/ipxe/src/include/ipxe/reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/reboot.h>
diff --git a/roms/ipxe/src/include/ipxe/refcnt.h b/roms/ipxe/src/include/ipxe/refcnt.h
index 0e8b8658c..7f489abc9 100644
--- a/roms/ipxe/src/include/ipxe/refcnt.h
+++ b/roms/ipxe/src/include/ipxe/refcnt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <assert.h>
diff --git a/roms/ipxe/src/include/ipxe/resolv.h b/roms/ipxe/src/include/ipxe/resolv.h
index d9868a5d7..ff48d35ca 100644
--- a/roms/ipxe/src/include/ipxe/resolv.h
+++ b/roms/ipxe/src/include/ipxe/resolv.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/interface.h>
 #include <ipxe/tables.h>
diff --git a/roms/ipxe/src/include/ipxe/retry.h b/roms/ipxe/src/include/ipxe/retry.h
index c514822b2..76d45fbd0 100644
--- a/roms/ipxe/src/include/ipxe/retry.h
+++ b/roms/ipxe/src/include/ipxe/retry.h
@@ -7,14 +7,14 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 
-/** Default timeout value */
+/** Default minimum timeout value (in ticks) */
 #define DEFAULT_MIN_TIMEOUT ( TICKS_PER_SEC / 4 )
 
-/** Limit after which the timeout will be deemed permanent */
+/** Default maximum timeout value (in ticks) */
 #define DEFAULT_MAX_TIMEOUT ( 10 * TICKS_PER_SEC )
 
 /** A retry timer */
@@ -25,16 +25,18 @@ struct retry_timer {
 	unsigned int running;
 	/** Timeout value (in ticks) */
 	unsigned long timeout;
-	/** Minimum timeout value (in ticks)
+	/** Minimum timeout value (in ticks), or zero to use default
 	 *
-	 * A value of zero means "use default timeout."
+	 * The timeout will never be reduced below this value.
 	 */
-	unsigned long min_timeout;
-	/** Maximum timeout value before failure (in ticks)
+	unsigned long min;
+	/** Maximum timeout value (in ticks), or zero to use default
 	 *
-	 * A value of zero means "use default timeout."
+	 * The timeout will be deemed permanent (according to the
+	 * failure indicator passed to expired()) when it exceeds this
+	 * value.
 	 */
-	unsigned long max_timeout;
+	unsigned long max;
 	/** Start time (in ticks) */
 	unsigned long start;
 	/** Retry count */
@@ -46,7 +48,7 @@ struct retry_timer {
 	 *
 	 * The timer will already be stopped when this method is
 	 * called.  The failure indicator will be True if the retry
-	 * timeout has already exceeded @c MAX_TIMEOUT.
+	 * timeout has already exceeded @c max_timeout.
 	 */
 	void ( * expired ) ( struct retry_timer *timer, int over );
 	/** Reference counter
@@ -109,4 +111,18 @@ timer_running ( struct retry_timer *timer ) {
 	return ( timer->running );
 }
 
+/**
+ * Set minimum and maximum timeouts
+ *
+ * @v timer		Retry timer
+ * @v min		Minimum timeout (in ticks), or zero to use default
+ * @v max		Maximum timeout (in ticks), or zero to use default
+ */
+static inline __attribute__ (( always_inline )) void
+set_timer_limits ( struct retry_timer *timer, unsigned long min,
+		   unsigned long max ) {
+	timer->min = min;
+	timer->max = max;
+}
+
 #endif /* _IPXE_RETRY_H */
diff --git a/roms/ipxe/src/include/ipxe/rndis.h b/roms/ipxe/src/include/ipxe/rndis.h
new file mode 100644
index 000000000..bcb6d8e6a
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/rndis.h
@@ -0,0 +1,370 @@
+#ifndef _IPXE_RNDIS_H
+#define _IPXE_RNDIS_H
+
+/** @file
+ *
+ * Remote Network Driver Interface Specification
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/iobuf.h>
+
+/** Maximum time to wait for a transaction to complete
+ *
+ * This is a policy decision.
+ */
+#define RNDIS_MAX_WAIT_MS 1000
+
+/** RNDIS message header */
+struct rndis_header {
+	/** Message type */
+	uint32_t type;
+	/** Message length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** RNDIS initialise message */
+#define RNDIS_INITIALISE_MSG 0x00000002UL
+
+/** RNDIS initialise message */
+struct rndis_initialise_message {
+	/** Request ID */
+	uint32_t id;
+	/** Major version */
+	uint32_t major;
+	/** Minor version */
+	uint32_t minor;
+	/** Maximum transfer size */
+	uint32_t mtu;
+} __attribute__ (( packed ));
+
+/** Request ID used for initialisation
+ *
+ * This is a policy decision.
+ */
+#define RNDIS_INIT_ID 0xe110e110UL
+
+/** RNDIS major version */
+#define RNDIS_VERSION_MAJOR 1
+
+/** RNDIS minor version */
+#define RNDIS_VERSION_MINOR 0
+
+/** RNDIS maximum transfer size
+ *
+ * This is a policy decision.
+ */
+#define RNDIS_MTU 2048
+
+/** RNDIS initialise completion */
+#define RNDIS_INITIALISE_CMPLT 0x80000002UL
+
+/** RNDIS initialise completion */
+struct rndis_initialise_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+	/** Major version */
+	uint32_t major;
+	/** Minor version */
+	uint32_t minor;
+	/** Device flags */
+	uint32_t flags;
+	/** Medium */
+	uint32_t medium;
+	/** Maximum packets per transfer */
+	uint32_t max_pkts;
+	/** Maximum transfer size */
+	uint32_t mtu;
+	/** Packet alignment factor */
+	uint32_t align;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS halt message */
+#define RNDIS_HALT_MSG 0x00000003UL
+
+/** RNDIS halt message */
+struct rndis_halt_message {
+	/** Request ID */
+	uint32_t id;
+} __attribute__ (( packed ));
+
+/** RNDIS query OID message */
+#define RNDIS_QUERY_MSG 0x00000004UL
+
+/** RNDIS set OID message */
+#define RNDIS_SET_MSG 0x00000005UL
+
+/** RNDIS query or set OID message */
+struct rndis_oid_message {
+	/** Request ID */
+	uint32_t id;
+	/** Object ID */
+	uint32_t oid;
+	/** Information buffer length */
+	uint32_t len;
+	/** Information buffer offset */
+	uint32_t offset;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS query OID completion */
+#define RNDIS_QUERY_CMPLT 0x80000004UL
+
+/** RNDIS query OID completion */
+struct rndis_query_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+	/** Information buffer length */
+	uint32_t len;
+	/** Information buffer offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** RNDIS set OID completion */
+#define RNDIS_SET_CMPLT 0x80000005UL
+
+/** RNDIS set OID completion */
+struct rndis_set_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** RNDIS reset message */
+#define RNDIS_RESET_MSG 0x00000006UL
+
+/** RNDIS reset message */
+struct rndis_reset_message {
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS reset completion */
+#define RNDIS_RESET_CMPLT 0x80000006UL
+
+/** RNDIS reset completion */
+struct rndis_reset_completion {
+	/** Status */
+	uint32_t status;
+	/** Addressing reset */
+	uint32_t addr;
+} __attribute__ (( packed ));
+
+/** RNDIS indicate status message */
+#define RNDIS_INDICATE_STATUS_MSG 0x00000007UL
+
+/** RNDIS diagnostic information */
+struct rndis_diagnostic_info {
+	/** Status */
+	uint32_t status;
+	/** Error offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** RNDIS indicate status message */
+struct rndis_indicate_status_message {
+	/** Status */
+	uint32_t status;
+	/** Status buffer length */
+	uint32_t len;
+	/** Status buffer offset */
+	uint32_t offset;
+	/** Diagnostic information (optional) */
+	struct rndis_diagnostic_info diag[0];
+} __attribute__ (( packed ));
+
+/** RNDIS status codes */
+enum rndis_status {
+	/** Device is connected to a network medium */
+	RNDIS_STATUS_MEDIA_CONNECT = 0x4001000bUL,
+	/** Device is disconnected from the medium */
+	RNDIS_STATUS_MEDIA_DISCONNECT = 0x4001000cUL,
+	/** Unknown start-of-day status code */
+	RNDIS_STATUS_WTF_WORLD = 0x40020006UL,
+};
+
+/** RNDIS keepalive message */
+#define RNDIS_KEEPALIVE_MSG 0x00000008UL
+
+/** RNDIS keepalive message */
+struct rndis_keepalive_message {
+	/** Request ID */
+	uint32_t id;
+} __attribute__ (( packed ));
+
+/** RNDIS keepalive completion */
+#define RNDIS_KEEPALIVE_CMPLT 0x80000008UL
+
+/** RNDIS keepalive completion */
+struct rndis_keepalive_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** RNDIS packet message */
+#define RNDIS_PACKET_MSG 0x00000001UL
+
+/** RNDIS packet field */
+struct rndis_packet_field {
+	/** Offset */
+	uint32_t offset;
+	/** Length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** RNDIS packet message */
+struct rndis_packet_message {
+	/** Data */
+	struct rndis_packet_field data;
+	/** Out-of-band data records */
+	struct rndis_packet_field oob;
+	/** Number of out-of-band data records */
+	uint32_t oob_count;
+	/** Per-packet information record */
+	struct rndis_packet_field ppi;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS packet record */
+struct rndis_packet_record {
+	/** Length */
+	uint32_t len;
+	/** Type */
+	uint32_t type;
+	/** Offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** OID for packet filter */
+#define RNDIS_OID_GEN_CURRENT_PACKET_FILTER 0x0001010eUL
+
+/** Packet filter bits */
+enum rndis_packet_filter {
+	/** Unicast packets */
+	RNDIS_FILTER_UNICAST = 0x00000001UL,
+	/** Multicast packets */
+	RNDIS_FILTER_MULTICAST = 0x00000002UL,
+	/** All multicast packets */
+	RNDIS_FILTER_ALL_MULTICAST = 0x00000004UL,
+	/** Broadcast packets */
+	RNDIS_FILTER_BROADCAST = 0x00000008UL,
+	/** All packets */
+	RNDIS_FILTER_PROMISCUOUS = 0x00000020UL
+};
+
+/** OID for media status */
+#define RNDIS_OID_GEN_MEDIA_CONNECT_STATUS 0x00010114UL
+
+/** OID for permanent MAC address */
+#define RNDIS_OID_802_3_PERMANENT_ADDRESS 0x01010101UL
+
+/** OID for current MAC address */
+#define RNDIS_OID_802_3_CURRENT_ADDRESS	0x01010102UL
+
+struct rndis_device;
+
+/** RNDIS device operations */
+struct rndis_operations {
+	/**
+	 * Open RNDIS device
+	 *
+	 * @v rndis		RNDIS device
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct rndis_device *rndis );
+	/**
+	 * Close RNDIS device
+	 *
+	 * @v rndis		RNDIS device
+	 */
+	void ( * close ) ( struct rndis_device *rndis );
+	/**
+	 * Transmit packet
+	 *
+	 * @v rndis		RNDIS device
+	 * @v iobuf		I/O buffer
+	 * @ret rc		Return status code
+	 *
+	 * If this method returns success then the RNDIS device must
+	 * eventually report completion via rndis_tx_complete().
+	 */
+	int ( * transmit ) ( struct rndis_device *rndis,
+			     struct io_buffer *iobuf );
+	/**
+	 * Poll for completed and received packets
+	 *
+	 * @v rndis		RNDIS device
+	 */
+	void ( * poll ) ( struct rndis_device *rndis );
+};
+
+/** An RNDIS device */
+struct rndis_device {
+	/** Network device */
+	struct net_device *netdev;
+	/** Device name */
+	const char *name;
+	/** RNDIS operations */
+	struct rndis_operations *op;
+	/** Driver private data */
+	void *priv;
+
+	/** Request ID for current blocking request */
+	unsigned int wait_id;
+	/** Return status code for current blocking request */
+	int wait_rc;
+};
+
+/**
+ * Initialise an RNDIS device
+ *
+ * @v rndis		RNDIS device
+ * @v op		RNDIS device operations
+ */
+static inline void rndis_init ( struct rndis_device *rndis,
+				struct rndis_operations *op ) {
+
+	rndis->op = op;
+}
+
+extern void rndis_tx_complete_err ( struct rndis_device *rndis,
+				    struct io_buffer *iobuf, int rc );
+extern int rndis_tx_defer ( struct rndis_device *rndis,
+			    struct io_buffer *iobuf );
+extern void rndis_rx ( struct rndis_device *rndis, struct io_buffer *iobuf );
+extern void rndis_rx_err ( struct rndis_device *rndis, struct io_buffer *iobuf,
+			   int rc );
+
+extern struct rndis_device * alloc_rndis ( size_t priv_len );
+extern int register_rndis ( struct rndis_device *rndis );
+extern void unregister_rndis ( struct rndis_device *rndis );
+extern void free_rndis ( struct rndis_device *rndis );
+
+/**
+ * Complete message transmission
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static inline void rndis_tx_complete ( struct rndis_device *rndis,
+				       struct io_buffer *iobuf ) {
+
+	rndis_tx_complete_err ( rndis, iobuf, 0 );
+}
+
+#endif /* _IPXE_RNDIS_H */
diff --git a/roms/ipxe/src/include/ipxe/rootcert.h b/roms/ipxe/src/include/ipxe/rootcert.h
index 6525df87a..d4be2e1bc 100644
--- a/roms/ipxe/src/include/ipxe/rootcert.h
+++ b/roms/ipxe/src/include/ipxe/rootcert.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/x509.h>
 
diff --git a/roms/ipxe/src/include/ipxe/rotate.h b/roms/ipxe/src/include/ipxe/rotate.h
index ba271ca74..b5693e3ca 100644
--- a/roms/ipxe/src/include/ipxe/rotate.h
+++ b/roms/ipxe/src/include/ipxe/rotate.h
@@ -6,10 +6,30 @@
  * Bit operations
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
+static inline __attribute__ (( always_inline )) uint8_t
+rol8 ( uint8_t data, unsigned int rotation ) {
+        return ( ( data << rotation ) | ( data >> ( 8 - rotation ) ) );
+}
+
+static inline __attribute__ (( always_inline )) uint8_t
+ror8 ( uint8_t data, unsigned int rotation ) {
+        return ( ( data >> rotation ) | ( data << ( 8 - rotation ) ) );
+}
+
+static inline __attribute__ (( always_inline )) uint16_t
+rol16 ( uint16_t data, unsigned int rotation ) {
+        return ( ( data << rotation ) | ( data >> ( 16 - rotation ) ) );
+}
+
+static inline __attribute__ (( always_inline )) uint16_t
+ror16 ( uint16_t data, unsigned int rotation ) {
+        return ( ( data >> rotation ) | ( data << ( 16 - rotation ) ) );
+}
+
 static inline __attribute__ (( always_inline )) uint32_t
 rol32 ( uint32_t data, unsigned int rotation ) {
         return ( ( data << rotation ) | ( data >> ( 32 - rotation ) ) );
diff --git a/roms/ipxe/src/include/ipxe/rsa.h b/roms/ipxe/src/include/ipxe/rsa.h
index 1a5ad8bab..d947eec73 100644
--- a/roms/ipxe/src/include/ipxe/rsa.h
+++ b/roms/ipxe/src/include/ipxe/rsa.h
@@ -6,8 +6,9 @@
  * RSA public-key cryptography
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stdarg.h>
 #include <ipxe/crypto.h>
 #include <ipxe/bigint.h>
 #include <ipxe/asn1.h>
diff --git a/roms/ipxe/src/include/ipxe/sanboot.h b/roms/ipxe/src/include/ipxe/sanboot.h
index 14c8a5da4..57025f2c6 100644
--- a/roms/ipxe/src/include/ipxe/sanboot.h
+++ b/roms/ipxe/src/include/ipxe/sanboot.h
@@ -16,7 +16,7 @@
  * the address parameter.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/sanboot.h>
diff --git a/roms/ipxe/src/include/ipxe/script.h b/roms/ipxe/src/include/ipxe/script.h
index 33420dae4..7e7a9a3a4 100644
--- a/roms/ipxe/src/include/ipxe/script.h
+++ b/roms/ipxe/src/include/ipxe/script.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/image.h>
 
diff --git a/roms/ipxe/src/include/ipxe/scsi.h b/roms/ipxe/src/include/ipxe/scsi.h
index 4428daac3..28b55b2d5 100644
--- a/roms/ipxe/src/include/ipxe/scsi.h
+++ b/roms/ipxe/src/include/ipxe/scsi.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Maximum block for READ/WRITE (10) commands */
 #define SCSI_MAX_BLOCK_10 0xffffffffULL
diff --git a/roms/ipxe/src/include/ipxe/segment.h b/roms/ipxe/src/include/ipxe/segment.h
index 37bed0e19..9d5ecbd9b 100644
--- a/roms/ipxe/src/include/ipxe/segment.h
+++ b/roms/ipxe/src/include/ipxe/segment.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 
diff --git a/roms/ipxe/src/include/ipxe/serial.h b/roms/ipxe/src/include/ipxe/serial.h
index b47b1d125..83be59c31 100644
--- a/roms/ipxe/src/include/ipxe/serial.h
+++ b/roms/ipxe/src/include/ipxe/serial.h
@@ -3,15 +3,14 @@
 
 /** @file
  *
- * Serial driver functions
+ * Serial console
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-extern void serial_putc ( int ch );
-extern int serial_getc ( void );
-extern int serial_ischar ( void );
-extern int serial_initialized;
+#include <ipxe/uart.h>
+
+extern struct uart serial_console;
 
 #endif /* _IPXE_SERIAL_H */
diff --git a/roms/ipxe/src/include/ipxe/settings.h b/roms/ipxe/src/include/ipxe/settings.h
index d6929ecd0..95a553cc8 100644
--- a/roms/ipxe/src/include/ipxe/settings.h
+++ b/roms/ipxe/src/include/ipxe/settings.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
@@ -415,6 +415,7 @@ extern const struct setting_type setting_type_uint32 __setting_type;
 extern const struct setting_type setting_type_hex __setting_type;
 extern const struct setting_type setting_type_hexhyp __setting_type;
 extern const struct setting_type setting_type_hexraw __setting_type;
+extern const struct setting_type setting_type_base64 __setting_type;
 extern const struct setting_type setting_type_uuid __setting_type;
 extern const struct setting_type setting_type_busdevfn __setting_type;
 extern const struct setting_type setting_type_dnssl __setting_type;
diff --git a/roms/ipxe/src/include/ipxe/settings_ui.h b/roms/ipxe/src/include/ipxe/settings_ui.h
index 5f7be30cc..0bf21935d 100644
--- a/roms/ipxe/src/include/ipxe/settings_ui.h
+++ b/roms/ipxe/src/include/ipxe/settings_ui.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct settings;
 
diff --git a/roms/ipxe/src/include/ipxe/sha256.h b/roms/ipxe/src/include/ipxe/sha256.h
index 9aa9f3e57..e234cce33 100644
--- a/roms/ipxe/src/include/ipxe/sha256.h
+++ b/roms/ipxe/src/include/ipxe/sha256.h
@@ -7,11 +7,14 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
 
+/** SHA-256 number of rounds */
+#define SHA256_ROUNDS 64
+
 /** An SHA-256 digest */
 struct sha256_digest {
 	/** Hash output */
@@ -58,6 +61,8 @@ union sha256_digest_data_dwords {
 struct sha256_context {
 	/** Amount of accumulated data */
 	size_t len;
+	/** Digest size */
+	size_t digestsize;
 	/** Digest and accumulated data */
 	union sha256_digest_data_dwords ddd;
 } __attribute__ (( packed ));
@@ -68,6 +73,16 @@ struct sha256_context {
 /** SHA-256 digest size */
 #define SHA256_DIGEST_SIZE sizeof ( struct sha256_digest )
 
+/** SHA-224 digest size */
+#define SHA224_DIGEST_SIZE ( SHA256_DIGEST_SIZE * 224 / 256 )
+
+extern void sha256_family_init ( struct sha256_context *context,
+				 const struct sha256_digest *init,
+				 size_t digestsize );
+extern void sha256_update ( void *ctx, const void *data, size_t len );
+extern void sha256_final ( void *ctx, void *out );
+
 extern struct digest_algorithm sha256_algorithm;
+extern struct digest_algorithm sha224_algorithm;
 
 #endif /* _IPXE_SHA256_H */
diff --git a/roms/ipxe/src/include/ipxe/sha512.h b/roms/ipxe/src/include/ipxe/sha512.h
new file mode 100644
index 000000000..8e22d8357
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/sha512.h
@@ -0,0 +1,98 @@
+#ifndef _IPXE_SHA512_H
+#define _IPXE_SHA512_H
+
+/** @file
+ *
+ * SHA-512 algorithm
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/crypto.h>
+
+/** SHA-512 number of rounds */
+#define SHA512_ROUNDS 80
+
+/** An SHA-512 digest */
+struct sha512_digest {
+	/** Hash output */
+	uint64_t h[8];
+};
+
+/** An SHA-512 data block */
+union sha512_block {
+	/** Raw bytes */
+	uint8_t byte[128];
+	/** Raw qwords */
+	uint64_t qword[16];
+	/** Final block structure */
+	struct {
+		/** Padding */
+		uint8_t pad[112];
+		/** High 64 bits of length in bits */
+		uint64_t len_hi;
+		/** Low 64 bits of length in bits */
+		uint64_t len_lo;
+	} final;
+};
+
+/** SHA-512 digest and data block
+ *
+ * The order of fields within this structure is designed to minimise
+ * code size.
+ */
+struct sha512_digest_data {
+	/** Digest of data already processed */
+	struct sha512_digest digest;
+	/** Accumulated data */
+	union sha512_block data;
+} __attribute__ (( packed ));
+
+/** SHA-512 digest and data block */
+union sha512_digest_data_qwords {
+	/** Digest and data block */
+	struct sha512_digest_data dd;
+	/** Raw qwords */
+	uint64_t qword[ sizeof ( struct sha512_digest_data ) /
+			sizeof ( uint64_t ) ];
+};
+
+/** An SHA-512 context */
+struct sha512_context {
+	/** Amount of accumulated data */
+	size_t len;
+	/** Digest size */
+	size_t digestsize;
+	/** Digest and accumulated data */
+	union sha512_digest_data_qwords ddq;
+} __attribute__ (( packed ));
+
+/** SHA-512 context size */
+#define SHA512_CTX_SIZE sizeof ( struct sha512_context )
+
+/** SHA-512 digest size */
+#define SHA512_DIGEST_SIZE sizeof ( struct sha512_digest )
+
+/** SHA-384 digest size */
+#define SHA384_DIGEST_SIZE ( SHA512_DIGEST_SIZE * 384 / 512 )
+
+/** SHA-512/256 digest size */
+#define SHA512_256_DIGEST_SIZE ( SHA512_DIGEST_SIZE * 256 / 512 )
+
+/** SHA-512/224 digest size */
+#define SHA512_224_DIGEST_SIZE ( SHA512_DIGEST_SIZE * 224 / 512 )
+
+extern void sha512_family_init ( struct sha512_context *context,
+				 const struct sha512_digest *init,
+				 size_t digestsize );
+extern void sha512_update ( void *ctx, const void *data, size_t len );
+extern void sha512_final ( void *ctx, void *out );
+
+extern struct digest_algorithm sha512_algorithm;
+extern struct digest_algorithm sha384_algorithm;
+extern struct digest_algorithm sha512_256_algorithm;
+extern struct digest_algorithm sha512_224_algorithm;
+
+#endif /* IPXE_SHA512_H */
diff --git a/roms/ipxe/src/include/ipxe/shell.h b/roms/ipxe/src/include/ipxe/shell.h
index faa32f422..0d574e028 100644
--- a/roms/ipxe/src/include/ipxe/shell.h
+++ b/roms/ipxe/src/include/ipxe/shell.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Shell stop states */
 enum shell_stop_state {
diff --git a/roms/ipxe/src/include/ipxe/smbios.h b/roms/ipxe/src/include/ipxe/smbios.h
index ef5892a21..24b05ed62 100644
--- a/roms/ipxe/src/include/ipxe/smbios.h
+++ b/roms/ipxe/src/include/ipxe/smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/api.h>
diff --git a/roms/ipxe/src/include/ipxe/socket.h b/roms/ipxe/src/include/ipxe/socket.h
index 7cb3912f4..8c70ea4c0 100644
--- a/roms/ipxe/src/include/ipxe/socket.h
+++ b/roms/ipxe/src/include/ipxe/socket.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
diff --git a/roms/ipxe/src/include/ipxe/spi.h b/roms/ipxe/src/include/ipxe/spi.h
index d92d1aec9..83b53bce3 100644
--- a/roms/ipxe/src/include/ipxe/spi.h
+++ b/roms/ipxe/src/include/ipxe/spi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/nvs.h>
 
diff --git a/roms/ipxe/src/include/ipxe/spi_bit.h b/roms/ipxe/src/include/ipxe/spi_bit.h
index 9cfa7b825..049d30a22 100644
--- a/roms/ipxe/src/include/ipxe/spi_bit.h
+++ b/roms/ipxe/src/include/ipxe/spi_bit.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/spi.h>
 #include <ipxe/bitbash.h>
diff --git a/roms/ipxe/src/include/ipxe/stp.h b/roms/ipxe/src/include/ipxe/stp.h
new file mode 100644
index 000000000..3d85e5ba4
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/stp.h
@@ -0,0 +1,76 @@
+#ifndef _IPXE_STP_H
+#define _IPXE_STP_H
+
+/** @file
+ *
+ * Spanning Tree Protocol (STP)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/if_ether.h>
+
+/** "Protocol" value for STP
+ *
+ * This is the concatenated {DSAP,SSAP} value used internally by iPXE
+ * as the network-layer protocol for LLC frames.
+ */
+#define ETH_P_STP 0x4242
+
+/** A switch identifier */
+struct stp_switch {
+	/** Priotity */
+	uint16_t priority;
+	/** MAC address */
+	uint8_t mac[ETH_ALEN];
+} __attribute__ (( packed ));
+
+/** A Spanning Tree bridge protocol data unit */
+struct stp_bpdu {
+	/** LLC DSAP */
+	uint8_t dsap;
+	/** LLC SSAP */
+	uint8_t ssap;
+	/** LLC control field */
+	uint8_t control;
+	/** Protocol ID */
+	uint16_t protocol;
+	/** Protocol version */
+	uint8_t version;
+	/** Message type */
+	uint8_t type;
+	/** Flags */
+	uint8_t flags;
+	/** Root switch */
+	struct stp_switch root;
+	/** Root path cost */
+	uint32_t cost;
+	/** Sender switch */
+	struct stp_switch sender;
+	/** Port */
+	uint16_t port;
+	/** Message age */
+	uint16_t age;
+	/** Maximum age */
+	uint16_t max;
+	/** Hello time */
+	uint16_t hello;
+	/** Forward delay */
+	uint16_t delay;
+} __attribute__ (( packed ));
+
+/** Spanning Tree protocol ID */
+#define STP_PROTOCOL 0x0000
+
+/** Rapid Spanning Tree protocol version */
+#define STP_VERSION_RSTP 0x02
+
+/** Rapid Spanning Tree bridge PDU type */
+#define STP_TYPE_RSTP 0x02
+
+/** Port is forwarding */
+#define STP_FL_FORWARDING 0x20
+
+#endif /* _IPXE_STP_H */
diff --git a/roms/ipxe/src/include/ipxe/string.h b/roms/ipxe/src/include/ipxe/string.h
new file mode 100644
index 000000000..a8cbe8faa
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/string.h
@@ -0,0 +1,14 @@
+#ifndef _IPXE_STRING_H
+#define _IPXE_STRING_H
+
+/** @file
+ *
+ * String functions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern unsigned int digit_value ( unsigned int digit );
+
+#endif /* _IPXE_STRING_H */
diff --git a/roms/ipxe/src/include/ipxe/syslog.h b/roms/ipxe/src/include/ipxe/syslog.h
index 131692654..138440d66 100644
--- a/roms/ipxe/src/include/ipxe/syslog.h
+++ b/roms/ipxe/src/include/ipxe/syslog.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <syslog.h>
 
diff --git a/roms/ipxe/src/include/ipxe/tables.h b/roms/ipxe/src/include/ipxe/tables.h
index e35ce8220..60f8efdea 100644
--- a/roms/ipxe/src/include/ipxe/tables.h
+++ b/roms/ipxe/src/include/ipxe/tables.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_TABLES_H
 #define _IPXE_TABLES_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @page ifdef_harmful #ifdef considered harmful
  *
diff --git a/roms/ipxe/src/include/ipxe/tcp.h b/roms/ipxe/src/include/ipxe/tcp.h
index 9baa6391c..063ebaa4b 100644
--- a/roms/ipxe/src/include/ipxe/tcp.h
+++ b/roms/ipxe/src/include/ipxe/tcp.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tcpip.h>
 
@@ -79,6 +79,48 @@ struct tcp_window_scale_padded_option {
  */
 #define TCP_RX_WINDOW_SCALE 9
 
+/** TCP selective acknowledgement permitted option */
+struct tcp_sack_permitted_option {
+	uint8_t kind;
+	uint8_t length;
+} __attribute__ (( packed ));
+
+/** Padded TCP selective acknowledgement permitted option (used for sending) */
+struct tcp_sack_permitted_padded_option {
+	uint8_t nop[2];
+	struct tcp_sack_permitted_option spopt;
+} __attribute__ (( packed ));
+
+/** Code for the TCP selective acknowledgement permitted option */
+#define TCP_OPTION_SACK_PERMITTED 4
+
+/** TCP selective acknowledgement option */
+struct tcp_sack_option {
+	uint8_t kind;
+	uint8_t length;
+} __attribute__ (( packed ));
+
+/** TCP selective acknowledgement block */
+struct tcp_sack_block {
+	uint32_t left;
+	uint32_t right;
+} __attribute__ (( packed ));
+
+/** Maximum number of selective acknowledgement blocks
+ *
+ * This allows for the presence of the TCP timestamp option.
+ */
+#define TCP_SACK_MAX 3
+
+/** Padded TCP selective acknowledgement option (used for sending) */
+struct tcp_sack_padded_option {
+	uint8_t nop[2];
+	struct tcp_sack_option sackopt;
+} __attribute__ (( packed ));
+
+/** Code for the TCP selective acknowledgement option */
+#define TCP_OPTION_SACK 5
+
 /** TCP timestamp option */
 struct tcp_timestamp_option {
 	uint8_t kind;
@@ -102,6 +144,8 @@ struct tcp_options {
 	const struct tcp_mss_option *mssopt;
 	/** Window scale option, if present */
 	const struct tcp_window_scale_option *wsopt;
+	/** SACK permitted option, if present */
+	const struct tcp_sack_permitted_option *spopt;
 	/** Timestamp option, if present */
 	const struct tcp_timestamp_option *tsopt;
 };
@@ -376,6 +420,13 @@ static inline int tcp_in_window ( uint32_t seq, uint32_t start,
 	return ( ( seq - start ) < len );
 }
 
+/** TCP finish wait time
+ *
+ * Currently set to one second, since we should not allow a slowly
+ * responding server to substantially delay a call to shutdown().
+ */
+#define TCP_FINISH_TIMEOUT ( 1 * TICKS_PER_SEC )
+
 extern struct tcpip_protocol tcp_protocol __tcpip_protocol;
 
 #endif /* _IPXE_TCP_H */
diff --git a/roms/ipxe/src/include/ipxe/tcpip.h b/roms/ipxe/src/include/ipxe/tcpip.h
index 200630d6b..3cfc8e3ac 100644
--- a/roms/ipxe/src/include/ipxe/tcpip.h
+++ b/roms/ipxe/src/include/ipxe/tcpip.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/socket.h>
@@ -48,6 +48,12 @@ struct sockaddr_tcpip {
 	uint16_t st_flags;
 	/** TCP/IP port */
 	uint16_t st_port;
+	/** Scope ID
+	 *
+	 * For link-local or multicast addresses, this is the network
+	 * device index.
+	 */
+        uint16_t st_scope_id;
 	/** Padding
 	 *
 	 * This ensures that a struct @c sockaddr_tcpip is large
@@ -57,7 +63,8 @@ struct sockaddr_tcpip {
 	char pad[ sizeof ( struct sockaddr ) -
 		  ( sizeof ( sa_family_t ) /* st_family */ +
 		    sizeof ( uint16_t ) /* st_flags */ +
-		    sizeof ( uint16_t ) /* st_port */ ) ];
+		    sizeof ( uint16_t ) /* st_port */ +
+		    sizeof ( uint16_t ) /* st_scope_id */ ) ];
 } __attribute__ (( packed, may_alias ));
 
 /** 
diff --git a/roms/ipxe/src/include/ipxe/test.h b/roms/ipxe/src/include/ipxe/test.h
index 028ee29fb..0b65c299c 100644
--- a/roms/ipxe/src/include/ipxe/test.h
+++ b/roms/ipxe/src/include/ipxe/test.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_TEST_H
 #define _IPXE_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/include/ipxe/tftp.h b/roms/ipxe/src/include/ipxe/tftp.h
index aecafa2ae..e3661e1ac 100644
--- a/roms/ipxe/src/include/ipxe/tftp.h
+++ b/roms/ipxe/src/include/ipxe/tftp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/ipxe/time.h b/roms/ipxe/src/include/ipxe/time.h
index 673fe098a..4c5bb2a00 100644
--- a/roms/ipxe/src/include/ipxe/time.h
+++ b/roms/ipxe/src/include/ipxe/time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <sys/time.h>
 #include <ipxe/api.h>
@@ -44,6 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 /* Include all architecture-independent time API headers */
 #include <ipxe/null_time.h>
+#include <ipxe/efi/efi_time.h>
 #include <ipxe/linux/linux_time.h>
 
 /* Include all architecture-dependent time API headers */
diff --git a/roms/ipxe/src/include/ipxe/timer.h b/roms/ipxe/src/include/ipxe/timer.h
index d0309655d..82fbb6764 100644
--- a/roms/ipxe/src/include/ipxe/timer.h
+++ b/roms/ipxe/src/include/ipxe/timer.h
@@ -9,7 +9,7 @@
  * for a monotonically increasing tick counter.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/timer.h>
diff --git a/roms/ipxe/src/include/ipxe/tls.h b/roms/ipxe/src/include/ipxe/tls.h
index 586da26ec..7d982c326 100644
--- a/roms/ipxe/src/include/ipxe/tls.h
+++ b/roms/ipxe/src/include/ipxe/tls.h
@@ -7,7 +7,7 @@
  * Transport Layer Security Protocol
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
@@ -20,6 +20,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/x509.h>
 #include <ipxe/pending.h>
 #include <ipxe/iobuf.h>
+#include <ipxe/tables.h>
 
 /** A TLS header */
 struct tls_header {
@@ -85,7 +86,10 @@ struct tls_header {
 /* TLS hash algorithm identifiers */
 #define TLS_MD5_ALGORITHM 1
 #define TLS_SHA1_ALGORITHM 2
+#define TLS_SHA224_ALGORITHM 3
 #define TLS_SHA256_ALGORITHM 4
+#define TLS_SHA384_ALGORITHM 5
+#define TLS_SHA512_ALGORITHM 6
 
 /* TLS signature algorithm identifiers */
 #define TLS_RSA_ALGORITHM 1
@@ -101,6 +105,9 @@ struct tls_header {
 #define TLS_MAX_FRAGMENT_LENGTH_2048 3
 #define TLS_MAX_FRAGMENT_LENGTH_4096 4
 
+/* TLS signature algorithms extension */
+#define TLS_SIGNATURE_ALGORITHMS 13
+
 /** TLS RX state machine state */
 enum tls_rx_state {
 	TLS_RX_HEADER = 0,
@@ -131,6 +138,14 @@ struct tls_cipher_suite {
 	uint16_t code;
 };
 
+/** TLS cipher suite table */
+#define TLS_CIPHER_SUITES						\
+	__table ( struct tls_cipher_suite, "tls_cipher_suites" )
+
+/** Declare a TLS cipher suite */
+#define __tls_cipher_suite( pref )					\
+	__table_entry ( TLS_CIPHER_SUITES, pref )
+
 /** A TLS cipher specification */
 struct tls_cipherspec {
 	/** Cipher suite */
@@ -165,6 +180,19 @@ struct tls_signature_hash_algorithm {
 	struct tls_signature_hash_id code;
 };
 
+/** TLS signature hash algorithm table
+ *
+ * Note that the default (TLSv1.1 and earlier) algorithm using
+ * MD5+SHA1 is never explicitly specified.
+ */
+#define TLS_SIG_HASH_ALGORITHMS						\
+	__table ( struct tls_signature_hash_algorithm,			\
+		  "tls_sig_hash_algorithms" )
+
+/** Declare a TLS signature hash algorithm */
+#define __tls_sig_hash_algorithm					\
+	__table_entry ( TLS_SIG_HASH_ALGORITHMS, 01 )
+
 /** TLS pre-master secret */
 struct tls_pre_master_secret {
 	/** TLS version */
diff --git a/roms/ipxe/src/include/ipxe/uaccess.h b/roms/ipxe/src/include/ipxe/uaccess.h
index 055bb2ca7..a3f78566a 100644
--- a/roms/ipxe/src/include/ipxe/uaccess.h
+++ b/roms/ipxe/src/include/ipxe/uaccess.h
@@ -19,7 +19,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/include/ipxe/uart.h b/roms/ipxe/src/include/ipxe/uart.h
new file mode 100644
index 000000000..c63eae615
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/uart.h
@@ -0,0 +1,132 @@
+#ifndef _IPXE_UART_H
+#define _IPXE_UART_H
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** Transmitter holding register */
+#define UART_THR 0x00
+
+/** Receiver buffer register */
+#define UART_RBR 0x00
+
+/** Interrupt enable register */
+#define UART_IER 0x01
+
+/** FIFO control register */
+#define UART_FCR 0x02
+#define UART_FCR_FE	0x01	/**< FIFO enable */
+
+/** Line control register */
+#define UART_LCR 0x03
+#define UART_LCR_WLS0	0x01	/**< Word length select bit 0 */
+#define UART_LCR_WLS1	0x02	/**< Word length select bit 1 */
+#define UART_LCR_STB	0x04	/**< Number of stop bits */
+#define UART_LCR_PEN	0x08	/**< Parity enable */
+#define UART_LCR_EPS	0x10	/**< Even parity select */
+#define UART_LCR_DLAB	0x80	/**< Divisor latch access bit */
+
+#define UART_LCR_WORD_LEN(x)	( ( (x) - 5 ) << 0 )	/**< Word length */
+#define UART_LCR_STOP_BITS(x)	( ( (x) - 1 ) << 2 )	/**< Stop bits */
+#define UART_LCR_PARITY(x)	( ( (x) - 0 ) << 3 )	/**< Parity */
+
+/**
+ * Calculate line control register value
+ *
+ * @v word_len		Word length (5-8)
+ * @v parity		Parity (0=none, 1=odd, 3=even)
+ * @v stop_bits		Stop bits (1-2)
+ * @ret lcr		Line control register value
+ */
+#define UART_LCR_WPS( word_len, parity, stop_bits )	\
+	( UART_LCR_WORD_LEN ( (word_len) ) |		\
+	  UART_LCR_PARITY ( (parity) ) |		\
+	  UART_LCR_STOP_BITS ( (stop_bits) ) )
+
+/** Default LCR value: 8 data bits, no parity, one stop bit */
+#define UART_LCR_8N1 UART_LCR_WPS ( 8, 0, 1 )
+
+/** Modem control register */
+#define UART_MCR 0x04
+#define UART_MCR_DTR	0x01	/**< Data terminal ready */
+#define UART_MCR_RTS	0x02	/**< Request to send */
+
+/** Line status register */
+#define UART_LSR 0x05
+#define UART_LSR_DR	0x01	/**< Data ready */
+#define UART_LSR_THRE	0x20	/**< Transmitter holding register empty */
+#define UART_LSR_TEMT	0x40	/**< Transmitter empty */
+
+/** Scratch register */
+#define UART_SCR 0x07
+
+/** Divisor latch (least significant byte) */
+#define UART_DLL 0x00
+
+/** Divisor latch (most significant byte) */
+#define UART_DLM 0x01
+
+/** Maximum baud rate */
+#define UART_MAX_BAUD 115200
+
+/** A 16550-compatible UART */
+struct uart {
+	/** I/O port base address */
+	void *base;
+	/** Baud rate divisor */
+	uint16_t divisor;
+	/** Line control register */
+	uint8_t lcr;
+};
+
+/** Symbolic names for port indexes */
+enum uart_port {
+	COM1 = 1,
+	COM2 = 2,
+	COM3 = 3,
+	COM4 = 4,
+};
+
+#include <bits/uart.h>
+
+void uart_write ( struct uart *uart, unsigned int addr, uint8_t data );
+uint8_t uart_read ( struct uart *uart, unsigned int addr );
+int uart_select ( struct uart *uart, unsigned int port );
+
+/**
+ * Check if received data is ready
+ *
+ * @v uart		UART
+ * @ret ready		Data is ready
+ */
+static inline int uart_data_ready ( struct uart *uart ) {
+	uint8_t lsr;
+
+	lsr = uart_read ( uart, UART_LSR );
+	return ( lsr & UART_LSR_DR );
+}
+
+/**
+ * Receive data
+ *
+ * @v uart		UART
+ * @ret data		Data
+ */
+static inline uint8_t uart_receive ( struct uart *uart ) {
+
+	return uart_read ( uart, UART_RBR );
+}
+
+extern void uart_transmit ( struct uart *uart, uint8_t data );
+extern void uart_flush ( struct uart *uart );
+extern int uart_exists ( struct uart *uart );
+extern int uart_init ( struct uart *uart, unsigned int baud, uint8_t lcr );
+
+#endif /* _IPXE_UART_H */
diff --git a/roms/ipxe/src/include/ipxe/udp.h b/roms/ipxe/src/include/ipxe/udp.h
index 5717ef213..7b0de4dc0 100644
--- a/roms/ipxe/src/include/ipxe/udp.h
+++ b/roms/ipxe/src/include/ipxe/udp.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/iobuf.h>
diff --git a/roms/ipxe/src/include/ipxe/umalloc.h b/roms/ipxe/src/include/ipxe/umalloc.h
index 4b25e182a..3892ef53b 100644
--- a/roms/ipxe/src/include/ipxe/umalloc.h
+++ b/roms/ipxe/src/include/ipxe/umalloc.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/umalloc.h>
diff --git a/roms/ipxe/src/include/ipxe/uri.h b/roms/ipxe/src/include/ipxe/uri.h
index 7613d578d..00e5a24c4 100644
--- a/roms/ipxe/src/include/ipxe/uri.h
+++ b/roms/ipxe/src/include/ipxe/uri.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
@@ -203,7 +203,7 @@ extern char * resolve_path ( const char *base_path,
 			     const char *relative_path );
 extern struct uri * resolve_uri ( const struct uri *base_uri,
 				  struct uri *relative_uri );
-extern struct uri * tftp_uri ( struct in_addr next_server,
+extern struct uri * tftp_uri ( struct in_addr next_server, unsigned int port,
 			       const char *filename );
 extern void churi ( struct uri *uri );
 
diff --git a/roms/ipxe/src/include/ipxe/usb.h b/roms/ipxe/src/include/ipxe/usb.h
new file mode 100644
index 000000000..ab060b8f4
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/usb.h
@@ -0,0 +1,1319 @@
+#ifndef _IPXE_USB_H
+#define _IPXE_USB_H
+
+/** @file
+ *
+ * Universal Serial Bus (USB)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/list.h>
+#include <ipxe/device.h>
+#include <ipxe/process.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/tables.h>
+
+/** USB protocols */
+enum usb_protocol {
+	/** USB 2.0 */
+	USB_PROTO_2_0 = 0x0200,
+	/** USB 3.0 */
+	USB_PROTO_3_0 = 0x0300,
+	/** USB 3.1 */
+	USB_PROTO_3_1 = 0x0301,
+};
+
+/** Define a USB speed
+ *
+ * @v mantissa		Mantissa
+ * @v exponent		Exponent (in engineering terms: 1=k, 2=M, 3=G)
+ * @ret speed		USB speed
+ */
+#define USB_SPEED( mantissa, exponent ) ( (exponent << 16) | (mantissa) )
+
+/** Extract USB speed mantissa */
+#define USB_SPEED_MANTISSA(speed) ( (speed) & 0xffff )
+
+/** Extract USB speed exponent */
+#define USB_SPEED_EXPONENT(speed) ( ( (speed) >> 16 ) & 0x3 )
+
+/** USB device speeds */
+enum usb_speed {
+	/** Not connected */
+	USB_SPEED_NONE = 0,
+	/** Low speed (1.5Mbps) */
+	USB_SPEED_LOW = USB_SPEED ( 1500, 1 ),
+	/** Full speed (12Mbps) */
+	USB_SPEED_FULL = USB_SPEED ( 12, 2 ),
+	/** High speed (480Mbps) */
+	USB_SPEED_HIGH = USB_SPEED ( 480, 2 ),
+	/** Super speed (5Gbps) */
+	USB_SPEED_SUPER = USB_SPEED ( 5, 3 ),
+};
+
+/** USB packet IDs */
+enum usb_pid {
+	/** IN PID */
+	USB_PID_IN = 0x69,
+	/** OUT PID */
+	USB_PID_OUT = 0xe1,
+	/** SETUP PID */
+	USB_PID_SETUP = 0x2d,
+};
+
+/** A USB setup data packet */
+struct usb_setup_packet {
+	/** Request */
+	uint16_t request;
+	/** Value paramer */
+	uint16_t value;
+	/** Index parameter */
+	uint16_t index;
+	/** Length of data stage */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** Data transfer is from host to device */
+#define USB_DIR_OUT ( 0 << 7 )
+
+/** Data transfer is from device to host */
+#define USB_DIR_IN ( 1 << 7 )
+
+/** Standard request type */
+#define USB_TYPE_STANDARD ( 0 << 5 )
+
+/** Class-specific request type */
+#define USB_TYPE_CLASS ( 1 << 5 )
+
+/** Vendor-specific request type */
+#define USB_TYPE_VENDOR ( 2 << 5 )
+
+/** Request recipient is the device */
+#define USB_RECIP_DEVICE ( 0 << 0 )
+
+/** Request recipient is an interface */
+#define USB_RECIP_INTERFACE ( 1 << 0 )
+
+/** Request recipient is an endpoint */
+#define USB_RECIP_ENDPOINT ( 2 << 0 )
+
+/** Construct USB request type */
+#define USB_REQUEST_TYPE(type) ( (type) << 8 )
+
+/** Get status */
+#define USB_GET_STATUS ( USB_DIR_IN | USB_REQUEST_TYPE ( 0 ) )
+
+/** Clear feature */
+#define USB_CLEAR_FEATURE ( USB_DIR_OUT | USB_REQUEST_TYPE ( 1 ) )
+
+/** Set feature */
+#define USB_SET_FEATURE ( USB_DIR_OUT | USB_REQUEST_TYPE ( 3 ) )
+
+/** Set address */
+#define USB_SET_ADDRESS ( USB_DIR_OUT | USB_REQUEST_TYPE ( 5 ) )
+
+/** Get descriptor */
+#define USB_GET_DESCRIPTOR ( USB_DIR_IN | USB_REQUEST_TYPE ( 6 ) )
+
+/** Set descriptor */
+#define USB_SET_DESCRIPTOR ( USB_DIR_OUT | USB_REQUEST_TYPE ( 7 ) )
+
+/** Get configuration */
+#define USB_GET_CONFIGURATION ( USB_DIR_IN | USB_REQUEST_TYPE ( 8 ) )
+
+/** Set configuration */
+#define USB_SET_CONFIGURATION ( USB_DIR_OUT | USB_REQUEST_TYPE ( 9 ) )
+
+/** Get interface */
+#define USB_GET_INTERFACE \
+	( USB_DIR_IN | USB_RECIP_INTERFACE | USB_REQUEST_TYPE ( 10 ) )
+
+/** Set interface */
+#define USB_SET_INTERFACE \
+	( USB_DIR_OUT | USB_RECIP_INTERFACE | USB_REQUEST_TYPE ( 11 ) )
+
+/** Endpoint halt feature */
+#define USB_ENDPOINT_HALT 0
+
+/** A USB class code tuple */
+struct usb_class {
+	/** Class code */
+	uint8_t class;
+	/** Subclass code */
+	uint8_t subclass;
+	/** Protocol code */
+	uint8_t protocol;
+} __attribute__ (( packed ));
+
+/** Class code for USB hubs */
+#define USB_CLASS_HUB 9
+
+/** A USB descriptor header */
+struct usb_descriptor_header {
+	/** Length of descriptor */
+	uint8_t len;
+	/** Descriptor type */
+	uint8_t type;
+} __attribute__ (( packed ));
+
+/** A USB device descriptor */
+struct usb_device_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** USB specification release number in BCD */
+	uint16_t protocol;
+	/** Device class */
+	struct usb_class class;
+	/** Maximum packet size for endpoint zero */
+	uint8_t mtu;
+	/** Vendor ID */
+	uint16_t vendor;
+	/** Product ID */
+	uint16_t product;
+	/** Device release number in BCD */
+	uint16_t release;
+	/** Manufacturer string */
+	uint8_t manufacturer;
+	/** Product string */
+	uint8_t name;
+	/** Serial number string */
+	uint8_t serial;
+	/** Number of possible configurations */
+	uint8_t configurations;
+} __attribute__ (( packed ));
+
+/** A USB device descriptor */
+#define USB_DEVICE_DESCRIPTOR 1
+
+/** A USB configuration descriptor */
+struct usb_configuration_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Total length */
+	uint16_t len;
+	/** Number of interfaces */
+	uint8_t interfaces;
+	/** Configuration value */
+	uint8_t config;
+	/** Configuration string */
+	uint8_t name;
+	/** Attributes */
+	uint8_t attributes;
+	/** Maximum power consumption */
+	uint8_t power;
+} __attribute__ (( packed ));
+
+/** A USB configuration descriptor */
+#define USB_CONFIGURATION_DESCRIPTOR 2
+
+/** A USB string descriptor */
+struct usb_string_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** String */
+	char string[0];
+} __attribute__ (( packed ));
+
+/** A USB string descriptor */
+#define USB_STRING_DESCRIPTOR 3
+
+/** A USB interface descriptor */
+struct usb_interface_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Interface number */
+	uint8_t interface;
+	/** Alternate setting */
+	uint8_t alternate;
+	/** Number of endpoints */
+	uint8_t endpoints;
+	/** Interface class */
+	struct usb_class class;
+	/** Interface name */
+	uint8_t name;
+} __attribute__ (( packed ));
+
+/** A USB interface descriptor */
+#define USB_INTERFACE_DESCRIPTOR 4
+
+/** A USB endpoint descriptor */
+struct usb_endpoint_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Endpoint address */
+	uint8_t endpoint;
+	/** Attributes */
+	uint8_t attributes;
+	/** Maximum packet size and burst size */
+	uint16_t sizes;
+	/** Polling interval */
+	uint8_t interval;
+} __attribute__ (( packed ));
+
+/** A USB endpoint descriptor */
+#define USB_ENDPOINT_DESCRIPTOR 5
+
+/** Endpoint attribute transfer type mask */
+#define USB_ENDPOINT_ATTR_TYPE_MASK 0x03
+
+/** Endpoint periodic type */
+#define USB_ENDPOINT_ATTR_PERIODIC 0x01
+
+/** Control endpoint transfer type */
+#define USB_ENDPOINT_ATTR_CONTROL 0x00
+
+/** Bulk endpoint transfer type */
+#define USB_ENDPOINT_ATTR_BULK 0x02
+
+/** Interrupt endpoint transfer type */
+#define USB_ENDPOINT_ATTR_INTERRUPT 0x03
+
+/** Bulk OUT endpoint (internal) type */
+#define USB_BULK_OUT ( USB_ENDPOINT_ATTR_BULK | USB_DIR_OUT )
+
+/** Bulk IN endpoint (internal) type */
+#define USB_BULK_IN ( USB_ENDPOINT_ATTR_BULK | USB_DIR_IN )
+
+/** Interrupt IN endpoint (internal) type */
+#define USB_INTERRUPT_IN ( USB_ENDPOINT_ATTR_INTERRUPT | USB_DIR_IN )
+
+/** Interrupt OUT endpoint (internal) type */
+#define USB_INTERRUPT_OUT ( USB_ENDPOINT_ATTR_INTERRUPT | USB_DIR_OUT )
+
+/** USB endpoint MTU */
+#define USB_ENDPOINT_MTU(sizes) ( ( (sizes) >> 0 ) & 0x07ff )
+
+/** USB endpoint maximum burst size */
+#define USB_ENDPOINT_BURST(sizes) ( ( (sizes) >> 11 ) & 0x0003 )
+
+/** A USB endpoint companion descriptor */
+struct usb_endpoint_companion_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Maximum burst size */
+	uint8_t burst;
+	/** Extended attributes */
+	uint8_t extended;
+	/** Number of bytes per service interval */
+	uint16_t periodic;
+} __attribute__ (( packed ));
+
+/** A USB endpoint companion descriptor */
+#define USB_ENDPOINT_COMPANION_DESCRIPTOR 48
+
+/** A USB interface association descriptor */
+struct usb_interface_association_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** First interface number */
+	uint8_t first;
+	/** Interface count */
+	uint8_t count;
+	/** Association class */
+	struct usb_class class;
+	/** Association name */
+	uint8_t name;
+} __attribute__ (( packed ));
+
+/** A USB interface association descriptor */
+#define USB_INTERFACE_ASSOCIATION_DESCRIPTOR 11
+
+/** A class-specific interface descriptor */
+#define USB_CS_INTERFACE_DESCRIPTOR 36
+
+/** A class-specific endpoint descriptor */
+#define USB_CS_ENDPOINT_DESCRIPTOR 37
+
+/**
+ * Get next USB descriptor
+ *
+ * @v desc		USB descriptor header
+ * @ret next		Next USB descriptor header
+ */
+static inline __attribute__ (( always_inline )) struct usb_descriptor_header *
+usb_next_descriptor ( struct usb_descriptor_header *desc ) {
+
+	return ( ( ( void * ) desc ) + desc->len );
+}
+
+/**
+ * Check that descriptor lies within a configuration descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v desc		Descriptor header
+ * @v is_within		Descriptor is within the configuration descriptor
+ */
+static inline __attribute__ (( always_inline )) int
+usb_is_within_config ( struct usb_configuration_descriptor *config,
+		       struct usb_descriptor_header *desc ) {
+	struct usb_descriptor_header *end =
+		( ( ( void * ) config ) + le16_to_cpu ( config->len ) );
+
+	/* Check that descriptor starts within the configuration
+	 * descriptor, and that the length does not exceed the
+	 * configuration descriptor.  This relies on the fact that
+	 * usb_next_descriptor() needs to access only the first byte
+	 * of the descriptor in order to determine the length.
+	 */
+	return ( ( desc < end ) && ( usb_next_descriptor ( desc ) <= end ) );
+}
+
+/** Iterate over all configuration descriptors */
+#define for_each_config_descriptor( desc, config )			   \
+	for ( desc = container_of ( &(config)->header,			   \
+				    typeof ( *desc ), header ) ;	   \
+	      usb_is_within_config ( (config), &desc->header ) ;	   \
+	      desc = container_of ( usb_next_descriptor ( &desc->header ), \
+				    typeof ( *desc ), header ) )
+
+/** Iterate over all configuration descriptors within an interface descriptor */
+#define for_each_interface_descriptor( desc, config, interface )	   \
+	for ( desc = container_of ( usb_next_descriptor ( &(interface)->   \
+							  header ),	   \
+				    typeof ( *desc ), header ) ;	   \
+	      ( usb_is_within_config ( (config), &desc->header ) &&	   \
+		( desc->header.type != USB_INTERFACE_DESCRIPTOR ) ) ;	   \
+	      desc = container_of ( usb_next_descriptor ( &desc->header ), \
+				    typeof ( *desc ), header ) )
+
+/** A USB endpoint */
+struct usb_endpoint {
+	/** USB device */
+	struct usb_device *usb;
+	/** Endpoint address */
+	unsigned int address;
+	/** Attributes */
+	unsigned int attributes;
+	/** Maximum transfer size */
+	size_t mtu;
+	/** Maximum burst size */
+	unsigned int burst;
+	/** Interval (in microframes) */
+	unsigned int interval;
+
+	/** Endpoint is open */
+	int open;
+	/** Buffer fill level */
+	unsigned int fill;
+
+	/** List of halted endpoints */
+	struct list_head halted;
+
+	/** Host controller operations */
+	struct usb_endpoint_host_operations *host;
+	/** Host controller private data */
+	void *priv;
+	/** Driver operations */
+	struct usb_endpoint_driver_operations *driver;
+
+	/** Recycled I/O buffer list */
+	struct list_head recycled;
+	/** Refill buffer length */
+	size_t len;
+	/** Maximum fill level */
+	unsigned int max;
+};
+
+/** USB endpoint host controller operations */
+struct usb_endpoint_host_operations {
+	/** Open endpoint
+	 *
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_endpoint *ep );
+	/** Close endpoint
+	 *
+	 * @v ep		USB endpoint
+	 */
+	void ( * close ) ( struct usb_endpoint *ep );
+	/**
+	 * Reset endpoint
+	 *
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * reset ) ( struct usb_endpoint *ep );
+	/** Update MTU
+	 *
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * mtu ) ( struct usb_endpoint *ep );
+	/** Enqueue message transfer
+	 *
+	 * @v ep		USB endpoint
+	 * @v iobuf		I/O buffer
+	 * @ret rc		Return status code
+	 */
+	int ( * message ) ( struct usb_endpoint *ep,
+			    struct io_buffer *iobuf );
+	/** Enqueue stream transfer
+	 *
+	 * @v ep		USB endpoint
+	 * @v iobuf		I/O buffer
+	 * @v terminate		Terminate using a short packet
+	 * @ret rc		Return status code
+	 */
+	int ( * stream ) ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			   int terminate );
+};
+
+/** USB endpoint driver operations */
+struct usb_endpoint_driver_operations {
+	/** Complete transfer
+	 *
+	 * @v ep		USB endpoint
+	 * @v iobuf		I/O buffer
+	 * @v rc		Completion status code
+	 */
+	void ( * complete ) ( struct usb_endpoint *ep,
+			      struct io_buffer *iobuf, int rc );
+};
+
+/** Control endpoint address */
+#define USB_EP0_ADDRESS 0x00
+
+/** Control endpoint attributes */
+#define USB_EP0_ATTRIBUTES 0x00
+
+/** Calculate default MTU based on device speed
+ *
+ * @v speed		Device speed
+ * @ret mtu		Default MTU
+ */
+#define USB_EP0_DEFAULT_MTU(speed)			\
+	( ( (speed) >= USB_SPEED_SUPER ) ? 512 :	\
+	  ( ( (speed) >= USB_SPEED_FULL ) ? 64 : 8 ) )
+
+/** Control endpoint maximum burst size */
+#define USB_EP0_BURST 0
+
+/** Control endpoint interval */
+#define USB_EP0_INTERVAL 0
+
+/** Maximum endpoint number */
+#define USB_ENDPOINT_MAX 0x0f
+
+/** Endpoint direction is in */
+#define USB_ENDPOINT_IN 0x80
+
+/** Construct endpoint index from endpoint address */
+#define USB_ENDPOINT_IDX(address)					\
+	( ( (address) & USB_ENDPOINT_MAX ) |				\
+	  ( ( (address) & USB_ENDPOINT_IN ) >> 3 ) )
+
+/**
+ * Initialise USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @v usb		USB device
+ * @v driver		Driver operations
+ */
+static inline __attribute__ (( always_inline )) void
+usb_endpoint_init ( struct usb_endpoint *ep, struct usb_device *usb,
+		    struct usb_endpoint_driver_operations *driver ) {
+
+	ep->usb = usb;
+	ep->driver = driver;
+}
+
+/**
+ * Describe USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @v address		Endpoint address
+ * @v attributes	Attributes
+ * @v mtu		Maximum packet size
+ * @v burst		Maximum burst size
+ * @v interval		Interval (in microframes)
+ */
+static inline __attribute__ (( always_inline )) void
+usb_endpoint_describe ( struct usb_endpoint *ep, unsigned int address,
+			unsigned int attributes, size_t mtu,
+			unsigned int burst, unsigned int interval ) {
+
+	ep->address = address;
+	ep->attributes = attributes;
+	ep->mtu = mtu;
+	ep->burst = burst;
+	ep->interval = interval;
+}
+
+/**
+ * Set USB endpoint host controller private data
+ *
+ * @v ep		USB endpoint
+ * @v priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_endpoint_set_hostdata ( struct usb_endpoint *ep, void *priv ) {
+	ep->priv = priv;
+}
+
+/**
+ * Get USB endpoint host controller private data
+ *
+ * @v ep		USB endpoint
+ * @ret priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_endpoint_get_hostdata ( struct usb_endpoint *ep ) {
+	return ep->priv;
+}
+
+extern const char * usb_endpoint_name ( struct usb_endpoint *ep );
+extern int
+usb_endpoint_described ( struct usb_endpoint *ep,
+			 struct usb_configuration_descriptor *config,
+			 struct usb_interface_descriptor *interface,
+			 unsigned int type, unsigned int index );
+extern int usb_endpoint_open ( struct usb_endpoint *ep );
+extern void usb_endpoint_close ( struct usb_endpoint *ep );
+extern int usb_message ( struct usb_endpoint *ep, unsigned int request,
+			 unsigned int value, unsigned int index,
+			 struct io_buffer *iobuf );
+extern int usb_stream ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			int terminate );
+extern void usb_complete_err ( struct usb_endpoint *ep,
+			       struct io_buffer *iobuf, int rc );
+
+/**
+ * Initialise USB endpoint refill
+ *
+ * @v ep		USB endpoint
+ * @v len		Refill buffer length (or zero to use endpoint's MTU)
+ * @v max		Maximum fill level
+ */
+static inline __attribute__ (( always_inline )) void
+usb_refill_init ( struct usb_endpoint *ep, size_t len, unsigned int max ) {
+
+	INIT_LIST_HEAD ( &ep->recycled );
+	ep->len = len;
+	ep->max = max;
+}
+
+/**
+ * Recycle I/O buffer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+usb_recycle ( struct usb_endpoint *ep, struct io_buffer *iobuf ) {
+
+	list_add_tail ( &iobuf->list, &ep->recycled );
+}
+
+extern int usb_prefill ( struct usb_endpoint *ep );
+extern int usb_refill ( struct usb_endpoint *ep );
+extern void usb_flush ( struct usb_endpoint *ep );
+
+/**
+ * A USB function
+ *
+ * A USB function represents an association of interfaces within a USB
+ * device.
+ */
+struct usb_function {
+	/** Name */
+	const char *name;
+	/** USB device */
+	struct usb_device *usb;
+	/** Class */
+	struct usb_class class;
+	/** Number of interfaces */
+	unsigned int count;
+	/** Generic device */
+	struct device dev;
+	/** List of functions within this USB device */
+	struct list_head list;
+
+	/** Driver */
+	struct usb_driver *driver;
+	/** Driver private data */
+	void *priv;
+
+	/** List of interface numbers
+	 *
+	 * This must be the last field within the structure.
+	 */
+	uint8_t interface[0];
+};
+
+/**
+ * Set USB function driver private data
+ *
+ * @v func		USB function
+ * @v priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_func_set_drvdata ( struct usb_function *func, void *priv ) {
+	func->priv = priv;
+}
+
+/**
+ * Get USB function driver private data
+ *
+ * @v function		USB function
+ * @ret priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_func_get_drvdata ( struct usb_function *func ) {
+	return func->priv;
+}
+
+/** A USB device */
+struct usb_device {
+	/** Name */
+	char name[32];
+	/** USB port */
+	struct usb_port *port;
+	/** List of devices on this bus */
+	struct list_head list;
+	/** Device address, if assigned */
+	unsigned int address;
+	/** Device descriptor */
+	struct usb_device_descriptor device;
+	/** List of functions */
+	struct list_head functions;
+
+	/** Host controller operations */
+	struct usb_device_host_operations *host;
+	/** Host controller private data */
+	void *priv;
+
+	/** Endpoint list */
+	struct usb_endpoint *ep[32];
+
+	/** Control endpoint */
+	struct usb_endpoint control;
+	/** Completed control transfers */
+	struct list_head complete;
+};
+
+/** USB device host controller operations */
+struct usb_device_host_operations {
+	/** Open device
+	 *
+	 * @v usb		USB device
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_device *usb );
+	/** Close device
+	 *
+	 * @v usb		USB device
+	 */
+	void ( * close ) ( struct usb_device *usb );
+	/** Assign device address
+	 *
+	 * @v usb		USB device
+	 * @ret rc		Return status code
+	 */
+	int ( * address ) ( struct usb_device *usb );
+};
+
+/**
+ * Set USB device host controller private data
+ *
+ * @v usb		USB device
+ * @v priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_set_hostdata ( struct usb_device *usb, void *priv ) {
+	usb->priv = priv;
+}
+
+/**
+ * Get USB device host controller private data
+ *
+ * @v usb		USB device
+ * @ret priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_get_hostdata ( struct usb_device *usb ) {
+	return usb->priv;
+}
+
+/**
+ * Get USB endpoint
+ *
+ * @v usb		USB device
+ * @v address		Endpoint address
+ * @ret ep		USB endpoint, or NULL if not opened
+ */
+static inline struct usb_endpoint * usb_endpoint ( struct usb_device *usb,
+						   unsigned int address ) {
+
+	return usb->ep[ USB_ENDPOINT_IDX ( address ) ];
+}
+
+/** A USB port */
+struct usb_port {
+	/** USB hub */
+	struct usb_hub *hub;
+	/** Port address */
+	unsigned int address;
+	/** Port protocol */
+	unsigned int protocol;
+	/** Port speed */
+	unsigned int speed;
+	/** Port disconnection has been detected
+	 *
+	 * This should be set whenever the underlying hardware reports
+	 * a connection status change.
+	 */
+	int disconnected;
+	/** Port has an attached device */
+	int attached;
+	/** Currently attached device (if in use)
+	 *
+	 * Note that this field will be NULL if the attached device
+	 * has been freed (e.g. because there were no drivers found).
+	 */
+	struct usb_device *usb;
+	/** List of changed ports */
+	struct list_head changed;
+};
+
+/** A USB hub */
+struct usb_hub {
+	/** Name */
+	const char *name;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Underlying USB device, if any */
+	struct usb_device *usb;
+	/** Hub protocol */
+	unsigned int protocol;
+	/** Number of ports */
+	unsigned int ports;
+
+	/** List of hubs */
+	struct list_head list;
+
+	/** Host controller operations */
+	struct usb_hub_host_operations *host;
+	/** Driver operations */
+	struct usb_hub_driver_operations *driver;
+	/** Driver private data */
+	void *priv;
+
+	/** Port list
+	 *
+	 * This must be the last field within the structure.
+	 */
+	struct usb_port port[0];
+};
+
+/** USB hub host controller operations */
+struct usb_hub_host_operations {
+	/** Open hub
+	 *
+	 * @v hub		USB hub
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_hub *hub );
+	/** Close hub
+	 *
+	 * @v hub		USB hub
+	 */
+	void ( * close ) ( struct usb_hub *hub );
+};
+
+/** USB hub driver operations */
+struct usb_hub_driver_operations {
+	/** Open hub
+	 *
+	 * @v hub		USB hub
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_hub *hub );
+	/** Close hub
+	 *
+	 * @v hub		USB hub
+	 */
+	void ( * close ) ( struct usb_hub *hub );
+	/** Enable port
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @ret rc		Return status code
+	 */
+	int ( * enable ) ( struct usb_hub *hub, struct usb_port *port );
+	/** Disable port
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @ret rc		Return status code
+	 */
+	int ( * disable ) ( struct usb_hub *hub, struct usb_port *port );
+	/** Update port speed
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @ret rc		Return status code
+	 */
+	int ( * speed ) ( struct usb_hub *hub, struct usb_port *port );
+	/** Clear transaction translator buffer
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * clear_tt ) ( struct usb_hub *hub, struct usb_port *port,
+			     struct usb_endpoint *ep );
+};
+
+/**
+ * Set USB hub driver private data
+ *
+ * @v hub		USB hub
+ * @v priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_hub_set_drvdata ( struct usb_hub *hub, void *priv ) {
+	hub->priv = priv;
+}
+
+/**
+ * Get USB hub driver private data
+ *
+ * @v hub		USB hub
+ * @ret priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_hub_get_drvdata ( struct usb_hub *hub ) {
+	return hub->priv;
+}
+
+/**
+ * Get USB port
+ *
+ * @v hub		USB hub
+ * @v address		Port address
+ * @ret port		USB port
+ */
+static inline __attribute__ (( always_inline )) struct usb_port *
+usb_port ( struct usb_hub *hub, unsigned int address ) {
+
+	return &hub->port[ address - 1 ];
+}
+
+/** A USB bus */
+struct usb_bus {
+	/** Name */
+	const char *name;
+	/** Underlying hardware device */
+	struct device *dev;
+	/** Host controller operations set */
+	struct usb_host_operations *op;
+
+	/** Largest transfer allowed on the bus */
+	size_t mtu;
+	/** Address in-use mask
+	 *
+	 * This is used only by buses which perform manual address
+	 * assignment.  USB allows for addresses in the range [1,127].
+	 * We use a simple bitmask which restricts us to the range
+	 * [1,64]; this is unlikely to be a problem in practice.  For
+	 * comparison: controllers which perform autonomous address
+	 * assignment (such as xHCI) typically allow for only 32
+	 * devices per bus anyway.
+	 */
+	unsigned long long addresses;
+
+	/** Root hub */
+	struct usb_hub *hub;
+
+	/** List of USB buses */
+	struct list_head list;
+	/** List of devices */
+	struct list_head devices;
+	/** List of hubs */
+	struct list_head hubs;
+
+	/** Host controller operations */
+	struct usb_bus_host_operations *host;
+	/** Host controller private data */
+	void *priv;
+};
+
+/** USB bus host controller operations */
+struct usb_bus_host_operations {
+	/** Open bus
+	 *
+	 * @v bus		USB bus
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_bus *bus );
+	/** Close bus
+	 *
+	 * @v bus		USB bus
+	 */
+	void ( * close ) ( struct usb_bus *bus );
+	/** Poll bus
+	 *
+	 * @v bus		USB bus
+	 */
+	void ( * poll ) ( struct usb_bus *bus );
+};
+
+/** USB host controller operations */
+struct usb_host_operations {
+	/** Endpoint operations */
+	struct usb_endpoint_host_operations endpoint;
+	/** Device operations */
+	struct usb_device_host_operations device;
+	/** Bus operations */
+	struct usb_bus_host_operations bus;
+	/** Hub operations */
+	struct usb_hub_host_operations hub;
+	/** Root hub operations */
+	struct usb_hub_driver_operations root;
+};
+
+/**
+ * Set USB bus host controller private data
+ *
+ * @v bus		USB bus
+ * @v priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_bus_set_hostdata ( struct usb_bus *bus, void *priv ) {
+	bus->priv = priv;
+}
+
+/**
+ * Get USB bus host controller private data
+ *
+ * @v bus		USB bus
+ * @ret priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_bus_get_hostdata ( struct usb_bus *bus ) {
+	return bus->priv;
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static inline __attribute__ (( always_inline )) void
+usb_poll ( struct usb_bus *bus ) {
+	bus->host->poll ( bus );
+}
+
+/** Iterate over all USB buses */
+#define for_each_usb_bus( bus ) \
+	list_for_each_entry ( (bus), &usb_buses, list )
+
+/**
+ * Complete transfer (without error)
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+usb_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf ) {
+	usb_complete_err ( ep, iobuf, 0 );
+}
+
+extern int usb_control ( struct usb_device *usb, unsigned int request,
+			 unsigned int value, unsigned int index, void *data,
+			 size_t len );
+extern int usb_get_string_descriptor ( struct usb_device *usb,
+				       unsigned int index,
+				       unsigned int language,
+				       char *buf, size_t len );
+
+/**
+ * Get status
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v index		Target index
+ * @v data		Status to fill in
+ * @v len		Length of status descriptor
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_status ( struct usb_device *usb, unsigned int type, unsigned int index,
+		 void *data, size_t len ) {
+
+	return usb_control ( usb, ( USB_GET_STATUS | type ), 0, index,
+			     data, len );
+}
+
+/**
+ * Clear feature
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v feature		Feature selector
+ * @v index		Target index
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_clear_feature ( struct usb_device *usb, unsigned int type,
+		    unsigned int feature, unsigned int index ) {
+
+	return usb_control ( usb, ( USB_CLEAR_FEATURE | type ),
+			     feature, index, NULL, 0 );
+}
+
+/**
+ * Set feature
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v feature		Feature selector
+ * @v index		Target index
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_feature ( struct usb_device *usb, unsigned int type,
+		  unsigned int feature, unsigned int index ) {
+
+	return usb_control ( usb, ( USB_SET_FEATURE | type ),
+			     feature, index, NULL, 0 );
+}
+
+/**
+ * Set address
+ *
+ * @v usb		USB device
+ * @v address		Device address
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_address ( struct usb_device *usb, unsigned int address ) {
+
+	return usb_control ( usb, USB_SET_ADDRESS, address, 0, NULL, 0 );
+}
+
+/**
+ * Get USB descriptor
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v desc		Descriptor type
+ * @v index		Descriptor index
+ * @v language		Language ID (for string descriptors)
+ * @v data		Descriptor to fill in
+ * @v len		Maximum length of descriptor
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_descriptor ( struct usb_device *usb, unsigned int type,
+		     unsigned int desc, unsigned int index,
+		     unsigned int language, struct usb_descriptor_header *data,
+		     size_t len ) {
+
+	return usb_control ( usb, ( USB_GET_DESCRIPTOR | type ),
+			     ( ( desc << 8 ) | index ), language, data, len );
+}
+
+/**
+ * Get first part of USB device descriptor (up to and including MTU)
+ *
+ * @v usb		USB device
+ * @v data		Device descriptor to (partially) fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_mtu ( struct usb_device *usb, struct usb_device_descriptor *data ) {
+
+	return usb_get_descriptor ( usb, 0, USB_DEVICE_DESCRIPTOR, 0, 0,
+				    &data->header,
+				    ( offsetof ( typeof ( *data ), mtu ) +
+				      sizeof ( data->mtu ) ) );
+}
+
+/**
+ * Get USB device descriptor
+ *
+ * @v usb		USB device
+ * @v data		Device descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_device_descriptor ( struct usb_device *usb,
+			    struct usb_device_descriptor *data ) {
+
+	return usb_get_descriptor ( usb, 0, USB_DEVICE_DESCRIPTOR, 0, 0,
+				    &data->header, sizeof ( *data ) );
+}
+
+/**
+ * Get USB configuration descriptor
+ *
+ * @v usb		USB device
+ * @v index		Configuration index
+ * @v data		Configuration descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute (( always_inline )) int
+usb_get_config_descriptor ( struct usb_device *usb, unsigned int index,
+			    struct usb_configuration_descriptor *data,
+			    size_t len ) {
+
+	return usb_get_descriptor ( usb, 0, USB_CONFIGURATION_DESCRIPTOR, index,
+				    0, &data->header, len );
+}
+
+/**
+ * Set USB configuration
+ *
+ * @v usb		USB device
+ * @v index		Configuration index
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_configuration ( struct usb_device *usb, unsigned int index ) {
+
+	return usb_control ( usb, USB_SET_CONFIGURATION, index, 0, NULL, 0 );
+}
+
+/**
+ * Set USB interface alternate setting
+ *
+ * @v usb		USB device
+ * @v interface		Interface number
+ * @v alternate		Alternate setting
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_interface ( struct usb_device *usb, unsigned int interface,
+		    unsigned int alternate ) {
+
+	return usb_control ( usb, USB_SET_INTERFACE, alternate, interface,
+			     NULL, 0 );
+}
+
+extern struct list_head usb_buses;
+
+extern struct usb_interface_descriptor *
+usb_interface_descriptor ( struct usb_configuration_descriptor *config,
+			   unsigned int interface, unsigned int alternate );
+extern struct usb_endpoint_descriptor *
+usb_endpoint_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface,
+			  unsigned int type, unsigned int index );
+extern struct usb_endpoint_companion_descriptor *
+usb_endpoint_companion_descriptor ( struct usb_configuration_descriptor *config,
+				    struct usb_endpoint_descriptor *desc );
+
+extern struct usb_hub * alloc_usb_hub ( struct usb_bus *bus,
+					struct usb_device *usb,
+					unsigned int ports,
+					struct usb_hub_driver_operations *op );
+extern int register_usb_hub ( struct usb_hub *hub );
+extern void unregister_usb_hub ( struct usb_hub *hub );
+extern void free_usb_hub ( struct usb_hub *hub );
+
+extern void usb_port_changed ( struct usb_port *port );
+
+extern struct usb_bus * alloc_usb_bus ( struct device *dev,
+					unsigned int ports, size_t mtu,
+					struct usb_host_operations *op );
+extern int register_usb_bus ( struct usb_bus *bus );
+extern void unregister_usb_bus ( struct usb_bus *bus );
+extern void free_usb_bus ( struct usb_bus *bus );
+extern struct usb_bus * find_usb_bus_by_location ( unsigned int bus_type,
+						   unsigned int location );
+
+extern int usb_alloc_address ( struct usb_bus *bus );
+extern void usb_free_address ( struct usb_bus *bus, unsigned int address );
+extern unsigned int usb_route_string ( struct usb_device *usb );
+extern unsigned int usb_depth ( struct usb_device *usb );
+extern struct usb_port * usb_root_hub_port ( struct usb_device *usb );
+extern struct usb_port * usb_transaction_translator ( struct usb_device *usb );
+
+/** Minimum reset time
+ *
+ * Section 7.1.7.5 of the USB2 specification states that root hub
+ * ports should assert reset signalling for at least 50ms.
+ */
+#define USB_RESET_DELAY_MS 50
+
+/** Reset recovery time
+ *
+ * Section 9.2.6.2 of the USB2 specification states that the
+ * "recovery" interval after a port reset is 10ms.
+ */
+#define USB_RESET_RECOVER_DELAY_MS 10
+
+/** Maximum time to wait for a control transaction to complete
+ *
+ * Section 9.2.6.1 of the USB2 specification states that the upper
+ * limit for commands to be processed is 5 seconds.
+ */
+#define USB_CONTROL_MAX_WAIT_MS 5000
+
+/** Set address recovery time
+ *
+ * Section 9.2.6.3 of the USB2 specification states that devices are
+ * allowed a 2ms recovery interval after receiving a new address.
+ */
+#define USB_SET_ADDRESS_RECOVER_DELAY_MS 2
+
+/** Time to wait for ports to stabilise
+ *
+ * Section 7.1.7.3 of the USB specification states that we must allow
+ * 100ms for devices to signal attachment, and an additional 100ms for
+ * connection debouncing.  (This delay is parallelised across all
+ * ports on a hub; we do not delay separately for each port.)
+ */
+#define USB_PORT_DELAY_MS 200
+
+/** A USB device ID */
+struct usb_device_id {
+	/** Name */
+	const char *name;
+	/** Vendor ID */
+	uint16_t vendor;
+	/** Product ID */
+	uint16_t product;
+	/** Class */
+	struct usb_class class;
+};
+
+/** Match-anything ID */
+#define USB_ANY_ID 0xffff
+
+/** A USB driver */
+struct usb_driver {
+	/** USB ID table */
+	struct usb_device_id *ids;
+	/** Number of entries in ID table */
+	unsigned int id_count;
+	/**
+	 * Probe device
+	 *
+	 * @v func		USB function
+	 * @v config		Configuration descriptor
+	 * @ret rc		Return status code
+	 */
+	int ( * probe ) ( struct usb_function *func,
+			  struct usb_configuration_descriptor *config );
+	/**
+	 * Remove device
+	 *
+	 * @v func		USB function
+	 */
+	void ( * remove ) ( struct usb_function *func );
+};
+
+/** USB driver table */
+#define USB_DRIVERS __table ( struct usb_driver, "usb_drivers" )
+
+/** Declare a USB driver */
+#define __usb_driver __table_entry ( USB_DRIVERS, 01 )
+
+#endif /* _IPXE_USB_H */
diff --git a/roms/ipxe/src/include/ipxe/usbhid.h b/roms/ipxe/src/include/ipxe/usbhid.h
new file mode 100644
index 000000000..fe9d84455
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/usbhid.h
@@ -0,0 +1,106 @@
+#ifndef _IPXE_USBHID_H
+#define _IPXE_USBHID_H
+
+/** @file
+ *
+ * USB human interface devices (HID)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+
+/** Class code for human interface devices */
+#define USB_CLASS_HID 3
+
+/** Subclass code for boot devices */
+#define USB_SUBCLASS_HID_BOOT 1
+
+/** Set protocol */
+#define USBHID_SET_PROTOCOL						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x0b ) )
+
+/** Boot protocol */
+#define USBHID_PROTOCOL_BOOT 0
+
+/** Report protocol */
+#define USBHID_PROTOCOL_REPORT 1
+
+/** Set idle time */
+#define USBHID_SET_IDLE							\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x0a ) )
+
+/** A USB human interface device */
+struct usb_hid {
+	/** USB function */
+	struct usb_function *func;
+	/** Interrupt IN endpoint */
+	struct usb_endpoint in;
+	/** Interrupt OUT endpoint (optional) */
+	struct usb_endpoint out;
+};
+
+/**
+ * Initialise USB human interface device
+ *
+ * @v hid		USB human interface device
+ * @v func		USB function
+ * @v in		Interrupt IN endpoint operations
+ * @v out		Interrupt OUT endpoint operations (or NULL)
+ */
+static inline __attribute__ (( always_inline )) void
+usbhid_init ( struct usb_hid *hid, struct usb_function *func,
+	      struct usb_endpoint_driver_operations *in,
+	      struct usb_endpoint_driver_operations *out ) {
+	struct usb_device *usb = func->usb;
+
+	hid->func = func;
+	usb_endpoint_init ( &hid->in, usb, in );
+	if ( out )
+		usb_endpoint_init ( &hid->out, usb, out );
+}
+
+/**
+ * Set protocol
+ *
+ * @v usb		USB device
+ * @v interface		Interface number
+ * @v protocol		HID protocol
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usbhid_set_protocol ( struct usb_device *usb, unsigned int interface,
+		      unsigned int protocol ) {
+
+	return usb_control ( usb, USBHID_SET_PROTOCOL, protocol, interface,
+			     NULL, 0 );
+}
+
+/**
+ * Set idle time
+ *
+ * @v usb		USB device
+ * @v interface		Interface number
+ * @v report		Report ID
+ * @v duration		Duration (in 4ms units)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usbhid_set_idle ( struct usb_device *usb, unsigned int interface,
+		  unsigned int report, unsigned int duration ) {
+
+	return usb_control ( usb, USBHID_SET_IDLE,
+			     ( ( duration << 8 ) | report ),
+			     interface, NULL, 0 );
+}
+
+extern int usbhid_open ( struct usb_hid *hid );
+extern void usbhid_close ( struct usb_hid *hid );
+extern int usbhid_refill ( struct usb_hid *hid );
+extern int usbhid_describe ( struct usb_hid *hid,
+			     struct usb_configuration_descriptor *config );
+
+#endif /* _IPXE_USBHID_H */
diff --git a/roms/ipxe/src/include/ipxe/usbnet.h b/roms/ipxe/src/include/ipxe/usbnet.h
new file mode 100644
index 000000000..33a8f3f58
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/usbnet.h
@@ -0,0 +1,62 @@
+#ifndef _IPXE_USBNET_H
+#define _IPXE_USBNET_H
+
+/** @file
+ *
+ * USB network devices
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+
+/** A USB network device */
+struct usbnet_device {
+	/** USB function */
+	struct usb_function *func;
+
+	/** Communications interface */
+	unsigned int comms;
+	/** Data interface */
+	unsigned int data;
+	/** Alternate setting for data interface */
+	unsigned int alternate;
+
+	/** Interrupt endpoint */
+	struct usb_endpoint intr;
+	/** Bulk IN endpoint */
+	struct usb_endpoint in;
+	/** Bulk OUT endpoint */
+	struct usb_endpoint out;
+};
+
+/**
+ * Initialise USB network device
+ *
+ * @v usbnet		USB network device
+ * @v func		USB function
+ * @v intr		Interrupt endpoint operations
+ * @v in		Bulk IN endpoint operations
+ * @v out		Bulk OUT endpoint operations
+ */
+static inline __attribute__ (( always_inline )) void
+usbnet_init ( struct usbnet_device *usbnet, struct usb_function *func,
+	      struct usb_endpoint_driver_operations *intr,
+	      struct usb_endpoint_driver_operations *in,
+	      struct usb_endpoint_driver_operations *out ) {
+	struct usb_device *usb = func->usb;
+
+	usbnet->func = func;
+	usb_endpoint_init ( &usbnet->intr, usb, intr );
+	usb_endpoint_init ( &usbnet->in, usb, in );
+	usb_endpoint_init ( &usbnet->out, usb, out );
+}
+
+extern int usbnet_open ( struct usbnet_device *usbnet );
+extern void usbnet_close ( struct usbnet_device *usbnet );
+extern int usbnet_refill ( struct usbnet_device *usbnet );
+extern int usbnet_describe ( struct usbnet_device *usbnet,
+			     struct usb_configuration_descriptor *config );
+
+#endif /* _IPXE_USBNET_H */
diff --git a/roms/ipxe/src/include/ipxe/uuid.h b/roms/ipxe/src/include/ipxe/uuid.h
index ad515d0cb..6c45eb9aa 100644
--- a/roms/ipxe/src/include/ipxe/uuid.h
+++ b/roms/ipxe/src/include/ipxe/uuid.h
@@ -6,7 +6,7 @@
  * Universally unique IDs
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/roms/ipxe/src/include/ipxe/validator.h b/roms/ipxe/src/include/ipxe/validator.h
index 23bdab423..0aee56eb0 100644
--- a/roms/ipxe/src/include/ipxe/validator.h
+++ b/roms/ipxe/src/include/ipxe/validator.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/interface.h>
 #include <ipxe/x509.h>
diff --git a/roms/ipxe/src/include/ipxe/version.h b/roms/ipxe/src/include/ipxe/version.h
index ae4275db1..a43a33425 100644
--- a/roms/ipxe/src/include/ipxe/version.h
+++ b/roms/ipxe/src/include/ipxe/version.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <wchar.h>
 
diff --git a/roms/ipxe/src/include/ipxe/vlan.h b/roms/ipxe/src/include/ipxe/vlan.h
index 083c21916..439e0c16d 100644
--- a/roms/ipxe/src/include/ipxe/vlan.h
+++ b/roms/ipxe/src/include/ipxe/vlan.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A VLAN header */
 struct vlan_header {
diff --git a/roms/ipxe/src/include/ipxe/vmbus.h b/roms/ipxe/src/include/ipxe/vmbus.h
new file mode 100644
index 000000000..26fc578c6
--- /dev/null
+++ b/roms/ipxe/src/include/ipxe/vmbus.h
@@ -0,0 +1,634 @@
+#ifndef _IPXE_VMBUS_H
+#define _IPXE_VMBUS_H
+
+/** @file
+ *
+ * Hyper-V virtual machine bus
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/uuid.h>
+#include <ipxe/device.h>
+#include <ipxe/tables.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/hyperv.h>
+
+/** VMBus message connection ID */
+#define VMBUS_MESSAGE_ID 1
+
+/** VMBus event connection ID */
+#define VMBUS_EVENT_ID 2
+
+/** VMBus message type */
+#define VMBUS_MESSAGE_TYPE 1
+
+/** VMBus message synthetic interrupt */
+#define VMBUS_MESSAGE_SINT 2
+
+/** VMBus version number */
+union vmbus_version {
+	/** Raw version */
+	uint32_t raw;
+	/** Major/minor version */
+	struct {
+		/** Minor version */
+		uint16_t minor;
+		/** Major version */
+		uint16_t major;
+	};
+} __attribute__ (( packed ));
+
+/** Known VMBus protocol versions */
+enum vmbus_raw_version {
+	/** Windows Server 2008 */
+	VMBUS_VERSION_WS2008 = ( ( 0 << 16 ) | ( 13 << 0 ) ),
+	/** Windows 7 */
+	VMBUS_VERSION_WIN7 = ( ( 1 << 16 ) | ( 1 << 0 ) ),
+	/** Windows 8 */
+	VMBUS_VERSION_WIN8 = ( ( 2 << 16 ) | ( 4 << 0 ) ),
+	/** Windows 8.1 */
+	VMBUS_VERSION_WIN8_1 = ( ( 3 << 16 ) | ( 0 << 0 ) ),
+};
+
+/** Guest physical address range descriptor */
+struct vmbus_gpa_range {
+	/** Byte count */
+	uint32_t len;
+	/** Starting byte offset */
+	uint32_t offset;
+	/** Page frame numbers
+	 *
+	 * The length of this array is implied by the byte count and
+	 * starting offset.
+	 */
+	uint64_t pfn[0];
+} __attribute__ (( packed ));
+
+/** VMBus message header */
+struct vmbus_message_header {
+	/** Message type */
+	uint32_t type;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** VMBus message types */
+enum vmbus_message_type {
+	VMBUS_OFFER_CHANNEL = 1,
+	VMBUS_REQUEST_OFFERS = 3,
+	VMBUS_ALL_OFFERS_DELIVERED = 4,
+	VMBUS_OPEN_CHANNEL = 5,
+	VMBUS_OPEN_CHANNEL_RESULT = 6,
+	VMBUS_CLOSE_CHANNEL = 7,
+	VMBUS_GPADL_HEADER = 8,
+	VMBUS_GPADL_CREATED = 10,
+	VMBUS_GPADL_TEARDOWN = 11,
+	VMBUS_GPADL_TORNDOWN = 12,
+	VMBUS_INITIATE_CONTACT = 14,
+	VMBUS_VERSION_RESPONSE = 15,
+	VMBUS_UNLOAD = 16,
+	VMBUS_UNLOAD_RESPONSE = 17,
+};
+
+/** VMBus "offer channel" message */
+struct vmbus_offer_channel {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel type */
+	union uuid type;
+	/** Channel instance */
+	union uuid instance;
+	/** Reserved */
+	uint8_t reserved_a[16];
+	/** Flags */
+	uint16_t flags;
+	/** Reserved */
+	uint8_t reserved_b[2];
+	/** User data */
+	uint8_t data[120];
+	/** Reserved */
+	uint8_t reserved_c[4];
+	/** Channel ID */
+	uint32_t channel;
+	/** Monitor ID */
+	uint8_t monitor;
+	/** Monitor exists */
+	uint8_t monitored;
+	/** Reserved */
+	uint8_t reserved[2];
+	/** Connection ID */
+	uint32_t connection;
+} __attribute__ (( packed ));
+
+/** VMBus "open channel" message */
+struct vmbus_open_channel {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** Open ID */
+	uint32_t id;
+	/** Ring buffer GPADL ID */
+	uint32_t gpadl;
+	/** Reserved */
+	uint32_t reserved;
+	/** Outbound ring buffer size (in pages) */
+	uint32_t out_pages;
+	/** User-specific data */
+	uint8_t data[120];
+} __attribute__ (( packed ));
+
+/** VMBus "open channel result" message */
+struct vmbus_open_channel_result {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** Open ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** VMBus "close channel" message */
+struct vmbus_close_channel {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL header" message */
+struct vmbus_gpadl_header {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** GPADL ID */
+	uint32_t gpadl;
+	/** Length of range descriptors */
+	uint16_t range_len;
+	/** Number of range descriptors */
+	uint16_t range_count;
+	/** Range descriptors */
+	struct vmbus_gpa_range range[0];
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL created" message */
+struct vmbus_gpadl_created {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** GPADL ID */
+	uint32_t gpadl;
+	/** Creation status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL teardown" message */
+struct vmbus_gpadl_teardown {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** GPADL ID */
+	uint32_t gpadl;
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL torndown" message */
+struct vmbus_gpadl_torndown {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** GPADL ID */
+	uint32_t gpadl;
+} __attribute__ (( packed ));
+
+/** VMBus "initiate contact" message */
+struct vmbus_initiate_contact {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Requested version */
+	union vmbus_version version;
+	/** Target virtual CPU */
+	uint32_t vcpu;
+	/** Interrupt page base address */
+	uint64_t intr;
+	/** Parent to child monitor page base address */
+	uint64_t monitor_in;
+	/** Child to parent monitor page base address */
+	uint64_t monitor_out;
+} __attribute__ (( packed ));
+
+/** VMBus "version response" message */
+struct vmbus_version_response {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Version is supported */
+	uint8_t supported;
+	/** Reserved */
+	uint8_t reserved[3];
+	/** Version */
+	union vmbus_version version;
+} __attribute__ (( packed ));
+
+/** VMBus message */
+union vmbus_message {
+	/** Common message header */
+	struct vmbus_message_header header;
+	/** "Offer channel" message */
+	struct vmbus_offer_channel offer;
+	/** "Open channel" message */
+	struct vmbus_open_channel open;
+	/** "Open channel result" message */
+	struct vmbus_open_channel_result opened;
+	/** "Close channel" message */
+	struct vmbus_close_channel close;
+	/** "GPADL header" message */
+	struct vmbus_gpadl_header gpadlhdr;
+	/** "GPADL created" message */
+	struct vmbus_gpadl_created created;
+	/** "GPADL teardown" message */
+	struct vmbus_gpadl_teardown teardown;
+	/** "GPADL torndown" message */
+	struct vmbus_gpadl_torndown torndown;
+	/** "Initiate contact" message */
+	struct vmbus_initiate_contact initiate;
+	/** "Version response" message */
+	struct vmbus_version_response version;
+};
+
+/** VMBus packet header */
+struct vmbus_packet_header {
+	/** Type */
+	uint16_t type;
+	/** Length of packet header (in quadwords) */
+	uint16_t hdr_qlen;
+	/** Length of packet (in quadwords) */
+	uint16_t qlen;
+	/** Flags */
+	uint16_t flags;
+	/** Transaction ID
+	 *
+	 * This is an opaque token: we therefore treat it as
+	 * native-endian and don't worry about byte-swapping.
+	 */
+	uint64_t xid;
+} __attribute__ (( packed ));
+
+/** VMBus packet types */
+enum vmbus_packet_type {
+	VMBUS_DATA_INBAND = 6,
+	VMBUS_DATA_XFER_PAGES = 7,
+	VMBUS_DATA_GPA_DIRECT = 9,
+	VMBUS_CANCELLATION = 10,
+	VMBUS_COMPLETION = 11,
+};
+
+/** VMBus packet flags */
+enum vmbus_packet_flags {
+	VMBUS_COMPLETION_REQUESTED = 0x0001,
+};
+
+/** VMBus GPA direct header */
+struct vmbus_gpa_direct_header {
+	/** Packet header */
+	struct vmbus_packet_header header;
+	/** Reserved */
+	uint32_t reserved;
+	/** Number of range descriptors */
+	uint32_t range_count;
+	/** Range descriptors */
+	struct vmbus_gpa_range range[0];
+} __attribute__ (( packed ));
+
+/** VMBus transfer page range */
+struct vmbus_xfer_page_range {
+	/** Length */
+	uint32_t len;
+	/** Offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** VMBus transfer page header */
+struct vmbus_xfer_page_header {
+	/** Packet header */
+	struct vmbus_packet_header header;
+	/** Page set ID */
+	uint16_t pageset;
+	/** Sender owns page set */
+	uint8_t owner;
+	/** Reserved */
+	uint8_t reserved;
+	/** Number of range descriptors */
+	uint32_t range_count;
+	/** Range descriptors */
+	struct vmbus_xfer_page_range range[0];
+} __attribute__ (( packed ));
+
+/** Maximum expected size of VMBus packet header */
+#define VMBUS_PACKET_MAX_HEADER_LEN 64
+
+/** VMBus maximum-sized packet header */
+union vmbus_packet_header_max {
+	/** Common header */
+	struct vmbus_packet_header header;
+	/** GPA direct header */
+	struct vmbus_gpa_direct_header gpa;
+	/** Transfer page header */
+	struct vmbus_xfer_page_header xfer;
+	/** Padding to maximum supported size */
+	uint8_t padding[VMBUS_PACKET_MAX_HEADER_LEN];
+} __attribute__ (( packed ));
+
+/** VMBus packet footer */
+struct vmbus_packet_footer {
+	/** Reserved */
+	uint32_t reserved;
+	/** Producer index of the first byte of the packet */
+	uint32_t prod;
+} __attribute__ (( packed ));
+
+/** VMBus ring buffer
+ *
+ * This is the structure of the each of the ring buffers created when
+ * a VMBus channel is opened.
+ */
+struct vmbus_ring {
+	/** Producer index (modulo ring length) */
+	uint32_t prod;
+	/** Consumer index (modulo ring length) */
+	uint32_t cons;
+	/** Interrupt mask */
+	uint32_t intr_mask;
+	/** Reserved */
+	uint8_t reserved[4084];
+	/** Ring buffer contents */
+	uint8_t data[0];
+} __attribute__ (( packed ));
+
+/** VMBus interrupt page */
+struct vmbus_interrupt {
+	/** Inbound interrupts */
+	uint8_t in[ PAGE_SIZE / 2 ];
+	/** Outbound interrupts */
+	uint8_t out[ PAGE_SIZE / 2 ];
+} __attribute__ (( packed ));
+
+/** A virtual machine bus */
+struct vmbus {
+	/** Interrupt page */
+	struct vmbus_interrupt *intr;
+	/** Inbound notifications */
+	struct hv_monitor *monitor_in;
+	/** Outbound notifications */
+	struct hv_monitor *monitor_out;
+	/** Received message buffer */
+	const union vmbus_message *message;
+};
+
+struct vmbus_device;
+
+/** VMBus channel operations */
+struct vmbus_channel_operations {
+	/**
+	 * Handle received control packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @v data		Data
+	 * @v len		Length of data
+	 * @ret rc		Return status code
+	 */
+	int ( * recv_control ) ( struct vmbus_device *vmdev, uint64_t xid,
+				 const void *data, size_t len );
+	/**
+	 * Handle received data packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @v data		Data
+	 * @v len		Length of data
+	 * @v list		List of I/O buffers
+	 * @ret rc		Return status code
+	 *
+	 * This function takes ownership of the I/O buffer.  It should
+	 * eventually call vmbus_send_completion() to indicate to the
+	 * host that the buffer can be reused.
+	 */
+	int ( * recv_data ) ( struct vmbus_device *vmdev, uint64_t xid,
+			      const void *data, size_t len,
+			      struct list_head *list );
+	/**
+	 * Handle received completion packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @v data		Data
+	 * @v len		Length of data
+	 * @ret rc		Return status code
+	 */
+	int ( * recv_completion ) ( struct vmbus_device *vmdev, uint64_t xid,
+				    const void *data, size_t len );
+	/**
+	 * Handle received cancellation packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @ret rc		Return status code
+	 */
+	int ( * recv_cancellation ) ( struct vmbus_device *vmdev,
+				      uint64_t xid );
+};
+
+struct vmbus_xfer_pages;
+
+/** VMBus transfer page set operations */
+struct vmbus_xfer_pages_operations {
+	/**
+	 * Copy data from transfer page
+	 *
+	 * @v pages		Transfer page set
+	 * @v data		Data buffer
+	 * @v offset		Offset within page set
+	 * @v len		Length within page set
+	 * @ret rc		Return status code
+	 */
+	int ( * copy ) ( struct vmbus_xfer_pages *pages, void *data,
+			 size_t offset, size_t len );
+};
+
+/** VMBus transfer page set */
+struct vmbus_xfer_pages {
+	/** List of all transfer page sets */
+	struct list_head list;
+	/** Page set ID (in protocol byte order) */
+	uint16_t pageset;
+	/** Page set operations */
+	struct vmbus_xfer_pages_operations *op;
+};
+
+/** A VMBus device */
+struct vmbus_device {
+	/** Generic iPXE device */
+	struct device dev;
+	/** Hyper-V hypervisor */
+	struct hv_hypervisor *hv;
+
+	/** Channel ID */
+	unsigned int channel;
+	/** Monitor ID */
+	unsigned int monitor;
+	/** Signal channel
+	 *
+	 * @v vmdev		VMBus device
+	 */
+	void ( * signal ) ( struct vmbus_device *vmdev );
+
+	/** Outbound ring buffer length */
+	uint32_t out_len;
+	/** Inbound ring buffer length */
+	uint32_t in_len;
+	/** Outbound ring buffer */
+	struct vmbus_ring *out;
+	/** Inbound ring buffer */
+	struct vmbus_ring *in;
+	/** Ring buffer GPADL ID */
+	unsigned int gpadl;
+
+	/** Channel operations */
+	struct vmbus_channel_operations *op;
+	/** Maximum expected data packet length */
+	size_t mtu;
+	/** Packet buffer */
+	void *packet;
+	/** List of transfer page sets */
+	struct list_head pages;
+
+	/** Driver */
+	struct vmbus_driver *driver;
+	/** Driver-private data */
+	void *priv;
+};
+
+/** A VMBus device driver */
+struct vmbus_driver {
+	/** Name */
+	const char *name;
+	/** Device type */
+	union uuid type;
+	/** Probe device
+	 *
+	 * @v vmdev		VMBus device
+	 * @ret rc		Return status code
+	 */
+	int ( * probe ) ( struct vmbus_device *vmdev );
+	/** Remove device
+	 *
+	 * @v vmdev		VMBus device
+	 */
+	void ( * remove ) ( struct vmbus_device *vmdev );
+};
+
+/** VMBus device driver table */
+#define VMBUS_DRIVERS __table ( struct vmbus_driver, "vmbus_drivers" )
+
+/** Declare a VMBus device driver */
+#define __vmbus_driver __table_entry ( VMBUS_DRIVERS, 01 )
+
+/**
+ * Set VMBus device driver-private data
+ *
+ * @v vmdev		VMBus device
+ * @v priv		Private data
+ */
+static inline void vmbus_set_drvdata ( struct vmbus_device *vmdev, void *priv ){
+	vmdev->priv = priv;
+}
+
+/**
+ * Get VMBus device driver-private data
+ *
+ * @v vmdev		VMBus device
+ * @ret priv		Private data
+ */
+static inline void * vmbus_get_drvdata ( struct vmbus_device *vmdev ) {
+	return vmdev->priv;
+}
+
+/** Construct VMBus type */
+#define VMBUS_TYPE( a, b, c, d, e0, e1, e2, e3, e4, e5 ) {		\
+	.canonical = {							\
+		cpu_to_le32 ( a ), cpu_to_le16 ( b ),			\
+		cpu_to_le16 ( c ), cpu_to_be16 ( d ),			\
+		{ e0, e1, e2, e3, e4, e5 }				\
+	 } }
+
+/**
+ * Check if data is present in ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v has_data		Data is present
+ */
+static inline __attribute__ (( always_inline )) int
+vmbus_has_data ( struct vmbus_device *vmdev ) {
+
+	return ( vmdev->in->prod != vmdev->in->cons );
+}
+
+/**
+ * Register transfer page set
+ *
+ * @v vmdev		VMBus device
+ * @v pages		Transfer page set
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+vmbus_register_pages ( struct vmbus_device *vmdev,
+		       struct vmbus_xfer_pages *pages ) {
+
+	list_add ( &pages->list, &vmdev->pages );
+	return 0;
+}
+
+/**
+ * Unregister transfer page set
+ *
+ * @v vmdev		VMBus device
+ * @v pages		Transfer page set
+ */
+static inline __attribute__ (( always_inline )) void
+vmbus_unregister_pages ( struct vmbus_device *vmdev,
+			 struct vmbus_xfer_pages *pages ) {
+
+	list_check_contains_entry ( pages, &vmdev->pages, list );
+	list_del ( &pages->list );
+}
+
+extern int vmbus_establish_gpadl ( struct vmbus_device *vmdev, userptr_t data,
+				   size_t len );
+extern int vmbus_gpadl_teardown ( struct vmbus_device *vmdev,
+				  unsigned int gpadl );
+extern int vmbus_open ( struct vmbus_device *vmdev,
+			struct vmbus_channel_operations *op,
+			size_t out_len, size_t in_len, size_t mtu );
+extern void vmbus_close ( struct vmbus_device *vmdev );
+extern int vmbus_send_control ( struct vmbus_device *vmdev, uint64_t xid,
+				const void *data, size_t len );
+extern int vmbus_send_data ( struct vmbus_device *vmdev, uint64_t xid,
+			     const void *data, size_t len,
+			     struct io_buffer *iobuf );
+extern int vmbus_send_completion ( struct vmbus_device *vmdev, uint64_t xid,
+				   const void *data, size_t len );
+extern int vmbus_send_cancellation ( struct vmbus_device *vmdev, uint64_t xid );
+extern int vmbus_poll ( struct vmbus_device *vmdev );
+extern void vmbus_dump_channel ( struct vmbus_device *vmdev );
+
+extern int vmbus_probe ( struct hv_hypervisor *hv, struct device *parent );
+extern void vmbus_remove ( struct hv_hypervisor *hv, struct device *parent );
+
+#endif /* _IPXE_VMBUS_H */
diff --git a/roms/ipxe/src/include/ipxe/vsprintf.h b/roms/ipxe/src/include/ipxe/vsprintf.h
index c48c97a87..9e6297715 100644
--- a/roms/ipxe/src/include/ipxe/vsprintf.h
+++ b/roms/ipxe/src/include/ipxe/vsprintf.h
@@ -31,7 +31,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
diff --git a/roms/ipxe/src/include/ipxe/x509.h b/roms/ipxe/src/include/ipxe/x509.h
index 802480e54..0daaf5e59 100644
--- a/roms/ipxe/src/include/ipxe/x509.h
+++ b/roms/ipxe/src/include/ipxe/x509.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/roms/ipxe/src/include/ipxe/xen.h b/roms/ipxe/src/include/ipxe/xen.h
index 60aabe03e..eac1145ad 100644
--- a/roms/ipxe/src/include/ipxe/xen.h
+++ b/roms/ipxe/src/include/ipxe/xen.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Define Xen interface version before including any Xen header files */
 #define __XEN_INTERFACE_VERSION__ 0x00040400
diff --git a/roms/ipxe/src/include/ipxe/xenbus.h b/roms/ipxe/src/include/ipxe/xenbus.h
index ef2b5496f..ec5782eed 100644
--- a/roms/ipxe/src/include/ipxe/xenbus.h
+++ b/roms/ipxe/src/include/ipxe/xenbus.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 #include <ipxe/tables.h>
diff --git a/roms/ipxe/src/include/ipxe/xenevent.h b/roms/ipxe/src/include/ipxe/xenevent.h
index 1dd6a0c0b..f0bd3465e 100644
--- a/roms/ipxe/src/include/ipxe/xenevent.h
+++ b/roms/ipxe/src/include/ipxe/xenevent.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/event_channel.h>
diff --git a/roms/ipxe/src/include/ipxe/xengrant.h b/roms/ipxe/src/include/ipxe/xengrant.h
index f9b3beb21..451a3ceee 100644
--- a/roms/ipxe/src/include/ipxe/xengrant.h
+++ b/roms/ipxe/src/include/ipxe/xengrant.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/include/ipxe/xenmem.h b/roms/ipxe/src/include/ipxe/xenmem.h
index 9b9aeda9c..dcc38d460 100644
--- a/roms/ipxe/src/include/ipxe/xenmem.h
+++ b/roms/ipxe/src/include/ipxe/xenmem.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/memory.h>
diff --git a/roms/ipxe/src/include/ipxe/xenstore.h b/roms/ipxe/src/include/ipxe/xenstore.h
index f25f15704..892640755 100644
--- a/roms/ipxe/src/include/ipxe/xenstore.h
+++ b/roms/ipxe/src/include/ipxe/xenstore.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 
diff --git a/roms/ipxe/src/include/ipxe/xenver.h b/roms/ipxe/src/include/ipxe/xenver.h
index 5d678c5a3..b29dfb321 100644
--- a/roms/ipxe/src/include/ipxe/xenver.h
+++ b/roms/ipxe/src/include/ipxe/xenver.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/version.h>
diff --git a/roms/ipxe/src/include/ipxe/xfer.h b/roms/ipxe/src/include/ipxe/xfer.h
index 1167e5cba..3a35fa924 100644
--- a/roms/ipxe/src/include/ipxe/xfer.h
+++ b/roms/ipxe/src/include/ipxe/xfer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
@@ -103,5 +103,7 @@ extern int xfer_vprintf ( struct interface *intf,
 extern int __attribute__ (( format ( printf, 2, 3 ) ))
 xfer_printf ( struct interface *intf, const char *format, ... );
 extern int xfer_seek ( struct interface *intf, off_t offset );
+extern int xfer_check_order ( struct xfer_metadata *meta, size_t *pos,
+			      size_t len );
 
 #endif /* _IPXE_XFER_H */
diff --git a/roms/ipxe/src/include/ipxe/xferbuf.h b/roms/ipxe/src/include/ipxe/xferbuf.h
index 2ca871e59..cb0b1a0e8 100644
--- a/roms/ipxe/src/include/ipxe/xferbuf.h
+++ b/roms/ipxe/src/include/ipxe/xferbuf.h
@@ -7,10 +7,12 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/iobuf.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/interface.h>
 #include <ipxe/xfer.h>
 
 /** A data transfer buffer */
@@ -21,11 +23,83 @@ struct xfer_buffer {
 	size_t len;
 	/** Current offset within data */
 	size_t pos;
+	/** Data transfer buffer operations */
+	struct xfer_buffer_operations *op;
 };
 
-extern void xferbuf_done ( struct xfer_buffer *xferbuf );
+/** Data transfer buffer operations */
+struct xfer_buffer_operations {
+	/** Reallocate data buffer
+	 *
+	 * @v xferbuf		Data transfer buffer
+	 * @v len		New length (or zero to free buffer)
+	 * @ret rc		Return status code
+	 */
+	int ( * realloc ) ( struct xfer_buffer *xferbuf, size_t len );
+	/** Write data to buffer
+	 *
+	 * @v xferbuf		Data transfer buffer
+	 * @v offset		Starting offset
+	 * @v data		Data to write
+	 * @v len		Length of data
+	 *
+	 * This call is simply a wrapper for the appropriate
+	 * memcpy()-like operation: the caller is responsible for
+	 * ensuring that the write does not exceed the buffer length.
+	 */
+	void ( * write ) ( struct xfer_buffer *xferbuf, size_t offset,
+			   const void *data, size_t len );
+	/** Read data from buffer
+	 *
+	 * @v xferbuf		Data transfer buffer
+	 * @v offset		Starting offset
+	 * @v data		Data to read
+	 * @v len		Length of data
+	 *
+	 * This call is simply a wrapper for the appropriate
+	 * memcpy()-like operation: the caller is responsible for
+	 * ensuring that the read does not exceed the buffer length.
+	 */
+	void ( * read ) ( struct xfer_buffer *xferbuf, size_t offset,
+			  void *data, size_t len );
+};
+
+extern struct xfer_buffer_operations xferbuf_malloc_operations;
+extern struct xfer_buffer_operations xferbuf_umalloc_operations;
+
+/**
+ * Initialise malloc()-based data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ */
+static inline __attribute__ (( always_inline )) void
+xferbuf_malloc_init ( struct xfer_buffer *xferbuf ) {
+	xferbuf->op = &xferbuf_malloc_operations;
+}
+
+/**
+ * Initialise umalloc()-based data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v data		User pointer
+ */
+static inline __attribute__ (( always_inline )) void
+xferbuf_umalloc_init ( struct xfer_buffer *xferbuf, userptr_t *data ) {
+	xferbuf->data = data;
+	xferbuf->op = &xferbuf_umalloc_operations;
+}
+
+extern void xferbuf_free ( struct xfer_buffer *xferbuf );
+extern int xferbuf_write ( struct xfer_buffer *xferbuf, size_t offset,
+			   const void *data, size_t len );
+extern int xferbuf_read ( struct xfer_buffer *xferbuf, size_t offset,
+			  void *data, size_t len );
 extern int xferbuf_deliver ( struct xfer_buffer *xferbuf,
 			     struct io_buffer *iobuf,
 			     struct xfer_metadata *meta );
 
+extern struct xfer_buffer * xfer_buffer ( struct interface *intf );
+#define xfer_buffer_TYPE( object_type ) \
+	typeof ( struct xfer_buffer * ( object_type ) )
+
 #endif /* _IPXE_XFERBUF_H */
diff --git a/roms/ipxe/src/include/libgen.h b/roms/ipxe/src/include/libgen.h
index 7e94881a9..ae0861270 100644
--- a/roms/ipxe/src/include/libgen.h
+++ b/roms/ipxe/src/include/libgen.h
@@ -1,7 +1,7 @@
 #ifndef _LIBGEN_H
 #define _LIBGEN_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern char * basename ( char *path );
 extern char * dirname ( char *path );
diff --git a/roms/ipxe/src/include/little_bswap.h b/roms/ipxe/src/include/little_bswap.h
deleted file mode 100644
index 92dd26ba1..000000000
--- a/roms/ipxe/src/include/little_bswap.h
+++ /dev/null
@@ -1,37 +0,0 @@
-#ifndef ETHERBOOT_LITTLE_BSWAP_H
-#define ETHERBOOT_LITTLE_BSWAP_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#define htonll(x)	__bswap_64(x)
-#define ntohll(x)	__bswap_64(x)
-#define ntohl(x)	__bswap_32(x)
-#define htonl(x) 	__bswap_32(x)
-#define ntohs(x) 	__bswap_16(x)
-#define htons(x) 	__bswap_16(x)
-#define cpu_to_le64(x)	(x)
-#define cpu_to_le32(x)	(x)
-#define cpu_to_le16(x)	(x)
-#define cpu_to_be64(x)	__bswap_64(x)
-#define cpu_to_be32(x)	__bswap_32(x)
-#define cpu_to_be16(x)	__bswap_16(x)
-#define le64_to_cpu(x)	(x)
-#define le32_to_cpu(x)	(x)
-#define le16_to_cpu(x)	(x)
-#define be64_to_cpu(x)	__bswap_64(x)
-#define be32_to_cpu(x)	__bswap_32(x)
-#define be16_to_cpu(x)	__bswap_16(x)
-#define cpu_to_le64s(x) do {} while (0)
-#define cpu_to_le32s(x) do {} while (0)
-#define cpu_to_le16s(x) do {} while (0)
-#define cpu_to_be64s(x) __bswap_64s(x)
-#define cpu_to_be32s(x) __bswap_32s(x)
-#define cpu_to_be16s(x) __bswap_16s(x)
-#define le64_to_cpus(x) do {} while (0)
-#define le32_to_cpus(x) do {} while (0)
-#define le16_to_cpus(x) do {} while (0)
-#define be64_to_cpus(x) __bswap_64s(x)
-#define be32_to_cpus(x) __bswap_32s(x)
-#define be16_to_cpus(x) __bswap_16s(x)
-
-#endif /* ETHERBOOT_LITTLE_BSWAP_H */
diff --git a/roms/ipxe/src/include/nic.h b/roms/ipxe/src/include/nic.h
index 9aaede8a7..4c91f57a6 100644
--- a/roms/ipxe/src/include/nic.h
+++ b/roms/ipxe/src/include/nic.h
@@ -1,8 +1,18 @@
- /*
+/*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -266,6 +276,7 @@ static inline void * legacy_isa_get_drvdata ( void *hwdev ) {
 	_name ## _isa_legacy_remove ( struct isa_device *isa ) {	  \
 		return legacy_remove ( isa, legacy_isa_get_drvdata,	  \
 				       _name ## _disable );		  \
-	}
+	}								  \
+	PROVIDE_REQUIRING_SYMBOL()
 
 #endif	/* NIC_H */
diff --git a/roms/ipxe/src/include/readline/readline.h b/roms/ipxe/src/include/readline/readline.h
index 0449a3f98..afafbbdf5 100644
--- a/roms/ipxe/src/include/readline/readline.h
+++ b/roms/ipxe/src/include/readline/readline.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A readline history entry */
 struct readline_history_entry {
diff --git a/roms/ipxe/src/include/stdarg.h b/roms/ipxe/src/include/stdarg.h
index f317238a9..89e94ce22 100644
--- a/roms/ipxe/src/include/stdarg.h
+++ b/roms/ipxe/src/include/stdarg.h
@@ -1,7 +1,7 @@
 #ifndef _STDARG_H
 #define _STDARG_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 typedef __builtin_va_list va_list;
 #define va_start( ap, last ) __builtin_va_start ( ap, last )
diff --git a/roms/ipxe/src/include/stddef.h b/roms/ipxe/src/include/stddef.h
index bf792771f..3c056294f 100644
--- a/roms/ipxe/src/include/stddef.h
+++ b/roms/ipxe/src/include/stddef.h
@@ -1,25 +1,43 @@
 #ifndef STDDEF_H
 #define STDDEF_H
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-/* for size_t */
 #include <stdint.h>
 
+/** EFI headers also define NULL */
 #undef NULL
-#define NULL ((void *)0)
 
-#undef offsetof
-#if ( defined ( __GNUC__ ) && ( __GNUC__ > 3 ) )
-#define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
+/** Null pointer */
+#define NULL ( ( void * ) 0 )
+
+/**
+ * Get offset of a field within a structure
+ *
+ * @v type		Structure type
+ * @v field		Field within structure
+ * @ret offset		Offset within structure
+ */
+#if defined ( __GNUC__ ) && ( __GNUC__ > 3 )
+#define offsetof( type, field ) __builtin_offsetof ( type, field )
 #else
-#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#define offsetof( type, field ) ( ( size_t ) &( ( ( type * ) NULL )->field ) )
 #endif
 
-#undef container_of
-#define container_of(ptr, type, member) ({                      \
-	const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
-	(type *)( (char *)__mptr - offsetof(type,member) );})
+/**
+ * Get containing structure
+ *
+ * @v ptr		Pointer to contained field
+ * @v type		Containing structure type
+ * @v field		Field within containing structure
+ * @ret container	Pointer to containing structure
+ */
+#define container_of( ptr, type, field ) ( {				\
+	type *__container;						\
+	const typeof ( __container->field ) *__field = (ptr);		\
+	__container = ( ( ( void * ) __field ) -			\
+			offsetof ( type, field ) );			\
+	__container; } )
 
 /* __WCHAR_TYPE__ is defined by gcc and will change if -fshort-wchar is used */
 #ifndef __WCHAR_TYPE__
diff --git a/roms/ipxe/src/include/stdint.h b/roms/ipxe/src/include/stdint.h
index 8cc9b84a5..0a239a517 100644
--- a/roms/ipxe/src/include/stdint.h
+++ b/roms/ipxe/src/include/stdint.h
@@ -1,7 +1,7 @@
 #ifndef _STDINT_H
 #define _STDINT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * This is a standard predefined macro on all gcc's I've seen. It's
diff --git a/roms/ipxe/src/include/stdio.h b/roms/ipxe/src/include/stdio.h
index 91840af5b..a618482ce 100644
--- a/roms/ipxe/src/include/stdio.h
+++ b/roms/ipxe/src/include/stdio.h
@@ -1,7 +1,7 @@
 #ifndef _STDIO_H
 #define _STDIO_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
diff --git a/roms/ipxe/src/include/stdlib.h b/roms/ipxe/src/include/stdlib.h
index 2951522b8..d7748a07e 100644
--- a/roms/ipxe/src/include/stdlib.h
+++ b/roms/ipxe/src/include/stdlib.h
@@ -1,7 +1,7 @@
 #ifndef STDLIB_H
 #define STDLIB_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <assert.h>
@@ -13,31 +13,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
  ****************************************************************************
  */
 
-static inline int strtoul_base ( const char **pp, int base )
-{
-	const char *p = *pp;
-
-	if ( base == 0 ) {
-		base = 10;
-		if ( *p == '0' ) {
-			p++;
-			base = 8;
-			if ( ( *p | 0x20 ) == 'x' ) {
-				p++;
-				base = 16;
-			}
-		}
-	}
-
-	*pp = p;
-
-	return base;
-}
-
-extern unsigned int strtoul_charval ( unsigned int charval );
-extern unsigned long strtoul ( const char *p, char **endp, int base );
-extern unsigned long long strtoull ( const char *p, char **endp, int base );
-
+extern unsigned long strtoul ( const char *string, char **endp, int base );
+extern unsigned long long strtoull ( const char *string, char **endp,
+				     int base );
 
 /*****************************************************************************
  *
diff --git a/roms/ipxe/src/include/string.h b/roms/ipxe/src/include/string.h
index 3482e1b22..0fab6c74b 100644
--- a/roms/ipxe/src/include/string.h
+++ b/roms/ipxe/src/include/string.h
@@ -1,52 +1,53 @@
-/*
- *  Copyright (C) 1991, 1992  Linus Torvalds
- *  Copyright (C) 2004 Tobias Lorenz
+#ifndef _STRING_H
+#define _STRING_H
+
+/** @file
  *
- *  string handling functions
- *  based on linux/include/linux/ctype.h
- *       and linux/include/linux/string.h
+ * String functions
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
-FILE_LICENCE ( GPL2_ONLY );
-
-#ifndef ETHERBOOT_STRING_H
-#define ETHERBOOT_STRING_H
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <bits/string.h>
 
-int __pure strnicmp(const char *s1, const char *s2, size_t len) __nonnull;
-char * strcpy(char * dest,const char *src) __nonnull;
-char * strncpy(char * dest,const char *src,size_t count) __nonnull;
-char * strcat(char * dest, const char * src) __nonnull;
-char * strncat(char *dest, const char *src, size_t count) __nonnull;
-int __pure strcmp(const char * cs,const char * ct) __nonnull;
-int __pure strncmp(const char * cs,const char * ct,
-				     size_t count) __nonnull;
-char * __pure strchr(const char * s, int c) __nonnull;
-char * __pure strrchr(const char * s, int c) __nonnull;
-size_t __pure strlen(const char * s) __nonnull;
-size_t __pure strnlen(const char * s, size_t count) __nonnull;
-size_t __pure strspn(const char *s, const char *accept) __nonnull;
-size_t __pure strcspn(const char *s, const char *reject) __nonnull;
-char * __pure strpbrk(const char * cs,const char * ct) __nonnull;
-char * strtok(char * s,const char * ct) __nonnull;
-char * strsep(char **s, const char *ct) __nonnull;
-void * memset(void * s,int c,size_t count) __nonnull;
+/* Architecture-specific code is expected to provide these functions,
+ * but may instead explicitly choose to use the generic versions.
+ */
+void * memset ( void *dest, int character, size_t len ) __nonnull;
 void * memcpy ( void *dest, const void *src, size_t len ) __nonnull;
-void * memmove(void * dest,const void *src,size_t count) __nonnull;
-int __pure memcmp(const void * cs,const void * ct,
-				    size_t count) __nonnull;
-void * __pure memscan(const void * addr, int c, size_t size) __nonnull;
-char * __pure strstr(const char * s1,const char * s2) __nonnull;
-void * __pure memchr(const void *s, int c, size_t n) __nonnull;
-char * __malloc strdup(const char *s) __nonnull;
-char * __malloc strndup(const char *s, size_t n) __nonnull;
+void * memmove ( void *dest, const void *src, size_t len ) __nonnull;
+extern void * generic_memset ( void *dest, int character,
+			       size_t len ) __nonnull;
+extern void * generic_memcpy ( void *dest, const void *src,
+			       size_t len ) __nonnull;
+extern void * generic_memmove ( void *dest, const void *src,
+				size_t len ) __nonnull;
+
+extern int __pure memcmp ( const void *first, const void *second,
+			   size_t len ) __nonnull;
+extern void * __pure memchr ( const void *src, int character,
+			      size_t len ) __nonnull;
+extern void * memswap ( void *dest, void *src, size_t len ) __nonnull;
+extern int __pure strcmp ( const char *first, const char *second ) __nonnull;
+extern int __pure strncmp ( const char *first, const char *second,
+			    size_t max ) __nonnull;
+extern size_t __pure strlen ( const char *src ) __nonnull;
+extern size_t __pure strnlen ( const char *src, size_t max ) __nonnull;
+extern char * __pure strchr ( const char *src, int character ) __nonnull;
+extern char * __pure strrchr ( const char *src, int character ) __nonnull;
+extern char * __pure strstr ( const char *haystack,
+			      const char *needle ) __nonnull;
+extern char * strcpy ( char *dest, const char *src ) __nonnull;
+extern char * strncpy ( char *dest, const char *src, size_t max ) __nonnull;
+extern char * strcat ( char *dest, const char *src ) __nonnull;
+extern char * __malloc strdup ( const char *src ) __nonnull;
+extern char * __malloc strndup ( const char *src, size_t max ) __nonnull;
+extern char * __pure strpbrk ( const char *string,
+			       const char *delim ) __nonnull;
+extern char * strsep ( char **string, const char *delim ) __nonnull;
 
-extern const char * __pure strerror ( int errno );
+extern char * __pure strerror ( int errno );
 
-#endif /* ETHERBOOT_STRING */
+#endif /* _STRING_H */
diff --git a/roms/ipxe/src/include/strings.h b/roms/ipxe/src/include/strings.h
index 6912a1e45..fab26dc28 100644
--- a/roms/ipxe/src/include/strings.h
+++ b/roms/ipxe/src/include/strings.h
@@ -1,12 +1,71 @@
 #ifndef _STRINGS_H
 #define _STRINGS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+/** @file
+ *
+ * String functions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <limits.h>
 #include <string.h>
 #include <bits/strings.h>
 
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int
+__constant_ffsll ( unsigned long long x ) {
+	int r = 0;
+
+	if ( ! ( x & 0x00000000ffffffffULL ) ) {
+		x >>= 32;
+		r += 32;
+	}
+	if ( ! ( x & 0x0000ffffUL ) ) {
+		x >>= 16;
+		r += 16;
+	}
+	if ( ! ( x & 0x00ff ) ) {
+		x >>= 8;
+		r += 8;
+	}
+	if ( ! ( x & 0x0f ) ) {
+		x >>= 4;
+		r += 4;
+	}
+	if ( ! ( x & 0x3 ) ) {
+		x >>= 2;
+		r += 2;
+	}
+	if ( ! ( x & 0x1 ) ) {
+		x >>= 1;
+		r += 1;
+	}
+	return ( x ? ( r + 1 ) : 0 );
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int
+__constant_ffsl ( unsigned long x ) {
+	return __constant_ffsll ( x );
+}
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 static inline __attribute__ (( always_inline )) int
 __constant_flsll ( unsigned long long x ) {
 	int r = 0;
@@ -35,38 +94,100 @@ __constant_flsll ( unsigned long long x ) {
 		x >>= 1;
 		r += 1;
 	}
-	if ( x & 0x1 ) {
-		r += 1;
-	}
-	return r;
+	return ( x ? ( r + 1 ) : 0 );
 }
 
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 static inline __attribute__ (( always_inline )) int
 __constant_flsl ( unsigned long x ) {
 	return __constant_flsll ( x );
 }
 
+int __ffsll ( long long x );
+int __ffsl ( long x );
 int __flsll ( long long x );
 int __flsl ( long x );
 
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+#define ffsll( x ) \
+	( __builtin_constant_p ( x ) ? __constant_ffsll ( x ) : __ffsll ( x ) )
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+#define ffsl( x ) \
+	( __builtin_constant_p ( x ) ? __constant_ffsl ( x ) : __ffsl ( x ) )
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+#define ffs( x ) ffsl ( x )
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 #define flsll( x ) \
 	( __builtin_constant_p ( x ) ? __constant_flsll ( x ) : __flsll ( x ) )
 
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 #define flsl( x ) \
 	( __builtin_constant_p ( x ) ? __constant_flsl ( x ) : __flsl ( x ) )
 
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 #define fls( x ) flsl ( x )
 
-extern int strcasecmp ( const char *s1, const char *s2 );
-
+/**
+ * Copy memory
+ *
+ * @v src		Source
+ * @v dest		Destination
+ * @v len		Length
+ */
 static inline __attribute__ (( always_inline )) void
-bcopy ( const void *src, void *dest, size_t n ) {
-	memmove ( dest, src, n );
+bcopy ( const void *src, void *dest, size_t len ) {
+	memmove ( dest, src, len );
 }
 
+/**
+ * Zero memory
+ *
+ * @v dest		Destination
+ * @v len		Length
+ */
 static inline __attribute__ (( always_inline )) void
-bzero ( void *s, size_t n ) {
-	memset ( s, 0, n );
+bzero ( void *dest, size_t len ) {
+	memset ( dest, 0, len );
 }
 
+int __pure strcasecmp ( const char *first, const char *second ) __nonnull;
+
 #endif /* _STRINGS_H */
diff --git a/roms/ipxe/src/include/sys/time.h b/roms/ipxe/src/include/sys/time.h
index 2647d3588..6e2a24447 100644
--- a/roms/ipxe/src/include/sys/time.h
+++ b/roms/ipxe/src/include/sys/time.h
@@ -6,7 +6,7 @@
  * Date and time
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/syslog.h b/roms/ipxe/src/include/syslog.h
index 93f32f867..748a4faec 100644
--- a/roms/ipxe/src/include/syslog.h
+++ b/roms/ipxe/src/include/syslog.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <ipxe/ansiesc.h>
diff --git a/roms/ipxe/src/include/time.h b/roms/ipxe/src/include/time.h
index 452a544bb..462ac6999 100644
--- a/roms/ipxe/src/include/time.h
+++ b/roms/ipxe/src/include/time.h
@@ -6,7 +6,7 @@
  * Date and time
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <sys/time.h>
 #include <ipxe/time.h>
diff --git a/roms/ipxe/src/include/unistd.h b/roms/ipxe/src/include/unistd.h
index 3a50a2521..d09e1ae30 100644
--- a/roms/ipxe/src/include/unistd.h
+++ b/roms/ipxe/src/include/unistd.h
@@ -1,7 +1,7 @@
 #ifndef _UNISTD_H
 #define _UNISTD_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
diff --git a/roms/ipxe/src/include/usr/autoboot.h b/roms/ipxe/src/include/usr/autoboot.h
index bc51aae79..4db226b9c 100644
--- a/roms/ipxe/src/include/usr/autoboot.h
+++ b/roms/ipxe/src/include/usr/autoboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 
@@ -35,7 +35,7 @@ extern int uriboot ( struct uri *filename, struct uri *root_path, int drive,
 extern struct uri *
 fetch_next_server_and_filename ( struct settings *settings );
 extern int netboot ( struct net_device *netdev );
-extern void ipxe ( struct net_device *netdev );
+extern int ipxe ( struct net_device *netdev );
 
 extern int pxe_menu_boot ( struct net_device *netdev );
 
diff --git a/roms/ipxe/src/include/usr/dhcpmgmt.h b/roms/ipxe/src/include/usr/dhcpmgmt.h
index af1eceb17..ed669eb9d 100644
--- a/roms/ipxe/src/include/usr/dhcpmgmt.h
+++ b/roms/ipxe/src/include/usr/dhcpmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct net_device;
 
diff --git a/roms/ipxe/src/include/usr/fcmgmt.h b/roms/ipxe/src/include/usr/fcmgmt.h
index 9441cefb4..eb568fd20 100644
--- a/roms/ipxe/src/include/usr/fcmgmt.h
+++ b/roms/ipxe/src/include/usr/fcmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct fc_port;
 struct fc_peer;
diff --git a/roms/ipxe/src/include/usr/ifmgmt.h b/roms/ipxe/src/include/usr/ifmgmt.h
index db77f1f1b..5c386327b 100644
--- a/roms/ipxe/src/include/usr/ifmgmt.h
+++ b/roms/ipxe/src/include/usr/ifmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct net_device;
 struct net_device_configurator;
diff --git a/roms/ipxe/src/include/usr/imgmgmt.h b/roms/ipxe/src/include/usr/imgmgmt.h
index 5e25c562b..806df0bfb 100644
--- a/roms/ipxe/src/include/usr/imgmgmt.h
+++ b/roms/ipxe/src/include/usr/imgmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/image.h>
 
diff --git a/roms/ipxe/src/include/usr/imgtrust.h b/roms/ipxe/src/include/usr/imgtrust.h
index f47105af0..414e07a80 100644
--- a/roms/ipxe/src/include/usr/imgtrust.h
+++ b/roms/ipxe/src/include/usr/imgtrust.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/image.h>
 
diff --git a/roms/ipxe/src/include/usr/ipstat.h b/roms/ipxe/src/include/usr/ipstat.h
index 5ff8b40c3..803254bcb 100644
--- a/roms/ipxe/src/include/usr/ipstat.h
+++ b/roms/ipxe/src/include/usr/ipstat.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void ipstat ( void );
 
diff --git a/roms/ipxe/src/include/usr/lotest.h b/roms/ipxe/src/include/usr/lotest.h
index aa4bbac4d..ce0fe5eda 100644
--- a/roms/ipxe/src/include/usr/lotest.h
+++ b/roms/ipxe/src/include/usr/lotest.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int loopback_test ( struct net_device *sender,
 			   struct net_device *receiver, size_t mtu );
diff --git a/roms/ipxe/src/include/usr/neighmgmt.h b/roms/ipxe/src/include/usr/neighmgmt.h
index 3c2b704af..06f03716e 100644
--- a/roms/ipxe/src/include/usr/neighmgmt.h
+++ b/roms/ipxe/src/include/usr/neighmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void nstat ( void );
 
diff --git a/roms/ipxe/src/include/usr/pingmgmt.h b/roms/ipxe/src/include/usr/pingmgmt.h
index d4c2d6cd5..c7a8434be 100644
--- a/roms/ipxe/src/include/usr/pingmgmt.h
+++ b/roms/ipxe/src/include/usr/pingmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/roms/ipxe/src/include/usr/profstat.h b/roms/ipxe/src/include/usr/profstat.h
index 06ea251a0..b7812ca7f 100644
--- a/roms/ipxe/src/include/usr/profstat.h
+++ b/roms/ipxe/src/include/usr/profstat.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void profstat ( void );
 
diff --git a/roms/ipxe/src/include/usr/prompt.h b/roms/ipxe/src/include/usr/prompt.h
index 57e43d2dc..8d3eeee3c 100644
--- a/roms/ipxe/src/include/usr/prompt.h
+++ b/roms/ipxe/src/include/usr/prompt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int prompt ( const char *text, unsigned long timeout, int key );
 
diff --git a/roms/ipxe/src/include/usr/route.h b/roms/ipxe/src/include/usr/route.h
index b914f4b84..7ec4a3509 100644
--- a/roms/ipxe/src/include/usr/route.h
+++ b/roms/ipxe/src/include/usr/route.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/roms/ipxe/src/include/usr/sync.h b/roms/ipxe/src/include/usr/sync.h
index 0047d4ed9..b6f12ad6e 100644
--- a/roms/ipxe/src/include/usr/sync.h
+++ b/roms/ipxe/src/include/usr/sync.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int sync ( unsigned long timeout );
 
diff --git a/roms/ipxe/src/arch/x86/include/valgrind/memcheck.h b/roms/ipxe/src/include/valgrind/memcheck.h
index 7d4b56d31..7d4b56d31 100644
--- a/roms/ipxe/src/arch/x86/include/valgrind/memcheck.h
+++ b/roms/ipxe/src/include/valgrind/memcheck.h
diff --git a/roms/ipxe/src/arch/x86/include/valgrind/valgrind.h b/roms/ipxe/src/include/valgrind/valgrind.h
index d48bbccae..d48bbccae 100644
--- a/roms/ipxe/src/arch/x86/include/valgrind/valgrind.h
+++ b/roms/ipxe/src/include/valgrind/valgrind.h
diff --git a/roms/ipxe/src/include/wchar.h b/roms/ipxe/src/include/wchar.h
index ba349aae8..d054b8d5b 100644
--- a/roms/ipxe/src/include/wchar.h
+++ b/roms/ipxe/src/include/wchar.h
@@ -1,7 +1,7 @@
 #ifndef WCHAR_H
 #define WCHAR_H
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 
diff --git a/roms/ipxe/src/interface/bofm/bofm.c b/roms/ipxe/src/interface/bofm/bofm.c
index b0e92b27c..545088dc6 100644
--- a/roms/ipxe/src/interface/bofm/bofm.c
+++ b/roms/ipxe/src/interface/bofm/bofm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/interface/efi/efi_autoboot.c b/roms/ipxe/src/interface/efi/efi_autoboot.c
index ab0f36541..a9e807e23 100644
--- a/roms/ipxe/src/interface/efi/efi_autoboot.c
+++ b/roms/ipxe/src/interface/efi/efi_autoboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_autoboot.h>
diff --git a/roms/ipxe/src/interface/efi/efi_bofm.c b/roms/ipxe/src/interface/efi/efi_bofm.c
index bdb705196..ea0e15f7f 100644
--- a/roms/ipxe/src/interface/efi/efi_bofm.c
+++ b/roms/ipxe/src/interface/efi/efi_bofm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/bofm.h>
diff --git a/roms/ipxe/src/interface/efi/efi_debug.c b/roms/ipxe/src/interface/efi/efi_debug.c
index d23960140..473803951 100644
--- a/roms/ipxe/src/interface/efi/efi_debug.c
+++ b/roms/ipxe/src/interface/efi/efi_debug.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -326,7 +330,7 @@ const char * efi_devpath_text ( EFI_DEVICE_PATH_PROTOCOL *path ) {
 		max_len = ( ( sizeof ( text ) - 1 /* NUL */ ) / 2 /* "xx" */ );
 		if ( len > max_len )
 			len = max_len;
-		base16_encode ( start, len, text );
+		base16_encode ( start, len, text, sizeof ( text ) );
 		return text;
 	}
 
diff --git a/roms/ipxe/src/interface/efi/efi_file.c b/roms/ipxe/src/interface/efi/efi_file.c
index 2ef3c5734..3715b70bf 100644
--- a/roms/ipxe/src/interface/efi/efi_file.c
+++ b/roms/ipxe/src/interface/efi/efi_file.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/interface/efi/efi_guid.c b/roms/ipxe/src/interface/efi/efi_guid.c
index 52ba58ae4..ab1c91e9f 100644
--- a/roms/ipxe/src/interface/efi/efi_guid.c
+++ b/roms/ipxe/src/interface/efi/efi_guid.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/Protocol/Arp.h>
diff --git a/roms/ipxe/src/interface/efi/efi_hii.c b/roms/ipxe/src/interface/efi/efi_hii.c
index 834060b54..0ea970e67 100644
--- a/roms/ipxe/src/interface/efi/efi_hii.c
+++ b/roms/ipxe/src/interface/efi/efi_hii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stddef.h>
diff --git a/roms/ipxe/src/interface/efi/efi_pci.c b/roms/ipxe/src/interface/efi/efi_pci.c
index 86c781d82..97ea72bb9 100644
--- a/roms/ipxe/src/interface/efi/efi_pci.c
+++ b/roms/ipxe/src/interface/efi/efi_pci.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/interface/efi/efi_reboot.c b/roms/ipxe/src/interface/efi/efi_reboot.c
index 96638c48e..35919221e 100644
--- a/roms/ipxe/src/interface/efi/efi_reboot.c
+++ b/roms/ipxe/src/interface/efi/efi_reboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/interface/efi/efi_snp.c b/roms/ipxe/src/interface/efi/efi_snp.c
index 67fba342e..3dfcc5e16 100644
--- a/roms/ipxe/src/interface/efi/efi_snp.c
+++ b/roms/ipxe/src/interface/efi/efi_snp.c
@@ -32,8 +32,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_strings.h>
 #include <ipxe/efi/efi_utils.h>
+#include <ipxe/efi/efi_watchdog.h>
 #include <ipxe/efi/efi_snp.h>
 #include <usr/autoboot.h>
+#include <config/general.h>
 
 /** List of SNP devices */
 static LIST_HEAD ( efi_snp_devices );
@@ -41,6 +43,39 @@ static LIST_HEAD ( efi_snp_devices );
 /** Network devices are currently claimed for use by iPXE */
 static int efi_snp_claimed;
 
+/* Downgrade user experience if configured to do so
+ *
+ * The default UEFI user experience for network boot is somewhat
+ * excremental: only TFTP is available as a download protocol, and if
+ * anything goes wrong the user will be shown just a dot on an
+ * otherwise blank screen.  (Some programmer was clearly determined to
+ * win a bet that they could outshine Apple at producing uninformative
+ * error messages.)
+ *
+ * For comparison, the default iPXE user experience provides the
+ * option to use protocols designed more recently than 1980 (such as
+ * HTTP and iSCSI), and if anything goes wrong the the user will be
+ * shown one of over 1200 different error messages, complete with a
+ * link to a wiki page describing that specific error.
+ *
+ * We default to upgrading the user experience to match that available
+ * in a "legacy" BIOS environment, by installing our own instance of
+ * EFI_LOAD_FILE_PROTOCOL.
+ *
+ * Note that unfortunately we can't sensibly provide the choice of
+ * both options to the user in the same build, because the UEFI boot
+ * menu ignores the multitude of ways in which a network device handle
+ * can be described and opaquely labels both menu entries as just "EFI
+ * Network".
+ */
+#ifdef EFI_DOWNGRADE_UX
+static EFI_GUID dummy_load_file_protocol_guid = {
+	0x6f6c7323, 0x2077, 0x7523,
+	{ 0x6e, 0x68, 0x65, 0x6c, 0x70, 0x66, 0x75, 0x6c }
+};
+#define efi_load_file_protocol_guid dummy_load_file_protocol_guid
+#endif
+
 /**
  * Set EFI SNP mode state
  *
@@ -98,28 +133,43 @@ static void efi_snp_set_mode ( struct efi_snp_device *snpdev ) {
 }
 
 /**
+ * Flush transmit ring and receive queue
+ *
+ * @v snpdev		SNP device
+ */
+static void efi_snp_flush ( struct efi_snp_device *snpdev ) {
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
+	/* Reset transmit completion ring */
+	snpdev->tx_prod = 0;
+	snpdev->tx_cons = 0;
+
+	/* Discard any queued receive buffers */
+	list_for_each_entry_safe ( iobuf, tmp, &snpdev->rx, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+}
+
+/**
  * Poll net device and count received packets
  *
  * @v snpdev		SNP device
  */
 static void efi_snp_poll ( struct efi_snp_device *snpdev ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	struct io_buffer *iobuf;
-	unsigned int before = 0;
-	unsigned int after = 0;
-	unsigned int arrived;
 
-	/* We have to report packet arrivals, and this is the easiest
-	 * way to fake it.
-	 */
-	list_for_each_entry ( iobuf, &snpdev->netdev->rx_queue, list )
-		before++;
+	/* Poll network device */
 	netdev_poll ( snpdev->netdev );
-	list_for_each_entry ( iobuf, &snpdev->netdev->rx_queue, list )
-		after++;
-	arrived = ( after - before );
 
-	snpdev->rx_count_interrupts += arrived;
-	snpdev->rx_count_events += arrived;
+	/* Retrieve any received packets */
+	while ( ( iobuf = netdev_rx_dequeue ( snpdev->netdev ) ) ) {
+		list_add_tail ( &iobuf->list, &snpdev->rx );
+		snpdev->interrupts |= EFI_SIMPLE_NETWORK_RECEIVE_INTERRUPT;
+		bs->SignalEvent ( &snpdev->snp.WaitForPacket );
+	}
 }
 
 /**
@@ -221,6 +271,7 @@ efi_snp_reset ( EFI_SIMPLE_NETWORK_PROTOCOL *snp, BOOLEAN ext_verify ) {
 
 	netdev_close ( snpdev->netdev );
 	efi_snp_set_state ( snpdev );
+	efi_snp_flush ( snpdev );
 
 	if ( ( rc = netdev_open ( snpdev->netdev ) ) != 0 ) {
 		DBGC ( snpdev, "SNPDEV %p could not reopen %s: %s\n",
@@ -251,6 +302,7 @@ efi_snp_shutdown ( EFI_SIMPLE_NETWORK_PROTOCOL *snp ) {
 
 	netdev_close ( snpdev->netdev );
 	efi_snp_set_state ( snpdev );
+	efi_snp_flush ( snpdev );
 
 	return 0;
 }
@@ -446,20 +498,22 @@ efi_snp_nvdata ( EFI_SIMPLE_NETWORK_PROTOCOL *snp, BOOLEAN read,
  *
  * @v snp		SNP interface
  * @v interrupts	Interrupt status, or NULL
- * @v txbufs		Recycled transmit buffer address, or NULL
+ * @v txbuf		Recycled transmit buffer address, or NULL
  * @ret efirc		EFI status code
  */
 static EFI_STATUS EFIAPI
 efi_snp_get_status ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
-		     UINT32 *interrupts, VOID **txbufs ) {
+		     UINT32 *interrupts, VOID **txbuf ) {
 	struct efi_snp_device *snpdev =
 		container_of ( snp, struct efi_snp_device, snp );
 
 	DBGC2 ( snpdev, "SNPDEV %p GET_STATUS", snpdev );
 
 	/* Fail if net device is currently claimed for use by iPXE */
-	if ( efi_snp_claimed )
+	if ( efi_snp_claimed ) {
+		DBGC2 ( snpdev, "\n" );
 		return EFI_NOT_READY;
+	}
 
 	/* Poll the network device */
 	efi_snp_poll ( snpdev );
@@ -468,47 +522,19 @@ efi_snp_get_status ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 	 * to detect TX completions.
 	 */
 	if ( interrupts ) {
-		*interrupts = 0;
-		/* Report TX completions once queue is empty; this
-		 * avoids having to add hooks in the net device layer.
-		 */
-		if ( snpdev->tx_count_interrupts &&
-		     list_empty ( &snpdev->netdev->tx_queue ) ) {
-			*interrupts |= EFI_SIMPLE_NETWORK_TRANSMIT_INTERRUPT;
-			snpdev->tx_count_interrupts--;
-		}
-		/* Report RX */
-		if ( snpdev->rx_count_interrupts ) {
-			*interrupts |= EFI_SIMPLE_NETWORK_RECEIVE_INTERRUPT;
-			snpdev->rx_count_interrupts--;
-		}
+		*interrupts = snpdev->interrupts;
 		DBGC2 ( snpdev, " INTS:%02x", *interrupts );
+		snpdev->interrupts = 0;
 	}
 
-	/* TX completions.  It would be possible to design a more
-	 * idiotic scheme for this, but it would be a challenge.
-	 * According to the UEFI header file, txbufs will be filled in
-	 * with a list of "recycled transmit buffers" (i.e. completed
-	 * TX buffers).  Observant readers may care to note that
-	 * *txbufs is a void pointer.  Precisely how a list of
-	 * completed transmit buffers is meant to be represented as an
-	 * array of voids is left as an exercise for the reader.
-	 *
-	 * The only users of this interface (MnpDxe/MnpIo.c and
-	 * PxeBcDxe/Bc.c within the EFI dev kit) both just poll until
-	 * seeing a non-NULL result return in txbufs.  This is valid
-	 * provided that they do not ever attempt to transmit more
-	 * than one packet concurrently (and that TX never times out).
-	 */
-	if ( txbufs ) {
-		if ( snpdev->tx_count_txbufs &&
-		     list_empty ( &snpdev->netdev->tx_queue ) ) {
-			*txbufs = "Which idiot designed this API?";
-			snpdev->tx_count_txbufs--;
+	/* TX completions */
+	if ( txbuf ) {
+		if ( snpdev->tx_prod != snpdev->tx_cons ) {
+			*txbuf = snpdev->tx[snpdev->tx_cons++ % EFI_SNP_NUM_TX];
 		} else {
-			*txbufs = NULL;
+			*txbuf = NULL;
 		}
-		DBGC2 ( snpdev, " TX:%s", ( *txbufs ? "some" : "none" ) );
+		DBGC2 ( snpdev, " TX:%p", *txbuf );
 	}
 
 	DBGC2 ( snpdev, "\n" );
@@ -537,6 +563,7 @@ efi_snp_transmit ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 	struct ll_protocol *ll_protocol = snpdev->netdev->ll_protocol;
 	struct io_buffer *iobuf;
 	size_t payload_len;
+	unsigned int tx_fill;
 	int rc;
 
 	DBGC2 ( snpdev, "SNPDEV %p TRANSMIT %p+%lx", snpdev, data,
@@ -624,12 +651,27 @@ efi_snp_transmit ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 		goto err_tx;
 	}
 
-	/* Record transmission as outstanding */
-	snpdev->tx_count_interrupts++;
-	snpdev->tx_count_txbufs++;
+	/* Record in transmit completion ring.  If we run out of
+	 * space, report the failure even though we have already
+	 * transmitted the packet.
+	 *
+	 * This allows us to report completions only for packets for
+	 * which we had reported successfully initiating transmission,
+	 * while continuing to support clients that never poll for
+	 * transmit completions.
+	 */
+	tx_fill = ( snpdev->tx_prod - snpdev->tx_cons );
+	if ( tx_fill >= EFI_SNP_NUM_TX ) {
+		DBGC ( snpdev, "SNPDEV %p TX completion ring full\n", snpdev );
+		rc = -ENOBUFS;
+		goto err_ring_full;
+	}
+	snpdev->tx[ snpdev->tx_prod++ % EFI_SNP_NUM_TX ] = data;
+	snpdev->interrupts |= EFI_SIMPLE_NETWORK_TRANSMIT_INTERRUPT;
 
 	return 0;
 
+ err_ring_full:
  err_tx:
  err_ll_push:
 	free_iob ( iobuf );
@@ -676,12 +718,13 @@ efi_snp_receive ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 	efi_snp_poll ( snpdev );
 
 	/* Dequeue a packet, if one is available */
-	iobuf = netdev_rx_dequeue ( snpdev->netdev );
+	iobuf = list_first_entry ( &snpdev->rx, struct io_buffer, list );
 	if ( ! iobuf ) {
 		DBGC2 ( snpdev, "\n" );
 		rc = -EAGAIN;
 		goto out_no_packet;
 	}
+	list_del ( &iobuf->list );
 	DBGC2 ( snpdev, "+%zx\n", iob_len ( iobuf ) );
 
 	/* Return packet to caller */
@@ -721,9 +764,8 @@ efi_snp_receive ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
  * @v event		Event
  * @v context		Event context
  */
-static VOID EFIAPI efi_snp_wait_for_packet ( EFI_EVENT event,
+static VOID EFIAPI efi_snp_wait_for_packet ( EFI_EVENT event __unused,
 					     VOID *context ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	struct efi_snp_device *snpdev = context;
 
 	DBGCP ( snpdev, "SNPDEV %p WAIT_FOR_PACKET\n", snpdev );
@@ -738,14 +780,6 @@ static VOID EFIAPI efi_snp_wait_for_packet ( EFI_EVENT event,
 
 	/* Poll the network device */
 	efi_snp_poll ( snpdev );
-
-	/* Fire event if packets have been received */
-	if ( snpdev->rx_count_events != 0 ) {
-		DBGC2 ( snpdev, "SNPDEV %p firing WaitForPacket event\n",
-			snpdev );
-		bs->SignalEvent ( event );
-		snpdev->rx_count_events--;
-	}
 }
 
 /** SNP interface */
@@ -837,6 +871,7 @@ efi_snp_load_file ( EFI_LOAD_FILE_PROTOCOL *load_file,
 	struct efi_snp_device *snpdev =
 		container_of ( load_file, struct efi_snp_device, load_file );
 	struct net_device *netdev = snpdev->netdev;
+	int rc;
 
 	/* Fail unless this is a boot attempt */
 	if ( ! booting ) {
@@ -848,14 +883,17 @@ efi_snp_load_file ( EFI_LOAD_FILE_PROTOCOL *load_file,
 	/* Claim network devices for use by iPXE */
 	efi_snp_claim();
 
+	/* Start watchdog holdoff timer */
+	efi_watchdog_start();
+
 	/* Boot from network device */
-	ipxe ( netdev );
+	if ( ( rc = ipxe ( netdev ) ) != 0 )
+		goto err_ipxe;
 
-	/* Release network devices for use via SNP */
+ err_ipxe:
+	efi_watchdog_stop();
 	efi_snp_release();
-
-	/* Assume boot process was aborted */
-	return EFI_ABORTED;
+	return EFIRC ( rc );
 }
 
 /** Load file protocol */
@@ -922,6 +960,7 @@ static int efi_snp_probe ( struct net_device *netdev ) {
 	}
 	snpdev->netdev = netdev_get ( netdev );
 	snpdev->efidev = efidev;
+	INIT_LIST_HEAD ( &snpdev->rx );
 
 	/* Sanity check */
 	if ( netdev->ll_protocol->ll_addr_len > sizeof ( EFI_MAC_ADDRESS ) ) {
diff --git a/roms/ipxe/src/interface/efi/efi_snp_hii.c b/roms/ipxe/src/interface/efi/efi_snp_hii.c
index c49c76a32..720402bdb 100644
--- a/roms/ipxe/src/interface/efi/efi_snp_hii.c
+++ b/roms/ipxe/src/interface/efi/efi_snp_hii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -59,6 +63,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/efi_hii.h>
 #include <ipxe/efi/efi_snp.h>
 #include <ipxe/efi/efi_strings.h>
+#include <config/branding.h>
 
 /** EFI platform setup formset GUID */
 static EFI_GUID efi_hii_platform_setup_formset_guid
@@ -136,7 +141,7 @@ static void efi_snp_hii_questions ( struct efi_snp_device *snpdev,
 		previous = setting;
 		name_id = efi_ifr_string ( ifr, "%s", setting->name );
 		prompt_id = efi_ifr_string ( ifr, "%s", setting->description );
-		help_id = efi_ifr_string ( ifr, "http://ipxe.org/cfg/%s",
+		help_id = efi_ifr_string ( ifr, PRODUCT_SETTING_URI,
 					   setting->name );
 		question_id = setting->tag;
 		efi_ifr_string_op ( ifr, prompt_id, help_id,
diff --git a/roms/ipxe/src/interface/efi/efi_strings.c b/roms/ipxe/src/interface/efi/efi_strings.c
index 751589b46..aa3afc64f 100644
--- a/roms/ipxe/src/interface/efi/efi_strings.c
+++ b/roms/ipxe/src/interface/efi/efi_strings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
diff --git a/roms/ipxe/src/interface/efi/efi_time.c b/roms/ipxe/src/interface/efi/efi_time.c
new file mode 100644
index 000000000..983a0ef5c
--- /dev/null
+++ b/roms/ipxe/src/interface/efi/efi_time.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <time.h>
+#include <ipxe/time.h>
+#include <ipxe/efi/efi.h>
+
+/** @file
+ *
+ * EFI time source
+ *
+ */
+
+/**
+ * Get current time in seconds
+ *
+ * @ret time		Time, in seconds
+ */
+static time_t efi_get_time ( void ) {
+	EFI_RUNTIME_SERVICES *rs = efi_systab->RuntimeServices;
+	EFI_TIME time;
+	struct tm tm;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Get current time and date */
+	if ( ( efirc = rs->GetTime ( &time, NULL ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( rs, "EFITIME could not get system time: %s\n",
+		       strerror ( rc ) );
+		/* Nothing meaningful we can return */
+		return 0;
+	}
+
+	/* Construct broken-down time */
+	memset ( &tm, 0, sizeof ( tm ) );
+	tm.tm_sec = time.Second;
+	tm.tm_min = time.Minute;
+	tm.tm_hour = time.Hour;
+	tm.tm_mday = time.Day;
+	tm.tm_mon = ( time.Month - 1 );
+	tm.tm_year = ( time.Year - 1900 );
+	DBGC ( rs, "EFITIME is %04d-%02d-%02d %02d:%02d:%02d\n",
+	       ( tm.tm_year + 1900 ), ( tm.tm_mon + 1 ),
+	       tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec );
+
+	/* Convert to seconds since the Epoch */
+	return mktime ( &tm );
+}
+
+PROVIDE_TIME ( efi, time_now, efi_get_time );
diff --git a/roms/ipxe/src/interface/efi/efi_timer.c b/roms/ipxe/src/interface/efi/efi_timer.c
index 7a1ff7869..81620c92c 100644
--- a/roms/ipxe/src/interface/efi/efi_timer.c
+++ b/roms/ipxe/src/interface/efi/efi_timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/interface/efi/efi_uaccess.c b/roms/ipxe/src/interface/efi/efi_uaccess.c
index 8b429b9ee..e058be66b 100644
--- a/roms/ipxe/src/interface/efi/efi_uaccess.c
+++ b/roms/ipxe/src/interface/efi/efi_uaccess.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 #include <ipxe/efi/efi.h>
diff --git a/roms/ipxe/src/interface/efi/efi_umalloc.c b/roms/ipxe/src/interface/efi/efi_umalloc.c
index 356efaa6f..e3f1dacc2 100644
--- a/roms/ipxe/src/interface/efi/efi_umalloc.c
+++ b/roms/ipxe/src/interface/efi/efi_umalloc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/interface/efi/efi_watchdog.c b/roms/ipxe/src/interface/efi/efi_watchdog.c
new file mode 100644
index 000000000..7061f81d8
--- /dev/null
+++ b/roms/ipxe/src/interface/efi/efi_watchdog.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * EFI watchdog holdoff timer
+ *
+ */
+
+#include <errno.h>
+#include <string.h>
+#include <ipxe/retry.h>
+#include <ipxe/timer.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_watchdog.h>
+
+/** Watchdog holdoff interval (in seconds) */
+#define WATCHDOG_HOLDOFF_SECS 10
+
+/** Watchdog timeout (in seconds) */
+#define WATCHDOG_TIMEOUT_SECS ( 5 * 60 )
+
+/** Watchdog code (to be logged on watchdog timeout) */
+#define WATCHDOG_CODE 0x6950584544454144ULL
+
+/** Watchdog data (to be logged on watchdog timeout) */
+#define WATCHDOG_DATA L"iPXE";
+
+/**
+ * Hold off watchdog timer
+ *
+ * @v retry		Retry timer
+ * @v over		Failure indicator
+ */
+static void efi_watchdog_expired ( struct retry_timer *timer,
+				   int over __unused ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	static CHAR16 data[] = WATCHDOG_DATA;
+	EFI_STATUS efirc;
+	int rc;
+
+	DBGC2 ( timer, "EFI holding off watchdog timer\n" );
+
+	/* Restart this holdoff timer */
+	start_timer_fixed ( timer, ( WATCHDOG_HOLDOFF_SECS * TICKS_PER_SEC ) );
+
+	/* Reset watchdog timer */
+	if ( ( efirc = bs->SetWatchdogTimer ( WATCHDOG_TIMEOUT_SECS,
+					      WATCHDOG_CODE, sizeof ( data ),
+					      data ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( timer, "EFI could not set watchdog timer: %s\n",
+		       strerror ( rc ) );
+		return;
+	}
+}
+
+/** Watchdog holdoff timer */
+struct retry_timer efi_watchdog = TIMER_INIT ( efi_watchdog_expired );
diff --git a/roms/ipxe/src/interface/efi/efi_wrap.c b/roms/ipxe/src/interface/efi/efi_wrap.c
index ff46b76ed..2ea184e97 100644
--- a/roms/ipxe/src/interface/efi/efi_wrap.c
+++ b/roms/ipxe/src/interface/efi/efi_wrap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/interface/hyperv/vmbus.c b/roms/ipxe/src/interface/hyperv/vmbus.c
new file mode 100644
index 000000000..795929eae
--- /dev/null
+++ b/roms/ipxe/src/interface/hyperv/vmbus.c
@@ -0,0 +1,1333 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Hyper-V virtual machine bus
+ *
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/nap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/hyperv.h>
+#include <ipxe/vmbus.h>
+
+/** VMBus initial GPADL ID
+ *
+ * This is an opaque value with no meaning.  The Linux kernel uses
+ * 0xe1e10.
+ */
+#define VMBUS_GPADL_MAGIC 0x18ae0000
+
+/**
+ * Post message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v header		Message header
+ * @v len		Length of message (including header)
+ * @ret rc		Return status code
+ */
+static int vmbus_post_message ( struct hv_hypervisor *hv,
+				const struct vmbus_message_header *header,
+				size_t len ) {
+	struct vmbus *vmbus = hv->vmbus;
+	int rc;
+
+	/* Post message */
+	if ( ( rc = hv_post_message ( hv, VMBUS_MESSAGE_ID, VMBUS_MESSAGE_TYPE,
+				      header, len ) ) != 0 ) {
+		DBGC ( vmbus, "VMBUS %p could not post message: %s\n",
+		       vmbus, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Post empty message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v type		Message type
+ * @ret rc		Return status code
+ */
+static int vmbus_post_empty_message ( struct hv_hypervisor *hv,
+				      unsigned int type ) {
+	struct vmbus_message_header header = { .type = cpu_to_le32 ( type ) };
+
+	return vmbus_post_message ( hv, &header, sizeof ( header ) );
+}
+
+/**
+ * Wait for received message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int vmbus_wait_for_message ( struct hv_hypervisor *hv ) {
+	struct vmbus *vmbus = hv->vmbus;
+	int rc;
+
+	/* Wait for message */
+	if ( ( rc = hv_wait_for_message ( hv, VMBUS_MESSAGE_SINT ) ) != 0 ) {
+		DBGC ( vmbus, "VMBUS %p failed waiting for message: %s\n",
+		       vmbus, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Sanity check */
+	if ( hv->message->received.type != cpu_to_le32 ( VMBUS_MESSAGE_TYPE ) ){
+		DBGC ( vmbus, "VMBUS %p invalid message type %d\n",
+		       vmbus, le32_to_cpu ( hv->message->received.type ) );
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * Initiate contact
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v raw		VMBus protocol (raw) version
+ * @ret rc		Return status code
+ */
+static int vmbus_initiate_contact ( struct hv_hypervisor *hv,
+				    unsigned int raw ) {
+	struct vmbus *vmbus = hv->vmbus;
+	const struct vmbus_version_response *version = &vmbus->message->version;
+	struct vmbus_initiate_contact initiate;
+	int rc;
+
+	/* Construct message */
+	memset ( &initiate, 0, sizeof ( initiate ) );
+	initiate.header.type = cpu_to_le32 ( VMBUS_INITIATE_CONTACT );
+	initiate.version.raw = cpu_to_le32 ( raw );
+	initiate.intr = virt_to_phys ( vmbus->intr );
+	initiate.monitor_in = virt_to_phys ( vmbus->monitor_in );
+	initiate.monitor_out = virt_to_phys ( vmbus->monitor_out );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &initiate.header,
+					 sizeof ( initiate ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( version->header.type != cpu_to_le32 ( VMBUS_VERSION_RESPONSE ) ) {
+		DBGC ( vmbus, "VMBUS %p unexpected version response type %d\n",
+		       vmbus, le32_to_cpu ( version->header.type ) );
+		return -EPROTO;
+	}
+	if ( ! version->supported ) {
+		DBGC ( vmbus, "VMBUS %p requested version not supported\n",
+		       vmbus );
+		return -ENOTSUP;
+	}
+	if ( version->version.raw != cpu_to_le32 ( raw ) ) {
+		DBGC ( vmbus, "VMBUS %p unexpected version %d.%d\n",
+		       vmbus, le16_to_cpu ( version->version.major ),
+		       le16_to_cpu ( version->version.minor ) );
+		return -EPROTO;
+	}
+
+	DBGC ( vmbus, "VMBUS %p initiated contact using version %d.%d\n",
+	       vmbus, le16_to_cpu ( version->version.major ),
+	       le16_to_cpu ( version->version.minor ) );
+	return 0;
+}
+
+/**
+ * Terminate contact
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int vmbus_unload ( struct hv_hypervisor *hv ) {
+	struct vmbus *vmbus = hv->vmbus;
+	const struct vmbus_message_header *header = &vmbus->message->header;
+	int rc;
+
+	/* Post message */
+	if ( ( rc = vmbus_post_empty_message ( hv, VMBUS_UNLOAD ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( header->type != cpu_to_le32 ( VMBUS_UNLOAD_RESPONSE ) ) {
+		DBGC ( vmbus, "VMBUS %p unexpected unload response type %d\n",
+		       vmbus, le32_to_cpu ( header->type ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Negotiate protocol version
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int vmbus_negotiate_version ( struct hv_hypervisor *hv ) {
+	int rc;
+
+	/* We require the ability to disconnect from and reconnect to
+	 * VMBus; if we don't have this then there is no (viable) way
+	 * for a loaded operating system to continue to use any VMBus
+	 * devices.  (There is also a small but non-zero risk that the
+	 * host will continue to write to our interrupt and monitor
+	 * pages, since the VMBUS_UNLOAD message in earlier versions
+	 * is essentially a no-op.)
+	 *
+	 * This requires us to ensure that the host supports protocol
+	 * version 3.0 (VMBUS_VERSION_WIN8_1).  However, we can't
+	 * actually _use_ protocol version 3.0, since doing so causes
+	 * an iSCSI-booted Windows Server 2012 R2 VM to crash due to a
+	 * NULL pointer dereference in vmbus.sys.
+	 *
+	 * To work around this problem, we first ensure that we can
+	 * connect using protocol v3.0, then disconnect and reconnect
+	 * using the oldest known protocol.
+	 */
+
+	/* Initiate contact to check for required protocol support */
+	if ( ( rc = vmbus_initiate_contact ( hv, VMBUS_VERSION_WIN8_1 ) ) != 0 )
+		return rc;
+
+	/* Terminate contact */
+	if ( ( rc = vmbus_unload ( hv ) ) != 0 )
+		return rc;
+
+	/* Reinitiate contact using the oldest known protocol version */
+	if ( ( rc = vmbus_initiate_contact ( hv, VMBUS_VERSION_WS2008 ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Establish GPA descriptor list
+ *
+ * @v vmdev		VMBus device
+ * @v data		Data buffer
+ * @v len		Length of data buffer
+ * @ret gpadl		GPADL ID, or negative error
+ */
+int vmbus_establish_gpadl ( struct vmbus_device *vmdev, userptr_t data,
+			    size_t len ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	physaddr_t addr = user_to_phys ( data, 0 );
+	unsigned int pfn_count = hv_pfn_count ( addr, len );
+	struct {
+		struct vmbus_gpadl_header gpadlhdr;
+		struct vmbus_gpa_range range;
+		uint64_t pfn[pfn_count];
+	} __attribute__ (( packed )) gpadlhdr;
+	const struct vmbus_gpadl_created *created = &vmbus->message->created;
+	static unsigned int gpadl = VMBUS_GPADL_MAGIC;
+	unsigned int i;
+	int rc;
+
+	/* Allocate GPADL ID */
+	gpadl++;
+
+	/* Construct message */
+	memset ( &gpadlhdr, 0, sizeof ( gpadlhdr ) );
+	gpadlhdr.gpadlhdr.header.type = cpu_to_le32 ( VMBUS_GPADL_HEADER );
+	gpadlhdr.gpadlhdr.channel = cpu_to_le32 ( vmdev->channel );
+	gpadlhdr.gpadlhdr.gpadl = cpu_to_le32 ( gpadl );
+	gpadlhdr.gpadlhdr.range_len =
+		cpu_to_le16 ( ( sizeof ( gpadlhdr.range ) +
+				sizeof ( gpadlhdr.pfn ) ) );
+	gpadlhdr.gpadlhdr.range_count = cpu_to_le16 ( 1 );
+	gpadlhdr.range.len = cpu_to_le32 ( len );
+	gpadlhdr.range.offset = cpu_to_le32 ( addr & ( PAGE_SIZE - 1 ) );
+	for ( i = 0 ; i < pfn_count ; i++ )
+		gpadlhdr.pfn[i] = ( ( addr / PAGE_SIZE ) + i );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &gpadlhdr.gpadlhdr.header,
+					 sizeof ( gpadlhdr ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( created->header.type != cpu_to_le32 ( VMBUS_GPADL_CREATED ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL response type %d\n",
+		       vmdev->dev.name, le32_to_cpu ( created->header.type ) );
+		return -EPROTO;
+	}
+	if ( created->channel != cpu_to_le32 ( vmdev->channel ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL channel %d\n",
+		       vmdev->dev.name, le32_to_cpu ( created->channel ) );
+		return -EPROTO;
+	}
+	if ( created->gpadl != cpu_to_le32 ( gpadl ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL ID %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( created->gpadl ) );
+		return -EPROTO;
+	}
+	if ( created->status != 0 ) {
+		DBGC ( vmdev, "VMBUS %s GPADL creation failed: %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( created->status ) );
+		return -EPROTO;
+	}
+
+	DBGC ( vmdev, "VMBUS %s GPADL %#08x is [%08lx,%08lx)\n",
+	       vmdev->dev.name, gpadl, addr, ( addr + len ) );
+	return gpadl;
+}
+
+/**
+ * Tear down GPA descriptor list
+ *
+ * @v vmdev		VMBus device
+ * @v gpadl		GPADL ID
+ * @ret rc		Return status code
+ */
+int vmbus_gpadl_teardown ( struct vmbus_device *vmdev, unsigned int gpadl ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	struct vmbus_gpadl_teardown teardown;
+	const struct vmbus_gpadl_torndown *torndown = &vmbus->message->torndown;
+	int rc;
+
+	/* Construct message */
+	memset ( &teardown, 0, sizeof ( teardown ) );
+	teardown.header.type = cpu_to_le32 ( VMBUS_GPADL_TEARDOWN );
+	teardown.channel = cpu_to_le32 ( vmdev->channel );
+	teardown.gpadl = cpu_to_le32 ( gpadl );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &teardown.header,
+					 sizeof ( teardown ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( torndown->header.type != cpu_to_le32 ( VMBUS_GPADL_TORNDOWN ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL response type %d\n",
+		       vmdev->dev.name, le32_to_cpu ( torndown->header.type ) );
+		return -EPROTO;
+	}
+	if ( torndown->gpadl != cpu_to_le32 ( gpadl ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL ID %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( torndown->gpadl ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Open VMBus channel
+ *
+ * @v vmdev		VMBus device
+ * @v op		Channel operations
+ * @v out_len		Outbound ring buffer length
+ * @v in_len		Inbound ring buffer length
+ * @v mtu		Maximum expected data packet length (including headers)
+ * @ret rc		Return status code
+ *
+ * Both outbound and inbound ring buffer lengths must be a power of
+ * two and a multiple of PAGE_SIZE.  The requirement to be a power of
+ * two is a policy decision taken to simplify the ring buffer indexing
+ * logic.
+ */
+int vmbus_open ( struct vmbus_device *vmdev,
+		 struct vmbus_channel_operations *op,
+		 size_t out_len, size_t in_len, size_t mtu ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	struct vmbus_open_channel open;
+	const struct vmbus_open_channel_result *opened =
+		&vmbus->message->opened;
+	size_t len;
+	void *ring;
+	void *packet;
+	int gpadl;
+	uint32_t open_id;
+	int rc;
+
+	/* Sanity checks */
+	assert ( ( out_len % PAGE_SIZE ) == 0 );
+	assert ( ( out_len & ( out_len - 1 ) ) == 0 );
+	assert ( ( in_len % PAGE_SIZE ) == 0 );
+	assert ( ( in_len & ( in_len - 1 ) ) == 0 );
+	assert ( mtu >= ( sizeof ( struct vmbus_packet_header ) +
+			  sizeof ( struct vmbus_packet_footer ) ) );
+
+	/* Allocate packet buffer */
+	packet = malloc ( mtu );
+	if ( ! packet ) {
+		rc = -ENOMEM;
+		goto err_alloc_packet;
+	}
+
+	/* Allocate ring buffer */
+	len = ( sizeof ( *vmdev->out ) + out_len +
+		sizeof ( *vmdev->in ) + in_len );
+	assert ( ( len % PAGE_SIZE ) == 0 );
+	ring = malloc_dma ( len, PAGE_SIZE );
+	if ( ! ring ) {
+		rc = -ENOMEM;
+		goto err_alloc_ring;
+	}
+	memset ( ring, 0, len );
+
+	/* Establish GPADL for ring buffer */
+	gpadl = vmbus_establish_gpadl ( vmdev, virt_to_user ( ring ), len );
+	if ( gpadl < 0 ) {
+		rc = gpadl;
+		goto err_establish;
+	}
+
+	/* Construct message */
+	memset ( &open, 0, sizeof ( open ) );
+	open.header.type = cpu_to_le32 ( VMBUS_OPEN_CHANNEL );
+	open.channel = cpu_to_le32 ( vmdev->channel );
+	open_id = random();
+	open.id = open_id; /* Opaque random value: endianness irrelevant */
+	open.gpadl = cpu_to_le32 ( gpadl );
+	open.out_pages = ( ( sizeof ( *vmdev->out ) / PAGE_SIZE ) +
+			   ( out_len / PAGE_SIZE ) );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &open.header,
+					 sizeof ( open ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( opened->header.type != cpu_to_le32 ( VMBUS_OPEN_CHANNEL_RESULT ) ){
+		DBGC ( vmdev, "VMBUS %s unexpected open response type %d\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->header.type ) );
+		return -EPROTO;
+	}
+	if ( opened->channel != cpu_to_le32 ( vmdev->channel ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected opened channel %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->channel ) );
+		return -EPROTO;
+	}
+	if ( opened->id != open_id /* Non-endian */ ) {
+		DBGC ( vmdev, "VMBUS %s unexpected open ID %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->id ) );
+		return -EPROTO;
+	}
+	if ( opened->status != 0 ) {
+		DBGC ( vmdev, "VMBUS %s open failed: %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->status ) );
+		return -EPROTO;
+	}
+
+	/* Store channel parameters */
+	vmdev->out_len = out_len;
+	vmdev->in_len = in_len;
+	vmdev->out = ring;
+	vmdev->in = ( ring + sizeof ( *vmdev->out ) + out_len );
+	vmdev->gpadl = gpadl;
+	vmdev->op = op;
+	vmdev->mtu = mtu;
+	vmdev->packet = packet;
+
+	DBGC ( vmdev, "VMBUS %s channel GPADL %#08x ring "
+		"[%#08lx,%#08lx,%#08lx)\n", vmdev->dev.name, vmdev->gpadl,
+		virt_to_phys ( vmdev->out ), virt_to_phys ( vmdev->in ),
+		( virt_to_phys ( vmdev->out ) + len ) );
+	return 0;
+
+	vmbus_gpadl_teardown ( vmdev, vmdev->gpadl );
+ err_establish:
+	free_dma ( ring, len );
+ err_alloc_ring:
+	free ( packet );
+ err_alloc_packet:
+	return rc;
+}
+
+/**
+ * Close VMBus channel
+ *
+ * @v vmdev		VMBus device
+ */
+void vmbus_close ( struct vmbus_device *vmdev ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus_close_channel close;
+	size_t len;
+	int rc;
+
+	/* Construct message */
+	memset ( &close, 0, sizeof ( close ) );
+	close.header.type = cpu_to_le32 ( VMBUS_CLOSE_CHANNEL );
+	close.channel = cpu_to_le32 ( vmdev->channel );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &close.header,
+					 sizeof ( close ) ) ) != 0 ) {
+		DBGC ( vmdev, "VMBUS %s failed to close: %s\n",
+		       vmdev->dev.name, strerror ( rc ) );
+		/* Continue to attempt to tear down GPADL, so that our
+		 * memory is no longer accessible by the remote VM.
+		 */
+	}
+
+	/* Tear down GPADL */
+	if ( ( rc = vmbus_gpadl_teardown ( vmdev,
+					   vmdev->gpadl ) ) != 0 ) {
+		DBGC ( vmdev, "VMBUS %s failed to tear down channel GPADL: "
+		       "%s\n", vmdev->dev.name, strerror ( rc ) );
+		/* We can't prevent the remote VM from continuing to
+		 * access this memory, so leak it.
+		 */
+		return;
+	}
+
+	/* Free ring buffer */
+	len = ( sizeof ( *vmdev->out ) + vmdev->out_len +
+		sizeof ( *vmdev->in ) + vmdev->in_len );
+	free_dma ( vmdev->out, len );
+	vmdev->out = NULL;
+	vmdev->in = NULL;
+
+	/* Free packet buffer */
+	free ( vmdev->packet );
+	vmdev->packet = NULL;
+
+	DBGC ( vmdev, "VMBUS %s closed\n", vmdev->dev.name );
+}
+
+/**
+ * Signal channel via monitor page
+ *
+ * @v vmdev		VMBus device
+ */
+static void vmbus_signal_monitor ( struct vmbus_device *vmdev ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	struct hv_monitor_trigger *trigger;
+	unsigned int group;
+	unsigned int bit;
+
+	/* Set bit in monitor trigger group */
+	group = ( vmdev->monitor / ( 8 * sizeof ( trigger->pending ) ));
+	bit = ( vmdev->monitor % ( 8 * sizeof ( trigger->pending ) ) );
+	trigger = &vmbus->monitor_out->trigger[group];
+	hv_set_bit ( trigger, bit );
+}
+
+/**
+ * Signal channel via hypervisor event
+ *
+ * @v vmdev		VMBus device
+ */
+static void vmbus_signal_event ( struct vmbus_device *vmdev ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	int rc;
+
+	/* Signal hypervisor event */
+	if ( ( rc = hv_signal_event ( hv, VMBUS_EVENT_ID, 0 ) ) != 0 ) {
+		DBGC ( vmdev, "VMBUS %s could not signal event: %s\n",
+		       vmdev->dev.name, strerror ( rc ) );
+		return;
+	}
+}
+
+/**
+ * Fill outbound ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v prod		Producer index
+ * @v data		Data
+ * @v len		Length
+ * @ret prod		New producer index
+ *
+ * The caller must ensure that there is sufficient space in the ring
+ * buffer.
+ */
+static size_t vmbus_produce ( struct vmbus_device *vmdev, size_t prod,
+			      const void *data, size_t len ) {
+	size_t first;
+	size_t second;
+
+	/* Determine fragment lengths */
+	first = ( vmdev->out_len - prod );
+	if ( first > len )
+		first = len;
+	second = ( len - first );
+
+	/* Copy fragment(s) */
+	memcpy ( &vmdev->out->data[prod], data, first );
+	if ( second )
+		memcpy ( &vmdev->out->data[0], ( data + first ), second );
+
+	return ( ( prod + len ) & ( vmdev->out_len - 1 ) );
+}
+
+/**
+ * Consume inbound ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v cons		Consumer index
+ * @v data		Data buffer, or NULL
+ * @v len		Length to consume
+ * @ret cons		New consumer index
+ */
+static size_t vmbus_consume ( struct vmbus_device *vmdev, size_t cons,
+			      void *data, size_t len ) {
+	size_t first;
+	size_t second;
+
+	/* Determine fragment lengths */
+	first = ( vmdev->in_len - cons );
+	if ( first > len )
+		first = len;
+	second = ( len - first );
+
+	/* Copy fragment(s) */
+	memcpy ( data, &vmdev->in->data[cons], first );
+	if ( second )
+		memcpy ( ( data + first ), &vmdev->in->data[0], second );
+
+	return ( ( cons + len ) & ( vmdev->in_len - 1 ) );
+}
+
+/**
+ * Send packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v header		Packet header
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ *
+ * Send a packet via the outbound ring buffer.  All fields in the
+ * packet header must be filled in, with the exception of the total
+ * packet length.
+ */
+static int vmbus_send ( struct vmbus_device *vmdev,
+			struct vmbus_packet_header *header,
+			const void *data, size_t len ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	static uint8_t padding[ 8 - 1 ];
+	struct vmbus_packet_footer footer;
+	size_t header_len;
+	size_t pad_len;
+	size_t footer_len;
+	size_t ring_len;
+	size_t cons;
+	size_t prod;
+	size_t old_prod;
+	size_t fill;
+
+	/* Sanity check */
+	assert ( vmdev->out != NULL );
+
+	/* Calculate lengths */
+	header_len = ( le16_to_cpu ( header->hdr_qlen ) * 8 );
+	pad_len = ( ( -len ) & ( 8 - 1 ) );
+	footer_len = sizeof ( footer );
+	ring_len = ( header_len + len + pad_len + footer_len );
+
+	/* Check that we have enough room in the outbound ring buffer */
+	cons = le32_to_cpu ( vmdev->out->cons );
+	prod = le32_to_cpu ( vmdev->out->prod );
+	old_prod = prod;
+	fill = ( ( prod - cons ) & ( vmdev->out_len - 1 ) );
+	if ( ( fill + ring_len ) >= vmdev->out_len ) {
+		DBGC ( vmdev, "VMBUS %s ring buffer full\n", vmdev->dev.name );
+		return -ENOBUFS;
+	}
+
+	/* Complete header */
+	header->qlen = cpu_to_le16 ( ( ring_len - footer_len ) / 8 );
+
+	/* Construct footer */
+	footer.reserved = 0;
+	footer.prod = vmdev->out->prod;
+
+	/* Copy packet to buffer */
+	DBGC2 ( vmdev, "VMBUS %s sending:\n", vmdev->dev.name );
+	DBGC2_HDA ( vmdev, prod, header, header_len );
+	prod = vmbus_produce ( vmdev, prod, header, header_len );
+	DBGC2_HDA ( vmdev, prod, data, len );
+	prod = vmbus_produce ( vmdev, prod, data, len );
+	prod = vmbus_produce ( vmdev, prod, padding, pad_len );
+	DBGC2_HDA ( vmdev, prod, &footer, sizeof ( footer ) );
+	prod = vmbus_produce ( vmdev, prod, &footer, sizeof ( footer ) );
+	assert ( ( ( prod - old_prod ) & ( vmdev->out_len - 1 ) ) == ring_len );
+
+	/* Update producer index */
+	wmb();
+	vmdev->out->prod = cpu_to_le32 ( prod );
+
+	/* Return if we do not need to signal the host.  This follows
+	 * the logic of hv_need_to_signal() in the Linux driver.
+	 */
+	mb();
+	if ( vmdev->out->intr_mask )
+		return 0;
+	rmb();
+	cons = le32_to_cpu ( vmdev->out->cons );
+	if ( cons != old_prod )
+		return 0;
+
+	/* Set channel bit in interrupt page */
+	hv_set_bit ( vmbus->intr->out, vmdev->channel );
+
+	/* Signal the host */
+	vmdev->signal ( vmdev );
+
+	return 0;
+}
+
+/**
+ * Send control packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID (or zero to not request completion)
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_DATA_INBAND packet.
+ */
+int vmbus_send_control ( struct vmbus_device *vmdev, uint64_t xid,
+			 const void *data, size_t len ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+
+	/* Construct header in packet buffer */
+	assert ( header != NULL );
+	header->type = cpu_to_le16 ( VMBUS_DATA_INBAND );
+	header->hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->flags = ( xid ?
+			  cpu_to_le16 ( VMBUS_COMPLETION_REQUESTED ) : 0 );
+	header->xid = xid; /* Non-endian */
+
+	return vmbus_send ( vmdev, header, data, len );
+}
+
+/**
+ * Send data packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_DATA_GPA_DIRECT packet.  The caller is
+ * responsible for ensuring that the I/O buffer remains untouched
+ * until the corresponding completion has been received.
+ */
+int vmbus_send_data ( struct vmbus_device *vmdev, uint64_t xid,
+		      const void *data, size_t len, struct io_buffer *iobuf ) {
+	physaddr_t addr = virt_to_phys ( iobuf->data );
+	unsigned int pfn_count = hv_pfn_count ( addr, iob_len ( iobuf ) );
+	struct {
+		struct vmbus_gpa_direct_header gpa;
+		struct vmbus_gpa_range range;
+		uint64_t pfn[pfn_count];
+	} __attribute__ (( packed )) *header = vmdev->packet;
+	unsigned int i;
+
+	/* Sanity check */
+	assert ( header != NULL );
+	assert ( sizeof ( *header ) <= vmdev->mtu );
+
+	/* Construct header in packet buffer */
+	header->gpa.header.type = cpu_to_le16 ( VMBUS_DATA_GPA_DIRECT );
+	header->gpa.header.hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->gpa.header.flags = cpu_to_le16 ( VMBUS_COMPLETION_REQUESTED );
+	header->gpa.header.xid = xid; /* Non-endian */
+	header->gpa.range_count = 1;
+	header->range.len = cpu_to_le32 ( iob_len ( iobuf ) );
+	header->range.offset = cpu_to_le32 ( addr & ( PAGE_SIZE - 1 ) );
+	for ( i = 0 ; i < pfn_count ; i++ )
+		header->pfn[i] = ( ( addr / PAGE_SIZE ) + i );
+
+	return vmbus_send ( vmdev, &header->gpa.header, data, len );
+}
+
+/**
+ * Send completion packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_COMPLETION packet.
+ */
+int vmbus_send_completion ( struct vmbus_device *vmdev, uint64_t xid,
+			    const void *data, size_t len ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+
+	/* Construct header in packet buffer */
+	assert ( header != NULL );
+	header->type = cpu_to_le16 ( VMBUS_COMPLETION );
+	header->hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->flags = 0;
+	header->xid = xid; /* Non-endian */
+
+	return vmbus_send ( vmdev, header, data, len );
+}
+
+/**
+ * Send cancellation packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_CANCELLATION packet.
+ */
+int vmbus_send_cancellation ( struct vmbus_device *vmdev, uint64_t xid ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+
+	/* Construct header in packet buffer */
+	assert ( header != NULL );
+	header->type = cpu_to_le16 ( VMBUS_CANCELLATION );
+	header->hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->flags = 0;
+	header->xid = xid; /* Non-endian */
+
+	return vmbus_send ( vmdev, header, NULL, 0 );
+}
+
+/**
+ * Get transfer page set from pageset ID
+ *
+ * @v vmdev		VMBus device
+ * @v pageset		Page set ID (in protocol byte order)
+ * @ret pages		Page set, or NULL if not found
+ */
+static struct vmbus_xfer_pages * vmbus_xfer_pages ( struct vmbus_device *vmdev,
+						    uint16_t pageset ) {
+	struct vmbus_xfer_pages *pages;
+
+	/* Locate page set */
+	list_for_each_entry ( pages, &vmdev->pages, list ) {
+		if ( pages->pageset == pageset )
+			return pages;
+	}
+
+	DBGC ( vmdev, "VMBUS %s unrecognised page set ID %#04x\n",
+	       vmdev->dev.name, le16_to_cpu ( pageset ) );
+	return NULL;
+}
+
+/**
+ * Construct I/O buffer list from transfer pages
+ *
+ * @v vmdev		VMBus device
+ * @v header		Transfer page header
+ * @v list		I/O buffer list to populate
+ * @ret rc		Return status code
+ */
+static int vmbus_xfer_page_iobufs ( struct vmbus_device *vmdev,
+				    struct vmbus_packet_header *header,
+				    struct list_head *list ) {
+	struct vmbus_xfer_page_header *page_header =
+		container_of ( header, struct vmbus_xfer_page_header, header );
+	struct vmbus_xfer_pages *pages;
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+	size_t len;
+	size_t offset;
+	unsigned int range_count;
+	unsigned int i;
+	int rc;
+
+	/* Sanity check */
+	assert ( header->type == cpu_to_le16 ( VMBUS_DATA_XFER_PAGES ) );
+
+	/* Locate page set */
+	pages = vmbus_xfer_pages ( vmdev, page_header->pageset );
+	if ( ! pages ) {
+		rc = -ENOENT;
+		goto err_pages;
+	}
+
+	/* Allocate and populate I/O buffers */
+	range_count = le32_to_cpu ( page_header->range_count );
+	for ( i = 0 ; i < range_count ; i++ ) {
+
+		/* Parse header */
+		len = le32_to_cpu ( page_header->range[i].len );
+		offset = le32_to_cpu ( page_header->range[i].offset );
+
+		/* Allocate I/O buffer */
+		iobuf = alloc_iob ( len );
+		if ( ! iobuf ) {
+			DBGC ( vmdev, "VMBUS %s could not allocate %zd-byte "
+			       "I/O buffer\n", vmdev->dev.name, len );
+			rc = -ENOMEM;
+			goto err_alloc;
+		}
+
+		/* Add I/O buffer to list */
+		list_add ( &iobuf->list, list );
+
+		/* Populate I/O buffer */
+		if ( ( rc = pages->op->copy ( pages, iob_put ( iobuf, len ),
+					      offset, len ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not populate I/O buffer "
+			       "range [%zd,%zd): %s\n",
+			       vmdev->dev.name, offset, len, strerror ( rc ) );
+			goto err_copy;
+		}
+	}
+
+	return 0;
+
+ err_copy:
+ err_alloc:
+	list_for_each_entry_safe ( iobuf, tmp, list, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+ err_pages:
+	return rc;
+}
+
+/**
+ * Poll ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @ret rc		Return status code
+ */
+int vmbus_poll ( struct vmbus_device *vmdev ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+	struct list_head list;
+	void *data;
+	size_t header_len;
+	size_t len;
+	size_t footer_len;
+	size_t ring_len;
+	size_t cons;
+	size_t old_cons;
+	uint64_t xid;
+	int rc;
+
+	/* Sanity checks */
+	assert ( vmdev->packet != NULL );
+	assert ( vmdev->in != NULL );
+
+	/* Return immediately if buffer is empty */
+	if ( ! vmbus_has_data ( vmdev ) )
+		return 0;
+	cons = le32_to_cpu ( vmdev->in->cons );
+	old_cons = cons;
+
+	/* Consume (start of) header */
+	cons = vmbus_consume ( vmdev, cons, header, sizeof ( *header ) );
+
+	/* Parse and sanity check header */
+	header_len = ( le16_to_cpu ( header->hdr_qlen ) * 8 );
+	if ( header_len < sizeof ( *header ) ) {
+		DBGC ( vmdev, "VMBUS %s received underlength header (%zd "
+		       "bytes)\n", vmdev->dev.name, header_len );
+		return -EINVAL;
+	}
+	len = ( ( le16_to_cpu ( header->qlen ) * 8 ) - header_len );
+	footer_len = sizeof ( struct vmbus_packet_footer );
+	ring_len = ( header_len + len + footer_len );
+	if ( ring_len > vmdev->mtu ) {
+		DBGC ( vmdev, "VMBUS %s received overlength packet (%zd "
+		       "bytes)\n", vmdev->dev.name, ring_len );
+		return -ERANGE;
+	}
+	xid = le64_to_cpu ( header->xid );
+
+	/* Consume remainder of packet */
+	cons = vmbus_consume ( vmdev, cons,
+			       ( ( ( void * ) header ) + sizeof ( *header ) ),
+			       ( ring_len - sizeof ( *header ) ) );
+	DBGC2 ( vmdev, "VMBUS %s received:\n", vmdev->dev.name );
+	DBGC2_HDA ( vmdev, old_cons, header, ring_len );
+	assert ( ( ( cons - old_cons ) & ( vmdev->in_len - 1 ) ) == ring_len );
+
+	/* Allocate I/O buffers, if applicable */
+	INIT_LIST_HEAD ( &list );
+	if ( header->type == cpu_to_le16 ( VMBUS_DATA_XFER_PAGES ) ) {
+		if ( ( rc = vmbus_xfer_page_iobufs ( vmdev, header,
+						     &list ) ) != 0 )
+			return rc;
+	}
+
+	/* Update producer index */
+	rmb();
+	vmdev->in->cons = cpu_to_le32 ( cons );
+
+	/* Handle packet */
+	data = ( ( ( void * ) header ) + header_len );
+	switch ( header->type ) {
+
+	case cpu_to_le16 ( VMBUS_DATA_INBAND ) :
+		if ( ( rc = vmdev->op->recv_control ( vmdev, xid, data,
+						      len ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not handle control "
+			       "packet: %s\n",
+			       vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	case cpu_to_le16 ( VMBUS_DATA_XFER_PAGES ) :
+		if ( ( rc = vmdev->op->recv_data ( vmdev, xid, data, len,
+						   &list ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not handle data packet: "
+			       "%s\n", vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	case cpu_to_le16 ( VMBUS_COMPLETION ) :
+		if ( ( rc = vmdev->op->recv_completion ( vmdev, xid, data,
+							 len ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not handle completion: "
+			       "%s\n", vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	case cpu_to_le16 ( VMBUS_CANCELLATION ) :
+		if ( ( rc = vmdev->op->recv_cancellation ( vmdev, xid ) ) != 0){
+			DBGC ( vmdev, "VMBUS %s could not handle cancellation: "
+			       "%s\n", vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	default:
+		DBGC ( vmdev, "VMBUS %s unknown packet type %d\n",
+		       vmdev->dev.name, le16_to_cpu ( header->type ) );
+		return -ENOTSUP;
+	}
+
+	return 0;
+}
+
+/**
+ * Dump channel status (for debugging)
+ *
+ * @v vmdev		VMBus device
+ */
+void vmbus_dump_channel ( struct vmbus_device *vmdev ) {
+	size_t out_prod = le32_to_cpu ( vmdev->out->prod );
+	size_t out_cons = le32_to_cpu ( vmdev->out->cons );
+	size_t in_prod = le32_to_cpu ( vmdev->in->prod );
+	size_t in_cons = le32_to_cpu ( vmdev->in->cons );
+	size_t in_len;
+	size_t first;
+	size_t second;
+
+	/* Dump ring status */
+	DBGC ( vmdev, "VMBUS %s out %03zx:%03zx%s in %03zx:%03zx%s\n",
+	       vmdev->dev.name, out_prod, out_cons,
+	       ( vmdev->out->intr_mask ? "(m)" : "" ), in_prod, in_cons,
+	       ( vmdev->in->intr_mask ? "(m)" : "" ) );
+
+	/* Dump inbound ring contents, if any */
+	if ( in_prod != in_cons ) {
+		in_len = ( ( in_prod - in_cons ) &
+			   ( vmdev->in_len - 1 ) );
+		first = ( vmdev->in_len - in_cons );
+		if ( first > in_len )
+			first = in_len;
+		second = ( in_len - first );
+		DBGC_HDA ( vmdev, in_cons, &vmdev->in->data[in_cons], first );
+		DBGC_HDA ( vmdev, 0, &vmdev->in->data[0], second );
+	}
+}
+
+/**
+ * Find driver for VMBus device
+ *
+ * @v vmdev		VMBus device
+ * @ret driver		Driver, or NULL
+ */
+static struct vmbus_driver * vmbus_find_driver ( const union uuid *type ) {
+	struct vmbus_driver *vmdrv;
+
+	for_each_table_entry ( vmdrv, VMBUS_DRIVERS ) {
+		if ( memcmp ( &vmdrv->type, type, sizeof ( *type ) ) == 0 )
+			return vmdrv;
+	}
+	return NULL;
+}
+
+/**
+ * Probe channels
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ * @ret rc		Return status code
+ */
+static int vmbus_probe_channels ( struct hv_hypervisor *hv,
+				  struct device *parent ) {
+	struct vmbus *vmbus = hv->vmbus;
+	const struct vmbus_message_header *header = &vmbus->message->header;
+	const struct vmbus_offer_channel *offer = &vmbus->message->offer;
+	const union uuid *type;
+	struct vmbus_driver *driver;
+	struct vmbus_device *vmdev;
+	struct vmbus_device *tmp;
+	unsigned int channel;
+	int rc;
+
+	/* Post message */
+	if ( ( rc = vmbus_post_empty_message ( hv, VMBUS_REQUEST_OFFERS ) ) !=0)
+		goto err_post_message;
+
+	/* Collect responses */
+	while ( 1 ) {
+
+		/* Wait for response */
+		if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+			goto err_wait_for_message;
+
+		/* Handle response */
+		if ( header->type == cpu_to_le32 ( VMBUS_OFFER_CHANNEL ) ) {
+
+			/* Parse offer */
+			type = &offer->type;
+			channel = le32_to_cpu ( offer->channel );
+			DBGC2 ( vmbus, "VMBUS %p offer %d type %s",
+				vmbus, channel, uuid_ntoa ( type ) );
+			if ( offer->monitored )
+				DBGC2 ( vmbus, " monitor %d", offer->monitor );
+			DBGC2 ( vmbus, "\n" );
+
+			/* Look for a driver */
+			driver = vmbus_find_driver ( type );
+			if ( ! driver ) {
+				DBGC2 ( vmbus, "VMBUS %p has no driver for "
+					"type %s\n", vmbus, uuid_ntoa ( type ));
+				/* Not a fatal error */
+				continue;
+			}
+
+			/* Allocate and initialise device */
+			vmdev = zalloc ( sizeof ( *vmdev ) );
+			if ( ! vmdev ) {
+				rc = -ENOMEM;
+				goto err_alloc_vmdev;
+			}
+			snprintf ( vmdev->dev.name, sizeof ( vmdev->dev.name ),
+				   "vmbus:%02x", channel );
+			vmdev->dev.desc.bus_type = BUS_TYPE_HV;
+			INIT_LIST_HEAD ( &vmdev->dev.children );
+			list_add_tail ( &vmdev->dev.siblings,
+					&parent->children );
+			vmdev->dev.parent = parent;
+			vmdev->hv = hv;
+			vmdev->channel = channel;
+			vmdev->monitor = offer->monitor;
+			vmdev->signal = ( offer->monitored ?
+					  vmbus_signal_monitor :
+					  vmbus_signal_event );
+			INIT_LIST_HEAD ( &vmdev->pages );
+			vmdev->driver = driver;
+			vmdev->dev.driver_name = driver->name;
+			DBGC ( vmdev, "VMBUS %s has driver \"%s\"\n",
+			       vmdev->dev.name, vmdev->driver->name );
+
+		} else if ( header->type ==
+			    cpu_to_le32 ( VMBUS_ALL_OFFERS_DELIVERED ) ) {
+
+			break;
+
+		} else {
+			DBGC ( vmbus, "VMBUS %p unexpected offer response type "
+			       "%d\n", vmbus, le32_to_cpu ( header->type ) );
+			rc = -EPROTO;
+			goto err_unexpected_offer;
+		}
+	}
+
+	/* Probe all devices.  We do this only after completing
+	 * enumeration since devices will need to send and receive
+	 * VMBus messages.
+	 */
+	list_for_each_entry ( vmdev, &parent->children, dev.siblings ) {
+		if ( ( rc = vmdev->driver->probe ( vmdev ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not probe: %s\n",
+			       vmdev->dev.name, strerror ( rc ) );
+			goto err_probe;
+		}
+	}
+
+	return 0;
+
+ err_probe:
+	/* Remove driver from each device that was already probed */
+	list_for_each_entry_continue_reverse ( vmdev, &parent->children,
+					       dev.siblings ) {
+		vmdev->driver->remove ( vmdev );
+	}
+ err_unexpected_offer:
+ err_alloc_vmdev:
+ err_wait_for_message:
+	/* Free any devices allocated (but potentially not yet probed) */
+	list_for_each_entry_safe ( vmdev, tmp, &parent->children,
+				   dev.siblings ) {
+		list_del ( &vmdev->dev.siblings );
+		free ( vmdev );
+	}
+ err_post_message:
+	return rc;
+}
+
+/**
+ * Remove channels
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ */
+static void vmbus_remove_channels ( struct hv_hypervisor *hv __unused,
+				    struct device *parent ) {
+	struct vmbus_device *vmdev;
+	struct vmbus_device *tmp;
+
+	/* Remove devices */
+	list_for_each_entry_safe ( vmdev, tmp, &parent->children,
+				   dev.siblings ) {
+		vmdev->driver->remove ( vmdev );
+		assert ( list_empty ( &vmdev->dev.children ) );
+		assert ( vmdev->out == NULL );
+		assert ( vmdev->in == NULL );
+		assert ( vmdev->packet == NULL );
+		assert ( list_empty ( &vmdev->pages ) );
+		list_del ( &vmdev->dev.siblings );
+		free ( vmdev );
+	}
+}
+
+/**
+ * Probe Hyper-V virtual machine bus
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ * @ret rc		Return status code
+ */
+int vmbus_probe ( struct hv_hypervisor *hv, struct device *parent ) {
+	struct vmbus *vmbus;
+	int rc;
+
+	/* Allocate and initialise structure */
+	vmbus = zalloc ( sizeof ( *vmbus ) );
+	if ( ! vmbus ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	hv->vmbus = vmbus;
+
+	/* Initialise message buffer pointer
+	 *
+	 * We use a pointer to the fixed-size Hyper-V received message
+	 * buffer.  This allows us to access fields within received
+	 * messages without first checking the message size: any
+	 * fields beyond the end of the message will read as zero.
+	 */
+	vmbus->message = ( ( void * ) hv->message->received.data );
+	assert ( sizeof ( *vmbus->message ) <=
+		 sizeof ( hv->message->received.data ) );
+
+	/* Allocate interrupt and monitor pages */
+	if ( ( rc = hv_alloc_pages ( hv, &vmbus->intr, &vmbus->monitor_in,
+				     &vmbus->monitor_out, NULL ) ) != 0 )
+		goto err_alloc_pages;
+
+	/* Enable message interrupt */
+	hv_enable_sint ( hv, VMBUS_MESSAGE_SINT );
+
+	/* Negotiate protocol version */
+	if ( ( rc = vmbus_negotiate_version ( hv ) ) != 0 )
+		goto err_negotiate_version;
+
+	/* Enumerate channels */
+	if ( ( rc = vmbus_probe_channels ( hv, parent ) ) != 0 )
+		goto err_probe_channels;
+
+	return 0;
+
+	vmbus_remove_channels ( hv, parent );
+ err_probe_channels:
+	vmbus_unload ( hv );
+ err_negotiate_version:
+	hv_disable_sint ( hv, VMBUS_MESSAGE_SINT );
+	hv_free_pages ( hv, vmbus->intr, vmbus->monitor_in, vmbus->monitor_out,
+			NULL );
+ err_alloc_pages:
+	free ( vmbus );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove Hyper-V virtual machine bus
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ */
+void vmbus_remove ( struct hv_hypervisor *hv, struct device *parent ) {
+	struct vmbus *vmbus = hv->vmbus;
+
+	vmbus_remove_channels ( hv, parent );
+	vmbus_unload ( hv );
+	hv_disable_sint ( hv, VMBUS_MESSAGE_SINT );
+	hv_free_pages ( hv, vmbus->intr, vmbus->monitor_in, vmbus->monitor_out,
+			NULL );
+	free ( vmbus );
+}
diff --git a/roms/ipxe/src/interface/linux/linux_entropy.c b/roms/ipxe/src/interface/linux/linux_entropy.c
index 4671a48da..0f8e45d36 100644
--- a/roms/ipxe/src/interface/linux/linux_entropy.c
+++ b/roms/ipxe/src/interface/linux/linux_entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/interface/linux/linux_pci.c b/roms/ipxe/src/interface/linux/linux_pci.c
index cbd825c18..0c140cb89 100644
--- a/roms/ipxe/src/interface/linux/linux_pci.c
+++ b/roms/ipxe/src/interface/linux/linux_pci.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/interface/linux/linux_time.c b/roms/ipxe/src/interface/linux/linux_time.c
index e3cbafec6..9e99fe9cd 100644
--- a/roms/ipxe/src/interface/linux/linux_time.c
+++ b/roms/ipxe/src/interface/linux/linux_time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/interface/linux/linux_uaccess.c b/roms/ipxe/src/interface/linux/linux_uaccess.c
index 5ab0b6b65..ea2d8057c 100644
--- a/roms/ipxe/src/interface/linux/linux_uaccess.c
+++ b/roms/ipxe/src/interface/linux/linux_uaccess.c
@@ -27,7 +27,6 @@ FILE_LICENCE(GPL2_OR_LATER);
  *
  */
 
-PROVIDE_UACCESS_INLINE(linux, phys_to_user);
 PROVIDE_UACCESS_INLINE(linux, user_to_phys);
 PROVIDE_UACCESS_INLINE(linux, virt_to_user);
 PROVIDE_UACCESS_INLINE(linux, user_to_virt);
diff --git a/roms/ipxe/src/interface/smbios/smbios.c b/roms/ipxe/src/interface/smbios/smbios.c
index 856943428..1dcf819c2 100644
--- a/roms/ipxe/src/interface/smbios/smbios.c
+++ b/roms/ipxe/src/interface/smbios/smbios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/interface/smbios/smbios_settings.c b/roms/ipxe/src/interface/smbios/smbios_settings.c
index 83e4320e9..5eadfa081 100644
--- a/roms/ipxe/src/interface/smbios/smbios_settings.c
+++ b/roms/ipxe/src/interface/smbios/smbios_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/roms/ipxe/src/interface/xen/xenbus.c b/roms/ipxe/src/interface/xen/xenbus.c
index ffc8aba3e..c328af443 100644
--- a/roms/ipxe/src/interface/xen/xenbus.c
+++ b/roms/ipxe/src/interface/xen/xenbus.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/interface/xen/xengrant.c b/roms/ipxe/src/interface/xen/xengrant.c
index be12b23dc..269cd5836 100644
--- a/roms/ipxe/src/interface/xen/xengrant.c
+++ b/roms/ipxe/src/interface/xen/xengrant.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <strings.h>
diff --git a/roms/ipxe/src/interface/xen/xenstore.c b/roms/ipxe/src/interface/xen/xenstore.c
index b96982927..23424a926 100644
--- a/roms/ipxe/src/interface/xen/xenstore.c
+++ b/roms/ipxe/src/interface/xen/xenstore.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
@@ -238,6 +242,10 @@ static int xenstore_response ( struct xen_hypervisor *xen, uint32_t req_id,
 	char *string;
 	int rc;
 
+	/* Wait for response to become available */
+	while ( ! xenevent_pending ( xen, xen->store.port ) )
+		cpu_nap();
+
 	/* Receive message header */
 	xenstore_recv ( xen, &msg, sizeof ( msg ) );
 	*len = msg.len;
diff --git a/roms/ipxe/src/net/80211/net80211.c b/roms/ipxe/src/net/80211/net80211.c
index 434944523..d4970ad5c 100644
--- a/roms/ipxe/src/net/80211/net80211.c
+++ b/roms/ipxe/src/net/80211/net80211.c
@@ -805,6 +805,10 @@ int net80211_register ( struct net80211_device *dev,
 		 NET80211_MAX_CHANNELS * sizeof ( dev->channels[0] ) );
 	dev->channel = 0;
 
+	/* Mark device as not supporting interrupts, if applicable */
+	if ( ! ops->irq )
+		dev->netdev->state |= NETDEV_IRQ_UNSUPPORTED;
+
 	list_add_tail ( &dev->list, &net80211_devices );
 	return register_netdev ( dev->netdev );
 }
@@ -2826,3 +2830,9 @@ struct errortab common_wireless_errors[] __errortab = {
 	__einfo_errortab ( EINFO_ECONNREFUSED_ASSOC_DENIED ),
 	__einfo_errortab ( EINFO_ECONNREFUSED_AUTH_ALGO_UNSUPP ),
 };
+
+/* Drag in objects via net80211_ll_protocol */
+REQUIRING_SYMBOL ( net80211_ll_protocol );
+
+/* Drag in 802.11 configuration */
+REQUIRE_OBJECT ( config_net80211 );
diff --git a/roms/ipxe/src/net/80211/wpa.c b/roms/ipxe/src/net/80211/wpa.c
index e2c4945f9..77f66d825 100644
--- a/roms/ipxe/src/net/80211/wpa.c
+++ b/roms/ipxe/src/net/80211/wpa.c
@@ -912,4 +912,5 @@ struct eapol_handler eapol_key_handler __eapol_handler = {
 };
 
 /* WPA always needs EAPOL in order to be useful */
+REQUIRING_SYMBOL ( eapol_key_handler );
 REQUIRE_OBJECT ( eapol );
diff --git a/roms/ipxe/src/net/80211/wpa_ccmp.c b/roms/ipxe/src/net/80211/wpa_ccmp.c
index f98ebea26..a073c6a3c 100644
--- a/roms/ipxe/src/net/80211/wpa_ccmp.c
+++ b/roms/ipxe/src/net/80211/wpa_ccmp.c
@@ -480,7 +480,7 @@ static void ccmp_kie_mic ( const void *kck, const void *msg, size_t len,
 {
 	u8 sha1_ctx[SHA1_CTX_SIZE];
 	u8 kckb[16];
-	u8 hash[SHA1_SIZE];
+	u8 hash[SHA1_DIGEST_SIZE];
 	size_t kck_len = 16;
 
 	memcpy ( kckb, kck, kck_len );
diff --git a/roms/ipxe/src/net/80211/wpa_tkip.c b/roms/ipxe/src/net/80211/wpa_tkip.c
index fa3e0763b..3b1934b59 100644
--- a/roms/ipxe/src/net/80211/wpa_tkip.c
+++ b/roms/ipxe/src/net/80211/wpa_tkip.c
@@ -136,7 +136,7 @@ static const u16 Sbox[256] = {
  */
 static inline u16 S ( u16 v )
 {
-	return Sbox[v & 0xFF] ^ swap16 ( Sbox[v >> 8] );
+	return Sbox[v & 0xFF] ^ bswap_16 ( Sbox[v >> 8] );
 }
 
 /**
diff --git a/roms/ipxe/src/net/aoe.c b/roms/ipxe/src/net/aoe.c
index a6d7b3e7b..2da8655b4 100644
--- a/roms/ipxe/src/net/aoe.c
+++ b/roms/ipxe/src/net/aoe.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
diff --git a/roms/ipxe/src/net/arp.c b/roms/ipxe/src/net/arp.c
index 261e681e1..1e27c44e7 100644
--- a/roms/ipxe/src/net/arp.c
+++ b/roms/ipxe/src/net/arp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -52,9 +56,9 @@ struct net_protocol arp_protocol __net_protocol;
  * @v net_source	Source network-layer address
  * @ret rc		Return status code
  */
-static int arp_tx_request ( struct net_device *netdev,
-			    struct net_protocol *net_protocol,
-			    const void *net_dest, const void *net_source ) {
+int arp_tx_request ( struct net_device *netdev,
+		     struct net_protocol *net_protocol,
+		     const void *net_dest, const void *net_source ) {
 	struct ll_protocol *ll_protocol = netdev->ll_protocol;
 	struct io_buffer *iobuf;
 	struct arphdr *arphdr;
diff --git a/roms/ipxe/src/net/dhcpopts.c b/roms/ipxe/src/net/dhcpopts.c
index 8cd19cf80..cdb632b46 100644
--- a/roms/ipxe/src/net/dhcpopts.c
+++ b/roms/ipxe/src/net/dhcpopts.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/dhcppkt.c b/roms/ipxe/src/net/dhcppkt.c
index a9a6d3a94..4e64f85e4 100644
--- a/roms/ipxe/src/net/dhcppkt.c
+++ b/roms/ipxe/src/net/dhcppkt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/eth_slow.c b/roms/ipxe/src/net/eth_slow.c
index db54b55a4..049c26cb3 100644
--- a/roms/ipxe/src/net/eth_slow.c
+++ b/roms/ipxe/src/net/eth_slow.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/roms/ipxe/src/net/ethernet.c b/roms/ipxe/src/net/ethernet.c
index 03978c2a8..6ddf05344 100644
--- a/roms/ipxe/src/net/ethernet.c
+++ b/roms/ipxe/src/net/ethernet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -43,6 +47,24 @@ FILE_LICENCE ( GPL2_OR_LATER );
 uint8_t eth_broadcast[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
 /**
+ * Check if Ethernet packet has an 802.3 LLC header
+ *
+ * @v ethhdr		Ethernet header
+ * @ret is_llc		Packet has 802.3 LLC header
+ */
+static inline int eth_is_llc_packet ( struct ethhdr *ethhdr ) {
+	uint8_t len_msb;
+
+	/* Check if the protocol field contains a value short enough
+	 * to be a frame length.  The slightly convoluted form of the
+	 * comparison is designed to reduce to a single x86
+	 * instruction.
+	 */
+	len_msb = *( ( uint8_t * ) &ethhdr->h_protocol );
+	return ( len_msb < 0x06 );
+}
+
+/**
  * Add Ethernet link-layer header
  *
  * @v netdev		Network device
@@ -80,9 +102,14 @@ int eth_pull ( struct net_device *netdev __unused, struct io_buffer *iobuf,
 	       const void **ll_dest, const void **ll_source,
 	       uint16_t *net_proto, unsigned int *flags ) {
 	struct ethhdr *ethhdr = iobuf->data;
+	uint16_t *llc_proto;
 
-	/* Sanity check */
-	if ( iob_len ( iobuf ) < sizeof ( *ethhdr ) ) {
+	/* Sanity check.  While in theory we could receive a one-byte
+	 * packet, this will never happen in practice and performing
+	 * the combined length check here avoids the need for an
+	 * additional comparison if we detect an LLC frame.
+	 */
+	if ( iob_len ( iobuf ) < ( sizeof ( *ethhdr ) + sizeof ( *llc_proto ))){
 		DBG ( "Ethernet packet too short (%zd bytes)\n",
 		      iob_len ( iobuf ) );
 		return -EINVAL;
@@ -100,6 +127,17 @@ int eth_pull ( struct net_device *netdev __unused, struct io_buffer *iobuf,
 		   ( is_broadcast_ether_addr ( ethhdr->h_dest ) ?
 		     LL_BROADCAST : 0 ) );
 
+	/* If this is an LLC frame (with a length in place of the
+	 * protocol field), then use the next two bytes (which happen
+	 * to be the LLC DSAP and SSAP) as the protocol.  This allows
+	 * for minimal-overhead support for receiving (rare) LLC
+	 * frames, without requiring a full LLC protocol layer.
+	 */
+	if ( eth_is_llc_packet ( ethhdr ) ) {
+		llc_proto = ( &ethhdr->h_protocol + 1 );
+		*net_proto = *llc_proto;
+	}
+
 	return 0;
 }
 
@@ -235,5 +273,11 @@ struct net_device * alloc_etherdev ( size_t priv_size ) {
 	return netdev;
 }
 
+/* Drag in objects via ethernet_protocol */
+REQUIRING_SYMBOL ( ethernet_protocol );
+
+/* Drag in Ethernet configuration */
+REQUIRE_OBJECT ( config_ethernet );
+
 /* Drag in Ethernet slow protocols */
 REQUIRE_OBJECT ( eth_slow );
diff --git a/roms/ipxe/src/net/fakedhcp.c b/roms/ipxe/src/net/fakedhcp.c
index 3dec88b11..b6c456a59 100644
--- a/roms/ipxe/src/net/fakedhcp.c
+++ b/roms/ipxe/src/net/fakedhcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/fc.c b/roms/ipxe/src/net/fc.c
index 58008995c..2e8070272 100644
--- a/roms/ipxe/src/net/fc.c
+++ b/roms/ipxe/src/net/fc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
@@ -1935,3 +1939,9 @@ struct fc_ulp * fc_ulp_get_port_id_type ( struct fc_port *port,
  err_peer_get_wwn:
 	return NULL;
 }
+
+/* Drag in objects via fc_ports */
+REQUIRING_SYMBOL ( fc_ports );
+
+/* Drag in Fibre Channel configuration */
+REQUIRE_OBJECT ( config_fc );
diff --git a/roms/ipxe/src/net/fcels.c b/roms/ipxe/src/net/fcels.c
index 1cfe90727..5fc27cef4 100644
--- a/roms/ipxe/src/net/fcels.c
+++ b/roms/ipxe/src/net/fcels.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/fcns.c b/roms/ipxe/src/net/fcns.c
index 3ca4ad557..be4dfea24 100644
--- a/roms/ipxe/src/net/fcns.c
+++ b/roms/ipxe/src/net/fcns.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/fcoe.c b/roms/ipxe/src/net/fcoe.c
index e9e404ec3..c3258f15e 100644
--- a/roms/ipxe/src/net/fcoe.c
+++ b/roms/ipxe/src/net/fcoe.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/fcp.c b/roms/ipxe/src/net/fcp.c
index 9c36a4c72..930bf7dd4 100644
--- a/roms/ipxe/src/net/fcp.c
+++ b/roms/ipxe/src/net/fcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/roms/ipxe/src/net/fragment.c b/roms/ipxe/src/net/fragment.c
index 410915b3b..781b9bc60 100644
--- a/roms/ipxe/src/net/fragment.c
+++ b/roms/ipxe/src/net/fragment.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/icmp.c b/roms/ipxe/src/net/icmp.c
index 1bbf8bd30..5371277e4 100644
--- a/roms/ipxe/src/net/icmp.c
+++ b/roms/ipxe/src/net/icmp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <byteswap.h>
diff --git a/roms/ipxe/src/net/icmpv4.c b/roms/ipxe/src/net/icmpv4.c
index 996ba1490..0858ff37f 100644
--- a/roms/ipxe/src/net/icmpv4.c
+++ b/roms/ipxe/src/net/icmpv4.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/net/icmpv6.c b/roms/ipxe/src/net/icmpv6.c
index 479800e7d..8555aaf0b 100644
--- a/roms/ipxe/src/net/icmpv6.c
+++ b/roms/ipxe/src/net/icmpv6.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
@@ -34,6 +38,65 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+/* Disambiguate the various error causes */
+#define EHOSTUNREACH_ROUTE						\
+	__einfo_error ( EINFO_EHOSTUNREACH_ROUTE )
+#define EINFO_EHOSTUNREACH_ROUTE					\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 0,			\
+			  "No route to destination" )
+#define EHOSTUNREACH_PROHIBITED						\
+	__einfo_error ( EINFO_EHOSTUNREACH_PROHIBITED )
+#define EINFO_EHOSTUNREACH_PROHIBITED					\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 1,			\
+			  "Communication administratively prohibited" )
+#define EHOSTUNREACH_ADDRESS						\
+	__einfo_error ( EINFO_EHOSTUNREACH_ADDRESS )
+#define EINFO_EHOSTUNREACH_ADDRESS					\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 3,			\
+			  "Address unreachable" )
+#define EHOSTUNREACH_PORT						\
+	__einfo_error ( EINFO_EHOSTUNREACH_PORT )
+#define EINFO_EHOSTUNREACH_PORT						\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 4,			\
+			  "Port unreachable" )
+#define EHOSTUNREACH_CODE( code )					\
+	EUNIQ ( EINFO_EHOSTUNREACH, ( (code) & 0x1f ),			\
+		EHOSTUNREACH_ROUTE, EHOSTUNREACH_PROHIBITED,		\
+		EHOSTUNREACH_ADDRESS, EHOSTUNREACH_PORT )
+
+#define ETIMEDOUT_HOP							\
+	__einfo_error ( EINFO_ETIMEDOUT_HOP )
+#define EINFO_ETIMEDOUT_HOP						\
+	__einfo_uniqify ( EINFO_ETIMEDOUT, 0,				\
+			  "Hop limit exceeded in transit" )
+#define ETIMEDOUT_REASSEMBLY						\
+	__einfo_error ( EINFO_ETIMEDOUT_REASSEMBLY )
+#define EINFO_ETIMEDOUT_REASSEMBLY					\
+	__einfo_uniqify ( EINFO_ETIMEDOUT, 1,				\
+			  "Fragment reassembly time exceeded" )
+#define ETIMEDOUT_CODE( code )						\
+	EUNIQ ( EINFO_ETIMEDOUT, ( (code) & 0x1f ),			\
+		ETIMEDOUT_HOP, ETIMEDOUT_REASSEMBLY )
+
+#define EPROTO_BAD_HEADER						\
+	__einfo_error ( EINFO_EPROTO_BAD_HEADER )
+#define EINFO_EPROTO_BAD_HEADER						\
+	__einfo_uniqify ( EINFO_EPROTO, 0,				\
+			  "Erroneous header field" )
+#define EPROTO_NEXT_HEADER						\
+	__einfo_error ( EINFO_EPROTO_NEXT_HEADER )
+#define EINFO_EPROTO_NEXT_HEADER					\
+	__einfo_uniqify ( EINFO_EPROTO, 1,				\
+			  "Unrecognised next header type" )
+#define EPROTO_OPTION							\
+	__einfo_error ( EINFO_EPROTO_OPTION )
+#define EINFO_EPROTO_OPTION						\
+	__einfo_uniqify ( EINFO_EPROTO, 2,				\
+			  "Unrecognised IPv6 option" )
+#define EPROTO_CODE( code )						\
+	EUNIQ ( EINFO_EPROTO, ( (code) & 0x1f ),			\
+		EPROTO_BAD_HEADER, EPROTO_NEXT_HEADER, EPROTO_OPTION )
+
 struct icmp_echo_protocol icmpv6_echo_protocol __icmp_echo_protocol;
 
 /**
@@ -144,8 +207,25 @@ static int icmpv6_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 	/* Identify handler */
 	handler = icmpv6_handler ( icmp->type );
 	if ( ! handler ) {
-		DBGC ( netdev, "ICMPv6 unrecognised type %d\n", icmp->type );
-		rc = -ENOTSUP;
+		switch ( icmp->type ) {
+		case ICMPV6_DESTINATION_UNREACHABLE:
+			rc = -EHOSTUNREACH_CODE ( icmp->code );
+			break;
+		case ICMPV6_PACKET_TOO_BIG:
+			rc = -ERANGE;
+			break;
+		case ICMPV6_TIME_EXCEEDED:
+			rc = -ETIMEDOUT_CODE ( icmp->code );
+			break;
+		case ICMPV6_PARAMETER_PROBLEM:
+			rc = -EPROTO_CODE ( icmp->code );
+			break;
+		default:
+			DBGC ( netdev, "ICMPv6 unrecognised type %d code %d\n",
+			       icmp->type, icmp->code );
+			rc = -ENOTSUP;
+			break;
+		};
 		goto done;
 	}
 
diff --git a/roms/ipxe/src/net/infiniband.c b/roms/ipxe/src/net/infiniband.c
index 12d1d83ce..2e3d76d54 100644
--- a/roms/ipxe/src/net/infiniband.c
+++ b/roms/ipxe/src/net/infiniband.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -714,6 +718,9 @@ int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	struct ib_multicast_gid *mgid;
 	int rc;
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Add to software multicast GID list */
 	mgid = zalloc ( sizeof ( *mgid ) );
 	if ( ! mgid ) {
@@ -747,6 +754,9 @@ void ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 		       union ib_gid *gid ) {
 	struct ib_multicast_gid *mgid;
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Remove from hardware multicast GID list */
 	ibdev->op->mcast_detach ( ibdev, qp, gid );
 
@@ -995,5 +1005,11 @@ struct ib_device * last_opened_ibdev ( void ) {
 	return ibdev;
 }
 
+/* Drag in objects via register_ibdev() */
+REQUIRING_SYMBOL ( register_ibdev );
+
+/* Drag in Infiniband configuration */
+REQUIRE_OBJECT ( config_infiniband );
+
 /* Drag in IPoIB */
 REQUIRE_OBJECT ( ipoib );
diff --git a/roms/ipxe/src/net/infiniband/ib_cm.c b/roms/ipxe/src/net/infiniband/ib_cm.c
index 797639bc8..85982f09d 100644
--- a/roms/ipxe/src/net/infiniband/ib_cm.c
+++ b/roms/ipxe/src/net/infiniband/ib_cm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/infiniband/ib_mcast.c b/roms/ipxe/src/net/infiniband/ib_mcast.c
index 0a5e72a37..fc4ff7f0a 100644
--- a/roms/ipxe/src/net/infiniband/ib_mcast.c
+++ b/roms/ipxe/src/net/infiniband/ib_mcast.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -146,6 +150,9 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	DBGC ( ibdev, "IBDEV %p QPN %lx joining " IB_GID_FMT "\n",
 	       ibdev, qp->qpn, IB_GID_ARGS ( gid ) );
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Initialise structure */
 	membership->qp = qp;
 	memcpy ( &membership->gid, gid, sizeof ( membership->gid ) );
@@ -195,6 +202,9 @@ void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	DBGC ( ibdev, "IBDEV %p QPN %lx leaving " IB_GID_FMT "\n",
 	       ibdev, qp->qpn, IB_GID_ARGS ( gid ) );
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Detach from multicast GID */
 	ib_mcast_detach ( ibdev, qp, &membership->gid );
 
diff --git a/roms/ipxe/src/net/infiniband/ib_mi.c b/roms/ipxe/src/net/infiniband/ib_mi.c
index ef6d539f1..b43212974 100644
--- a/roms/ipxe/src/net/infiniband/ib_mi.c
+++ b/roms/ipxe/src/net/infiniband/ib_mi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/infiniband/ib_packet.c b/roms/ipxe/src/net/infiniband/ib_packet.c
index 6c850e39b..d3a22d309 100644
--- a/roms/ipxe/src/net/infiniband/ib_packet.c
+++ b/roms/ipxe/src/net/infiniband/ib_packet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/infiniband/ib_pathrec.c b/roms/ipxe/src/net/infiniband/ib_pathrec.c
index 1b95cbfa8..f9cbab87f 100644
--- a/roms/ipxe/src/net/infiniband/ib_pathrec.c
+++ b/roms/ipxe/src/net/infiniband/ib_pathrec.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/infiniband/ib_sma.c b/roms/ipxe/src/net/infiniband/ib_sma.c
index 86553732a..a05d7c924 100644
--- a/roms/ipxe/src/net/infiniband/ib_sma.c
+++ b/roms/ipxe/src/net/infiniband/ib_sma.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/infiniband/ib_smc.c b/roms/ipxe/src/net/infiniband/ib_smc.c
index 4d947d568..c1741b26c 100644
--- a/roms/ipxe/src/net/infiniband/ib_smc.c
+++ b/roms/ipxe/src/net/infiniband/ib_smc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/infiniband/ib_srp.c b/roms/ipxe/src/net/infiniband/ib_srp.c
index 7b2b2b4ea..3700184c0 100644
--- a/roms/ipxe/src/net/infiniband/ib_srp.c
+++ b/roms/ipxe/src/net/infiniband/ib_srp.c
@@ -291,7 +291,7 @@ static int ib_srp_parse_byte_string ( const char *rp_comp, uint8_t *bytes,
 		return -EINVAL_BYTE_STRING_LEN;
 
 	/* Parse byte string */
-	decoded_size = base16_decode ( rp_comp, bytes );
+	decoded_size = base16_decode ( rp_comp, bytes, size );
 	if ( decoded_size < 0 )
 		return decoded_size;
 
diff --git a/roms/ipxe/src/net/iobpad.c b/roms/ipxe/src/net/iobpad.c
index 9cc8328e9..936b4bde4 100644
--- a/roms/ipxe/src/net/iobpad.c
+++ b/roms/ipxe/src/net/iobpad.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/roms/ipxe/src/net/ipv4.c b/roms/ipxe/src/net/ipv4.c
index 9c5cf2eb4..a54784049 100644
--- a/roms/ipxe/src/net/ipv4.c
+++ b/roms/ipxe/src/net/ipv4.c
@@ -1,3 +1,27 @@
+/*
+ * Copyright (C) 2006 Michael Brown <mbrown@fensystems.co.uk>.
+ * Copyright (C) 2006 Nikhil Chandru Rao
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
 #include <string.h>
 #include <stdint.h>
 #include <stdlib.h>
@@ -24,7 +48,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Unique IP datagram identification number (high byte) */
 static uint8_t next_ident_high = 0;
@@ -115,6 +139,7 @@ static void del_ipv4_miniroute ( struct ipv4_miniroute *miniroute ) {
 /**
  * Perform IPv4 routing
  *
+ * @v scope_id		Destination address scope ID
  * @v dest		Final destination address
  * @ret dest		Next hop destination address
  * @ret miniroute	Routing table entry to use, or NULL if no route
@@ -122,22 +147,42 @@ static void del_ipv4_miniroute ( struct ipv4_miniroute *miniroute ) {
  * If the route requires use of a gateway, the next hop destination
  * address will be overwritten with the gateway address.
  */
-static struct ipv4_miniroute * ipv4_route ( struct in_addr *dest ) {
+static struct ipv4_miniroute * ipv4_route ( unsigned int scope_id,
+					    struct in_addr *dest ) {
 	struct ipv4_miniroute *miniroute;
-	int local;
-	int has_gw;
 
 	/* Find first usable route in routing table */
 	list_for_each_entry ( miniroute, &ipv4_miniroutes, list ) {
+
+		/* Skip closed network devices */
 		if ( ! netdev_is_open ( miniroute->netdev ) )
 			continue;
-		local = ( ( ( dest->s_addr ^ miniroute->address.s_addr )
-			    & miniroute->netmask.s_addr ) == 0 );
-		has_gw = ( miniroute->gateway.s_addr );
-		if ( local || has_gw ) {
-			if ( ! local )
+
+		if ( IN_IS_MULTICAST ( dest->s_addr ) ) {
+
+			/* If destination is non-global, and the scope ID
+			 * matches this network device, then use this route.
+			 */
+			if ( miniroute->netdev->index == scope_id )
+				return miniroute;
+
+		} else {
+
+			/* If destination is an on-link global
+			 * address, then use this route.
+			 */
+			if ( ( ( dest->s_addr ^ miniroute->address.s_addr )
+			       & miniroute->netmask.s_addr ) == 0 )
+				return miniroute;
+
+			/* If destination is an off-link global
+			 * address, and we have a default gateway,
+			 * then use this route.
+			 */
+			if ( miniroute->gateway.s_addr ) {
 				*dest = miniroute->gateway;
-			return miniroute;
+				return miniroute;
+			}
 		}
 	}
 
@@ -156,7 +201,7 @@ static struct net_device * ipv4_netdev ( struct sockaddr_tcpip *st_dest ) {
 	struct ipv4_miniroute *miniroute;
 
 	/* Find routing table entry */
-	miniroute = ipv4_route ( &dest );
+	miniroute = ipv4_route ( sin_dest->sin_scope_id, &dest );
 	if ( ! miniroute )
 		return NULL;
 
@@ -290,8 +335,8 @@ static int ipv4_tx ( struct io_buffer *iobuf,
 	if ( sin_src )
 		iphdr->src = sin_src->sin_addr;
 	if ( ( next_hop.s_addr != INADDR_BROADCAST ) &&
-	     ( ! IN_MULTICAST ( ntohl ( next_hop.s_addr ) ) ) &&
-	     ( ( miniroute = ipv4_route ( &next_hop ) ) != NULL ) ) {
+	     ( ( miniroute = ipv4_route ( sin_dest->sin_scope_id,
+					  &next_hop ) ) != NULL ) ) {
 		iphdr->src = miniroute->address;
 		netmask = miniroute->netmask;
 		netdev = miniroute->netdev;
@@ -329,7 +374,7 @@ static int ipv4_tx ( struct io_buffer *iobuf,
 		/* Broadcast address */
 		ipv4_stats.out_bcast_pkts++;
 		ll_dest = netdev->ll_broadcast;
-	} else if ( IN_MULTICAST ( ntohl ( next_hop.s_addr ) ) ) {
+	} else if ( IN_IS_MULTICAST ( next_hop.s_addr ) ) {
 		/* Multicast address */
 		ipv4_stats.out_mcast_pkts++;
 		if ( ( rc = netdev->ll_protocol->mc_hash ( AF_INET, &next_hop,
@@ -569,10 +614,42 @@ static int ipv4_arp_check ( struct net_device *netdev, const void *net_addr ) {
 }
 
 /**
+ * Parse IPv4 address
+ *
+ * @v string		IPv4 address string
+ * @ret in		IPv4 address to fill in
+ * @ret ok		IPv4 address is valid
+ *
+ * Note that this function returns nonzero iff the address is valid,
+ * to match the standard BSD API function of the same name.  Unlike
+ * most other iPXE functions, a zero therefore indicates failure.
+ */
+int inet_aton ( const char *string, struct in_addr *in ) {
+	const char *separator = "...";
+	uint8_t *byte = ( ( uint8_t * ) in );
+	char *endp;
+	unsigned long value;
+
+	while ( 1 ) {
+		value = strtoul ( string, &endp, 0 );
+		if ( string == endp )
+			return 0;
+		if ( value > 0xff )
+			return 0;
+		*(byte++) = value;
+		if ( *endp != *separator )
+			return 0;
+		if ( ! *(separator++) )
+			return 1;
+		string = ( endp + 1 );
+	}
+}
+
+/**
  * Convert IPv4 address to dotted-quad notation
  *
- * @v in	IP address
- * @ret string	IP address in dotted-quad notation
+ * @v in		IPv4 address
+ * @ret string		IPv4 address in dotted-quad notation
  */
 char * inet_ntoa ( struct in_addr in ) {
 	static char buf[16]; /* "xxx.xxx.xxx.xxx" */
@@ -583,10 +660,10 @@ char * inet_ntoa ( struct in_addr in ) {
 }
 
 /**
- * Transcribe IP address
+ * Transcribe IPv4 address
  *
- * @v net_addr	IP address
- * @ret string	IP address in dotted-quad notation
+ * @v net_addr		IPv4 address
+ * @ret string		IPv4 address in dotted-quad notation
  *
  */
 static const char * ipv4_ntoa ( const void *net_addr ) {
@@ -760,12 +837,12 @@ static int ipv4_create_routes ( void ) {
 		fetch_ipv4_setting ( settings, &netmask_setting, &netmask );
 		/* Calculate default netmask, if necessary */
 		if ( ! netmask.s_addr ) {
-			if ( IN_CLASSA ( ntohl ( address.s_addr ) ) ) {
-				netmask.s_addr = htonl ( IN_CLASSA_NET );
-			} else if ( IN_CLASSB ( ntohl ( address.s_addr ) ) ) {
-				netmask.s_addr = htonl ( IN_CLASSB_NET );
-			} else if ( IN_CLASSC ( ntohl ( address.s_addr ) ) ) {
-				netmask.s_addr = htonl ( IN_CLASSC_NET );
+			if ( IN_IS_CLASSA ( address.s_addr ) ) {
+				netmask.s_addr = INADDR_NET_CLASSA;
+			} else if ( IN_IS_CLASSB ( address.s_addr ) ) {
+				netmask.s_addr = INADDR_NET_CLASSB;
+			} else if ( IN_IS_CLASSC ( address.s_addr ) ) {
+				netmask.s_addr = INADDR_NET_CLASSC;
 			}
 		}
 		/* Get default gateway, if present */
@@ -785,5 +862,8 @@ struct settings_applicator ipv4_settings_applicator __settings_applicator = {
 	.apply = ipv4_create_routes,
 };
 
+/* Drag in objects via ipv4_protocol */
+REQUIRING_SYMBOL ( ipv4_protocol );
+
 /* Drag in ICMPv4 */
 REQUIRE_OBJECT ( icmpv4 );
diff --git a/roms/ipxe/src/net/ipv6.c b/roms/ipxe/src/net/ipv6.c
index 3c374168c..a75e72ddb 100644
--- a/roms/ipxe/src/net/ipv6.c
+++ b/roms/ipxe/src/net/ipv6.c
@@ -290,8 +290,7 @@ static struct ipv6_miniroute * ipv6_route ( unsigned int scope_id,
 		if ( ! ( miniroute->flags & IPV6_HAS_ADDRESS ) )
 			continue;
 
-		if ( IN6_IS_ADDR_LINKLOCAL ( *dest ) ||
-		     IN6_IS_ADDR_MULTICAST ( *dest ) ) {
+		if ( IN6_IS_ADDR_NONGLOBAL ( *dest ) ) {
 
 			/* If destination is non-global, and the scope ID
 			 * matches this network device, then use this route.
@@ -901,7 +900,7 @@ static const char * ipv6_sock_ntoa ( struct sockaddr *sa ) {
 	const char *netdev_name;
 
 	/* Identify network device, if applicable */
-	if ( IN6_IS_ADDR_LINKLOCAL ( in ) || IN6_IS_ADDR_MULTICAST ( in ) ) {
+	if ( IN6_IS_ADDR_NONGLOBAL ( in ) ) {
 		netdev = find_netdev_by_index ( sin6->sin6_scope_id );
 		netdev_name = ( netdev ? netdev->name : "UNKNOWN" );
 	} else {
@@ -956,14 +955,26 @@ static int ipv6_sock_aton ( const char *string, struct sockaddr *sa ) {
 	if ( ( rc = inet6_aton ( in_string, &in ) ) != 0 )
 		goto err_inet6_aton;
 
-	/* Parse network device name, if present */
+	/* Parse scope ID, if applicable */
 	if ( netdev_string ) {
+
+		/* Parse explicit network device name, if present */
 		netdev = find_netdev ( netdev_string );
 		if ( ! netdev ) {
 			rc = -ENODEV;
 			goto err_find_netdev;
 		}
 		sin6->sin6_scope_id = netdev->index;
+
+	} else if ( IN6_IS_ADDR_NONGLOBAL ( &in ) ) {
+
+		/* If no network device is explicitly specified for a
+		 * link-local or multicast address, default to using
+		 * "netX" (if existent).
+		 */
+		netdev = last_opened_netdev();
+		if ( netdev )
+			sin6->sin6_scope_id = netdev->index;
 	}
 
 	/* Copy IPv6 address portion to socket address */
@@ -1104,6 +1115,9 @@ struct net_driver ipv6_driver __net_driver = {
 	.remove = ipv6_remove,
 };
 
+/* Drag in objects via ipv6_protocol */
+REQUIRING_SYMBOL ( ipv6_protocol );
+
 /* Drag in ICMPv6 */
 REQUIRE_OBJECT ( icmpv6 );
 
diff --git a/roms/ipxe/src/net/neighbour.c b/roms/ipxe/src/net/neighbour.c
index e3026ce46..7f66d9992 100644
--- a/roms/ipxe/src/net/neighbour.c
+++ b/roms/ipxe/src/net/neighbour.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -91,8 +95,8 @@ static struct neighbour * neighbour_create ( struct net_device *netdev,
 	memcpy ( neighbour->net_dest, net_dest,
 		 net_protocol->net_addr_len );
 	timer_init ( &neighbour->timer, neighbour_expired, &neighbour->refcnt );
-	neighbour->timer.min_timeout = NEIGHBOUR_MIN_TIMEOUT;
-	neighbour->timer.max_timeout = NEIGHBOUR_MAX_TIMEOUT;
+	set_timer_limits ( &neighbour->timer, NEIGHBOUR_MIN_TIMEOUT,
+			   NEIGHBOUR_MAX_TIMEOUT );
 	INIT_LIST_HEAD ( &neighbour->tx_queue );
 
 	/* Transfer ownership to cache */
@@ -318,7 +322,7 @@ int neighbour_tx ( struct io_buffer *iobuf, struct net_device *netdev,
 			netdev->name, net_protocol->name,
 			net_protocol->ntoa ( net_dest ) );
 		list_add_tail ( &iobuf->list, &neighbour->tx_queue );
-		return -EAGAIN;
+		return 0;
 	}
 }
 
diff --git a/roms/ipxe/src/net/netdev_settings.c b/roms/ipxe/src/net/netdev_settings.c
index b3b2e68d8..edd4c4b9f 100644
--- a/roms/ipxe/src/net/netdev_settings.c
+++ b/roms/ipxe/src/net/netdev_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
@@ -121,6 +125,10 @@ static int netdev_fetch_bustype ( struct net_device *netdev, void *data,
 		[BUS_TYPE_MCA] = "MCA",
 		[BUS_TYPE_ISA] = "ISA",
 		[BUS_TYPE_TAP] = "TAP",
+		[BUS_TYPE_EFI] = "EFI",
+		[BUS_TYPE_XEN] = "XEN",
+		[BUS_TYPE_HV] = "HV",
+		[BUS_TYPE_USB] = "USB",
 	};
 	struct device_description *desc = &netdev->dev->desc;
 	const char *bustype;
diff --git a/roms/ipxe/src/net/netdevice.c b/roms/ipxe/src/net/netdevice.c
index a55e6b7d7..7c40a2ac8 100644
--- a/roms/ipxe/src/net/netdevice.c
+++ b/roms/ipxe/src/net/netdevice.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -35,6 +39,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/device.h>
 #include <ipxe/errortab.h>
 #include <ipxe/profile.h>
+#include <ipxe/fault.h>
 #include <ipxe/vlan.h>
 #include <ipxe/netdevice.h>
 
@@ -157,6 +162,9 @@ void netdev_rx_unfreeze ( struct net_device *netdev ) {
  */
 void netdev_link_err ( struct net_device *netdev, int rc ) {
 
+	/* Stop link block timer */
+	stop_timer ( &netdev->link_block );
+
 	/* Record link state */
 	netdev->link_rc = rc;
 	if ( netdev->link_rc == 0 ) {
@@ -187,6 +195,50 @@ void netdev_link_down ( struct net_device *netdev ) {
 }
 
 /**
+ * Mark network device link as being blocked
+ *
+ * @v netdev		Network device
+ * @v timeout		Timeout (in ticks)
+ */
+void netdev_link_block ( struct net_device *netdev, unsigned long timeout ) {
+
+	/* Start link block timer */
+	if ( ! netdev_link_blocked ( netdev ) ) {
+		DBGC ( netdev, "NETDEV %s link blocked for %ld ticks\n",
+		       netdev->name, timeout );
+	}
+	start_timer_fixed ( &netdev->link_block, timeout );
+}
+
+/**
+ * Mark network device link as being unblocked
+ *
+ * @v netdev		Network device
+ */
+void netdev_link_unblock ( struct net_device *netdev ) {
+
+	/* Stop link block timer */
+	if ( netdev_link_blocked ( netdev ) )
+		DBGC ( netdev, "NETDEV %s link unblocked\n", netdev->name );
+	stop_timer ( &netdev->link_block );
+}
+
+/**
+ * Handle network device link block timer expiry
+ *
+ * @v timer		Link block timer
+ * @v fail		Failure indicator
+ */
+static void netdev_link_block_expired ( struct retry_timer *timer,
+					int fail __unused ) {
+	struct net_device *netdev =
+		container_of ( timer, struct net_device, link_block );
+
+	/* Assume link is no longer blocked */
+	DBGC ( netdev, "NETDEV %s link block expired\n", netdev->name );
+}
+
+/**
  * Record network device statistic
  *
  * @v stats		Network device statistics
@@ -252,11 +304,8 @@ int netdev_tx ( struct net_device *netdev, struct io_buffer *iobuf ) {
 	}
 
 	/* Discard packet (for test purposes) if applicable */
-	if ( ( NETDEV_DISCARD_RATE > 0 ) &&
-	     ( ( random() % NETDEV_DISCARD_RATE ) == 0 ) ) {
-		rc = -EAGAIN;
+	if ( ( rc = inject_fault ( NETDEV_DISCARD_RATE ) ) != 0 )
 		goto err;
-	}
 
 	/* Transmit packet */
 	if ( ( rc = netdev->op->transmit ( netdev, iobuf ) ) != 0 )
@@ -406,14 +455,14 @@ static void netdev_tx_flush ( struct net_device *netdev ) {
  * function takes ownership of the I/O buffer.
  */
 void netdev_rx ( struct net_device *netdev, struct io_buffer *iobuf ) {
+	int rc;
 
 	DBGC2 ( netdev, "NETDEV %s received %p (%p+%zx)\n",
 		netdev->name, iobuf, iobuf->data, iob_len ( iobuf ) );
 
 	/* Discard packet (for test purposes) if applicable */
-	if ( ( NETDEV_DISCARD_RATE > 0 ) &&
-	     ( ( random() % NETDEV_DISCARD_RATE ) == 0 ) ) {
-		netdev_rx_err ( netdev, iobuf, -EAGAIN );
+	if ( ( rc = inject_fault ( NETDEV_DISCARD_RATE ) ) != 0 ) {
+		netdev_rx_err ( netdev, iobuf, rc );
 		return;
 	}
 
@@ -541,7 +590,8 @@ static struct interface_descriptor netdev_config_desc =
 static void free_netdev ( struct refcnt *refcnt ) {
 	struct net_device *netdev =
 		container_of ( refcnt, struct net_device, refcnt );
-	
+
+	stop_timer ( &netdev->link_block );
 	netdev_tx_flush ( netdev );
 	netdev_rx_flush ( netdev );
 	clear_settings ( netdev_settings ( netdev ) );
@@ -571,6 +621,8 @@ struct net_device * alloc_netdev ( size_t priv_len ) {
 	if ( netdev ) {
 		ref_init ( &netdev->refcnt, free_netdev );
 		netdev->link_rc = -EUNKNOWN_LINK_STATUS;
+		timer_init ( &netdev->link_block, netdev_link_block_expired,
+			     &netdev->refcnt );
 		INIT_LIST_HEAD ( &netdev->tx_queue );
 		INIT_LIST_HEAD ( &netdev->tx_deferred );
 		INIT_LIST_HEAD ( &netdev->rx_queue );
@@ -624,11 +676,11 @@ int register_netdev ( struct net_device *netdev ) {
 	}
 
 	/* Record device index and create device name */
-	netdev->index = netdev_index++;
 	if ( netdev->name[0] == '\0' ) {
 		snprintf ( netdev->name, sizeof ( netdev->name ), "net%d",
-			   netdev->index );
+			   netdev_index );
 	}
+	netdev->index = ++netdev_index;
 
 	/* Use least significant bits of the link-layer address to
 	 * improve the randomness of the (non-cryptographic) random
diff --git a/roms/ipxe/src/net/nullnet.c b/roms/ipxe/src/net/nullnet.c
index 4ac50f64b..2948b38c0 100644
--- a/roms/ipxe/src/net/nullnet.c
+++ b/roms/ipxe/src/net/nullnet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/net/pccrc.c b/roms/ipxe/src/net/pccrc.c
new file mode 100644
index 000000000..4cd82cd1c
--- /dev/null
+++ b/roms/ipxe/src/net/pccrc.c
@@ -0,0 +1,818 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/sha256.h>
+#include <ipxe/sha512.h>
+#include <ipxe/hmac.h>
+#include <ipxe/base16.h>
+#include <ipxe/pccrc.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Content Identification [MS-PCCRC]
+ *
+ */
+
+/******************************************************************************
+ *
+ * Utility functions
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Transcribe hash value (for debugging)
+ *
+ * @v info		Content information
+ * @v hash		Hash value
+ * @ret string		Hash value string
+ */
+static inline const char *
+peerdist_info_hash_ntoa ( const struct peerdist_info *info, const void *hash ) {
+	static char buf[ ( 2 * PEERDIST_DIGEST_MAX_SIZE ) + 1 /* NUL */ ];
+	size_t digestsize = info->digestsize;
+
+	/* Sanity check */
+	assert ( info != NULL );
+	assert ( digestsize != 0 );
+	assert ( base16_encoded_len ( digestsize ) < sizeof ( buf ) );
+
+	/* Transcribe hash value */
+	base16_encode ( hash, digestsize, buf, sizeof ( buf ) );
+	return buf;
+}
+
+/**
+ * Get raw data
+ *
+ * @v info		Content information
+ * @v data		Data buffer
+ * @v offset		Starting offset
+ * @v len		Length
+ * @ret rc		Return status code
+ */
+static int peerdist_info_get ( const struct peerdist_info *info, void *data,
+			       size_t offset, size_t len ) {
+
+	/* Sanity check */
+	if ( ( offset > info->raw.len ) ||
+	     ( len > ( info->raw.len - offset ) ) ) {
+		DBGC ( info, "PCCRC %p data underrun at [%zx,%zx) of %zx\n",
+		       info, offset, ( offset + len ), info->raw.len );
+		return -ERANGE;
+	}
+
+	/* Copy data */
+	copy_from_user ( data, info->raw.data, offset, len );
+
+	return 0;
+}
+
+/**
+ * Populate segment hashes
+ *
+ * @v segment		Content information segment to fill in
+ * @v hash		Segment hash of data
+ * @v secret		Segment secret
+ */
+static void peerdist_info_segment_hash ( struct peerdist_info_segment *segment,
+					 const void *hash, const void *secret ){
+	const struct peerdist_info *info = segment->info;
+	struct digest_algorithm *digest = info->digest;
+	uint8_t ctx[digest->ctxsize];
+	size_t digestsize = info->digestsize;
+	size_t secretsize = digestsize;
+	static const uint16_t magic[] = PEERDIST_SEGMENT_ID_MAGIC;
+
+	/* Sanity check */
+	assert ( digestsize <= sizeof ( segment->hash ) );
+	assert ( digestsize <= sizeof ( segment->secret ) );
+	assert ( digestsize <= sizeof ( segment->id ) );
+
+	/* Get segment hash of data */
+	memcpy ( segment->hash, hash, digestsize );
+
+	/* Get segment secret */
+	memcpy ( segment->secret, secret, digestsize );
+
+	/* Calculate segment identifier */
+	hmac_init ( digest, ctx, segment->secret, &secretsize );
+	assert ( secretsize == digestsize );
+	hmac_update ( digest, ctx, segment->hash, digestsize );
+	hmac_update ( digest, ctx, magic, sizeof ( magic ) );
+	hmac_final ( digest, ctx, segment->secret, &secretsize, segment->id );
+	assert ( secretsize == digestsize );
+}
+
+/******************************************************************************
+ *
+ * Content Information version 1
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get number of blocks within a block description
+ *
+ * @v info		Content information
+ * @v offset		Block description offset
+ * @ret blocks		Number of blocks, or negative error
+ */
+static int peerdist_info_v1_blocks ( const struct peerdist_info *info,
+				     size_t offset ) {
+	struct peerdist_info_v1_block raw;
+	unsigned int blocks;
+	int rc;
+
+	/* Get block description header */
+	if ( ( rc = peerdist_info_get ( info, &raw, offset,
+					sizeof ( raw ) ) ) != 0 )
+		return rc;
+
+	/* Calculate number of blocks */
+	blocks = le32_to_cpu ( raw.blocks );
+
+	return blocks;
+}
+
+/**
+ * Locate block description
+ *
+ * @v info		Content information
+ * @v index		Segment index
+ * @ret offset		Block description offset, or negative error
+ */
+static ssize_t peerdist_info_v1_block_offset ( const struct peerdist_info *info,
+					       unsigned int index ) {
+	size_t digestsize = info->digestsize;
+	unsigned int i;
+	size_t offset;
+	int blocks;
+	int rc;
+
+	/* Sanity check */
+	assert ( index < info->segments );
+
+	/* Calculate offset of first block description */
+	offset = ( sizeof ( struct peerdist_info_v1 ) +
+		   ( info->segments *
+		     sizeof ( peerdist_info_v1_segment_t ( digestsize ) ) ) );
+
+	/* Iterate over block descriptions until we find this segment */
+	for ( i = 0 ; i < index ; i++ ) {
+
+		/* Get number of blocks */
+		blocks = peerdist_info_v1_blocks ( info, offset );
+		if ( blocks < 0 ) {
+			rc = blocks;
+			DBGC ( info, "PCCRC %p segment %d could not get number "
+			       "of blocks: %s\n", info, i, strerror ( rc ) );
+			return rc;
+		}
+
+		/* Move to next block description */
+		offset += sizeof ( peerdist_info_v1_block_t ( digestsize,
+							      blocks ) );
+	}
+
+	return offset;
+}
+
+/**
+ * Populate content information
+ *
+ * @v info		Content information to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v1 ( struct peerdist_info *info ) {
+	struct peerdist_info_v1 raw;
+	struct peerdist_info_segment first;
+	struct peerdist_info_segment last;
+	size_t first_skip;
+	size_t last_skip;
+	size_t last_read;
+	int rc;
+
+	/* Get raw header */
+	if ( ( rc = peerdist_info_get ( info, &raw, 0, sizeof ( raw ) ) ) != 0){
+		DBGC ( info, "PCCRC %p could not get V1 content information: "
+		       "%s\n", info, strerror ( rc ) );
+		return rc;
+	}
+	assert ( raw.version.raw == cpu_to_le16 ( PEERDIST_INFO_V1 ) );
+
+	/* Determine hash algorithm */
+	switch ( raw.hash ) {
+	case cpu_to_le32 ( PEERDIST_INFO_V1_HASH_SHA256 ) :
+		info->digest = &sha256_algorithm;
+		break;
+	case cpu_to_le32 ( PEERDIST_INFO_V1_HASH_SHA384 ) :
+		info->digest = &sha384_algorithm;
+		break;
+	case cpu_to_le32 ( PEERDIST_INFO_V1_HASH_SHA512 ) :
+		info->digest = &sha512_algorithm;
+		break;
+	default:
+		DBGC ( info, "PCCRC %p unsupported hash algorithm %#08x\n",
+		       info, le32_to_cpu ( raw.hash ) );
+		return -ENOTSUP;
+	}
+	info->digestsize = info->digest->digestsize;
+	assert ( info->digest != NULL );
+	DBGC2 ( info, "PCCRC %p using %s[%zd]\n",
+		info, info->digest->name, ( info->digestsize * 8 ) );
+
+	/* Calculate number of segments */
+	info->segments = le32_to_cpu ( raw.segments );
+
+	/* Get first segment */
+	if ( ( rc = peerdist_info_segment ( info, &first, 0 ) ) != 0 )
+		return rc;
+
+	/* Calculate range start offset */
+	info->range.start = first.range.start;
+
+	/* Calculate trimmed range start offset */
+	first_skip = le32_to_cpu ( raw.first );
+	info->trim.start = ( first.range.start + first_skip );
+
+	/* Get last segment */
+	if ( ( rc = peerdist_info_segment ( info, &last,
+					    ( info->segments - 1 ) ) ) != 0 )
+		return rc;
+
+	/* Calculate range end offset */
+	info->range.end = last.range.end;
+
+	/* Calculate trimmed range end offset */
+	if ( raw.last ) {
+		/* Explicit length to include from last segment is given */
+		last_read = le32_to_cpu ( raw.last );
+		last_skip = ( last.index ? 0 : first_skip );
+		info->trim.end = ( last.range.start + last_skip + last_read );
+	} else {
+		/* No explicit length given: range extends to end of segment */
+		info->trim.end = last.range.end;
+	}
+
+	return 0;
+}
+
+/**
+ * Populate content information segment
+ *
+ * @v segment		Content information segment to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v1_segment ( struct peerdist_info_segment *segment ) {
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+	peerdist_info_v1_segment_t ( digestsize ) raw;
+	ssize_t raw_offset;
+	int blocks;
+	int rc;
+
+	/* Sanity checks */
+	assert ( segment->index < info->segments );
+
+	/* Get raw description */
+	raw_offset = ( sizeof ( struct peerdist_info_v1 ) +
+		       ( segment->index * sizeof ( raw ) ) );
+	if ( ( rc = peerdist_info_get ( info, &raw, raw_offset,
+					sizeof ( raw ) ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p segment %d could not get segment "
+		       "description: %s\n", info, segment->index,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Calculate start offset of this segment */
+	segment->range.start = le64_to_cpu ( raw.segment.offset );
+
+	/* Calculate end offset of this segment */
+	segment->range.end = ( segment->range.start +
+			       le32_to_cpu ( raw.segment.len ) );
+
+	/* Calculate block size of this segment */
+	segment->blksize = le32_to_cpu ( raw.segment.blksize );
+
+	/* Locate block description for this segment */
+	raw_offset = peerdist_info_v1_block_offset ( info, segment->index );
+	if ( raw_offset < 0 ) {
+		rc = raw_offset;
+		return rc;
+	}
+
+	/* Get number of blocks */
+	blocks = peerdist_info_v1_blocks ( info, raw_offset );
+	if ( blocks < 0 ) {
+		rc = blocks;
+		DBGC ( info, "PCCRC %p segment %d could not get number of "
+		       "blocks: %s\n", info, segment->index, strerror ( rc ) );
+		return rc;
+	}
+	segment->blocks = blocks;
+
+	/* Calculate segment hashes */
+	peerdist_info_segment_hash ( segment, raw.hash, raw.secret );
+
+	return 0;
+}
+
+/**
+ * Populate content information block
+ *
+ * @v block		Content information block to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v1_block ( struct peerdist_info_block *block ) {
+	const struct peerdist_info_segment *segment = block->segment;
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+	peerdist_info_v1_block_t ( digestsize, segment->blocks ) raw;
+	ssize_t raw_offset;
+	int rc;
+
+	/* Sanity checks */
+	assert ( block->index < segment->blocks );
+
+	/* Calculate start offset of this block */
+	block->range.start = ( segment->range.start +
+			       ( block->index * segment->blksize ) );
+
+	/* Calculate end offset of this block */
+	block->range.end = ( block->range.start + segment->blksize );
+	if ( block->range.end > segment->range.end )
+		block->range.end = segment->range.end;
+
+	/* Locate block description */
+	raw_offset = peerdist_info_v1_block_offset ( info, segment->index );
+	if ( raw_offset < 0 ) {
+		rc = raw_offset;
+		return rc;
+	}
+
+	/* Get block hash */
+	raw_offset += offsetof ( typeof ( raw ), hash[block->index] );
+	if ( ( rc = peerdist_info_get ( info, block->hash, raw_offset,
+					digestsize ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p segment %d block %d could not get "
+		       "hash: %s\n", info, segment->index, block->index,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/** Content information version 1 operations */
+static struct peerdist_info_operations peerdist_info_v1_operations = {
+	.info = peerdist_info_v1,
+	.segment = peerdist_info_v1_segment,
+	.block = peerdist_info_v1_block,
+};
+
+/******************************************************************************
+ *
+ * Content Information version 2
+ *
+ ******************************************************************************
+ */
+
+/** A segment cursor */
+struct peerdist_info_v2_cursor {
+	/** Raw data offset */
+	size_t offset;
+	/** Number of segments remaining within this chunk */
+	unsigned int remaining;
+	/** Accumulated segment length */
+	size_t len;
+};
+
+/**
+ * Initialise segment cursor
+ *
+ * @v cursor		Segment cursor
+ */
+static inline void
+peerdist_info_v2_cursor_init ( struct peerdist_info_v2_cursor *cursor ) {
+
+	/* Initialise cursor */
+	cursor->offset = ( sizeof ( struct peerdist_info_v2 ) +
+			   sizeof ( struct peerdist_info_v2_chunk ) );
+	cursor->remaining = 0;
+	cursor->len = 0;
+}
+
+/**
+ * Update segment cursor to next segment description
+ *
+ * @v info		Content information
+ * @v offset		Current offset
+ * @v remaining		Number of segments remaining within this chunk
+ * @ret rc		Return status code
+ */
+static int
+peerdist_info_v2_cursor_next ( const struct peerdist_info *info,
+			       struct peerdist_info_v2_cursor *cursor ) {
+	size_t digestsize = info->digestsize;
+	peerdist_info_v2_segment_t ( digestsize ) raw;
+	struct peerdist_info_v2_chunk chunk;
+	int rc;
+
+	/* Get chunk description if applicable */
+	if ( ! cursor->remaining ) {
+
+		/* Get chunk description */
+		if ( ( rc = peerdist_info_get ( info, &chunk,
+						( cursor->offset -
+						  sizeof ( chunk ) ),
+						sizeof ( chunk ) ) ) != 0 )
+			return rc;
+
+		/* Update number of segments remaining */
+		cursor->remaining = ( be32_to_cpu ( chunk.len ) /
+				      sizeof ( raw ) );
+	}
+
+	/* Get segment description header */
+	if ( ( rc = peerdist_info_get ( info, &raw.segment, cursor->offset,
+					sizeof ( raw.segment ) ) ) != 0 )
+		return rc;
+
+	/* Update cursor */
+	cursor->offset += sizeof ( raw );
+	cursor->remaining--;
+	if ( ! cursor->remaining )
+		cursor->offset += sizeof ( chunk );
+	cursor->len += be32_to_cpu ( raw.segment.len );
+
+	return 0;
+}
+
+/**
+ * Get number of segments and total length
+ *
+ * @v info		Content information
+ * @v len		Length to fill in
+ * @ret rc		Number of segments, or negative error
+ */
+static int peerdist_info_v2_segments ( const struct peerdist_info *info,
+				       size_t *len ) {
+	struct peerdist_info_v2_cursor cursor;
+	unsigned int segments;
+	int rc;
+
+	/* Iterate over all segments */
+	for ( peerdist_info_v2_cursor_init ( &cursor ), segments = 0 ;
+	      cursor.offset < info->raw.len ; segments++ ) {
+
+		/* Update segment cursor */
+		if ( ( rc = peerdist_info_v2_cursor_next ( info,
+							   &cursor ) ) != 0 ) {
+			DBGC ( info, "PCCRC %p segment %d could not update "
+			       "segment cursor: %s\n",
+			       info, segments, strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	/* Record accumulated length */
+	*len = cursor.len;
+
+	return segments;
+}
+
+/**
+ * Populate content information
+ *
+ * @v info		Content information to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v2 ( struct peerdist_info *info ) {
+	struct peerdist_info_v2 raw;
+	size_t len = 0;
+	int segments;
+	int rc;
+
+	/* Get raw header */
+	if ( ( rc = peerdist_info_get ( info, &raw, 0, sizeof ( raw ) ) ) != 0){
+		DBGC ( info, "PCCRC %p could not get V2 content information: "
+		       "%s\n", info, strerror ( rc ) );
+		return rc;
+	}
+	assert ( raw.version.raw == cpu_to_le16 ( PEERDIST_INFO_V2 ) );
+
+	/* Determine hash algorithm */
+	switch ( raw.hash ) {
+	case PEERDIST_INFO_V2_HASH_SHA512_TRUNC :
+		info->digest = &sha512_algorithm;
+		info->digestsize = ( 256 / 8 );
+		break;
+	default:
+		DBGC ( info, "PCCRC %p unsupported hash algorithm %#02x\n",
+		       info, raw.hash );
+		return -ENOTSUP;
+	}
+	assert ( info->digest != NULL );
+	DBGC2 ( info, "PCCRC %p using %s[%zd]\n",
+		info, info->digest->name, ( info->digestsize * 8 ) );
+
+	/* Calculate number of segments and total length */
+	segments = peerdist_info_v2_segments ( info, &len );
+	if ( segments < 0 ) {
+		rc = segments;
+		DBGC ( info, "PCCRC %p could not get segment count and length: "
+		       "%s\n", info, strerror ( rc ) );
+		return rc;
+	}
+	info->segments = segments;
+
+	/* Calculate range start offset */
+	info->range.start = be64_to_cpu ( raw.offset );
+
+	/* Calculate trimmed range start offset */
+	info->trim.start = ( info->range.start + be32_to_cpu ( raw.first ) );
+
+	/* Calculate range end offset */
+	info->range.end = ( info->range.start + len );
+
+	/* Calculate trimmed range end offset */
+	info->trim.end = ( raw.len ? be64_to_cpu ( raw.len ) :
+			   info->range.end );
+
+	return 0;
+}
+
+/**
+ * Populate content information segment
+ *
+ * @v segment		Content information segment to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v2_segment ( struct peerdist_info_segment *segment ) {
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+	peerdist_info_v2_segment_t ( digestsize ) raw;
+	struct peerdist_info_v2_cursor cursor;
+	unsigned int index;
+	size_t len;
+	int rc;
+
+	/* Sanity checks */
+	assert ( segment->index < info->segments );
+
+	/* Iterate over all segments before the target segment */
+	for ( peerdist_info_v2_cursor_init ( &cursor ), index = 0 ;
+	      index < segment->index ; index++ ) {
+
+		/* Update segment cursor */
+		if ( ( rc = peerdist_info_v2_cursor_next ( info,
+							   &cursor ) ) != 0 ) {
+			DBGC ( info, "PCCRC %p segment %d could not update "
+			       "segment cursor: %s\n",
+			       info, index, strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	/* Get raw description */
+	if ( ( rc = peerdist_info_get ( info, &raw, cursor.offset,
+					sizeof ( raw ) ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p segment %d could not get segment "
+		       "description: %s\n",
+		       info, segment->index, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Calculate start offset of this segment */
+	segment->range.start = ( info->range.start + cursor.len );
+
+	/* Calculate end offset of this segment */
+	len = be32_to_cpu ( raw.segment.len );
+	segment->range.end = ( segment->range.start + len );
+
+	/* Model as a segment containing a single block */
+	segment->blocks = 1;
+	segment->blksize = len;
+
+	/* Calculate segment hashes */
+	peerdist_info_segment_hash ( segment, raw.hash, raw.secret );
+
+	return 0;
+}
+
+/**
+ * Populate content information block
+ *
+ * @v block		Content information block to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v2_block ( struct peerdist_info_block *block ) {
+	const struct peerdist_info_segment *segment = block->segment;
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+
+	/* Sanity checks */
+	assert ( block->index < segment->blocks );
+
+	/* Model as a block covering the whole segment */
+	memcpy ( &block->range, &segment->range, sizeof ( block->range ) );
+	memcpy ( block->hash, segment->hash, digestsize );
+
+	return 0;
+}
+
+/** Content information version 2 operations */
+static struct peerdist_info_operations peerdist_info_v2_operations = {
+	.block = peerdist_info_v2_block,
+	.segment = peerdist_info_v2_segment,
+	.info = peerdist_info_v2,
+};
+
+/******************************************************************************
+ *
+ * Content Information
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Populate content information
+ *
+ * @v data		Raw data
+ * @v len		Length of raw data
+ * @v info		Content information to fill in
+ * @ret rc		Return status code
+ */
+int peerdist_info ( userptr_t data, size_t len, struct peerdist_info *info ) {
+	union peerdist_info_version version;
+	int rc;
+
+	/* Initialise structure */
+	memset ( info, 0, sizeof ( *info ) );
+	info->raw.data = data;
+	info->raw.len = len;
+
+	/* Get version */
+	if ( ( rc = peerdist_info_get ( info, &version, 0,
+					sizeof ( version ) ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p could not get version: %s\n",
+		       info, strerror ( rc ) );
+		return rc;
+	}
+	DBGC2 ( info, "PCCRC %p version %d.%d\n",
+		info, version.major, version.minor );
+
+	/* Determine version */
+	switch ( version.raw ) {
+	case cpu_to_le16 ( PEERDIST_INFO_V1 ) :
+		info->op = &peerdist_info_v1_operations;
+		break;
+	case cpu_to_le16 ( PEERDIST_INFO_V2 ) :
+		info->op = &peerdist_info_v2_operations;
+		break;
+	default:
+		DBGC ( info, "PCCRC %p unsupported version %d.%d\n",
+		       info, version.major, version.minor );
+		return -ENOTSUP;
+	}
+	assert ( info->op != NULL );
+	assert ( info->op->info != NULL );
+
+	/* Populate content information */
+	if ( ( rc = info->op->info ( info ) ) != 0 )
+		return rc;
+
+	DBGC2 ( info, "PCCRC %p range [%08zx,%08zx) covers [%08zx,%08zx) with "
+		"%d segments\n", info, info->range.start, info->range.end,
+		info->trim.start, info->trim.end, info->segments );
+	return 0;
+}
+
+/**
+ * Populate content information segment
+ *
+ * @v info		Content information
+ * @v segment		Content information segment to fill in
+ * @v index		Segment index
+ * @ret rc		Return status code
+ */
+int peerdist_info_segment ( const struct peerdist_info *info,
+			    struct peerdist_info_segment *segment,
+			    unsigned int index ) {
+	int rc;
+
+	/* Sanity checks */
+	assert ( info != NULL );
+	assert ( info->op != NULL );
+	assert ( info->op->segment != NULL );
+	if ( index >= info->segments ) {
+		DBGC ( info, "PCCRC %p segment %d of [0,%d) out of range\n",
+		       info, index, info->segments );
+		return -ERANGE;
+	}
+
+	/* Initialise structure */
+	memset ( segment, 0, sizeof ( *segment ) );
+	segment->info = info;
+	segment->index = index;
+
+	/* Populate content information segment */
+	if ( ( rc = info->op->segment ( segment ) ) != 0 )
+		return rc;
+
+	DBGC2 ( info, "PCCRC %p segment %d range [%08zx,%08zx) with %d "
+		"blocks\n", info, segment->index, segment->range.start,
+		segment->range.end, segment->blocks );
+	DBGC2 ( info, "PCCRC %p segment %d digest %s\n", info, segment->index,
+		peerdist_info_hash_ntoa ( info, segment->hash ) );
+	DBGC2 ( info, "PCCRC %p segment %d secret %s\n", info, segment->index,
+		peerdist_info_hash_ntoa ( info, segment->secret ) );
+	DBGC2 ( info, "PCCRC %p segment %d identf %s\n", info, segment->index,
+		peerdist_info_hash_ntoa ( info, segment->id ) );
+	return 0;
+}
+
+/**
+ * Populate content information block
+ *
+ * @v segment		Content information segment
+ * @v block		Content information block to fill in
+ * @v index		Block index
+ * @ret rc		Return status code
+ */
+int peerdist_info_block ( const struct peerdist_info_segment *segment,
+			  struct peerdist_info_block *block,
+			  unsigned int index ) {
+	const struct peerdist_info *info = segment->info;
+	size_t start;
+	size_t end;
+	int rc;
+
+	/* Sanity checks */
+	assert ( segment != NULL );
+	assert ( info != NULL );
+	assert ( info->op != NULL );
+	assert ( info->op->block != NULL );
+	if ( index >= segment->blocks ) {
+		DBGC ( info, "PCCRC %p segment %d block %d of [0,%d) out of "
+		       "range\n", info, segment->index, index, segment->blocks);
+		return -ERANGE;
+	}
+
+	/* Initialise structure */
+	memset ( block, 0, sizeof ( *block ) );
+	block->segment = segment;
+	block->index = index;
+
+	/* Populate content information block */
+	if ( ( rc = info->op->block ( block ) ) != 0 )
+		return rc;
+
+	/* Calculate trimmed range */
+	start = block->range.start;
+	if ( start < info->trim.start )
+		start = info->trim.start;
+	end = block->range.end;
+	if ( end > info->trim.end )
+		end = info->trim.end;
+	if ( end < start )
+		end = start;
+	block->trim.start = start;
+	block->trim.end = end;
+
+	DBGC2 ( info, "PCCRC %p segment %d block %d hash %s\n",
+		info, segment->index, block->index,
+		peerdist_info_hash_ntoa ( info, block->hash ) );
+	DBGC2 ( info, "PCCRC %p segment %d block %d range [%08zx,%08zx) covers "
+		"[%08zx,%08zx)\n", info, segment->index, block->index,
+		block->range.start, block->range.end, block->trim.start,
+		block->trim.end );
+	return 0;
+}
diff --git a/roms/ipxe/src/net/pccrd.c b/roms/ipxe/src/net/pccrd.c
new file mode 100644
index 000000000..04b5dd86c
--- /dev/null
+++ b/roms/ipxe/src/net/pccrd.c
@@ -0,0 +1,286 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/pccrd.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Discovery Protocol [MS-PCCRD]
+ *
+ * This protocol manages to ingeniously combine the excessive
+ * verbosity of XML with a paucity of actual information.  For
+ * example: even in version 2.0 of the protocol it is still not
+ * possible to discover which peers hold a specific block within a
+ * given segment.
+ *
+ * For added bonus points, version 1.0 of the protocol is specified to
+ * use a case-sensitive string comparison (for SHA2 digest values) but
+ * nothing specifies whether the strings in question should be in
+ * upper or lower case.  There are example strings given in the
+ * specification, but the author skilfully manages to leave the issue
+ * unresolved by using the somewhat implausible digest value of
+ * "0200000000000000000000000000000000000000000000000000000000000000".
+ *
+ * Just in case you were thinking that the silver lining of the choice
+ * to use an XML-based protocol would be the ability to generate and
+ * process messages with standard tools, version 2.0 of the protocol
+ * places most of the critical information inside a Base64-encoded
+ * custom binary data structure.  Within an XML element, naturally.
+ *
+ * I hereby announce this specification to be the 2015 winner of the
+ * prestigious "UEFI HII API" award for incompetent design.
+ */
+
+/** Discovery request format */
+#define PEERDIST_DISCOVERY_REQUEST					      \
+	"<?xml version=\"1.0\" encoding=\"utf-8\"?>"			      \
+	"<soap:Envelope "						      \
+	    "xmlns:soap=\"http://www.w3.org/2003/05/soap-envelope\" "	      \
+	    "xmlns:wsa=\"http://schemas.xmlsoap.org/ws/2004/08/addressing\" " \
+	    "xmlns:wsd=\"http://schemas.xmlsoap.org/ws/2005/04/discovery\" "  \
+	    "xmlns:PeerDist=\"http://schemas.microsoft.com/p2p/"	      \
+			     "2007/09/PeerDistributionDiscovery\">"	      \
+	  "<soap:Header>"						      \
+	    "<wsa:To>"							      \
+	      "urn:schemas-xmlsoap-org:ws:2005:04:discovery"		      \
+	    "</wsa:To>"							      \
+	    "<wsa:Action>"						      \
+	      "http://schemas.xmlsoap.org/ws/2005/04/discovery/Probe"	      \
+	    "</wsa:Action>"						      \
+	    "<wsa:MessageID>"						      \
+	      "urn:uuid:%s"						      \
+	    "</wsa:MessageID>"						      \
+	  "</soap:Header>"						      \
+	  "<soap:Body>"							      \
+	    "<wsd:Probe>"						      \
+	      "<wsd:Types>"						      \
+		"PeerDist:PeerDistData"					      \
+	      "</wsd:Types>"						      \
+	      "<wsd:Scopes MatchBy=\"http://schemas.xmlsoap.org/ws/"	      \
+				    "2005/04/discovery/strcmp0\">"	      \
+		"%s"							      \
+	      "</wsd:Scopes>"						      \
+	    "</wsd:Probe>"						      \
+	  "</soap:Body>"						      \
+	"</soap:Envelope>"
+
+/**
+ * Construct discovery request
+ *
+ * @v uuid		Message UUID string
+ * @v id		Segment identifier string
+ * @ret request		Discovery request, or NULL on failure
+ *
+ * The request is dynamically allocated; the caller must eventually
+ * free() the request.
+ */
+char * peerdist_discovery_request ( const char *uuid, const char *id ) {
+	char *request;
+	int len;
+
+	/* Construct request */
+	len = asprintf ( &request, PEERDIST_DISCOVERY_REQUEST, uuid, id );
+	if ( len < 0 )
+		return NULL;
+
+	return request;
+}
+
+/**
+ * Locate discovery reply tag
+ *
+ * @v data		Reply data (not NUL-terminated)
+ * @v len		Length of reply data
+ * @v tag		XML tag
+ * @ret found		Found tag (or NULL if not found)
+ */
+static char * peerdist_discovery_reply_tag ( char *data, size_t len,
+					     const char *tag ) {
+	size_t tag_len = strlen ( tag );
+
+	/* Search, allowing for the fact that the reply data is not
+	 * cleanly NUL-terminated and may contain embedded NULs due to
+	 * earlier parsing.
+	 */
+	for ( ; len >= tag_len ; data++, len-- ) {
+		if ( strncmp ( data, tag, tag_len ) == 0 )
+			return data;
+	}
+	return NULL;
+}
+
+/**
+ * Locate discovery reply values
+ *
+ * @v data		Reply data (not NUL-terminated, will be modified)
+ * @v len		Length of reply data
+ * @v name		XML tag name
+ * @ret values		Tag values (or NULL if not found)
+ *
+ * The reply data is modified by adding NULs and moving characters as
+ * needed to produce a NUL-separated list of values, terminated with a
+ * zero-length string.
+ *
+ * This is not supposed to be a full XML parser; it's supposed to
+ * include just enough functionality to allow PeerDist discovery to
+ * work with existing implementations.
+ */
+static char * peerdist_discovery_reply_values ( char *data, size_t len,
+						const char *name ) {
+	char buf[ 2 /* "</" */ + strlen ( name ) + 1 /* ">" */ + 1 /* NUL */ ];
+	char *open;
+	char *close;
+	char *start;
+	char *end;
+	char *in;
+	char *out;
+	char c;
+
+	/* Locate opening tag */
+	snprintf ( buf, sizeof ( buf ), "<%s>", name );
+	open = peerdist_discovery_reply_tag ( data, len, buf );
+	if ( ! open )
+		return NULL;
+	start = ( open + strlen ( buf ) );
+	len -= ( start - data );
+	data = start;
+
+	/* Locate closing tag */
+	snprintf ( buf, sizeof ( buf ), "</%s>", name );
+	close = peerdist_discovery_reply_tag ( data, len, buf );
+	if ( ! close )
+		return NULL;
+	assert ( close >= open );
+	end = close;
+
+	/* Strip initial whitespace, convert other whitespace
+	 * sequences to single NULs, add terminating pair of NULs.
+	 * This will probably overwrite part of the closing tag.
+	 */
+	for ( in = start, out = start ; in < end ; in++ ) {
+		c = *in;
+		if ( isspace ( c ) ) {
+			if ( ( out > start ) && ( out[-1] != '\0' ) )
+				*(out++) = '\0';
+		} else {
+			*(out++) = c;
+		}
+	}
+	*(out++) = '\0';
+	*(out++) = '\0';
+	assert ( out < ( close + strlen ( buf ) ) );
+
+	return start;
+}
+
+/**
+ * Parse discovery reply
+ *
+ * @v data		Reply data (not NUL-terminated, will be modified)
+ * @v len		Length of reply data
+ * @v reply		Discovery reply to fill in
+ * @ret rc		Return status code
+ *
+ * The discovery reply includes pointers to strings within the
+ * modified reply data.
+ */
+int peerdist_discovery_reply ( char *data, size_t len,
+			       struct peerdist_discovery_reply *reply ) {
+	static const struct peerdist_discovery_block_count zcount = {
+		.hex = "00000000",
+	};
+	struct peerdist_discovery_block_count *count;
+	unsigned int max;
+	unsigned int i;
+	char *scopes;
+	char *xaddrs;
+	char *blockcount;
+	char *in;
+	char *out;
+	size_t skip;
+
+	/* Find <wsd:Scopes> tag */
+	scopes = peerdist_discovery_reply_values ( data, len, "wsd:Scopes" );
+	if ( ! scopes ) {
+		DBGC ( reply, "PCCRD %p missing <wsd:Scopes> tag\n", reply );
+		return -ENOENT;
+	}
+
+	/* Find <wsd:XAddrs> tag */
+	xaddrs = peerdist_discovery_reply_values ( data, len, "wsd:XAddrs" );
+	if ( ! xaddrs ) {
+		DBGC ( reply, "PCCRD %p missing <wsd:XAddrs> tag\n", reply );
+		return -ENOENT;
+	}
+
+	/* Find <PeerDist:BlockCount> tag */
+	blockcount = peerdist_discovery_reply_values ( data, len,
+						       "PeerDist:BlockCount" );
+	if ( ! blockcount ) {
+		DBGC ( reply, "PCCRD %p missing <PeerDist:BlockCount> tag\n",
+		       reply );
+		return -ENOENT;
+	}
+
+	/* Determine maximum number of segments (according to number
+	 * of entries in the block count list).
+	 */
+	max = ( strlen ( blockcount ) / sizeof ( *count ) );
+	count = container_of ( blockcount,
+			       struct peerdist_discovery_block_count, hex[0] );
+
+	/* Eliminate any segments with a zero block count */
+	for ( i = 0, in = scopes, out = scopes ; *in ; i++, in += skip ) {
+
+		/* Fail if we have overrun the maximum number of segments */
+		if ( i >= max ) {
+			DBGC ( reply, "PCCRD %p too many segment IDs\n",
+			       reply );
+			return -EPROTO;
+		}
+
+		/* Delete segment if block count is zero */
+		skip = ( strlen ( in ) + 1 /* NUL */ );
+		if ( memcmp ( count[i].hex, zcount.hex,
+			      sizeof ( zcount.hex ) ) == 0 )
+			continue;
+		strcpy ( out, in );
+		out += skip;
+	}
+	out[0] = '\0'; /* Ensure list is terminated with a zero-length string */
+
+	/* Fill in discovery reply */
+	reply->ids = scopes;
+	reply->locations = xaddrs;
+
+	return 0;
+}
diff --git a/roms/ipxe/src/net/peerblk.c b/roms/ipxe/src/net/peerblk.c
new file mode 100644
index 000000000..fd7ea0893
--- /dev/null
+++ b/roms/ipxe/src/net/peerblk.c
@@ -0,0 +1,1366 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <ipxe/http.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/xfer.h>
+#include <ipxe/uri.h>
+#include <ipxe/timer.h>
+#include <ipxe/profile.h>
+#include <ipxe/fault.h>
+#include <ipxe/pccrr.h>
+#include <ipxe/peerblk.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol block downloads
+ *
+ */
+
+/** PeerDist decryption chunksize
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_DECRYPT_CHUNKSIZE 2048
+
+/** PeerDist raw block download attempt initial progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RAW_OPEN_TIMEOUT ( 10 * TICKS_PER_SEC )
+
+/** PeerDist raw block download attempt ongoing progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RAW_RX_TIMEOUT ( 15 * TICKS_PER_SEC )
+
+/** PeerDist retrieval protocol block download attempt initial progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RETRIEVAL_OPEN_TIMEOUT ( 3 * TICKS_PER_SEC )
+
+/** PeerDist retrieval protocol block download attempt ongoing progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RETRIEVAL_RX_TIMEOUT ( 5 * TICKS_PER_SEC )
+
+/** PeerDist maximum number of full download attempt cycles
+ *
+ * This is the maximum number of times that we will try a full cycle
+ * of download attempts (i.e. a retrieval protocol download attempt
+ * from each discovered peer plus a raw download attempt from the
+ * origin server).
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_MAX_ATTEMPT_CYCLES 4
+
+/** PeerDist block download profiler */
+static struct profiler peerblk_download_profiler __profiler =
+	{ .name = "peerblk.download" };
+
+/** PeerDist block download attempt success profiler */
+static struct profiler peerblk_attempt_success_profiler __profiler =
+	{ .name = "peerblk.attempt.success" };
+
+/** PeerDist block download attempt failure profiler */
+static struct profiler peerblk_attempt_failure_profiler __profiler =
+	{ .name = "peerblk.attempt.failure" };
+
+/** PeerDist block download attempt timeout profiler */
+static struct profiler peerblk_attempt_timeout_profiler __profiler =
+	{ .name = "peerblk.attempt.timeout" };
+
+/** PeerDist block download discovery success profiler */
+static struct profiler peerblk_discovery_success_profiler __profiler =
+	{ .name = "peerblk.discovery.success" };
+
+/** PeerDist block download discovery timeout profiler */
+static struct profiler peerblk_discovery_timeout_profiler __profiler =
+	{ .name = "peerblk.discovery.timeout" };
+
+/**
+ * Get profiling timestamp
+ *
+ * @ret timestamp	Timestamp
+ */
+static inline __attribute__ (( always_inline )) unsigned long
+peerblk_timestamp ( void ) {
+
+	if ( PROFILING ) {
+		return currticks();
+	} else {
+		return 0;
+	}
+}
+
+/**
+ * Free PeerDist block download
+ *
+ * @v refcnt		Reference count
+ */
+static void peerblk_free ( struct refcnt *refcnt ) {
+	struct peerdist_block *peerblk =
+		container_of ( refcnt, struct peerdist_block, refcnt );
+
+	uri_put ( peerblk->uri );
+	free ( peerblk->cipherctx );
+	free ( peerblk );
+}
+
+/**
+ * Reset PeerDist block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for reset
+ */
+static void peerblk_reset ( struct peerdist_block *peerblk, int rc ) {
+
+	/* Stop decryption process */
+	process_del ( &peerblk->process );
+
+	/* Stop timer */
+	stop_timer ( &peerblk->timer );
+
+	/* Abort any current download attempt */
+	intf_restart ( &peerblk->raw, rc );
+	intf_restart ( &peerblk->retrieval, rc );
+
+	/* Empty received data buffer */
+	xferbuf_free ( &peerblk->buffer );
+	peerblk->pos = 0;
+
+	/* Reset digest and free cipher context */
+	digest_init ( peerblk->digest, peerblk->digestctx );
+	free ( peerblk->cipherctx );
+	peerblk->cipherctx = NULL;
+	peerblk->cipher = NULL;
+
+	/* Reset trim thresholds */
+	peerblk->start = ( peerblk->trim.start - peerblk->range.start );
+	peerblk->end = ( peerblk->trim.end - peerblk->range.start );
+	assert ( peerblk->start <= peerblk->end );
+}
+
+/**
+ * Close PeerDist block download
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_close ( struct peerdist_block *peerblk, int rc ) {
+	unsigned long now = peerblk_timestamp();
+
+	/* Profile overall block download */
+	profile_custom ( &peerblk_download_profiler,
+			 ( now - peerblk->started ) );
+
+	/* Reset download attempt */
+	peerblk_reset ( peerblk, rc );
+
+	/* Close discovery */
+	peerdisc_close ( &peerblk->discovery );
+
+	/* Shut down all interfaces */
+	intf_shutdown ( &peerblk->retrieval, rc );
+	intf_shutdown ( &peerblk->raw, rc );
+	intf_shutdown ( &peerblk->xfer, rc );
+}
+
+/**
+ * Calculate offset within overall download
+ *
+ * @v peerblk		PeerDist block download
+ * @v pos		Position within incoming data stream
+ * @ret offset		Offset within overall download
+ */
+static inline __attribute__ (( always_inline )) size_t
+peerblk_offset ( struct peerdist_block *peerblk, size_t pos ) {
+
+	return ( ( pos - peerblk->start ) + peerblk->offset );
+}
+
+/**
+ * Deliver download attempt data block
+ *
+ * @v peerblk		PeerDist block download
+ * @v iobuf		I/O buffer
+ * @v meta		Original data transfer metadata
+ * @v pos		Position within incoming data stream
+ * @ret rc		Return status code
+ */
+static int peerblk_deliver ( struct peerdist_block *peerblk,
+			     struct io_buffer *iobuf,
+			     struct xfer_metadata *meta, size_t pos ) {
+	struct xfer_metadata xfer_meta;
+	size_t len = iob_len ( iobuf );
+	size_t start = pos;
+	size_t end = ( pos + len );
+	int rc;
+
+	/* Discard zero-length packets and packets which lie entirely
+	 * outside the trimmed range.
+	 */
+	if ( ( start >= peerblk->end ) || ( end <= peerblk->start ) ||
+	     ( len == 0 ) ) {
+		free_iob ( iobuf );
+		return 0;
+	}
+
+	/* Truncate data to within trimmed range */
+	if ( start < peerblk->start ) {
+		iob_pull ( iobuf, ( peerblk->start - start ) );
+		start = peerblk->start;
+	}
+	if ( end > peerblk->end ) {
+		iob_unput ( iobuf, ( end - peerblk->end ) );
+		end = peerblk->end;
+	}
+
+	/* Construct metadata */
+	memcpy ( &xfer_meta, meta, sizeof ( xfer_meta ) );
+	xfer_meta.flags |= XFER_FL_ABS_OFFSET;
+	xfer_meta.offset = peerblk_offset ( peerblk, start );
+
+	/* Deliver data */
+	if ( ( rc = xfer_deliver ( &peerblk->xfer, iob_disown ( iobuf ),
+				   &xfer_meta ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not deliver data: %s\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Finish PeerDist block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_done ( struct peerdist_block *peerblk, int rc ) {
+	struct digest_algorithm *digest = peerblk->digest;
+	uint8_t hash[digest->digestsize];
+	unsigned long now = peerblk_timestamp();
+
+	/* Check for errors on completion */
+	if ( rc != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d attempt failed: %s\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		goto err;
+	}
+
+	/* Check digest */
+	digest_final ( digest, peerblk->digestctx, hash );
+	if ( memcmp ( hash, peerblk->hash, peerblk->digestsize ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d digest mismatch:\n",
+		       peerblk, peerblk->segment, peerblk->block );
+		DBGC_HDA ( peerblk, 0, hash, peerblk->digestsize );
+		DBGC_HDA ( peerblk, 0, peerblk->hash, peerblk->digestsize );
+		rc = -EIO;
+		goto err;
+	}
+
+	/* Profile successful attempt */
+	profile_custom ( &peerblk_attempt_success_profiler,
+			 ( now - peerblk->attempted ) );
+
+	/* Close download */
+	peerblk_close ( peerblk, 0 );
+	return;
+
+ err:
+	/* Record failure reason and schedule a retry attempt */
+	profile_custom ( &peerblk_attempt_failure_profiler,
+			 ( now - peerblk->attempted ) );
+	peerblk_reset ( peerblk, rc );
+	peerblk->rc = rc;
+	start_timer_nodelay ( &peerblk->timer );
+}
+
+/******************************************************************************
+ *
+ * Raw block download attempts (using an HTTP range request)
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open PeerDist raw block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @ret rc		Return status code
+ */
+static int peerblk_raw_open ( struct peerdist_block *peerblk ) {
+	struct http_request_range range;
+	int rc;
+
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d attempting raw range request\n",
+		peerblk, peerblk->segment, peerblk->block );
+
+	/* Construct HTTP range */
+	memset ( &range, 0, sizeof ( range ) );
+	range.start = peerblk->range.start;
+	range.len = ( peerblk->range.end - peerblk->range.start );
+
+	/* Initiate range request to retrieve block */
+	if ( ( rc = http_open ( &peerblk->raw, &http_get, peerblk->uri,
+				&range, NULL ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not create range "
+		       "request: %s\n", peerblk, peerblk->segment,
+		       peerblk->block, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Annul HTTP connection (for testing) if applicable.  Do not
+	 * report as an immediate error, in order to test our ability
+	 * to recover from a totally unresponsive HTTP server.
+	 */
+	if ( inject_fault ( PEERBLK_ANNUL_RATE ) )
+		intf_restart ( &peerblk->raw, 0 );
+
+	return 0;
+}
+
+/**
+ * Receive PeerDist raw data
+ *
+ * @v peerblk		PeerDist block download
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peerblk_raw_rx ( struct peerdist_block *peerblk,
+			    struct io_buffer *iobuf,
+			    struct xfer_metadata *meta ) {
+	size_t len = iob_len ( iobuf );
+	size_t pos = peerblk->pos;
+	size_t mid = ( ( peerblk->range.end - peerblk->range.start ) / 2 );
+	int rc;
+
+	/* Corrupt received data (for testing) if applicable */
+	inject_corruption ( PEERBLK_CORRUPT_RATE, iobuf->data, len );
+
+	/* Fail if data is delivered out of order, since the streaming
+	 * digest requires strict ordering.
+	 */
+	if ( ( rc = xfer_check_order ( meta, &peerblk->pos, len ) ) != 0 )
+		goto err;
+
+	/* Add data to digest */
+	digest_update ( peerblk->digest, peerblk->digestctx, iobuf->data, len );
+
+	/* Deliver data */
+	if ( ( rc = peerblk_deliver ( peerblk, iob_disown ( iobuf ), meta,
+				      pos ) ) != 0 )
+		goto err;
+
+	/* Extend download attempt timer */
+	start_timer_fixed ( &peerblk->timer, PEERBLK_RAW_RX_TIMEOUT );
+
+	/* Stall download attempt (for testing) if applicable */
+	if ( ( pos < mid ) && ( ( pos + len ) >= mid ) &&
+	     ( ( rc = inject_fault ( PEERBLK_STALL_RATE ) ) != 0 ) ) {
+		intf_restart ( &peerblk->raw, rc );
+	}
+
+	return 0;
+
+ err:
+	free_iob ( iobuf );
+	peerblk_done ( peerblk, rc );
+	return rc;
+}
+
+/**
+ * Close PeerDist raw block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_raw_close ( struct peerdist_block *peerblk, int rc ) {
+
+	/* Restart interface */
+	intf_restart ( &peerblk->raw, rc );
+
+	/* Fail immediately if we have an error */
+	if ( rc != 0 )
+		goto done;
+
+	/* Abort download attempt (for testing) if applicable */
+	if ( ( rc = inject_fault ( PEERBLK_ABORT_RATE ) ) != 0 )
+		goto done;
+
+ done:
+	/* Complete download attempt */
+	peerblk_done ( peerblk, rc );
+}
+
+/******************************************************************************
+ *
+ * Retrieval protocol block download attempts (using HTTP POST)
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Construct PeerDist retrieval protocol URI
+ *
+ * @v location		Peer location
+ * @ret uri		Retrieval URI, or NULL on error
+ */
+static struct uri * peerblk_retrieval_uri ( const char *location ) {
+	char uri_string[ 7 /* "http://" */ + strlen ( location ) +
+			 sizeof ( PEERDIST_MAGIC_PATH /* includes NUL */ ) ];
+
+	/* Construct URI string */
+	snprintf ( uri_string, sizeof ( uri_string ),
+		   ( "http://%s" PEERDIST_MAGIC_PATH ), location );
+
+	/* Parse URI string */
+	return parse_uri ( uri_string );
+}
+
+/**
+ * Open PeerDist retrieval protocol block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v location		Peer location
+ * @ret rc		Return status code
+ */
+static int peerblk_retrieval_open ( struct peerdist_block *peerblk,
+				    const char *location ) {
+	size_t digestsize = peerblk->digestsize;
+	peerdist_msg_getblks_t ( digestsize, 1, 0 ) req;
+	peerblk_msg_blk_t ( digestsize, 0, 0, 0 ) *rsp;
+	struct http_request_content content;
+	struct uri *uri;
+	int rc;
+
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d attempting retrieval from %s\n",
+		peerblk, peerblk->segment, peerblk->block, location );
+
+	/* Construct block fetch request */
+	memset ( &req, 0, sizeof ( req ) );
+	req.getblks.hdr.version.raw = htonl ( PEERDIST_MSG_GETBLKS_VERSION );
+	req.getblks.hdr.type = htonl ( PEERDIST_MSG_GETBLKS_TYPE );
+	req.getblks.hdr.len = htonl ( sizeof ( req ) );
+	req.getblks.hdr.algorithm = htonl ( PEERDIST_MSG_AES_128_CBC );
+	req.segment.segment.digestsize = htonl ( digestsize );
+	memcpy ( req.segment.id, peerblk->id, digestsize );
+	req.ranges.ranges.count = htonl ( 1 );
+	req.ranges.range[0].first = htonl ( peerblk->block );
+	req.ranges.range[0].count = htonl ( 1 );
+
+	/* Construct POST request content */
+	memset ( &content, 0, sizeof ( content ) );
+	content.data = &req;
+	content.len = sizeof ( req );
+
+	/* Construct URI */
+	if ( ( uri = peerblk_retrieval_uri ( location ) ) == NULL ) {
+		rc = -ENOMEM;
+		goto err_uri;
+	}
+
+	/* Update trim thresholds */
+	peerblk->start += offsetof ( typeof ( *rsp ), msg.vrf );
+	peerblk->end += offsetof ( typeof ( *rsp ), msg.vrf );
+
+	/* Initiate HTTP POST to retrieve block */
+	if ( ( rc = http_open ( &peerblk->retrieval, &http_post, uri,
+				NULL, &content ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not create retrieval "
+		       "request: %s\n", peerblk, peerblk->segment,
+		       peerblk->block, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Annul HTTP connection (for testing) if applicable.  Do not
+	 * report as an immediate error, in order to test our ability
+	 * to recover from a totally unresponsive HTTP server.
+	 */
+	if ( inject_fault ( PEERBLK_ANNUL_RATE ) )
+		intf_restart ( &peerblk->retrieval, 0 );
+
+ err_open:
+	uri_put ( uri );
+ err_uri:
+	return rc;
+}
+
+/**
+ * Receive PeerDist retrieval protocol data
+ *
+ * @v peerblk		PeerDist block download
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peerblk_retrieval_rx ( struct peerdist_block *peerblk,
+				  struct io_buffer *iobuf,
+				  struct xfer_metadata *meta ) {
+	size_t len = iob_len ( iobuf );
+	size_t start;
+	size_t end;
+	size_t before;
+	size_t after;
+	size_t cut;
+	int rc;
+
+	/* Some genius at Microsoft thought it would be a great idea
+	 * to place the AES-CBC initialisation vector *after* the
+	 * encrypted data, thereby making it logically impossible to
+	 * decrypt each packet as it arrives.
+	 *
+	 * To work around this mindless stupidity, we deliver the
+	 * ciphertext as-is and later use xfer_buffer() to obtain
+	 * access to the underlying data transfer buffer in order to
+	 * perform the decryption.
+	 *
+	 * There will be some data both before and after the bytes
+	 * corresponding to the trimmed plaintext: a MSG_BLK
+	 * header/footer, some block padding for the AES-CBC cipher,
+	 * and a possibly large quantity of unwanted ciphertext which
+	 * is excluded from the trimmed content range.  We store this
+	 * data in a local data transfer buffer.  If the amount of
+	 * data to be stored is too large, we will fail allocation and
+	 * so eventually fall back to using a range request (which
+	 * does not require this kind of temporary storage
+	 * allocation).
+	 */
+
+	/* Corrupt received data (for testing) if applicable */
+	inject_corruption ( PEERBLK_CORRUPT_RATE, iobuf->data, len );
+
+	/* Calculate start and end positions of this buffer */
+	start = peerblk->pos;
+	if ( meta->flags & XFER_FL_ABS_OFFSET )
+		start = 0;
+	start += meta->offset;
+	end = ( start + len );
+
+	/* Buffer any data before the trimmed content */
+	if ( ( start < peerblk->start ) && ( len > 0 ) ) {
+
+		/* Calculate length of data before the trimmed content */
+		before = ( peerblk->start - start );
+		if ( before > len )
+			before = len;
+
+		/* Buffer data before the trimmed content */
+		if ( ( rc = xferbuf_write ( &peerblk->buffer, start,
+					    iobuf->data, before ) ) != 0 ) {
+			DBGC ( peerblk, "PEERBLK %p %d.%d could not buffer "
+			       "data: %s\n", peerblk, peerblk->segment,
+			       peerblk->block, strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	/* Buffer any data after the trimmed content */
+	if ( ( end > peerblk->end ) && ( len > 0 ) ) {
+
+		/* Calculate length of data after the trimmed content */
+		after = ( end - peerblk->end );
+		if ( after > len )
+			after = len;
+
+		/* Buffer data after the trimmed content */
+		cut = ( peerblk->end - peerblk->start );
+		if ( ( rc = xferbuf_write ( &peerblk->buffer,
+					    ( end - after - cut ),
+					    ( iobuf->data + len - after ),
+					    after ) ) != 0 ) {
+			DBGC ( peerblk, "PEERBLK %p %d.%d could not buffer "
+			       "data: %s\n", peerblk, peerblk->segment,
+			       peerblk->block, strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	/* Deliver any remaining data */
+	if ( ( rc = peerblk_deliver ( peerblk, iob_disown ( iobuf ), meta,
+				      start ) ) != 0 )
+		goto err;
+
+	/* Update position */
+	peerblk->pos = end;
+
+	/* Extend download attempt timer */
+	start_timer_fixed ( &peerblk->timer, PEERBLK_RETRIEVAL_RX_TIMEOUT );
+
+	/* Stall download attempt (for testing) if applicable */
+	if ( ( start < peerblk->end ) && ( end >= peerblk->end ) &&
+	     ( ( rc = inject_fault ( PEERBLK_STALL_RATE ) ) != 0 ) ) {
+		intf_restart ( &peerblk->retrieval, rc );
+	}
+
+	return 0;
+
+ err:
+	free_iob ( iobuf );
+	peerblk_done ( peerblk, rc );
+	return rc;
+}
+
+/**
+ * Parse retrieval protocol message header
+ *
+ * @v peerblk		PeerDist block download
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_header ( struct peerdist_block *peerblk ) {
+	struct {
+		struct peerdist_msg_transport_header hdr;
+		struct peerdist_msg_header msg;
+	} __attribute__ (( packed )) *msg = peerblk->buffer.data;
+	struct cipher_algorithm *cipher;
+	size_t len = peerblk->buffer.len;
+	size_t keylen = 0;
+	int rc;
+
+	/* Check message length */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for header "
+		       "(%zd bytes)\n", peerblk, peerblk->segment,
+		       peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Check message type */
+	if ( msg->msg.type != htonl ( PEERDIST_MSG_BLK_TYPE ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d unexpected message type "
+		       "%#08x\n", peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.type ) );
+		return -EPROTO;
+	}
+
+	/* Determine cipher algorithm and key length */
+	cipher = &aes_cbc_algorithm;
+	switch ( msg->msg.algorithm ) {
+	case htonl ( PEERDIST_MSG_PLAINTEXT ) :
+		cipher = NULL;
+		break;
+	case htonl ( PEERDIST_MSG_AES_128_CBC ) :
+		keylen = ( 128 / 8 );
+		break;
+	case htonl ( PEERDIST_MSG_AES_192_CBC ) :
+		keylen = ( 192 / 8 );
+		break;
+	case htonl ( PEERDIST_MSG_AES_256_CBC ) :
+		keylen = ( 256 / 8 );
+		break;
+	default:
+		DBGC ( peerblk, "PEERBLK %p %d.%d unrecognised algorithm "
+		       "%#08x\n", peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.algorithm ) );
+		return -ENOTSUP;
+	}
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d using %s with %zd-bit key\n",
+		peerblk, peerblk->segment, peerblk->block,
+		( cipher ? cipher->name : "plaintext" ), ( 8 * keylen ) );
+
+	/* Sanity check key length against maximum secret length */
+	if ( keylen > peerblk->digestsize ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d %zd-byte secret too short "
+		       "for %zd-bit key\n", peerblk, peerblk->segment,
+		       peerblk->block, peerblk->digestsize, ( 8 * keylen ) );
+		return -EPROTO;
+	}
+
+	/* Allocate cipher context.  Freeing the cipher context (on
+	 * error or otherwise) is handled by peerblk_reset().
+	 */
+	peerblk->cipher = cipher;
+	assert ( peerblk->cipherctx == NULL );
+	peerblk->cipherctx = malloc ( cipher->ctxsize );
+	if ( ! peerblk->cipherctx )
+		return -ENOMEM;
+
+	/* Initialise cipher */
+	if ( ( rc = cipher_setkey ( cipher, peerblk->cipherctx, peerblk->secret,
+				    keylen ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not set key: %s\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Parse retrieval protocol message segment and block details
+ *
+ * @v peerblk		PeerDist block download
+ * @v buf_len		Length of buffered data to fill in
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_block ( struct peerdist_block *peerblk,
+				 size_t *buf_len ) {
+	size_t digestsize = peerblk->digestsize;
+	peerblk_msg_blk_t ( digestsize, 0, 0, 0 ) *msg = peerblk->buffer.data;
+	size_t len = peerblk->buffer.len;
+	size_t data_len;
+	size_t total;
+
+	/* Check message length */
+	if ( len < offsetof ( typeof ( *msg ), msg.block.data ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for "
+		       "zero-length data (%zd bytes)\n", peerblk,
+		       peerblk->segment, peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Check digest size */
+	if ( ntohl ( msg->msg.segment.segment.digestsize ) != digestsize ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d incorrect digest size %d\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.segment.segment.digestsize ) );
+		return -EPROTO;
+	}
+
+	/* Check segment ID */
+	if ( memcmp ( msg->msg.segment.id, peerblk->id, digestsize ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d segment ID mismatch\n",
+		       peerblk, peerblk->segment, peerblk->block );
+		return -EPROTO;
+	}
+
+	/* Check block ID */
+	if ( ntohl ( msg->msg.index ) != peerblk->block ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d block ID mismatch (got %d)\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.index ) );
+		return -EPROTO;
+	}
+
+	/* Check for missing blocks */
+	data_len = be32_to_cpu ( msg->msg.block.block.len );
+	if ( ! data_len ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d block not found\n",
+		       peerblk, peerblk->segment, peerblk->block );
+		return -ENOENT;
+	}
+
+	/* Check for underlength blocks */
+	if ( data_len < ( peerblk->range.end - peerblk->range.start ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d underlength block (%zd "
+		       "bytes)\n", peerblk, peerblk->segment, peerblk->block,
+		       data_len );
+		return -ERANGE;
+	}
+
+	/* Calculate buffered data length (i.e. excluding data which
+	 * was delivered to the final data transfer buffer).
+	 */
+	*buf_len = ( data_len - ( peerblk->end - peerblk->start ) );
+
+	/* Describe data before the trimmed content */
+	peerblk->decrypt[PEERBLK_BEFORE].xferbuf = &peerblk->buffer;
+	peerblk->decrypt[PEERBLK_BEFORE].offset =
+		offsetof ( typeof ( *msg ), msg.block.data );
+	peerblk->decrypt[PEERBLK_BEFORE].len =
+		( peerblk->start -
+		  offsetof ( typeof ( *msg ), msg.block.data ) );
+	total = peerblk->decrypt[PEERBLK_BEFORE].len;
+
+	/* Describe data within the trimmed content */
+	peerblk->decrypt[PEERBLK_DURING].offset =
+		peerblk_offset ( peerblk, peerblk->start );
+	peerblk->decrypt[PEERBLK_DURING].len =
+		( peerblk->end - peerblk->start );
+	total += peerblk->decrypt[PEERBLK_DURING].len;
+
+	/* Describe data after the trimmed content */
+	peerblk->decrypt[PEERBLK_AFTER].xferbuf = &peerblk->buffer;
+	peerblk->decrypt[PEERBLK_AFTER].offset = peerblk->start;
+	peerblk->decrypt[PEERBLK_AFTER].len =
+		( offsetof ( typeof ( *msg ), msg.block.data )
+		  + *buf_len - peerblk->start );
+	total += peerblk->decrypt[PEERBLK_AFTER].len;
+
+	/* Sanity check */
+	assert ( total == be32_to_cpu ( msg->msg.block.block.len ) );
+
+	/* Initialise cipher and digest lengths */
+	peerblk->cipher_remaining = total;
+	peerblk->digest_remaining =
+		( peerblk->range.end - peerblk->range.start );
+	assert ( peerblk->cipher_remaining >= peerblk->digest_remaining );
+
+	return 0;
+}
+
+/**
+ * Parse retrieval protocol message useless details
+ *
+ * @v peerblk		PeerDist block download
+ * @v buf_len		Length of buffered data
+ * @v vrf_len		Length of uselessness to fill in
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_useless ( struct peerdist_block *peerblk,
+				   size_t buf_len, size_t *vrf_len ) {
+	size_t digestsize = peerblk->digestsize;
+	peerblk_msg_blk_t ( digestsize, buf_len, 0, 0 ) *msg =
+		peerblk->buffer.data;
+	size_t len = peerblk->buffer.len;
+
+	/* Check message length */
+	if ( len < offsetof ( typeof ( *msg ), msg.vrf.data ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for "
+		       "zero-length uselessness (%zd bytes)\n", peerblk,
+		       peerblk->segment, peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Extract length of uselessness */
+	*vrf_len = be32_to_cpu ( msg->msg.vrf.vrf.len );
+
+	return 0;
+}
+
+/**
+ * Parse retrieval protocol message initialisation vector details
+ *
+ * @v peerblk		PeerDist block download
+ * @v buf_len		Length of buffered data
+ * @v vrf_len		Length of uselessness
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_iv ( struct peerdist_block *peerblk, size_t buf_len,
+			      size_t vrf_len ) {
+	size_t digestsize = peerblk->digestsize;
+	size_t blksize = peerblk->cipher->blocksize;
+	peerblk_msg_blk_t ( digestsize, buf_len, vrf_len, blksize ) *msg =
+		peerblk->buffer.data;
+	size_t len = peerblk->buffer.len;
+
+	/* Check message length */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for "
+		       "initialisation vector (%zd bytes)\n", peerblk,
+		       peerblk->segment, peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Check initialisation vector size */
+	if ( ntohl ( msg->msg.iv.iv.blksize ) != blksize ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d incorrect IV size %d\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.iv.iv.blksize ) );
+		return -EPROTO;
+	}
+
+	/* Set initialisation vector */
+	cipher_setiv ( peerblk->cipher, peerblk->cipherctx, msg->msg.iv.data );
+
+	return 0;
+}
+
+/**
+ * Read from decryption buffers
+ *
+ * @v peerblk		PeerDist block download
+ * @v data		Data buffer
+ * @v len		Length to read
+ * @ret rc		Return status code
+ */
+static int peerblk_decrypt_read ( struct peerdist_block *peerblk,
+				  void *data, size_t len ) {
+	struct peerdist_block_decrypt *decrypt = peerblk->decrypt;
+	size_t frag_len;
+	int rc;
+
+	/* Read from each decryption buffer in turn */
+	for ( ; len ; decrypt++, data += frag_len, len -= frag_len ) {
+
+		/* Calculate length to use from this buffer */
+		frag_len = decrypt->len;
+		if ( frag_len > len )
+			frag_len = len;
+		if ( ! frag_len )
+			continue;
+
+		/* Read from this buffer */
+		if ( ( rc = xferbuf_read ( decrypt->xferbuf, decrypt->offset,
+					   data, frag_len ) ) != 0 )
+			return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Write to decryption buffers and update offsets and lengths
+ *
+ * @v peerblk		PeerDist block download
+ * @v data		Data buffer
+ * @v len		Length to read
+ * @ret rc		Return status code
+ */
+static int peerblk_decrypt_write ( struct peerdist_block *peerblk,
+				   const void *data, size_t len ) {
+	struct peerdist_block_decrypt *decrypt = peerblk->decrypt;
+	size_t frag_len;
+	int rc;
+
+	/* Write to each decryption buffer in turn */
+	for ( ; len ; decrypt++, data += frag_len, len -= frag_len ) {
+
+		/* Calculate length to use from this buffer */
+		frag_len = decrypt->len;
+		if ( frag_len > len )
+			frag_len = len;
+		if ( ! frag_len )
+			continue;
+
+		/* Write to this buffer */
+		if ( ( rc = xferbuf_write ( decrypt->xferbuf, decrypt->offset,
+					    data, frag_len ) ) != 0 )
+			return rc;
+
+		/* Update offset and length */
+		decrypt->offset += frag_len;
+		decrypt->len -= frag_len;
+	}
+
+	return 0;
+}
+
+/**
+ * Decrypt one chunk of PeerDist retrieval protocol data
+ *
+ * @v peerblk		PeerDist block download
+ */
+static void peerblk_decrypt ( struct peerdist_block *peerblk ) {
+	struct cipher_algorithm *cipher = peerblk->cipher;
+	struct digest_algorithm *digest = peerblk->digest;
+	struct xfer_buffer *xferbuf;
+	size_t cipher_len;
+	size_t digest_len;
+	void *data;
+	int rc;
+
+	/* Sanity check */
+	assert ( ( PEERBLK_DECRYPT_CHUNKSIZE % cipher->blocksize ) == 0 );
+
+	/* Get the underlying data transfer buffer */
+	xferbuf = xfer_buffer ( &peerblk->xfer );
+	if ( ! xferbuf ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d has no underlying data "
+		       "transfer buffer\n", peerblk, peerblk->segment,
+		       peerblk->block );
+		rc = -ENOTSUP;
+		goto err_xfer_buffer;
+	}
+	peerblk->decrypt[PEERBLK_DURING].xferbuf = xferbuf;
+
+	/* Calculate cipher and digest lengths */
+	cipher_len = PEERBLK_DECRYPT_CHUNKSIZE;
+	if ( cipher_len > peerblk->cipher_remaining )
+		cipher_len = peerblk->cipher_remaining;
+	digest_len = cipher_len;
+	if ( digest_len > peerblk->digest_remaining )
+		digest_len = peerblk->digest_remaining;
+	assert ( ( cipher_len & ( cipher->blocksize - 1 ) ) == 0 );
+
+	/* Allocate temporary data buffer */
+	data = malloc ( cipher_len );
+	if ( ! data ) {
+		rc = -ENOMEM;
+		goto err_alloc_data;
+	}
+
+	/* Read ciphertext */
+	if ( ( rc = peerblk_decrypt_read ( peerblk, data, cipher_len ) ) != 0 ){
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not read ciphertext: "
+		       "%s\n", peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		goto err_read;
+	}
+
+	/* Decrypt data */
+	cipher_decrypt ( cipher, peerblk->cipherctx, data, data, cipher_len );
+
+	/* Add data to digest */
+	digest_update ( digest, peerblk->digestctx, data, digest_len );
+
+	/* Write plaintext */
+	if ( ( rc = peerblk_decrypt_write ( peerblk, data, cipher_len ) ) != 0){
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not write plaintext: "
+		       "%s\n", peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		goto err_write;
+	}
+
+	/* Consume input */
+	peerblk->cipher_remaining -= cipher_len;
+	peerblk->digest_remaining -= digest_len;
+
+	/* Free temporary data buffer */
+	free ( data );
+
+	/* Continue processing until all input is consumed */
+	if ( peerblk->cipher_remaining )
+		return;
+
+	/* Complete download attempt */
+	peerblk_done ( peerblk, 0 );
+	return;
+
+ err_write:
+ err_read:
+	free ( data );
+ err_alloc_data:
+ err_xfer_buffer:
+	peerblk_done ( peerblk, rc );
+}
+
+/**
+ * Close PeerDist retrieval protocol block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_retrieval_close ( struct peerdist_block *peerblk, int rc ) {
+	size_t buf_len;
+	size_t vrf_len;
+
+	/* Restart interface */
+	intf_restart ( &peerblk->retrieval, rc );
+
+	/* Fail immediately if we have an error */
+	if ( rc != 0 )
+		goto done;
+
+	/* Abort download attempt (for testing) if applicable */
+	if ( ( rc = inject_fault ( PEERBLK_ABORT_RATE ) ) != 0 )
+		goto done;
+
+	/* Parse message header */
+	if ( ( rc = peerblk_parse_header ( peerblk ) ) != 0 )
+		goto done;
+
+	/* Parse message segment and block details */
+	if ( ( rc = peerblk_parse_block ( peerblk, &buf_len ) ) != 0 )
+		goto done;
+
+	/* If the block was plaintext, then there is nothing more to do */
+	if ( ! peerblk->cipher )
+		goto done;
+
+	/* Parse message useless details */
+	if ( ( rc = peerblk_parse_useless ( peerblk, buf_len, &vrf_len ) ) != 0)
+		goto done;
+
+	/* Parse message initialisation vector details */
+	if ( ( rc = peerblk_parse_iv ( peerblk, buf_len, vrf_len ) ) != 0 )
+		goto done;
+
+	/* Fail if decryption length is not aligned to the cipher block size */
+	if ( peerblk->cipher_remaining & ( peerblk->cipher->blocksize - 1 ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d unaligned data length %zd\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       peerblk->cipher_remaining );
+		rc = -EPROTO;
+		goto done;
+	}
+
+	/* Stop the download attempt timer: there is no point in
+	 * timing out while decrypting.
+	 */
+	stop_timer ( &peerblk->timer );
+
+	/* Start decryption process */
+	process_add ( &peerblk->process );
+	return;
+
+ done:
+	/* Complete download attempt */
+	peerblk_done ( peerblk, rc );
+}
+
+/******************************************************************************
+ *
+ * Retry policy
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Handle PeerDist retry timer expiry
+ *
+ * @v timer		Retry timer
+ * @v over		Failure indicator
+ */
+static void peerblk_expired ( struct retry_timer *timer, int over __unused ) {
+	struct peerdist_block *peerblk =
+		container_of ( timer, struct peerdist_block, timer );
+	struct peerdisc_segment *segment = peerblk->discovery.segment;
+	struct peerdisc_peer *head;
+	unsigned long now = peerblk_timestamp();
+	const char *location;
+	int rc;
+
+	/* Profile discovery timeout, if applicable */
+	if ( ( peerblk->peer == NULL ) && ( timer->timeout != 0 ) ) {
+		profile_custom ( &peerblk_discovery_timeout_profiler,
+				 ( now - peerblk->started ) );
+		DBGC ( peerblk, "PEERBLK %p %d.%d discovery timed out after "
+		       "%ld ticks\n", peerblk, peerblk->segment,
+		       peerblk->block, timer->timeout );
+	}
+
+	/* Profile download timeout, if applicable */
+	if ( ( peerblk->peer != NULL ) && ( timer->timeout != 0 ) ) {
+		profile_custom ( &peerblk_attempt_timeout_profiler,
+				 ( now - peerblk->attempted ) );
+		DBGC ( peerblk, "PEERBLK %p %d.%d timed out after %ld ticks\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       timer->timeout );
+	}
+
+	/* Abort any current download attempt */
+	peerblk_reset ( peerblk, -ETIMEDOUT );
+
+	/* Record attempt start time */
+	peerblk->attempted = now;
+
+	/* If we have exceeded our maximum number of attempt cycles
+	 * (each cycle comprising a retrieval protocol download from
+	 * each peer in the list followed by a raw download from the
+	 * origin server), then abort the overall download.
+	 */
+	head = list_entry ( &segment->peers, struct peerdisc_peer, list );
+	if ( ( peerblk->peer == head ) &&
+	     ( ++peerblk->cycles >= PEERBLK_MAX_ATTEMPT_CYCLES ) ) {
+		rc = peerblk->rc;
+		assert ( rc != 0 );
+		goto err;
+	}
+
+	/* If we have not yet made any download attempts, then move to
+	 * the start of the peer list.
+	 */
+	if ( peerblk->peer == NULL )
+		peerblk->peer = head;
+
+	/* Attempt retrieval protocol download from next usable peer */
+	list_for_each_entry_continue ( peerblk->peer, &segment->peers, list ) {
+
+		/* Attempt retrieval protocol download from this peer */
+		location = peerblk->peer->location;
+		if ( ( rc = peerblk_retrieval_open ( peerblk,
+						     location ) ) != 0 ) {
+			/* Non-fatal: continue to try next peer */
+			continue;
+		}
+
+		/* Start download attempt timer */
+		peerblk->rc = -ETIMEDOUT;
+		start_timer_fixed ( &peerblk->timer,
+				    PEERBLK_RETRIEVAL_OPEN_TIMEOUT );
+		return;
+	}
+
+	/* Attempt raw download */
+	if ( ( rc = peerblk_raw_open ( peerblk ) ) != 0 )
+		goto err;
+
+	/* Start download attempt timer */
+	peerblk->rc = -ETIMEDOUT;
+	start_timer_fixed ( &peerblk->timer, PEERBLK_RAW_OPEN_TIMEOUT );
+	return;
+
+ err:
+	peerblk_close ( peerblk, rc );
+}
+
+/**
+ * Handle PeerDist peer discovery
+ *
+ * @v discovery		PeerDist discovery client
+ */
+static void peerblk_discovered ( struct peerdisc_client *discovery ) {
+	struct peerdist_block *peerblk =
+		container_of ( discovery, struct peerdist_block, discovery );
+	unsigned long now = peerblk_timestamp();
+
+	/* Do nothing unless we are still waiting for the initial
+	 * discovery timeout.
+	 */
+	if ( ( peerblk->peer != NULL ) || ( peerblk->timer.timeout == 0 ) )
+		return;
+
+	/* Schedule an immediate retry */
+	start_timer_nodelay ( &peerblk->timer );
+
+	/* Profile discovery success */
+	profile_custom ( &peerblk_discovery_success_profiler,
+			 ( now - peerblk->started ) );
+}
+
+/******************************************************************************
+ *
+ * Opener
+ *
+ ******************************************************************************
+ */
+
+/** PeerDist block download data transfer interface operations */
+static struct interface_operation peerblk_xfer_operations[] = {
+	INTF_OP ( intf_close, struct peerdist_block *, peerblk_close ),
+};
+
+/** PeerDist block download data transfer interface descriptor */
+static struct interface_descriptor peerblk_xfer_desc =
+	INTF_DESC ( struct peerdist_block, xfer, peerblk_xfer_operations );
+
+/** PeerDist block download raw data interface operations */
+static struct interface_operation peerblk_raw_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_block *, peerblk_raw_rx ),
+	INTF_OP ( intf_close, struct peerdist_block *, peerblk_raw_close ),
+};
+
+/** PeerDist block download raw data interface descriptor */
+static struct interface_descriptor peerblk_raw_desc =
+	INTF_DESC ( struct peerdist_block, raw, peerblk_raw_operations );
+
+/** PeerDist block download retrieval protocol interface operations */
+static struct interface_operation peerblk_retrieval_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_block *, peerblk_retrieval_rx ),
+	INTF_OP ( intf_close, struct peerdist_block *, peerblk_retrieval_close),
+};
+
+/** PeerDist block download retrieval protocol interface descriptor */
+static struct interface_descriptor peerblk_retrieval_desc =
+	INTF_DESC ( struct peerdist_block, retrieval,
+		    peerblk_retrieval_operations );
+
+/** PeerDist block download decryption process descriptor */
+static struct process_descriptor peerblk_process_desc =
+	PROC_DESC ( struct peerdist_block, process, peerblk_decrypt );
+
+/** PeerDist block download discovery operations */
+static struct peerdisc_client_operations peerblk_discovery_operations = {
+	.discovered = peerblk_discovered,
+};
+
+/**
+ * Open PeerDist block download
+ *
+ * @v xfer		Data transfer interface
+ * @v uri		Original URI
+ * @v info		Content information block
+ * @ret rc		Return status code
+ */
+int peerblk_open ( struct interface *xfer, struct uri *uri,
+		   struct peerdist_info_block *block ) {
+	const struct peerdist_info_segment *segment = block->segment;
+	const struct peerdist_info *info = segment->info;
+	struct digest_algorithm *digest = info->digest;
+	struct peerdist_block *peerblk;
+	unsigned long timeout;
+	size_t digestsize;
+	int rc;
+
+	/* Allocate and initialise structure */
+	peerblk = zalloc ( sizeof ( *peerblk ) + digest->ctxsize );
+	if ( ! peerblk ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	ref_init ( &peerblk->refcnt, peerblk_free );
+	intf_init ( &peerblk->xfer, &peerblk_xfer_desc, &peerblk->refcnt );
+	intf_init ( &peerblk->raw, &peerblk_raw_desc, &peerblk->refcnt );
+	intf_init ( &peerblk->retrieval, &peerblk_retrieval_desc,
+		    &peerblk->refcnt );
+	peerblk->uri = uri_get ( uri );
+	memcpy ( &peerblk->range, &block->range, sizeof ( peerblk->range ) );
+	memcpy ( &peerblk->trim, &block->trim, sizeof ( peerblk->trim ) );
+	peerblk->offset = ( block->trim.start - info->trim.start );
+	peerblk->digest = info->digest;
+	peerblk->digestsize = digestsize = info->digestsize;
+	peerblk->digestctx = ( ( ( void * ) peerblk ) + sizeof ( *peerblk ) );
+	peerblk->segment = segment->index;
+	memcpy ( peerblk->id, segment->id, sizeof ( peerblk->id ) );
+	memcpy ( peerblk->secret, segment->secret, sizeof ( peerblk->secret ) );
+	peerblk->block = block->index;
+	memcpy ( peerblk->hash, block->hash, sizeof ( peerblk->hash ) );
+	xferbuf_malloc_init ( &peerblk->buffer );
+	process_init_stopped ( &peerblk->process, &peerblk_process_desc,
+			       &peerblk->refcnt );
+	peerdisc_init ( &peerblk->discovery, &peerblk_discovery_operations );
+	timer_init ( &peerblk->timer, peerblk_expired, &peerblk->refcnt );
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d id %02x%02x%02x%02x%02x..."
+		"%02x%02x%02x [%08zx,%08zx)", peerblk, peerblk->segment,
+		peerblk->block, peerblk->id[0], peerblk->id[1], peerblk->id[2],
+		peerblk->id[3], peerblk->id[4], peerblk->id[ digestsize - 3 ],
+		peerblk->id[ digestsize - 2 ], peerblk->id[ digestsize - 1 ],
+		peerblk->range.start, peerblk->range.end );
+	if ( ( peerblk->trim.start != peerblk->range.start ) ||
+	     ( peerblk->trim.end != peerblk->range.end ) ) {
+		DBGC2 ( peerblk, " covers [%08zx,%08zx)",
+			peerblk->trim.start, peerblk->trim.end );
+	}
+	DBGC2 ( peerblk, "\n" );
+
+	/* Open discovery */
+	if ( ( rc = peerdisc_open ( &peerblk->discovery, peerblk->id,
+				    peerblk->digestsize ) ) != 0 )
+		goto err_open_discovery;
+
+	/* Schedule a retry attempt either immediately (if we already
+	 * have some peers) or after the discovery timeout.
+	 */
+	timeout = ( list_empty ( &peerblk->discovery.segment->peers ) ?
+		    ( peerdisc_timeout_secs * TICKS_PER_SEC ) : 0 );
+	start_timer_fixed ( &peerblk->timer, timeout );
+
+	/* Record start time */
+	peerblk->started = peerblk_timestamp();
+
+	/* Attach to parent interface, mortalise self, and return */
+	intf_plug_plug ( xfer, &peerblk->xfer );
+	ref_put ( &peerblk->refcnt );
+	return 0;
+
+ err_open_discovery:
+	peerblk_close ( peerblk, rc );
+ err_alloc:
+	return rc;
+}
diff --git a/roms/ipxe/src/net/peerdisc.c b/roms/ipxe/src/net/peerdisc.c
new file mode 100644
index 000000000..5b0e98911
--- /dev/null
+++ b/roms/ipxe/src/net/peerdisc.c
@@ -0,0 +1,551 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/xfer.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/open.h>
+#include <ipxe/tcpip.h>
+#include <ipxe/uuid.h>
+#include <ipxe/base16.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/timer.h>
+#include <ipxe/fault.h>
+#include <ipxe/pccrd.h>
+#include <ipxe/peerdisc.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol peer discovery
+ *
+ */
+
+/** List of discovery segments */
+static LIST_HEAD ( peerdisc_segments );
+
+/** Number of repeated discovery attempts */
+#define PEERDISC_REPEAT_COUNT 2
+
+/** Time between repeated discovery attempts */
+#define PEERDISC_REPEAT_TIMEOUT ( 1 * TICKS_PER_SEC )
+
+/** Default discovery timeout (in seconds) */
+#define PEERDISC_DEFAULT_TIMEOUT_SECS 2
+
+/** Recommended discovery timeout (in seconds)
+ *
+ * We reduce the recommended discovery timeout whenever a segment
+ * fails to discover any peers, and restore the default value whenever
+ * a valid discovery reply is received.  We continue to send discovery
+ * requests even if the recommended timeout is reduced to zero.
+ *
+ * This strategy is intended to minimise discovery delays when no
+ * peers are available on the network, while allowing downloads to
+ * quickly switch back to using PeerDist acceleration if new peers
+ * become available.
+ */
+unsigned int peerdisc_timeout_secs = PEERDISC_DEFAULT_TIMEOUT_SECS;
+
+static struct peerdisc_segment * peerdisc_find ( const char *id );
+static int peerdisc_discovered ( struct peerdisc_segment *segment,
+				 const char *location );
+
+/******************************************************************************
+ *
+ * Discovery sockets
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open all PeerDist discovery sockets
+ *
+ * @ret rc		Return status code
+ */
+static int peerdisc_socket_open ( void ) {
+	struct peerdisc_socket *socket;
+	int rc;
+
+	/* Open each socket */
+	for_each_table_entry ( socket, PEERDISC_SOCKETS ) {
+		if ( ( rc = xfer_open_socket ( &socket->xfer, SOCK_DGRAM,
+					       &socket->address.sa,
+					       NULL ) ) != 0 ) {
+			DBGC ( socket, "PEERDISC %s could not open socket: "
+			       "%s\n", socket->name, strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	return 0;
+
+ err:
+	for_each_table_entry_continue_reverse ( socket, PEERDISC_SOCKETS )
+		intf_restart ( &socket->xfer, rc );
+	return rc;
+}
+
+/**
+ * Attempt to transmit PeerDist discovery requests on all sockets
+ *
+ * @v uuid		Message UUID string
+ * @v id		Segment identifier string
+ */
+static void peerdisc_socket_tx ( const char *uuid, const char *id ) {
+	struct peerdisc_socket *socket;
+	struct net_device *netdev;
+	struct xfer_metadata meta;
+	union {
+		struct sockaddr sa;
+		struct sockaddr_tcpip st;
+	} address;
+	char *request;
+	size_t len;
+	int rc;
+
+	/* Construct discovery request */
+	request = peerdist_discovery_request ( uuid, id );
+	if ( ! request )
+		goto err_request;
+	len = strlen ( request );
+
+	/* Initialise data transfer metadata */
+	memset ( &meta, 0, sizeof ( meta ) );
+	meta.dest = &address.sa;
+
+	/* Send message on each socket */
+	for_each_table_entry ( socket, PEERDISC_SOCKETS ) {
+
+		/* Initialise socket address */
+		memcpy ( &address.sa, &socket->address.sa,
+			 sizeof ( address.sa ) );
+
+		/* Send message on each open network device */
+		for_each_netdev ( netdev ) {
+
+			/* Skip unopened network devices */
+			if ( ! netdev_is_open ( netdev ) )
+				continue;
+			address.st.st_scope_id = netdev->index;
+
+			/* Discard request (for test purposes) if applicable */
+			if ( inject_fault ( PEERDISC_DISCARD_RATE ) )
+				continue;
+
+			/* Transmit request */
+			if ( ( rc = xfer_deliver_raw_meta ( &socket->xfer,
+							    request, len,
+							    &meta ) ) != 0 ) {
+				DBGC ( socket, "PEERDISC %s could not transmit "
+				       "via %s: %s\n", socket->name,
+				       netdev->name, strerror ( rc ) );
+				/* Contine to try other net devices/sockets */
+				continue;
+			}
+		}
+	}
+
+	free ( request );
+ err_request:
+	return;
+}
+
+/**
+ * Handle received PeerDist discovery reply
+ *
+ * @v socket		PeerDist discovery socket
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peerdisc_socket_rx ( struct peerdisc_socket *socket,
+				struct io_buffer *iobuf,
+				struct xfer_metadata *meta __unused ) {
+	struct peerdist_discovery_reply reply;
+	struct peerdisc_segment *segment;
+	char *id;
+	char *location;
+	int rc;
+
+	/* Discard reply (for test purposes) if applicable */
+	if ( ( rc = inject_fault ( PEERDISC_DISCARD_RATE ) ) != 0 )
+		goto err;
+
+	/* Parse reply */
+	if ( ( rc = peerdist_discovery_reply ( iobuf->data, iob_len ( iobuf ),
+					       &reply ) ) != 0 ) {
+		DBGC ( socket, "PEERDISC %s could not parse reply: %s\n",
+		       socket->name, strerror ( rc ) );
+		DBGC_HDA ( socket, 0, iobuf->data, iob_len ( iobuf ) );
+		goto err;
+	}
+
+	/* Any kind of discovery reply indicates that there are active
+	 * peers on a local network, so restore the recommended
+	 * discovery timeout to its default value for future requests.
+	 */
+	if ( peerdisc_timeout_secs != PEERDISC_DEFAULT_TIMEOUT_SECS ) {
+		DBGC ( socket, "PEERDISC %s restoring timeout to %d seconds\n",
+		       socket->name, PEERDISC_DEFAULT_TIMEOUT_SECS );
+	}
+	peerdisc_timeout_secs = PEERDISC_DEFAULT_TIMEOUT_SECS;
+
+	/* Iterate over segment IDs */
+	for ( id = reply.ids ; *id ; id += ( strlen ( id ) + 1 /* NUL */ ) ) {
+
+		/* Find corresponding segment */
+		segment = peerdisc_find ( id );
+		if ( ! segment ) {
+			DBGC ( socket, "PEERDISC %s ignoring reply for %s\n",
+			       socket->name, id );
+			continue;
+		}
+
+		/* Report all discovered peer locations */
+		for ( location = reply.locations ; *location ;
+		      location += ( strlen ( location ) + 1 /* NUL */ ) ) {
+
+			/* Report discovered peer location */
+			if ( ( rc = peerdisc_discovered ( segment,
+							  location ) ) != 0 )
+				goto err;
+		}
+	}
+
+ err:
+	free_iob ( iobuf );
+	return rc;
+}
+
+/**
+ * Close all PeerDist discovery sockets
+ *
+ * @v rc		Reason for close
+ */
+static void peerdisc_socket_close ( int rc ) {
+	struct peerdisc_socket *socket;
+
+	/* Close all sockets */
+	for_each_table_entry ( socket, PEERDISC_SOCKETS )
+		intf_restart ( &socket->xfer, rc );
+}
+
+/** PeerDist discovery socket interface operations */
+static struct interface_operation peerdisc_socket_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdisc_socket *, peerdisc_socket_rx ),
+};
+
+/** PeerDist discovery socket interface descriptor */
+static struct interface_descriptor peerdisc_socket_desc =
+	INTF_DESC ( struct peerdisc_socket, xfer, peerdisc_socket_operations );
+
+/** PeerDist discovery IPv4 socket */
+struct peerdisc_socket peerdisc_socket_ipv4 __peerdisc_socket = {
+	.name = "IPv4",
+	.address = {
+		.sin = {
+			.sin_family = AF_INET,
+			.sin_port = htons ( PEERDIST_DISCOVERY_PORT ),
+			.sin_addr.s_addr = htonl ( PEERDIST_DISCOVERY_IPV4 ),
+		},
+	},
+	.xfer = INTF_INIT ( peerdisc_socket_desc ),
+};
+
+/** PeerDist discovery IPv6 socket */
+struct peerdisc_socket peerdisc_socket_ipv6 __peerdisc_socket = {
+	.name = "IPv6",
+	.address = {
+		.sin6 = {
+			.sin6_family = AF_INET6,
+			.sin6_port = htons ( PEERDIST_DISCOVERY_PORT ),
+			.sin6_addr.s6_addr = PEERDIST_DISCOVERY_IPV6,
+		},
+	},
+	.xfer = INTF_INIT ( peerdisc_socket_desc ),
+};
+
+/******************************************************************************
+ *
+ * Discovery segments
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Free PeerDist discovery segment
+ *
+ * @v refcnt		Reference count
+ */
+static void peerdisc_free ( struct refcnt *refcnt ) {
+	struct peerdisc_segment *segment =
+		container_of ( refcnt, struct peerdisc_segment, refcnt );
+	struct peerdisc_peer *peer;
+	struct peerdisc_peer *tmp;
+
+	/* Free all discovered peers */
+	list_for_each_entry_safe ( peer, tmp, &segment->peers, list ) {
+		list_del ( &peer->list );
+		free ( peer );
+	}
+
+	/* Free segment */
+	free ( segment );
+}
+
+/**
+ * Find PeerDist discovery segment
+ *
+ * @v id		Segment ID
+ * @ret segment		PeerDist discovery segment, or NULL if not found
+ */
+static struct peerdisc_segment * peerdisc_find ( const char *id ) {
+	struct peerdisc_segment *segment;
+
+	/* Look for a matching segment */
+	list_for_each_entry ( segment, &peerdisc_segments, list ) {
+		if ( strcmp ( id, segment->id ) == 0 )
+			return segment;
+	}
+
+	return NULL;
+}
+
+/**
+ * Add discovered PeerDist peer
+ *
+ * @v segment		PeerDist discovery segment
+ * @v location		Peer location
+ * @ret rc		Return status code
+ */
+static int peerdisc_discovered ( struct peerdisc_segment *segment,
+				 const char *location ) {
+	struct peerdisc_peer *peer;
+	struct peerdisc_client *peerdisc;
+	struct peerdisc_client *tmp;
+
+	/* Ignore duplicate peers */
+	list_for_each_entry ( peer, &segment->peers, list ) {
+		if ( strcmp ( peer->location, location ) == 0 ) {
+			DBGC2 ( segment, "PEERDISC %p duplicate %s\n",
+				segment, location );
+			return 0;
+		}
+	}
+	DBGC2 ( segment, "PEERDISC %p discovered %s\n", segment, location );
+
+	/* Allocate and initialise structure */
+	peer = zalloc ( sizeof ( *peer ) + strlen ( location ) + 1 /* NUL */ );
+	if ( ! peer )
+		return -ENOMEM;
+	strcpy ( peer->location, location );
+
+	/* Add to end of list of peers */
+	list_add_tail ( &peer->list, &segment->peers );
+
+	/* Notify all clients */
+	list_for_each_entry_safe ( peerdisc, tmp, &segment->clients, list )
+		peerdisc->op->discovered ( peerdisc );
+
+	return 0;
+}
+
+/**
+ * Handle discovery timer expiry
+ *
+ * @v timer		Discovery timer
+ * @v over		Failure indicator
+ */
+static void peerdisc_expired ( struct retry_timer *timer, int over __unused ) {
+	struct peerdisc_segment *segment =
+		container_of ( timer, struct peerdisc_segment, timer );
+
+	/* Attempt to transmit discovery requests */
+	peerdisc_socket_tx ( segment->uuid, segment->id );
+
+	/* Schedule next transmission, if applicable */
+	if ( timer->count < PEERDISC_REPEAT_COUNT )
+		start_timer_fixed ( &segment->timer, PEERDISC_REPEAT_TIMEOUT );
+}
+
+/**
+ * Create PeerDist discovery segment
+ *
+ * @v id		Segment ID
+ * @ret segment		PeerDist discovery segment, or NULL on error
+ */
+static struct peerdisc_segment * peerdisc_create ( const char *id ) {
+	struct peerdisc_segment *segment;
+	union {
+		union uuid uuid;
+		uint32_t dword[ sizeof ( union uuid ) / sizeof ( uint32_t ) ];
+	} random_uuid;
+	size_t uuid_len;
+	size_t id_len;
+	char *uuid;
+	char *uuid_copy;
+	char *id_copy;
+	unsigned int i;
+
+	/* Generate a random message UUID.  This does not require high
+	 * quality randomness.
+	 */
+	for ( i = 0 ; i < ( sizeof ( random_uuid.dword ) /
+			    sizeof ( random_uuid.dword[0] ) ) ; i++ )
+		random_uuid.dword[i] = random();
+	uuid = uuid_ntoa ( &random_uuid.uuid );
+
+	/* Calculate string lengths */
+	id_len = ( strlen ( id ) + 1 /* NUL */ );
+	uuid_len = ( strlen ( uuid ) + 1 /* NUL */ );
+
+	/* Allocate and initialise structure */
+	segment = zalloc ( sizeof ( *segment ) + id_len + uuid_len );
+	if ( ! segment )
+		return NULL;
+	id_copy = ( ( ( void * ) segment ) + sizeof ( *segment ) );
+	memcpy ( id_copy, id, id_len );
+	uuid_copy = ( ( ( void * ) id_copy ) + id_len );
+	memcpy ( uuid_copy, uuid, uuid_len );
+	ref_init ( &segment->refcnt, peerdisc_free );
+	segment->id = id_copy;
+	segment->uuid = uuid_copy;
+	INIT_LIST_HEAD ( &segment->peers );
+	INIT_LIST_HEAD ( &segment->clients );
+	timer_init ( &segment->timer, peerdisc_expired, &segment->refcnt );
+	DBGC2 ( segment, "PEERDISC %p discovering %s\n", segment, segment->id );
+
+	/* Start discovery timer */
+	start_timer_nodelay ( &segment->timer );
+
+	/* Add to list of segments, transfer reference to list, and return */
+	list_add_tail ( &segment->list, &peerdisc_segments );
+	return segment;
+}
+
+/**
+ * Destroy PeerDist discovery segment
+ *
+ * @v segment		PeerDist discovery segment
+ */
+static void peerdisc_destroy ( struct peerdisc_segment *segment ) {
+
+	/* Sanity check */
+	assert ( list_empty ( &segment->clients ) );
+
+	/* Stop timer */
+	stop_timer ( &segment->timer );
+
+	/* Remove from list of segments and drop list's reference */
+	list_del ( &segment->list );
+	ref_put ( &segment->refcnt );
+}
+
+/******************************************************************************
+ *
+ * Discovery clients
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open PeerDist discovery client
+ *
+ * @v peerdisc		PeerDist discovery client
+ * @v id		Segment ID
+ * @v len		Length of segment ID
+ * @ret rc		Return status code
+ */
+int peerdisc_open ( struct peerdisc_client *peerdisc, const void *id,
+		    size_t len ) {
+	struct peerdisc_segment *segment;
+	char id_string[ base16_encoded_len ( len ) + 1 /* NUL */ ];
+	char *id_chr;
+	int rc;
+
+	/* Construct ID string */
+	base16_encode ( id, len, id_string, sizeof ( id_string ) );
+	for ( id_chr = id_string ; *id_chr ; id_chr++ )
+		*id_chr = toupper ( *id_chr );
+
+	/* Sanity check */
+	assert ( peerdisc->segment == NULL );
+
+	/* Open socket if this is the first segment */
+	if ( list_empty ( &peerdisc_segments ) &&
+	     ( ( rc = peerdisc_socket_open() ) != 0 ) )
+		return rc;
+
+	/* Find or create segment */
+	if ( ! ( ( segment = peerdisc_find ( id_string ) ) ||
+		 ( segment = peerdisc_create ( id_string ) ) ) )
+		return -ENOMEM;
+
+	/* Add to list of clients */
+	ref_get ( &segment->refcnt );
+	peerdisc->segment = segment;
+	list_add_tail ( &peerdisc->list, &segment->clients );
+
+	return 0;
+}
+
+/**
+ * Close PeerDist discovery client
+ *
+ * @v peerdisc		PeerDist discovery client
+ */
+void peerdisc_close ( struct peerdisc_client *peerdisc ) {
+	struct peerdisc_segment *segment = peerdisc->segment;
+
+	/* Ignore if discovery is already closed */
+	if ( ! segment )
+		return;
+
+	/* If no peers were discovered, reduce the recommended
+	 * discovery timeout to minimise delays on future requests.
+	 */
+	if ( list_empty ( &segment->peers ) && peerdisc_timeout_secs ) {
+		peerdisc_timeout_secs--;
+		DBGC ( segment, "PEERDISC %p reducing timeout to %d "
+		       "seconds\n", peerdisc, peerdisc_timeout_secs );
+	}
+
+	/* Remove from list of clients */
+	peerdisc->segment = NULL;
+	list_del ( &peerdisc->list );
+	ref_put ( &segment->refcnt );
+
+	/* If this was the last clients, destroy the segment */
+	if ( list_empty ( &segment->clients ) )
+		peerdisc_destroy ( segment );
+
+	/* If there are no more segments, close the socket */
+	if ( list_empty ( &peerdisc_segments ) )
+		peerdisc_socket_close ( 0 );
+}
diff --git a/roms/ipxe/src/net/peerdist.c b/roms/ipxe/src/net/peerdist.c
new file mode 100644
index 000000000..48933f951
--- /dev/null
+++ b/roms/ipxe/src/net/peerdist.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdio.h>
+#include <ipxe/http.h>
+#include <ipxe/peermux.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol
+ *
+ * This is quite possibly the ugliest protocol I have ever had the
+ * misfortune to encounter, and I've encountered multicast TFTP.
+ */
+
+/**
+ * Check whether or not to support PeerDist encoding for this request
+ *
+ * @v http		HTTP transaction
+ * @ret supported	PeerDist encoding is supported for this request
+ */
+static int http_peerdist_supported ( struct http_transaction *http ) {
+
+	/* Support PeerDist encoding only if we can directly access an
+	 * underlying data transfer buffer.  Direct access is required
+	 * in order to support decryption of data received via the
+	 * retrieval protocol (which provides the AES initialisation
+	 * vector only after all of the encrypted data has been
+	 * received).
+	 *
+	 * This test simultaneously ensures that we do not attempt to
+	 * use PeerDist encoding on a request which is itself a
+	 * PeerDist individual block download, since the individual
+	 * block downloads do not themselves provide direct access to
+	 * an underlying data transfer buffer.
+	 */
+	return ( xfer_buffer ( &http->xfer ) != NULL );
+}
+
+/**
+ * Format HTTP "X-P2P-PeerDist" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_p2p_peerdist ( struct http_transaction *http,
+				      char *buf, size_t len ) {
+	int supported = http_peerdist_supported ( http );
+	int missing;
+
+	/* PeerDist wants us to inform the server whenever we make a
+	 * request for data that was missing from local peers
+	 * (presumably for statistical purposes only).  We use the
+	 * heuristic of assuming that the combination of "this request
+	 * may not itself use PeerDist content encoding" and "this is
+	 * a range request" probably indicates that we are making a
+	 * PeerDist block raw range request for missing data.
+	 */
+	missing = ( http->request.range.len && ( ! supported ) );
+
+	/* Omit header if PeerDist encoding is not supported and we
+	 * are not reporting a missing data request.
+	 */
+	if ( ! ( supported || missing ) )
+		return 0;
+
+	/* Construct header */
+	return snprintf ( buf, len, "Version=1.1%s",
+			  ( missing ? ", MissingDataRequest=true" : "" ) );
+}
+
+/** HTTP "X-P2P-PeerDist" header */
+struct http_request_header http_request_p2p_peerdist __http_request_header = {
+	.name = "X-P2P-PeerDist",
+	.format = http_format_p2p_peerdist,
+};
+
+/**
+ * Format HTTP "X-P2P-PeerDistEx" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_p2p_peerdistex ( struct http_transaction *http,
+					char *buf, size_t len ) {
+	int supported = http_peerdist_supported ( http );
+
+	/* Omit header if PeerDist encoding is not supported */
+	if ( ! supported )
+		return 0;
+
+	/* Construct header */
+	return snprintf ( buf, len, ( "MinContentInformation=1.0, "
+				      "MaxContentInformation=2.0" ) );
+}
+
+/** HTTP "X-P2P-PeerDist" header */
+struct http_request_header http_request_p2p_peerdistex __http_request_header = {
+	.name = "X-P2P-PeerDistEx",
+	.format = http_format_p2p_peerdistex,
+};
+
+/**
+ * Initialise PeerDist content encoding
+ *
+ * @v http		HTTP transaction
+ * @ret rc		Return status code
+ */
+static int http_peerdist_init ( struct http_transaction *http ) {
+
+	return peermux_filter ( &http->content, &http->transfer, http->uri );
+}
+
+/** PeerDist HTTP content encoding */
+struct http_content_encoding peerdist_encoding __http_content_encoding = {
+	.name = "peerdist",
+	.supported = http_peerdist_supported,
+	.init = http_peerdist_init,
+};
diff --git a/roms/ipxe/src/net/peermux.c b/roms/ipxe/src/net/peermux.c
new file mode 100644
index 000000000..634c69992
--- /dev/null
+++ b/roms/ipxe/src/net/peermux.c
@@ -0,0 +1,387 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <errno.h>
+#include <ipxe/uri.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/peerblk.h>
+#include <ipxe/peermux.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol multiplexer
+ *
+ */
+
+/**
+ * Free PeerDist download multiplexer
+ *
+ * @v refcnt		Reference count
+ */
+static void peermux_free ( struct refcnt *refcnt ) {
+	struct peerdist_multiplexer *peermux =
+		container_of ( refcnt, struct peerdist_multiplexer, refcnt );
+
+	uri_put ( peermux->uri );
+	xferbuf_free ( &peermux->buffer );
+	free ( peermux );
+}
+
+/**
+ * Close PeerDist download multiplexer
+ *
+ * @v peermux		PeerDist download multiplexer
+ * @v rc		Reason for close
+ */
+static void peermux_close ( struct peerdist_multiplexer *peermux, int rc ) {
+	unsigned int i;
+
+	/* Stop block download initiation process */
+	process_del ( &peermux->process );
+
+	/* Shut down all block downloads */
+	for ( i = 0 ; i < PEERMUX_MAX_BLOCKS ; i++ )
+		intf_shutdown ( &peermux->block[i].xfer, rc );
+
+	/* Shut down all other interfaces (which may be connected to
+	 * the same object).
+	 */
+	intf_nullify ( &peermux->info ); /* avoid potential loops */
+	intf_shutdown ( &peermux->xfer, rc );
+	intf_shutdown ( &peermux->info, rc );
+}
+
+/**
+ * Receive content information
+ *
+ * @v peermux		PeerDist download multiplexer
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peermux_info_deliver ( struct peerdist_multiplexer *peermux,
+				  struct io_buffer *iobuf,
+				  struct xfer_metadata *meta ) {
+	int rc;
+
+	/* Add data to buffer */
+	if ( ( rc = xferbuf_deliver ( &peermux->buffer, iobuf, meta ) ) != 0 )
+		goto err;
+
+	return 0;
+
+ err:
+	peermux_close ( peermux, rc );
+	return rc;
+}
+
+/**
+ * Close content information interface
+ *
+ * @v peermux		PeerDist download multiplexer
+ * @v rc		Reason for close
+ */
+static void peermux_info_close ( struct peerdist_multiplexer *peermux, int rc ){
+	struct peerdist_info *info = &peermux->cache.info;
+	size_t len;
+
+	/* Terminate download on error */
+	if ( rc != 0 )
+		goto err;
+
+	/* Successfully closing the content information interface
+	 * indicates that the content information has been fully
+	 * received, and initiates the actual PeerDist download.
+	 */
+
+	/* Shut down content information interface */
+	intf_shutdown ( &peermux->info, rc );
+
+	/* Parse content information */
+	if ( ( rc = peerdist_info ( info->raw.data, peermux->buffer.len,
+				    info ) ) != 0 ) {
+		DBGC ( peermux, "PEERMUX %p could not parse content info: %s\n",
+		       peermux, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Notify recipient of total download size */
+	len = ( info->trim.end - info->trim.start );
+	if ( ( rc = xfer_seek ( &peermux->xfer, len ) ) != 0 ) {
+		DBGC ( peermux, "PEERMUX %p could not presize buffer: %s\n",
+		       peermux, strerror ( rc ) );
+		goto err;
+	}
+	xfer_seek ( &peermux->xfer, 0 );
+
+	/* Start block download process */
+	process_add ( &peermux->process );
+
+	return;
+
+ err:
+	peermux_close ( peermux, rc );
+}
+
+/**
+ * Initiate multiplexed block download
+ *
+ * @v peermux		PeerDist download multiplexer
+ */
+static void peermux_step ( struct peerdist_multiplexer *peermux ) {
+	struct peerdist_info *info = &peermux->cache.info;
+	struct peerdist_info_segment *segment = &peermux->cache.segment;
+	struct peerdist_info_block *block = &peermux->cache.block;
+	struct peerdist_multiplexed_block *peermblk;
+	unsigned int next_segment;
+	unsigned int next_block;
+	int rc;
+
+	/* Stop initiation process if all block downloads are busy */
+	peermblk = list_first_entry ( &peermux->idle,
+				      struct peerdist_multiplexed_block, list );
+	if ( ! peermblk ) {
+		process_del ( &peermux->process );
+		return;
+	}
+
+	/* Increment block index */
+	next_block = ( block->index + 1 );
+
+	/* Move to first/next segment, if applicable */
+	if ( next_block >= segment->blocks ) {
+
+		/* Reset block index */
+		next_block = 0;
+
+		/* Calculate segment index */
+		next_segment = ( segment->info ? ( segment->index + 1 ) : 0 );
+
+		/* If we have finished all segments and have no
+		 * remaining block downloads, then we are finished.
+		 */
+		if ( next_segment >= info->segments ) {
+			process_del ( &peermux->process );
+			if ( list_empty ( &peermux->busy ) )
+				peermux_close ( peermux, 0 );
+			return;
+		}
+
+		/* Get content information segment */
+		if ( ( rc = peerdist_info_segment ( info, segment,
+						    next_segment ) ) != 0 ) {
+			DBGC ( peermux, "PEERMUX %p could not get segment %d "
+			       "information: %s\n", peermux, next_segment,
+			       strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	/* Get content information block */
+	if ( ( rc = peerdist_info_block ( segment, block, next_block ) ) != 0 ){
+		DBGC ( peermux, "PEERMUX %p could not get segment %d block "
+		       "%d information: %s\n", peermux, segment->index,
+		       next_block, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Ignore block if it lies entirely outside the trimmed range */
+	if ( block->trim.start == block->trim.end ) {
+		DBGC ( peermux, "PEERMUX %p skipping segment %d block %d\n",
+		       peermux, segment->index, block->index );
+		return;
+	}
+
+	/* Start downloading this block */
+	if ( ( rc = peerblk_open ( &peermblk->xfer, peermux->uri,
+				   block ) ) != 0 ) {
+		DBGC ( peermux, "PEERMUX %p could not start download for "
+		       "segment %d block %d: %s\n", peermux, segment->index,
+		       block->index, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Move to list of busy block downloads */
+	list_del ( &peermblk->list );
+	list_add_tail ( &peermblk->list, &peermux->busy );
+
+	return;
+
+ err:
+	peermux_close ( peermux, rc );
+}
+
+/**
+ * Receive data from multiplexed block download
+ *
+ * @v peermblk		PeerDist multiplexed block download
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peermux_block_deliver ( struct peerdist_multiplexed_block *peermblk,
+				   struct io_buffer *iobuf,
+				   struct xfer_metadata *meta ) {
+	struct peerdist_multiplexer *peermux = peermblk->peermux;
+
+	/* Sanity check: all block downloads must use absolute
+	 * positions for all deliveries, since they run concurrently.
+	 */
+	assert ( meta->flags & XFER_FL_ABS_OFFSET );
+
+	/* We can't use a simple passthrough interface descriptor,
+	 * since there are multiple block download interfaces.
+	 */
+	return xfer_deliver ( &peermux->xfer, iob_disown ( iobuf ), meta );
+}
+
+/**
+ * Get multiplexed block download underlying data transfer buffer
+ *
+ * @v peermblk		PeerDist multiplexed download block
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ */
+static struct xfer_buffer *
+peermux_block_buffer ( struct peerdist_multiplexed_block *peermblk ) {
+	struct peerdist_multiplexer *peermux = peermblk->peermux;
+
+	/* We can't use a simple passthrough interface descriptor,
+	 * since there are multiple block download interfaces.
+	 */
+	return xfer_buffer ( &peermux->xfer );
+}
+
+/**
+ * Close multiplexed block download
+ *
+ * @v peermblk		PeerDist multiplexed block download
+ * @v rc		Reason for close
+ */
+static void peermux_block_close ( struct peerdist_multiplexed_block *peermblk,
+				  int rc ) {
+	struct peerdist_multiplexer *peermux = peermblk->peermux;
+
+	/* Move to list of idle downloads */
+	list_del ( &peermblk->list );
+	list_add_tail ( &peermblk->list, &peermux->idle );
+
+	/* If any error occurred, terminate the whole multiplexer */
+	if ( rc != 0 ) {
+		peermux_close ( peermux, rc );
+		return;
+	}
+
+	/* Restart data transfer interface */
+	intf_restart ( &peermblk->xfer, rc );
+
+	/* Restart block download initiation process */
+	process_add ( &peermux->process );
+}
+
+/** Data transfer interface operations */
+static struct interface_operation peermux_xfer_operations[] = {
+	INTF_OP ( intf_close, struct peerdist_multiplexer *, peermux_close ),
+};
+
+/** Data transfer interface descriptor */
+static struct interface_descriptor peermux_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct peerdist_multiplexer, xfer,
+			     peermux_xfer_operations, info );
+
+/** Content information interface operations */
+static struct interface_operation peermux_info_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_multiplexer *,
+		  peermux_info_deliver ),
+	INTF_OP ( intf_close, struct peerdist_multiplexer *,
+		  peermux_info_close ),
+};
+
+/** Content information interface descriptor */
+static struct interface_descriptor peermux_info_desc =
+	INTF_DESC_PASSTHRU ( struct peerdist_multiplexer, info,
+			     peermux_info_operations, xfer );
+
+/** Block download data transfer interface operations */
+static struct interface_operation peermux_block_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_multiplexed_block *,
+		  peermux_block_deliver ),
+	INTF_OP ( xfer_buffer, struct peerdist_multiplexed_block *,
+		  peermux_block_buffer ),
+	INTF_OP ( intf_close, struct peerdist_multiplexed_block *,
+		  peermux_block_close ),
+};
+
+/** Block download data transfer interface descriptor */
+static struct interface_descriptor peermux_block_desc =
+	INTF_DESC ( struct peerdist_multiplexed_block, xfer,
+		    peermux_block_operations );
+
+/** Block download initiation process descriptor */
+static struct process_descriptor peermux_process_desc =
+	PROC_DESC ( struct peerdist_multiplexer, process, peermux_step );
+
+/**
+ * Add PeerDist content-encoding filter
+ *
+ * @v xfer		Data transfer interface
+ * @v info		Content information interface
+ * @v uri		Original URI
+ * @ret rc		Return status code
+ */
+int peermux_filter ( struct interface *xfer, struct interface *info,
+		     struct uri *uri ) {
+	struct peerdist_multiplexer *peermux;
+	struct peerdist_multiplexed_block *peermblk;
+	unsigned int i;
+
+	/* Allocate and initialise structure */
+	peermux = zalloc ( sizeof ( *peermux ) );
+	if ( ! peermux )
+		return -ENOMEM;
+	ref_init ( &peermux->refcnt, peermux_free );
+	intf_init ( &peermux->xfer, &peermux_xfer_desc, &peermux->refcnt );
+	intf_init ( &peermux->info, &peermux_info_desc, &peermux->refcnt );
+	peermux->uri = uri_get ( uri );
+	xferbuf_umalloc_init ( &peermux->buffer,
+			       &peermux->cache.info.raw.data );
+	process_init_stopped ( &peermux->process, &peermux_process_desc,
+			       &peermux->refcnt );
+	INIT_LIST_HEAD ( &peermux->busy );
+	INIT_LIST_HEAD ( &peermux->idle );
+	for ( i = 0 ; i < PEERMUX_MAX_BLOCKS ; i++ ) {
+		peermblk = &peermux->block[i];
+		peermblk->peermux = peermux;
+		list_add_tail ( &peermblk->list, &peermux->idle );
+		intf_init ( &peermblk->xfer, &peermux_block_desc,
+			    &peermux->refcnt );
+	}
+
+	/* Attach to parent interfaces, mortalise self, and return */
+	intf_plug_plug ( &peermux->xfer, xfer );
+	intf_plug_plug ( &peermux->info, info );
+	ref_put ( &peermux->refcnt );
+	return 0;
+}
diff --git a/roms/ipxe/src/net/ping.c b/roms/ipxe/src/net/ping.c
index d9da87ade..3f4fa5c11 100644
--- a/roms/ipxe/src/net/ping.c
+++ b/roms/ipxe/src/net/ping.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/roms/ipxe/src/net/rarp.c b/roms/ipxe/src/net/rarp.c
index 371145015..c194a404f 100644
--- a/roms/ipxe/src/net/rarp.c
+++ b/roms/ipxe/src/net/rarp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/roms/ipxe/src/net/retry.c b/roms/ipxe/src/net/retry.c
index 8f210bdcc..734567be5 100644
--- a/roms/ipxe/src/net/retry.c
+++ b/roms/ipxe/src/net/retry.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/timer.h>
@@ -35,7 +39,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  * This implementation of the timer is designed to satisfy RFC 2988
  * and therefore be usable as a TCP retransmission timer.
- *
  * 
  */
 
@@ -49,47 +52,59 @@ FILE_LICENCE ( GPL2_OR_LATER );
 static LIST_HEAD ( timers );
 
 /**
- * Start timer
+ * Start timer with a specified timeout
  *
  * @v timer		Retry timer
+ * @v timeout		Timeout, in ticks
  *
- * This starts the timer running with the current timeout value.  If
+ * This starts the timer running with the specified timeout value.  If
  * stop_timer() is not called before the timer expires, the timer will
  * be stopped and the timer's callback function will be called.
  */
-void start_timer ( struct retry_timer *timer ) {
+void start_timer_fixed ( struct retry_timer *timer, unsigned long timeout ) {
+
+	/* Add to list of running timers (if applicable) */
 	if ( ! timer->running ) {
 		list_add ( &timer->list, &timers );
 		ref_get ( timer->refcnt );
+		timer->running = 1;
 	}
+
+	/* Record start time */
 	timer->start = currticks();
-	timer->running = 1;
-
-	/* 0 means "use default timeout" */
-	if ( timer->min_timeout == 0 )
-		timer->min_timeout = DEFAULT_MIN_TIMEOUT;
-	/* We must never be less than MIN_TIMEOUT under any circumstances */
-	if ( timer->min_timeout < MIN_TIMEOUT )
-		timer->min_timeout = MIN_TIMEOUT;
-	/* Honor user-specified minimum timeout */
-	if ( timer->timeout < timer->min_timeout )
-		timer->timeout = timer->min_timeout;
-
-	DBG2 ( "Timer %p started at time %ld (expires at %ld)\n",
-	       timer, timer->start, ( timer->start + timer->timeout ) );
+
+	/* Record timeout */
+	timer->timeout = timeout;
+
+	DBGC2 ( timer, "Timer %p started at time %ld (expires at %ld)\n",
+		timer, timer->start, ( timer->start + timer->timeout ) );
 }
 
 /**
- * Start timer with a specified fixed timeout
+ * Start timer
  *
  * @v timer		Retry timer
- * @v timeout		Timeout, in ticks
+ *
+ * This starts the timer running with the current timeout value
+ * (rounded up to the minimum timeout value).  If stop_timer() is not
+ * called before the timer expires, the timer will be stopped and the
+ * timer's callback function will be called.
  */
-void start_timer_fixed ( struct retry_timer *timer, unsigned long timeout ) {
-	start_timer ( timer );
-	timer->timeout = timeout;
-	DBG2 ( "Timer %p expiry time changed to %ld\n",
-	       timer, ( timer->start + timer->timeout ) );
+void start_timer ( struct retry_timer *timer ) {
+	unsigned long timeout = timer->timeout;
+	unsigned long min;
+
+	/* Calculate minimum timeout */
+	min = ( timer->min ? timer->min : DEFAULT_MIN_TIMEOUT );
+	if ( min < MIN_TIMEOUT )
+		min = MIN_TIMEOUT;
+
+	/* Ensure timeout is at least the minimum */
+	if ( timeout < min )
+		timeout = min;
+
+	/* Start timer with this timeout */
+	start_timer_fixed ( timer, timeout );
 }
 
 /**
@@ -111,8 +126,8 @@ void stop_timer ( struct retry_timer *timer ) {
 	list_del ( &timer->list );
 	runtime = ( now - timer->start );
 	timer->running = 0;
-	DBG2 ( "Timer %p stopped at time %ld (ran for %ld)\n",
-	       timer, now, runtime );
+	DBGC2 ( timer, "Timer %p stopped at time %ld (ran for %ld)\n",
+		timer, now, runtime );
 
 	/* Update timer.  Variables are:
 	 *
@@ -135,8 +150,8 @@ void stop_timer ( struct retry_timer *timer ) {
 		timer->timeout -= ( timer->timeout >> 3 );
 		timer->timeout += ( runtime >> 1 );
 		if ( timer->timeout != old_timeout ) {
-			DBG ( "Timer %p timeout updated to %ld\n",
-			      timer, timer->timeout );
+			DBGC ( timer, "Timer %p timeout updated to %ld\n",
+			       timer, timer->timeout );
 		}
 	}
 
@@ -150,11 +165,12 @@ void stop_timer ( struct retry_timer *timer ) {
  */
 static void timer_expired ( struct retry_timer *timer ) {
 	struct refcnt *refcnt = timer->refcnt;
+	unsigned long max = ( timer->max ? timer->max : DEFAULT_MAX_TIMEOUT );
 	int fail;
 
 	/* Stop timer without performing RTT calculations */
-	DBG2 ( "Timer %p stopped at time %ld on expiry\n",
-	       timer, currticks() );
+	DBGC2 ( timer, "Timer %p stopped at time %ld on expiry\n",
+		timer, currticks() );
 	assert ( timer->running );
 	list_del ( &timer->list );
 	timer->running = 0;
@@ -162,12 +178,10 @@ static void timer_expired ( struct retry_timer *timer ) {
 
 	/* Back off the timeout value */
 	timer->timeout <<= 1;
-	if ( timer->max_timeout == 0 ) /* 0 means "use default timeout" */
-		timer->max_timeout = DEFAULT_MAX_TIMEOUT;
-	if ( ( fail = ( timer->timeout > timer->max_timeout ) ) )
-		timer->timeout = timer->max_timeout;
-	DBG ( "Timer %p timeout backed off to %ld\n",
-	      timer, timer->timeout );
+	if ( ( fail = ( timer->timeout > max ) ) )
+		timer->timeout = max;
+	DBGC ( timer, "Timer %p timeout backed off to %ld\n",
+	       timer, timer->timeout );
 
 	/* Call expiry callback */
 	timer->expired ( timer, fail );
diff --git a/roms/ipxe/src/net/rndis.c b/roms/ipxe/src/net/rndis.c
new file mode 100644
index 000000000..8c4fe8b30
--- /dev/null
+++ b/roms/ipxe/src/net/rndis.c
@@ -0,0 +1,1052 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Remote Network Driver Interface Specification
+ *
+ */
+
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/device.h>
+#include <ipxe/rndis.h>
+
+/**
+ * Allocate I/O buffer
+ *
+ * @v len		Length
+ * @ret iobuf		I/O buffer, or NULL
+ */
+static struct io_buffer * rndis_alloc_iob ( size_t len ) {
+	struct rndis_header *header;
+	struct io_buffer *iobuf;
+
+	/* Allocate I/O buffer and reserve space */
+	iobuf = alloc_iob ( sizeof ( *header ) + len );
+	if ( iobuf )
+		iob_reserve ( iobuf, sizeof ( *header ) );
+
+	return iobuf;
+}
+
+/**
+ * Wait for completion
+ *
+ * @v rndis		RNDIS device
+ * @v wait_id		Request ID
+ * @ret rc		Return status code
+ */
+static int rndis_wait ( struct rndis_device *rndis, unsigned int wait_id ) {
+	unsigned int i;
+
+	/* Record query ID */
+	rndis->wait_id = wait_id;
+
+	/* Wait for operation to complete */
+	for ( i = 0 ; i < RNDIS_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for completion */
+		if ( ! rndis->wait_id )
+			return rndis->wait_rc;
+
+		/* Poll RNDIS device */
+		rndis->op->poll ( rndis );
+
+		/* Delay for 1ms */
+		mdelay ( 1 );
+	}
+
+	DBGC ( rndis, "RNDIS %s timed out waiting for ID %#08x\n",
+	       rndis->name, wait_id );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Transmit message
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v type		Message type
+ * @ret rc		Return status code
+ */
+static int rndis_tx_message ( struct rndis_device *rndis,
+			      struct io_buffer *iobuf, unsigned int type ) {
+	struct rndis_header *header;
+	int rc;
+
+	/* Prepend RNDIS header */
+	header = iob_push ( iobuf, sizeof ( *header ) );
+	header->type = cpu_to_le32 ( type );
+	header->len = cpu_to_le32 ( iob_len ( iobuf ) );
+
+	/* Transmit message */
+	if ( ( rc = rndis->op->transmit ( rndis, iobuf ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not transmit: %s\n",
+		       rndis->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Complete message transmission
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v rc		Packet status code
+ */
+void rndis_tx_complete_err ( struct rndis_device *rndis,
+			     struct io_buffer *iobuf, int rc ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_header *header;
+	size_t len = iob_len ( iobuf );
+
+	/* Sanity check */
+	if ( len < sizeof ( *header ) ) {
+		DBGC ( rndis, "RNDIS %s completed underlength transmission:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		netdev_tx_err ( netdev, NULL, -EINVAL );
+		return;
+	}
+	header = iobuf->data;
+
+	/* Complete buffer */
+	if ( header->type == cpu_to_le32 ( RNDIS_PACKET_MSG ) ) {
+		netdev_tx_complete_err ( netdev, iobuf, rc );
+	} else {
+		free_iob ( iobuf );
+	}
+}
+
+/**
+ * Transmit data packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int rndis_tx_data ( struct rndis_device *rndis,
+			   struct io_buffer *iobuf ) {
+	struct rndis_packet_message *msg;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Prepend packet message header */
+	msg = iob_push ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->data.offset = cpu_to_le32 ( sizeof ( *msg ) );
+	msg->data.len = cpu_to_le32 ( len );
+
+	/* Transmit message */
+	if ( ( rc = rndis_tx_message ( rndis, iobuf, RNDIS_PACKET_MSG ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Defer transmitted packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * As with netdev_tx_defer(), the caller must ensure that space in the
+ * transmit descriptor ring is freed up before calling
+ * rndis_tx_complete().
+ *
+ * Unlike netdev_tx_defer(), this call may fail.
+ */
+int rndis_tx_defer ( struct rndis_device *rndis, struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_header *header;
+	struct rndis_packet_message *msg;
+
+	/* Fail unless this was a packet message.  Only packet
+	 * messages correspond to I/O buffers in the network device's
+	 * TX queue; other messages cannot be deferred in this way.
+	 */
+	assert ( iob_len ( iobuf ) >= sizeof ( *header ) );
+	header = iobuf->data;
+	if ( header->type != cpu_to_le32 ( RNDIS_PACKET_MSG ) )
+		return -ENOTSUP;
+
+	/* Strip RNDIS header and packet message header, to return
+	 * this packet to the state in which we received it.
+	 */
+	iob_pull ( iobuf, ( sizeof ( *header ) + sizeof ( *msg ) ) );
+
+	/* Defer packet */
+	netdev_tx_defer ( netdev, iobuf );
+
+	return 0;
+}
+
+/**
+ * Receive data packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_data ( struct rndis_device *rndis,
+			    struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_packet_message *msg;
+	size_t len = iob_len ( iobuf );
+	size_t data_offset;
+	size_t data_len;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength data packet:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	msg = iobuf->data;
+
+	/* Locate and sanity check data buffer */
+	data_offset = le32_to_cpu ( msg->data.offset );
+	data_len = le32_to_cpu ( msg->data.len );
+	if ( ( data_offset > len ) || ( data_len > ( len - data_offset ) ) ) {
+		DBGC ( rndis, "RNDIS %s data packet data exceeds packet:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_data;
+	}
+
+	/* Strip non-data portions */
+	iob_pull ( iobuf, data_offset );
+	iob_unput ( iobuf, ( iob_len ( iobuf ) - data_len ) );
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+
+	return;
+
+ err_data:
+ err_len:
+	/* Report error to network stack */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/**
+ * Transmit initialisation message
+ *
+ * @v rndis		RNDIS device
+ * @v id		Request ID
+ * @ret rc		Return status code
+ */
+static int rndis_tx_initialise ( struct rndis_device *rndis, unsigned int id ) {
+	struct io_buffer *iobuf;
+	struct rndis_initialise_message *msg;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = rndis_alloc_iob ( sizeof ( *msg ) );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct message */
+	msg = iob_put ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->id = id; /* Non-endian */
+	msg->major = cpu_to_le32 ( RNDIS_VERSION_MAJOR );
+	msg->minor = cpu_to_le32 ( RNDIS_VERSION_MINOR );
+	msg->mtu = cpu_to_le32 ( RNDIS_MTU );
+
+	/* Transmit message */
+	if ( ( rc = rndis_tx_message ( rndis, iobuf,
+				       RNDIS_INITIALISE_MSG ) ) != 0 )
+		goto err_tx;
+
+	return 0;
+
+ err_tx:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Receive initialisation completion
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_initialise ( struct rndis_device *rndis,
+				  struct io_buffer *iobuf ) {
+	struct rndis_initialise_completion *cmplt;
+	size_t len = iob_len ( iobuf );
+	unsigned int id;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength initialisation "
+		       "completion:\n", rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	cmplt = iobuf->data;
+
+	/* Extract request ID */
+	id = cmplt->id; /* Non-endian */
+
+	/* Check status */
+	if ( cmplt->status ) {
+		DBGC ( rndis, "RNDIS %s received initialisation completion "
+		       "failure %#08x\n", rndis->name,
+		       le32_to_cpu ( cmplt->status ) );
+		rc = -EIO;
+		goto err_status;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_status:
+	/* Record completion result if applicable */
+	if ( id == rndis->wait_id ) {
+		rndis->wait_id = 0;
+		rndis->wait_rc = rc;
+	}
+ err_len:
+	free_iob ( iobuf );
+}
+
+/**
+ * Initialise RNDIS
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int rndis_initialise ( struct rndis_device *rndis ) {
+	int rc;
+
+	/* Transmit initialisation message */
+	if ( ( rc = rndis_tx_initialise ( rndis, RNDIS_INIT_ID ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = rndis_wait ( rndis, RNDIS_INIT_ID ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Transmit halt message
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int rndis_tx_halt ( struct rndis_device *rndis ) {
+	struct io_buffer *iobuf;
+	struct rndis_halt_message *msg;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = rndis_alloc_iob ( sizeof ( *msg ) );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct message */
+	msg = iob_put ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+
+	/* Transmit message */
+	if ( ( rc = rndis_tx_message ( rndis, iobuf, RNDIS_HALT_MSG ) ) != 0 )
+		goto err_tx;
+
+	return 0;
+
+ err_tx:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Halt RNDIS
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int rndis_halt ( struct rndis_device *rndis ) {
+	int rc;
+
+	/* Transmit halt message */
+	if ( ( rc = rndis_tx_halt ( rndis ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Transmit OID message
+ *
+ * @v rndis		RNDIS device
+ * @v oid		Object ID
+ * @v data		New OID value (or NULL to query current value)
+ * @v len		Length of new OID value
+ * @ret rc		Return status code
+ */
+static int rndis_tx_oid ( struct rndis_device *rndis, unsigned int oid,
+			  const void *data, size_t len ) {
+	struct io_buffer *iobuf;
+	struct rndis_oid_message *msg;
+	unsigned int type;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = rndis_alloc_iob ( sizeof ( *msg ) + len );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct message.  We use the OID as the request ID. */
+	msg = iob_put ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->id = oid; /* Non-endian */
+	msg->oid = cpu_to_le32 ( oid );
+	msg->offset = cpu_to_le32 ( sizeof ( *msg ) );
+	msg->len = cpu_to_le32 ( len );
+	memcpy ( iob_put ( iobuf, len ), data, len );
+
+	/* Transmit message */
+	type = ( data ? RNDIS_SET_MSG : RNDIS_QUERY_MSG );
+	if ( ( rc = rndis_tx_message ( rndis, iobuf, type ) ) != 0 )
+		goto err_tx;
+
+	return 0;
+
+ err_tx:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Receive query OID completion
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_query_oid ( struct rndis_device *rndis,
+				 struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_query_completion *cmplt;
+	size_t len = iob_len ( iobuf );
+	size_t info_offset;
+	size_t info_len;
+	unsigned int id;
+	void *info;
+	uint32_t *link_status;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength query "
+		       "completion:\n", rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	cmplt = iobuf->data;
+
+	/* Extract request ID */
+	id = cmplt->id; /* Non-endian */
+
+	/* Check status */
+	if ( cmplt->status ) {
+		DBGC ( rndis, "RNDIS %s received query completion failure "
+		       "%#08x\n", rndis->name, le32_to_cpu ( cmplt->status ) );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EIO;
+		goto err_status;
+	}
+
+	/* Locate and sanity check information buffer */
+	info_offset = le32_to_cpu ( cmplt->offset );
+	info_len = le32_to_cpu ( cmplt->len );
+	if ( ( info_offset > len ) || ( info_len > ( len - info_offset ) ) ) {
+		DBGC ( rndis, "RNDIS %s query completion information exceeds "
+		       "packet:\n", rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_info;
+	}
+	info = ( ( ( void * ) cmplt ) + info_offset );
+
+	/* Handle OID */
+	switch ( id ) {
+
+	case RNDIS_OID_802_3_PERMANENT_ADDRESS:
+		if ( info_len > sizeof ( netdev->hw_addr ) )
+			info_len = sizeof ( netdev->hw_addr );
+		memcpy ( netdev->hw_addr, info, info_len );
+		break;
+
+	case RNDIS_OID_802_3_CURRENT_ADDRESS:
+		if ( info_len > sizeof ( netdev->ll_addr ) )
+			info_len = sizeof ( netdev->ll_addr );
+		memcpy ( netdev->ll_addr, info, info_len );
+		break;
+
+	case RNDIS_OID_GEN_MEDIA_CONNECT_STATUS:
+		if ( info_len != sizeof ( *link_status ) ) {
+			DBGC ( rndis, "RNDIS %s invalid link status:\n",
+			       rndis->name );
+			DBGC_HDA ( rndis, 0, iobuf->data, len );
+			rc = -EPROTO;
+			goto err_link_status;
+		}
+		link_status = info;
+		if ( *link_status == 0 ) {
+			DBGC ( rndis, "RNDIS %s link is up\n", rndis->name );
+			netdev_link_up ( netdev );
+		} else {
+			DBGC ( rndis, "RNDIS %s link is down: %#08x\n",
+			       rndis->name, le32_to_cpu ( *link_status ) );
+			netdev_link_down ( netdev );
+		}
+		break;
+
+	default:
+		DBGC ( rndis, "RNDIS %s unexpected query completion ID %#08x\n",
+		       rndis->name, id );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EPROTO;
+		goto err_id;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_id:
+ err_link_status:
+ err_info:
+ err_status:
+	/* Record completion result if applicable */
+	if ( id == rndis->wait_id ) {
+		rndis->wait_id = 0;
+		rndis->wait_rc = rc;
+	}
+ err_len:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/**
+ * Receive set OID completion
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_set_oid ( struct rndis_device *rndis,
+			       struct io_buffer *iobuf ) {
+	struct rndis_set_completion *cmplt;
+	size_t len = iob_len ( iobuf );
+	unsigned int id;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength set completion:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	cmplt = iobuf->data;
+
+	/* Extract request ID */
+	id = cmplt->id; /* Non-endian */
+
+	/* Check status */
+	if ( cmplt->status ) {
+		DBGC ( rndis, "RNDIS %s received set completion failure "
+		       "%#08x\n", rndis->name, le32_to_cpu ( cmplt->status ) );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EIO;
+		goto err_status;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_status:
+	/* Record completion result if applicable */
+	if ( id == rndis->wait_id ) {
+		rndis->wait_id = 0;
+		rndis->wait_rc = rc;
+	}
+ err_len:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/**
+ * Query or set OID
+ *
+ * @v rndis		RNDIS device
+ * @v oid		Object ID
+ * @v data		New OID value (or NULL to query current value)
+ * @v len		Length of new OID value
+ * @ret rc		Return status code
+ */
+static int rndis_oid ( struct rndis_device *rndis, unsigned int oid,
+		       const void *data, size_t len ) {
+	int rc;
+
+	/* Transmit query */
+	if ( ( rc = rndis_tx_oid ( rndis, oid, data, len ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = rndis_wait ( rndis, oid ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Receive indicate status message
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_status ( struct rndis_device *rndis,
+			      struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_indicate_status_message *msg;
+	size_t len = iob_len ( iobuf );
+	unsigned int status;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength status message:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	msg = iobuf->data;
+
+	/* Extract status */
+	status = le32_to_cpu ( msg->status );
+
+	/* Handle status */
+	switch ( msg->status ) {
+
+	case RNDIS_STATUS_MEDIA_CONNECT:
+		DBGC ( rndis, "RNDIS %s link is up\n", rndis->name );
+		netdev_link_up ( netdev );
+		break;
+
+	case RNDIS_STATUS_MEDIA_DISCONNECT:
+		DBGC ( rndis, "RNDIS %s link is down\n", rndis->name );
+		netdev_link_down ( netdev );
+		break;
+
+	case RNDIS_STATUS_WTF_WORLD:
+		/* Ignore */
+		break;
+
+	default:
+		DBGC ( rndis, "RNDIS %s unexpected status %#08x:\n",
+		       rndis->name, status );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -ENOTSUP;
+		goto err_status;
+	}
+
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+
+	return;
+
+ err_status:
+ err_len:
+	/* Report error via network device statistics */
+	netdev_rx_err ( netdev, iobuf, rc );
+}
+
+/**
+ * Receive RNDIS message
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v type		Message type
+ */
+static void rndis_rx_message ( struct rndis_device *rndis,
+			       struct io_buffer *iobuf, unsigned int type ) {
+	struct net_device *netdev = rndis->netdev;
+	int rc;
+
+	/* Handle packet */
+	switch ( type ) {
+
+	case RNDIS_PACKET_MSG:
+		rndis_rx_data ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_INITIALISE_CMPLT:
+		rndis_rx_initialise ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_QUERY_CMPLT:
+		rndis_rx_query_oid ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_SET_CMPLT:
+		rndis_rx_set_oid ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_INDICATE_STATUS_MSG:
+		rndis_rx_status ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	default:
+		DBGC ( rndis, "RNDIS %s received unexpected type %#08x\n",
+		       rndis->name, type );
+		DBGC_HDA ( rndis, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EPROTO;
+		goto err_type;
+	}
+
+	return;
+
+ err_type:
+	/* Report error via network device statistics */
+	netdev_rx_err ( netdev, iobuf, rc );
+}
+
+/**
+ * Receive packet from underlying transport layer
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+void rndis_rx ( struct rndis_device *rndis, struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_header *header;
+	unsigned int type;
+	int rc;
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < sizeof ( *header ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength packet:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto drop;
+	}
+	header = iobuf->data;
+
+	/* Parse and strip header */
+	type = le32_to_cpu ( header->type );
+	iob_pull ( iobuf, sizeof ( *header ) );
+
+	/* Handle message */
+	rndis_rx_message ( rndis, iob_disown ( iobuf ), type );
+
+	return;
+
+ drop:
+	/* Record error */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/**
+ * Discard packet from underlying transport layer
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v rc		Packet status code
+ */
+void rndis_rx_err ( struct rndis_device *rndis, struct io_buffer *iobuf,
+		    int rc ) {
+	struct net_device *netdev = rndis->netdev;
+
+	/* Record error */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/**
+ * Set receive filter
+ *
+ * @v rndis		RNDIS device
+ * @v filter		Receive filter
+ * @ret rc		Return status code
+ */
+static int rndis_filter ( struct rndis_device *rndis, unsigned int filter ) {
+	uint32_t value = cpu_to_le32 ( filter );
+	int rc;
+
+	/* Set receive filter */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_GEN_CURRENT_PACKET_FILTER,
+				&value, sizeof ( value ) ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not set receive filter to %#08x: "
+		       "%s\n", rndis->name, filter, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int rndis_open ( struct net_device *netdev ) {
+	struct rndis_device *rndis = netdev->priv;
+	int rc;
+
+	/* Open RNDIS device */
+	if ( ( rc = rndis->op->open ( rndis ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not open: %s\n",
+		       rndis->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Initialise RNDIS */
+	if ( ( rc = rndis_initialise ( rndis ) ) != 0 )
+		goto err_initialise;
+
+	/* Set receive filter */
+	if ( ( rc = rndis_filter ( rndis, ( RNDIS_FILTER_UNICAST |
+					    RNDIS_FILTER_MULTICAST |
+					    RNDIS_FILTER_ALL_MULTICAST |
+					    RNDIS_FILTER_BROADCAST |
+					    RNDIS_FILTER_PROMISCUOUS ) ) ) != 0)
+		goto err_set_filter;
+
+	/* Update link status */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_link;
+
+	return 0;
+
+ err_query_link:
+ err_set_filter:
+	rndis_halt ( rndis );
+ err_initialise:
+	rndis->op->close ( rndis );
+ err_open:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void rndis_close ( struct net_device *netdev ) {
+	struct rndis_device *rndis = netdev->priv;
+
+	/* Clear receive filter */
+	rndis_filter ( rndis, 0 );
+
+	/* Halt RNDIS device */
+	rndis_halt ( rndis );
+
+	/* Close RNDIS device */
+	rndis->op->close ( rndis );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int rndis_transmit ( struct net_device *netdev,
+			    struct io_buffer *iobuf ) {
+	struct rndis_device *rndis = netdev->priv;
+
+	/* Transmit data packet */
+	return rndis_tx_data ( rndis, iobuf );
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void rndis_poll ( struct net_device *netdev ) {
+	struct rndis_device *rndis = netdev->priv;
+
+	/* Poll RNDIS device */
+	rndis->op->poll ( rndis );
+}
+
+/** Network device operations */
+static struct net_device_operations rndis_operations = {
+	.open		= rndis_open,
+	.close		= rndis_close,
+	.transmit	= rndis_transmit,
+	.poll		= rndis_poll,
+};
+
+/**
+ * Allocate RNDIS device
+ *
+ * @v priv_len		Length of private data
+ * @ret rndis		RNDIS device, or NULL on allocation failure
+ */
+struct rndis_device * alloc_rndis ( size_t priv_len ) {
+	struct net_device *netdev;
+	struct rndis_device *rndis;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *rndis ) + priv_len );
+	if ( ! netdev )
+		return NULL;
+	netdev_init ( netdev, &rndis_operations );
+	rndis = netdev->priv;
+	rndis->netdev = netdev;
+	rndis->priv = ( ( ( void * ) rndis ) + sizeof ( *rndis ) );
+
+	return rndis;
+}
+
+/**
+ * Register RNDIS device
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ *
+ * Note that this routine will open and use the RNDIS device in order
+ * to query the MAC address.  The device must be immediately ready for
+ * use prior to registration.
+ */
+int register_rndis ( struct rndis_device *rndis ) {
+	struct net_device *netdev = rndis->netdev;
+	int rc;
+
+	/* Assign device name (for debugging) */
+	rndis->name = netdev->dev->name;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not register: %s\n",
+		       rndis->name, strerror ( rc ) );
+		goto err_register;
+	}
+
+	/* Open RNDIS device to read MAC addresses */
+	if ( ( rc = rndis->op->open ( rndis ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not open: %s\n",
+		       rndis->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Initialise RNDIS */
+	if ( ( rc = rndis_initialise ( rndis ) ) != 0 )
+		goto err_initialise;
+
+	/* Query permanent MAC address */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_802_3_PERMANENT_ADDRESS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_permanent;
+
+	/* Query current MAC address */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_802_3_CURRENT_ADDRESS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_current;
+
+	/* Get link status */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_link;
+
+	/* Halt RNDIS device */
+	rndis_halt ( rndis );
+
+	/* Close RNDIS device */
+	rndis->op->close ( rndis );
+
+	return 0;
+
+ err_query_link:
+ err_query_current:
+ err_query_permanent:
+	rndis_halt ( rndis );
+ err_initialise:
+	rndis->op->close ( rndis );
+ err_open:
+	unregister_netdev ( netdev );
+ err_register:
+	return rc;
+}
+
+/**
+ * Unregister RNDIS device
+ *
+ * @v rndis		RNDIS device
+ */
+void unregister_rndis ( struct rndis_device *rndis ) {
+	struct net_device *netdev = rndis->netdev;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+}
+
+/**
+ * Free RNDIS device
+ *
+ * @v rndis		RNDIS device
+ */
+void free_rndis ( struct rndis_device *rndis ) {
+	struct net_device *netdev = rndis->netdev;
+
+	/* Free network device */
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
diff --git a/roms/ipxe/src/net/socket.c b/roms/ipxe/src/net/socket.c
index 24f6a0892..2009ab237 100644
--- a/roms/ipxe/src/net/socket.c
+++ b/roms/ipxe/src/net/socket.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/net/stp.c b/roms/ipxe/src/net/stp.c
new file mode 100644
index 000000000..d4e65a1a2
--- /dev/null
+++ b/roms/ipxe/src/net/stp.c
@@ -0,0 +1,152 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/timer.h>
+#include <ipxe/stp.h>
+
+/** @file
+ *
+ * Spanning Tree Protocol (STP)
+ *
+ */
+
+/* Disambiguate the various error causes */
+#define ENOTSUP_PROTOCOL __einfo_error ( EINFO_ENOTSUP_PROTOCOL )
+#define EINFO_ENOTSUP_PROTOCOL					\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01,			\
+			  "Non-STP packet received" )
+#define ENOTSUP_VERSION __einfo_error ( EINFO_ENOTSUP_VERSION )
+#define EINFO_ENOTSUP_VERSION					\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01,			\
+			  "Legacy STP packet received" )
+#define ENOTSUP_TYPE __einfo_error ( EINFO_ENOTSUP_TYPE )
+#define EINFO_ENOTSUP_TYPE					\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01,			\
+			  "Non-RSTP packet received" )
+
+/**
+ * Process incoming STP packets
+ *
+ * @v iobuf		I/O buffer
+ * @v netdev		Network device
+ * @v ll_source		Link-layer source address
+ * @v flags		Packet flags
+ * @ret rc		Return status code
+ */
+static int stp_rx ( struct io_buffer *iobuf, struct net_device *netdev,
+		    const void *ll_dest __unused,
+		    const void *ll_source __unused,
+		    unsigned int flags __unused ) {
+	struct stp_bpdu *stp;
+	unsigned int hello;
+	int rc;
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < sizeof ( *stp ) ) {
+		DBGC ( netdev, "STP %s received underlength packet (%zd "
+		       "bytes):\n", netdev->name, iob_len ( iobuf ) );
+		DBGC_HDA ( netdev, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto done;
+	}
+	stp = iobuf->data;
+
+	/* Ignore non-RSTP packets */
+	if ( stp->protocol != htons ( STP_PROTOCOL ) ) {
+		DBGC ( netdev, "STP %s ignoring non-STP packet (protocol "
+		       "%#04x)\n", netdev->name, ntohs ( stp->protocol ) );
+		rc = -ENOTSUP_PROTOCOL;
+		goto done;
+	}
+	if ( stp->version < STP_VERSION_RSTP ) {
+		DBGC ( netdev, "STP %s received legacy STP packet (version "
+		       "%#02x)\n", netdev->name, stp->version );
+		rc = -ENOTSUP_VERSION;
+		goto done;
+	}
+	if ( stp->type != STP_TYPE_RSTP ) {
+		DBGC ( netdev, "STP %s received non-RSTP packet (type %#02x)\n",
+		       netdev->name, stp->type );
+		rc = -ENOTSUP_TYPE;
+		goto done;
+	}
+
+	/* Dump information */
+	DBGC2 ( netdev, "STP %s %s port %#04x flags %#02x hello %d delay %d\n",
+		netdev->name, eth_ntoa ( stp->sender.mac ), ntohs ( stp->port ),
+		stp->flags, ntohs ( stp->hello ), ntohs ( stp->delay ) );
+
+	/* Check if port is forwarding */
+	if ( ! ( stp->flags & STP_FL_FORWARDING ) ) {
+		/* Port is not forwarding: block link for two hello times */
+		DBGC ( netdev, "STP %s %s port %#04x flags %#02x is not "
+		       "forwarding\n",
+		       netdev->name, eth_ntoa ( stp->sender.mac ),
+		       ntohs ( stp->port ), stp->flags );
+		hello = ( ( ntohs ( stp->hello ) * TICKS_PER_SEC ) / 256 );
+		netdev_link_block ( netdev, ( hello * 2 ) );
+		rc = -ENETUNREACH;
+		goto done;
+	}
+
+	/* Success */
+	if ( netdev_link_blocked ( netdev ) ) {
+		DBGC ( netdev, "STP %s %s port %#04x flags %#02x is "
+		       "forwarding\n",
+		       netdev->name, eth_ntoa ( stp->sender.mac ),
+		       ntohs ( stp->port ), stp->flags );
+	}
+	netdev_link_unblock ( netdev );
+	rc = 0;
+
+ done:
+	free_iob ( iobuf );
+	return rc;
+}
+
+/**
+ * Transcribe STP address
+ *
+ * @v net_addr		STP address
+ * @ret string		"<STP>"
+ *
+ * This operation is meaningless for the STP protocol.
+ */
+static const char * stp_ntoa ( const void *net_addr __unused ) {
+	return "<STP>";
+}
+
+/** STP network protocol */
+struct net_protocol stp_protocol __net_protocol = {
+	.name = "STP",
+	.net_proto = htons ( ETH_P_STP ),
+	.rx = stp_rx,
+	.ntoa = stp_ntoa,
+};
diff --git a/roms/ipxe/src/net/tcp.c b/roms/ipxe/src/net/tcp.c
index 987cb63e1..c69c83b85 100644
--- a/roms/ipxe/src/net/tcp.c
+++ b/roms/ipxe/src/net/tcp.c
@@ -26,7 +26,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A TCP connection */
 struct tcp_connection {
@@ -101,8 +101,9 @@ struct tcp_connection {
 	 * Equivalent to Rcv.Wind.Scale in RFC 1323 terminology
 	 */
 	uint8_t rcv_win_scale;
-	/** Maximum receive window */
-	uint32_t max_rcv_win;
+
+	/** Selective acknowledgement list (in host-endian order) */
+	struct tcp_sack_block sack[TCP_SACK_MAX];
 
 	/** Transmit queue */
 	struct list_head tx_queue;
@@ -129,6 +130,8 @@ enum tcp_flags {
 	TCP_TS_ENABLED = 0x0002,
 	/** TCP acknowledgement is pending */
 	TCP_ACK_PENDING = 0x0004,
+	/** TCP selective acknowledgement is enabled */
+	TCP_SACK_ENABLED = 0x0008,
 };
 
 /** TCP internal header
@@ -143,6 +146,8 @@ struct tcp_rx_queued_header {
 	 * enqueued, and so excludes the SYN, if present.
 	 */
 	uint32_t seq;
+	/** Next SEQ value, in host-endian order */
+	uint32_t nxt;
 	/** Flags
 	 *
 	 * Only FIN is valid within this flags byte; all other flags
@@ -284,7 +289,6 @@ static int tcp_open ( struct interface *xfer, struct sockaddr *peer,
 	tcp->tcp_state = TCP_STATE_SENT ( TCP_SYN );
 	tcp_dump_state ( tcp );
 	tcp->snd_seq = random();
-	tcp->max_rcv_win = TCP_MAX_WINDOW_SIZE;
 	INIT_LIST_HEAD ( &tcp->tx_queue );
 	INIT_LIST_HEAD ( &tcp->rx_queue );
 	memcpy ( &tcp->peer, st_peer, sizeof ( tcp->peer ) );
@@ -396,6 +400,7 @@ static void tcp_close ( struct tcp_connection *tcp, int rc ) {
 
 		tcp->tcp_state |= TCP_STATE_SENT ( TCP_FIN );
 		tcp_dump_state ( tcp );
+		process_add ( &tcp->process );
 
 		/* Add a pending operation for the FIN */
 		pending_get ( &tcp->pending_flags );
@@ -450,6 +455,94 @@ static size_t tcp_xfer_window ( struct tcp_connection *tcp ) {
 }
 
 /**
+ * Find selective acknowledgement block
+ *
+ * @v tcp		TCP connection
+ * @v seq		SEQ value in SACK block (in host-endian order)
+ * @v sack		SACK block to fill in (in host-endian order)
+ * @ret len		Length of SACK block
+ */
+static uint32_t tcp_sack_block ( struct tcp_connection *tcp, uint32_t seq,
+				 struct tcp_sack_block *sack ) {
+	struct io_buffer *iobuf;
+	struct tcp_rx_queued_header *tcpqhdr;
+	uint32_t left = tcp->rcv_ack;
+	uint32_t right = left;
+
+	/* Find highest block which does not start after SEQ */
+	list_for_each_entry ( iobuf, &tcp->rx_queue, list ) {
+		tcpqhdr = iobuf->data;
+		if ( tcp_cmp ( tcpqhdr->seq, right ) > 0 ) {
+			if ( tcp_cmp ( tcpqhdr->seq, seq ) > 0 )
+				break;
+			left = tcpqhdr->seq;
+		}
+		if ( tcp_cmp ( tcpqhdr->nxt, right ) > 0 )
+			right = tcpqhdr->nxt;
+	}
+
+	/* Fail if this block does not contain SEQ */
+	if ( tcp_cmp ( right, seq ) < 0 )
+		return 0;
+
+	/* Populate SACK block */
+	sack->left = left;
+	sack->right = right;
+	return ( right - left );
+}
+
+/**
+ * Update TCP selective acknowledgement list
+ *
+ * @v tcp		TCP connection
+ * @v seq		SEQ value in first SACK block (in host-endian order)
+ * @ret count		Number of SACK blocks
+ */
+static unsigned int tcp_sack ( struct tcp_connection *tcp, uint32_t seq ) {
+	struct tcp_sack_block sack[TCP_SACK_MAX];
+	unsigned int old = 0;
+	unsigned int new = 0;
+	unsigned int i;
+	uint32_t len;
+
+	/* Populate first new SACK block */
+	len = tcp_sack_block ( tcp, seq, &sack[0] );
+	if ( len )
+		new++;
+
+	/* Populate remaining new SACK blocks based on old SACK blocks */
+	for ( old = 0 ; old < TCP_SACK_MAX ; old++ ) {
+
+		/* Stop if we run out of space in the new list */
+		if ( new == TCP_SACK_MAX )
+			break;
+
+		/* Skip empty old SACK blocks */
+		if ( tcp->sack[old].left == tcp->sack[old].right )
+			continue;
+
+		/* Populate new SACK block */
+		len = tcp_sack_block ( tcp, tcp->sack[old].left, &sack[new] );
+		if ( len == 0 )
+			continue;
+
+		/* Eliminate duplicates */
+		for ( i = 0 ; i < new ; i++ ) {
+			if ( sack[i].left == sack[new].left ) {
+				new--;
+				break;
+			}
+		}
+		new++;
+	}
+
+	/* Update SACK list */
+	memset ( tcp->sack, 0, sizeof ( tcp->sack ) );
+	memcpy ( tcp->sack, sack, ( new * sizeof ( tcp->sack[0] ) ) );
+	return new;
+}
+
+/**
  * Process TCP transmit queue
  *
  * @v tcp		TCP connection
@@ -493,9 +586,10 @@ static size_t tcp_process_tx_queue ( struct tcp_connection *tcp, size_t max_len,
 }
 
 /**
- * Transmit any outstanding data
+ * Transmit any outstanding data (with selective acknowledgement)
  *
  * @v tcp		TCP connection
+ * @v sack_seq		SEQ for first selective acknowledgement (if any)
  * 
  * Transmits any outstanding data on the connection.
  *
@@ -503,17 +597,22 @@ static size_t tcp_process_tx_queue ( struct tcp_connection *tcp, size_t max_len,
  * will have been started if necessary, and so the stack will
  * eventually attempt to retransmit the failed packet.
  */
-static void tcp_xmit ( struct tcp_connection *tcp ) {
+static void tcp_xmit_sack ( struct tcp_connection *tcp, uint32_t sack_seq ) {
 	struct io_buffer *iobuf;
 	struct tcp_header *tcphdr;
 	struct tcp_mss_option *mssopt;
 	struct tcp_window_scale_padded_option *wsopt;
 	struct tcp_timestamp_padded_option *tsopt;
+	struct tcp_sack_permitted_padded_option *spopt;
+	struct tcp_sack_padded_option *sackopt;
+	struct tcp_sack_block *sack;
 	void *payload;
 	unsigned int flags;
+	unsigned int sack_count;
+	unsigned int i;
 	size_t len = 0;
+	size_t sack_len;
 	uint32_t seq_len;
-	uint32_t app_win;
 	uint32_t max_rcv_win;
 	uint32_t max_representable_win;
 	int rc;
@@ -567,10 +666,9 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 	tcp_process_tx_queue ( tcp, len, iobuf, 0 );
 
 	/* Expand receive window if possible */
-	max_rcv_win = tcp->max_rcv_win;
-	app_win = xfer_window ( &tcp->xfer );
-	if ( max_rcv_win > app_win )
-		max_rcv_win = app_win;
+	max_rcv_win = xfer_window ( &tcp->xfer );
+	if ( max_rcv_win > TCP_MAX_WINDOW_SIZE )
+		max_rcv_win = TCP_MAX_WINDOW_SIZE;
 	max_representable_win = ( 0xffff << tcp->rcv_win_scale );
 	if ( max_rcv_win > max_representable_win )
 		max_rcv_win = max_representable_win;
@@ -590,6 +688,10 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 		wsopt->wsopt.kind = TCP_OPTION_WS;
 		wsopt->wsopt.length = sizeof ( wsopt->wsopt );
 		wsopt->wsopt.scale = TCP_RX_WINDOW_SCALE;
+		spopt = iob_push ( iobuf, sizeof ( *spopt ) );
+		memset ( spopt->nop, TCP_OPTION_NOP, sizeof ( spopt ) );
+		spopt->spopt.kind = TCP_OPTION_SACK_PERMITTED;
+		spopt->spopt.length = sizeof ( spopt->spopt );
 	}
 	if ( ( flags & TCP_SYN ) || ( tcp->flags & TCP_TS_ENABLED ) ) {
 		tsopt = iob_push ( iobuf, sizeof ( *tsopt ) );
@@ -599,6 +701,21 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 		tsopt->tsopt.tsval = htonl ( currticks() );
 		tsopt->tsopt.tsecr = htonl ( tcp->ts_recent );
 	}
+	if ( ( tcp->flags & TCP_SACK_ENABLED ) &&
+	     ( ! list_empty ( &tcp->rx_queue ) ) &&
+	     ( ( sack_count = tcp_sack ( tcp, sack_seq ) ) != 0 ) ) {
+		sack_len = ( sack_count * sizeof ( *sack ) );
+		sackopt = iob_push ( iobuf, ( sizeof ( *sackopt ) + sack_len ));
+		memset ( sackopt->nop, TCP_OPTION_NOP, sizeof ( sackopt->nop ));
+		sackopt->sackopt.kind = TCP_OPTION_SACK;
+		sackopt->sackopt.length =
+			( sizeof ( sackopt->sackopt ) + sack_len );
+		sack = ( ( ( void * ) sackopt ) + sizeof ( *sackopt ) );
+		for ( i = 0 ; i < sack_count ; i++, sack++ ) {
+			sack->left = htonl ( tcp->sack[i].left );
+			sack->right = htonl ( tcp->sack[i].right );
+		}
+	}
 	if ( len != 0 )
 		flags |= TCP_PSH;
 	tcphdr = iob_push ( iobuf, sizeof ( *tcphdr ) );
@@ -635,6 +752,17 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 	profile_stop ( &tcp_tx_profiler );
 }
 
+/**
+ * Transmit any outstanding data
+ *
+ * @v tcp		TCP connection
+ */
+static void tcp_xmit ( struct tcp_connection *tcp ) {
+
+	/* Transmit without an explicit first SACK */
+	tcp_xmit_sack ( tcp, tcp->rcv_ack );
+}
+
 /** TCP process descriptor */
 static struct process_descriptor tcp_process_desc =
 	PROC_DESC_ONCE ( struct tcp_connection, process, tcp_xmit );
@@ -804,6 +932,12 @@ static void tcp_rx_opts ( struct tcp_connection *tcp, const void *data,
 		case TCP_OPTION_WS:
 			options->wsopt = data;
 			break;
+		case TCP_OPTION_SACK_PERMITTED:
+			options->spopt = data;
+			break;
+		case TCP_OPTION_SACK:
+			/* Ignore received SACKs */
+			break;
 		case TCP_OPTION_TS:
 			options->tsopt = data;
 			break;
@@ -823,6 +957,7 @@ static void tcp_rx_opts ( struct tcp_connection *tcp, const void *data,
  * @v seq_len		Sequence space length to consume
  */
 static void tcp_rx_seq ( struct tcp_connection *tcp, uint32_t seq_len ) {
+	unsigned int sack;
 
 	/* Sanity check */
 	assert ( seq_len > 0 );
@@ -840,6 +975,16 @@ static void tcp_rx_seq ( struct tcp_connection *tcp, uint32_t seq_len ) {
 	/* Update timestamp */
 	tcp->ts_recent = tcp->ts_val;
 
+	/* Update SACK list */
+	for ( sack = 0 ; sack < TCP_SACK_MAX ; sack++ ) {
+		if ( tcp->sack[sack].left == tcp->sack[sack].right )
+			continue;
+		if ( tcp_cmp ( tcp->sack[sack].left, tcp->rcv_ack ) < 0 )
+			tcp->sack[sack].left = tcp->rcv_ack;
+		if ( tcp_cmp ( tcp->sack[sack].right, tcp->rcv_ack ) < 0 )
+			tcp->sack[sack].right = tcp->rcv_ack;
+	}
+
 	/* Mark ACK as pending */
 	tcp->flags |= TCP_ACK_PENDING;
 }
@@ -860,6 +1005,8 @@ static int tcp_rx_syn ( struct tcp_connection *tcp, uint32_t seq,
 		tcp->rcv_ack = seq;
 		if ( options->tsopt )
 			tcp->flags |= TCP_TS_ENABLED;
+		if ( options->spopt )
+			tcp->flags |= TCP_SACK_ENABLED;
 		if ( options->wsopt ) {
 			tcp->snd_win_scale = options->wsopt->scale;
 			tcp->rcv_win_scale = TCP_RX_WINDOW_SCALE;
@@ -1070,6 +1217,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	struct io_buffer *queued;
 	size_t len;
 	uint32_t seq_len;
+	uint32_t nxt;
 
 	/* Calculate remaining flags and sequence length.  Note that
 	 * SYN, if present, has already been processed by this point.
@@ -1077,6 +1225,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	flags &= TCP_FIN;
 	len = iob_len ( iobuf );
 	seq_len = ( len + ( flags ? 1 : 0 ) );
+	nxt = ( seq + seq_len );
 
 	/* Discard immediately (to save memory) if:
 	 *
@@ -1087,7 +1236,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	 */
 	if ( ( ! ( tcp->tcp_state & TCP_STATE_RCVD ( TCP_SYN ) ) ) ||
 	     ( tcp_cmp ( seq, tcp->rcv_ack + tcp->rcv_win ) >= 0 ) ||
-	     ( tcp_cmp ( seq + seq_len, tcp->rcv_ack ) < 0 ) ||
+	     ( tcp_cmp ( nxt, tcp->rcv_ack ) < 0 ) ||
 	     ( seq_len == 0 ) ) {
 		free_iob ( iobuf );
 		return;
@@ -1096,6 +1245,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	/* Add internal header */
 	tcpqhdr = iob_push ( iobuf, sizeof ( *tcpqhdr ) );
 	tcpqhdr->seq = seq;
+	tcpqhdr->nxt = nxt;
 	tcpqhdr->flags = flags;
 
 	/* Add to RX queue */
@@ -1289,7 +1439,7 @@ static int tcp_rx ( struct io_buffer *iobuf,
 	if ( list_empty ( &tcp->rx_queue ) ) {
 		process_add ( &tcp->process );
 	} else {
-		tcp_xmit ( tcp );
+		tcp_xmit_sack ( tcp, seq );
 	}
 
 	/* If this packet was the last we expect to receive, set up
@@ -1328,24 +1478,12 @@ struct tcpip_protocol tcp_protocol __tcpip_protocol = {
 static unsigned int tcp_discard ( void ) {
 	struct tcp_connection *tcp;
 	struct io_buffer *iobuf;
-	struct tcp_rx_queued_header *tcpqhdr;
-	uint32_t max_win;
 	unsigned int discarded = 0;
 
 	/* Try to drop one queued RX packet from each connection */
 	list_for_each_entry ( tcp, &tcp_conns, list ) {
 		list_for_each_entry_reverse ( iobuf, &tcp->rx_queue, list ) {
 
-			/* Limit window to prevent future discards */
-			tcpqhdr = iobuf->data;
-			max_win = ( tcpqhdr->seq - tcp->rcv_ack );
-			if ( max_win < tcp->max_rcv_win ) {
-				DBGC ( tcp, "TCP %p reducing maximum window "
-				       "from %d to %d\n",
-				       tcp, tcp->max_rcv_win, max_win );
-				tcp->max_rcv_win = max_win;
-			}
-
 			/* Remove packet from queue */
 			list_del ( &iobuf->list );
 			free_iob ( iobuf );
@@ -1365,12 +1503,67 @@ struct cache_discarder tcp_discarder __cache_discarder ( CACHE_NORMAL ) = {
 };
 
 /**
+ * Find first TCP connection that has not yet been closed
+ *
+ * @ret tcp		First unclosed connection, or NULL
+ */
+static struct tcp_connection * tcp_first_unclosed ( void ) {
+	struct tcp_connection *tcp;
+
+	/* Find first connection which has not yet been closed */
+	list_for_each_entry ( tcp, &tcp_conns, list ) {
+		if ( ! ( tcp->flags & TCP_XFER_CLOSED ) )
+			return tcp;
+	}
+	return NULL;
+}
+
+/**
+ * Find first TCP connection that has not yet finished all operations
+ *
+ * @ret tcp		First unfinished connection, or NULL
+ */
+static struct tcp_connection * tcp_first_unfinished ( void ) {
+	struct tcp_connection *tcp;
+
+	/* Find first connection which has not yet closed gracefully,
+	 * or which still has a pending transmission (e.g. to ACK the
+	 * received FIN).
+	 */
+	list_for_each_entry ( tcp, &tcp_conns, list ) {
+		if ( ( ! TCP_CLOSED_GRACEFULLY ( tcp->tcp_state ) ) ||
+		     process_running ( &tcp->process ) ) {
+			return tcp;
+		}
+	}
+	return NULL;
+}
+
+/**
  * Shut down all TCP connections
  *
  */
 static void tcp_shutdown ( int booting __unused ) {
 	struct tcp_connection *tcp;
+	unsigned long start;
+	unsigned long elapsed;
+
+	/* Initiate a graceful close of all connections, allowing for
+	 * the fact that the connection list may change as we do so.
+	 */
+	while ( ( tcp = tcp_first_unclosed() ) ) {
+		DBGC ( tcp, "TCP %p closing for shutdown\n", tcp );
+		tcp_close ( tcp, -ECANCELED );
+	}
+
+	/* Wait for all connections to finish closing gracefully */
+	start = currticks();
+	while ( ( tcp = tcp_first_unfinished() ) &&
+		( ( elapsed = ( currticks() - start ) ) < TCP_FINISH_TIMEOUT )){
+		step();
+	}
 
+	/* Forcibly close any remaining connections */
 	while ( ( tcp = list_first_entry ( &tcp_conns, struct tcp_connection,
 					   list ) ) != NULL ) {
 		tcp->tcp_state = TCP_CLOSED;
@@ -1380,7 +1573,7 @@ static void tcp_shutdown ( int booting __unused ) {
 }
 
 /** TCP shutdown function */
-struct startup_fn tcp_startup_fn __startup_fn ( STARTUP_EARLY ) = {
+struct startup_fn tcp_startup_fn __startup_fn ( STARTUP_LATE ) = {
 	.shutdown = tcp_shutdown,
 };
 
diff --git a/roms/ipxe/src/net/tcp/http.c b/roms/ipxe/src/net/tcp/http.c
index 90bae9d7a..b000ed80f 100644
--- a/roms/ipxe/src/net/tcp/http.c
+++ b/roms/ipxe/src/net/tcp/http.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -26,26 +30,20 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#include <stddef.h>
 #include <ipxe/open.h>
 #include <ipxe/http.h>
 #include <ipxe/features.h>
 
 FEATURE ( FEATURE_PROTOCOL, "HTTP", DHCP_EB_FEATURE_HTTP, 1 );
 
-/**
- * Initiate an HTTP connection
- *
- * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @ret rc		Return status code
- */
-static int http_open ( struct interface *xfer, struct uri *uri ) {
-	return http_open_filter ( xfer, uri, HTTP_PORT, NULL );
-}
-
 /** HTTP URI opener */
 struct uri_opener http_uri_opener __uri_opener = {
 	.scheme	= "http",
-	.open	= http_open,
+	.open	= http_open_uri,
+};
+
+/** HTTP URI scheme */
+struct http_scheme http_scheme __http_scheme = {
+	.name = "http",
+	.port = HTTP_PORT,
 };
diff --git a/roms/ipxe/src/net/tcp/httpauth.c b/roms/ipxe/src/net/tcp/httpauth.c
new file mode 100644
index 000000000..fb6dcd035
--- /dev/null
+++ b/roms/ipxe/src/net/tcp/httpauth.c
@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) authentication
+ *
+ */
+
+#include <stdio.h>
+#include <strings.h>
+#include <errno.h>
+#include <ipxe/http.h>
+
+/**
+ * Identify authentication scheme
+ *
+ * @v http		HTTP transaction
+ * @v name		Scheme name
+ * @ret auth		Authentication scheme, or NULL
+ */
+static struct http_authentication * http_authentication ( const char *name ) {
+	struct http_authentication *auth;
+
+	/* Identify authentication scheme */
+	for_each_table_entry ( auth, HTTP_AUTHENTICATIONS ) {
+		if ( strcasecmp ( name, auth->name ) == 0 )
+			return auth;
+	}
+
+	return NULL;
+}
+
+/** An HTTP "WWW-Authenticate" response field */
+struct http_www_authenticate_field {
+	/** Name */
+	const char *name;
+	/** Offset */
+	size_t offset;
+};
+
+/** Define an HTTP "WWW-Authenticate" response field */
+#define HTTP_WWW_AUTHENTICATE_FIELD( _name ) {				\
+		.name = #_name,						\
+		.offset = offsetof ( struct http_transaction,		\
+				     response.auth._name ),		\
+	}
+
+/**
+ * Set HTTP "WWW-Authenticate" response field value
+ *
+ * @v http		HTTP transaction
+ * @v field		Response field
+ * @v value		Field value
+ */
+static inline void
+http_www_auth_field ( struct http_transaction *http,
+		      struct http_www_authenticate_field *field, char *value ) {
+	char **ptr;
+
+	ptr = ( ( ( void * ) http ) + field->offset );
+	*ptr = value;
+}
+
+/** HTTP "WWW-Authenticate" fields */
+static struct http_www_authenticate_field http_www_auth_fields[] = {
+	HTTP_WWW_AUTHENTICATE_FIELD ( realm ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( qop ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( algorithm ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( nonce ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( opaque ),
+};
+
+/**
+ * Parse HTTP "WWW-Authenticate" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_www_authenticate ( struct http_transaction *http,
+					 char *line ) {
+	struct http_www_authenticate_field *field;
+	char *name;
+	char *key;
+	char *value;
+	unsigned int i;
+
+	/* Get scheme name */
+	name = http_token ( &line, NULL );
+	if ( ! name ) {
+		DBGC ( http, "HTTP %p malformed WWW-Authenticate \"%s\"\n",
+		       http, value );
+		return -EPROTO;
+	}
+
+	/* Identify scheme */
+	http->response.auth.auth = http_authentication ( name );
+	if ( ! http->response.auth.auth ) {
+		DBGC ( http, "HTTP %p unrecognised authentication scheme "
+		       "\"%s\"\n", http, name );
+		return -ENOTSUP;
+	}
+
+	/* Process fields */
+	while ( ( key = http_token ( &line, &value ) ) ) {
+		for ( i = 0 ; i < ( sizeof ( http_www_auth_fields ) /
+				    sizeof ( http_www_auth_fields[0] ) ) ; i++){
+			field = &http_www_auth_fields[i];
+			if ( strcasecmp ( key, field->name ) == 0 )
+				http_www_auth_field ( http, field, value );
+		}
+	}
+
+	/* Allow HTTP request to be retried if the request had not
+	 * already tried authentication.
+	 */
+	if ( ! http->request.auth.auth )
+		http->response.flags |= HTTP_RESPONSE_RETRY;
+
+	return 0;
+}
+
+/** HTTP "WWW-Authenticate" header */
+struct http_response_header
+http_response_www_authenticate __http_response_header = {
+	.name = "WWW-Authenticate",
+	.parse = http_parse_www_authenticate,
+};
+
+/**
+ * Construct HTTP "Authorization" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_authorization ( struct http_transaction *http,
+				       char *buf, size_t len ) {
+	struct http_authentication *auth = http->request.auth.auth;
+	size_t used;
+	int auth_len;
+	int rc;
+
+	/* Do nothing unless we have an authentication scheme */
+	if ( ! auth )
+		return 0;
+
+	/* Construct header */
+	used = snprintf ( buf, len, "%s ", auth->name );
+	auth_len = auth->format ( http, ( buf + used ),
+				  ( ( used < len ) ? ( len - used ) : 0 ) );
+	if ( auth_len < 0 ) {
+		rc = auth_len;
+		return rc;
+	}
+	used += auth_len;
+
+	return used;
+}
+
+/** HTTP "Authorization" header */
+struct http_request_header http_request_authorization __http_request_header = {
+	.name = "Authorization",
+	.format = http_format_authorization,
+};
diff --git a/roms/ipxe/src/net/tcp/httpbasic.c b/roms/ipxe/src/net/tcp/httpbasic.c
new file mode 100644
index 000000000..7ed7de9e7
--- /dev/null
+++ b/roms/ipxe/src/net/tcp/httpbasic.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) Basic authentication
+ *
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <ipxe/uri.h>
+#include <ipxe/base64.h>
+#include <ipxe/http.h>
+
+/* Disambiguate the various error causes */
+#define EACCES_USERNAME __einfo_error ( EINFO_EACCES_USERNAME )
+#define EINFO_EACCES_USERNAME						\
+	__einfo_uniqify ( EINFO_EACCES, 0x01,				\
+			  "No username available for Basic authentication" )
+
+/**
+ * Perform HTTP Basic authentication
+ *
+ * @v http		HTTP transaction
+ * @ret rc		Return status code
+ */
+static int http_basic_authenticate ( struct http_transaction *http ) {
+	struct http_request_auth *req = &http->request.auth;
+
+	/* Record username and password */
+	if ( ! http->uri->user ) {
+		DBGC ( http, "HTTP %p has no username for Basic "
+		       "authentication\n", http );
+		return -EACCES_USERNAME;
+	}
+	req->username = http->uri->user;
+	req->password = ( http->uri->password ? http->uri->password : "" );
+
+	return 0;
+}
+
+/**
+ * Construct HTTP "Authorization" header for Basic authentication
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_basic_auth ( struct http_transaction *http,
+				    char *buf, size_t len ) {
+	struct http_request_auth *req = &http->request.auth;
+	size_t user_pw_len = ( strlen ( req->username ) + 1 /* ":" */ +
+			       strlen ( req->password ) );
+	char user_pw[ user_pw_len + 1 /* NUL */ ];
+
+	/* Sanity checks */
+	assert ( req->username != NULL );
+	assert ( req->password != NULL );
+
+	/* Construct "user:password" string */
+	snprintf ( user_pw, sizeof ( user_pw ), "%s:%s",
+		   req->username, req->password );
+
+	/* Construct response */
+	return base64_encode ( user_pw, user_pw_len, buf, len );
+}
+
+/** HTTP Basic authentication scheme */
+struct http_authentication http_basic_auth __http_authentication = {
+	.name = "Basic",
+	.authenticate = http_basic_authenticate,
+	.format = http_format_basic_auth,
+};
+
+/* Drag in HTTP authentication support */
+REQUIRING_SYMBOL ( http_basic_auth );
+REQUIRE_OBJECT ( httpauth );
diff --git a/roms/ipxe/src/net/tcp/httpblock.c b/roms/ipxe/src/net/tcp/httpblock.c
new file mode 100644
index 000000000..e124ad2d6
--- /dev/null
+++ b/roms/ipxe/src/net/tcp/httpblock.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) block device
+ *
+ */
+
+#include <stdint.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/blocktrans.h>
+#include <ipxe/blockdev.h>
+#include <ipxe/acpi.h>
+#include <ipxe/http.h>
+
+/** Block size used for HTTP block device requests */
+#define HTTP_BLKSIZE 512
+
+/**
+ * Read from block device
+ *
+ * @v http		HTTP transaction
+ * @v data		Data interface
+ * @v lba		Starting logical block address
+ * @v count		Number of logical blocks
+ * @v buffer		Data buffer
+ * @v len		Length of data buffer
+ * @ret rc		Return status code
+ */
+int http_block_read ( struct http_transaction *http, struct interface *data,
+		      uint64_t lba, unsigned int count, userptr_t buffer,
+		      size_t len ) {
+	struct http_request_range range;
+	int rc;
+
+	/* Sanity check */
+	assert ( len == ( count * HTTP_BLKSIZE ) );
+
+	/* Construct request range descriptor */
+	range.start = ( lba * HTTP_BLKSIZE );
+	range.len = len;
+
+	/* Start a range request to retrieve the block(s) */
+	if ( ( rc = http_open ( data, &http_get, http->uri, &range,
+				NULL ) ) != 0 )
+		goto err_open;
+
+	/* Insert block device translator */
+	if ( ( rc = block_translate ( data, buffer, len ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not insert block translator: %s\n",
+		       http, strerror ( rc ) );
+		goto err_translate;
+	}
+
+	return 0;
+
+ err_translate:
+	intf_restart ( data, rc );
+ err_open:
+	return rc;
+}
+
+/**
+ * Read block device capacity
+ *
+ * @v control		Control interface
+ * @v data		Data interface
+ * @ret rc		Return status code
+ */
+int http_block_read_capacity ( struct http_transaction *http,
+			       struct interface *data ) {
+	int rc;
+
+	/* Start a HEAD request to retrieve the capacity */
+	if ( ( rc = http_open ( data, &http_head, http->uri, NULL,
+				NULL ) ) != 0 )
+		goto err_open;
+
+	/* Insert block device translator */
+	if ( ( rc = block_translate ( data, UNULL, HTTP_BLKSIZE ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not insert block translator: %s\n",
+		       http, strerror ( rc ) );
+		goto err_translate;
+	}
+
+	return 0;
+
+ err_translate:
+	intf_restart ( data, rc );
+ err_open:
+	return rc;
+}
+
+/**
+ * Describe device in ACPI table
+ *
+ * @v http		HTTP transaction
+ * @v acpi		ACPI table
+ * @v len		Length of ACPI table
+ * @ret rc		Return status code
+ */
+int http_acpi_describe ( struct http_transaction *http,
+			 struct acpi_description_header *acpi, size_t len ) {
+
+	DBGC ( http, "HTTP %p cannot yet describe device in an ACPI table\n",
+	       http );
+	( void ) acpi;
+	( void ) len;
+	return 0;
+}
diff --git a/roms/ipxe/src/net/tcp/httpconn.c b/roms/ipxe/src/net/tcp/httpconn.c
new file mode 100644
index 000000000..7e4877b7b
--- /dev/null
+++ b/roms/ipxe/src/net/tcp/httpconn.c
@@ -0,0 +1,309 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) connection management
+ *
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/tcpip.h>
+#include <ipxe/uri.h>
+#include <ipxe/timer.h>
+#include <ipxe/xfer.h>
+#include <ipxe/open.h>
+#include <ipxe/pool.h>
+#include <ipxe/http.h>
+
+/** HTTP pooled connection expiry time */
+#define HTTP_CONN_EXPIRY ( 10 * TICKS_PER_SEC )
+
+/** HTTP connection pool */
+static LIST_HEAD ( http_connection_pool );
+
+/**
+ * Identify HTTP scheme
+ *
+ * @v uri		URI
+ * @ret scheme		HTTP scheme, or NULL
+ */
+static struct http_scheme * http_scheme ( struct uri *uri ) {
+	struct http_scheme *scheme;
+
+	/* Sanity check */
+	if ( ! uri->scheme )
+		return NULL;
+
+	/* Identify scheme */
+	for_each_table_entry ( scheme, HTTP_SCHEMES ) {
+		if ( strcmp ( uri->scheme, scheme->name ) == 0 )
+			return scheme;
+	}
+
+	return NULL;
+}
+
+/**
+ * Free HTTP connection
+ *
+ * @v refcnt		Reference count
+ */
+static void http_conn_free ( struct refcnt *refcnt ) {
+	struct http_connection *conn =
+		container_of ( refcnt, struct http_connection, refcnt );
+
+	/* Free connection */
+	uri_put ( conn->uri );
+	free ( conn );
+}
+
+/**
+ * Close HTTP connection
+ *
+ * @v conn		HTTP connection
+ * @v rc		Reason for close
+ */
+static void http_conn_close ( struct http_connection *conn, int rc ) {
+
+	/* Remove from connection pool, if applicable */
+	pool_del ( &conn->pool );
+
+	/* Shut down interfaces */
+	intf_shutdown ( &conn->socket, rc );
+	intf_shutdown ( &conn->xfer, rc );
+	if ( rc == 0 ) {
+		DBGC2 ( conn, "HTTPCONN %p closed %s://%s\n",
+			conn, conn->scheme->name, conn->uri->host );
+	} else {
+		DBGC ( conn, "HTTPCONN %p closed %s://%s: %s\n",
+		       conn, conn->scheme->name, conn->uri->host,
+		       strerror ( rc ) );
+	}
+}
+
+/**
+ * Disconnect idle HTTP connection
+ *
+ * @v pool		Pooled connection
+ */
+static void http_conn_expired ( struct pooled_connection *pool ) {
+	struct http_connection *conn =
+		container_of ( pool, struct http_connection, pool );
+
+	/* Close connection */
+	http_conn_close ( conn, 0 /* Not an error to close idle connection */ );
+}
+
+/**
+ * Receive data from transport layer interface
+ *
+ * @v http		HTTP connection
+ * @v iobuf		I/O buffer
+ * @v meta		Transfer metadata
+ * @ret rc		Return status code
+ */
+static int http_conn_socket_deliver ( struct http_connection *conn,
+				      struct io_buffer *iobuf,
+				      struct xfer_metadata *meta ) {
+
+	/* Mark connection as alive */
+	pool_alive ( &conn->pool );
+
+	/* Pass on to data transfer interface */
+	return xfer_deliver ( &conn->xfer, iobuf, meta );
+}
+
+/**
+ * Close HTTP connection transport layer interface
+ *
+ * @v http		HTTP connection
+ * @v rc		Reason for close
+ */
+static void http_conn_socket_close ( struct http_connection *conn, int rc ) {
+
+	/* If we are reopenable (i.e. we are a recycled connection
+	 * from the connection pool, and we have received no data from
+	 * the underlying socket since we were pooled), then suggest
+	 * that the client should reopen the connection.
+	 */
+	if ( pool_is_reopenable ( &conn->pool ) )
+		pool_reopen ( &conn->xfer );
+
+	/* Close the connection */
+	http_conn_close ( conn, rc );
+}
+
+/**
+ * Recycle this connection after closing
+ *
+ * @v http		HTTP connection
+ */
+static void http_conn_xfer_recycle ( struct http_connection *conn ) {
+
+	/* Mark connection as recyclable */
+	pool_recyclable ( &conn->pool );
+	DBGC2 ( conn, "HTTPCONN %p keepalive enabled\n", conn );
+}
+
+/**
+ * Close HTTP connection data transfer interface
+ *
+ * @v conn		HTTP connection
+ * @v rc		Reason for close
+ */
+static void http_conn_xfer_close ( struct http_connection *conn, int rc ) {
+
+	/* Add to the connection pool if keepalive is enabled and no
+	 * error occurred.
+	 */
+	if ( ( rc == 0 ) && pool_is_recyclable ( &conn->pool ) ) {
+		intf_restart ( &conn->xfer, rc );
+		pool_add ( &conn->pool, &http_connection_pool,
+			   HTTP_CONN_EXPIRY );
+		DBGC2 ( conn, "HTTPCONN %p pooled %s://%s\n",
+			conn, conn->scheme->name, conn->uri->host );
+		return;
+	}
+
+	/* Otherwise, close the connection */
+	http_conn_close ( conn, rc );
+}
+
+/** HTTP connection socket interface operations */
+static struct interface_operation http_conn_socket_operations[] = {
+	INTF_OP ( xfer_deliver, struct http_connection *,
+		  http_conn_socket_deliver ),
+	INTF_OP ( intf_close, struct http_connection *,
+		  http_conn_socket_close ),
+};
+
+/** HTTP connection socket interface descriptor */
+static struct interface_descriptor http_conn_socket_desc =
+	INTF_DESC_PASSTHRU ( struct http_connection, socket,
+			     http_conn_socket_operations, xfer );
+
+/** HTTP connection data transfer interface operations */
+static struct interface_operation http_conn_xfer_operations[] = {
+	INTF_OP ( pool_recycle, struct http_connection *,
+		  http_conn_xfer_recycle ),
+	INTF_OP ( intf_close, struct http_connection *,
+		  http_conn_xfer_close ),
+};
+
+/** HTTP connection data transfer interface descriptor */
+static struct interface_descriptor http_conn_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct http_connection, xfer,
+			     http_conn_xfer_operations, socket );
+
+/**
+ * Connect to an HTTP server
+ *
+ * @v xfer		Data transfer interface
+ * @v uri		Connection URI
+ * @ret rc		Return status code
+ *
+ * HTTP connections are pooled.  The caller should be prepared to
+ * receive a pool_reopen() message.
+ */
+int http_connect ( struct interface *xfer, struct uri *uri ) {
+	struct http_connection *conn;
+	struct http_scheme *scheme;
+	struct sockaddr_tcpip server;
+	struct interface *socket;
+	int rc;
+
+	/* Identify scheme */
+	scheme = http_scheme ( uri );
+	if ( ! scheme )
+		return -ENOTSUP;
+
+	/* Sanity check */
+	if ( ! uri->host )
+		return -EINVAL;
+
+	/* Look for a reusable connection in the pool */
+	list_for_each_entry ( conn, &http_connection_pool, pool.list ) {
+
+		/* Sanity checks */
+		assert ( conn->uri != NULL );
+		assert ( conn->uri->host != NULL );
+
+		/* Reuse connection, if possible */
+		if ( ( scheme == conn->scheme ) &&
+		     ( strcmp ( uri->host, conn->uri->host ) == 0 ) ) {
+
+			/* Remove from connection pool, stop timer,
+			 * attach to parent interface, and return.
+			 */
+			pool_del ( &conn->pool );
+			intf_plug_plug ( &conn->xfer, xfer );
+			DBGC2 ( conn, "HTTPCONN %p reused %s://%s\n",
+				conn, conn->scheme->name, conn->uri->host );
+			return 0;
+		}
+	}
+
+	/* Allocate and initialise structure */
+	conn = zalloc ( sizeof ( *conn ) );
+	ref_init ( &conn->refcnt, http_conn_free );
+	conn->uri = uri_get ( uri );
+	conn->scheme = scheme;
+	intf_init ( &conn->socket, &http_conn_socket_desc, &conn->refcnt );
+	intf_init ( &conn->xfer, &http_conn_xfer_desc, &conn->refcnt );
+	pool_init ( &conn->pool, http_conn_expired, &conn->refcnt );
+
+	/* Open socket */
+	memset ( &server, 0, sizeof ( server ) );
+	server.st_port = htons ( uri_port ( uri, scheme->port ) );
+	socket = &conn->socket;
+	if ( scheme->filter &&
+	     ( ( rc = scheme->filter ( socket, uri->host, &socket ) ) != 0 ) )
+		goto err_filter;
+	if ( ( rc = xfer_open_named_socket ( socket, SOCK_STREAM,
+					     ( struct sockaddr * ) &server,
+					     uri->host, NULL ) ) != 0 )
+		goto err_open;
+
+	/* Attach to parent interface, mortalise self, and return */
+	intf_plug_plug ( &conn->xfer, xfer );
+	ref_put ( &conn->refcnt );
+
+	DBGC2 ( conn, "HTTPCONN %p created %s://%s:%d\n", conn,
+		conn->scheme->name, conn->uri->host, ntohs ( server.st_port ) );
+	return 0;
+
+ err_open:
+ err_filter:
+	DBGC2 ( conn, "HTTPCONN %p could not create %s://%s: %s\n",
+		conn, conn->scheme->name, conn->uri->host, strerror ( rc ) );
+	http_conn_close ( conn, rc );
+	ref_put ( &conn->refcnt );
+	return rc;
+}
diff --git a/roms/ipxe/src/net/tcp/httpcore.c b/roms/ipxe/src/net/tcp/httpcore.c
index 1d1953e61..af3ca9780 100644
--- a/roms/ipxe/src/net/tcp/httpcore.c
+++ b/roms/ipxe/src/net/tcp/httpcore.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -40,35 +44,26 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/iobuf.h>
 #include <ipxe/xfer.h>
 #include <ipxe/open.h>
-#include <ipxe/socket.h>
-#include <ipxe/tcpip.h>
 #include <ipxe/process.h>
 #include <ipxe/retry.h>
 #include <ipxe/timer.h>
 #include <ipxe/linebuf.h>
-#include <ipxe/base64.h>
-#include <ipxe/base16.h>
-#include <ipxe/md5.h>
+#include <ipxe/xferbuf.h>
 #include <ipxe/blockdev.h>
 #include <ipxe/acpi.h>
 #include <ipxe/version.h>
 #include <ipxe/params.h>
 #include <ipxe/profile.h>
+#include <ipxe/vsprintf.h>
 #include <ipxe/http.h>
 
 /* Disambiguate the various error causes */
 #define EACCES_401 __einfo_error ( EINFO_EACCES_401 )
 #define EINFO_EACCES_401 \
 	__einfo_uniqify ( EINFO_EACCES, 0x01, "HTTP 401 Unauthorized" )
-#define EIO_OTHER __einfo_error ( EINFO_EIO_OTHER )
-#define EINFO_EIO_OTHER \
-	__einfo_uniqify ( EINFO_EIO, 0x01, "Unrecognised HTTP response code" )
-#define EIO_CONTENT_LENGTH __einfo_error ( EINFO_EIO_CONTENT_LENGTH )
-#define EINFO_EIO_CONTENT_LENGTH \
-	__einfo_uniqify ( EINFO_EIO, 0x02, "Content length mismatch" )
-#define EINVAL_RESPONSE __einfo_error ( EINFO_EINVAL_RESPONSE )
-#define EINFO_EINVAL_RESPONSE \
-	__einfo_uniqify ( EINFO_EINVAL, 0x01, "Invalid content length" )
+#define EINVAL_STATUS __einfo_error ( EINFO_EINVAL_STATUS )
+#define EINFO_EINVAL_STATUS \
+	__einfo_uniqify ( EINFO_EINVAL, 0x01, "Invalid status line" )
 #define EINVAL_HEADER __einfo_error ( EINFO_EINVAL_HEADER )
 #define EINFO_EINVAL_HEADER \
 	__einfo_uniqify ( EINFO_EINVAL, 0x02, "Invalid header" )
@@ -78,9 +73,27 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define EINVAL_CHUNK_LENGTH __einfo_error ( EINFO_EINVAL_CHUNK_LENGTH )
 #define EINFO_EINVAL_CHUNK_LENGTH \
 	__einfo_uniqify ( EINFO_EINVAL, 0x04, "Invalid chunk length" )
+#define EIO_OTHER __einfo_error ( EINFO_EIO_OTHER )
+#define EINFO_EIO_OTHER \
+	__einfo_uniqify ( EINFO_EIO, 0x01, "Unrecognised HTTP response code" )
+#define EIO_CONTENT_LENGTH __einfo_error ( EINFO_EIO_CONTENT_LENGTH )
+#define EINFO_EIO_CONTENT_LENGTH \
+	__einfo_uniqify ( EINFO_EIO, 0x02, "Content length mismatch" )
+#define EIO_4XX __einfo_error ( EINFO_EIO_4XX )
+#define EINFO_EIO_4XX \
+	__einfo_uniqify ( EINFO_EIO, 0x04, "HTTP 4xx Client Error" )
+#define EIO_5XX __einfo_error ( EINFO_EIO_5XX )
+#define EINFO_EIO_5XX \
+	__einfo_uniqify ( EINFO_EIO, 0x05, "HTTP 5xx Server Error" )
 #define ENOENT_404 __einfo_error ( EINFO_ENOENT_404 )
 #define EINFO_ENOENT_404 \
 	__einfo_uniqify ( EINFO_ENOENT, 0x01, "HTTP 404 Not Found" )
+#define ENOTSUP_CONNECTION __einfo_error ( EINFO_ENOTSUP_CONNECTION )
+#define EINFO_ENOTSUP_CONNECTION \
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01, "Unsupported connection header" )
+#define ENOTSUP_TRANSFER __einfo_error ( EINFO_ENOTSUP_TRANSFER )
+#define EINFO_ENOTSUP_TRANSFER \
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x02, "Unsupported transfer encoding" )
 #define EPERM_403 __einfo_error ( EINFO_EPERM_403 )
 #define EINFO_EPERM_403 \
 	__einfo_uniqify ( EINFO_EPERM, 0x01, "HTTP 403 Forbidden" )
@@ -88,9 +101,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define EINFO_EPROTO_UNSOLICITED \
 	__einfo_uniqify ( EINFO_EPROTO, 0x01, "Unsolicited data" )
 
-/** Block size used for HTTP block device request */
-#define HTTP_BLKSIZE 512
-
 /** Retry delay used when we cannot understand the Retry-After header */
 #define HTTP_RETRY_SECONDS 5
 
@@ -100,1069 +110,1711 @@ static struct profiler http_rx_profiler __profiler = { .name = "http.rx" };
 /** Data transfer profiler */
 static struct profiler http_xfer_profiler __profiler = { .name = "http.xfer" };
 
-/** HTTP flags */
-enum http_flags {
-	/** Request is waiting to be transmitted */
-	HTTP_TX_PENDING = 0x0001,
-	/** Fetch header only */
-	HTTP_HEAD_ONLY = 0x0002,
-	/** Client would like to keep connection alive */
-	HTTP_CLIENT_KEEPALIVE = 0x0004,
-	/** Server will keep connection alive */
-	HTTP_SERVER_KEEPALIVE = 0x0008,
-	/** Discard the current request and try again */
-	HTTP_TRY_AGAIN = 0x0010,
-	/** Provide Basic authentication details */
-	HTTP_BASIC_AUTH = 0x0020,
-	/** Provide Digest authentication details */
-	HTTP_DIGEST_AUTH = 0x0040,
-	/** Socket must be reopened */
-	HTTP_REOPEN_SOCKET = 0x0080,
+static struct http_state http_request;
+static struct http_state http_headers;
+static struct http_state http_trailers;
+static struct http_transfer_encoding http_transfer_identity;
+
+/******************************************************************************
+ *
+ * Methods
+ *
+ ******************************************************************************
+ */
+
+/** HTTP HEAD method */
+struct http_method http_head = {
+	.name = "HEAD",
 };
 
-/** HTTP receive state */
-enum http_rx_state {
-	HTTP_RX_RESPONSE = 0,
-	HTTP_RX_HEADER,
-	HTTP_RX_CHUNK_LEN,
-	/* In HTTP_RX_DATA, it is acceptable for the server to close
-	 * the connection (unless we are in the middle of a chunked
-	 * transfer).
-	 */
-	HTTP_RX_DATA,
-	/* In the following states, it is acceptable for the server to
-	 * close the connection.
-	 */
-	HTTP_RX_TRAILER,
-	HTTP_RX_IDLE,
-	HTTP_RX_DEAD,
+/** HTTP GET method */
+struct http_method http_get = {
+	.name = "GET",
 };
 
+/** HTTP POST method */
+struct http_method http_post = {
+	.name = "POST",
+};
+
+/******************************************************************************
+ *
+ * Utility functions
+ *
+ ******************************************************************************
+ */
+
 /**
- * An HTTP request
+ * Handle received HTTP line-buffered data
  *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer
+ * @v linebuf		Line buffer
+ * @ret rc		Return status code
  */
-struct http_request {
-	/** Reference count */
-	struct refcnt refcnt;
-	/** Data transfer interface */
-	struct interface xfer;
-	/** Partial transfer interface */
-	struct interface partial;
-
-	/** URI being fetched */
-	struct uri *uri;
-	/** Default port */
-	unsigned int default_port;
-	/** Filter (if any) */
-	int ( * filter ) ( struct interface *xfer,
-			   const char *name,
-			   struct interface **next );
-	/** Transport layer interface */
-	struct interface socket;
-
-	/** Flags */
-	unsigned int flags;
-	/** Starting offset of partial transfer (if applicable) */
-	size_t partial_start;
-	/** Length of partial transfer (if applicable) */
-	size_t partial_len;
-
-	/** TX process */
-	struct process process;
-
-	/** RX state */
-	enum http_rx_state rx_state;
-	/** Response code */
-	unsigned int code;
-	/** Received length */
-	size_t rx_len;
-	/** Length remaining (or 0 if unknown) */
-	size_t remaining;
-	/** HTTP is using Transfer-Encoding: chunked */
-	int chunked;
-	/** Current chunk length remaining (if applicable) */
-	size_t chunk_remaining;
-	/** Line buffer for received header lines */
-	struct line_buffer linebuf;
-	/** Receive data buffer (if applicable) */
-	userptr_t rx_buffer;
-
-	/** Authentication realm (if any) */
-	char *auth_realm;
-	/** Authentication nonce (if any) */
-	char *auth_nonce;
-	/** Authentication opaque string (if any) */
-	char *auth_opaque;
-
-	/** Request retry timer */
-	struct retry_timer timer;
-	/** Retry delay (in timer ticks) */
-	unsigned long retry_delay;
-};
+static int http_rx_linebuf ( struct http_transaction *http,
+			     struct io_buffer *iobuf,
+			     struct line_buffer *linebuf ) {
+	int consumed;
+	int rc;
+
+	/* Buffer received line */
+	consumed = line_buffer ( linebuf, iobuf->data, iob_len ( iobuf ) );
+	if ( consumed < 0 ) {
+		rc = consumed;
+		DBGC ( http, "HTTP %p could not buffer line: %s\n",
+		       http, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Consume line */
+	iob_pull ( iobuf, consumed );
+
+	return 0;
+}
 
 /**
- * Free HTTP request
+ * Get HTTP response token
  *
- * @v refcnt		Reference counter
+ * @v line		Line position
+ * @v value		Token value to fill in (if any)
+ * @ret token		Token, or NULL
+ */
+char * http_token ( char **line, char **value ) {
+	char *token;
+	char quote = '\0';
+	char c;
+
+	/* Avoid returning uninitialised data */
+	if ( value )
+		*value = NULL;
+
+	/* Skip any initial whitespace */
+	while ( isspace ( **line ) )
+		(*line)++;
+
+	/* Check for end of line and record token position */
+	if ( ! **line )
+		return NULL;
+	token = *line;
+
+	/* Scan for end of token */
+	while ( ( c = **line ) ) {
+
+		/* Terminate if we hit an unquoted whitespace */
+		if ( isspace ( c ) && ! quote )
+			break;
+
+		/* Terminate if we hit a closing quote */
+		if ( c == quote )
+			break;
+
+		/* Check for value separator */
+		if ( value && ( ! *value ) && ( c == '=' ) ) {
+
+			/* Terminate key portion of token */
+			*((*line)++) = '\0';
+
+			/* Check for quote character */
+			c = **line;
+			if ( ( c == '"' ) || ( c == '\'' ) ) {
+				quote = c;
+				(*line)++;
+			}
+
+			/* Record value portion of token */
+			*value = *line;
+
+		} else {
+
+			/* Move to next character */
+			(*line)++;
+		}
+	}
+
+	/* Terminate token, if applicable */
+	if ( c )
+		*((*line)++) = '\0';
+
+	return token;
+}
+
+/******************************************************************************
+ *
+ * Transactions
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Free HTTP transaction
+ *
+ * @v refcnt		Reference count
  */
 static void http_free ( struct refcnt *refcnt ) {
-	struct http_request *http =
-		container_of ( refcnt, struct http_request, refcnt );
+	struct http_transaction *http =
+		container_of ( refcnt, struct http_transaction, refcnt );
 
-	uri_put ( http->uri );
+	empty_line_buffer ( &http->response.headers );
 	empty_line_buffer ( &http->linebuf );
-	free ( http->auth_realm );
-	free ( http->auth_nonce );
-	free ( http->auth_opaque );
+	uri_put ( http->uri );
 	free ( http );
-};
+}
 
 /**
- * Close HTTP request
+ * Close HTTP transaction
  *
- * @v http		HTTP request
- * @v rc		Return status code
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
  */
-static void http_close ( struct http_request *http, int rc ) {
-
-	/* Prevent further processing of any current packet */
-	http->rx_state = HTTP_RX_DEAD;
+static void http_close ( struct http_transaction *http, int rc ) {
 
-	/* Prevent reconnection */
-	http->flags &= ~HTTP_CLIENT_KEEPALIVE;
-
-	/* Remove process */
+	/* Stop process */
 	process_del ( &http->process );
 
-	/* Close all data transfer interfaces */
-	intf_shutdown ( &http->socket, rc );
-	intf_shutdown ( &http->partial, rc );
+	/* Stop timer */
+	stop_timer ( &http->timer );
+
+	/* Close all interfaces, allowing for the fact that the
+	 * content-decoded and transfer-decoded interfaces may be
+	 * connected to the same object.
+	 */
+	intf_shutdown ( &http->conn, rc );
+	intf_nullify ( &http->transfer );
+	intf_shutdown ( &http->content, rc );
+	intf_shutdown ( &http->transfer, rc );
 	intf_shutdown ( &http->xfer, rc );
 }
 
 /**
- * Open HTTP socket
+ * Close HTTP transaction with error (even if none specified)
  *
- * @v http		HTTP request
- * @ret rc		Return status code
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
  */
-static int http_socket_open ( struct http_request *http ) {
-	struct uri *uri = http->uri;
-	struct sockaddr_tcpip server;
-	struct interface *socket;
+static void http_close_error ( struct http_transaction *http, int rc ) {
+
+	/* Treat any close as an error */
+	http_close ( http, ( rc ? rc : -EPIPE ) );
+}
+
+/**
+ * Reopen stale HTTP connection
+ *
+ * @v http		HTTP transaction
+ */
+static void http_reopen ( struct http_transaction *http ) {
 	int rc;
 
-	/* Open socket */
-	memset ( &server, 0, sizeof ( server ) );
-	server.st_port = htons ( uri_port ( uri, http->default_port ) );
-	socket = &http->socket;
-	if ( http->filter ) {
-		if ( ( rc = http->filter ( socket, uri->host, &socket ) ) != 0 )
-			return rc;
+	/* Close existing connection */
+	intf_restart ( &http->conn, -ECANCELED );
+
+	/* Reopen connection */
+	if ( ( rc = http_connect ( &http->conn, http->uri ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not reconnect: %s\n",
+		       http, strerror ( rc ) );
+		goto err_connect;
 	}
-	if ( ( rc = xfer_open_named_socket ( socket, SOCK_STREAM,
-					     ( struct sockaddr * ) &server,
-					     uri->host, NULL ) ) != 0 )
-		return rc;
 
-	return 0;
+	/* Reset state */
+	http->state = &http_request;
+
+	/* Reschedule transmission process */
+	process_add ( &http->process );
+
+	return;
+
+ err_connect:
+	http_close ( http, rc );
 }
 
 /**
- * Retry HTTP request
+ * Handle retry timer expiry
  *
  * @v timer		Retry timer
- * @v fail		Failure indicator
+ * @v over		Failure indicator
+ */
+static void http_expired ( struct retry_timer *timer, int over __unused ) {
+	struct http_transaction *http =
+		container_of ( timer, struct http_transaction, timer );
+
+	/* Reopen connection */
+	http_reopen ( http );
+}
+
+/**
+ * HTTP transmit process
+ *
+ * @v http		HTTP transaction
  */
-static void http_retry ( struct retry_timer *timer, int fail __unused ) {
-	struct http_request *http =
-		container_of ( timer, struct http_request, timer );
+static void http_step ( struct http_transaction *http ) {
 	int rc;
 
-	/* Reopen socket if required */
-	if ( http->flags & HTTP_REOPEN_SOCKET ) {
-		http->flags &= ~HTTP_REOPEN_SOCKET;
-		DBGC ( http, "HTTP %p reopening connection\n", http );
-		if ( ( rc = http_socket_open ( http ) ) != 0 ) {
-			http_close ( http, rc );
-			return;
+	/* Do nothing if we have nothing to transmit */
+	if ( ! http->state->tx )
+		return;
+
+	/* Do nothing until connection is ready */
+	if ( ! xfer_window ( &http->conn ) )
+		return;
+
+	/* Do nothing until data transfer interface is ready */
+	if ( ! xfer_window ( &http->xfer ) )
+		return;
+
+	/* Transmit data */
+	if ( ( rc = http->state->tx ( http ) ) != 0 )
+		goto err;
+
+	return;
+
+ err:
+	http_close ( http, rc );
+}
+
+/**
+ * Handle received HTTP data
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer
+ * @v meta		Transfer metadata
+ * @ret rc		Return status code
+ *
+ * This function takes ownership of the I/O buffer.
+ */
+static int http_conn_deliver ( struct http_transaction *http,
+			       struct io_buffer *iobuf,
+			       struct xfer_metadata *meta __unused ) {
+	int rc;
+
+	/* Handle received data */
+	profile_start ( &http_rx_profiler );
+	while ( iobuf && iob_len ( iobuf ) ) {
+
+		/* Sanity check */
+		if ( ( ! http->state ) || ( ! http->state->rx ) ) {
+			DBGC ( http, "HTTP %p unexpected data\n", http );
+			rc = -EPROTO_UNSOLICITED;
+			goto err;
 		}
-	}
 
-	/* Retry the request if applicable */
-	if ( http->flags & HTTP_TRY_AGAIN ) {
-		http->flags &= ~HTTP_TRY_AGAIN;
-		DBGC ( http, "HTTP %p retrying request\n", http );
-		http->flags |= HTTP_TX_PENDING;
-		http->rx_state = HTTP_RX_RESPONSE;
-		process_add ( &http->process );
+		/* Receive (some) data */
+		if ( ( rc = http->state->rx ( http, &iobuf ) ) != 0 )
+			goto err;
 	}
+
+	/* Free I/O buffer, if applicable */
+	free_iob ( iobuf );
+
+	profile_stop ( &http_rx_profiler );
+	return 0;
+
+ err:
+	free_iob ( iobuf );
+	http_close ( http, rc );
+	return rc;
 }
 
 /**
- * Mark HTTP request as completed successfully
+ * Handle server connection close
  *
- * @v http		HTTP request
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
  */
-static void http_done ( struct http_request *http ) {
+static void http_conn_close ( struct http_transaction *http, int rc ) {
 
-	/* If we are not at an appropriate stage of the protocol
-	 * (including being in the middle of a chunked transfer),
-	 * force an error.
-	 */
-	if ( ( http->rx_state < HTTP_RX_DATA ) || ( http->chunked != 0 ) ) {
-		DBGC ( http, "HTTP %p connection closed unexpectedly in state "
-		       "%d\n", http, http->rx_state );
-		http_close ( http, -ECONNRESET );
-		return;
-	}
+	/* Sanity checks */
+	assert ( http->state != NULL );
+	assert ( http->state->close != NULL );
+
+	/* Restart server connection interface */
+	intf_restart ( &http->conn, rc );
+
+	/* Hand off to state-specific method */
+	http->state->close ( http, rc );
+}
 
-	/* If we had a Content-Length, and the received content length
-	 * isn't correct, force an error
+/**
+ * Handle received content-decoded data
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ */
+static int http_content_deliver ( struct http_transaction *http,
+				  struct io_buffer *iobuf,
+				  struct xfer_metadata *meta ) {
+	int rc;
+
+	/* Ignore content if this is anything other than a successful
+	 * transfer.
 	 */
-	if ( http->remaining != 0 ) {
-		DBGC ( http, "HTTP %p incorrect length %zd, should be %zd\n",
-		       http, http->rx_len, ( http->rx_len + http->remaining ) );
-		http_close ( http, -EIO_CONTENT_LENGTH );
-		return;
+	if ( http->response.rc != 0 ) {
+		free_iob ( iobuf );
+		return 0;
 	}
 
-	/* Enter idle state */
-	http->rx_state = HTTP_RX_IDLE;
-	http->rx_len = 0;
-	assert ( http->remaining == 0 );
-	assert ( http->chunked == 0 );
-	assert ( http->chunk_remaining == 0 );
+	/* Deliver to data transfer interface */
+	profile_start ( &http_xfer_profiler );
+	if ( ( rc = xfer_deliver ( &http->xfer, iob_disown ( iobuf ),
+				   meta ) ) != 0 )
+		return rc;
+	profile_stop ( &http_xfer_profiler );
 
-	/* Close partial transfer interface */
-	if ( ! ( http->flags & HTTP_TRY_AGAIN ) )
-		intf_restart ( &http->partial, 0 );
+	return 0;
+}
 
-	/* Close everything unless we want to keep the connection alive */
-	if ( ! ( http->flags & ( HTTP_CLIENT_KEEPALIVE | HTTP_TRY_AGAIN ) ) ) {
-		http_close ( http, 0 );
-		return;
-	}
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v http		HTTP transaction
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ */
+static struct xfer_buffer *
+http_content_buffer ( struct http_transaction *http ) {
 
-	/* If the server is not intending to keep the connection
-	 * alive, then close the socket and mark it as requiring
-	 * reopening.
+	/* Deny access to the data transfer buffer if this is anything
+	 * other than a successful transfer.
 	 */
-	if ( ! ( http->flags & HTTP_SERVER_KEEPALIVE ) ) {
-		intf_restart ( &http->socket, 0 );
-		http->flags &= ~HTTP_SERVER_KEEPALIVE;
-		http->flags |= HTTP_REOPEN_SOCKET;
-	}
+	if ( http->response.rc != 0 )
+		return NULL;
 
-	/* Start request retry timer */
-	start_timer_fixed ( &http->timer, http->retry_delay );
-	http->retry_delay = 0;
+	/* Hand off to data transfer interface */
+	return xfer_buffer ( &http->xfer );
 }
 
 /**
- * Convert HTTP response code to return status code
+ * Read from block device (when HTTP block device support is not present)
  *
- * @v response		HTTP response code
+ * @v http		HTTP transaction
+ * @v data		Data interface
+ * @v lba		Starting logical block address
+ * @v count		Number of logical blocks
+ * @v buffer		Data buffer
+ * @v len		Length of data buffer
  * @ret rc		Return status code
  */
-static int http_response_to_rc ( unsigned int response ) {
-	switch ( response ) {
-	case 200:
-	case 206:
-	case 301:
-	case 302:
-	case 303:
-		return 0;
-	case 404:
-		return -ENOENT_404;
-	case 403:
-		return -EPERM_403;
-	case 401:
-		return -EACCES_401;
-	default:
-		return -EIO_OTHER;
-	}
+__weak int http_block_read ( struct http_transaction *http __unused,
+			     struct interface *data __unused,
+			     uint64_t lba __unused, unsigned int count __unused,
+			     userptr_t buffer __unused, size_t len __unused ) {
+
+	return -ENOTSUP;
 }
 
 /**
- * Handle HTTP response
+ * Read block device capacity (when HTTP block device support is not present)
  *
- * @v http		HTTP request
- * @v response		HTTP response
+ * @v control		Control interface
+ * @v data		Data interface
  * @ret rc		Return status code
  */
-static int http_rx_response ( struct http_request *http, char *response ) {
-	char *spc;
-
-	DBGC ( http, "HTTP %p response \"%s\"\n", http, response );
+__weak int http_block_read_capacity ( struct http_transaction *http __unused,
+				      struct interface *data __unused ) {
 
-	/* Check response starts with "HTTP/" */
-	if ( strncmp ( response, "HTTP/", 5 ) != 0 )
-		return -EINVAL_RESPONSE;
+	return -ENOTSUP;
+}
 
-	/* Locate and store response code */
-	spc = strchr ( response, ' ' );
-	if ( ! spc )
-		return -EINVAL_RESPONSE;
-	http->code = strtoul ( spc, NULL, 10 );
+/**
+ * Describe device in ACPI table (when HTTP block device support is not present)
+ *
+ * @v http		HTTP transaction
+ * @v acpi		ACPI table
+ * @v len		Length of ACPI table
+ * @ret rc		Return status code
+ */
+__weak int http_acpi_describe ( struct http_transaction *http __unused,
+				struct acpi_description_header *acpi __unused,
+				size_t len __unused ) {
 
-	/* Move to receive headers */
-	http->rx_state = ( ( http->flags & HTTP_HEAD_ONLY ) ?
-			   HTTP_RX_TRAILER : HTTP_RX_HEADER );
-	return 0;
+	return -ENOTSUP;
 }
 
+/** HTTP data transfer interface operations */
+static struct interface_operation http_xfer_operations[] = {
+	INTF_OP ( block_read, struct http_transaction *, http_block_read ),
+	INTF_OP ( block_read_capacity, struct http_transaction *,
+		  http_block_read_capacity ),
+	INTF_OP ( acpi_describe, struct http_transaction *,
+		  http_acpi_describe ),
+	INTF_OP ( xfer_window_changed, struct http_transaction *, http_step ),
+	INTF_OP ( intf_close, struct http_transaction *, http_close ),
+};
+
+/** HTTP data transfer interface descriptor */
+static struct interface_descriptor http_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, xfer,
+			     http_xfer_operations, content );
+
+/** HTTP content-decoded interface operations */
+static struct interface_operation http_content_operations[] = {
+	INTF_OP ( xfer_deliver, struct http_transaction *,
+		  http_content_deliver ),
+	INTF_OP ( xfer_buffer, struct http_transaction *, http_content_buffer ),
+	INTF_OP ( intf_close, struct http_transaction *, http_close ),
+};
+
+/** HTTP content-decoded interface descriptor */
+static struct interface_descriptor http_content_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, content,
+			     http_content_operations, xfer );
+
+/** HTTP transfer-decoded interface operations */
+static struct interface_operation http_transfer_operations[] = {
+	INTF_OP ( intf_close, struct http_transaction *, http_close ),
+};
+
+/** HTTP transfer-decoded interface descriptor */
+static struct interface_descriptor http_transfer_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, transfer,
+			     http_transfer_operations, conn );
+
+/** HTTP server connection interface operations */
+static struct interface_operation http_conn_operations[] = {
+	INTF_OP ( xfer_deliver, struct http_transaction *, http_conn_deliver ),
+	INTF_OP ( xfer_window_changed, struct http_transaction *, http_step ),
+	INTF_OP ( pool_reopen, struct http_transaction *, http_reopen ),
+	INTF_OP ( intf_close, struct http_transaction *, http_conn_close ),
+};
+
+/** HTTP server connection interface descriptor */
+static struct interface_descriptor http_conn_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, conn,
+			     http_conn_operations, transfer );
+
+/** HTTP process descriptor */
+static struct process_descriptor http_process_desc =
+	PROC_DESC_ONCE ( struct http_transaction, process, http_step );
+
 /**
- * Handle HTTP Location header
+ * Open HTTP transaction
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v xfer		Data transfer interface
+ * @v method		Request method
+ * @v uri		Request URI
+ * @v range		Content range (if any)
+ * @v content		Request content (if any)
  * @ret rc		Return status code
  */
-static int http_rx_location ( struct http_request *http, char *value ) {
+int http_open ( struct interface *xfer, struct http_method *method,
+		struct uri *uri, struct http_request_range *range,
+		struct http_request_content *content ) {
+	struct http_transaction *http;
+	struct uri request_uri;
+	struct uri request_host;
+	size_t request_uri_len;
+	size_t request_host_len;
+	size_t content_len;
+	char *request_uri_string;
+	char *request_host_string;
+	void *content_data;
 	int rc;
 
-	/* Redirect to new location */
-	DBGC ( http, "HTTP %p redirecting to %s\n", http, value );
-	if ( ( rc = xfer_redirect ( &http->xfer, LOCATION_URI_STRING,
-				    value ) ) != 0 ) {
-		DBGC ( http, "HTTP %p could not redirect: %s\n",
+	/* Calculate request URI length */
+	memset ( &request_uri, 0, sizeof ( request_uri ) );
+	request_uri.path = ( uri->path ? uri->path : "/" );
+	request_uri.query = uri->query;
+	request_uri_len =
+		( format_uri ( &request_uri, NULL, 0 ) + 1 /* NUL */);
+
+	/* Calculate host name length */
+	memset ( &request_host, 0, sizeof ( request_host ) );
+	request_host.host = uri->host;
+	request_host.port = uri->port;
+	request_host_len =
+		( format_uri ( &request_host, NULL, 0 ) + 1 /* NUL */ );
+
+	/* Calculate request content length */
+	content_len = ( content ? content->len : 0 );
+
+	/* Allocate and initialise structure */
+	http = zalloc ( sizeof ( *http ) + request_uri_len + request_host_len +
+			content_len );
+	if ( ! http ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	request_uri_string = ( ( ( void * ) http ) + sizeof ( *http ) );
+	request_host_string = ( request_uri_string + request_uri_len );
+	content_data = ( request_host_string + request_host_len );
+	format_uri ( &request_uri, request_uri_string, request_uri_len );
+	format_uri ( &request_host, request_host_string, request_host_len );
+	ref_init ( &http->refcnt, http_free );
+	intf_init ( &http->xfer, &http_xfer_desc, &http->refcnt );
+	intf_init ( &http->content, &http_content_desc, &http->refcnt );
+	intf_init ( &http->transfer, &http_transfer_desc, &http->refcnt );
+	intf_init ( &http->conn, &http_conn_desc, &http->refcnt );
+	intf_plug_plug ( &http->transfer, &http->content );
+	process_init ( &http->process, &http_process_desc, &http->refcnt );
+	timer_init ( &http->timer, http_expired, &http->refcnt );
+	http->uri = uri_get ( uri );
+	http->request.method = method;
+	http->request.uri = request_uri_string;
+	http->request.host = request_host_string;
+	if ( range ) {
+		memcpy ( &http->request.range, range,
+			 sizeof ( http->request.range ) );
+	}
+	if ( content ) {
+		http->request.content.type = content->type;
+		http->request.content.data = content_data;
+		http->request.content.len = content_len;
+		memcpy ( content_data, content->data, content_len );
+	}
+	http->state = &http_request;
+	DBGC2 ( http, "HTTP %p %s://%s%s\n", http, http->uri->scheme,
+		http->request.host, http->request.uri );
+
+	/* Open connection */
+	if ( ( rc = http_connect ( &http->conn, uri ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not connect: %s\n",
 		       http, strerror ( rc ) );
-		return rc;
+		goto err_connect;
 	}
 
+	/* Attach to parent interface, mortalise self, and return */
+	intf_plug_plug ( &http->xfer, xfer );
+	ref_put ( &http->refcnt );
 	return 0;
+
+ err_connect:
+	http_close ( http, rc );
+	ref_put ( &http->refcnt );
+ err_alloc:
+	return rc;
 }
 
 /**
- * Handle HTTP Content-Length header
+ * Handle successful transfer completion
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_content_length ( struct http_request *http, char *value ) {
-	struct block_device_capacity capacity;
-	size_t content_len;
-	char *endp;
+static int http_transfer_complete ( struct http_transaction *http ) {
+	struct http_authentication *auth;
+	const char *location;
+	int rc;
 
-	/* Parse content length */
-	content_len = strtoul ( value, &endp, 10 );
-	if ( ! ( ( *endp == '\0' ) || isspace ( *endp ) ) ) {
-		DBGC ( http, "HTTP %p invalid Content-Length \"%s\"\n",
-		       http, value );
-		return -EINVAL_CONTENT_LENGTH;
+	/* Keep connection alive if applicable */
+	if ( http->response.flags & HTTP_RESPONSE_KEEPALIVE )
+		pool_recycle ( &http->conn );
+
+	/* Restart server connection interface */
+	intf_restart ( &http->conn, 0 );
+
+	/* No more data is expected */
+	http->state = NULL;
+
+	/* If transaction is successful, then close the
+	 * transfer-decoded interface.  The content encoding may
+	 * choose whether or not to immediately terminate the
+	 * transaction.
+	 */
+	if ( http->response.rc == 0 ) {
+		intf_shutdown ( &http->transfer, 0 );
+		return 0;
 	}
 
-	/* If we already have an expected content length, and this
-	 * isn't it, then complain
+	/* Perform redirection, if applicable */
+	if ( ( location = http->response.location ) ) {
+		DBGC2 ( http, "HTTP %p redirecting to \"%s\"\n",
+			http, location );
+		if ( ( rc = xfer_redirect ( &http->xfer, LOCATION_URI_STRING,
+					    location ) ) != 0 ) {
+			DBGC ( http, "HTTP %p could not redirect: %s\n",
+			       http, strerror ( rc ) );
+			return rc;
+		}
+		http_close ( http, 0 );
+		return 0;
+	}
+
+	/* Fail unless a retry is permitted */
+	if ( ! ( http->response.flags & HTTP_RESPONSE_RETRY ) )
+		return http->response.rc;
+
+	/* Perform authentication, if applicable */
+	if ( ( auth = http->response.auth.auth ) ) {
+		http->request.auth.auth = auth;
+		DBGC2 ( http, "HTTP %p performing %s authentication\n",
+			http, auth->name );
+		if ( ( rc = auth->authenticate ( http ) ) != 0 ) {
+			DBGC ( http, "HTTP %p could not authenticate: %s\n",
+			       http, strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	/* Restart content decoding interfaces (which may be attached
+	 * to the same object).
 	 */
-	if ( http->remaining && ( http->remaining != content_len ) ) {
-		DBGC ( http, "HTTP %p incorrect Content-Length %zd (expected "
-		       "%zd)\n", http, content_len, http->remaining );
-		return -EIO_CONTENT_LENGTH;
+	intf_nullify ( &http->content );
+	intf_nullify ( &http->transfer );
+	intf_restart ( &http->content, http->response.rc );
+	intf_restart ( &http->transfer, http->response.rc );
+	http->content.desc = &http_content_desc;
+	http->transfer.desc = &http_transfer_desc;
+	intf_plug_plug ( &http->transfer, &http->content );
+	http->len = 0;
+	assert ( http->remaining == 0 );
+
+	/* Start timer to initiate retry */
+	DBGC2 ( http, "HTTP %p retrying after %d seconds\n",
+		http, http->response.retry_after );
+	start_timer_fixed ( &http->timer,
+			    ( http->response.retry_after * TICKS_PER_SEC ) );
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Requests
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Construct HTTP request headers
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length, or negative error
+ */
+static int http_format_headers ( struct http_transaction *http, char *buf,
+				 size_t len ) {
+	struct http_request_header *header;
+	size_t used;
+	size_t remaining;
+	char *line;
+	int value_len;
+	int rc;
+
+	/* Construct request line */
+	used = ssnprintf ( buf, len, "%s %s HTTP/1.1",
+			   http->request.method->name, http->request.uri );
+	if ( used < len )
+		DBGC2 ( http, "HTTP %p TX %s\n", http, buf );
+	used += ssnprintf ( ( buf + used ), ( len - used ), "\r\n" );
+
+	/* Construct all headers */
+	for_each_table_entry ( header, HTTP_REQUEST_HEADERS ) {
+
+		/* Determine header value length */
+		value_len = header->format ( http, NULL, 0 );
+		if ( value_len < 0 ) {
+			rc = value_len;
+			return rc;
+		}
+
+		/* Skip zero-length headers */
+		if ( ! value_len )
+			continue;
+
+		/* Construct header */
+		line = ( buf + used );
+		used += ssnprintf ( ( buf + used ), ( len - used ), "%s: ",
+				    header->name );
+		remaining = ( ( used < len ) ? ( len - used ) : 0 );
+		used += header->format ( http, ( buf + used ), remaining );
+		if ( used < len )
+			DBGC2 ( http, "HTTP %p TX %s\n", http, line );
+		used += ssnprintf ( ( buf + used ), ( len - used ), "\r\n" );
 	}
-	if ( ! ( http->flags & HTTP_HEAD_ONLY ) )
-		http->remaining = content_len;
 
-	/* Do nothing more if we are retrying the request */
-	if ( http->flags & HTTP_TRY_AGAIN )
+	/* Construct terminating newline */
+	used += ssnprintf ( ( buf + used ), ( len - used ), "\r\n" );
+
+	return used;
+}
+
+/**
+ * Construct HTTP "Host" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_host ( struct http_transaction *http, char *buf,
+			      size_t len ) {
+
+	/* Construct host URI */
+	return snprintf ( buf, len, "%s", http->request.host );
+}
+
+/** HTTP "Host" header "*/
+struct http_request_header http_request_host __http_request_header = {
+	.name = "Host",
+	.format = http_format_host,
+};
+
+/**
+ * Construct HTTP "User-Agent" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_user_agent ( struct http_transaction *http __unused,
+				    char *buf, size_t len ) {
+
+	/* Construct user agent */
+	return snprintf ( buf, len, "iPXE/%s", product_version );
+}
+
+/** HTTP "User-Agent" header */
+struct http_request_header http_request_user_agent __http_request_header = {
+	.name = "User-Agent",
+	.format = http_format_user_agent,
+};
+
+/**
+ * Construct HTTP "Connection" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_connection ( struct http_transaction *http __unused,
+				    char *buf, size_t len ) {
+
+	/* Always request keep-alive */
+	return snprintf ( buf, len, "keep-alive" );
+}
+
+/** HTTP "Connection" header */
+struct http_request_header http_request_connection __http_request_header = {
+	.name = "Connection",
+	.format = http_format_connection,
+};
+
+/**
+ * Construct HTTP "Range" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_range ( struct http_transaction *http,
+			       char *buf, size_t len ) {
+
+	/* Construct range, if applicable */
+	if ( http->request.range.len ) {
+		return snprintf ( buf, len, "bytes=%zd-%zd",
+				  http->request.range.start,
+				  ( http->request.range.start +
+				    http->request.range.len - 1 ) );
+	} else {
 		return 0;
+	}
+}
+
+/** HTTP "Range" header */
+struct http_request_header http_request_range __http_request_header = {
+	.name = "Range",
+	.format = http_format_range,
+};
 
-	/* Use seek() to notify recipient of filesize */
-	xfer_seek ( &http->xfer, http->remaining );
-	xfer_seek ( &http->xfer, 0 );
+/**
+ * Construct HTTP "Content-Type" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_content_type ( struct http_transaction *http,
+				      char *buf, size_t len ) {
 
-	/* Report block device capacity if applicable */
-	if ( http->flags & HTTP_HEAD_ONLY ) {
-		capacity.blocks = ( content_len / HTTP_BLKSIZE );
-		capacity.blksize = HTTP_BLKSIZE;
-		capacity.max_count = -1U;
-		block_capacity ( &http->partial, &capacity );
+	/* Construct content type, if applicable */
+	if ( http->request.content.type ) {
+		return snprintf ( buf, len, "%s", http->request.content.type );
+	} else {
+		return 0;
 	}
-	return 0;
 }
 
+/** HTTP "Content-Type" header */
+struct http_request_header http_request_content_type __http_request_header = {
+	.name = "Content-Type",
+	.format = http_format_content_type,
+};
+
 /**
- * Handle HTTP Transfer-Encoding header
+ * Construct HTTP "Content-Length" header
  *
- * @v http		HTTP request
- * @v value		HTTP header value
- * @ret rc		Return status code
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
  */
-static int http_rx_transfer_encoding ( struct http_request *http, char *value ){
+static int http_format_content_length ( struct http_transaction *http,
+					char *buf, size_t len ) {
 
-	if ( strcasecmp ( value, "chunked" ) == 0 ) {
-		/* Mark connection as using chunked transfer encoding */
-		http->chunked = 1;
+	/* Construct content length, if applicable */
+	if ( http->request.content.len ) {
+		return snprintf ( buf, len, "%zd", http->request.content.len );
+	} else {
+		return 0;
 	}
+}
 
-	return 0;
+/** HTTP "Content-Length" header */
+struct http_request_header http_request_content_length __http_request_header = {
+	.name = "Content-Length",
+	.format = http_format_content_length,
+};
+
+/**
+ * Construct HTTP "Accept-Encoding" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_accept_encoding ( struct http_transaction *http,
+					 char *buf, size_t len ) {
+	struct http_content_encoding *encoding;
+	const char *sep = "";
+	size_t used = 0;
+
+	/* Construct list of content encodings */
+	for_each_table_entry ( encoding, HTTP_CONTENT_ENCODINGS ) {
+		if ( encoding->supported && ( ! encoding->supported ( http ) ) )
+			continue;
+		used += ssnprintf ( ( buf + used ), ( len - used ),
+				    "%s%s", sep, encoding->name );
+		sep = ", ";
+	}
+
+	return used;
 }
 
+/** HTTP "Accept-Encoding" header */
+struct http_request_header http_request_accept_encoding __http_request_header ={
+	.name = "Accept-Encoding",
+	.format = http_format_accept_encoding,
+};
+
 /**
- * Handle HTTP Connection header
+ * Transmit request
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_connection ( struct http_request *http, char *value ) {
+static int http_tx_request ( struct http_transaction *http ) {
+	struct io_buffer *iobuf;
+	int len;
+	int check_len;
+	int rc;
+
+	/* Calculate request length */
+	len = http_format_headers ( http, NULL, 0 );
+	if ( len < 0 ) {
+		rc = len;
+		DBGC ( http, "HTTP %p could not construct request: %s\n",
+		       http, strerror ( rc ) );
+		goto err_len;
+	}
+
+	/* Allocate I/O buffer */
+	iobuf = alloc_iob ( len + 1 /* NUL */ + http->request.content.len );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
 
-	if ( strcasecmp ( value, "keep-alive" ) == 0 ) {
-		/* Mark connection as being kept alive by the server */
-		http->flags |= HTTP_SERVER_KEEPALIVE;
+	/* Construct request */
+	check_len = http_format_headers ( http, iob_put ( iobuf, len ),
+					  ( len + 1 /* NUL */ ) );
+	assert ( check_len == len );
+	memcpy ( iob_put ( iobuf, http->request.content.len ),
+		 http->request.content.data, http->request.content.len );
+
+	/* Deliver request */
+	if ( ( rc = xfer_deliver_iob ( &http->conn,
+				       iob_disown ( iobuf ) ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not deliver request: %s\n",
+		       http, strerror ( rc ) );
+		goto err_deliver;
 	}
 
+	/* Clear any previous response */
+	empty_line_buffer ( &http->response.headers );
+	memset ( &http->response, 0, sizeof ( http->response ) );
+
+	/* Move to response headers state */
+	http->state = &http_headers;
+
 	return 0;
+
+ err_deliver:
+	free_iob ( iobuf );
+ err_alloc:
+ err_len:
+	return rc;
 }
 
+/** HTTP request state */
+static struct http_state http_request = {
+	.tx = http_tx_request,
+	.close = http_close_error,
+};
+
+/******************************************************************************
+ *
+ * Response headers
+ *
+ ******************************************************************************
+ */
+
 /**
- * Handle WWW-Authenticate Basic header
+ * Parse HTTP status line
  *
- * @v http		HTTP request
- * @v params		Parameters
+ * @v http		HTTP transaction
+ * @v line		Status line
  * @ret rc		Return status code
  */
-static int http_rx_basic_auth ( struct http_request *http, char *params ) {
+static int http_parse_status ( struct http_transaction *http, char *line ) {
+	char *endp;
+	char *version;
+	char *vernum;
+	char *status;
+	int response_rc;
+
+	DBGC2 ( http, "HTTP %p RX %s\n", http, line );
+
+	/* Parse HTTP version */
+	version = http_token ( &line, NULL );
+	if ( ( ! version ) || ( strncmp ( version, "HTTP/", 5 ) != 0 ) ) {
+		DBGC ( http, "HTTP %p malformed version \"%s\"\n", http, line );
+		return -EINVAL_STATUS;
+	}
 
-	DBGC ( http, "HTTP %p Basic authentication required (%s)\n",
-	       http, params );
+	/* Keepalive is enabled by default for anything newer than HTTP/1.0 */
+	vernum = ( version + 5 /* "HTTP/" (presence already checked) */ );
+	if ( vernum[0] == '0' ) {
+		/* HTTP/0.x : keepalive not enabled by default */
+	} else if ( strncmp ( vernum, "1.0", 3 ) == 0 ) {
+		/* HTTP/1.0 : keepalive not enabled by default */
+	} else {
+		/* HTTP/1.1 or newer: keepalive enabled by default */
+		http->response.flags |= HTTP_RESPONSE_KEEPALIVE;
+	}
 
-	/* If we received a 401 Unauthorized response, then retry
-	 * using Basic authentication
-	 */
-	if ( ( http->code == 401 ) &&
-	     ( ! ( http->flags & HTTP_BASIC_AUTH ) ) &&
-	     ( http->uri->user != NULL ) ) {
-		http->flags |= ( HTTP_TRY_AGAIN | HTTP_BASIC_AUTH );
+	/* Parse status code */
+	status = line;
+	http->response.status = strtoul ( status, &endp, 10 );
+	if ( *endp != ' ' ) {
+		DBGC ( http, "HTTP %p malformed status code \"%s\"\n",
+		       http, status );
+		return -EINVAL_STATUS;
+	}
+
+	/* Convert HTTP status code to iPXE return status code */
+	if ( status[0] == '2' ) {
+		/* 2xx Success */
+		response_rc = 0;
+	} else if ( status[0] == '3' ) {
+		/* 3xx Redirection */
+		response_rc = -EXDEV;
+	} else if ( http->response.status == 401 ) {
+		/* 401 Unauthorized */
+		response_rc = -EACCES_401;
+	} else if ( http->response.status == 403 ) {
+		/* 403 Forbidden */
+		response_rc = -EPERM_403;
+	} else if ( http->response.status == 404 ) {
+		/* 404 Not Found */
+		response_rc = -ENOENT_404;
+	} else if ( status[0] == '4' ) {
+		/* 4xx Client Error (not already specified) */
+		response_rc = -EIO_4XX;
+	} else if ( status[0] == '5' ) {
+		/* 5xx Server Error */
+		response_rc = -EIO_5XX;
+	} else {
+		/* Unrecognised */
+		response_rc = -EIO_OTHER;
 	}
+	http->response.rc = response_rc;
 
 	return 0;
 }
 
 /**
- * Parse Digest authentication parameter
+ * Parse HTTP header
  *
- * @v params		Parameters
- * @v name		Parameter name (including trailing "=\"")
- * @ret value		Parameter value, or NULL
+ * @v http		HTTP transaction
+ * @v line		Header line
+ * @ret rc		Return status code
  */
-static char * http_digest_param ( char *params, const char *name ) {
-	char *key;
-	char *value;
-	char *terminator;
-
-	/* Locate parameter */
-	key = strstr ( params, name );
-	if ( ! key )
-		return NULL;
+static int http_parse_header ( struct http_transaction *http, char *line ) {
+	struct http_response_header *header;
+	char *name = line;
+	char *sep;
 
-	/* Extract value */
-	value = ( key + strlen ( name ) );
-	terminator = strchr ( value, '"' );
-	if ( ! terminator )
-		return NULL;
-	return strndup ( value, ( terminator - value ) );
+	DBGC2 ( http, "HTTP %p RX %s\n", http, line );
+
+	/* Extract header name */
+	sep = strstr ( line, ": " );
+	if ( ! sep ) {
+		DBGC ( http, "HTTP %p malformed header \"%s\"\n", http, line );
+		return -EINVAL_HEADER;
+	}
+	*sep = '\0';
+	line = ( sep + 2 /* ": " */ );
+
+	/* Process header, if recognised */
+	for_each_table_entry ( header, HTTP_RESPONSE_HEADERS ) {
+		if ( strcasecmp ( name, header->name ) == 0 )
+			return header->parse ( http, line );
+	}
+
+	/* Unrecognised headers should be ignored */
+	return 0;
 }
 
 /**
- * Handle WWW-Authenticate Digest header
+ * Parse HTTP response headers
  *
- * @v http		HTTP request
- * @v params		Parameters
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_digest_auth ( struct http_request *http, char *params ) {
+static int http_parse_headers ( struct http_transaction *http ) {
+	char *line;
+	char *next;
+	int rc;
 
-	DBGC ( http, "HTTP %p Digest authentication required (%s)\n",
-	       http, params );
+	/* Get status line */
+	line = http->response.headers.data;
+	assert ( line != NULL );
+	next = ( line + strlen ( line ) + 1 /* NUL */ );
 
-	/* If we received a 401 Unauthorized response, then retry
-	 * using Digest authentication
-	 */
-	if ( ( http->code == 401 ) &&
-	     ( ! ( http->flags & HTTP_DIGEST_AUTH ) ) &&
-	     ( http->uri->user != NULL ) ) {
-
-		/* Extract realm */
-		free ( http->auth_realm );
-		http->auth_realm = http_digest_param ( params, "realm=\"" );
-		if ( ! http->auth_realm ) {
-			DBGC ( http, "HTTP %p Digest prompt missing realm\n",
-			       http );
-			return -EINVAL_HEADER;
-		}
+	/* Parse status line */
+	if ( ( rc = http_parse_status ( http, line ) ) != 0 )
+		return rc;
 
-		/* Extract nonce */
-		free ( http->auth_nonce );
-		http->auth_nonce = http_digest_param ( params, "nonce=\"" );
-		if ( ! http->auth_nonce ) {
-			DBGC ( http, "HTTP %p Digest prompt missing nonce\n",
-			       http );
-			return -EINVAL_HEADER;
-		}
+	/* Process header lines */
+	while ( 1 ) {
 
-		/* Extract opaque */
-		free ( http->auth_opaque );
-		http->auth_opaque = http_digest_param ( params, "opaque=\"" );
-		if ( ! http->auth_opaque ) {
-			/* Not an error; "opaque" is optional */
-		}
+		/* Move to next line */
+		line = next;
+		next = ( line + strlen ( line ) + 1 /* NUL */ );
+
+		/* Stop on terminating blank line */
+		if ( ! line[0] )
+			return 0;
 
-		http->flags |= ( HTTP_TRY_AGAIN | HTTP_DIGEST_AUTH );
+		/* Process header line */
+		if ( ( rc = http_parse_header ( http, line ) ) != 0 )
+			return rc;
 	}
+}
 
+/**
+ * Parse HTTP "Location" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_location ( struct http_transaction *http, char *line ) {
+
+	/* Store location */
+	http->response.location = line;
 	return 0;
 }
 
-/** An HTTP WWW-Authenticate header handler */
-struct http_auth_header_handler {
-	/** Scheme (e.g. "Basic") */
-	const char *scheme;
-	/** Handle received parameters
-	 *
-	 * @v http	HTTP request
-	 * @v params	Parameters
-	 * @ret rc	Return status code
-	 */
-	int ( * rx ) ( struct http_request *http, char *params );
+/** HTTP "Location" header */
+struct http_response_header http_response_location __http_response_header = {
+	.name = "Location",
+	.parse = http_parse_location,
 };
 
-/** List of HTTP WWW-Authenticate header handlers */
-static struct http_auth_header_handler http_auth_header_handlers[] = {
-	{
-		.scheme = "Basic",
-		.rx = http_rx_basic_auth,
-	},
-	{
-		.scheme = "Digest",
-		.rx = http_rx_digest_auth,
-	},
-	{ NULL, NULL },
+/**
+ * Parse HTTP "Transfer-Encoding" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_transfer_encoding ( struct http_transaction *http,
+					  char *line ) {
+	struct http_transfer_encoding *encoding;
+
+	/* Check for known transfer encodings */
+	for_each_table_entry ( encoding, HTTP_TRANSFER_ENCODINGS ) {
+		if ( strcasecmp ( line, encoding->name ) == 0 ) {
+			http->response.transfer.encoding = encoding;
+			return 0;
+		}
+	}
+
+	DBGC ( http, "HTTP %p unrecognised Transfer-Encoding \"%s\"\n",
+	       http, line );
+	return -ENOTSUP_TRANSFER;
+}
+
+/** HTTP "Transfer-Encoding" header */
+struct http_response_header
+http_response_transfer_encoding __http_response_header = {
+	.name = "Transfer-Encoding",
+	.parse = http_parse_transfer_encoding,
 };
 
 /**
- * Handle HTTP WWW-Authenticate header
+ * Parse HTTP "Connection" header
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
  * @ret rc		Return status code
  */
-static int http_rx_www_authenticate ( struct http_request *http, char *value ) {
-	struct http_auth_header_handler *handler;
-	char *separator;
-	char *scheme;
-	char *params;
-	int rc;
+static int http_parse_connection ( struct http_transaction *http, char *line ) {
 
-	/* Extract scheme */
-	separator = strchr ( value, ' ' );
-	if ( ! separator ) {
-		DBGC ( http, "HTTP %p malformed WWW-Authenticate header\n",
-		       http );
-		return -EINVAL_HEADER;
+	/* Check for known connection intentions */
+	if ( strcasecmp ( line, "keep-alive" ) == 0 ) {
+		http->response.flags |= HTTP_RESPONSE_KEEPALIVE;
+		return 0;
 	}
-	*separator = '\0';
-	scheme = value;
-	params = ( separator + 1 );
-
-	/* Hand off to header handler, if one exists */
-	for ( handler = http_auth_header_handlers; handler->scheme; handler++ ){
-		if ( strcasecmp ( scheme, handler->scheme ) == 0 ) {
-			if ( ( rc = handler->rx ( http, params ) ) != 0 )
-				return rc;
-			break;
-		}
+	if ( strcasecmp ( line, "close" ) == 0 ) {
+		http->response.flags &= ~HTTP_RESPONSE_KEEPALIVE;
+		return 0;
 	}
-	return 0;
+
+	DBGC ( http, "HTTP %p unrecognised Connection \"%s\"\n", http, line );
+	return -ENOTSUP_CONNECTION;
 }
 
+/** HTTP "Connection" header */
+struct http_response_header http_response_connection __http_response_header = {
+	.name = "Connection",
+	.parse = http_parse_connection,
+};
+
 /**
- * Handle HTTP Retry-After header
+ * Parse HTTP "Content-Length" header
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
  * @ret rc		Return status code
  */
-static int http_rx_retry_after ( struct http_request *http, char *value ) {
-	unsigned long seconds;
+static int http_parse_content_length ( struct http_transaction *http,
+				       char *line ) {
 	char *endp;
 
-	DBGC ( http, "HTTP %p retry requested (%s)\n", http, value );
+	/* Parse length */
+	http->response.content.len = strtoul ( line, &endp, 10 );
+	if ( *endp != '\0' ) {
+		DBGC ( http, "HTTP %p invalid Content-Length \"%s\"\n",
+		       http, line );
+		return -EINVAL_CONTENT_LENGTH;
+	}
 
-	/* If we received a 503 Service Unavailable response, then
-	 * retry after the specified number of seconds.  If the value
-	 * is not a simple number of seconds (e.g. a full HTTP date),
-	 * then retry after a fixed delay, since we don't have code
-	 * able to parse full HTTP dates.
-	 */
-	if ( http->code == 503 ) {
-		seconds = strtoul ( value, &endp, 10 );
-		if ( *endp != '\0' ) {
-			seconds = HTTP_RETRY_SECONDS;
-			DBGC ( http, "HTTP %p cannot understand \"%s\"; "
-			       "using %ld seconds\n", http, value, seconds );
+	/* Record that we have a content length (since it may be zero) */
+	http->response.flags |= HTTP_RESPONSE_CONTENT_LEN;
+
+	return 0;
+}
+
+/** HTTP "Content-Length" header */
+struct http_response_header
+http_response_content_length __http_response_header = {
+	.name = "Content-Length",
+	.parse = http_parse_content_length,
+};
+
+/**
+ * Parse HTTP "Content-Encoding" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_content_encoding ( struct http_transaction *http,
+					 char *line ) {
+	struct http_content_encoding *encoding;
+
+	/* Check for known content encodings */
+	for_each_table_entry ( encoding, HTTP_CONTENT_ENCODINGS ) {
+		if ( encoding->supported && ( ! encoding->supported ( http ) ) )
+			continue;
+		if ( strcasecmp ( line, encoding->name ) == 0 ) {
+			http->response.content.encoding = encoding;
+			return 0;
 		}
-		http->flags |= HTTP_TRY_AGAIN;
-		http->retry_delay = ( seconds * TICKS_PER_SEC );
 	}
 
+	/* Some servers (e.g. Apache) have a habit of specifying
+	 * unwarranted content encodings.  For example, if Apache
+	 * detects (via /etc/httpd/conf/magic) that a file's contents
+	 * are gzip-compressed, it will set "Content-Encoding: x-gzip"
+	 * regardless of the client's Accept-Encoding header.  The
+	 * only viable way to handle such servers is to treat unknown
+	 * content encodings as equivalent to "identity".
+	 */
+	DBGC ( http, "HTTP %p unrecognised Content-Encoding \"%s\"\n",
+	       http, line );
 	return 0;
 }
 
-/** An HTTP header handler */
-struct http_header_handler {
-	/** Name (e.g. "Content-Length") */
-	const char *header;
-	/** Handle received header
-	 *
-	 * @v http	HTTP request
-	 * @v value	HTTP header value
-	 * @ret rc	Return status code
-	 *
-	 * If an error is returned, the download will be aborted.
-	 */
-	int ( * rx ) ( struct http_request *http, char *value );
+/** HTTP "Content-Encoding" header */
+struct http_response_header
+http_response_content_encoding __http_response_header = {
+	.name = "Content-Encoding",
+	.parse = http_parse_content_encoding,
 };
 
-/** List of HTTP header handlers */
-static struct http_header_handler http_header_handlers[] = {
-	{
-		.header = "Location",
-		.rx = http_rx_location,
-	},
-	{
-		.header = "Content-Length",
-		.rx = http_rx_content_length,
-	},
-	{
-		.header = "Transfer-Encoding",
-		.rx = http_rx_transfer_encoding,
-	},
-	{
-		.header = "Connection",
-		.rx = http_rx_connection,
-	},
-	{
-		.header = "WWW-Authenticate",
-		.rx = http_rx_www_authenticate,
-	},
-	{
-		.header = "Retry-After",
-		.rx = http_rx_retry_after,
-	},
-	{ NULL, NULL }
+/**
+ * Parse HTTP "Retry-After" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_retry_after ( struct http_transaction *http,
+				    char *line ) {
+	char *endp;
+
+	/* Try to parse value as a simple number of seconds */
+	http->response.retry_after = strtoul ( line, &endp, 10 );
+	if ( *endp != '\0' ) {
+		/* For any value which is not a simple number of
+		 * seconds (e.g. a full HTTP date), just retry after a
+		 * fixed delay, since we don't have code able to parse
+		 * full HTTP dates.
+		 */
+		http->response.retry_after = HTTP_RETRY_SECONDS;
+		DBGC ( http, "HTTP %p cannot understand Retry-After \"%s\"; "
+		       "using %d seconds\n", http, line, HTTP_RETRY_SECONDS );
+	}
+
+	/* Allow HTTP request to be retried after specified delay */
+	http->response.flags |= HTTP_RESPONSE_RETRY;
+
+	return 0;
+}
+
+/** HTTP "Retry-After" header */
+struct http_response_header http_response_retry_after __http_response_header = {
+	.name = "Retry-After",
+	.parse = http_parse_retry_after,
 };
 
 /**
- * Handle HTTP header
+ * Handle received HTTP headers
  *
- * @v http		HTTP request
- * @v header		HTTP header
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
  * @ret rc		Return status code
  */
-static int http_rx_header ( struct http_request *http, char *header ) {
-	struct http_header_handler *handler;
-	char *separator;
-	char *value;
+static int http_rx_headers ( struct http_transaction *http,
+			     struct io_buffer **iobuf ) {
+	struct http_transfer_encoding *transfer;
+	struct http_content_encoding *content;
+	char *line;
 	int rc;
 
-	/* An empty header line marks the end of this phase */
-	if ( ! header[0] ) {
-		empty_line_buffer ( &http->linebuf );
+	/* Buffer header line */
+	if ( ( rc = http_rx_linebuf ( http, *iobuf,
+				      &http->response.headers ) ) != 0 )
+		return rc;
 
-		/* Handle response code */
-		if ( ! ( http->flags & HTTP_TRY_AGAIN ) ) {
-			if ( ( rc = http_response_to_rc ( http->code ) ) != 0 )
-				return rc;
-		}
+	/* Wait until we see the empty line marking end of headers */
+	line = buffered_line ( &http->response.headers );
+	if ( ( line == NULL ) || ( line[0] != '\0' ) )
+		return 0;
 
-		/* Move to next state */
-		if ( http->rx_state == HTTP_RX_HEADER ) {
-			DBGC ( http, "HTTP %p start of data\n", http );
-			http->rx_state = ( http->chunked ?
-					   HTTP_RX_CHUNK_LEN : HTTP_RX_DATA );
-			if ( ( http->partial_len != 0 ) &&
-			     ( ! ( http->flags & HTTP_TRY_AGAIN ) ) ) {
-				http->remaining = http->partial_len;
-			}
-			return 0;
-		} else {
-			DBGC ( http, "HTTP %p end of trailer\n", http );
-			http_done ( http );
-			return 0;
-		}
+	/* Process headers */
+	if ( ( rc = http_parse_headers ( http ) ) != 0 )
+		return rc;
+
+	/* Initialise content encoding, if applicable */
+	if ( ( content = http->response.content.encoding ) &&
+	     ( ( rc = content->init ( http ) ) != 0 ) ) {
+		DBGC ( http, "HTTP %p could not initialise %s content "
+		       "encoding: %s\n", http, content->name, strerror ( rc ) );
+		return rc;
 	}
 
-	DBGC ( http, "HTTP %p header \"%s\"\n", http, header );
+	/* Presize receive buffer, if we have a content length */
+	if ( http->response.content.len ) {
+		xfer_seek ( &http->transfer, http->response.content.len );
+		xfer_seek ( &http->transfer, 0 );
+	}
 
-	/* Split header at the ": " */
-	separator = strstr ( header, ": " );
-	if ( ! separator ) {
-		DBGC ( http, "HTTP %p malformed header\n", http );
-		return -EINVAL_HEADER;
+	/* Complete transfer if this is a HEAD request */
+	if ( http->request.method == &http_head ) {
+		if ( ( rc = http_transfer_complete ( http ) ) != 0 )
+			return rc;
+		return 0;
 	}
-	*separator = '\0';
-	value = ( separator + 2 );
-
-	/* Hand off to header handler, if one exists */
-	for ( handler = http_header_handlers ; handler->header ; handler++ ) {
-		if ( strcasecmp ( header, handler->header ) == 0 ) {
-			if ( ( rc = handler->rx ( http, value ) ) != 0 )
-				return rc;
-			break;
-		}
+
+	/* Default to identity transfer encoding, if none specified */
+	if ( ! http->response.transfer.encoding )
+		http->response.transfer.encoding = &http_transfer_identity;
+
+	/* Move to transfer encoding-specific data state */
+	transfer = http->response.transfer.encoding;
+	http->state = &transfer->state;
+
+	/* Initialise transfer encoding */
+	if ( ( rc = transfer->init ( http ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not initialise %s transfer "
+		       "encoding: %s\n", http, transfer->name, strerror ( rc ));
+		return rc;
 	}
+
 	return 0;
 }
 
+/** HTTP response headers state */
+static struct http_state http_headers = {
+	.rx = http_rx_headers,
+	.close = http_close_error,
+};
+
+/******************************************************************************
+ *
+ * Identity transfer encoding
+ *
+ ******************************************************************************
+ */
+
 /**
- * Handle HTTP chunk length
+ * Initialise transfer encoding
  *
- * @v http		HTTP request
- * @v length		HTTP chunk length
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_chunk_len ( struct http_request *http, char *length ) {
-	char *endp;
+static int http_init_transfer_identity ( struct http_transaction *http ) {
+	int rc;
 
-	/* Skip blank lines between chunks */
-	if ( length[0] == '\0' )
-		return 0;
+	/* Complete transfer immediately if we have a zero content length */
+	if ( ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) &&
+	     ( http->response.content.len == 0 ) &&
+	     ( ( rc = http_transfer_complete ( http ) ) != 0 ) )
+		return rc;
 
-	/* Parse chunk length */
-	http->chunk_remaining = strtoul ( length, &endp, 16 );
-	if ( *endp != '\0' ) {
-		DBGC ( http, "HTTP %p invalid chunk length \"%s\"\n",
-		       http, length );
-		return -EINVAL_CHUNK_LENGTH;
-	}
+	return 0;
+}
 
-	/* Terminate chunked encoding if applicable */
-	if ( http->chunk_remaining == 0 ) {
-		DBGC ( http, "HTTP %p end of chunks\n", http );
-		http->chunked = 0;
-		http->rx_state = HTTP_RX_TRAILER;
-		return 0;
-	}
+/**
+ * Handle received data
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
+ */
+static int http_rx_transfer_identity ( struct http_transaction *http,
+				       struct io_buffer **iobuf ) {
+	size_t len = iob_len ( *iobuf );
+	int rc;
+
+	/* Update lengths */
+	http->len += len;
 
-	/* Use seek() to notify recipient of new filesize */
-	DBGC ( http, "HTTP %p start of chunk of length %zd\n",
-	       http, http->chunk_remaining );
-	if ( ! ( http->flags & HTTP_TRY_AGAIN ) ) {
-		xfer_seek ( &http->xfer,
-			    ( http->rx_len + http->chunk_remaining ) );
-		xfer_seek ( &http->xfer, http->rx_len );
+	/* Fail if this transfer would overrun the expected content
+	 * length (if any).
+	 */
+	if ( ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) &&
+	     ( http->len > http->response.content.len ) ) {
+		DBGC ( http, "HTTP %p content length overrun\n", http );
+		return -EIO_CONTENT_LENGTH;
 	}
 
-	/* Start receiving data */
-	http->rx_state = HTTP_RX_DATA;
+	/* Hand off to content encoding */
+	if ( ( rc = xfer_deliver_iob ( &http->transfer,
+				       iob_disown ( *iobuf ) ) ) != 0 )
+		return rc;
+
+	/* Complete transfer if we have received the expected content
+	 * length (if any).
+	 */
+	if ( ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) &&
+	     ( http->len == http->response.content.len ) &&
+	     ( ( rc = http_transfer_complete ( http ) ) != 0 ) )
+		return rc;
 
 	return 0;
 }
 
-/** An HTTP line-based data handler */
-struct http_line_handler {
-	/** Handle line
-	 *
-	 * @v http	HTTP request
-	 * @v line	Line to handle
-	 * @ret rc	Return status code
+/**
+ * Handle server connection close
+ *
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
+ */
+static void http_close_transfer_identity ( struct http_transaction *http,
+					   int rc ) {
+
+	/* Fail if any error occurred */
+	if ( rc != 0 )
+		goto err;
+
+	/* Fail if we have a content length (since we would have
+	 * already closed the connection if we had received the
+	 * correct content length).
 	 */
-	int ( * rx ) ( struct http_request *http, char *line );
-};
+	if ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) {
+		DBGC ( http, "HTTP %p content length underrun\n", http );
+		rc = EIO_CONTENT_LENGTH;
+		goto err;
+	}
+
+	/* Indicate that transfer is complete */
+	if ( ( rc = http_transfer_complete ( http ) ) != 0 )
+		goto err;
 
-/** List of HTTP line-based data handlers */
-static struct http_line_handler http_line_handlers[] = {
-	[HTTP_RX_RESPONSE]	= { .rx = http_rx_response },
-	[HTTP_RX_HEADER]	= { .rx = http_rx_header },
-	[HTTP_RX_CHUNK_LEN]	= { .rx = http_rx_chunk_len },
-	[HTTP_RX_TRAILER]	= { .rx = http_rx_header },
+	return;
+
+ err:
+	http_close ( http, rc );
+}
+
+/** Identity transfer encoding */
+static struct http_transfer_encoding http_transfer_identity = {
+	.name = "identity",
+	.init = http_init_transfer_identity,
+	.state = {
+		.rx = http_rx_transfer_identity,
+		.close = http_close_transfer_identity,
+	},
 };
 
+/******************************************************************************
+ *
+ * Chunked transfer encoding
+ *
+ ******************************************************************************
+ */
+
 /**
- * Handle new data arriving via HTTP connection
+ * Initialise transfer encoding
  *
- * @v http		HTTP request
- * @v iobuf		I/O buffer
- * @v meta		Data transfer metadata
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_socket_deliver ( struct http_request *http,
-				 struct io_buffer *iobuf,
-				 struct xfer_metadata *meta __unused ) {
-	struct http_line_handler *lh;
+static int http_init_transfer_chunked ( struct http_transaction *http ) {
+
+	/* Sanity checks */
+	assert ( http->remaining == 0 );
+	assert ( http->linebuf.len == 0 );
+
+	return 0;
+}
+
+/**
+ * Handle received chunk length
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
+ */
+static int http_rx_chunk_len ( struct http_transaction *http,
+			       struct io_buffer **iobuf ) {
 	char *line;
-	size_t data_len;
-	ssize_t line_len;
-	int rc = 0;
+	char *endp;
+	size_t len;
+	int rc;
 
-	profile_start ( &http_rx_profiler );
-	while ( iobuf && iob_len ( iobuf ) ) {
+	/* Receive into temporary line buffer */
+	if ( ( rc = http_rx_linebuf ( http, *iobuf, &http->linebuf ) ) != 0 )
+		return rc;
 
-		switch ( http->rx_state ) {
-		case HTTP_RX_IDLE:
-			/* Receiving any data in this state is an error */
-			DBGC ( http, "HTTP %p received %zd bytes while %s\n",
-			       http, iob_len ( iobuf ),
-			       ( ( http->rx_state == HTTP_RX_IDLE ) ?
-				 "idle" : "dead" ) );
-			rc = -EPROTO_UNSOLICITED;
-			goto done;
-		case HTTP_RX_DEAD:
-			/* Do no further processing */
-			goto done;
-		case HTTP_RX_DATA:
-			/* Pass received data to caller */
-			data_len = iob_len ( iobuf );
-			if ( http->chunk_remaining &&
-			     ( http->chunk_remaining < data_len ) ) {
-				data_len = http->chunk_remaining;
-			}
-			if ( http->remaining &&
-			     ( http->remaining < data_len ) ) {
-				data_len = http->remaining;
-			}
-			if ( http->flags & HTTP_TRY_AGAIN ) {
-				/* Discard all received data */
-				iob_pull ( iobuf, data_len );
-			} else if ( http->rx_buffer != UNULL ) {
-				/* Copy to partial transfer buffer */
-				copy_to_user ( http->rx_buffer, http->rx_len,
-					       iobuf->data, data_len );
-				iob_pull ( iobuf, data_len );
-			} else if ( data_len < iob_len ( iobuf ) ) {
-				/* Deliver partial buffer as raw data */
-				profile_start ( &http_xfer_profiler );
-				rc = xfer_deliver_raw ( &http->xfer,
-							iobuf->data, data_len );
-				iob_pull ( iobuf, data_len );
-				if ( rc != 0 )
-					goto done;
-				profile_stop ( &http_xfer_profiler );
-			} else {
-				/* Deliver whole I/O buffer */
-				profile_start ( &http_xfer_profiler );
-				if ( ( rc = xfer_deliver_iob ( &http->xfer,
-						 iob_disown ( iobuf ) ) ) != 0 )
-					goto done;
-				profile_stop ( &http_xfer_profiler );
-			}
-			http->rx_len += data_len;
-			if ( http->chunk_remaining ) {
-				http->chunk_remaining -= data_len;
-				if ( http->chunk_remaining == 0 )
-					http->rx_state = HTTP_RX_CHUNK_LEN;
-			}
-			if ( http->remaining ) {
-				http->remaining -= data_len;
-				if ( ( http->remaining == 0 ) &&
-				     ( http->rx_state == HTTP_RX_DATA ) ) {
-					http_done ( http );
-				}
-			}
-			break;
-		case HTTP_RX_RESPONSE:
-		case HTTP_RX_HEADER:
-		case HTTP_RX_CHUNK_LEN:
-		case HTTP_RX_TRAILER:
-			/* In the other phases, buffer and process a
-			 * line at a time
-			 */
-			line_len = line_buffer ( &http->linebuf, iobuf->data,
-						 iob_len ( iobuf ) );
-			if ( line_len < 0 ) {
-				rc = line_len;
-				DBGC ( http, "HTTP %p could not buffer line: "
-				       "%s\n", http, strerror ( rc ) );
-				goto done;
-			}
-			iob_pull ( iobuf, line_len );
-			line = buffered_line ( &http->linebuf );
-			if ( line ) {
-				lh = &http_line_handlers[http->rx_state];
-				if ( ( rc = lh->rx ( http, line ) ) != 0 )
-					goto done;
-			}
-			break;
-		default:
-			assert ( 0 );
-			break;
-		}
+	/* Wait until we receive a non-empty line */
+	line = buffered_line ( &http->linebuf );
+	if ( ( line == NULL ) || ( line[0] == '\0' ) )
+		return 0;
+
+	/* Parse chunk length */
+	http->remaining = strtoul ( line, &endp, 16 );
+	if ( *endp != '\0' ) {
+		DBGC ( http, "HTTP %p invalid chunk length \"%s\"\n",
+		       http, line );
+		return -EINVAL_CHUNK_LENGTH;
 	}
 
- done:
-	if ( rc )
-		http_close ( http, rc );
-	free_iob ( iobuf );
-	profile_stop ( &http_rx_profiler );
-	return rc;
+	/* Empty line buffer */
+	empty_line_buffer ( &http->linebuf );
+
+	/* Update expected length */
+	len = ( http->len + http->remaining );
+	xfer_seek ( &http->transfer, len );
+	xfer_seek ( &http->transfer, http->len );
+
+	/* If chunk length is zero, then move to response trailers state */
+	if ( ! http->remaining )
+		http->state = &http_trailers;
+
+	return 0;
 }
 
 /**
- * Check HTTP socket flow control window
+ * Handle received chunk data
  *
- * @v http		HTTP request
- * @ret len		Length of window
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
  */
-static size_t http_socket_window ( struct http_request *http __unused ) {
+static int http_rx_chunk_data ( struct http_transaction *http,
+				struct io_buffer **iobuf ) {
+	struct io_buffer *payload;
+	uint8_t *crlf;
+	size_t len;
+	int rc;
 
-	/* Window is always open.  This is to prevent TCP from
-	 * stalling if our parent window is not currently open.
+	/* In the common case of a final chunk in a packet which also
+	 * includes the terminating CRLF, strip the terminating CRLF
+	 * (which we would ignore anyway) and hence avoid
+	 * unnecessarily copying the data.
 	 */
-	return ( ~( ( size_t ) 0 ) );
+	if ( iob_len ( *iobuf ) == ( http->remaining + 2 /* CRLF */ ) ) {
+		crlf = ( (*iobuf)->data + http->remaining );
+		if ( ( crlf[0] == '\r' ) && ( crlf[1] == '\n' ) )
+			iob_unput ( (*iobuf), 2 /* CRLF */ );
+	}
+	len = iob_len ( *iobuf );
+
+	/* Use whole/partial buffer as applicable */
+	if ( len <= http->remaining ) {
+
+		/* Whole buffer is to be consumed: decrease remaining
+		 * length and use original I/O buffer as payload.
+		 */
+		payload = iob_disown ( *iobuf );
+		http->len += len;
+		http->remaining -= len;
+
+	} else {
+
+		/* Partial buffer is to be consumed: copy data to a
+		 * temporary I/O buffer.
+		 */
+		payload = alloc_iob ( http->remaining );
+		if ( ! payload ) {
+			rc = -ENOMEM;
+			goto err;
+		}
+		memcpy ( iob_put ( payload, http->remaining ), (*iobuf)->data,
+			 http->remaining );
+		iob_pull ( *iobuf, http->remaining );
+		http->len += http->remaining;
+		http->remaining = 0;
+	}
+
+	/* Hand off to content encoding */
+	if ( ( rc = xfer_deliver_iob ( &http->transfer,
+				       iob_disown ( payload ) ) ) != 0 )
+		goto err;
+
+	return 0;
+
+ err:
+	assert ( payload == NULL );
+	return rc;
 }
 
 /**
- * Close HTTP socket
+ * Handle received chunked data
  *
- * @v http		HTTP request
- * @v rc		Reason for close
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
  */
-static void http_socket_close ( struct http_request *http, int rc ) {
+static int http_rx_transfer_chunked ( struct http_transaction *http,
+				      struct io_buffer **iobuf ) {
 
-	/* If we have an error, terminate */
-	if ( rc != 0 ) {
-		http_close ( http, rc );
-		return;
+	/* Handle as chunk length or chunk data as appropriate */
+	if ( http->remaining ) {
+		return http_rx_chunk_data ( http, iobuf );
+	} else {
+		return http_rx_chunk_len ( http, iobuf );
 	}
-
-	/* Mark HTTP request as complete */
-	http_done ( http );
 }
 
-/**
- * Generate HTTP Basic authorisation string
+/** Chunked transfer encoding */
+struct http_transfer_encoding http_transfer_chunked __http_transfer_encoding = {
+	.name = "chunked",
+	.init = http_init_transfer_chunked,
+	.state = {
+		.rx = http_rx_transfer_chunked,
+		.close = http_close_error,
+	},
+};
+
+/******************************************************************************
  *
- * @v http		HTTP request
- * @ret auth		Authorisation string, or NULL on error
+ * Response trailers
  *
- * The authorisation string is dynamically allocated, and must be
- * freed by the caller.
+ ******************************************************************************
  */
-static char * http_basic_auth ( struct http_request *http ) {
-	const char *user = http->uri->user;
-	const char *password = ( http->uri->password ?
-				 http->uri->password : "" );
-	size_t user_pw_len =
-		( strlen ( user ) + 1 /* ":" */ + strlen ( password ) );
-	char user_pw[ user_pw_len + 1 /* NUL */ ];
-	size_t user_pw_base64_len = base64_encoded_len ( user_pw_len );
-	char user_pw_base64[ user_pw_base64_len + 1 /* NUL */ ];
-	char *auth;
-	int len;
 
-	/* Sanity check */
-	assert ( user != NULL );
+/**
+ * Handle received HTTP trailer
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
+ */
+static int http_rx_trailers ( struct http_transaction *http,
+			      struct io_buffer **iobuf ) {
+	char *line;
+	int rc;
 
-	/* Make "user:password" string from decoded fields */
-	snprintf ( user_pw, sizeof ( user_pw ), "%s:%s", user, password );
+	/* Buffer trailer line */
+	if ( ( rc = http_rx_linebuf ( http, *iobuf, &http->linebuf ) ) != 0 )
+		return rc;
 
-	/* Base64-encode the "user:password" string */
-	base64_encode ( ( void * ) user_pw, user_pw_len, user_pw_base64 );
+	/* Wait until we see the empty line marking end of trailers */
+	line = buffered_line ( &http->linebuf );
+	if ( ( line == NULL ) || ( line[0] != '\0' ) )
+		return 0;
 
-	/* Generate the authorisation string */
-	len = asprintf ( &auth, "Authorization: Basic %s\r\n",
-			 user_pw_base64 );
-	if ( len < 0 )
-		return NULL;
+	/* Empty line buffer */
+	empty_line_buffer ( &http->linebuf );
 
-	return auth;
+	/* Transfer is complete */
+	if ( ( rc = http_transfer_complete ( http ) ) != 0 )
+		return rc;
+
+	return 0;
 }
 
-/**
- * Generate HTTP Digest authorisation string
+/** HTTP response trailers state */
+static struct http_state http_trailers = {
+	.rx = http_rx_trailers,
+	.close = http_close_error,
+};
+
+/******************************************************************************
  *
- * @v http		HTTP request
- * @v method		HTTP method (e.g. "GET")
- * @v uri		HTTP request URI (e.g. "/index.html")
- * @ret auth		Authorisation string, or NULL on error
+ * Simple URI openers
  *
- * The authorisation string is dynamically allocated, and must be
- * freed by the caller.
+ ******************************************************************************
  */
-static char * http_digest_auth ( struct http_request *http,
-				 const char *method, const char *uri ) {
-	const char *user = http->uri->user;
-	const char *password = ( http->uri->password ?
-				 http->uri->password : "" );
-	const char *realm = http->auth_realm;
-	const char *nonce = http->auth_nonce;
-	const char *opaque = http->auth_opaque;
-	static const char colon = ':';
-	uint8_t ctx[MD5_CTX_SIZE];
-	uint8_t digest[MD5_DIGEST_SIZE];
-	char ha1[ base16_encoded_len ( sizeof ( digest ) ) + 1 /* NUL */ ];
-	char ha2[ base16_encoded_len ( sizeof ( digest ) ) + 1 /* NUL */ ];
-	char response[ base16_encoded_len ( sizeof ( digest ) ) + 1 /* NUL */ ];
-	char *auth;
-	int len;
-
-	/* Sanity checks */
-	assert ( user != NULL );
-	assert ( realm != NULL );
-	assert ( nonce != NULL );
-
-	/* Generate HA1 */
-	digest_init ( &md5_algorithm, ctx );
-	digest_update ( &md5_algorithm, ctx, user, strlen ( user ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, realm, strlen ( realm ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, password, strlen ( password ) );
-	digest_final ( &md5_algorithm, ctx, digest );
-	base16_encode ( digest, sizeof ( digest ), ha1 );
-
-	/* Generate HA2 */
-	digest_init ( &md5_algorithm, ctx );
-	digest_update ( &md5_algorithm, ctx, method, strlen ( method ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, uri, strlen ( uri ) );
-	digest_final ( &md5_algorithm, ctx, digest );
-	base16_encode ( digest, sizeof ( digest ), ha2 );
-
-	/* Generate response */
-	digest_init ( &md5_algorithm, ctx );
-	digest_update ( &md5_algorithm, ctx, ha1, strlen ( ha1 ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, nonce, strlen ( nonce ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, ha2, strlen ( ha2 ) );
-	digest_final ( &md5_algorithm, ctx, digest );
-	base16_encode ( digest, sizeof ( digest ), response );
-
-	/* Generate the authorisation string */
-	len = asprintf ( &auth, "Authorization: Digest username=\"%s\", "
-			 "realm=\"%s\", nonce=\"%s\", uri=\"%s\", "
-			 "%s%s%sresponse=\"%s\"\r\n", user, realm, nonce, uri,
-			 ( opaque ? "opaque=\"" : "" ),
-			 ( opaque ? opaque : "" ),
-			 ( opaque ? "\", " : "" ), response );
-	if ( len < 0 )
-		return NULL;
-
-	return auth;
-}
 
 /**
- * Generate HTTP POST parameter list
+ * Construct HTTP parameter list
  *
- * @v http		HTTP request
+ * @v params		Parameter list
  * @v buf		Buffer to contain HTTP POST parameters
  * @v len		Length of buffer
  * @ret len		Length of parameter list (excluding terminating NUL)
  */
-static size_t http_post_params ( struct http_request *http,
-				 char *buf, size_t len ) {
+static size_t http_params ( struct parameters *params, char *buf, size_t len ) {
 	struct parameter *param;
 	ssize_t remaining = len;
 	size_t frag_len;
 
 	/* Add each parameter in the form "key=value", joined with "&" */
 	len = 0;
-	for_each_param ( param, http->uri->params ) {
+	for_each_param ( param, params ) {
 
 		/* Add the "&", if applicable */
 		if ( len ) {
@@ -1201,374 +1853,78 @@ static size_t http_post_params ( struct http_request *http,
 }
 
 /**
- * Generate HTTP POST body
+ * Open HTTP transaction for simple GET URI
  *
- * @v http		HTTP request
- * @ret post		I/O buffer containing POST body, or NULL on error
+ * @v xfer		Data transfer interface
+ * @v uri		Request URI
+ * @ret rc		Return status code
  */
-static struct io_buffer * http_post ( struct http_request *http ) {
-	struct io_buffer *post;
-	size_t len;
-	size_t check_len;
+static int http_open_get_uri ( struct interface *xfer, struct uri *uri ) {
 
-	/* Calculate length of parameter list */
-	len = http_post_params ( http, NULL, 0 );
-
-	/* Allocate parameter list */
-	post = alloc_iob ( len + 1 /* NUL */ );
-	if ( ! post )
-		return NULL;
-
-	/* Fill parameter list */
-	check_len = http_post_params ( http, iob_put ( post, len ),
-				       ( len + 1 /* NUL */ ) );
-	assert ( len == check_len );
-	DBGC ( http, "HTTP %p POST %s\n", http, ( ( char * ) post->data ) );
-
-	return post;
+	return http_open ( xfer, &http_get, uri, NULL, NULL );
 }
 
 /**
- * HTTP process
+ * Open HTTP transaction for simple POST URI
  *
- * @v http		HTTP request
+ * @v xfer		Data transfer interface
+ * @v uri		Request URI
+ * @ret rc		Return status code
  */
-static void http_step ( struct http_request *http ) {
-	struct io_buffer *post;
-	struct uri host_uri;
-	struct uri path_uri;
-	char *host_uri_string;
-	char *path_uri_string;
-	char *method;
-	char *range;
-	char *auth;
-	char *content;
-	int len;
+static int http_open_post_uri ( struct interface *xfer, struct uri *uri ) {
+	struct parameters *params = uri->params;
+	struct http_request_content content;
+	void *data;
+	size_t len;
+	size_t check_len;
 	int rc;
 
-	/* Do nothing if we have already transmitted the request */
-	if ( ! ( http->flags & HTTP_TX_PENDING ) )
-		return;
-
-	/* Do nothing until socket is ready */
-	if ( ! xfer_window ( &http->socket ) )
-		return;
-
-	/* Force a HEAD request if we have nowhere to send any received data */
-	if ( ( xfer_window ( &http->xfer ) == 0 ) &&
-	     ( http->rx_buffer == UNULL ) ) {
-		http->flags |= ( HTTP_HEAD_ONLY | HTTP_CLIENT_KEEPALIVE );
-	}
-
-	/* Determine method */
-	method = ( ( http->flags & HTTP_HEAD_ONLY ) ? "HEAD" :
-		   ( http->uri->params ? "POST" : "GET" ) );
+	/* Calculate length of parameter list */
+	len = http_params ( params, NULL, 0 );
 
-	/* Construct host URI */
-	memset ( &host_uri, 0, sizeof ( host_uri ) );
-	host_uri.host = http->uri->host;
-	host_uri.port = http->uri->port;
-	host_uri_string = format_uri_alloc ( &host_uri );
-	if ( ! host_uri_string ) {
+	/* Allocate temporary parameter list */
+	data = zalloc ( len + 1 /* NUL */ );
+	if ( ! data ) {
 		rc = -ENOMEM;
-		goto err_host_uri;
+		goto err_alloc;
 	}
 
-	/* Construct path URI */
-	memset ( &path_uri, 0, sizeof ( path_uri ) );
-	path_uri.path = ( http->uri->path ? http->uri->path : "/" );
-	path_uri.query = http->uri->query;
-	path_uri_string = format_uri_alloc ( &path_uri );
-	if ( ! path_uri_string ) {
-		rc = -ENOMEM;
-		goto err_path_uri;
-	}
+	/* Construct temporary parameter list */
+	check_len = http_params ( params, data, ( len + 1 /* NUL */ ) );
+	assert ( check_len == len );
 
-	/* Calculate range request parameters if applicable */
-	if ( http->partial_len ) {
-		len = asprintf ( &range, "Range: bytes=%zd-%zd\r\n",
-				 http->partial_start,
-				 ( http->partial_start + http->partial_len
-				   - 1 ) );
-		if ( len < 0 ) {
-			rc = len;
-			goto err_range;
-		}
-	} else {
-		range = NULL;
-	}
+	/* Construct request content */
+	content.type = "application/x-www-form-urlencoded";
+	content.data = data;
+	content.len = len;
 
-	/* Construct authorisation, if applicable */
-	if ( http->flags & HTTP_BASIC_AUTH ) {
-		auth = http_basic_auth ( http );
-		if ( ! auth ) {
-			rc = -ENOMEM;
-			goto err_auth;
-		}
-	} else if ( http->flags & HTTP_DIGEST_AUTH ) {
-		auth = http_digest_auth ( http, method, path_uri_string );
-		if ( ! auth ) {
-			rc = -ENOMEM;
-			goto err_auth;
-		}
-	} else {
-		auth = NULL;
-	}
+	/* Open HTTP transaction */
+	if ( ( rc = http_open ( xfer, &http_post, uri, NULL, &content ) ) != 0 )
+		goto err_open;
 
-	/* Construct POST content, if applicable */
-	if ( http->uri->params ) {
-		post = http_post ( http );
-		if ( ! post ) {
-			rc = -ENOMEM;
-			goto err_post;
-		}
-		len = asprintf ( &content, "Content-Type: "
-				 "application/x-www-form-urlencoded\r\n"
-				 "Content-Length: %zd\r\n", iob_len ( post ) );
-		if ( len < 0 ) {
-			rc = len;
-			goto err_content;
-		}
-	} else {
-		post = NULL;
-		content = NULL;
-	}
-
-	/* Mark request as transmitted */
-	http->flags &= ~HTTP_TX_PENDING;
-
-	/* Send request */
-	if ( ( rc = xfer_printf ( &http->socket,
-				  "%s %s HTTP/1.1\r\n"
-				  "User-Agent: iPXE/%s\r\n"
-				  "Host: %s\r\n"
-				  "%s%s%s%s"
-				  "\r\n",
-				  method, path_uri_string, product_version,
-				  host_uri_string,
-				  ( ( http->flags & HTTP_CLIENT_KEEPALIVE ) ?
-				    "Connection: keep-alive\r\n" : "" ),
-				  ( range ? range : "" ),
-				  ( auth ? auth : "" ),
-				  ( content ? content : "" ) ) ) != 0 ) {
-		goto err_xfer;
-	}
-
-	/* Send POST content, if applicable */
-	if ( post ) {
-		if ( ( rc = xfer_deliver_iob ( &http->socket,
-					       iob_disown ( post ) ) ) != 0 )
-			goto err_xfer_post;
-	}
-
- err_xfer_post:
- err_xfer:
-	free ( content );
- err_content:
-	free ( post );
- err_post:
-	free ( auth );
- err_auth:
-	free ( range );
- err_range:
-	free ( path_uri_string );
- err_path_uri:
-	free ( host_uri_string );
- err_host_uri:
-	if ( rc != 0 )
-		http_close ( http, rc );
-}
-
-/**
- * Check HTTP data transfer flow control window
- *
- * @v http		HTTP request
- * @ret len		Length of window
- */
-static size_t http_xfer_window ( struct http_request *http ) {
-
-	/* New block commands may be issued only when we are idle */
-	return ( ( http->rx_state == HTTP_RX_IDLE ) ? 1 : 0 );
-}
-
-/**
- * Initiate HTTP partial read
- *
- * @v http		HTTP request
- * @v partial		Partial transfer interface
- * @v offset		Starting offset
- * @v buffer		Data buffer
- * @v len		Length
- * @ret rc		Return status code
- */
-static int http_partial_read ( struct http_request *http,
-			       struct interface *partial,
-			       size_t offset, userptr_t buffer, size_t len ) {
-
-	/* Sanity check */
-	if ( http_xfer_window ( http ) == 0 )
-		return -EBUSY;
-
-	/* Initialise partial transfer parameters */
-	http->rx_buffer = buffer;
-	http->partial_start = offset;
-	http->partial_len = len;
-
-	/* Schedule request */
-	http->rx_state = HTTP_RX_RESPONSE;
-	http->flags = ( HTTP_TX_PENDING | HTTP_CLIENT_KEEPALIVE );
-	if ( ! len )
-		http->flags |= HTTP_HEAD_ONLY;
-	process_add ( &http->process );
-
-	/* Attach to parent interface and return */
-	intf_plug_plug ( &http->partial, partial );
-
-	return 0;
-}
-
-/**
- * Issue HTTP block device read
- *
- * @v http		HTTP request
- * @v block		Block data interface
- * @v lba		Starting logical block address
- * @v count		Number of blocks to transfer
- * @v buffer		Data buffer
- * @v len		Length of data buffer
- * @ret rc		Return status code
- */
-static int http_block_read ( struct http_request *http,
-			     struct interface *block,
-			     uint64_t lba, unsigned int count,
-			     userptr_t buffer, size_t len __unused ) {
-
-	return http_partial_read ( http, block, ( lba * HTTP_BLKSIZE ),
-				   buffer, ( count * HTTP_BLKSIZE ) );
-}
-
-/**
- * Read HTTP block device capacity
- *
- * @v http		HTTP request
- * @v block		Block data interface
- * @ret rc		Return status code
- */
-static int http_block_read_capacity ( struct http_request *http,
-				      struct interface *block ) {
-
-	return http_partial_read ( http, block, 0, 0, 0 );
-}
-
-/**
- * Describe HTTP device in an ACPI table
- *
- * @v http		HTTP request
- * @v acpi		ACPI table
- * @v len		Length of ACPI table
- * @ret rc		Return status code
- */
-static int http_acpi_describe ( struct http_request *http,
-				struct acpi_description_header *acpi,
-				size_t len ) {
-
-	DBGC ( http, "HTTP %p cannot yet describe device in an ACPI table\n",
-	       http );
-	( void ) acpi;
-	( void ) len;
-	return 0;
+ err_open:
+	free ( data );
+ err_alloc:
+	return rc;
 }
 
-/** HTTP socket interface operations */
-static struct interface_operation http_socket_operations[] = {
-	INTF_OP ( xfer_window, struct http_request *, http_socket_window ),
-	INTF_OP ( xfer_deliver, struct http_request *, http_socket_deliver ),
-	INTF_OP ( xfer_window_changed, struct http_request *, http_step ),
-	INTF_OP ( intf_close, struct http_request *, http_socket_close ),
-};
-
-/** HTTP socket interface descriptor */
-static struct interface_descriptor http_socket_desc =
-	INTF_DESC_PASSTHRU ( struct http_request, socket,
-			     http_socket_operations, xfer );
-
-/** HTTP partial transfer interface operations */
-static struct interface_operation http_partial_operations[] = {
-	INTF_OP ( intf_close, struct http_request *, http_close ),
-};
-
-/** HTTP partial transfer interface descriptor */
-static struct interface_descriptor http_partial_desc =
-	INTF_DESC ( struct http_request, partial, http_partial_operations );
-
-/** HTTP data transfer interface operations */
-static struct interface_operation http_xfer_operations[] = {
-	INTF_OP ( xfer_window, struct http_request *, http_xfer_window ),
-	INTF_OP ( block_read, struct http_request *, http_block_read ),
-	INTF_OP ( block_read_capacity, struct http_request *,
-		  http_block_read_capacity ),
-	INTF_OP ( intf_close, struct http_request *, http_close ),
-	INTF_OP ( acpi_describe, struct http_request *, http_acpi_describe ),
-};
-
-/** HTTP data transfer interface descriptor */
-static struct interface_descriptor http_xfer_desc =
-	INTF_DESC_PASSTHRU ( struct http_request, xfer,
-			     http_xfer_operations, socket );
-
-/** HTTP process descriptor */
-static struct process_descriptor http_process_desc =
-	PROC_DESC_ONCE ( struct http_request, process, http_step );
-
 /**
- * Initiate an HTTP connection, with optional filter
+ * Open HTTP transaction for simple URI
  *
  * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @v default_port	Default port number
- * @v filter		Filter to apply to socket, or NULL
+ * @v uri		Request URI
  * @ret rc		Return status code
  */
-int http_open_filter ( struct interface *xfer, struct uri *uri,
-		       unsigned int default_port,
-		       int ( * filter ) ( struct interface *xfer,
-					  const char *name,
-					  struct interface **next ) ) {
-	struct http_request *http;
-	int rc;
-
-	/* Sanity checks */
-	if ( ! uri->host )
-		return -EINVAL;
-
-	/* Allocate and populate HTTP structure */
-	http = zalloc ( sizeof ( *http ) );
-	if ( ! http )
-		return -ENOMEM;
-	ref_init ( &http->refcnt, http_free );
-	intf_init ( &http->xfer, &http_xfer_desc, &http->refcnt );
-	intf_init ( &http->partial, &http_partial_desc, &http->refcnt );
-	http->uri = uri_get ( uri );
-	http->default_port = default_port;
-	http->filter = filter;
-	intf_init ( &http->socket, &http_socket_desc, &http->refcnt );
-	process_init ( &http->process, &http_process_desc, &http->refcnt );
-	timer_init ( &http->timer, http_retry, &http->refcnt );
-	http->flags = HTTP_TX_PENDING;
+int http_open_uri ( struct interface *xfer, struct uri *uri ) {
 
-	/* Open socket */
-	if ( ( rc = http_socket_open ( http ) ) != 0 )
-		goto err;
-
-	/* Attach to parent interface, mortalise self, and return */
-	intf_plug_plug ( &http->xfer, xfer );
-	ref_put ( &http->refcnt );
-	return 0;
-
- err:
-	DBGC ( http, "HTTP %p could not create request: %s\n",
-	       http, strerror ( rc ) );
-	http_close ( http, rc );
-	ref_put ( &http->refcnt );
-	return rc;
+	/* Open GET/POST URI as applicable */
+	if ( uri->params ) {
+		return http_open_post_uri ( xfer, uri );
+	} else {
+		return http_open_get_uri ( xfer, uri );
+	}
 }
+
+/* Drag in HTTP extensions */
+REQUIRING_SYMBOL ( http_open );
+REQUIRE_OBJECT ( config_http );
diff --git a/roms/ipxe/src/net/tcp/httpdigest.c b/roms/ipxe/src/net/tcp/httpdigest.c
new file mode 100644
index 000000000..626dd7e9d
--- /dev/null
+++ b/roms/ipxe/src/net/tcp/httpdigest.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) Digest authentication
+ *
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <strings.h>
+#include <ipxe/uri.h>
+#include <ipxe/md5.h>
+#include <ipxe/base16.h>
+#include <ipxe/vsprintf.h>
+#include <ipxe/http.h>
+
+/* Disambiguate the various error causes */
+#define EACCES_USERNAME __einfo_error ( EINFO_EACCES_USERNAME )
+#define EINFO_EACCES_USERNAME						\
+	__einfo_uniqify ( EINFO_EACCES, 0x01,				\
+			  "No username available for Digest authentication" )
+
+/**
+ * Initialise HTTP Digest
+ *
+ * @v ctx		Digest context
+ * @v string		Initial string
+ */
+static void http_digest_init ( struct md5_context *ctx ) {
+
+	/* Initialise MD5 digest */
+	digest_init ( &md5_algorithm, ctx );
+}
+
+/**
+ * Update HTTP Digest with new data
+ *
+ * @v ctx		Digest context
+ * @v string		String to append
+ */
+static void http_digest_update ( struct md5_context *ctx, const char *string ) {
+	static const char colon = ':';
+
+	/* Add (possibly colon-separated) field to MD5 digest */
+	if ( ctx->len )
+		digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
+	digest_update ( &md5_algorithm, ctx, string, strlen ( string ) );
+}
+
+/**
+ * Finalise HTTP Digest
+ *
+ * @v ctx		Digest context
+ * @v out		Buffer for digest output
+ * @v len		Buffer length
+ */
+static void http_digest_final ( struct md5_context *ctx, char *out,
+				size_t len ) {
+	uint8_t digest[MD5_DIGEST_SIZE];
+
+	/* Finalise and base16-encode MD5 digest */
+	digest_final ( &md5_algorithm, ctx, digest );
+	base16_encode ( digest, sizeof ( digest ), out, len );
+}
+
+/**
+ * Perform HTTP Digest authentication
+ *
+ * @v http		HTTP transaction
+ * @ret rc		Return status code
+ */
+static int http_digest_authenticate ( struct http_transaction *http ) {
+	struct http_request_auth *req = &http->request.auth;
+	struct http_response_auth *rsp = &http->response.auth;
+	char ha1[ base16_encoded_len ( MD5_DIGEST_SIZE ) + 1 /* NUL */ ];
+	char ha2[ base16_encoded_len ( MD5_DIGEST_SIZE ) + 1 /* NUL */ ];
+	static const char md5sess[] = "MD5-sess";
+	static const char md5[] = "MD5";
+	struct md5_context ctx;
+
+	/* Check for required response parameters */
+	if ( ! rsp->realm ) {
+		DBGC ( http, "HTTP %p has no realm for Digest authentication\n",
+		       http );
+		return -EINVAL;
+	}
+	if ( ! rsp->nonce ) {
+		DBGC ( http, "HTTP %p has no nonce for Digest authentication\n",
+		       http );
+		return -EINVAL;
+	}
+
+	/* Record username and password */
+	if ( ! http->uri->user ) {
+		DBGC ( http, "HTTP %p has no username for Digest "
+		       "authentication\n", http );
+		return -EACCES_USERNAME;
+	}
+	req->username = http->uri->user;
+	req->password = ( http->uri->password ? http->uri->password : "" );
+
+	/* Handle quality of protection */
+	if ( rsp->qop ) {
+
+		/* Use "auth" in subsequent request */
+		req->qop = "auth";
+
+		/* Generate a client nonce */
+		snprintf ( req->cnonce, sizeof ( req->cnonce ),
+			   "%08lx", random() );
+
+		/* Determine algorithm */
+		req->algorithm = md5;
+		if ( rsp->algorithm &&
+		     ( strcasecmp ( rsp->algorithm, md5sess ) == 0 ) ) {
+			req->algorithm = md5sess;
+		}
+	}
+
+	/* Generate HA1 */
+	http_digest_init ( &ctx );
+	http_digest_update ( &ctx, req->username );
+	http_digest_update ( &ctx, rsp->realm );
+	http_digest_update ( &ctx, req->password );
+	http_digest_final ( &ctx, ha1, sizeof ( ha1 ) );
+	if ( req->algorithm == md5sess ) {
+		http_digest_init ( &ctx );
+		http_digest_update ( &ctx, ha1 );
+		http_digest_update ( &ctx, rsp->nonce );
+		http_digest_update ( &ctx, req->cnonce );
+		http_digest_final ( &ctx, ha1, sizeof ( ha1 ) );
+	}
+
+	/* Generate HA2 */
+	http_digest_init ( &ctx );
+	http_digest_update ( &ctx, http->request.method->name );
+	http_digest_update ( &ctx, http->request.uri );
+	http_digest_final ( &ctx, ha2, sizeof ( ha2 ) );
+
+	/* Generate response */
+	http_digest_init ( &ctx );
+	http_digest_update ( &ctx, ha1 );
+	http_digest_update ( &ctx, rsp->nonce );
+	if ( req->qop ) {
+		http_digest_update ( &ctx, HTTP_DIGEST_NC );
+		http_digest_update ( &ctx, req->cnonce );
+		http_digest_update ( &ctx, req->qop );
+	}
+	http_digest_update ( &ctx, ha2 );
+	http_digest_final ( &ctx, req->response, sizeof ( req->response ) );
+
+	return 0;
+}
+
+/**
+ * Construct HTTP "Authorization" header for Digest authentication
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_digest_auth ( struct http_transaction *http,
+				     char *buf, size_t len ) {
+	struct http_request_auth *req = &http->request.auth;
+	struct http_response_auth *rsp = &http->response.auth;
+	size_t used = 0;
+
+	/* Sanity checks */
+	assert ( rsp->realm != NULL );
+	assert ( rsp->nonce != NULL );
+	assert ( req->username != NULL );
+	if ( req->qop ) {
+		assert ( req->algorithm != NULL );
+		assert ( req->cnonce[0] != '\0' );
+	}
+	assert ( req->response[0] != '\0' );
+
+	/* Construct response */
+	used += ssnprintf ( ( buf + used ), ( len - used ),
+			    "realm=\"%s\", nonce=\"%s\", uri=\"%s\", "
+			    "username=\"%s\"", rsp->realm, rsp->nonce,
+			    http->request.uri, req->username );
+	if ( rsp->opaque ) {
+		used += ssnprintf ( ( buf + used ), ( len - used ),
+				    ", opaque=\"%s\"", rsp->opaque );
+	}
+	if ( req->qop ) {
+		used += ssnprintf ( ( buf + used ), ( len - used ),
+				    ", qop=%s, algorithm=%s, cnonce=\"%s\", "
+				    "nc=" HTTP_DIGEST_NC, req->qop,
+				    req->algorithm, req->cnonce );
+	}
+	used += ssnprintf ( ( buf + used ), ( len - used ),
+			    ", response=\"%s\"", req->response );
+
+	return used;
+}
+
+/** HTTP Digest authentication scheme */
+struct http_authentication http_digest_auth __http_authentication = {
+	.name = "Digest",
+	.authenticate = http_digest_authenticate,
+	.format = http_format_digest_auth,
+};
+
+/* Drag in HTTP authentication support */
+REQUIRING_SYMBOL ( http_digest_auth );
+REQUIRE_OBJECT ( httpauth );
diff --git a/roms/ipxe/src/net/tcp/https.c b/roms/ipxe/src/net/tcp/https.c
index 6112acdae..e91000322 100644
--- a/roms/ipxe/src/net/tcp/https.c
+++ b/roms/ipxe/src/net/tcp/https.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -26,7 +30,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#include <stddef.h>
 #include <ipxe/open.h>
 #include <ipxe/tls.h>
 #include <ipxe/http.h>
@@ -34,19 +37,15 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 FEATURE ( FEATURE_PROTOCOL, "HTTPS", DHCP_EB_FEATURE_HTTPS, 1 );
 
-/**
- * Initiate an HTTPS connection
- *
- * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @ret rc		Return status code
- */
-static int https_open ( struct interface *xfer, struct uri *uri ) {
-	return http_open_filter ( xfer, uri, HTTPS_PORT, add_tls );
-}
-
 /** HTTPS URI opener */
 struct uri_opener https_uri_opener __uri_opener = {
 	.scheme	= "https",
-	.open	= https_open,
+	.open	= http_open_uri,
+};
+
+/** HTTP URI scheme */
+struct http_scheme https_scheme __http_scheme = {
+	.name = "https",
+	.port = HTTPS_PORT,
+	.filter = add_tls,
 };
diff --git a/roms/ipxe/src/net/tcp/iscsi.c b/roms/ipxe/src/net/tcp/iscsi.c
index 03c6d0f23..019a4c14e 100644
--- a/roms/ipxe/src/net/tcp/iscsi.c
+++ b/roms/ipxe/src/net/tcp/iscsi.c
@@ -15,14 +15,19 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <ctype.h>
 #include <errno.h>
 #include <assert.h>
 #include <byteswap.h>
@@ -127,7 +132,7 @@ FEATURE ( FEATURE_PROTOCOL, "iSCSI", DHCP_EB_FEATURE_ISCSI, 1 );
 #define EPROTO_INVALID_LARGE_BINARY \
 	__einfo_error ( EINFO_EPROTO_INVALID_LARGE_BINARY )
 #define EINFO_EPROTO_INVALID_LARGE_BINARY \
-	__einfo_uniqify ( EINFO_EPROTO, 0x03, "Invalid large binary" )
+	__einfo_uniqify ( EINFO_EPROTO, 0x03, "Invalid large binary value" )
 #define EPROTO_INVALID_CHAP_RESPONSE \
 	__einfo_error ( EINFO_EPROTO_INVALID_CHAP_RESPONSE )
 #define EINFO_EPROTO_INVALID_CHAP_RESPONSE \
@@ -704,7 +709,7 @@ static int iscsi_build_login_request_strings ( struct iscsi_session *iscsi,
 		char buf[ base16_encoded_len ( iscsi->chap.response_len ) + 1 ];
 		assert ( iscsi->initiator_username != NULL );
 		base16_encode ( iscsi->chap.response, iscsi->chap.response_len,
-				buf );
+				buf, sizeof ( buf ) );
 		used += ssnprintf ( data + used, len - used,
 				    "CHAP_N=%s%cCHAP_R=0x%s%c",
 				    iscsi->initiator_username, 0, buf, 0 );
@@ -714,7 +719,7 @@ static int iscsi_build_login_request_strings ( struct iscsi_session *iscsi,
 		size_t challenge_len = ( sizeof ( iscsi->chap_challenge ) - 1 );
 		char buf[ base16_encoded_len ( challenge_len ) + 1 ];
 		base16_encode ( ( iscsi->chap_challenge + 1 ), challenge_len,
-				buf );
+				buf, sizeof ( buf ) );
 		used += ssnprintf ( data + used, len - used,
 				    "CHAP_I=%d%cCHAP_C=0x%s%c",
 				    iscsi->chap_challenge[0], 0, buf, 0 );
@@ -824,38 +829,27 @@ static int iscsi_tx_login_request ( struct iscsi_session *iscsi ) {
 }
 
 /**
- * Calculate maximum length of decoded large binary value
- *
- * @v encoded		Encoded large binary value
- * @v max_raw_len	Maximum length of raw data
- */
-static inline size_t
-iscsi_large_binary_decoded_max_len ( const char *encoded ) {
-	return ( strlen ( encoded ) ); /* Decoding never expands data */
-}
-
-/**
  * Decode large binary value
  *
  * @v encoded		Encoded large binary value
  * @v raw		Raw data
+ * @v len		Length of data buffer
  * @ret len		Length of raw data, or negative error
  */
-static int iscsi_large_binary_decode ( const char *encoded, uint8_t *raw ) {
-
-	if ( encoded[0] != '0' )
-		return -EPROTO_INVALID_LARGE_BINARY;
-
-	switch ( encoded[1] ) {
-	case 'x' :
-	case 'X' :
-		return base16_decode ( ( encoded + 2 ), raw );
-	case 'b' :
-	case 'B' :
-		return base64_decode ( ( encoded + 2 ), raw );
-	default:
-		return -EPROTO_INVALID_LARGE_BINARY;
+static int iscsi_large_binary_decode ( const char *encoded, uint8_t *raw,
+				       size_t len ) {
+
+	/* Check for initial '0x' or '0b' and decode as appropriate */
+	if ( *(encoded++) == '0' ) {
+		switch ( tolower ( *(encoded++) ) ) {
+		case 'x' :
+			return base16_decode ( encoded, raw, len );
+		case 'b' :
+			return base64_decode ( encoded, raw, len );
+		}
 	}
+
+	return -EPROTO_INVALID_LARGE_BINARY;
 }
 
 /**
@@ -982,19 +976,19 @@ static int iscsi_handle_chap_i_value ( struct iscsi_session *iscsi,
  */
 static int iscsi_handle_chap_c_value ( struct iscsi_session *iscsi,
 				       const char *value ) {
-	uint8_t buf[ iscsi_large_binary_decoded_max_len ( value ) ];
+	uint8_t buf[ strlen ( value ) ]; /* Decoding never expands data */
 	unsigned int i;
-	size_t len;
+	int len;
 	int rc;
 
 	/* Process challenge */
-	rc = iscsi_large_binary_decode ( value, buf );
-	if ( rc < 0 ) {
+	len = iscsi_large_binary_decode ( value, buf, sizeof ( buf ) );
+	if ( len < 0 ) {
+		rc = len;
 		DBGC ( iscsi, "iSCSI %p invalid CHAP challenge \"%s\": %s\n",
 		       iscsi, value, strerror ( rc ) );
 		return rc;
 	}
-	len = rc;
 	chap_update ( &iscsi->chap, buf, len );
 
 	/* Build CHAP response */
@@ -1052,8 +1046,8 @@ static int iscsi_handle_chap_n_value ( struct iscsi_session *iscsi,
  */
 static int iscsi_handle_chap_r_value ( struct iscsi_session *iscsi,
 				       const char *value ) {
-	uint8_t buf[ iscsi_large_binary_decoded_max_len ( value ) ];
-	size_t len;
+	uint8_t buf[ strlen ( value ) ]; /* Decoding never expands data */
+	int len;
 	int rc;
 
 	/* Generate CHAP response for verification */
@@ -1073,16 +1067,16 @@ static int iscsi_handle_chap_r_value ( struct iscsi_session *iscsi,
 	chap_respond ( &iscsi->chap );
 
 	/* Process response */
-	rc = iscsi_large_binary_decode ( value, buf );
-	if ( rc < 0 ) {
+	len = iscsi_large_binary_decode ( value, buf, sizeof ( buf ) );
+	if ( len < 0 ) {
+		rc = len;
 		DBGC ( iscsi, "iSCSI %p invalid CHAP response \"%s\": %s\n",
 		       iscsi, value, strerror ( rc ) );
 		return rc;
 	}
-	len = rc;
 
 	/* Check CHAP response */
-	if ( len != iscsi->chap.response_len ) {
+	if ( len != ( int ) iscsi->chap.response_len ) {
 		DBGC ( iscsi, "iSCSI %p invalid CHAP response length\n",
 		       iscsi );
 		return -EPROTO_INVALID_CHAP_RESPONSE;
@@ -1445,8 +1439,10 @@ static void iscsi_tx_done ( struct iscsi_session *iscsi ) {
 	switch ( common->opcode & ISCSI_OPCODE_MASK ) {
 	case ISCSI_OPCODE_DATA_OUT:
 		iscsi_data_out_done ( iscsi );
+		break;
 	case ISCSI_OPCODE_LOGIN_REQUEST:
 		iscsi_login_request_done ( iscsi );
+		break;
 	default:
 		/* No action */
 		break;
diff --git a/roms/ipxe/src/net/tcp/syslogs.c b/roms/ipxe/src/net/tcp/syslogs.c
index 095afc543..0c07f86d5 100644
--- a/roms/ipxe/src/net/tcp/syslogs.c
+++ b/roms/ipxe/src/net/tcp/syslogs.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/net/tcpip.c b/roms/ipxe/src/net/tcpip.c
index 4bcbe64bb..5ad982fd1 100644
--- a/roms/ipxe/src/net/tcpip.c
+++ b/roms/ipxe/src/net/tcpip.c
@@ -17,7 +17,7 @@
  * TCP/IP transport-network layer interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Process a received TCP/IP packet
@@ -235,7 +235,7 @@ int tcpip_bind ( struct sockaddr_tcpip *st_local,
 	/* Otherwise, find an available port in the range [1,1023] or
 	 * [1025,65535] as appropriate.
 	 */
-	min_port = ( ( ( ! flags ) & TCPIP_BIND_PRIVILEGED ) + 1 );
+	min_port = ( ( ( ~flags ) & TCPIP_BIND_PRIVILEGED ) + 1 );
 	max_port = ( ( flags & TCPIP_BIND_PRIVILEGED ) - 1 );
 	offset = random();
 	for ( i = 0 ; i <= max_port ; i++ ) {
diff --git a/roms/ipxe/src/net/tls.c b/roms/ipxe/src/net/tls.c
index 30ccc932e..db01fb291 100644
--- a/roms/ipxe/src/net/tls.c
+++ b/roms/ipxe/src/net/tls.c
@@ -179,20 +179,29 @@ static void tls_clear_cipher ( struct tls_session *tls,
  ******************************************************************************
  */
 
+/** A TLS 24-bit integer
+ *
+ * TLS uses 24-bit integers in several places, which are awkward to
+ * parse in C.
+ */
+typedef struct {
+	/** High byte */
+	uint8_t high;
+	/** Low word */
+	uint16_t low;
+} __attribute__ (( packed )) tls24_t;
+
 /**
  * Extract 24-bit field value
  *
  * @v field24		24-bit field
  * @ret value		Field value
  *
- * TLS uses 24-bit integers in several places, which are awkward to
- * parse in C.
  */
 static inline __attribute__ (( always_inline )) unsigned long
-tls_uint24 ( const uint8_t field24[3] ) {
-	const uint32_t *field32 __attribute__ (( may_alias )) =
-		( ( const void * ) field24 );
-	return ( be32_to_cpu ( *field32 ) >> 8 );
+tls_uint24 ( const tls24_t *field24 ) {
+
+	return ( ( field24->high << 16 ) | be16_to_cpu ( field24->low ) );
 }
 
 /**
@@ -200,13 +209,11 @@ tls_uint24 ( const uint8_t field24[3] ) {
  *
  * @v field24		24-bit field
  * @v value		Field value
- *
- * The field must be pre-zeroed.
  */
-static void tls_set_uint24 ( uint8_t field24[3], unsigned long value ) {
-	uint32_t *field32 __attribute__ (( may_alias )) =
-		( ( void * ) field24 );
-	*field32 |= cpu_to_be32 ( value << 8 );
+static void tls_set_uint24 ( tls24_t *field24, unsigned long value ) {
+
+	field24->high = ( value >> 16 );
+	field24->low = cpu_to_be16 ( value );
 }
 
 /**
@@ -659,41 +666,8 @@ struct tls_cipher_suite tls_cipher_suite_null = {
 	.digest = &digest_null,
 };
 
-/** Supported cipher suites, in order of preference */
-struct tls_cipher_suite tls_cipher_suites[] = {
-	{
-		.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA256 ),
-		.key_len = ( 256 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha256_algorithm,
-	},
-	{
-		.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA256 ),
-		.key_len = ( 128 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha256_algorithm,
-	},
-	{
-		.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA ),
-		.key_len = ( 256 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha1_algorithm,
-	},
-	{
-		.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA ),
-		.key_len = ( 128 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha1_algorithm,
-	},
-};
-
 /** Number of supported cipher suites */
-#define TLS_NUM_CIPHER_SUITES \
-	( sizeof ( tls_cipher_suites ) / sizeof ( tls_cipher_suites[0] ) )
+#define TLS_NUM_CIPHER_SUITES table_num_entries ( TLS_CIPHER_SUITES )
 
 /**
  * Identify cipher suite
@@ -704,11 +678,9 @@ struct tls_cipher_suite tls_cipher_suites[] = {
 static struct tls_cipher_suite *
 tls_find_cipher_suite ( unsigned int cipher_suite ) {
 	struct tls_cipher_suite *suite;
-	unsigned int i;
 
 	/* Identify cipher suite */
-	for ( i = 0 ; i < TLS_NUM_CIPHER_SUITES ; i++ ) {
-		suite = &tls_cipher_suites[i];
+	for_each_table_entry ( suite, TLS_CIPHER_SUITES ) {
 		if ( suite->code == cipher_suite )
 			return suite;
 	}
@@ -841,26 +813,9 @@ static int tls_change_cipher ( struct tls_session *tls,
  ******************************************************************************
  */
 
-/** Supported signature and hash algorithms
- *
- * Note that the default (TLSv1.1 and earlier) algorithm using
- * MD5+SHA1 is never explicitly specified.
- */
-struct tls_signature_hash_algorithm tls_signature_hash_algorithms[] = {
-	{
-		.code = {
-			.signature = TLS_RSA_ALGORITHM,
-			.hash = TLS_SHA256_ALGORITHM,
-		},
-		.pubkey = &rsa_algorithm,
-		.digest = &sha256_algorithm,
-	},
-};
-
 /** Number of supported signature and hash algorithms */
-#define TLS_NUM_SIG_HASH_ALGORITHMS			\
-	( sizeof ( tls_signature_hash_algorithms ) /	\
-	  sizeof ( tls_signature_hash_algorithms[0] ) )
+#define TLS_NUM_SIG_HASH_ALGORITHMS \
+	table_num_entries ( TLS_SIG_HASH_ALGORITHMS )
 
 /**
  * Find TLS signature and hash algorithm
@@ -873,11 +828,9 @@ static struct tls_signature_hash_algorithm *
 tls_signature_hash_algorithm ( struct pubkey_algorithm *pubkey,
 			       struct digest_algorithm *digest ) {
 	struct tls_signature_hash_algorithm *sig_hash;
-	unsigned int i;
 
 	/* Identify signature and hash algorithm */
-	for ( i = 0 ; i < TLS_NUM_SIG_HASH_ALGORITHMS ; i++ ) {
-		sig_hash = &tls_signature_hash_algorithms[i];
+	for_each_table_entry ( sig_hash, TLS_SIG_HASH_ALGORITHMS ) {
 		if ( ( sig_hash->pubkey == pubkey ) &&
 		     ( sig_hash->digest == digest ) ) {
 			return sig_hash;
@@ -994,8 +947,17 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 			struct {
 				uint8_t max;
 			} __attribute__ (( packed )) max_fragment_length;
+			uint16_t signature_algorithms_type;
+			uint16_t signature_algorithms_len;
+			struct {
+				uint16_t len;
+				struct tls_signature_hash_id
+					code[TLS_NUM_SIG_HASH_ALGORITHMS];
+			} __attribute__ (( packed )) signature_algorithms;
 		} __attribute__ (( packed )) extensions;
 	} __attribute__ (( packed )) hello;
+	struct tls_cipher_suite *suite;
+	struct tls_signature_hash_algorithm *sighash;
 	unsigned int i;
 
 	memset ( &hello, 0, sizeof ( hello ) );
@@ -1005,8 +967,8 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 	hello.version = htons ( tls->version );
 	memcpy ( &hello.random, &tls->client_random, sizeof ( hello.random ) );
 	hello.cipher_suite_len = htons ( sizeof ( hello.cipher_suites ) );
-	for ( i = 0 ; i < TLS_NUM_CIPHER_SUITES ; i++ )
-		hello.cipher_suites[i] = tls_cipher_suites[i].code;
+	i = 0 ; for_each_table_entry ( suite, TLS_CIPHER_SUITES )
+		hello.cipher_suites[i++] = suite->code;
 	hello.compression_methods_len = sizeof ( hello.compression_methods );
 	hello.extensions_len = htons ( sizeof ( hello.extensions ) );
 	hello.extensions.server_name_type = htons ( TLS_SERVER_NAME );
@@ -1025,6 +987,14 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 		= htons ( sizeof ( hello.extensions.max_fragment_length ) );
 	hello.extensions.max_fragment_length.max
 		= TLS_MAX_FRAGMENT_LENGTH_4096;
+	hello.extensions.signature_algorithms_type
+		= htons ( TLS_SIGNATURE_ALGORITHMS );
+	hello.extensions.signature_algorithms_len
+		= htons ( sizeof ( hello.extensions.signature_algorithms ) );
+	hello.extensions.signature_algorithms.len
+		= htons ( sizeof ( hello.extensions.signature_algorithms.code));
+	i = 0 ; for_each_table_entry ( sighash, TLS_SIG_HASH_ALGORITHMS )
+		hello.extensions.signature_algorithms.code[i++] = sighash->code;
 
 	return tls_send_handshake ( tls, &hello, sizeof ( hello ) );
 }
@@ -1038,9 +1008,9 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 static int tls_send_certificate ( struct tls_session *tls ) {
 	struct {
 		uint32_t type_length;
-		uint8_t length[3];
+		tls24_t length;
 		struct {
-			uint8_t length[3];
+			tls24_t length;
 			uint8_t data[ tls->cert->raw.len ];
 		} __attribute__ (( packed )) certificates[1];
 	} __attribute__ (( packed )) *certificate;
@@ -1058,9 +1028,9 @@ static int tls_send_certificate ( struct tls_session *tls ) {
 		( cpu_to_le32 ( TLS_CERTIFICATE ) |
 		  htonl ( sizeof ( *certificate ) -
 			  sizeof ( certificate->type_length ) ) );
-	tls_set_uint24 ( certificate->length,
+	tls_set_uint24 ( &certificate->length,
 			 sizeof ( certificate->certificates ) );
-	tls_set_uint24 ( certificate->certificates[0].length,
+	tls_set_uint24 ( &certificate->certificates[0].length,
 			 sizeof ( certificate->certificates[0].data ) );
 	memcpy ( certificate->certificates[0].data,
 		 tls->cert->raw.data,
@@ -1412,7 +1382,7 @@ static int tls_parse_chain ( struct tls_session *tls,
 			     const void *data, size_t len ) {
 	const void *end = ( data + len );
 	const struct {
-		uint8_t length[3];
+		tls24_t length;
 		uint8_t data[0];
 	} __attribute__ (( packed )) *certificate;
 	size_t certificate_len;
@@ -1436,7 +1406,7 @@ static int tls_parse_chain ( struct tls_session *tls,
 
 		/* Extract raw certificate data */
 		certificate = data;
-		certificate_len = tls_uint24 ( certificate->length );
+		certificate_len = tls_uint24 ( &certificate->length );
 		next = ( certificate->data + certificate_len );
 		if ( next > end ) {
 			DBGC ( tls, "TLS %p overlength certificate:\n", tls );
@@ -1482,10 +1452,10 @@ static int tls_parse_chain ( struct tls_session *tls,
 static int tls_new_certificate ( struct tls_session *tls,
 				 const void *data, size_t len ) {
 	const struct {
-		uint8_t length[3];
+		tls24_t length;
 		uint8_t certificates[0];
 	} __attribute__ (( packed )) *certificate = data;
-	size_t certificates_len = tls_uint24 ( certificate->length );
+	size_t certificates_len = tls_uint24 ( &certificate->length );
 	const void *end = ( certificate->certificates + certificates_len );
 	int rc;
 
@@ -1634,12 +1604,12 @@ static int tls_new_handshake ( struct tls_session *tls,
 	while ( data != end ) {
 		const struct {
 			uint8_t type;
-			uint8_t length[3];
+			tls24_t length;
 			uint8_t payload[0];
 		} __attribute__ (( packed )) *handshake = data;
-		void *payload = &handshake->payload;
-		size_t payload_len = tls_uint24 ( handshake->length );
-		void *next = ( payload + payload_len );
+		const void *payload = &handshake->payload;
+		size_t payload_len = tls_uint24 ( &handshake->length );
+		const void *next = ( payload + payload_len );
 
 		/* Sanity check */
 		if ( next > end ) {
@@ -2637,3 +2607,9 @@ int add_tls ( struct interface *xfer, const char *name,
  err_alloc:
 	return rc;
 }
+
+/* Drag in objects via add_tls() */
+REQUIRING_SYMBOL ( add_tls );
+
+/* Drag in crypto configuration */
+REQUIRE_OBJECT ( config_crypto );
diff --git a/roms/ipxe/src/net/udp.c b/roms/ipxe/src/net/udp.c
index 76da67ecf..0f7dfb24a 100644
--- a/roms/ipxe/src/net/udp.c
+++ b/roms/ipxe/src/net/udp.c
@@ -17,7 +17,7 @@
  * UDP protocol
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * A UDP connection
diff --git a/roms/ipxe/src/net/udp/dhcp.c b/roms/ipxe/src/net/udp/dhcp.c
index 04fad04c2..aed5ee360 100644
--- a/roms/ipxe/src/net/udp/dhcp.c
+++ b/roms/ipxe/src/net/udp/dhcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdlib.h>
@@ -44,6 +48,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/dhcppkt.h>
 #include <ipxe/dhcp_arch.h>
 #include <ipxe/features.h>
+#include <config/dhcp.h>
 
 /** @file
  *
@@ -149,30 +154,32 @@ struct dhcp_session_state {
 	 * @v dhcppkt		DHCP packet
 	 * @v peer		Destination address
 	 */
-	int ( * tx ) ( struct dhcp_session *dhcp,
-		       struct dhcp_packet *dhcppkt,
+	int ( * tx ) ( struct dhcp_session *dhcp, struct dhcp_packet *dhcppkt,
 		       struct sockaddr_in *peer );
-	/** Handle received packet
+	/**
+	 * Handle received packet
 	 *
 	 * @v dhcp		DHCP session
 	 * @v dhcppkt		DHCP packet
 	 * @v peer		DHCP server address
 	 * @v msgtype		DHCP message type
 	 * @v server_id		DHCP server ID
+	 * @v pseudo_id		DHCP server pseudo-ID
 	 */
-	void ( * rx ) ( struct dhcp_session *dhcp,
-			struct dhcp_packet *dhcppkt,
-			struct sockaddr_in *peer,
-			uint8_t msgtype, struct in_addr server_id );
-	/** Handle timer expiry
+	void ( * rx ) ( struct dhcp_session *dhcp, struct dhcp_packet *dhcppkt,
+			struct sockaddr_in *peer, uint8_t msgtype,
+			struct in_addr server_id, struct in_addr pseudo_id );
+	/**
+	 * Handle timer expiry
 	 *
 	 * @v dhcp		DHCP session
 	 */
 	void ( * expired ) ( struct dhcp_session *dhcp );
 	/** Transmitted message type */
 	uint8_t tx_msgtype;
-	/** Apply minimum timeout */
-	uint8_t apply_min_timeout;
+	/** Timeout parameters */
+	uint8_t min_timeout_sec;
+	uint8_t max_timeout_sec;
 };
 
 static struct dhcp_session_state dhcp_state_discover;
@@ -272,9 +279,9 @@ static void dhcp_set_state ( struct dhcp_session *dhcp,
 	dhcp->state = state;
 	dhcp->start = currticks();
 	stop_timer ( &dhcp->timer );
-	dhcp->timer.min_timeout =
-		( state->apply_min_timeout ? DHCP_MIN_TIMEOUT : 0 );
-	dhcp->timer.max_timeout = DHCP_MAX_TIMEOUT;
+	set_timer_limits ( &dhcp->timer,
+			   ( state->min_timeout_sec * TICKS_PER_SEC ),
+			   ( state->max_timeout_sec * TICKS_PER_SEC ) );
 	start_timer_nodelay ( &dhcp->timer );
 }
 
@@ -334,11 +341,13 @@ static int dhcp_discovery_tx ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 				struct dhcp_packet *dhcppkt,
 				struct sockaddr_in *peer, uint8_t msgtype,
-				struct in_addr server_id ) {
+				struct in_addr server_id,
+				struct in_addr pseudo_id ) {
 	struct in_addr ip;
 	char vci[9]; /* "PXEClient" */
 	int vci_len;
@@ -350,8 +359,11 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
 
 	/* Identify offered IP address */
 	ip = dhcppkt->dhcphdr->yiaddr;
@@ -392,10 +404,10 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 	}
 
 	/* Select as ProxyDHCP offer, if applicable */
-	if ( server_id.s_addr && has_pxeclient &&
+	if ( pseudo_id.s_addr && has_pxeclient &&
 	     ( priority >= dhcp->proxy_priority ) ) {
 		dhcppkt_put ( dhcp->proxy_offer );
-		dhcp->proxy_server = server_id;
+		dhcp->proxy_server = pseudo_id;
 		dhcp->proxy_offer = dhcppkt_get ( dhcppkt );
 		dhcp->proxy_priority = priority;
 	}
@@ -415,7 +427,7 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 	/* If we can't yet transition to DHCPREQUEST, do nothing */
 	elapsed = ( currticks() - dhcp->start );
 	if ( ! ( dhcp->no_pxedhcp || dhcp->proxy_offer ||
-		 ( elapsed > PROXYDHCP_MAX_TIMEOUT ) ) )
+		 ( elapsed > DHCP_DISC_PROXY_TIMEOUT_SEC * TICKS_PER_SEC ) ) )
 		return;
 
 	/* Transition to DHCPREQUEST */
@@ -430,8 +442,18 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 static void dhcp_discovery_expired ( struct dhcp_session *dhcp ) {
 	unsigned long elapsed = ( currticks() - dhcp->start );
 
+	/* If link is blocked, defer DHCP discovery (and reset timeout) */
+	if ( netdev_link_blocked ( dhcp->netdev ) ) {
+		DBGC ( dhcp, "DHCP %p deferring discovery\n", dhcp );
+		start_timer_fixed ( &dhcp->timer,
+				    ( DHCP_DISC_START_TIMEOUT_SEC *
+				      TICKS_PER_SEC ) );
+		return;
+	}
+
 	/* Give up waiting for ProxyDHCP before we reach the failure point */
-	if ( dhcp->offer.s_addr && ( elapsed > PROXYDHCP_MAX_TIMEOUT ) ) {
+	if ( dhcp->offer.s_addr &&
+	     ( elapsed > DHCP_DISC_PROXY_TIMEOUT_SEC * TICKS_PER_SEC ) ) {
 		dhcp_set_state ( dhcp, &dhcp_state_request );
 		return;
 	}
@@ -447,7 +469,8 @@ static struct dhcp_session_state dhcp_state_discover = {
 	.rx			= dhcp_discovery_rx,
 	.expired		= dhcp_discovery_expired,
 	.tx_msgtype		= DHCPDISCOVER,
-	.apply_min_timeout	= 1,
+	.min_timeout_sec	= DHCP_DISC_START_TIMEOUT_SEC,
+	.max_timeout_sec	= DHCP_DISC_END_TIMEOUT_SEC,
 };
 
 /**
@@ -493,11 +516,13 @@ static int dhcp_request_tx ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_request_rx ( struct dhcp_session *dhcp,
 			      struct dhcp_packet *dhcppkt,
 			      struct sockaddr_in *peer, uint8_t msgtype,
-			      struct in_addr server_id ) {
+			      struct in_addr server_id,
+			      struct in_addr pseudo_id ) {
 	struct in_addr ip;
 	struct settings *parent;
 	struct settings *settings;
@@ -506,8 +531,11 @@ static void dhcp_request_rx ( struct dhcp_session *dhcp,
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
 
 	/* Identify leased IP address */
 	ip = dhcppkt->dhcphdr->yiaddr;
@@ -584,7 +612,8 @@ static struct dhcp_session_state dhcp_state_request = {
 	.rx			= dhcp_request_rx,
 	.expired		= dhcp_request_expired,
 	.tx_msgtype		= DHCPREQUEST,
-	.apply_min_timeout	= 0,
+	.min_timeout_sec	= DHCP_REQ_START_TIMEOUT_SEC,
+	.max_timeout_sec	= DHCP_REQ_END_TIMEOUT_SEC,
 };
 
 /**
@@ -623,19 +652,26 @@ static int dhcp_proxy_tx ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_proxy_rx ( struct dhcp_session *dhcp,
 			    struct dhcp_packet *dhcppkt,
 			    struct sockaddr_in *peer, uint8_t msgtype,
-			    struct in_addr server_id ) {
+			    struct in_addr server_id,
+			    struct in_addr pseudo_id ) {
 	struct settings *settings = &dhcppkt->settings;
 	int rc;
 
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
+	if ( dhcp_has_pxeopts ( dhcppkt ) )
+		DBGC ( dhcp, " pxe" );
 	DBGC ( dhcp, "\n" );
 
 	/* Filter out unacceptable responses */
@@ -643,8 +679,9 @@ static void dhcp_proxy_rx ( struct dhcp_session *dhcp,
 		return;
 	if ( ( msgtype != DHCPOFFER ) && ( msgtype != DHCPACK ) )
 		return;
-	if ( server_id.s_addr /* Linux PXE server omits server ID */ &&
-	     ( server_id.s_addr != dhcp->proxy_server.s_addr ) )
+	if ( ( pseudo_id.s_addr != dhcp->proxy_server.s_addr ) )
+		return;
+	if ( ! dhcp_has_pxeopts ( dhcppkt ) )
 		return;
 
 	/* Register settings */
@@ -669,7 +706,7 @@ static void dhcp_proxy_expired ( struct dhcp_session *dhcp ) {
 	unsigned long elapsed = ( currticks() - dhcp->start );
 
 	/* Give up waiting for ProxyDHCP before we reach the failure point */
-	if ( elapsed > PROXYDHCP_MAX_TIMEOUT ) {
+	if ( elapsed > DHCP_REQ_PROXY_TIMEOUT_SEC * TICKS_PER_SEC ) {
 		dhcp_finished ( dhcp, 0 );
 		return;
 	}
@@ -685,7 +722,8 @@ static struct dhcp_session_state dhcp_state_proxy = {
 	.rx			= dhcp_proxy_rx,
 	.expired		= dhcp_proxy_expired,
 	.tx_msgtype		= DHCPREQUEST,
-	.apply_min_timeout	= 0,
+	.min_timeout_sec	= DHCP_PROXY_START_TIMEOUT_SEC,
+	.max_timeout_sec	= DHCP_PROXY_END_TIMEOUT_SEC,
 };
 
 /**
@@ -753,19 +791,24 @@ static int dhcp_pxebs_accept ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_pxebs_rx ( struct dhcp_session *dhcp,
 			    struct dhcp_packet *dhcppkt,
 			    struct sockaddr_in *peer, uint8_t msgtype,
-			    struct in_addr server_id ) {
+			    struct in_addr server_id,
+			    struct in_addr pseudo_id ) {
 	struct dhcp_pxe_boot_menu_item menu_item = { 0, 0 };
 	int rc;
 
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
 
 	/* Identify boot menu item */
 	dhcppkt_fetch ( dhcppkt, DHCP_PXE_BOOT_MENU_ITEM,
@@ -782,8 +825,7 @@ static void dhcp_pxebs_rx ( struct dhcp_session *dhcp,
 		return;
 	if ( menu_item.type != dhcp->pxe_type )
 		return;
-	if ( ! dhcp_pxebs_accept ( dhcp, ( server_id.s_addr ?
-					   server_id : peer->sin_addr ) ) )
+	if ( ! dhcp_pxebs_accept ( dhcp, pseudo_id ) )
 		return;
 
 	/* Register settings */
@@ -810,7 +852,7 @@ static void dhcp_pxebs_expired ( struct dhcp_session *dhcp ) {
 	/* Give up waiting before we reach the failure point, and fail
 	 * over to the next server in the attempt list
 	 */
-	if ( elapsed > PXEBS_MAX_TIMEOUT ) {
+	if ( elapsed > PXEBS_MAX_TIMEOUT_SEC * TICKS_PER_SEC ) {
 		dhcp->pxe_attempt++;
 		if ( dhcp->pxe_attempt->s_addr ) {
 			dhcp_set_state ( dhcp, &dhcp_state_pxebs );
@@ -832,7 +874,8 @@ static struct dhcp_session_state dhcp_state_pxebs = {
 	.rx			= dhcp_pxebs_rx,
 	.expired		= dhcp_pxebs_expired,
 	.tx_msgtype		= DHCPREQUEST,
-	.apply_min_timeout	= 1,
+	.min_timeout_sec	= PXEBS_START_TIMEOUT_SEC,
+	.max_timeout_sec	= PXEBS_END_TIMEOUT_SEC,
 };
 
 /****************************************************************************
@@ -1114,6 +1157,7 @@ static int dhcp_deliver ( struct dhcp_session *dhcp,
 	struct dhcphdr *dhcphdr;
 	uint8_t msgtype = 0;
 	struct in_addr server_id = { 0 };
+	struct in_addr pseudo_id;
 	int rc = 0;
 
 	/* Sanity checks */
@@ -1148,6 +1192,13 @@ static int dhcp_deliver ( struct dhcp_session *dhcp,
 	dhcppkt_fetch ( dhcppkt, DHCP_SERVER_IDENTIFIER,
 			&server_id, sizeof ( server_id ) );
 
+	/* Identify server pseudo-ID */
+	pseudo_id = server_id;
+	if ( ! pseudo_id.s_addr )
+		pseudo_id = dhcppkt->dhcphdr->siaddr;
+	if ( ! pseudo_id.s_addr )
+		pseudo_id = peer->sin_addr;
+
 	/* Check for matching transaction ID */
 	if ( dhcphdr->xid != dhcp->xid ) {
 		DBGC ( dhcp, "DHCP %p %s from %s:%d has bad transaction "
@@ -1170,7 +1221,7 @@ static int dhcp_deliver ( struct dhcp_session *dhcp,
 	}
 
 	/* Handle packet based on current state */
-	dhcp->state->rx ( dhcp, dhcppkt, peer, msgtype, server_id );
+	dhcp->state->rx ( dhcp, dhcppkt, peer, msgtype, server_id, pseudo_id );
 
  err_chaddr:
  err_xid:
diff --git a/roms/ipxe/src/net/udp/dhcpv6.c b/roms/ipxe/src/net/udp/dhcpv6.c
index f7736d08e..a63543775 100644
--- a/roms/ipxe/src/net/udp/dhcpv6.c
+++ b/roms/ipxe/src/net/udp/dhcpv6.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/net/udp/dns.c b/roms/ipxe/src/net/udp/dns.c
index fffe6e697..2d77477f6 100644
--- a/roms/ipxe/src/net/udp/dns.c
+++ b/roms/ipxe/src/net/udp/dns.c
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/udp/slam.c b/roms/ipxe/src/net/udp/slam.c
index 3cb492d73..8b26bfb3c 100644
--- a/roms/ipxe/src/net/udp/slam.c
+++ b/roms/ipxe/src/net/udp/slam.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/net/udp/syslog.c b/roms/ipxe/src/net/udp/syslog.c
index d65d19ab8..b6eee6036 100644
--- a/roms/ipxe/src/net/udp/syslog.c
+++ b/roms/ipxe/src/net/udp/syslog.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/net/udp/tftp.c b/roms/ipxe/src/net/udp/tftp.c
index ee827ae3d..953bcb46a 100644
--- a/roms/ipxe/src/net/udp/tftp.c
+++ b/roms/ipxe/src/net/udp/tftp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -149,8 +153,6 @@ enum {
 	TFTP_FL_RRQ_MULTICAST = 0x0004,
 	/** Perform MTFTP recovery on timeout */
 	TFTP_FL_MTFTP_RECOVERY = 0x0008,
-	/** Only get filesize and then abort the transfer */
-	TFTP_FL_SIZEONLY = 0x0010,
 };
 
 /** Maximum number of MTFTP open requests before falling back to TFTP */
@@ -759,14 +761,6 @@ static int tftp_rx_oack ( struct tftp_request *tftp, void *buf, size_t len ) {
 			goto done;
 	}
 
-	/* Abort request if only trying to determine file size */
-	if ( tftp->flags & TFTP_FL_SIZEONLY ) {
-		rc = 0;
-		tftp_send_error ( tftp, 0, "TFTP Aborted" );
-		tftp_done ( tftp, rc );
-		return rc;
-	}
-
 	/* Request next data block */
 	tftp_send_packet ( tftp );
 
@@ -794,13 +788,6 @@ static int tftp_rx_data ( struct tftp_request *tftp,
 	size_t data_len;
 	int rc;
 
-	if ( tftp->flags & TFTP_FL_SIZEONLY ) {
-		/* If we get here then server doesn't support SIZE option */
-		rc = -ENOTSUP;
-		tftp_send_error ( tftp, 0, "TFTP Aborted" );
-		goto done;
-	}
-
 	/* Sanity check */
 	if ( iob_len ( iobuf ) < sizeof ( *data ) ) {
 		DBGC ( tftp, "TFTP %p received underlength DATA packet "
@@ -1036,10 +1023,25 @@ static size_t tftp_xfer_window ( struct tftp_request *tftp ) {
 	return tftp->blksize;
 }
 
+/**
+ * Terminate download
+ *
+ * @v tftp		TFTP connection
+ * @v rc		Reason for close
+ */
+static void tftp_close ( struct tftp_request *tftp, int rc ) {
+
+	/* Abort download */
+	tftp_send_error ( tftp, 0, "TFTP Aborted" );
+
+	/* Close TFTP request */
+	tftp_done ( tftp, rc );
+}
+
 /** TFTP data transfer interface operations */
 static struct interface_operation tftp_xfer_operations[] = {
 	INTF_OP ( xfer_window, struct tftp_request *, tftp_xfer_window ),
-	INTF_OP ( intf_close, struct tftp_request *, tftp_done ),
+	INTF_OP ( intf_close, struct tftp_request *, tftp_close ),
 };
 
 /** TFTP data transfer interface descriptor */
@@ -1126,26 +1128,6 @@ struct uri_opener tftp_uri_opener __uri_opener = {
 };
 
 /**
- * Initiate TFTP-size request
- *
- * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @ret rc		Return status code
- */
-static int tftpsize_open ( struct interface *xfer, struct uri *uri ) {
-	return tftp_core_open ( xfer, uri, TFTP_PORT, NULL,
-				( TFTP_FL_RRQ_SIZES |
-				  TFTP_FL_SIZEONLY ) );
-
-}
-
-/** TFTP URI opener */
-struct uri_opener tftpsize_uri_opener __uri_opener = {
-	.scheme	= "tftpsize",
-	.open	= tftpsize_open,
-};
-
-/**
  * Initiate TFTM download
  *
  * @v xfer		Data transfer interface
diff --git a/roms/ipxe/src/net/validator.c b/roms/ipxe/src/net/validator.c
index 74d70e312..db968398a 100644
--- a/roms/ipxe/src/net/validator.c
+++ b/roms/ipxe/src/net/validator.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
@@ -79,7 +83,7 @@ static void validator_free ( struct refcnt *refcnt ) {
 	DBGC2 ( validator, "VALIDATOR %p freed\n", validator );
 	x509_chain_put ( validator->chain );
 	ocsp_put ( validator->ocsp );
-	xferbuf_done ( &validator->buffer );
+	xferbuf_free ( &validator->buffer );
 	free ( validator );
 }
 
@@ -250,7 +254,8 @@ static int validator_start_download ( struct validator *validator,
 	/* Generate URI string */
 	len = snprintf ( uri_string, uri_string_len, "%s/%08x.der?subject=",
 			 crosscert, crc );
-	base64_encode ( issuer->data, issuer->len, ( uri_string + len ) );
+	base64_encode ( issuer->data, issuer->len, ( uri_string + len ),
+			( uri_string_len - len ) );
 	DBGC ( validator, "VALIDATOR %p downloading cross-signed certificate "
 	       "from %s\n", validator, uri_string );
 
@@ -387,7 +392,7 @@ static void validator_xfer_close ( struct validator *validator, int rc ) {
 		goto err_append;
 
 	/* Free downloaded data */
-	xferbuf_done ( &validator->buffer );
+	xferbuf_free ( &validator->buffer );
 
 	/* Resume validation process */
 	process_add ( &validator->process );
@@ -552,6 +557,7 @@ int create_validator ( struct interface *job, struct x509_chain *chain ) {
 	process_init ( &validator->process, &validator_process_desc,
 		       &validator->refcnt );
 	validator->chain = x509_chain_get ( chain );
+	xferbuf_malloc_init ( &validator->buffer );
 
 	/* Attach parent interface, mortalise self, and return */
 	intf_plug_plug ( &validator->job, job );
diff --git a/roms/ipxe/src/net/vlan.c b/roms/ipxe/src/net/vlan.c
index b4ddde42d..f515c2dc9 100644
--- a/roms/ipxe/src/net/vlan.c
+++ b/roms/ipxe/src/net/vlan.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -385,6 +389,10 @@ int vlan_create ( struct net_device *trunk, unsigned int tag,
 	snprintf ( netdev->name, sizeof ( netdev->name ), "%s-%d",
 		   trunk->name, vlan->tag );
 
+	/* Mark device as not supporting interrupts, if applicable */
+	if ( ! netdev_irq_supported ( trunk ) )
+		netdev->state |= NETDEV_IRQ_UNSUPPORTED;
+
 	/* Register VLAN device */
 	if ( ( rc = register_netdev ( netdev ) ) != 0 ) {
 		DBGC ( netdev, "VLAN %s could not register: %s\n",
diff --git a/roms/ipxe/src/tests/aes_cbc_test.c b/roms/ipxe/src/tests/aes_cbc_test.c
deleted file mode 100644
index 4ae3a92e5..000000000
--- a/roms/ipxe/src/tests/aes_cbc_test.c
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-/** @file
- *
- * AES-in-CBC-mode tests
- *
- * These test vectors are provided by NIST as part of the
- * Cryptographic Toolkit Examples, downloadable from:
- *
- *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_CBC.pdf
- *
- */
-
-/* Forcibly enable assertions */
-#undef NDEBUG
-
-#include <assert.h>
-#include <string.h>
-#include <ipxe/aes.h>
-#include <ipxe/test.h>
-#include "cbc_test.h"
-
-/** Define inline key */
-#define KEY(...) { __VA_ARGS__ }
-
-/** Define inline initialisation vector */
-#define IV(...) { __VA_ARGS__ }
-
-/** Define inline plaintext data */
-#define PLAINTEXT(...) { __VA_ARGS__ }
-
-/** Define inline ciphertext data */
-#define CIPHERTEXT(...) { __VA_ARGS__ }
-
-/** An AES-in-CBC-mode test */
-struct aes_cbc_test {
-	/** Key */
-	const void *key;
-	/** Length of key */
-	size_t key_len;
-	/** Initialisation vector */
-	const void *iv;
-	/** Length of initialisation vector */
-	size_t iv_len;
-	/** Plaintext */
-	const void *plaintext;
-	/** Length of plaintext */
-	size_t plaintext_len;
-	/** Ciphertext */
-	const void *ciphertext;
-	/** Length of ciphertext */
-	size_t ciphertext_len;
-};
-
-/**
- * Define an AES-in-CBC-mode test
- *
- * @v name		Test name
- * @v key_array		Key
- * @v iv_array		Initialisation vector
- * @v plaintext_array	Plaintext
- * @v ciphertext_array	Ciphertext
- * @ret test		AES-in-CBC-mode test
- */
-#define AES_CBC_TEST( name, key_array, iv_array, plaintext_array,	\
-		      ciphertext_array )				\
-	static const uint8_t name ## _key [] = key_array;		\
-	static const uint8_t name ## _iv [] = iv_array;			\
-	static const uint8_t name ## _plaintext [] = plaintext_array;	\
-	static const uint8_t name ## _ciphertext [] = ciphertext_array;	\
-	static struct aes_cbc_test name = {				\
-		.key = name ## _key,					\
-		.key_len = sizeof ( name ## _key ),			\
-		.iv = name ## _iv,					\
-		.iv_len = sizeof ( name ## _iv ),			\
-		.plaintext = name ## _plaintext,			\
-		.plaintext_len = sizeof ( name ## _plaintext ),		\
-		.ciphertext = name ## _ciphertext,			\
-		.ciphertext_len = sizeof ( name ## _ciphertext ),	\
-	}
-
-/**
- * Report AES-in-CBC-mode
- *
- * @v state		HMAC_DRBG internal state
- * @v test		Instantiation test
- */
-#define aes_cbc_ok( test ) do {						\
-	struct cipher_algorithm *cipher = &aes_cbc_algorithm;		\
-									\
-	assert ( (test)->iv_len == cipher->blocksize );			\
-	assert ( (test)->plaintext_len == (test)->ciphertext_len );	\
-	cbc_encrypt_ok ( cipher, (test)->key, (test)->key_len,		\
-			 (test)->iv, (test)->plaintext,			\
-			 (test)->ciphertext, (test)->plaintext_len );	\
-	cbc_decrypt_ok ( cipher, (test)->key, (test)->key_len,		\
-			 (test)->iv, (test)->ciphertext,		\
-			 (test)->plaintext, (test)->ciphertext_len );	\
-	} while ( 0 )
-
-/** CBC_AES128 */
-AES_CBC_TEST ( test_128,
-	KEY ( 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15,
-	      0x88, 0x09, 0xcf, 0x4f, 0x3c ),
-	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
-	     0x0b, 0x0c, 0x0d, 0x0e, 0x0f ),
-	PLAINTEXT ( 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
-		    0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
-		    0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,
-		    0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,
-		    0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,
-		    0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,
-		    0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17,
-		    0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 ),
-	CIPHERTEXT ( 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46,
-		     0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d,
-		     0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee,
-		     0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2,
-		     0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b,
-		     0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16,
-		     0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09,
-		     0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 ) );
-
-/** CBC_AES256 */
-AES_CBC_TEST ( test_256,
-	KEY ( 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae,
-	      0xf0, 0x85, 0x7d, 0x77, 0x81, 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61,
-	      0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 ),
-	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
-	     0x0b, 0x0c, 0x0d, 0x0e, 0x0f ),
-	PLAINTEXT ( 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
-		    0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
-		    0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,
-		    0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,
-		    0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,
-		    0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,
-		    0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17,
-		    0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 ),
-	CIPHERTEXT ( 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba,
-		     0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6,
-		     0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d,
-		     0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d,
-		     0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf,
-		     0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61,
-		     0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc,
-		     0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b ) );
-
-/**
- * Perform AES-in-CBC-mode self-test
- *
- */
-static void aes_cbc_test_exec ( void ) {
-	struct cipher_algorithm *cipher = &aes_cbc_algorithm;
-
-	/* Correctness tests */
-	aes_cbc_ok ( &test_128 );
-	aes_cbc_ok ( &test_256 );
-
-	/* Speed tests */
-	DBG ( "AES128 encryption required %ld cycles per byte\n",
-	      cbc_cost_encrypt ( cipher, test_128.key_len ) );
-	DBG ( "AES128 decryption required %ld cycles per byte\n",
-	      cbc_cost_decrypt ( cipher, test_128.key_len ) );
-	DBG ( "AES256 encryption required %ld cycles per byte\n",
-	      cbc_cost_encrypt ( cipher, test_256.key_len ) );
-	DBG ( "AES256 decryption required %ld cycles per byte\n",
-	      cbc_cost_decrypt ( cipher, test_256.key_len ) );
-}
-
-/** AES-in-CBC-mode self-test */
-struct self_test aes_cbc_test __self_test = {
-	.name = "aes_cbc",
-	.exec = aes_cbc_test_exec,
-};
diff --git a/roms/ipxe/src/tests/aes_test.c b/roms/ipxe/src/tests/aes_test.c
new file mode 100644
index 000000000..ad66c734c
--- /dev/null
+++ b/roms/ipxe/src/tests/aes_test.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * AES tests
+ *
+ * These test vectors are provided by NIST as part of the
+ * Cryptographic Toolkit Examples, downloadable from:
+ *
+ *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_Core_All.pdf
+ *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_ECB.pdf
+ *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_CBC.pdf
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <assert.h>
+#include <string.h>
+#include <ipxe/aes.h>
+#include <ipxe/test.h>
+#include "cipher_test.h"
+
+/** Key used for NIST 128-bit test vectors */
+#define AES_KEY_NIST_128						\
+	KEY ( 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab,	\
+	      0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c )
+
+/** Key used for NIST 192-bit test vectors */
+#define AES_KEY_NIST_192						\
+	KEY ( 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8,	\
+	      0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, 0x62, 0xf8,	\
+	      0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b )
+
+/** Key used for NIST 256-bit test vectors */
+#define AES_KEY_NIST_256						\
+	KEY ( 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b,	\
+	      0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, 0x1f, 0x35,	\
+	      0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10,	\
+	      0xa3, 0x09, 0x14, 0xdf, 0xf4 )
+
+/** Dummy initialisation vector used for NIST ECB-mode test vectors */
+#define AES_IV_NIST_DUMMY						\
+	IV ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	\
+	     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 )
+
+/** Initialisation vector used for NIST CBC-mode test vectors */
+#define AES_IV_NIST_CBC							\
+	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,	\
+	     0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f )
+
+/** Plaintext used for NIST test vectors */
+#define AES_PLAINTEXT_NIST						\
+	PLAINTEXT ( 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,	\
+		    0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,	\
+		    0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,	\
+		    0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,	\
+		    0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,	\
+		    0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,	\
+		    0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17,	\
+		    0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 )
+
+/** AES-128-ECB (same test as AES-128-Core) */
+CIPHER_TEST ( aes_128_ecb, &aes_ecb_algorithm,
+	AES_KEY_NIST_128, AES_IV_NIST_DUMMY, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60,
+		     0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97,
+		     0xf5, 0xd3, 0xd5, 0x85, 0x03, 0xb9, 0x69, 0x9d,
+		     0xe7, 0x85, 0x89, 0x5a, 0x96, 0xfd, 0xba, 0xaf,
+		     0x43, 0xb1, 0xcd, 0x7f, 0x59, 0x8e, 0xce, 0x23,
+		     0x88, 0x1b, 0x00, 0xe3, 0xed, 0x03, 0x06, 0x88,
+		     0x7b, 0x0c, 0x78, 0x5e, 0x27, 0xe8, 0xad, 0x3f,
+		     0x82, 0x23, 0x20, 0x71, 0x04, 0x72, 0x5d, 0xd4 ) );
+
+/** AES-128-CBC */
+CIPHER_TEST ( aes_128_cbc, &aes_cbc_algorithm,
+	AES_KEY_NIST_128, AES_IV_NIST_CBC, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46,
+		     0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d,
+		     0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee,
+		     0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2,
+		     0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b,
+		     0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16,
+		     0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09,
+		     0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 ) );
+
+/** AES-192-ECB (same test as AES-192-Core) */
+CIPHER_TEST ( aes_192_ecb, &aes_ecb_algorithm,
+	AES_KEY_NIST_192, AES_IV_NIST_DUMMY, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f,
+		     0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc,
+		     0x97, 0x41, 0x04, 0x84, 0x6d, 0x0a, 0xd3, 0xad,
+		     0x77, 0x34, 0xec, 0xb3, 0xec, 0xee, 0x4e, 0xef,
+		     0xef, 0x7a, 0xfd, 0x22, 0x70, 0xe2, 0xe6, 0x0a,
+		     0xdc, 0xe0, 0xba, 0x2f, 0xac, 0xe6, 0x44, 0x4e,
+		     0x9a, 0x4b, 0x41, 0xba, 0x73, 0x8d, 0x6c, 0x72,
+		     0xfb, 0x16, 0x69, 0x16, 0x03, 0xc1, 0x8e, 0x0e ) );
+
+/** AES-192-CBC */
+CIPHER_TEST ( aes_192_cbc, &aes_cbc_algorithm,
+	AES_KEY_NIST_192, AES_IV_NIST_CBC, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d,
+		     0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8,
+		     0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4,
+		     0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a,
+		     0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0,
+		     0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0,
+		     0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81,
+		     0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd ) );
+
+/** AES-256-ECB (same test as AES-256-Core) */
+CIPHER_TEST ( aes_256_ecb, &aes_ecb_algorithm,
+	AES_KEY_NIST_256, AES_IV_NIST_DUMMY, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c,
+		     0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8,
+		     0x59, 0x1c, 0xcb, 0x10, 0xd4, 0x10, 0xed, 0x26,
+		     0xdc, 0x5b, 0xa7, 0x4a, 0x31, 0x36, 0x28, 0x70,
+		     0xb6, 0xed, 0x21, 0xb9, 0x9c, 0xa6, 0xf4, 0xf9,
+		     0xf1, 0x53, 0xe7, 0xb1, 0xbe, 0xaf, 0xed, 0x1d,
+		     0x23, 0x30, 0x4b, 0x7a, 0x39, 0xf9, 0xf3, 0xff,
+		     0x06, 0x7d, 0x8d, 0x8f, 0x9e, 0x24, 0xec, 0xc7 ) );
+
+/** AES-256-CBC */
+CIPHER_TEST ( aes_256_cbc, &aes_cbc_algorithm,
+	AES_KEY_NIST_256, AES_IV_NIST_CBC, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba,
+		     0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6,
+		     0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d,
+		     0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d,
+		     0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf,
+		     0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61,
+		     0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc,
+		     0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b ) );
+
+/**
+ * Perform AES self-test
+ *
+ */
+static void aes_test_exec ( void ) {
+	struct cipher_algorithm *ecb = &aes_ecb_algorithm;
+	struct cipher_algorithm *cbc = &aes_cbc_algorithm;
+	unsigned int keylen;
+
+	/* Correctness tests */
+	cipher_ok ( &aes_128_ecb );
+	cipher_ok ( &aes_128_cbc );
+	cipher_ok ( &aes_192_ecb );
+	cipher_ok ( &aes_192_cbc );
+	cipher_ok ( &aes_256_ecb );
+	cipher_ok ( &aes_256_cbc );
+
+	/* Speed tests */
+	for ( keylen = 128 ; keylen <= 256 ; keylen += 64 ) {
+		DBG ( "AES-%d-ECB encryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_encrypt ( ecb, ( keylen / 8 ) ) );
+		DBG ( "AES-%d-ECB decryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_decrypt ( ecb, ( keylen / 8 ) ) );
+		DBG ( "AES-%d-CBC encryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_encrypt ( cbc, ( keylen / 8 ) ) );
+		DBG ( "AES-%d-CBC decryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_decrypt ( cbc, ( keylen / 8 ) ) );
+	}
+}
+
+/** AES self-test */
+struct self_test aes_test __self_test = {
+	.name = "aes",
+	.exec = aes_test_exec,
+};
diff --git a/roms/ipxe/src/tests/base16_test.c b/roms/ipxe/src/tests/base16_test.c
index 9b047b74c..46884aef7 100644
--- a/roms/ipxe/src/tests/base16_test.c
+++ b/roms/ipxe/src/tests/base16_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -73,30 +77,42 @@ BASE16 ( random_test,
  * Report a base16 encoding test result
  *
  * @v test		Base16 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base16_encode_ok( test ) do {					\
-	size_t len = base16_encoded_len ( (test)->len );		\
-	char buf[ len + 1 /* NUL */ ];					\
-	ok ( len == strlen ( (test)->encoded ) );			\
-	base16_encode ( (test)->data, (test)->len, buf );		\
-	ok ( strcmp ( (test)->encoded, buf ) == 0 );			\
-	} while ( 0 )
+static void base16_encode_okx ( struct base16_test *test, const char *file,
+				unsigned int line ) {
+	size_t len = base16_encoded_len ( test->len );
+	char buf[ len + 1 /* NUL */ ];
+	size_t check_len;
+
+	okx ( len == strlen ( test->encoded ), file, line );
+	check_len = base16_encode ( test->data, test->len, buf, sizeof ( buf ));
+	okx ( check_len == len, file, line );
+	okx ( strcmp ( test->encoded, buf ) == 0, file, line );
+}
+#define base16_encode_ok( test ) base16_encode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Report a base16 decoding test result
  *
  * @v test		Base16 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base16_decode_ok( test ) do {					\
-	size_t max_len = base16_decoded_max_len ( (test)->encoded );	\
-	uint8_t buf[max_len];						\
-	int len;							\
-	len = base16_decode ( (test)->encoded, buf );			\
-	ok ( len >= 0 );						\
-	ok ( ( size_t ) len <= max_len );				\
-	ok ( ( size_t ) len == (test)->len );				\
-	ok ( memcmp ( (test)->data, buf, len ) == 0 );			\
-	} while ( 0 )
+static void base16_decode_okx ( struct base16_test *test, const char *file,
+				unsigned int line ) {
+	size_t max_len = base16_decoded_max_len ( test->encoded );
+	uint8_t buf[max_len];
+	int len;
+
+	len = base16_decode ( test->encoded, buf, sizeof ( buf ) );
+	okx ( len >= 0, file, line );
+	okx ( ( size_t ) len <= max_len, file, line );
+	okx ( ( size_t ) len == test->len, file, line );
+	okx ( memcmp ( test->data, buf, len ) == 0, file, line );
+}
+#define base16_decode_ok( test ) base16_decode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Perform Base16 self-tests
diff --git a/roms/ipxe/src/tests/base64_test.c b/roms/ipxe/src/tests/base64_test.c
index c088298ca..0fc595d90 100644
--- a/roms/ipxe/src/tests/base64_test.c
+++ b/roms/ipxe/src/tests/base64_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -76,30 +80,42 @@ BASE64 ( random_test,
  * Report a base64 encoding test result
  *
  * @v test		Base64 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base64_encode_ok( test ) do {					\
-	size_t len = base64_encoded_len ( (test)->len );		\
-	char buf[ len + 1 /* NUL */ ];					\
-	ok ( len == strlen ( (test)->encoded ) );			\
-	base64_encode ( (test)->data, (test)->len, buf );		\
-	ok ( strcmp ( (test)->encoded, buf ) == 0 );			\
-	} while ( 0 )
+static void base64_encode_okx ( struct base64_test *test, const char *file,
+				unsigned int line ) {
+	size_t len = base64_encoded_len ( test->len );
+	char buf[ len + 1 /* NUL */ ];
+	size_t check_len;
+
+	okx ( len == strlen ( test->encoded ), file, line );
+	check_len = base64_encode ( test->data, test->len, buf, sizeof ( buf ));
+	okx ( check_len == len, file, line );
+	okx ( strcmp ( test->encoded, buf ) == 0, file, line );
+}
+#define base64_encode_ok( test ) base64_encode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Report a base64 decoding test result
  *
  * @v test		Base64 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base64_decode_ok( test ) do {					\
-	size_t max_len = base64_decoded_max_len ( (test)->encoded );	\
-	uint8_t buf[max_len];						\
-	int len;							\
-	len = base64_decode ( (test)->encoded, buf );			\
-	ok ( len >= 0 );						\
-	ok ( ( size_t ) len <= max_len );				\
-	ok ( ( size_t ) len == (test)->len );				\
-	ok ( memcmp ( (test)->data, buf, len ) == 0 );			\
-	} while ( 0 )
+static void base64_decode_okx ( struct base64_test *test, const char *file,
+				unsigned int line ) {
+	size_t max_len = base64_decoded_max_len ( test->encoded );
+	uint8_t buf[max_len];
+	int len;
+
+	len = base64_decode ( test->encoded, buf, sizeof ( buf ) );
+	okx ( len >= 0, file, line );
+	okx ( ( size_t ) len <= max_len, file, line );
+	okx ( ( size_t ) len == test->len, file, line );
+	okx ( memcmp ( test->data, buf, len ) == 0, file, line );
+}
+#define base64_decode_ok( test ) base64_decode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Perform Base64 self-tests
diff --git a/roms/ipxe/src/tests/bigint_test.c b/roms/ipxe/src/tests/bigint_test.c
index 75a80622f..8d40c3188 100644
--- a/roms/ipxe/src/tests/bigint_test.c
+++ b/roms/ipxe/src/tests/bigint_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/bofm_test.c b/roms/ipxe/src/tests/bofm_test.c
index e430d12d4..829924887 100644
--- a/roms/ipxe/src/tests/bofm_test.c
+++ b/roms/ipxe/src/tests/bofm_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/tests/byteswap_test.c b/roms/ipxe/src/tests/byteswap_test.c
index a500218be..92bdb1d59 100644
--- a/roms/ipxe/src/tests/byteswap_test.c
+++ b/roms/ipxe/src/tests/byteswap_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/cbc_test.h b/roms/ipxe/src/tests/cbc_test.h
deleted file mode 100644
index ad9e6f341..000000000
--- a/roms/ipxe/src/tests/cbc_test.h
+++ /dev/null
@@ -1,57 +0,0 @@
-#ifndef _CBC_TEST_H
-#define _CBC_TEST_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stdint.h>
-#include <ipxe/crypto.h>
-#include <ipxe/test.h>
-
-extern int cbc_test_encrypt ( struct cipher_algorithm *cipher, const void *key,
-			      size_t key_len, const void *iv,
-			      const void *plaintext,
-			      const void *expected_ciphertext, size_t len );
-extern int cbc_test_decrypt ( struct cipher_algorithm *cipher, const void *key,
-			      size_t key_len, const void *iv,
-			      const void *ciphertext,
-			      const void *expected_plaintext, size_t len );
-extern unsigned long cbc_cost_encrypt ( struct cipher_algorithm *cipher,
-					size_t key_len );
-extern unsigned long cbc_cost_decrypt ( struct cipher_algorithm *cipher,
-					size_t key_len );
-
-/**
- * Report CBC encryption test result
- *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v plaintext			Plaintext data
- * @v expected_ciphertext	Expected ciphertext data
- * @v len			Length of data
- */
-#define cbc_encrypt_ok( cipher, key, key_len, iv, plaintext,		\
-			expected_ciphertext, len ) do {			\
-	ok ( cbc_test_encrypt ( cipher, key, key_len, iv, plaintext,	\
-				expected_ciphertext, len ) );		\
-	} while ( 0 )
-
-/**
- * Report CBC decryption test result
- *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v ciphertext		Ciphertext data
- * @v expected_plaintext	Expected plaintext data
- * @v len			Length of data
- */
-#define cbc_decrypt_ok( cipher, key, key_len, iv, ciphertext,		\
-			expected_plaintext, len ) do {			\
-	ok ( cbc_test_decrypt ( cipher, key, key_len, iv, ciphertext,	\
-				expected_plaintext, len ) );		\
-	} while ( 0 )
-
-#endif /* _CBC_TEST_H */
diff --git a/roms/ipxe/src/tests/cbc_test.c b/roms/ipxe/src/tests/cipher_test.c
index cb0f7bdea..800d6c138 100644
--- a/roms/ipxe/src/tests/cbc_test.c
+++ b/roms/ipxe/src/tests/cipher_test.c
@@ -15,13 +15,17 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
- * CBC self-tests
+ * Cipher self-tests
  *
  */
 
@@ -34,86 +38,90 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <assert.h>
 #include <ipxe/crypto.h>
 #include <ipxe/profile.h>
-#include "cbc_test.h"
+#include <ipxe/test.h>
+#include "cipher_test.h"
 
 /** Number of sample iterations for profiling */
 #define PROFILE_COUNT 16
 
 /**
- * Test CBC encryption
+ * Report a cipher encryption test result
  *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v plaintext			Plaintext data
- * @v expected_ciphertext	Expected ciphertext data
- * @v len			Length of data
- * @ret ok			Ciphertext is as expected
+ * @v test		Cipher test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-int cbc_test_encrypt ( struct cipher_algorithm *cipher, const void *key,
-		       size_t key_len, const void *iv, const void *plaintext,
-		       const void *expected_ciphertext, size_t len ) {
+void cipher_encrypt_okx ( struct cipher_test *test, const char *file,
+			  unsigned int line ) {
+	struct cipher_algorithm *cipher = test->cipher;
+	size_t len = test->len;
 	uint8_t ctx[cipher->ctxsize];
 	uint8_t ciphertext[len];
-	int rc;
 
 	/* Initialise cipher */
-	rc = cipher_setkey ( cipher, ctx, key, key_len );
-	assert ( rc == 0 );
-	cipher_setiv ( cipher, ctx, iv );
+	okx ( cipher_setkey ( cipher, ctx, test->key, test->key_len ) == 0,
+	      file, line );
+	cipher_setiv ( cipher, ctx, test->iv );
 
 	/* Perform encryption */
-	cipher_encrypt ( cipher, ctx, plaintext, ciphertext, len );
+	cipher_encrypt ( cipher, ctx, test->plaintext, ciphertext, len );
 
-	/* Verify result */
-	return ( memcmp ( ciphertext, expected_ciphertext, len ) == 0 );
+	/* Compare against expected ciphertext */
+	okx ( memcmp ( ciphertext, test->ciphertext, len ) == 0, file, line );
 }
 
 /**
- * Test CBC decryption
+ * Report a cipher decryption test result
  *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v ciphertext		Ciphertext data
- * @v expected_plaintext	Expected plaintext data
- * @v len			Length of data
- * @ret ok			Plaintext is as expected
+ * @v test		Cipher test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-int cbc_test_decrypt ( struct cipher_algorithm *cipher, const void *key,
-		       size_t key_len, const void *iv, const void *ciphertext,
-		       const void *expected_plaintext, size_t len ) {
+void cipher_decrypt_okx ( struct cipher_test *test, const char *file,
+			  unsigned int line ) {
+	struct cipher_algorithm *cipher = test->cipher;
+	size_t len = test->len;
 	uint8_t ctx[cipher->ctxsize];
 	uint8_t plaintext[len];
-	int rc;
 
 	/* Initialise cipher */
-	rc = cipher_setkey ( cipher, ctx, key, key_len );
-	assert ( rc == 0 );
-	cipher_setiv ( cipher, ctx, iv );
+	okx ( cipher_setkey ( cipher, ctx, test->key, test->key_len ) == 0,
+	      file, line );
+	cipher_setiv ( cipher, ctx, test->iv );
 
 	/* Perform encryption */
-	cipher_decrypt ( cipher, ctx, ciphertext, plaintext, len );
+	cipher_decrypt ( cipher, ctx, test->ciphertext, plaintext, len );
+
+	/* Compare against expected plaintext */
+	okx ( memcmp ( plaintext, test->plaintext, len ) == 0, file, line );
+}
+
+/**
+ * Report a cipher encryption and decryption test result
+ *
+ * @v test		Cipher test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+void cipher_okx ( struct cipher_test *test, const char *file,
+		  unsigned int line ) {
 
-	/* Verify result */
-	return ( memcmp ( plaintext, expected_plaintext, len ) == 0 );
+	cipher_encrypt_okx ( test, file, line );
+	cipher_decrypt_okx ( test, file, line );
 }
 
 /**
- * Calculate CBC encryption or decryption cost
+ * Calculate cipher encryption or decryption cost
  *
  * @v cipher			Cipher algorithm
  * @v key_len			Length of key
  * @v op			Encryption or decryption operation
  * @ret cost			Cost (in cycles per byte)
  */
-static unsigned long cbc_cost ( struct cipher_algorithm *cipher,
-				size_t key_len,
-				void ( * op ) ( struct cipher_algorithm *cipher,
-						void *ctx, const void *src,
-						void *dst, size_t len ) ) {
+static unsigned long
+cipher_cost ( struct cipher_algorithm *cipher, size_t key_len,
+	      void ( * op ) ( struct cipher_algorithm *cipher, void *ctx,
+			      const void *src, void *dst, size_t len ) ) {
 	static uint8_t random[8192]; /* Too large for stack */
 	uint8_t key[key_len];
 	uint8_t iv[cipher->blocksize];
@@ -153,25 +161,25 @@ static unsigned long cbc_cost ( struct cipher_algorithm *cipher,
 }
 
 /**
- * Calculate CBC encryption cost
+ * Calculate cipher encryption cost
  *
  * @v cipher			Cipher algorithm
  * @v key_len			Length of key
  * @ret cost			Cost (in cycles per byte)
  */
-unsigned long cbc_cost_encrypt ( struct cipher_algorithm *cipher,
-				 size_t key_len ) {
-	return cbc_cost ( cipher, key_len, cipher_encrypt );
+unsigned long cipher_cost_encrypt ( struct cipher_algorithm *cipher,
+				    size_t key_len ) {
+	return cipher_cost ( cipher, key_len, cipher_encrypt );
 }
 
 /**
- * Calculate CBC decryption cost
+ * Calculate cipher decryption cost
  *
  * @v cipher			Cipher algorithm
  * @v key_len			Length of key
  * @ret cost			Cost (in cycles per byte)
  */
-unsigned long cbc_cost_decrypt ( struct cipher_algorithm *cipher,
-				 size_t key_len ) {
-	return cbc_cost ( cipher, key_len, cipher_decrypt );
+unsigned long cipher_cost_decrypt ( struct cipher_algorithm *cipher,
+				    size_t key_len ) {
+	return cipher_cost ( cipher, key_len, cipher_decrypt );
 }
diff --git a/roms/ipxe/src/tests/cipher_test.h b/roms/ipxe/src/tests/cipher_test.h
new file mode 100644
index 000000000..d7c5aef8f
--- /dev/null
+++ b/roms/ipxe/src/tests/cipher_test.h
@@ -0,0 +1,111 @@
+#ifndef _CIPHER_TEST_H
+#define _CIPHER_TEST_H
+
+/** @file
+ *
+ * Cipher self-tests
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/crypto.h>
+#include <ipxe/test.h>
+
+/** A cipher test */
+struct cipher_test {
+	/** Cipher algorithm */
+	struct cipher_algorithm *cipher;
+	/** Key */
+	const void *key;
+	/** Length of key */
+	size_t key_len;
+	/** Initialisation vector */
+	const void *iv;
+	/** Length of initialisation vector */
+	size_t iv_len;
+	/** Plaintext */
+	const void *plaintext;
+	/** Ciphertext */
+	const void *ciphertext;
+	/** Length of text */
+	size_t len;
+};
+
+/** Define inline key */
+#define KEY(...) { __VA_ARGS__ }
+
+/** Define inline initialisation vector */
+#define IV(...) { __VA_ARGS__ }
+
+/** Define inline plaintext data */
+#define PLAINTEXT(...) { __VA_ARGS__ }
+
+/** Define inline ciphertext data */
+#define CIPHERTEXT(...) { __VA_ARGS__ }
+
+/**
+ * Define a cipher test
+ *
+ * @v name		Test name
+ * @v CIPHER		Cipher algorithm
+ * @v KEY		Key
+ * @v IV		Initialisation vector
+ * @v PLAINTEXT		Plaintext
+ * @v CIPHERTEXT	Ciphertext
+ * @ret test		Cipher test
+ */
+#define CIPHER_TEST( name, CIPHER, KEY, IV, PLAINTEXT, CIPHERTEXT )	\
+	static const uint8_t name ## _key [] = KEY;			\
+	static const uint8_t name ## _iv [] = IV;			\
+	static const uint8_t name ## _plaintext [] = PLAINTEXT;		\
+	static const uint8_t name ## _ciphertext			\
+		[ sizeof ( name ## _plaintext ) ] = CIPHERTEXT;		\
+	static struct cipher_test name = {				\
+		.cipher = CIPHER,					\
+		.key = name ## _key,					\
+		.key_len = sizeof ( name ## _key ),			\
+		.iv = name ## _iv,					\
+		.iv_len = sizeof ( name ## _iv ),			\
+		.plaintext = name ## _plaintext,			\
+		.ciphertext = name ## _ciphertext,			\
+		.len = sizeof ( name ## _plaintext ),			\
+	}
+
+extern void cipher_encrypt_okx ( struct cipher_test *test, const char *file,
+				 unsigned int line );
+extern void cipher_decrypt_okx ( struct cipher_test *test, const char *file,
+				 unsigned int line );
+extern void cipher_okx ( struct cipher_test *test, const char *file,
+			 unsigned int line );
+extern unsigned long cipher_cost_encrypt ( struct cipher_algorithm *cipher,
+					   size_t key_len );
+extern unsigned long cipher_cost_decrypt ( struct cipher_algorithm *cipher,
+					   size_t key_len );
+
+/**
+ * Report a cipher encryption test result
+ *
+ * @v test		Cipher test
+ */
+#define cipher_encrypt_ok( test ) \
+	cipher_encrypt_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report a cipher decryption test result
+ *
+ * @v test		Cipher test
+ */
+#define cipher_decrypt_ok( test ) \
+	cipher_decrypt_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report a cipher encryption and decryption test result
+ *
+ * @v test		Cipher test
+ */
+#define cipher_ok( test ) \
+	cipher_okx ( test, __FILE__, __LINE__ )
+
+#endif /* _CIPHER_TEST_H */
diff --git a/roms/ipxe/src/tests/cms_test.c b/roms/ipxe/src/tests/cms_test.c
index 8767504c0..b805a9974 100644
--- a/roms/ipxe/src/tests/cms_test.c
+++ b/roms/ipxe/src/tests/cms_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -1470,6 +1474,7 @@ struct self_test cms_test __self_test = {
 };
 
 /* Drag in algorithms required for tests */
+REQUIRING_SYMBOL ( cms_test );
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( md5 );
 REQUIRE_OBJECT ( sha1 );
diff --git a/roms/ipxe/src/tests/crc32_test.c b/roms/ipxe/src/tests/crc32_test.c
index 873f633a5..46cde0f7b 100644
--- a/roms/ipxe/src/tests/crc32_test.c
+++ b/roms/ipxe/src/tests/crc32_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/deflate_test.c b/roms/ipxe/src/tests/deflate_test.c
index 68c1aad96..20ff5b9a2 100644
--- a/roms/ipxe/src/tests/deflate_test.c
+++ b/roms/ipxe/src/tests/deflate_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/digest_test.c b/roms/ipxe/src/tests/digest_test.c
index 4df26c099..c3a128853 100644
--- a/roms/ipxe/src/tests/digest_test.c
+++ b/roms/ipxe/src/tests/digest_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -34,27 +38,47 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/profile.h>
 #include "digest_test.h"
 
+/** Maximum number of digest test fragments */
+#define NUM_DIGEST_TEST_FRAG 8
+
+/** A digest test fragment list */
+struct digest_test_fragments {
+	/** Fragment lengths */
+	size_t len[NUM_DIGEST_TEST_FRAG];
+};
+
+/** Digest test fragment lists */
+static struct digest_test_fragments digest_test_fragments[] = {
+	{ { 0, -1UL, } },
+	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
+	{ { 2, 0, 23, 4, 6, 1, 0 } },
+};
+
 /** Number of sample iterations for profiling */
 #define PROFILE_COUNT 16
 
 /**
- * Test digest algorithm
+ * Report a digest fragmented test result
  *
- * @v digest		Digest algorithm
- * @v fragments		Digest test fragment list, or NULL
- * @v data		Test data
- * @v len		Length of test data
- * @v expected		Expected digest value
- * @ret ok		Digest value is as expected
+ * @v test		Digest test
+ * @v fragments		Fragment list
+ * @v file		Test code file
+ * @v line		Test code line
  */
-int digest_test ( struct digest_algorithm *digest,
-		  struct digest_test_fragments *fragments,
-		  void *data, size_t len, void *expected ) {
+void digest_frag_okx ( struct digest_test *test,
+		       struct digest_test_fragments *fragments,
+		       const char *file, unsigned int line ) {
+	struct digest_algorithm *digest = test->digest;
 	uint8_t ctx[digest->ctxsize];
 	uint8_t out[digest->digestsize];
+	const void *data = test->data;
+	size_t len = test->len;
 	size_t frag_len = 0;
 	unsigned int i;
 
+	/* Sanity check */
+	okx ( test->expected_len == sizeof ( out ), file, line );
+
 	/* Initialise digest */
 	digest_init ( digest, ctx );
 
@@ -74,7 +98,28 @@ int digest_test ( struct digest_algorithm *digest,
 	digest_final ( digest, ctx, out );
 
 	/* Compare against expected output */
-	return ( memcmp ( expected, out, sizeof ( out ) ) == 0 );
+	okx ( memcmp ( test->expected, out, sizeof ( out ) ) == 0, file, line );
+}
+
+/**
+ * Report a digest test result
+ *
+ * @v test		Digest test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+void digest_okx ( struct digest_test *test, const char *file,
+		  unsigned int line ) {
+	unsigned int i;
+
+	/* Test with a single pass */
+	digest_frag_okx ( test, NULL, file, line );
+
+	/* Test with fragment lists */
+	for ( i = 0 ; i < ( sizeof ( digest_test_fragments ) /
+			    sizeof ( digest_test_fragments[0] ) ) ; i++ ) {
+		digest_frag_okx ( test, &digest_test_fragments[i], file, line );
+	}
 }
 
 /**
diff --git a/roms/ipxe/src/tests/digest_test.h b/roms/ipxe/src/tests/digest_test.h
index 49e06d1cb..abf1b834f 100644
--- a/roms/ipxe/src/tests/digest_test.h
+++ b/roms/ipxe/src/tests/digest_test.h
@@ -1,37 +1,115 @@
 #ifndef _DIGEST_TEST_H
 #define _DIGEST_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
 #include <ipxe/test.h>
 
-/** Maximum number of digest test fragments */
-#define NUM_DIGEST_TEST_FRAG 8
-
-/** A digest test fragment list */
-struct digest_test_fragments {
-	/** Fragment lengths */
-	size_t len[NUM_DIGEST_TEST_FRAG];
+/** A digest test */
+struct digest_test {
+	/** Digest algorithm */
+	struct digest_algorithm *digest;
+	/** Test data */
+	const void *data;
+	/** Length of test data */
+	size_t len;
+	/** Expected digest value */
+	const void *expected;
+	/** Expected digest length */
+	size_t expected_len;
 };
 
-extern int digest_test ( struct digest_algorithm *digest,
-			 struct digest_test_fragments *fragments,
-			 void *data, size_t len, void *expected );
-extern unsigned long digest_cost ( struct digest_algorithm *digest );
+/** Define inline test data */
+#define DATA(...) { __VA_ARGS__ }
+
+/** Define inline expected digest value */
+#define DIGEST(...) { __VA_ARGS__ }
+
+/**
+ * Define a digest test
+ *
+ * @v name		Test name
+ * @v DIGEST		Digest algorithm
+ * @v DATA		Test data
+ * @v EXPECTED		Expected digest value
+ * @ret test		Digest test
+ */
+#define DIGEST_TEST( name, DIGEST, DATA, EXPECTED )			\
+	static const uint8_t name ## _data[] = DATA;			\
+	static const uint8_t name ## _expected[] = EXPECTED;		\
+	static struct digest_test name = {				\
+		.digest = DIGEST,					\
+		.data = name ## _data,					\
+		.len = sizeof ( name ## _data ),			\
+		.expected = name ## _expected,				\
+		.expected_len = sizeof ( name ## _expected ),		\
+	};
+
+/** Standard test vector: empty data */
+#define DIGEST_EMPTY DATA()
+
+/** Standard test vector: NIST string "abc"
+ *
+ * The NIST Cryptographic Toolkit examples for all digest algorithms
+ * include a test vector which is the unterminated string
+ *
+ *   "abc"
+ */
+#define DIGEST_NIST_ABC							\
+	DATA ( 0x61, 0x62, 0x63 )
+
+/** Standard test vector: NIST string "abc...opq"
+ *
+ * The NIST Cryptographic Toolkit examples for all 32-bit digest
+ * algorithms (SHA-1 and the SHA-256 family) include a test vector
+ * which is the unterminated string
+ *
+ *   "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"
+ */
+#define DIGEST_NIST_ABC_OPQ						\
+	DATA ( 0x61, 0x62, 0x63, 0x64, 0x62, 0x63, 0x64, 0x65, 0x63,	\
+	       0x64, 0x65, 0x66, 0x64, 0x65, 0x66, 0x67, 0x65, 0x66,	\
+	       0x67, 0x68, 0x66, 0x67, 0x68, 0x69, 0x67, 0x68, 0x69,	\
+	       0x6a, 0x68, 0x69, 0x6a, 0x6b, 0x69, 0x6a, 0x6b, 0x6c,	\
+	       0x6a, 0x6b, 0x6c, 0x6d, 0x6b, 0x6c, 0x6d, 0x6e, 0x6c,	\
+	       0x6d, 0x6e, 0x6f, 0x6d, 0x6e, 0x6f, 0x70, 0x6e, 0x6f,	\
+	       0x70, 0x71 )
+
+/** Standard test vector: NIST string "abc...stu"
+ *
+ * The NIST Cryptographic Toolkit examples for all 64-bit digest
+ * algorithms (SHA-512 family) include a test vector which is the
+ * unterminated string
+ *
+ *   "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmn"
+ *   "hijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"
+ */
+#define DIGEST_NIST_ABC_STU						\
+	DATA ( 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x62,	\
+	       0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x63, 0x64,	\
+	       0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x64, 0x65, 0x66,	\
+	       0x67, 0x68, 0x69, 0x6a, 0x6b, 0x65, 0x66, 0x67, 0x68,	\
+	       0x69, 0x6a, 0x6b, 0x6c, 0x66, 0x67, 0x68, 0x69, 0x6a,	\
+	       0x6b, 0x6c, 0x6d, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c,	\
+	       0x6d, 0x6e, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e,	\
+	       0x6f, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70,	\
+	       0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x6b,	\
+	       0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x6c, 0x6d,	\
+	       0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x6d, 0x6e, 0x6f,	\
+	       0x70, 0x71, 0x72, 0x73, 0x74, 0x6e, 0x6f, 0x70, 0x71,	\
+	       0x72, 0x73, 0x74, 0x75 )
 
 /**
- * Report digest test result
+ * Report a digest test result
  *
- * @v digest		Digest algorithm
- * @v fragments		Digest test fragment list, or NULL
- * @v data		Test data
- * @v len		Length of test data
- * @v expected		Expected digest value
+ * @v test		Digest test
  */
-#define digest_ok( digest, fragments, data, len, expected ) do {	\
-	ok ( digest_test ( digest, fragments, data, len, expected ) );	\
-	} while ( 0 )
+#define digest_ok(test) digest_okx ( test, __FILE__, __LINE__ )
+
+extern void digest_okx ( struct digest_test *test, const char *file,
+			 unsigned int line );
+extern unsigned long digest_cost ( struct digest_algorithm *digest );
 
 #endif /* _DIGEST_TEST_H */
diff --git a/roms/ipxe/src/tests/dns_test.c b/roms/ipxe/src/tests/dns_test.c
index 52f5f19f2..f08e7810f 100644
--- a/roms/ipxe/src/tests/dns_test.c
+++ b/roms/ipxe/src/tests/dns_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/entropy_sample.c b/roms/ipxe/src/tests/entropy_sample.c
index 95a662e3e..b45648c11 100644
--- a/roms/ipxe/src/tests/entropy_sample.c
+++ b/roms/ipxe/src/tests/entropy_sample.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/hash_df_test.c b/roms/ipxe/src/tests/hash_df_test.c
index 74c8d0f4d..0b7d56ad7 100644
--- a/roms/ipxe/src/tests/hash_df_test.c
+++ b/roms/ipxe/src/tests/hash_df_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/hmac_drbg_test.c b/roms/ipxe/src/tests/hmac_drbg_test.c
index 8cbf1cc8b..ddf9db2c5 100644
--- a/roms/ipxe/src/tests/hmac_drbg_test.c
+++ b/roms/ipxe/src/tests/hmac_drbg_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/ipv4_test.c b/roms/ipxe/src/tests/ipv4_test.c
new file mode 100644
index 000000000..f84a8b81f
--- /dev/null
+++ b/roms/ipxe/src/tests/ipv4_test.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * IPv4 tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <string.h>
+#include <byteswap.h>
+#include <ipxe/in.h>
+#include <ipxe/test.h>
+
+/** Define inline IPv4 address */
+#define IPV4(a,b,c,d) \
+	htonl ( ( (a) << 24 ) | ( (b) << 16 ) | ( (c) << 8 ) | (d) )
+
+/**
+ * Report an inet_ntoa() test result
+ *
+ * @v addr		IPv4 address
+ * @v text		Expected textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void inet_ntoa_okx ( uint32_t addr, const char *text, const char *file,
+			    unsigned int line ) {
+	struct in_addr in = { .s_addr = addr };
+	char *actual;
+
+	/* Format address */
+	actual = inet_ntoa ( in );
+	DBG ( "inet_ntoa ( %d.%d.%d.%d ) = %s\n",
+	      ( ( ntohl ( addr ) >> 24 ) & 0xff ),
+	      ( ( ntohl ( addr ) >> 16 ) & 0xff ),
+	      ( ( ntohl ( addr ) >> 8 ) & 0xff ),
+	      ( ( ntohl ( addr ) >> 0 ) & 0xff ), actual );
+	okx ( strcmp ( actual, text ) == 0, file, line );
+}
+#define inet_ntoa_ok( addr, text ) \
+	inet_ntoa_okx ( addr, text, __FILE__, __LINE__ )
+
+/**
+ * Report an inet_aton() test result
+ *
+ * @v text		Textual representation
+ * @v addr		Expected IPv4 address
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void inet_aton_okx ( const char *text, uint32_t addr, const char *file,
+			    unsigned int line ) {
+	struct in_addr actual;
+
+	/* Parse address */
+	okx ( inet_aton ( text, &actual ) != 0, file, line );
+	DBG ( "inet_aton ( \"%s\" ) = %s\n", text, inet_ntoa ( actual ) );
+	okx ( actual.s_addr == addr, file, line );
+};
+#define inet_aton_ok( text, addr ) \
+	inet_aton_okx ( text, addr, __FILE__, __LINE__ )
+
+/**
+ * Report an inet_aton() failure test result
+ *
+ * @v text		Textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void inet_aton_fail_okx ( const char *text, const char *file,
+				 unsigned int line ) {
+	struct in_addr actual;
+
+	/* Attempt to parse address */
+	okx ( inet_aton ( text, &actual ) == 0, file, line );
+}
+#define inet_aton_fail_ok( text ) \
+	inet_aton_fail_okx ( text, __FILE__, __LINE__ )
+
+/**
+ * Perform IPv4 self-tests
+ *
+ */
+static void ipv4_test_exec ( void ) {
+
+	/* Address testing macros */
+	ok (   IN_IS_CLASSA ( IPV4 ( 10, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSB ( IPV4 ( 10, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSC ( IPV4 ( 10, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSA ( IPV4 ( 172, 16, 0, 1 ) ) );
+	ok (   IN_IS_CLASSB ( IPV4 ( 172, 16, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSC ( IPV4 ( 172, 16, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSA ( IPV4 ( 192, 168, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSB ( IPV4 ( 192, 168, 0, 1 ) ) );
+	ok (   IN_IS_CLASSC ( IPV4 ( 192, 168, 0, 1 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 127, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 8, 8, 8, 8 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 0, 0, 0, 0 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 223, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 240, 0, 0, 1 ) ) );
+	ok (   IN_IS_MULTICAST ( IPV4 ( 224, 0, 0, 1 ) ) );
+	ok (   IN_IS_MULTICAST ( IPV4 ( 231, 89, 0, 2 ) ) );
+	ok (   IN_IS_MULTICAST ( IPV4 ( 239, 6, 1, 17 ) ) );
+
+	/* inet_ntoa() tests */
+	inet_ntoa_ok ( IPV4 ( 127, 0, 0, 1 ), "127.0.0.1" );
+	inet_ntoa_ok ( IPV4 ( 0, 0, 0, 0 ), "0.0.0.0" );
+	inet_ntoa_ok ( IPV4 ( 255, 255, 255, 255 ), "255.255.255.255" );
+	inet_ntoa_ok ( IPV4 ( 212, 13, 204, 60 ), "212.13.204.60" );
+
+	/* inet_aton() tests */
+	inet_aton_ok ( "212.13.204.60", IPV4 ( 212, 13, 204, 60 ) );
+	inet_aton_ok ( "127.0.0.1", IPV4 ( 127, 0, 0, 1 ) );
+
+	/* inet_aton() failure tests */
+	inet_aton_fail_ok ( "256.0.0.1" ); /* Byte out of range */
+	inet_aton_fail_ok ( "212.13.204.60.1" ); /* Too long */
+	inet_aton_fail_ok ( "127.0.0" ); /* Too short */
+	inet_aton_fail_ok ( "1.2.3.a" ); /* Invalid characters */
+	inet_aton_fail_ok ( "127.0..1" ); /* Missing bytes */
+}
+
+/** IPv4 self-test */
+struct self_test ipv4_test __self_test = {
+	.name = "ipv4",
+	.exec = ipv4_test_exec,
+};
diff --git a/roms/ipxe/src/tests/ipv6_test.c b/roms/ipxe/src/tests/ipv6_test.c
index e16fc7c3d..772eb1b82 100644
--- a/roms/ipxe/src/tests/ipv6_test.c
+++ b/roms/ipxe/src/tests/ipv6_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/linebuf_test.c b/roms/ipxe/src/tests/linebuf_test.c
index e06ac7d86..0dd486e9d 100644
--- a/roms/ipxe/src/tests/linebuf_test.c
+++ b/roms/ipxe/src/tests/linebuf_test.c
@@ -1,35 +1,320 @@
-#include <stdint.h>
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Line buffer self-tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <string.h>
-#include <stdio.h>
+#include <assert.h>
 #include <ipxe/linebuf.h>
+#include <ipxe/test.h>
 
-static const char data1[] = 
-"Hello world\r\n"
-"This is a reasonably nice set of lines\n"
-"with not many different terminators\r\n\r\n"
-"There should be exactly one blank line above\n"
-"and this line should never appear at all since it has no terminator";
+/** Define inline raw data */
+#define DATA(...) { __VA_ARGS__ }
 
-void linebuf_test ( void ) {
-	struct line_buffer linebuf;
-	const char *data = data1;
-	size_t len = ( sizeof ( data1 ) - 1 /* be mean; strip the NUL */ );
-	ssize_t frag_len;
-	char *line;
-
-	memset ( &linebuf, 0, sizeof ( linebuf ) );
-	while ( len ) {
-		frag_len = line_buffer ( &linebuf, data, len );
-		if ( frag_len < 0 ) {
-			printf ( "line_buffer() failed: %s\n",
-				 strerror ( frag_len ) );
+/** Define inline lines */
+#define LINES(...) { __VA_ARGS__ }
+
+/** A line buffer test */
+struct linebuf_test {
+	/** Raw data */
+	const void *data;
+	/** Length of raw data */
+	size_t len;
+	/** Expected sequence of lines */
+	const char **lines;
+	/** Number of expected lines */
+	unsigned int count;
+};
+
+/** Line buffer test expected failure indicator */
+static const char linebuf_failure[1];
+
+/**
+ * Define a line buffer test
+ *
+ * @v name		Test name
+ * @v DATA		Raw data
+ * @v LINES		Expected sequence of lines
+ * @ret test		Line buffer test
+ */
+#define LINEBUF_TEST( name, DATA, LINES )				\
+	static const char name ## _data[] = DATA;			\
+	static const char * name ## _lines[] = LINES;			\
+	static struct linebuf_test name = {				\
+		.data = name ## _data,					\
+		.len = ( sizeof ( name ## _data ) - 1 /* NUL */ ),	\
+		.lines = name ## _lines,				\
+		.count = ( sizeof ( name ## _lines ) /			\
+			   sizeof ( name ## _lines[0] ) ),		\
+	}
+
+/** Simple line buffer test */
+LINEBUF_TEST ( simple,
+	       ( "HTTP/1.1 200 OK\r\n"
+		 "Content-Length: 123\r\n"
+		 "Content-Type: text/plain\r\n"
+		 "\r\n" ),
+	       LINES ( "HTTP/1.1 200 OK",
+		       "Content-Length: 123",
+		       "Content-Type: text/plain",
+		       "" ) );
+
+/** Mixed line terminators */
+LINEBUF_TEST ( mixed,
+	       ( "LF only\n" "CRLF\r\n" "\n" "\n" "\r\n" "\r\n" "CR only\r" ),
+	       LINES ( "LF only", "CRLF", "", "", "", "",
+		       NULL /* \r should not be treated as a terminator */ ) );
+
+/** Split consumption: part 1 */
+LINEBUF_TEST ( split_1,
+	       ( "This line was" ),
+	       LINES ( NULL ) );
+
+/** Split consumption: part 2 */
+LINEBUF_TEST ( split_2,
+	       ( " split across" ),
+	       LINES ( NULL ) );
+
+/** Split consumption: part 3 */
+LINEBUF_TEST ( split_3,
+	       ( " multiple calls\r\nand so was this one\r" ),
+	       LINES ( "This line was split across multiple calls", NULL ) );
+
+/** Split consumption: part 4 */
+LINEBUF_TEST ( split_4,
+	       ( "\nbut not this one\r\n" ),
+	       LINES ( "and so was this one", "but not this one" ) );
+
+/** Split consumption: part 5 */
+LINEBUF_TEST ( split_5,
+	       ( "" ),
+	       LINES ( NULL ) );
+
+/** Split consumption: part 6 */
+LINEBUF_TEST ( split_6,
+	       ( "This line came after a zero-length call\r\n" ),
+	       LINES ( "This line came after a zero-length call" ) );
+
+/** Embedded NULs */
+LINEBUF_TEST ( embedded_nuls,
+	       ( "This\r\ntest\r\nincludes\r\n\r\nsome\0binary\0data\r\n" ),
+	       LINES ( "This", "test", "includes", "", linebuf_failure ) );
+
+/**
+ * Report line buffer initialisation test result
+ *
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_init_okx ( struct line_buffer *linebuf,
+				const char *file, unsigned int line ) {
+
+	/* Initialise line buffer */
+	memset ( linebuf, 0, sizeof ( *linebuf ) );
+	okx ( buffered_line ( linebuf ) == NULL, file, line );
+}
+#define linebuf_init_ok( linebuf ) \
+	linebuf_init_okx ( linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer consumption test result
+ *
+ * @v test		Line buffer test
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_consume_okx ( struct linebuf_test *test,
+				  struct line_buffer *linebuf,
+				  const char *file, unsigned int line ) {
+	const char *data = test->data;
+	size_t remaining = test->len;
+	int len;
+	unsigned int i;
+	const char *expected;
+	char *actual;
+	int rc;
+
+	DBGC ( test, "LINEBUF %p:\n", test );
+	DBGC_HDA ( test, 0, data, remaining );
+
+	/* Consume data one line at a time */
+	for ( i = 0 ; i < test->count ; i++ ) {
+
+		/* Add data to line buffer */
+		len = line_buffer ( linebuf, data, remaining );
+
+		/* Get buffered line, if any */
+		actual = buffered_line ( linebuf );
+		if ( len < 0 ) {
+			rc = len;
+			DBGC ( test, "LINEBUF %p %s\n", test, strerror ( rc ) );
+		} else if ( actual != NULL ) {
+			DBGC ( test, "LINEBUF %p \"%s\" (consumed %d)\n",
+			       test, actual, len );
+		} else {
+			DBGC ( test, "LINEBUF %p unterminated (consumed %d)\n",
+			       test, len );
+		}
+
+		/* Check for success/failure */
+		expected = test->lines[i];
+		if ( expected == linebuf_failure ) {
+			rc = len;
+			okx ( rc < 0, file, line );
+			okx ( remaining > 0, file, line );
 			return;
 		}
-		data += frag_len;
-		len -= frag_len;
-		if ( ( line = buffered_line ( &linebuf ) ) )
-			printf ( "\"%s\"\n", line );
+		okx ( len >= 0, file, line );
+		okx ( ( ( size_t ) len ) <= remaining, file, line );
+
+		/* Check expected result */
+		if ( expected == NULL ) {
+			okx ( actual == NULL, file, line );
+		} else {
+			okx ( actual != NULL, file, line );
+			okx ( strcmp ( actual, expected ) == 0, file, line );
+		}
+
+		/* Consume data */
+		data += len;
+		remaining -= len;
+	}
+
+	/* Check that all data was consumed */
+	okx ( remaining == 0, file, line );
+}
+#define linebuf_consume_ok( test, linebuf ) \
+	linebuf_consume_okx ( test, linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer accumulation test result
+ *
+ * @v test		Line buffer test
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_accumulated_okx ( struct linebuf_test *test,
+				      struct line_buffer *linebuf,
+				      const char *file, unsigned int line ) {
+	const char *actual;
+	const char *expected;
+	unsigned int i;
+
+	/* Check each accumulated line */
+	actual = linebuf->data;
+	for ( i = 0 ; i < test->count ; i++ ) {
+
+		/* Check accumulated line */
+		okx ( actual != NULL, file, line );
+		okx ( actual >= linebuf->data, file, line );
+		expected = test->lines[i];
+		if ( ( expected == NULL ) || ( expected == linebuf_failure ) )
+			return;
+		okx ( strcmp ( actual, expected ) == 0, file, line );
+
+		/* Move to next line */
+		actual += ( strlen ( actual ) + 1 /* NUL */ );
+		okx ( actual <= ( linebuf->data + linebuf->len ), file, line );
 	}
+}
+#define linebuf_accumulated_ok( test, linebuf ) \
+	linebuf_accumulated_okx ( test, linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer emptying test result
+ *
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_empty_okx ( struct line_buffer *linebuf,
+				const char *file, unsigned int line ) {
 
-	empty_line_buffer ( &linebuf );
+	/* Empty line buffer */
+	empty_line_buffer ( linebuf );
+	okx ( buffered_line ( linebuf ) == NULL, file, line );
 }
+#define linebuf_empty_ok( linebuf ) \
+	linebuf_empty_okx ( linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer combined test result
+ *
+ * @v test		Line buffer test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_okx ( struct linebuf_test *test, const char *file,
+			  unsigned int line ) {
+	struct line_buffer linebuf;
+
+	linebuf_init_okx ( &linebuf, file, line );
+	linebuf_consume_okx ( test, &linebuf, file, line );
+	linebuf_accumulated_okx ( test, &linebuf, file, line );
+	linebuf_empty_okx ( &linebuf, file, line );
+}
+#define linebuf_ok( test ) \
+	linebuf_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Perform line buffer self-tests
+ *
+ */
+static void linebuf_test_exec ( void ) {
+	struct line_buffer linebuf;
+
+	/* Basic tests */
+	linebuf_ok ( &simple );
+	linebuf_ok ( &mixed );
+
+	/* Split consumption test */
+	linebuf_init_ok ( &linebuf );
+	linebuf_consume_ok ( &split_1, &linebuf );
+	linebuf_consume_ok ( &split_2, &linebuf );
+	linebuf_consume_ok ( &split_3, &linebuf );
+	linebuf_consume_ok ( &split_4, &linebuf );
+	linebuf_consume_ok ( &split_5, &linebuf );
+	linebuf_consume_ok ( &split_6, &linebuf );
+	linebuf_empty_ok ( &linebuf );
+
+	/* Embedded NULs */
+	linebuf_ok ( &embedded_nuls );
+}
+
+/** Line buffer self-test */
+struct self_test linebuf_test __self_test = {
+	.name = "linebuf",
+	.exec = linebuf_test_exec,
+};
diff --git a/roms/ipxe/src/tests/list_test.c b/roms/ipxe/src/tests/list_test.c
index 35cbd5e5f..352c87da0 100644
--- a/roms/ipxe/src/tests/list_test.c
+++ b/roms/ipxe/src/tests/list_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/math_test.c b/roms/ipxe/src/tests/math_test.c
index e12b7939d..1a244f1eb 100644
--- a/roms/ipxe/src/tests/math_test.c
+++ b/roms/ipxe/src/tests/math_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -35,6 +39,26 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/isqrt.h>
 
 /**
+ * Force a call to the non-constant implementation of ffsl()
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+__attribute__ (( noinline )) int ffsl_var ( long value ) {
+	return ffsl ( value );
+}
+
+/**
+ * Force a call to the non-constant implementation of ffsll()
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+__attribute__ (( noinline )) int ffsll_var ( long long value ) {
+	return ffsll ( value );
+}
+
+/**
  * Force a call to the non-constant implementation of flsl()
  *
  * @v value		Value
@@ -173,6 +197,44 @@ __attribute__ (( noinline )) int64_t s64mod_var ( int64_t dividend,
 }
 
 /**
+ * Report a ffsl() test result
+ *
+ * @v value		Value
+ * @v lsb		Expected LSB
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static inline __attribute__ (( always_inline )) void
+ffsl_okx ( long value, int lsb, const char *file, unsigned int line ) {
+
+	/* Verify as a constant (requires to be inlined) */
+	okx ( ffsl ( value ) == lsb, file, line );
+
+	/* Verify as a non-constant */
+	okx ( ffsl_var ( value ) == lsb, file, line );
+}
+#define ffsl_ok( value, lsb ) ffsl_okx ( value, lsb, __FILE__, __LINE__ )
+
+/**
+ * Report a ffsll() test result
+ *
+ * @v value		Value
+ * @v lsb		Expected LSB
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static inline __attribute__ (( always_inline )) void
+ffsll_okx ( long long value, int lsb, const char *file, unsigned int line ) {
+
+	/* Verify as a constant (requires to be inlined) */
+	okx ( ffsll ( value ) == lsb, file, line );
+
+	/* Verify as a non-constant */
+	okx ( ffsll_var ( value ) == lsb, file, line );
+}
+#define ffsll_ok( value, lsb ) ffsll_okx ( value, lsb, __FILE__, __LINE__ )
+
+/**
  * Report a flsl() test result
  *
  * @v value		Value
@@ -270,6 +332,22 @@ static void s64divmod_okx ( int64_t dividend, int64_t divisor,
  */
 static void math_test_exec ( void ) {
 
+	/* Test ffsl() */
+	ffsl_ok ( 0, 0 );
+	ffsl_ok ( 1, 1 );
+	ffsl_ok ( 255, 1 );
+	ffsl_ok ( 256, 9 );
+	ffsl_ok ( 257, 1 );
+	ffsl_ok ( 0x54850596, 2 );
+	ffsl_ok ( 0x80000000, 32 );
+
+	/* Test ffsll() */
+	ffsll_ok ( 0, 0 );
+	ffsll_ok ( 1, 1 );
+	ffsll_ok ( 0x6d63623330ULL, 5 );
+	ffsll_ok ( 0x80000000UL, 32 );
+	ffsll_ok ( 0x8000000000000000ULL, 64 );
+
 	/* Test flsl() */
 	flsl_ok ( 0, 0 );
 	flsl_ok ( 1, 1 );
diff --git a/roms/ipxe/src/tests/md5_test.c b/roms/ipxe/src/tests/md5_test.c
index ba5f24c3e..e9ed2716a 100644
--- a/roms/ipxe/src/tests/md5_test.c
+++ b/roms/ipxe/src/tests/md5_test.c
@@ -15,82 +15,58 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
  * MD5 tests
  *
+ * Test inputs borrowed from NIST SHA-1 tests, with results calculated
+ * using md5sum.
  */
 
-#include <stdint.h>
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <ipxe/md5.h>
 #include <ipxe/test.h>
 #include "digest_test.h"
 
-/** An MD5 test vector */
-struct md5_test_vector {
-	/** Test data */
-	void *data;
-	/** Test data length */
-	size_t len;
-	/** Expected digest */
-	uint8_t digest[MD5_DIGEST_SIZE];
-};
+/* Empty test vector (digest obtained from "md5sum /dev/null") */
+DIGEST_TEST ( md5_empty, &md5_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9,
+		       0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e ) );
 
-/** MD5 test vectors */
-static struct md5_test_vector md5_test_vectors[] = {
-	/* Test inputs borrowed from SHA-1 tests, with results
-	 * calculated using md5sum.
-	 */
-	{ NULL, 0,
-	  { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
-	    0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } },
-	{ "abc", 3,
-	  { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0,
-	    0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } },
-	{ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56,
-	  { 0x82, 0x15, 0xef, 0x07, 0x96, 0xa2, 0x0b, 0xca,
-	    0xaa, 0xe1, 0x16, 0xd3, 0x87, 0x6c, 0x66, 0x4a } },
-};
+/* NIST test vector "abc" (digest obtained from "md5sum <data>") */
+DIGEST_TEST ( md5_nist_abc, &md5_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6,
+		       0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 ) );
 
-/** MD5 test fragment lists */
-static struct digest_test_fragments md5_test_fragments[] = {
-	{ { 0, -1UL, } },
-	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
-	{ { 2, 0, 23, 4, 6, 1, 0 } },
-};
+/* NIST test vector "abc...opq" (digest obtained from "md5sum <data>") */
+DIGEST_TEST ( md5_nist_abc_opq, &md5_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x82, 0x15, 0xef, 0x07, 0x96, 0xa2, 0x0b, 0xca, 0xaa,
+		       0xe1, 0x16, 0xd3, 0x87, 0x6c, 0x66, 0x4a ) );
 
 /**
  * Perform MD5 self-test
  *
  */
 static void md5_test_exec ( void ) {
-	struct digest_algorithm *digest = &md5_algorithm;
-	struct md5_test_vector *test;
-	unsigned long cost;
-	unsigned int i;
-	unsigned int j;
 
-	/* Correctness test */
-	for ( i = 0 ; i < ( sizeof ( md5_test_vectors ) /
-			    sizeof ( md5_test_vectors[0] ) ) ; i++ ) {
-		test = &md5_test_vectors[i];
-		/* Test with a single pass */
-		digest_ok ( digest, NULL, test->data, test->len, test->digest );
-		/* Test with fragment lists */
-		for ( j = 0 ; j < ( sizeof ( md5_test_fragments ) /
-				    sizeof ( md5_test_fragments[0] ) ) ; j++ ){
-			digest_ok ( digest, &md5_test_fragments[j],
-				    test->data, test->len, test->digest );
-		}
-	}
+	/* Correctness tests */
+	digest_ok ( &md5_empty );
+	digest_ok ( &md5_nist_abc );
+	digest_ok ( &md5_nist_abc_opq );
 
-	/* Speed test */
-	cost = digest_cost ( digest );
-	DBG ( "MD5 required %ld cycles per byte\n", cost );
+	/* Speed tests */
+	DBG ( "MD5 required %ld cycles per byte\n",
+	      digest_cost ( &md5_algorithm ) );
 }
 
 /** MD5 self-test */
diff --git a/roms/ipxe/src/tests/memcpy_test.c b/roms/ipxe/src/tests/memcpy_test.c
index f1e5503a6..0247c71d4 100644
--- a/roms/ipxe/src/tests/memcpy_test.c
+++ b/roms/ipxe/src/tests/memcpy_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/memset_test.c b/roms/ipxe/src/tests/memset_test.c
new file mode 100644
index 000000000..d96f83fa6
--- /dev/null
+++ b/roms/ipxe/src/tests/memset_test.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * memset() self-tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <string.h>
+#include <ipxe/test.h>
+
+/* Provide global functions to allow inspection of generated code */
+
+void memset_zero_0 ( void *dest ) { memset ( dest, 0, 0 ); }
+void memset_zero_1 ( void *dest ) { memset ( dest, 0, 1 ); }
+void memset_zero_2 ( void *dest ) { memset ( dest, 0, 2 ); }
+void memset_zero_3 ( void *dest ) { memset ( dest, 0, 3 ); }
+void memset_zero_4 ( void *dest ) { memset ( dest, 0, 4 ); }
+void memset_zero_5 ( void *dest ) { memset ( dest, 0, 5 ); }
+void memset_zero_6 ( void *dest ) { memset ( dest, 0, 6 ); }
+void memset_zero_7 ( void *dest ) { memset ( dest, 0, 7 ); }
+void memset_zero_8 ( void *dest ) { memset ( dest, 0, 8 ); }
+void memset_zero_9 ( void *dest ) { memset ( dest, 0, 9 ); }
+void memset_zero_10 ( void *dest ) { memset ( dest, 0, 10 ); }
+void memset_zero_11 ( void *dest ) { memset ( dest, 0, 11 ); }
+void memset_zero_12 ( void *dest ) { memset ( dest, 0, 12 ); }
+void memset_zero_13 ( void *dest ) { memset ( dest, 0, 13 ); }
+void memset_zero_14 ( void *dest ) { memset ( dest, 0, 14 ); }
+void memset_zero_15 ( void *dest ) { memset ( dest, 0, 15 ); }
+void memset_zero_16 ( void *dest ) { memset ( dest, 0, 16 ); }
+void memset_zero_17 ( void *dest ) { memset ( dest, 0, 17 ); }
+void memset_zero_18 ( void *dest ) { memset ( dest, 0, 18 ); }
+void memset_zero_19 ( void *dest ) { memset ( dest, 0, 19 ); }
+void memset_zero_20 ( void *dest ) { memset ( dest, 0, 20 ); }
+void memset_zero_21 ( void *dest ) { memset ( dest, 0, 21 ); }
+void memset_zero_22 ( void *dest ) { memset ( dest, 0, 22 ); }
+void memset_zero_23 ( void *dest ) { memset ( dest, 0, 23 ); }
+void memset_zero_24 ( void *dest ) { memset ( dest, 0, 24 ); }
+void memset_zero_25 ( void *dest ) { memset ( dest, 0, 25 ); }
+void memset_zero_26 ( void *dest ) { memset ( dest, 0, 26 ); }
+void memset_zero_27 ( void *dest ) { memset ( dest, 0, 27 ); }
+void memset_zero_28 ( void *dest ) { memset ( dest, 0, 28 ); }
+void memset_zero_29 ( void *dest ) { memset ( dest, 0, 29 ); }
+void memset_zero_30 ( void *dest ) { memset ( dest, 0, 30 ); }
+void memset_zero_31 ( void *dest ) { memset ( dest, 0, 31 ); }
+
+/**
+ * Force a call to the variable-length implementation of memset()
+ *
+ * @v dest		Destination address
+ * @v fill		Fill pattern
+ * @v len		Length of data
+ * @ret dest		Destination address
+ */
+__attribute__ (( noinline )) void * memset_var ( void *dest, unsigned int fill,
+						 size_t len ) {
+	return memset ( dest, fill, len );
+}
+
+/**
+ * Perform a constant-length memset() test
+ *
+ * @v len		Length of data
+ */
+#define MEMSET_TEST_CONSTANT( len ) do {				\
+		uint8_t dest_const[ 1 + len + 1 ];			\
+		uint8_t dest_var[ 1 + len + 1 ];			\
+		static uint8_t zero[len];				\
+		unsigned int i;						\
+									\
+		for ( i = 0 ; i < sizeof ( dest_const ) ; i++ )		\
+			dest_const[i] = 0xaa;				\
+		memset ( ( dest_const + 1 ), 0, len );			\
+		ok ( dest_const[0] == 0xaa );				\
+		ok ( dest_const[ sizeof ( dest_const ) - 1 ] == 0xaa );	\
+		ok ( memcmp ( ( dest_const + 1 ), zero, len ) == 0 );	\
+									\
+		for ( i = 0 ; i < sizeof ( dest_var ) ; i++ )		\
+			dest_var[i] = 0xbb;				\
+		memset_var ( ( dest_var + 1 ), 0, len );		\
+		ok ( dest_var[0] == 0xbb );				\
+		ok ( dest_var[ sizeof ( dest_var ) - 1 ] == 0xbb );	\
+		ok ( memcmp ( ( dest_var + 1 ), zero, len ) == 0 );	\
+	} while ( 0 )
+
+/**
+ * Perform memset() self-tests
+ *
+ */
+static void memset_test_exec ( void ) {
+
+	/* Constant-length tests */
+	MEMSET_TEST_CONSTANT ( 0 );
+	MEMSET_TEST_CONSTANT ( 1 );
+	MEMSET_TEST_CONSTANT ( 2 );
+	MEMSET_TEST_CONSTANT ( 3 );
+	MEMSET_TEST_CONSTANT ( 4 );
+	MEMSET_TEST_CONSTANT ( 5 );
+	MEMSET_TEST_CONSTANT ( 6 );
+	MEMSET_TEST_CONSTANT ( 7 );
+	MEMSET_TEST_CONSTANT ( 8 );
+	MEMSET_TEST_CONSTANT ( 9 );
+	MEMSET_TEST_CONSTANT ( 10 );
+	MEMSET_TEST_CONSTANT ( 11 );
+	MEMSET_TEST_CONSTANT ( 12 );
+	MEMSET_TEST_CONSTANT ( 13 );
+	MEMSET_TEST_CONSTANT ( 14 );
+	MEMSET_TEST_CONSTANT ( 15 );
+	MEMSET_TEST_CONSTANT ( 16 );
+	MEMSET_TEST_CONSTANT ( 17 );
+	MEMSET_TEST_CONSTANT ( 18 );
+	MEMSET_TEST_CONSTANT ( 19 );
+	MEMSET_TEST_CONSTANT ( 20 );
+	MEMSET_TEST_CONSTANT ( 21 );
+	MEMSET_TEST_CONSTANT ( 22 );
+	MEMSET_TEST_CONSTANT ( 23 );
+	MEMSET_TEST_CONSTANT ( 24 );
+	MEMSET_TEST_CONSTANT ( 25 );
+	MEMSET_TEST_CONSTANT ( 26 );
+	MEMSET_TEST_CONSTANT ( 27 );
+	MEMSET_TEST_CONSTANT ( 28 );
+	MEMSET_TEST_CONSTANT ( 29 );
+	MEMSET_TEST_CONSTANT ( 30 );
+	MEMSET_TEST_CONSTANT ( 31 );
+}
+
+/** memset() self-test */
+struct self_test memset_test __self_test = {
+	.name = "memset",
+	.exec = memset_test_exec,
+};
diff --git a/roms/ipxe/src/tests/ocsp_test.c b/roms/ipxe/src/tests/ocsp_test.c
index a318c185a..c6d458596 100644
--- a/roms/ipxe/src/tests/ocsp_test.c
+++ b/roms/ipxe/src/tests/ocsp_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -1857,5 +1861,6 @@ struct self_test ocsp_test __self_test = {
 };
 
 /* Drag in algorithms required for tests */
+REQUIRING_SYMBOL ( ocsp_test );
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( sha1 );
diff --git a/roms/ipxe/src/tests/pccrc_test.c b/roms/ipxe/src/tests/pccrc_test.c
new file mode 100644
index 000000000..f4ab573ac
--- /dev/null
+++ b/roms/ipxe/src/tests/pccrc_test.c
@@ -0,0 +1,529 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Content Identification [MS-PCCRC] tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/pccrc.h>
+#include <ipxe/sha256.h>
+#include <ipxe/sha512.h>
+#include <ipxe/hmac.h>
+#include <ipxe/test.h>
+
+/** Define inline raw data */
+#define DATA(...) { __VA_ARGS__ }
+
+/**
+ * Define an inline content range
+ *
+ * @v START		Start offset
+ * @v END		End offset
+ * @ret range		Content range
+ */
+#define RANGE( START, END ) { .start = START, .end = END }
+
+/**
+ * Define an inline trimmed content range
+ *
+ * @v START		Start offset
+ * @v END		End offset
+ * @ret trim		Trimmed content range
+ */
+#define TRIM( START, END ) { .start = START, .end = END }
+
+/** A content information test */
+struct peerdist_info_test {
+	/** Raw content information */
+	const void *data;
+	/** Length of raw content information */
+	size_t len;
+	/** Expected digest algorithm */
+	struct digest_algorithm *expected_digest;
+	/** Expected digest size */
+	size_t expected_digestsize;
+	/** Expected content range */
+	struct peerdist_range expected_range;
+	/** Expected trimmed content range */
+	struct peerdist_range expected_trim;
+	/** Expected number of segments */
+	unsigned int expected_segments;
+};
+
+/**
+ * Define a content information test
+ *
+ * @v name		Test name
+ * @v DATA		Raw content information
+ * @v DIGEST		Expected digest algorithm
+ * @v DIGESTSIZE	Expected digest size
+ * @v RANGE		Expected content range
+ * @v TRIM		Expected trimmer content range
+ * @v SEGMENTS		Expected number of segments
+ * @ret test		Content information test
+ *
+ * Raw content information can be obtained from PeerDist-capable web
+ * servers using wget's "--header" option to inject the relevant
+ * PeerDist headers.  For example:
+ *
+ *   wget --header "Accept-Encoding: peerdist" \
+ *        --header "X-P2P-PeerDist: Version=1.0" \
+ *	  http://peerdist.server.address/test.url -O - | xxd -i -c 11
+ *
+ * Version 1 content information can be retrieved using the headers:
+ *
+ *   Accept-Encoding: peerdist
+ *   X-P2P-PeerDist: Version=1.0
+ *
+ * Version 2 content information can be retrieved (from compatible
+ * servers) using the headers:
+ *
+ *   Accept-Encoding: peerdist
+ *   X-P2P-PeerDist: Version=1.1
+ *   X-P2P-PeerDistEx: MinContentInformation=2.0, MaxContentInformation=2.0
+ */
+#define PEERDIST_INFO_TEST( name, DATA, DIGEST, DIGESTSIZE, RANGE,	\
+			    TRIM, SEGMENTS )				\
+	static const uint8_t name ## _data[] = DATA;			\
+	static struct peerdist_info_test name = {			\
+		.data = name ## _data,					\
+		.len = sizeof ( name ## _data ),			\
+		.expected_digest = DIGEST,				\
+		.expected_digestsize = DIGESTSIZE,			\
+		.expected_range = RANGE,				\
+		.expected_trim = TRIM,					\
+		.expected_segments = SEGMENTS,				\
+	}
+
+/** A content information segment test */
+struct peerdist_info_segment_test {
+	/** Segment index */
+	unsigned int index;
+	/** Expected content range */
+	struct peerdist_range expected_range;
+	/** Expected number of blocks */
+	unsigned int expected_blocks;
+	/** Expected block size */
+	size_t expected_blksize;
+	/** Expected segment hash of data */
+	uint8_t expected_hash[PEERDIST_DIGEST_MAX_SIZE];
+	/** Expected segment secret */
+	uint8_t expected_secret[PEERDIST_DIGEST_MAX_SIZE];
+	/** Expected segment identifier */
+	uint8_t expected_id[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/**
+ * Define a content information segment test
+ *
+ * @v name		Test name
+ * @v INDEX		Segment index
+ * @v RANGE		Expected content range
+ * @v BLOCKS		Expected number of blocks
+ * @v BLKSIZE		Expected block size
+ * @v HASH		Expected segment hash of data
+ * @v SECRET		Expected segment secret
+ * @v ID		Expected segment identifier
+ * @ret test		Content information segment test
+ */
+#define PEERDIST_INFO_SEGMENT_TEST( name, INDEX, RANGE, BLOCKS,		\
+				    BLKSIZE, HASH, SECRET, ID )		\
+	static struct peerdist_info_segment_test name = {		\
+		.index = INDEX,						\
+		.expected_range = RANGE,				\
+		.expected_blocks = BLOCKS,				\
+		.expected_blksize = BLKSIZE,				\
+		.expected_hash = HASH,					\
+		.expected_secret = SECRET,				\
+		.expected_id = ID,					\
+	}
+
+/** A content information block test */
+struct peerdist_info_block_test {
+	/** Block index */
+	unsigned int index;
+	/** Expected content range */
+	struct peerdist_range expected_range;
+	/** Expected trimmed content range */
+	struct peerdist_range expected_trim;
+	/** Expected hash of data */
+	uint8_t expected_hash[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/**
+ * Define a content information block test
+ *
+ * @v name		Test name
+ * @v INDEX		Block index
+ * @v RANGE		Expected content range
+ * @v TRIM		Expected trimmed content range
+ * @v HASH		Expected hash of data
+ * @ret test		Content information block test
+ */
+#define PEERDIST_INFO_BLOCK_TEST( name, INDEX, RANGE, TRIM, HASH )	\
+	static struct peerdist_info_block_test name = {			\
+		.index = INDEX,						\
+		.expected_range = RANGE,				\
+		.expected_trim = TRIM,					\
+		.expected_hash = HASH,					\
+	}
+
+/**
+ * Define a server passphrase
+ *
+ * @v name		Server passphrase name
+ * @v DATA		Raw server passphrase
+ *
+ * The server passphrase can be exported from a Windows BranchCache
+ * server using the command:
+ *
+ *   netsh branchcache exportkey exported.key somepassword
+ *
+ * and this encrypted exported key can be decrypted using the
+ * oSSL_key_dx or mcrypt_key_dx utilities found in the (prototype)
+ * Prequel project at https://fedorahosted.org/prequel/ :
+ *
+ *   oSSL_key_dx exported.key somepassword
+ *     or
+ *   mcrypt_key_dx exported.key somepassword
+ *
+ * Either command will display both the server passphrase and the
+ * "Server Secret".  Note that this latter is the version 1 server
+ * secret (i.e. the SHA-256 of the server passphrase); the
+ * corresponding version 2 server secret can be obtained by
+ * calculating the truncated SHA-512 of the server passphrase.
+ *
+ * We do not know the server passphrase during normal operation.  We
+ * use it in the self-tests only to check for typos and other errors
+ * in the test vectors, by checking that the segment secret defined in
+ * a content information segment test is as expected.
+ */
+#define SERVER_PASSPHRASE( name, DATA )					\
+	static uint8_t name[] = DATA
+
+/** Server passphrase used for these test vectors */
+SERVER_PASSPHRASE ( passphrase,
+      DATA ( 0x2a, 0x3d, 0x73, 0xeb, 0x43, 0x5e, 0x9f, 0x2b, 0x8a, 0x34, 0x42,
+	     0x67, 0xe7, 0x46, 0x7a, 0x3c, 0x73, 0x85, 0xc6, 0xe0, 0x55, 0xe2,
+	     0xb4, 0xd3, 0x0d, 0xfe, 0xc7, 0xc3, 0x8b, 0x0e, 0xd7, 0x2c ) );
+
+/** IIS logo (iis-85.png) content information version 1 */
+PEERDIST_INFO_TEST ( iis_85_png_v1,
+	DATA ( 0x00, 0x01, 0x0c, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x7e, 0x85, 0x01, 0x00, 0x00, 0x00, 0x01,
+	       0x00, 0xd8, 0xd9, 0x76, 0x35, 0x4a, 0x48, 0x72, 0xe9, 0x25, 0x76,
+	       0x18, 0x03, 0xf4, 0x58, 0xd9, 0xda, 0xaa, 0x67, 0xf8, 0xe3, 0x1c,
+	       0x63, 0x0f, 0xb7, 0x4e, 0x6a, 0x31, 0x2e, 0xf8, 0xa2, 0x5a, 0xba,
+	       0x11, 0xaf, 0xc0, 0xd7, 0x94, 0x92, 0x43, 0xf9, 0x4f, 0x9c, 0x1f,
+	       0xab, 0x35, 0xd9, 0xfd, 0x1e, 0x33, 0x1f, 0xcf, 0x78, 0x11, 0xa2,
+	       0xe0, 0x1d, 0x35, 0x87, 0xb3, 0x8d, 0x77, 0x0a, 0x29, 0xe2, 0x02,
+	       0x00, 0x00, 0x00, 0x73, 0xc1, 0x8a, 0xb8, 0x54, 0x91, 0x10, 0xf8,
+	       0xe9, 0x0e, 0x71, 0xbb, 0xc3, 0xab, 0x2a, 0xa8, 0xc4, 0x4d, 0x13,
+	       0xf4, 0x92, 0x94, 0x99, 0x25, 0x5b, 0x66, 0x0f, 0x24, 0xec, 0x77,
+	       0x80, 0x0b, 0x97, 0x4b, 0xdd, 0x65, 0x56, 0x7f, 0xde, 0xec, 0xcd,
+	       0xaf, 0xe4, 0x57, 0xa9, 0x50, 0x3b, 0x45, 0x48, 0xf6, 0x6e, 0xd3,
+	       0xb1, 0x88, 0xdc, 0xfd, 0xa0, 0xac, 0x38, 0x2b, 0x09, 0x71, 0x1a,
+	       0xcc ),
+	&sha256_algorithm, 32, RANGE ( 0, 99710 ), TRIM ( 0, 99710 ), 1 );
+
+/** IIS logo (iis-85.png) content information version 1 segment 0 */
+PEERDIST_INFO_SEGMENT_TEST ( iis_85_png_v1_s0, 0,
+	RANGE ( 0, 99710 ), 2, 65536,
+	DATA ( 0xd8, 0xd9, 0x76, 0x35, 0x4a, 0x48, 0x72, 0xe9, 0x25, 0x76, 0x18,
+	       0x03, 0xf4, 0x58, 0xd9, 0xda, 0xaa, 0x67, 0xf8, 0xe3, 0x1c, 0x63,
+	       0x0f, 0xb7, 0x4e, 0x6a, 0x31, 0x2e, 0xf8, 0xa2, 0x5a, 0xba ),
+	DATA ( 0x11, 0xaf, 0xc0, 0xd7, 0x94, 0x92, 0x43, 0xf9, 0x4f, 0x9c, 0x1f,
+	       0xab, 0x35, 0xd9, 0xfd, 0x1e, 0x33, 0x1f, 0xcf, 0x78, 0x11, 0xa2,
+	       0xe0, 0x1d, 0x35, 0x87, 0xb3, 0x8d, 0x77, 0x0a, 0x29, 0xe2 ),
+	DATA ( 0x49, 0x1b, 0x21, 0x7d, 0xbe, 0xe2, 0xb5, 0xf1, 0x2c, 0xa7, 0x9b,
+	       0x01, 0x5e, 0x06, 0xf4, 0xbb, 0xe6, 0x4f, 0x97, 0x45, 0xba, 0xd7,
+	       0x86, 0x7a, 0xef, 0x17, 0xde, 0x59, 0x92, 0x7e, 0xdc, 0xe9 ) );
+
+/** IIS logo (iis-85.png) content information version 1 segment 0 block 0 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v1_s0_b0, 0,
+	RANGE ( 0, 65536 ),
+	TRIM ( 0, 65536 ),
+	DATA ( 0x73, 0xc1, 0x8a, 0xb8, 0x54, 0x91, 0x10, 0xf8, 0xe9, 0x0e, 0x71,
+	       0xbb, 0xc3, 0xab, 0x2a, 0xa8, 0xc4, 0x4d, 0x13, 0xf4, 0x92, 0x94,
+	       0x99, 0x25, 0x5b, 0x66, 0x0f, 0x24, 0xec, 0x77, 0x80, 0x0b ) );
+
+/** IIS logo (iis-85.png) content information version 1 segment 0 block 1 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v1_s0_b1, 1,
+	RANGE ( 65536, 99710 ),
+	TRIM ( 65536, 99710 ),
+	DATA ( 0x97, 0x4b, 0xdd, 0x65, 0x56, 0x7f, 0xde, 0xec, 0xcd, 0xaf, 0xe4,
+	       0x57, 0xa9, 0x50, 0x3b, 0x45, 0x48, 0xf6, 0x6e, 0xd3, 0xb1, 0x88,
+	       0xdc, 0xfd, 0xa0, 0xac, 0x38, 0x2b, 0x09, 0x71, 0x1a, 0xcc ) );
+
+/** IIS logo (iis-85.png) content information version 2 */
+PEERDIST_INFO_TEST ( iis_85_png_v2,
+	DATA ( 0x00, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x88, 0x00, 0x00, 0x99, 0xde, 0xe0, 0xd0, 0xc3, 0x58,
+	       0xe2, 0x68, 0x4b, 0x62, 0x33, 0x0d, 0x32, 0xb5, 0xf1, 0x97, 0x87,
+	       0x24, 0xa0, 0xd0, 0xa5, 0x2b, 0xdc, 0x5e, 0x78, 0x1f, 0xae, 0x71,
+	       0xff, 0x57, 0xa8, 0xbe, 0x3d, 0xd4, 0x58, 0x03, 0x7e, 0xd4, 0x04,
+	       0x11, 0x6b, 0xb6, 0x16, 0xd9, 0xb1, 0x41, 0x16, 0x08, 0x85, 0x20,
+	       0xc4, 0x7c, 0xdc, 0x50, 0xab, 0xce, 0xa3, 0xfa, 0xe1, 0x88, 0xa9,
+	       0x8e, 0xa2, 0x2d, 0xf3, 0xc0, 0x00, 0x00, 0xeb, 0xa0, 0x33, 0x81,
+	       0xd0, 0xd0, 0xcb, 0x74, 0xf4, 0xb6, 0x13, 0xd8, 0x21, 0x0f, 0x37,
+	       0xf0, 0x02, 0xa0, 0x6f, 0x39, 0x10, 0x58, 0x60, 0x96, 0xa1, 0x30,
+	       0xd3, 0x43, 0x98, 0xc0, 0x8e, 0x66, 0xd7, 0xbc, 0xb8, 0xb6, 0xeb,
+	       0x77, 0x83, 0xe4, 0xf8, 0x07, 0x64, 0x7b, 0x63, 0xf1, 0x46, 0xb5,
+	       0x2f, 0x4a, 0xc8, 0x9c, 0xcc, 0x7a, 0xbf, 0x5f, 0xa1, 0x1a, 0xca,
+	       0xfc, 0x2a, 0xcf, 0x50, 0x28, 0x58, 0x6c ),
+	&sha512_algorithm, 32, RANGE ( 0, 99710 ), TRIM ( 0, 99710 ), 2 );
+
+/** IIS logo (iis-85.png) content information version 2 segment 0 */
+PEERDIST_INFO_SEGMENT_TEST ( iis_85_png_v2_s0, 0,
+	RANGE ( 0, 39390 ), 1, 39390,
+	DATA ( 0xe0, 0xd0, 0xc3, 0x58, 0xe2, 0x68, 0x4b, 0x62, 0x33, 0x0d, 0x32,
+	       0xb5, 0xf1, 0x97, 0x87, 0x24, 0xa0, 0xd0, 0xa5, 0x2b, 0xdc, 0x5e,
+	       0x78, 0x1f, 0xae, 0x71, 0xff, 0x57, 0xa8, 0xbe, 0x3d, 0xd4 ),
+	DATA ( 0x58, 0x03, 0x7e, 0xd4, 0x04, 0x11, 0x6b, 0xb6, 0x16, 0xd9, 0xb1,
+	       0x41, 0x16, 0x08, 0x85, 0x20, 0xc4, 0x7c, 0xdc, 0x50, 0xab, 0xce,
+	       0xa3, 0xfa, 0xe1, 0x88, 0xa9, 0x8e, 0xa2, 0x2d, 0xf3, 0xc0 ),
+	DATA ( 0x33, 0x71, 0xbb, 0xea, 0xdd, 0xb6, 0x23, 0x53, 0xad, 0xce, 0xf9,
+	       0x70, 0xa0, 0x6f, 0xdf, 0x65, 0x00, 0x1e, 0x04, 0x21, 0xf4, 0xc7,
+	       0x10, 0x82, 0x76, 0xb0, 0xc3, 0x7a, 0x9f, 0x9e, 0xc1, 0x0f ) );
+
+/** IIS logo (iis-85.png) content information version 2 segment 0 block 0 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v2_s0_b0, 0,
+	RANGE ( 0, 39390 ),
+	TRIM ( 0, 39390 ),
+	DATA ( 0xe0, 0xd0, 0xc3, 0x58, 0xe2, 0x68, 0x4b, 0x62, 0x33, 0x0d, 0x32,
+	       0xb5, 0xf1, 0x97, 0x87, 0x24, 0xa0, 0xd0, 0xa5, 0x2b, 0xdc, 0x5e,
+	       0x78, 0x1f, 0xae, 0x71, 0xff, 0x57, 0xa8, 0xbe, 0x3d, 0xd4 ) );
+
+/** IIS logo (iis-85.png) content information version 2 segment 1 */
+PEERDIST_INFO_SEGMENT_TEST ( iis_85_png_v2_s1, 1,
+	RANGE ( 39390, 99710 ), 1, 60320,
+	DATA ( 0x33, 0x81, 0xd0, 0xd0, 0xcb, 0x74, 0xf4, 0xb6, 0x13, 0xd8, 0x21,
+	       0x0f, 0x37, 0xf0, 0x02, 0xa0, 0x6f, 0x39, 0x10, 0x58, 0x60, 0x96,
+	       0xa1, 0x30, 0xd3, 0x43, 0x98, 0xc0, 0x8e, 0x66, 0xd7, 0xbc ),
+	DATA ( 0xb8, 0xb6, 0xeb, 0x77, 0x83, 0xe4, 0xf8, 0x07, 0x64, 0x7b, 0x63,
+	       0xf1, 0x46, 0xb5, 0x2f, 0x4a, 0xc8, 0x9c, 0xcc, 0x7a, 0xbf, 0x5f,
+	       0xa1, 0x1a, 0xca, 0xfc, 0x2a, 0xcf, 0x50, 0x28, 0x58, 0x6c ),
+	DATA ( 0xd7, 0xe9, 0x24, 0x42, 0x5e, 0x8f, 0x4f, 0x88, 0xf0, 0x1d, 0xc6,
+	       0xa9, 0xbb, 0x1b, 0xc3, 0x7b, 0xe1, 0x13, 0xec, 0x79, 0x17, 0xc7,
+	       0x45, 0xd4, 0x96, 0x5c, 0x2b, 0x55, 0xfa, 0x16, 0x3a, 0x6e ) );
+
+/** IIS logo (iis-85.png) content information version 2 segment 1 block 0 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v2_s1_b0, 0,
+	RANGE ( 39390, 99710 ),
+	TRIM ( 39390, 99710 ),
+	DATA ( 0x33, 0x81, 0xd0, 0xd0, 0xcb, 0x74, 0xf4, 0xb6, 0x13, 0xd8, 0x21,
+	       0x0f, 0x37, 0xf0, 0x02, 0xa0, 0x6f, 0x39, 0x10, 0x58, 0x60, 0x96,
+	       0xa1, 0x30, 0xd3, 0x43, 0x98, 0xc0, 0x8e, 0x66, 0xd7, 0xbc ) );
+
+/**
+ * Report content information test result
+ *
+ * @v test		Content information test
+ * @v info		Content information to fill in
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void peerdist_info_okx ( struct peerdist_info_test *test,
+				struct peerdist_info *info,
+				const char *file, unsigned int line ) {
+
+	/* Parse content information */
+	okx ( peerdist_info ( virt_to_user ( test->data ), test->len,
+			      info ) == 0, file, line );
+
+	/* Verify content information */
+	okx ( info->raw.data == virt_to_user ( test->data ), file, line );
+	okx ( info->raw.len == test->len, file, line );
+	okx ( info->digest == test->expected_digest, file, line );
+	okx ( info->digestsize == test->expected_digestsize, file, line );
+	okx ( info->range.start == test->expected_range.start, file, line );
+	okx ( info->range.end == test->expected_range.end, file, line );
+	okx ( info->trim.start == test->expected_trim.start, file, line );
+	okx ( info->trim.end == test->expected_trim.end, file, line );
+	okx ( info->trim.start >= info->range.start, file, line );
+	okx ( info->trim.end <= info->range.end, file, line );
+	okx ( info->segments == test->expected_segments, file, line );
+}
+#define peerdist_info_ok( test, info ) \
+	peerdist_info_okx ( test, info, __FILE__, __LINE__ )
+
+/**
+ * Report content information segment test result
+ *
+ * @v test		Content information segment test
+ * @v info		Content information
+ * @v segment		Segment information to fill in
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void peerdist_info_segment_okx ( struct peerdist_info_segment_test *test,
+					const struct peerdist_info *info,
+					struct peerdist_info_segment *segment,
+					const char *file, unsigned int line ) {
+	size_t digestsize = info->digestsize;
+
+	/* Parse content information segment */
+	okx ( peerdist_info_segment ( info, segment, test->index ) == 0,
+	      file, line );
+
+	/* Verify content information segment */
+	okx ( segment->info == info, file, line );
+	okx ( segment->index == test->index, file, line );
+	okx ( segment->range.start == test->expected_range.start, file, line );
+	okx ( segment->range.end == test->expected_range.end, file, line );
+	okx ( segment->blocks == test->expected_blocks, file, line );
+	okx ( segment->blksize == test->expected_blksize, file, line );
+	okx ( memcmp ( segment->hash, test->expected_hash,
+		       digestsize ) == 0, file, line );
+	okx ( memcmp ( segment->secret, test->expected_secret,
+		       digestsize ) == 0, file, line );
+	okx ( memcmp ( segment->id, test->expected_id,
+		       digestsize ) == 0, file, line );
+}
+#define peerdist_info_segment_ok( test, info, segment ) \
+	peerdist_info_segment_okx ( test, info, segment, __FILE__, __LINE__ )
+
+/**
+ * Report content information block test result
+ *
+ * @v test		Content information block test
+ * @v segment		Segment information
+ * @v block		Block information to fill in
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void
+peerdist_info_block_okx ( struct peerdist_info_block_test *test,
+			  const struct peerdist_info_segment *segment,
+			  struct peerdist_info_block *block,
+			  const char *file, unsigned int line ) {
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+
+	/* Parse content information block */
+	okx ( peerdist_info_block ( segment, block, test->index ) == 0,
+	      file, line );
+
+	/* Verify content information block */
+	okx ( block->segment == segment, file, line );
+	okx ( block->index == test->index, file, line );
+	okx ( block->range.start == test->expected_range.start, file, line );
+	okx ( block->range.end == test->expected_range.end, file, line );
+	okx ( block->trim.start == test->expected_trim.start, file, line );
+	okx ( block->trim.end == test->expected_trim.end, file, line );
+	okx ( memcmp ( block->hash, test->expected_hash,
+		       digestsize ) == 0, file, line );
+}
+#define peerdist_info_block_ok( test, segment, block ) \
+	peerdist_info_block_okx ( test, segment, block, __FILE__, __LINE__ )
+
+/**
+ * Report server passphrase test result
+ *
+ * @v test		Content information segment test
+ * @v info		Content information
+ * @v pass		Server passphrase
+ * @v pass_len		Length of server passphrase
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void
+peerdist_info_passphrase_okx ( struct peerdist_info_segment_test *test,
+			       const struct peerdist_info *info,
+			       uint8_t *pass, size_t pass_len,
+			       const char *file, unsigned int line ) {
+	struct digest_algorithm *digest = info->digest;
+	uint8_t ctx[digest->ctxsize];
+	uint8_t secret[digest->digestsize];
+	uint8_t expected[digest->digestsize];
+	size_t digestsize = info->digestsize;
+	size_t secretsize = digestsize;
+
+	/* Calculate server secret */
+	digest_init ( digest, ctx );
+	digest_update ( digest, ctx, pass, pass_len );
+	digest_final ( digest, ctx, secret );
+
+	/* Calculate expected segment secret */
+	hmac_init ( digest, ctx, secret, &secretsize );
+	assert ( secretsize == digestsize );
+	hmac_update ( digest, ctx, test->expected_hash, digestsize );
+	hmac_final ( digest, ctx, secret, &secretsize, expected );
+	assert ( secretsize == digestsize );
+
+	/* Verify segment secret */
+	okx ( memcmp ( test->expected_secret, expected, digestsize ) == 0,
+	      file, line );
+}
+#define peerdist_info_passphrase_ok( test, info, pass, pass_len )	\
+	peerdist_info_passphrase_okx ( test, info, pass, pass_len,	\
+				       __FILE__, __LINE__ )
+
+/**
+ * Perform content information self-tests
+ *
+ */
+static void peerdist_info_test_exec ( void ) {
+	struct peerdist_info info;
+	struct peerdist_info_segment segment;
+	struct peerdist_info_block block;
+
+	/* IIS logo (iis-85.png) content information version 1 */
+	peerdist_info_ok ( &iis_85_png_v1, &info );
+	peerdist_info_passphrase_ok ( &iis_85_png_v1_s0, &info,
+				      passphrase, sizeof ( passphrase ) );
+	peerdist_info_segment_ok ( &iis_85_png_v1_s0, &info, &segment );
+	peerdist_info_block_ok ( &iis_85_png_v1_s0_b0, &segment, &block );
+	peerdist_info_block_ok ( &iis_85_png_v1_s0_b1, &segment, &block );
+
+	/* IIS logo (iis-85.png) content information version 2 */
+	peerdist_info_ok ( &iis_85_png_v2, &info );
+	peerdist_info_passphrase_ok ( &iis_85_png_v2_s0, &info,
+				      passphrase, sizeof ( passphrase ) );
+	peerdist_info_segment_ok ( &iis_85_png_v2_s0, &info, &segment );
+	peerdist_info_block_ok ( &iis_85_png_v2_s0_b0, &segment, &block );
+	peerdist_info_passphrase_ok ( &iis_85_png_v2_s1, &info,
+				      passphrase, sizeof ( passphrase ) );
+	peerdist_info_segment_ok ( &iis_85_png_v2_s1, &info, &segment );
+	peerdist_info_block_ok ( &iis_85_png_v2_s1_b0, &segment, &block );
+}
+
+/** Content information self-test */
+struct self_test peerdist_info_test __self_test = {
+	.name = "pccrc",
+	.exec = peerdist_info_test_exec,
+};
diff --git a/roms/ipxe/src/tests/pixbuf_test.c b/roms/ipxe/src/tests/pixbuf_test.c
index 15cd33dfd..aaa516bb2 100644
--- a/roms/ipxe/src/tests/pixbuf_test.c
+++ b/roms/ipxe/src/tests/pixbuf_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -53,8 +57,8 @@ void pixbuf_okx ( struct pixel_buffer_test *test, const char *file,
 	/* Correct image data pointer */
 	test->image->data = virt_to_user ( ( void * ) test->image->data );
 
-	/* Check that image is detected as PNM */
-	okx ( image_probe ( test->image ) == 0, file, line );
+	/* Check that image is detected as correct type */
+	okx ( register_image ( test->image ) == 0, file, line );
 	okx ( test->image->type == test->type, file, line );
 
 	/* Check that a pixel buffer can be created from the image */
@@ -73,4 +77,7 @@ void pixbuf_okx ( struct pixel_buffer_test *test, const char *file,
 
 		pixbuf_put ( pixbuf );
 	}
+
+	/* Unregister image */
+	unregister_image ( test->image );
 }
diff --git a/roms/ipxe/src/tests/pixbuf_test.h b/roms/ipxe/src/tests/pixbuf_test.h
index 394f7f5fa..d12829d89 100644
--- a/roms/ipxe/src/tests/pixbuf_test.h
+++ b/roms/ipxe/src/tests/pixbuf_test.h
@@ -1,7 +1,7 @@
 #ifndef _PIXBUF_TEST_H
 #define _PIXBUF_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/roms/ipxe/src/tests/png_test.c b/roms/ipxe/src/tests/png_test.c
index cf32f2034..e921aa2a6 100644
--- a/roms/ipxe/src/tests/png_test.c
+++ b/roms/ipxe/src/tests/png_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/pnm_test.c b/roms/ipxe/src/tests/pnm_test.c
index 26b0c0726..d57fdaaef 100644
--- a/roms/ipxe/src/tests/pnm_test.c
+++ b/roms/ipxe/src/tests/pnm_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/profile_test.c b/roms/ipxe/src/tests/profile_test.c
index 9d682bf2b..d2f8df211 100644
--- a/roms/ipxe/src/tests/profile_test.c
+++ b/roms/ipxe/src/tests/profile_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/pubkey_test.h b/roms/ipxe/src/tests/pubkey_test.h
index 7678453a9..cd65b8703 100644
--- a/roms/ipxe/src/tests/pubkey_test.h
+++ b/roms/ipxe/src/tests/pubkey_test.h
@@ -1,7 +1,7 @@
 #ifndef _PUBKEY_TEST_H
 #define _PUBKEY_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/roms/ipxe/src/tests/rsa_test.c b/roms/ipxe/src/tests/rsa_test.c
index 3b32c74bc..c0d05d263 100644
--- a/roms/ipxe/src/tests/rsa_test.c
+++ b/roms/ipxe/src/tests/rsa_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/setjmp_test.c b/roms/ipxe/src/tests/setjmp_test.c
new file mode 100644
index 000000000..50ad13f3c
--- /dev/null
+++ b/roms/ipxe/src/tests/setjmp_test.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * setjmp()/longjmp() tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stddef.h>
+#include <assert.h>
+#include <setjmp.h>
+#include <ipxe/test.h>
+
+/** A setjmp()/longjmp() test */
+struct setjmp_test {
+	/** Jump buffer */
+	jmp_buf env;
+	/** Expected value */
+	int expected;
+	/** Test code file */
+	const char *file;
+	/** Test code line */
+	unsigned int line;
+};
+
+/** Expected jump */
+static struct setjmp_test *jumped;
+
+/**
+ * Report a setjmp() test result
+ *
+ * @v test		setjmp()/longjmp() test
+ *
+ * This has to be implemented as a macro since if it were a function
+ * then the context saved by setjmp() would be invalidated when the
+ * function returned.
+ */
+#define setjmp_ok( test ) do {						\
+	int value;							\
+	/* Sanity check */						\
+	assert ( jumped == NULL );					\
+	/* Initialise test */						\
+	(test)->expected = 0;						\
+	(test)->file = __FILE__;					\
+	(test)->line = __LINE__;					\
+	/* Perform setjmp() */						\
+	value = setjmp ( (test)->env );					\
+	/* Report setjmp()/longjmp() result */				\
+	setjmp_return_ok ( (test), value );				\
+	} while ( 0 )
+
+/**
+ * Report a setjmp()/longjmp() test result
+ *
+ * @v test		setjmp()/longjmp() test
+ * @v value		Value returned from setjmp()
+ *
+ * This function ends up reporting results from either setjmp() or
+ * longjmp() tests (since calls to longjmp() will return via the
+ * corresponding setjmp()).  It therefore uses the test code file and
+ * line stored in the test structure, which will represent the line
+ * from which either setjmp() or longjmp() was called.
+ */
+static void setjmp_return_ok ( struct setjmp_test *test, int value ) {
+
+	/* Determine whether this was reached via setjmp() or longjmp() */
+	if ( value == 0 ) {
+		/* This is the initial call to setjmp() */
+		okx ( test->expected == 0, test->file, test->line );
+		okx ( jumped == NULL, test->file, test->line );
+	} else {
+		/* This is reached via a call to longjmp() */
+		okx ( value == test->expected, test->file, test->line );
+		okx ( jumped == test, test->file, test->line );
+	}
+
+	/* Clear expected jump */
+	jumped = NULL;
+}
+
+/**
+ * Report a longjmp() test result
+ *
+ * @v test		setjmp()/longjmp() test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void longjmp_okx ( struct setjmp_test *test, int value,
+			  const char *file, unsigned int line ) {
+
+	/* Record expected value.  A zero passed to longjmp() should
+	 * result in setjmp() returning a value of one.
+	 */
+	test->expected = ( value ? value : 1 );
+
+	/* Record test code file and line */
+	test->file = file;
+	test->line = line;
+
+	/* Record expected jump */
+	jumped = test;
+
+	/* Perform longjmp().  Should return via setjmp_okx() */
+	longjmp ( test->env, value );
+
+	/* longjmp() should never return */
+	assert ( 0 );
+}
+#define longjmp_ok( test, value ) \
+	longjmp_okx ( test, value, __FILE__, __LINE__ )
+
+/**
+ * Perform setjmp()/longjmp() self-tests
+ *
+ */
+static void setjmp_test_exec ( void ) {
+	static struct setjmp_test alpha;
+	static struct setjmp_test beta;
+	static int iteration;
+
+	/* This is one of the very few situations in which the
+	 * "for-case" pattern is justified.
+	 */
+	for ( iteration = 0 ; iteration < 10 ; iteration++ ) {
+		DBGC ( jumped, "SETJMP test iteration %d\n", iteration );
+		switch ( iteration ) {
+		case 0: setjmp_ok ( &alpha ); break;
+		case 1: setjmp_ok ( &beta ); break;
+		case 2:	longjmp_ok ( &alpha, 0 );
+		case 3: longjmp_ok ( &alpha, 1 );
+		case 4: longjmp_ok ( &alpha, 2 );
+		case 5: longjmp_ok ( &beta, 17 );
+		case 6: longjmp_ok ( &beta, 29 );
+		case 7: longjmp_ok ( &alpha, -1 );
+		case 8: longjmp_ok ( &beta, 0 );
+		case 9: longjmp_ok ( &beta, 42 );
+		}
+	}
+}
+
+/** setjmp()/longjmp() self-test */
+struct self_test setjmp_test __self_test = {
+	.name = "setjmp",
+	.exec = setjmp_test_exec,
+};
diff --git a/roms/ipxe/src/tests/settings_test.c b/roms/ipxe/src/tests/settings_test.c
index 4ee6a10fa..f7fb35d0d 100644
--- a/roms/ipxe/src/tests/settings_test.c
+++ b/roms/ipxe/src/tests/settings_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -228,6 +232,12 @@ static struct setting test_hexraw_setting = {
 	.type = &setting_type_hexraw,
 };
 
+/** Test Base64 setting type */
+static struct setting test_base64_setting = {
+	.name = "test_base64",
+	.type = &setting_type_base64,
+};
+
 /** Test UUID setting type */
 static struct setting test_uuid_setting = {
 	.name = "test_uuid",
@@ -379,6 +389,15 @@ static void settings_test_exec ( void ) {
 			  0x17, 0x06, 0x39, 0x6b, 0xf4, 0x48, 0x4e ),
 		    "9e4b6eef36b646fe8f1706396bf4484e" );
 
+	/* "base64" setting type */
+	storef_ok ( &test_settings, &test_base64_setting,
+		    "cGFzc6\nNwaHJhc2U= ",
+		    RAW ( 0x70, 0x61, 0x73, 0x73, 0xa3, 0x70, 0x68, 0x72, 0x61,
+			  0x73, 0x65 ) );
+	fetchf_ok ( &test_settings, &test_base64_setting,
+		    RAW ( 0x80, 0x81, 0x82, 0x83, 0x84, 0x00, 0xff ),
+		    "gIGCg4QA/w==" );
+
 	/* "uuid" setting type (no store capability) */
 	fetchf_ok ( &test_settings, &test_uuid_setting,
 		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a,0xa8,
@@ -399,3 +418,7 @@ struct self_test settings_test __self_test = {
 	.name = "settings",
 	.exec = settings_test_exec,
 };
+
+/* Include real IPv6 setting type */
+REQUIRING_SYMBOL ( settings_test );
+REQUIRE_OBJECT ( ipv6 );
diff --git a/roms/ipxe/src/tests/sha1_test.c b/roms/ipxe/src/tests/sha1_test.c
index bcf761bdd..9f1d75686 100644
--- a/roms/ipxe/src/tests/sha1_test.c
+++ b/roms/ipxe/src/tests/sha1_test.c
@@ -15,87 +15,63 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
  * SHA-1 tests
  *
+ * NIST test vectors are taken from
+ *
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA1.pdf
+ *
  */
 
-#include <stdint.h>
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <ipxe/sha1.h>
 #include <ipxe/test.h>
 #include "digest_test.h"
 
-/** An SHA-1 test vector */
-struct sha1_test_vector {
-	/** Test data */
-	void *data;
-	/** Test data length */
-	size_t len;
-	/** Expected digest */
-	uint8_t digest[SHA1_DIGEST_SIZE];
-};
+/* Empty test vector (digest obtained from "sha1sum /dev/null") */
+DIGEST_TEST ( sha1_empty, &sha1_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
+		       0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
+		       0x07, 0x09 ) );
 
-/** SHA-1 test vectors */
-static struct sha1_test_vector sha1_test_vectors[] = {
-	/* Empty test data
-	 *
-	 * Expected digest value obtained from "sha1sum /dev/null"
-	 */
-	{ NULL, 0,
-	  { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55,
-	    0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09 } },
-	/* Test data and expected digests taken from the NIST
-	 * Cryptographic Toolkit Algorithm Examples at
-	 * http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA1.pdf
-	 */
-	{ "abc", 3,
-	  { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e,
-	    0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d } },
-	{ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56,
-	  { 0x84, 0x98, 0x3e, 0x44, 0x1c, 0x3b, 0xd2, 0x6e, 0xba, 0xae,
-	    0x4a, 0xa1, 0xf9, 0x51, 0x29, 0xe5, 0xe5, 0x46, 0x70, 0xf1 } },
-};
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha1_nist_abc, &sha1_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba,
+		       0x3e, 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0,
+		       0xd8, 0x9d ) );
 
-/** SHA-1 test fragment lists */
-static struct digest_test_fragments sha1_test_fragments[] = {
-	{ { 0, -1UL, } },
-	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
-	{ { 2, 0, 23, 4, 6, 1, 0 } },
-};
+/* NIST test vector "abc...opq" */
+DIGEST_TEST ( sha1_nist_abc_opq, &sha1_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x84, 0x98, 0x3e, 0x44, 0x1c, 0x3b, 0xd2, 0x6e, 0xba,
+		       0xae, 0x4a, 0xa1, 0xf9, 0x51, 0x29, 0xe5, 0xe5, 0x46,
+		       0x70, 0xf1 ) );
 
 /**
  * Perform SHA-1 self-test
  *
  */
 static void sha1_test_exec ( void ) {
-	struct digest_algorithm *digest = &sha1_algorithm;
-	struct sha1_test_vector *test;
-	unsigned long cost;
-	unsigned int i;
-	unsigned int j;
 
-	/* Correctness test */
-	for ( i = 0 ; i < ( sizeof ( sha1_test_vectors ) /
-			    sizeof ( sha1_test_vectors[0] ) ) ; i++ ) {
-		test = &sha1_test_vectors[i];
-		/* Test with a single pass */
-		digest_ok ( digest, NULL, test->data, test->len, test->digest );
-		/* Test with fragment lists */
-		for ( j = 0 ; j < ( sizeof ( sha1_test_fragments ) /
-				    sizeof ( sha1_test_fragments[0] ) ) ; j++ ){
-			digest_ok ( digest, &sha1_test_fragments[j],
-				    test->data, test->len, test->digest );
-		}
-	}
+	/* Correctness tests */
+	digest_ok ( &sha1_empty );
+	digest_ok ( &sha1_nist_abc );
+	digest_ok ( &sha1_nist_abc_opq );
 
-	/* Speed test */
-	cost = digest_cost ( digest );
-	DBG ( "SHA1 required %ld cycles per byte\n", cost );
+	/* Speed tests */
+	DBG ( "SHA1 required %ld cycles per byte\n",
+	      digest_cost ( &sha1_algorithm ) );
 }
 
 /** SHA-1 self-test */
diff --git a/roms/ipxe/src/tests/sha256_test.c b/roms/ipxe/src/tests/sha256_test.c
index 06a8cae25..3b4c423fd 100644
--- a/roms/ipxe/src/tests/sha256_test.c
+++ b/roms/ipxe/src/tests/sha256_test.c
@@ -15,93 +15,96 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
- * SHA-256 tests
+ * SHA-256 family tests
+ *
+ * NIST test vectors are taken from
+ *
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA256.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA224.pdf
  *
  */
 
-#include <stdint.h>
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <ipxe/sha256.h>
 #include <ipxe/test.h>
 #include "digest_test.h"
 
-/** An SHA-256 test vector */
-struct sha256_test_vector {
-	/** Test data */
-	void *data;
-	/** Test data length */
-	size_t len;
-	/** Expected digest */
-	uint8_t digest[SHA256_DIGEST_SIZE];
-};
+/* Empty test vector (digest obtained from "sha256sum /dev/null") */
+DIGEST_TEST ( sha256_empty, &sha256_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
+		       0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
+		       0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
+		       0x1b, 0x78, 0x52, 0xb8, 0x55 ) );
 
-/** SHA-256 test vectors */
-static struct sha256_test_vector sha256_test_vectors[] = {
-	/* Empty test data
-	 *
-	 * Expected digest value obtained from "sha256sum /dev/null"
-	 */
-	{ NULL, 0,
-	  { 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4,
-	    0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b,
-	    0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 } },
-	/* Test data and expected digests taken from the NIST
-	 * Cryptographic Toolkit Algorithm Examples at
-	 * http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA256.pdf
-	 */
-	{ "abc", 3,
-	  { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40,
-	    0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17,
-	    0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad } },
-	{ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56,
-	  { 0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, 0xe5, 0xc0, 0x26,
-	    0x93, 0x0c, 0x3e, 0x60, 0x39, 0xa3, 0x3c, 0xe4, 0x59, 0x64, 0xff,
-	    0x21, 0x67, 0xf6, 0xec, 0xed, 0xd4, 0x19, 0xdb, 0x06, 0xc1 } },
-};
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha256_nist_abc, &sha256_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41,
+		       0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03,
+		       0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff,
+		       0x61, 0xf2, 0x00, 0x15, 0xad ) );
 
-/** SHA-256 test fragment lists */
-static struct digest_test_fragments sha256_test_fragments[] = {
-	{ { 0, -1UL, } },
-	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
-	{ { 2, 0, 23, 4, 6, 1, 0 } },
-};
+/* NIST test vector "abc...opq" */
+DIGEST_TEST ( sha256_nist_abc_opq, &sha256_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, 0xe5,
+		       0xc0, 0x26, 0x93, 0x0c, 0x3e, 0x60, 0x39, 0xa3, 0x3c,
+		       0xe4, 0x59, 0x64, 0xff, 0x21, 0x67, 0xf6, 0xec, 0xed,
+		       0xd4, 0x19, 0xdb, 0x06, 0xc1 ) );
+
+/* Empty test vector (digest obtained from "sha224sum /dev/null") */
+DIGEST_TEST ( sha224_empty, &sha224_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
+		       0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
+		       0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
+		       0x2f ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha224_nist_abc, &sha224_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x23, 0x09, 0x7d, 0x22, 0x34, 0x05, 0xd8, 0x22, 0x86,
+		       0x42, 0xa4, 0x77, 0xbd, 0xa2, 0x55, 0xb3, 0x2a, 0xad,
+		       0xbc, 0xe4, 0xbd, 0xa0, 0xb3, 0xf7, 0xe3, 0x6c, 0x9d,
+		       0xa7 ) );
+
+/* NIST test vector "abc...opq" */
+DIGEST_TEST ( sha224_nist_abc_opq, &sha224_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x75, 0x38, 0x8b, 0x16, 0x51, 0x27, 0x76, 0xcc, 0x5d,
+		       0xba, 0x5d, 0xa1, 0xfd, 0x89, 0x01, 0x50, 0xb0, 0xc6,
+		       0x45, 0x5c, 0xb4, 0xf5, 0x8b, 0x19, 0x52, 0x52, 0x25,
+		       0x25 ) );
 
 /**
- * Perform SHA-256 self-test
+ * Perform SHA-256 family self-test
  *
  */
 static void sha256_test_exec ( void ) {
-	struct digest_algorithm *digest = &sha256_algorithm;
-	struct sha256_test_vector *test;
-	unsigned long cost;
-	unsigned int i;
-	unsigned int j;
 
-	/* Correctness test */
-	for ( i = 0 ; i < ( sizeof ( sha256_test_vectors ) /
-			    sizeof ( sha256_test_vectors[0] ) ) ; i++ ) {
-		test = &sha256_test_vectors[i];
-		/* Test with a single pass */
-		digest_ok ( digest, NULL, test->data, test->len, test->digest );
-		/* Test with fragment lists */
-		for ( j = 0 ; j < ( sizeof ( sha256_test_fragments ) /
-				    sizeof ( sha256_test_fragments[0] )); j++ ){
-			digest_ok ( digest, &sha256_test_fragments[j],
-				    test->data, test->len, test->digest );
-		}
-	}
+	/* Correctness tests */
+	digest_ok ( &sha256_empty );
+	digest_ok ( &sha256_nist_abc );
+	digest_ok ( &sha256_nist_abc_opq );
+	digest_ok ( &sha224_empty );
+	digest_ok ( &sha224_nist_abc );
+	digest_ok ( &sha224_nist_abc_opq );
 
-	/* Speed test */
-	cost = digest_cost ( digest );
-	DBG ( "SHA256 required %ld cycles per byte\n", cost );
+	/* Speed tests */
+	DBG ( "SHA256 required %ld cycles per byte\n",
+	      digest_cost ( &sha256_algorithm ) );
+	DBG ( "SHA224 required %ld cycles per byte\n",
+	      digest_cost ( &sha224_algorithm ) );
 }
 
-/** SHA-256 self-test */
+/** SHA-256 family self-test */
 struct self_test sha256_test __self_test = {
 	.name = "sha256",
 	.exec = sha256_test_exec,
diff --git a/roms/ipxe/src/tests/sha512_test.c b/roms/ipxe/src/tests/sha512_test.c
new file mode 100644
index 000000000..be530ebad
--- /dev/null
+++ b/roms/ipxe/src/tests/sha512_test.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512 family tests
+ *
+ * NIST test vectors are taken from
+ *
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA512.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA384.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA512_256.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA512_224.pdf
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <ipxe/sha512.h>
+#include <ipxe/test.h>
+#include "digest_test.h"
+
+/* Empty test vector (digest obtained from "sha512sum /dev/null") */
+DIGEST_TEST ( sha512_empty, &sha512_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, 0xf1,
+		       0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, 0xd6, 0x20,
+		       0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc, 0x83, 0xf4, 0xa9,
+		       0x21, 0xd3, 0x6c, 0xe9, 0xce, 0x47, 0xd0, 0xd1, 0x3c,
+		       0x5d, 0x85, 0xf2, 0xb0, 0xff, 0x83, 0x18, 0xd2, 0x87,
+		       0x7e, 0xec, 0x2f, 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41,
+		       0x7a, 0x81, 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda,
+		       0x3e ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha512_nist_abc, &sha512_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xdd, 0xaf, 0x35, 0xa1, 0x93, 0x61, 0x7a, 0xba, 0xcc,
+		       0x41, 0x73, 0x49, 0xae, 0x20, 0x41, 0x31, 0x12, 0xe6,
+		       0xfa, 0x4e, 0x89, 0xa9, 0x7e, 0xa2, 0x0a, 0x9e, 0xee,
+		       0xe6, 0x4b, 0x55, 0xd3, 0x9a, 0x21, 0x92, 0x99, 0x2a,
+		       0x27, 0x4f, 0xc1, 0xa8, 0x36, 0xba, 0x3c, 0x23, 0xa3,
+		       0xfe, 0xeb, 0xbd, 0x45, 0x4d, 0x44, 0x23, 0x64, 0x3c,
+		       0xe8, 0x0e, 0x2a, 0x9a, 0xc9, 0x4f, 0xa5, 0x4c, 0xa4,
+		       0x9f ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha512_nist_abc_stu, &sha512_algorithm, DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x8e, 0x95, 0x9b, 0x75, 0xda, 0xe3, 0x13, 0xda, 0x8c,
+		       0xf4, 0xf7, 0x28, 0x14, 0xfc, 0x14, 0x3f, 0x8f, 0x77,
+		       0x79, 0xc6, 0xeb, 0x9f, 0x7f, 0xa1, 0x72, 0x99, 0xae,
+		       0xad, 0xb6, 0x88, 0x90, 0x18, 0x50, 0x1d, 0x28, 0x9e,
+		       0x49, 0x00, 0xf7, 0xe4, 0x33, 0x1b, 0x99, 0xde, 0xc4,
+		       0xb5, 0x43, 0x3a, 0xc7, 0xd3, 0x29, 0xee, 0xb6, 0xdd,
+		       0x26, 0x54, 0x5e, 0x96, 0xe5, 0x5b, 0x87, 0x4b, 0xe9,
+		       0x09 ) );
+
+/* Empty test vector (digest obtained from "sha384sum /dev/null") */
+DIGEST_TEST ( sha384_empty, &sha384_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38, 0x4c,
+		       0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a, 0x21, 0xfd,
+		       0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43, 0x4c, 0x0c, 0xc7,
+		       0xbf, 0x63, 0xf6, 0xe1, 0xda, 0x27, 0x4e, 0xde, 0xbf,
+		       0xe7, 0x6f, 0x65, 0xfb, 0xd5, 0x1a, 0xd2, 0xf1, 0x48,
+		       0x98, 0xb9, 0x5b ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha384_nist_abc, &sha384_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xcb, 0x00, 0x75, 0x3f, 0x45, 0xa3, 0x5e, 0x8b, 0xb5,
+		       0xa0, 0x3d, 0x69, 0x9a, 0xc6, 0x50, 0x07, 0x27, 0x2c,
+		       0x32, 0xab, 0x0e, 0xde, 0xd1, 0x63, 0x1a, 0x8b, 0x60,
+		       0x5a, 0x43, 0xff, 0x5b, 0xed, 0x80, 0x86, 0x07, 0x2b,
+		       0xa1, 0xe7, 0xcc, 0x23, 0x58, 0xba, 0xec, 0xa1, 0x34,
+		       0xc8, 0x25, 0xa7 ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha384_nist_abc_stu, &sha384_algorithm, DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x09, 0x33, 0x0c, 0x33, 0xf7, 0x11, 0x47, 0xe8, 0x3d,
+		       0x19, 0x2f, 0xc7, 0x82, 0xcd, 0x1b, 0x47, 0x53, 0x11,
+		       0x1b, 0x17, 0x3b, 0x3b, 0x05, 0xd2, 0x2f, 0xa0, 0x80,
+		       0x86, 0xe3, 0xb0, 0xf7, 0x12, 0xfc, 0xc7, 0xc7, 0x1a,
+		       0x55, 0x7e, 0x2d, 0xb9, 0x66, 0xc3, 0xe9, 0xfa, 0x91,
+		       0x74, 0x60, 0x39 ) );
+
+/* Empty test vector (digest obtained from "shasum -a 512256 /dev/null") */
+DIGEST_TEST ( sha512_256_empty, &sha512_256_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xc6, 0x72, 0xb8, 0xd1, 0xef, 0x56, 0xed, 0x28, 0xab,
+		       0x87, 0xc3, 0x62, 0x2c, 0x51, 0x14, 0x06, 0x9b, 0xdd,
+		       0x3a, 0xd7, 0xb8, 0xf9, 0x73, 0x74, 0x98, 0xd0, 0xc0,
+		       0x1e, 0xce, 0xf0, 0x96, 0x7a ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha512_256_nist_abc, &sha512_256_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x53, 0x04, 0x8e, 0x26, 0x81, 0x94, 0x1e, 0xf9, 0x9b,
+		       0x2e, 0x29, 0xb7, 0x6b, 0x4c, 0x7d, 0xab, 0xe4, 0xc2,
+		       0xd0, 0xc6, 0x34, 0xfc, 0x6d, 0x46, 0xe0, 0xe2, 0xf1,
+		       0x31, 0x07, 0xe7, 0xaf, 0x23 ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha512_256_nist_abc_stu, &sha512_256_algorithm,
+	      DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x39, 0x28, 0xe1, 0x84, 0xfb, 0x86, 0x90, 0xf8, 0x40,
+		       0xda, 0x39, 0x88, 0x12, 0x1d, 0x31, 0xbe, 0x65, 0xcb,
+		       0x9d, 0x3e, 0xf8, 0x3e, 0xe6, 0x14, 0x6f, 0xea, 0xc8,
+		       0x61, 0xe1, 0x9b, 0x56, 0x3a ) );
+
+/* Empty test vector (digest obtained from "shasum -a 512224 /dev/null") */
+DIGEST_TEST ( sha512_224_empty, &sha512_224_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0x6e, 0xd0, 0xdd, 0x02, 0x80, 0x6f, 0xa8, 0x9e, 0x25,
+		       0xde, 0x06, 0x0c, 0x19, 0xd3, 0xac, 0x86, 0xca, 0xbb,
+		       0x87, 0xd6, 0xa0, 0xdd, 0xd0, 0x5c, 0x33, 0x3b, 0x84,
+		       0xf4 ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha512_224_nist_abc, &sha512_224_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x46, 0x34, 0x27, 0x0f, 0x70, 0x7b, 0x6a, 0x54, 0xda,
+		       0xae, 0x75, 0x30, 0x46, 0x08, 0x42, 0xe2, 0x0e, 0x37,
+		       0xed, 0x26, 0x5c, 0xee, 0xe9, 0xa4, 0x3e, 0x89, 0x24,
+		       0xaa ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha512_224_nist_abc_stu, &sha512_224_algorithm,
+	      DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x23, 0xfe, 0xc5, 0xbb, 0x94, 0xd6, 0x0b, 0x23, 0x30,
+		       0x81, 0x92, 0x64, 0x0b, 0x0c, 0x45, 0x33, 0x35, 0xd6,
+		       0x64, 0x73, 0x4f, 0xe4, 0x0e, 0x72, 0x68, 0x67, 0x4a,
+		       0xf9 ) );
+
+/**
+ * Perform SHA-512 family self-test
+ *
+ */
+static void sha512_test_exec ( void ) {
+
+	/* Correctness tests */
+	digest_ok ( &sha512_empty );
+	digest_ok ( &sha512_nist_abc );
+	digest_ok ( &sha512_nist_abc_stu );
+	digest_ok ( &sha384_empty );
+	digest_ok ( &sha384_nist_abc );
+	digest_ok ( &sha384_nist_abc_stu );
+	digest_ok ( &sha512_256_empty );
+	digest_ok ( &sha512_256_nist_abc );
+	digest_ok ( &sha512_256_nist_abc_stu );
+	digest_ok ( &sha512_224_empty );
+	digest_ok ( &sha512_224_nist_abc );
+	digest_ok ( &sha512_224_nist_abc_stu );
+
+	/* Speed tests */
+	DBG ( "SHA512 required %ld cycles per byte\n",
+	      digest_cost ( &sha512_algorithm ) );
+	DBG ( "SHA384 required %ld cycles per byte\n",
+	      digest_cost ( &sha384_algorithm ) );
+	DBG ( "SHA512/256 required %ld cycles per byte\n",
+	      digest_cost ( &sha512_256_algorithm ) );
+	DBG ( "SHA512/224 required %ld cycles per byte\n",
+	      digest_cost ( &sha512_224_algorithm ) );
+}
+
+/** SHA-512 family self-test */
+struct self_test sha512_test __self_test = {
+	.name = "sha512",
+	.exec = sha512_test_exec,
+};
diff --git a/roms/ipxe/src/tests/string_test.c b/roms/ipxe/src/tests/string_test.c
index 3b48d9f3d..4693b5f65 100644
--- a/roms/ipxe/src/tests/string_test.c
+++ b/roms/ipxe/src/tests/string_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -31,7 +35,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <stdint.h>
 #include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
+#include <strings.h>
+#include <ipxe/string.h>
 #include <ipxe/test.h>
 
 /**
@@ -63,6 +70,18 @@ static void string_test_exec ( void ) {
 	ok ( *(strchr ( "Testing", 'g' )) == 'g' );
 	ok ( strchr ( "Testing", 'x' ) == NULL );
 
+	/* Test strrchr() */
+	ok ( strrchr ( "", 'a' ) == NULL );
+	ok ( *(strrchr ( "Haystack", 'a' )) == 'a' );
+	ok ( *(strrchr ( "Haystack", 'k' )) == 'k' );
+	ok ( strrchr ( "Haystack", 'x' ) == NULL );
+
+	/* Test memchr() */
+	ok ( memchr ( "", '\0', 0 ) == NULL );
+	ok ( *((uint8_t *)memchr ( "post\0null", 'l', 9 )) == 'l' );
+	ok ( *((uint8_t *)memchr ( "post\0null", '\0', 9 )) == '\0' );
+	ok ( memchr ( "thingy", 'z', 6 ) == NULL );
+
 	/* Test strcmp() */
 	ok ( strcmp ( "", "" ) == 0 );
 	ok ( strcmp ( "Hello", "Hello" ) == 0 );
@@ -78,11 +97,31 @@ static void string_test_exec ( void ) {
 	ok ( strncmp ( "Goodbye", "Goodbye world", 32 ) != 0 );
 	ok ( strncmp ( "Goodbye", "Goodbye world", 7 ) == 0 );
 
+	/* Test strcasecmp() */
+	ok ( strcasecmp ( "", "" ) == 0 );
+	ok ( strcasecmp ( "Uncle Jack", "Uncle jack" ) == 0 );
+	ok ( strcasecmp ( "Uncle Jack", "Uncle" ) != 0 );
+	ok ( strcasecmp ( "Uncle", "Uncle Jack" ) != 0 );
+	ok ( strcasecmp ( "not", "equal" ) != 0 );
+
 	/* Test memcmp() */
 	ok ( memcmp ( "", "", 0 ) == 0 );
 	ok ( memcmp ( "Foo", "Foo", 3 ) == 0 );
 	ok ( memcmp ( "Foo", "Bar", 3 ) != 0 );
 
+	/* Test strstr() */
+	{
+		const char haystack[] = "find me!";
+		char *found;
+
+		found = strstr ( haystack, "find" );
+		ok ( found == &haystack[0] );
+		found = strstr ( haystack, "me" );
+		ok ( found == &haystack[5] );
+		found = strstr ( haystack, "me." );
+		ok ( found == NULL );
+	}
+
 	/* Test memset() */
 	{
 		static uint8_t test[7] = { '>', 1, 1, 1, 1, 1, '<' };
@@ -154,6 +193,107 @@ static void string_test_exec ( void ) {
 		ok ( strcmp ( dup, "hello" ) == 0 );
 		free ( dup );
 	}
+
+	/* Test strcpy() */
+	{
+		const char longer[7] = "copyme";
+		const char shorter[3] = "hi";
+		char dest[7];
+		char *copy;
+
+		copy = strcpy ( dest, longer );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, longer, 7 ) == 0 );
+		copy = strcpy ( dest, shorter );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, shorter, 3 ) == 0 );
+		ok ( memcmp ( ( dest + 3 ), ( longer + 3 ), 4 ) == 0 );
+	}
+
+	/* Test strncpy() */
+	{
+		const char src[5] = "copy";
+		const char orig[8] = { 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x' };
+		const char zero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+		char dest[8];
+		char *copy;
+
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, src, 5 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, src, 5 ) == 0 );
+		ok ( memcmp ( dest + 5, orig + 5, 3 ) == 0 );
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, src, 4 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, src, 4 ) == 0 );
+		ok ( memcmp ( dest + 4, orig + 4, 4 ) == 0 );
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, src, 8 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, src, 5 ) == 0 );
+		ok ( memcmp ( dest + 5, zero + 5, 3 ) == 0 );
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, "", 8 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, zero, 8 ) == 0 );
+	}
+
+	/* Test strcat() */
+	{
+		char buf[16] = "append";
+		char *dest;
+
+		dest = strcat ( buf, " this" );
+		ok ( dest == buf );
+		ok ( strcmp ( buf, "append this" ) == 0 );
+	}
+
+	/* Test digit_value() */
+	{
+		unsigned int i;
+		char buf[2];
+		for ( i = 0 ; i < 16 ; i++ ) {
+			snprintf ( buf, sizeof ( buf ), "%x", i );
+			ok ( digit_value ( buf[0] ) == i );
+			snprintf ( buf, sizeof ( buf ), "%X", i );
+			ok ( digit_value ( buf[0] ) == i );
+		}
+		ok ( digit_value ( 0 ) >= 16 );
+		ok ( digit_value ( 9 ) >= 16 );
+		ok ( digit_value ( '0' - 1 ) >= 16 );
+		ok ( digit_value ( '9' + 1 ) >= 16 );
+		ok ( digit_value ( 'A' - 1 ) >= 16 );
+		ok ( digit_value ( 'F' + 1 ) >= 16 );
+		ok ( digit_value ( 'a' - 1 ) >= 16 );
+		ok ( digit_value ( 'f' + 1 ) >= 16 );
+	}
+
+	/* Test strtoul() */
+	ok ( strtoul ( "12345", NULL, 0 ) == 12345UL );
+	ok ( strtoul ( "  741", NULL, 10 ) == 741UL );
+	ok ( strtoul ( " 555a", NULL, 0 ) == 555UL );
+	ok ( strtoul ( " 555a", NULL, 16 ) == 0x555aUL );
+	ok ( strtoul ( "-12", NULL, 0 ) == -12UL );
+	ok ( strtoul ( "+3", NULL, 0 ) == 3UL );
+	ok ( strtoul ( "721", NULL, 0 ) == 721UL );
+	ok ( strtoul ( "721", NULL, 8 ) == 0721UL );
+	ok ( strtoul ( "0721", NULL, 0 ) == 0721UL );
+	ok ( strtoul ( "", NULL, 0 ) == 0UL );
+	ok ( strtoul ( "\t0xcAfe", NULL, 0 ) == 0xcafeUL );
+	ok ( strtoul ( "0xffffffff", NULL, 0 ) == 0xffffffffUL );
+	{
+		static const char string[] = "123aHa.world";
+		char *endp;
+		ok ( strtoul ( string, &endp, 0 ) == 123UL );
+		ok ( endp == &string[3] );
+		ok ( strtoul ( string, &endp, 16 ) == 0x123aUL );
+		ok ( endp == &string[4] );
+		ok ( strtoul ( string, &endp, 26 ) ==
+		     ( ( ( ( ( 1 * 26 + 2 ) * 26 + 3 ) * 26 + 10 ) * 26
+			 + 17 ) * 26 + 10 ) );
+		ok ( endp == &string[6] );
+	}
 }
 
 /** String self-test */
diff --git a/roms/ipxe/src/tests/tcpip_test.c b/roms/ipxe/src/tests/tcpip_test.c
index 00c88ae32..759f886bc 100644
--- a/roms/ipxe/src/tests/tcpip_test.c
+++ b/roms/ipxe/src/tests/tcpip_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/test.c b/roms/ipxe/src/tests/test.c
index c05e72a76..67bd4cf89 100644
--- a/roms/ipxe/src/tests/test.c
+++ b/roms/ipxe/src/tests/test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/tests.c b/roms/ipxe/src/tests/tests.c
index 2b4b78c7c..54ce86677 100644
--- a/roms/ipxe/src/tests/tests.c
+++ b/roms/ipxe/src/tests/tests.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -26,6 +30,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /* Drag in all applicable self-tests */
+PROVIDE_REQUIRING_SYMBOL();
+REQUIRE_OBJECT ( memset_test );
 REQUIRE_OBJECT ( memcpy_test );
 REQUIRE_OBJECT ( string_test );
 REQUIRE_OBJECT ( math_test );
@@ -37,12 +43,14 @@ REQUIRE_OBJECT ( base16_test );
 REQUIRE_OBJECT ( settings_test );
 REQUIRE_OBJECT ( time_test );
 REQUIRE_OBJECT ( tcpip_test );
+REQUIRE_OBJECT ( ipv4_test );
 REQUIRE_OBJECT ( ipv6_test );
 REQUIRE_OBJECT ( crc32_test );
 REQUIRE_OBJECT ( md5_test );
 REQUIRE_OBJECT ( sha1_test );
 REQUIRE_OBJECT ( sha256_test );
-REQUIRE_OBJECT ( aes_cbc_test );
+REQUIRE_OBJECT ( sha512_test );
+REQUIRE_OBJECT ( aes_test );
 REQUIRE_OBJECT ( hmac_drbg_test );
 REQUIRE_OBJECT ( hash_df_test );
 REQUIRE_OBJECT ( bigint_test );
@@ -56,3 +64,6 @@ REQUIRE_OBJECT ( png_test );
 REQUIRE_OBJECT ( dns_test );
 REQUIRE_OBJECT ( uri_test );
 REQUIRE_OBJECT ( profile_test );
+REQUIRE_OBJECT ( setjmp_test );
+REQUIRE_OBJECT ( pccrc_test );
+REQUIRE_OBJECT ( linebuf_test );
diff --git a/roms/ipxe/src/tests/time_test.c b/roms/ipxe/src/tests/time_test.c
index 28acebee6..3bf01dd1d 100644
--- a/roms/ipxe/src/tests/time_test.c
+++ b/roms/ipxe/src/tests/time_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/uri_test.c b/roms/ipxe/src/tests/uri_test.c
index 14f1b4ad0..da7fb8abe 100644
--- a/roms/ipxe/src/tests/uri_test.c
+++ b/roms/ipxe/src/tests/uri_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -66,6 +70,8 @@ struct uri_resolve_test {
 struct uri_tftp_test {
 	/** Next-server address */
 	struct in_addr next_server;
+	/** Port number */
+	unsigned int port;
 	/** Filename */
 	const char *filename;
 	/** URI */
@@ -330,7 +336,7 @@ static void uri_tftp_okx ( struct uri_tftp_test *test, const char *file,
 	size_t len;
 
 	/* Construct URI */
-	uri = tftp_uri ( test->next_server, test->filename );
+	uri = tftp_uri ( test->next_server, test->port, test->filename );
 	okx ( uri != NULL, file, line );
 	if ( uri ) {
 		uri_okx ( uri, &test->uri, file, line );
@@ -674,7 +680,7 @@ static struct uri_resolve_test uri_fragment = {
 
 /** TFTP URI with absolute path */
 static struct uri_tftp_test uri_tftp_absolute = {
-	{ .s_addr = htonl ( 0xc0a80002 ) /* 192.168.0.2 */ },
+	{ .s_addr = htonl ( 0xc0a80002 ) /* 192.168.0.2 */ }, 0,
 	"/absolute/path",
 	{
 		.scheme = "tftp",
@@ -686,7 +692,7 @@ static struct uri_tftp_test uri_tftp_absolute = {
 
 /** TFTP URI with relative path */
 static struct uri_tftp_test uri_tftp_relative = {
-	{ .s_addr = htonl ( 0xc0a80003 ) /* 192.168.0.3 */ },
+	{ .s_addr = htonl ( 0xc0a80003 ) /* 192.168.0.3 */ }, 0,
 	"relative/path",
 	{
 		.scheme = "tftp",
@@ -698,7 +704,7 @@ static struct uri_tftp_test uri_tftp_relative = {
 
 /** TFTP URI with path containing special characters */
 static struct uri_tftp_test uri_tftp_icky = {
-	{ .s_addr = htonl ( 0x0a000006 ) /* 10.0.0.6 */ },
+	{ .s_addr = htonl ( 0x0a000006 ) /* 10.0.0.6 */ }, 0,
 	"C:\\tftpboot\\icky#path",
 	{
 		.scheme = "tftp",
@@ -708,6 +714,19 @@ static struct uri_tftp_test uri_tftp_icky = {
 	"tftp://10.0.0.6/C%3A\\tftpboot\\icky%23path",
 };
 
+/** TFTP URI with custom port */
+static struct uri_tftp_test uri_tftp_port = {
+	{ .s_addr = htonl ( 0xc0a80001 ) /* 192.168.0.1 */ }, 4069,
+	"/another/path",
+	{
+		.scheme = "tftp",
+		.host = "192.168.0.1",
+		.port = "4069",
+		.path = "/another/path",
+	},
+	"tftp://192.168.0.1:4069/another/path",
+};
+
 /** Current working URI test */
 static struct uri_churi_test uri_churi[] = {
 	{
@@ -842,6 +861,7 @@ static void uri_test_exec ( void ) {
 	uri_tftp_ok ( &uri_tftp_absolute );
 	uri_tftp_ok ( &uri_tftp_relative );
 	uri_tftp_ok ( &uri_tftp_icky );
+	uri_tftp_ok ( &uri_tftp_port );
 
 	/* Current working URI tests */
 	uri_churi_ok ( uri_churi );
diff --git a/roms/ipxe/src/tests/vsprintf_test.c b/roms/ipxe/src/tests/vsprintf_test.c
index 11512ec8e..0ad4f1c56 100644
--- a/roms/ipxe/src/tests/vsprintf_test.c
+++ b/roms/ipxe/src/tests/vsprintf_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/tests/x509_test.c b/roms/ipxe/src/tests/x509_test.c
index fd39e12d2..658d5247c 100644
--- a/roms/ipxe/src/tests/x509_test.c
+++ b/roms/ipxe/src/tests/x509_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -1105,6 +1109,7 @@ struct self_test x509_test __self_test = {
 };
 
 /* Drag in algorithms required for tests */
+REQUIRING_SYMBOL ( x509_test );
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( sha1 );
 REQUIRE_OBJECT ( sha256 );
diff --git a/roms/ipxe/src/usr/autoboot.c b/roms/ipxe/src/usr/autoboot.c
index 47476ae40..912543828 100644
--- a/roms/ipxe/src/usr/autoboot.c
+++ b/roms/ipxe/src/usr/autoboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
@@ -42,6 +46,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <usr/prompt.h>
 #include <usr/autoboot.h>
 #include <config/general.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -101,7 +106,7 @@ static struct uri * parse_next_server_and_filename ( struct in_addr next_server,
 	/* Construct a TFTP URI for the filename, if applicable */
 	if ( next_server.s_addr && filename[0] && ! uri_is_absolute ( uri ) ) {
 		uri_put ( uri );
-		uri = tftp_uri ( next_server, filename );
+		uri = tftp_uri ( next_server, 0, filename );
 		if ( ! uri )
 			return NULL;
 	}
@@ -173,6 +178,7 @@ int uriboot ( struct uri *filename, struct uri *root_path, int drive,
 	if ( filename ) {
 		if ( ( rc = imgdownload ( filename, 0, &image ) ) != 0 )
 			goto err_download;
+		imgstat ( image );
 		image->flags |= IMAGE_AUTO_UNREGISTER;
 		if ( ( rc = image_exec ( image ) ) != 0 ) {
 			printf ( "Could not boot image: %s\n",
@@ -434,9 +440,14 @@ int netboot ( struct net_device *netdev ) {
  * @ret is_autoboot	Network device matches the autoboot device
  */
 static int is_autoboot_busloc ( struct net_device *netdev ) {
+	struct device *dev;
 
-	return ( ( netdev->dev->desc.bus_type == autoboot_desc.bus_type ) &&
-		 ( netdev->dev->desc.location == autoboot_desc.location ) );
+	for ( dev = netdev->dev ; dev ; dev = dev->parent ) {
+		if ( ( dev->desc.bus_type == autoboot_desc.bus_type ) &&
+		     ( dev->desc.location == autoboot_desc.location ) )
+			return 1;
+	}
+	return 0;
 }
 
 /**
@@ -522,7 +533,8 @@ static int shell_banner ( void ) {
 
 	/* Prompt user */
 	printf ( "\n" );
-	return ( prompt ( "Press Ctrl-B for the iPXE command line...",
+	return ( prompt ( "Press Ctrl-B for the " PRODUCT_SHORT_NAME
+			  " command line...",
 			  ( ( BANNER_TIMEOUT * TICKS_PER_SEC ) / 10 ),
 			  CTRL_B ) == 0 );
 }
@@ -531,28 +543,29 @@ static int shell_banner ( void ) {
  * Main iPXE flow of execution
  *
  * @v netdev		Network device, or NULL
+ * @ret rc		Return status code
  */
-void ipxe ( struct net_device *netdev ) {
+int ipxe ( struct net_device *netdev ) {
 	struct feature *feature;
 	struct image *image;
 	char *scriptlet;
+	int rc;
 
 	/*
 	 * Print welcome banner
 	 *
 	 *
 	 * If you wish to brand this build of iPXE, please do so by
-	 * defining the string PRODUCT_NAME in config/general.h.
+	 * defining the string PRODUCT_NAME in config/branding.h.
 	 *
 	 * While nothing in the GPL prevents you from removing all
 	 * references to iPXE or http://ipxe.org, we prefer you not to
 	 * do so.
 	 *
 	 */
-	printf ( NORMAL "\n\n%s\n" BOLD "iPXE %s"
-		 NORMAL " -- Open Source Network Boot Firmware -- "
-		 CYAN "http://ipxe.org" NORMAL "\n"
-		 "Features:", product_name, product_version );
+	printf ( NORMAL "\n\n" PRODUCT_NAME "\n" BOLD PRODUCT_SHORT_NAME " %s"
+		 NORMAL " -- " PRODUCT_TAG_LINE " -- "
+		 CYAN PRODUCT_URI NORMAL "\nFeatures:", product_version );
 	for_each_table_entry ( feature, FEATURES )
 		printf ( " %s", feature->name );
 	printf ( "\n" );
@@ -560,28 +573,30 @@ void ipxe ( struct net_device *netdev ) {
 	/* Boot system */
 	if ( ( image = first_image() ) != NULL ) {
 		/* We have an embedded image; execute it */
-		image_exec ( image );
+		return image_exec ( image );
 	} else if ( shell_banner() ) {
 		/* User wants shell; just give them a shell */
-		shell();
+		return shell();
 	} else {
 		fetch_string_setting_copy ( NULL, &scriptlet_setting,
 					    &scriptlet );
 		if ( scriptlet ) {
 			/* User has defined a scriptlet; execute it */
-			system ( scriptlet );
+			rc = system ( scriptlet );
 			free ( scriptlet );
+			return rc;
 		} else {
 			/* Try booting.  If booting fails, offer the
 			 * user another chance to enter the shell.
 			 */
 			if ( netdev ) {
-				netboot ( netdev );
+				rc = netboot ( netdev );
 			} else {
-				autoboot();
+				rc = autoboot();
 			}
 			if ( shell_banner() )
-				shell();
+				rc = shell();
+			return rc;
 		}
 	}
 }
diff --git a/roms/ipxe/src/usr/dhcpmgmt.c b/roms/ipxe/src/usr/dhcpmgmt.c
index 23982b19c..dcb360b23 100644
--- a/roms/ipxe/src/usr/dhcpmgmt.c
+++ b/roms/ipxe/src/usr/dhcpmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/usr/fcmgmt.c b/roms/ipxe/src/usr/fcmgmt.c
index a30f37a71..6f626143f 100644
--- a/roms/ipxe/src/usr/fcmgmt.c
+++ b/roms/ipxe/src/usr/fcmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/usr/ifmgmt.c b/roms/ipxe/src/usr/ifmgmt.c
index 3d05895c2..aefdaa45d 100644
--- a/roms/ipxe/src/usr/ifmgmt.c
+++ b/roms/ipxe/src/usr/ifmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
@@ -99,11 +103,12 @@ static void ifstat_errors ( struct net_device_stats *stats,
  */
 void ifstat ( struct net_device *netdev ) {
 	printf ( "%s: %s using %s on %s (%s)\n"
-		 "  [Link:%s, TX:%d TXE:%d RX:%d RXE:%d]\n",
+		 "  [Link:%s%s, TX:%d TXE:%d RX:%d RXE:%d]\n",
 		 netdev->name, netdev_addr ( netdev ),
 		 netdev->dev->driver_name, netdev->dev->name,
 		 ( netdev_is_open ( netdev ) ? "open" : "closed" ),
 		 ( netdev_link_ok ( netdev ) ? "up" : "down" ),
+		 ( netdev_link_blocked ( netdev ) ? " (blocked)" : "" ),
 		 netdev->tx_stats.good, netdev->tx_stats.bad,
 		 netdev->rx_stats.good, netdev->rx_stats.bad );
 	if ( ! netdev_link_ok ( netdev ) ) {
diff --git a/roms/ipxe/src/usr/imgmgmt.c b/roms/ipxe/src/usr/imgmgmt.c
index c9c571640..352dd0242 100644
--- a/roms/ipxe/src/usr/imgmgmt.c
+++ b/roms/ipxe/src/usr/imgmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/usr/imgtrust.c b/roms/ipxe/src/usr/imgtrust.c
index da7ff2ef0..a269833a6 100644
--- a/roms/ipxe/src/usr/imgtrust.c
+++ b/roms/ipxe/src/usr/imgtrust.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <errno.h>
diff --git a/roms/ipxe/src/usr/ipstat.c b/roms/ipxe/src/usr/ipstat.c
index 95ad799dc..0f09cc2ff 100644
--- a/roms/ipxe/src/usr/ipstat.c
+++ b/roms/ipxe/src/usr/ipstat.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/ipstat.h>
diff --git a/roms/ipxe/src/usr/lotest.c b/roms/ipxe/src/usr/lotest.c
index ad7a2fad7..6b328713c 100644
--- a/roms/ipxe/src/usr/lotest.c
+++ b/roms/ipxe/src/usr/lotest.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/usr/neighmgmt.c b/roms/ipxe/src/usr/neighmgmt.c
index e4d21a208..9fd88f82b 100644
--- a/roms/ipxe/src/usr/neighmgmt.c
+++ b/roms/ipxe/src/usr/neighmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/neighbour.h>
diff --git a/roms/ipxe/src/usr/pingmgmt.c b/roms/ipxe/src/usr/pingmgmt.c
index 16b3ec994..bb33c5d47 100644
--- a/roms/ipxe/src/usr/pingmgmt.c
+++ b/roms/ipxe/src/usr/pingmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/roms/ipxe/src/usr/profstat.c b/roms/ipxe/src/usr/profstat.c
index 991427473..d80fa26b2 100644
--- a/roms/ipxe/src/usr/profstat.c
+++ b/roms/ipxe/src/usr/profstat.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/profile.h>
diff --git a/roms/ipxe/src/usr/prompt.c b/roms/ipxe/src/usr/prompt.c
index 957b4ab3d..fca0a157c 100644
--- a/roms/ipxe/src/usr/prompt.c
+++ b/roms/ipxe/src/usr/prompt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/roms/ipxe/src/usr/pxemenu.c b/roms/ipxe/src/usr/pxemenu.c
index b69905df1..2d05d3f51 100644
--- a/roms/ipxe/src/usr/pxemenu.c
+++ b/roms/ipxe/src/usr/pxemenu.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/roms/ipxe/src/usr/route.c b/roms/ipxe/src/usr/route.c
index ba4cc3221..690ba3b6b 100644
--- a/roms/ipxe/src/usr/route.c
+++ b/roms/ipxe/src/usr/route.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/netdevice.h>
 #include <usr/route.h>
@@ -42,3 +46,7 @@ void route ( void ) {
 		}
 	}
 }
+
+/* Drag in routing management configuration */
+REQUIRING_SYMBOL ( route );
+REQUIRE_OBJECT ( config_route );
diff --git a/roms/ipxe/src/usr/route_ipv4.c b/roms/ipxe/src/usr/route_ipv4.c
index b4d1b7bf3..6260335ac 100644
--- a/roms/ipxe/src/usr/route_ipv4.c
+++ b/roms/ipxe/src/usr/route_ipv4.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/netdevice.h>
diff --git a/roms/ipxe/src/usr/route_ipv6.c b/roms/ipxe/src/usr/route_ipv6.c
index 6045f85bb..9e94b4a15 100644
--- a/roms/ipxe/src/usr/route_ipv6.c
+++ b/roms/ipxe/src/usr/route_ipv6.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/netdevice.h>
diff --git a/roms/ipxe/src/usr/sync.c b/roms/ipxe/src/usr/sync.c
index f7a04c44c..f599588ae 100644
--- a/roms/ipxe/src/usr/sync.c
+++ b/roms/ipxe/src/usr/sync.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/job.h>
diff --git a/roms/ipxe/src/util/Option/ROM.pm b/roms/ipxe/src/util/Option/ROM.pm
index 6c396730e..232cf16b8 100644
--- a/roms/ipxe/src/util/Option/ROM.pm
+++ b/roms/ipxe/src/util/Option/ROM.pm
@@ -529,6 +529,26 @@ sub new {
   return $hash;  
 }
 
+sub device_list {
+  my $hash = shift;
+  my $self = tied(%$hash);
+
+  my $device_list = $hash->{device_list};
+  return undef unless $device_list;
+
+  my @ids;
+  my $offset = ( $self->{offset} + $device_list );
+  while ( 1 ) {
+    my $raw = substr ( ${$self->{data}}, $offset, 2 );
+    my $id = unpack ( "S", $raw );
+    last unless $id;
+    push @ids, $id;
+    $offset += 2;
+  }
+
+  return @ids;
+}
+
 ##############################################################################
 #
 # Option::ROM::PnP
diff --git a/roms/ipxe/src/util/disrom.pl b/roms/ipxe/src/util/disrom.pl
index 574957acd..920a86b24 100755
--- a/roms/ipxe/src/util/disrom.pl
+++ b/roms/ipxe/src/util/disrom.pl
@@ -55,6 +55,10 @@ do {
     printf "  %-16s %s\n", "Signature:", $pci->{signature};
     printf "  %-16s 0x%04x\n", "Vendor ID:", $pci->{vendor_id};
     printf "  %-16s 0x%04x\n", "Device ID:", $pci->{device_id};
+    if ( $pci->{device_list} ) {
+      printf "  %-16s %s\n", "Device list:",
+	     ( join ( ", ", map { sprintf "0x%04x", $_ } $pci->device_list ) );
+    }
     printf "  %-16s 0x%02x%02x%02x\n", "Device class:",
 	   $pci->{base_class}, $pci->{sub_class}, $pci->{prog_intf};
     printf "  %-16s 0x%04x (%d)\n", "Image length:",
diff --git a/roms/ipxe/src/util/elf2efi.c b/roms/ipxe/src/util/elf2efi.c
index 45d539574..e68fa5d14 100644
--- a/roms/ipxe/src/util/elf2efi.c
+++ b/roms/ipxe/src/util/elf2efi.c
@@ -478,11 +478,13 @@ static void process_reloc ( bfd *bfd __attribute__ (( unused )),
 		/* Skip absolute symbols; the symbol value won't
 		 * change when the object is loaded.
 		 */
+	} else if ( ( strcmp ( howto->name, "R_386_NONE" ) == 0 ) ||
+		    ( strcmp ( howto->name, "R_X86_64_NONE" ) == 0 ) ) {
+		/* Ignore dummy relocations used by REQUIRE_SYMBOL() */
 	} else if ( strcmp ( howto->name, "R_X86_64_64" ) == 0 ) {
 		/* Generate an 8-byte PE relocation */
 		generate_pe_reloc ( pe_reltab, offset, 8 );
-	} else if ( ( strcmp ( howto->name, "R_386_32" ) == 0 ) ||
-		    ( strcmp ( howto->name, "R_X86_64_32" ) == 0 ) ) {
+	} else if ( strcmp ( howto->name, "R_386_32" ) == 0 ) {
 		/* Generate a 4-byte PE relocation */
 		generate_pe_reloc ( pe_reltab, offset, 4 );
 	} else if ( strcmp ( howto->name, "R_386_16" ) == 0 ) {
diff --git a/roms/ipxe/src/util/licence.pl b/roms/ipxe/src/util/licence.pl
index 0e43c7b4c..79e70fd65 100755
--- a/roms/ipxe/src/util/licence.pl
+++ b/roms/ipxe/src/util/licence.pl
@@ -37,6 +37,7 @@ my $known_licences = {
     desc => "GPL version 2 (or, at your option, any later version)",
     can_subsume => {
       gpl_any => 1,
+      gpl2_or_later_or_ubdl => 1,
       public_domain => 1,
       bsd3 => 1,
       bsd2 => 1,
@@ -49,6 +50,7 @@ my $known_licences = {
     can_subsume => {
       gpl_any => 1,
       gpl2_or_later => 1,
+      gpl2_or_later_or_ubdl => 1,
       public_domain => 1,
       bsd3 => 1,
       bsd2 => 1,
@@ -56,6 +58,17 @@ my $known_licences = {
       isc  => 1,
     },
   },
+  gpl2_or_later_or_ubdl => {
+    desc => ( "GPL version 2 (or, at your option, any later version) or ".
+	      "Unmodified Binary Distribution Licence" ),
+    can_subsume => {
+      public_domain => 1,
+      bsd3 => 1,
+      bsd2 => 1,
+      mit => 1,
+      isc => 1,
+    },
+  },
   public_domain => {
     desc => "Public Domain",
     can_subsume => {},
diff --git a/roms/ipxe/src/util/parserom.pl b/roms/ipxe/src/util/parserom.pl
index e278e6336..28df60652 100755
--- a/roms/ipxe/src/util/parserom.pl
+++ b/roms/ipxe/src/util/parserom.pl
@@ -1,66 +1,260 @@
 #!/usr/bin/env perl
 #
-# Parse PCI_ROM and ISA_ROM entries from a source file on stdin and
-# output the relevant Makefile variable definitions to stdout
+# Parse PCI_ROM and ISA_ROM entries from source file(s) specified as
+# arguments and output the relevant Makefile rules to STDOUT.
 #
-# Based upon portions of Ken Yap's genrules.pl
+# Originally based on portions of Ken Yap's genrules.pl. Completely
+# rewritten by Robin Smidsrød to be more maintainable.
 
 use strict;
 use warnings;
+use Getopt::Long;
 
-die "Syntax: $0 driver_source.c" unless @ARGV == 1;
-my $source = shift;
-open DRV, "<$source" or die "Could not open $source: $!\n";
+# Parse command-line options
+my @exclude_driver_classes = ();
+my @exclude_drivers = ();
+my $debug = 0;
+my $help = 0;
+GetOptions(
+    "exclude-driver-class=s" => \@exclude_driver_classes,
+    "exclude-driver=s"       => \@exclude_drivers,
+    "debug"                  => \$debug,
+    "help"                   => \$help,
+);
 
-( my $family, my $driver_name ) = ( $source =~ /^(.*?([^\/]+))\..$/ )
-    or die "Could not parse source file name \"$source\"\n";
+# Convert exclution arrays to lookup tables
+my $exclude_driver_class_map = { map { $_ => 1 } @exclude_driver_classes };
+my $exclude_driver_map       = { map { $_ => 1 } @exclude_drivers        };
 
-my $printed_family;
+# Ensure STDOUT and STDERR are synchronized if debugging
+if ( $debug ) {
+    STDOUT->autoflush(1);
+    STDERR->autoflush(1);
+}
+
+# Compile regular expressions here for slight performance boost
+my %RE = (
+    'parse_driver_class'    => qr{ drivers/ (\w+?) / }x,
+    'parse_family'          => qr{^ (?:\./)? (.*) \..+? $}x,
+    'find_rom_line'         => qr/^ \s* ( (PCI|ISA)_ROM \s* \( \s* (.*?) ) $/x,
+    'extract_pci_id'        => qr/^ \s* 0x([0-9A-Fa-f]{4}) \s* ,? \s* (.*) $/x,
+    'extract_quoted_string' => qr/^ \s* \" ([^\"]*?) \" \s* ,? \s* (.*) $/x,
+);
+
+# Show help if required arguments are missing or help was requested
+show_usage_and_exit() if $help or @ARGV < 1;
+
+# Process each source file specified
+process_source_file($_) for @ARGV;
+
+exit;
+
+sub show_usage_and_exit {
+    print STDERR <<"EOM";
+Syntax: $0 [<options>] <source-file> [<source-file>]
+Options:
+    --exclude-driver-class Exclude specified driver classes
+    --exclude-driver       Exclude specified drivers
+    --debug                Output debug information on STDERR
+    --help                 This help information
+EOM
+    exit 1;
+}
+
+# Figure out if source file is a driver and look for ROM declarations
+sub process_source_file {
+    my ($source_file) = @_;
+    return unless defined $source_file;
+    return unless length $source_file;
+    my $state = { 'source_file' => $source_file };
+    log_debug("SOURCE_FILE", $state->{source_file});
+    # Skip source files that aren't drivers
+    parse_driver_class( $state );
+    unless ( $state->{'driver_class'} ) {
+        log_debug("SKIP_NOT_DRIVER", $state->{source_file} );
+        return;
+    }
+    # Skip source files with driver classes that are explicitly excluded
+    if ( $exclude_driver_class_map->{ $state->{'driver_class'} } ) {
+        log_debug("SKIP_EXCL_CLASS", $state->{'driver_class'} );
+        return;
+    }
+    # Skip source files without driver information
+    parse_family( $state );
+    parse_driver_name( $state );
+    unless ( $state->{'family'} and $state->{'driver_name'} ) {
+        log_debug("SKIP_NO_DRV_INFO", $state->{source_file} );
+        return;
+    }
+    # Skip source files with drivers that are explicitly excluded
+    if ( $exclude_driver_map->{ $state->{'driver_name'} } ) {
+        log_debug("SKIP_EXCL_DRV", $state->{'driver_name'} );
+        return;
+    }
+    # Iterate through lines in source files looking for ROM declarations
+    # and # output Makefile rules
+    open( my $fh, "<", $state->{'source_file'} )
+        or die "Couldn't open $state->{source_file}: $!\n";
+    while (<$fh>) {
+        process_rom_decl($state, $1, $2, $3) if m/$RE{find_rom_line}/;
+    }
+    close($fh) or die "Couldn't close $source_file: $!\n";
+    return 1;
+}
+
+# Verify that the found ROM declaration is sane and dispatch to the right
+# handler depending on type
+sub process_rom_decl {
+    my ($state, $rom_line, $rom_type, $rom_decl) = @_;
+    return unless defined $rom_line;
+    return unless length $rom_line;
+    log_debug("ROM_LINE", $rom_line);
+    return unless defined $rom_type;
+    return unless length $rom_type;
+    log_debug("ROM_TYPE", $rom_type);
+    $state->{'type'} = lc $rom_type;
+    return process_pci_rom($state, $rom_decl) if $rom_type eq "PCI";
+    return process_isa_rom($state, $rom_decl) if $rom_type eq "ISA";
+    return;
+}
+
+# Extract values from PCI_ROM declaration lines and dispatch to
+# Makefile rule generator
+sub process_pci_rom {
+    my ($state, $decl) = @_;
+    return unless defined $decl;
+    return unless length $decl;
+    (my $vendor, $decl) = extract_pci_id($decl,        'PCI_VENDOR');
+    (my $device, $decl) = extract_pci_id($decl,        'PCI_DEVICE');
+    (my $image,  $decl) = extract_quoted_string($decl, 'IMAGE');
+    (my $desc,   $decl) = extract_quoted_string($decl, 'DESCRIPTION');
+    if ( $vendor and $device and $image and $desc ) {
+        print_make_rules( $state, "${vendor}${device}", $desc, $vendor, $device );
+        print_make_rules( $state, $image, $desc, $vendor, $device, 1 );
+    }
+    else {
+        log_debug("WARNING", "Malformed PCI_ROM macro on line $. of $state->{source_file}");
+    }
+    return 1;
+}
+
+# Extract values from ISA_ROM declaration lines and dispatch to
+# Makefile rule generator
+sub process_isa_rom {
+    my ($state, $decl) = @_;
+    return unless defined $decl;
+    return unless length $decl;
+    (my $image, $decl) = extract_quoted_string($decl, 'IMAGE');
+    (my $desc,  $decl) = extract_quoted_string($decl, 'DESCRIPTION');
+    if ( $image and $desc ) {
+        print_make_rules( $state, $image, $desc );
+    }
+    else {
+        log_debug("WARNING", "Malformed ISA_ROM macro on line $. of $state->{source_file}");
+    }
+    return 1;
+}
 
-sub rom {
-  ( my $type, my $image, my $desc, my $vendor, my $device, my $dup ) = @_;
-  my $ids = $vendor ? "$vendor,$device" : "-";
-  unless ( $printed_family ) {
+# Output Makefile rules for the specified ROM declarations
+sub print_make_rules {
+    my ( $state, my $image, my $desc, my $vendor, my $device, my $dup ) = @_;
+    unless ( $state->{'is_header_printed'} ) {
+        print "# NIC\t\n";
+        print "# NIC\tfamily\t$state->{family}\n";
+        print "DRIVERS_$state->{driver_class} += $state->{driver_name}\n";
+        print "DRIVERS += $state->{driver_name}\n";
+        print "\n";
+        $state->{'is_header_printed'} = 1;
+    }
+    return if $vendor and ( $vendor eq "ffff" or $device eq "ffff" );
+    my $ids = $vendor ? "$vendor,$device" : "-";
+    print "# NIC\t$image\t$ids\t$desc\n";
+    print "DRIVER_$image = $state->{driver_name}\n";
+    print "ROM_TYPE_$image = $state->{type}\n";
+    print "ROM_DESCRIPTION_$image = \"$desc\"\n";
+    print "PCI_VENDOR_$image = 0x$vendor\n" if $vendor;
+    print "PCI_DEVICE_$image = 0x$device\n" if $device;
+    print "ROMS += $image\n" unless $dup;
+    print "ROMS_$state->{driver_name} += $image\n" unless $dup;
     print "\n";
-    print "# NIC\t\n";
-    print "# NIC\tfamily\t$family\n";
-    print "DRIVERS += $driver_name\n";
-    $printed_family = 1;
-  }
-  print "\n";
-  return if ( $vendor && ( ( $vendor eq "ffff" ) || ( $device eq "ffff" ) ) );
-  print "# NIC\t$image\t$ids\t$desc\n";
-  print "DRIVER_$image = $driver_name\n";
-  print "ROM_TYPE_$image = $type\n";
-  print "ROM_DESCRIPTION_$image = \"$desc\"\n";
-  print "PCI_VENDOR_$image = 0x$vendor\n" if $vendor;
-  print "PCI_DEVICE_$image = 0x$device\n" if $device;
-  print "ROMS += $image\n" unless $dup;
-  print "ROMS_$driver_name += $image\n" unless $dup;
+    return 1;
+}
+
+# Driver class is whatever comes after the "drivers" part of the filename (relative path)
+sub parse_driver_class {
+    my ($state) = @_;
+    my $filename = $state->{'source_file'};
+    return unless defined $filename;
+    return unless length $filename;
+    if ( $filename =~ m/$RE{parse_driver_class}/ ) {
+        log_debug("DRIVER_CLASS", $1);
+        $state->{'driver_class'} = $1;
+    }
+    return;
+}
+
+# Family name is filename (relative path) without extension
+sub parse_family {
+    my ($state) = @_;
+    my $filename = $state->{'source_file'};
+    return unless defined $filename;
+    return unless length $filename;
+    if ( $filename =~ m/$RE{parse_family}/ ) {
+        log_debug("FAMILY", $1);
+        $state->{'family'} = $1;
+    }
+    return;
+}
+
+# Driver name is last part of family name
+sub parse_driver_name {
+    my ($state) = @_;
+    my $family = $state->{'family'};
+    return unless defined $family;
+    return unless length $family;
+    my @parts = split "/", $family;
+    $state->{'driver_name'} = $parts[-1];
+    log_debug("DRIVER", $state->{'driver_name'});
+    return;
 }
 
-while ( <DRV> ) {
-  next unless /(PCI|ISA)_ROM\s*\(/;
-
-  if ( /^\s*PCI_ROM\s*\(
-         \s*0x([0-9A-Fa-f]{4})\s*, # PCI vendor
-         \s*0x([0-9A-Fa-f]{4})\s*, # PCI device
-         \s*\"([^\"]*)\"\s*,	   # Image
-         \s*\"([^\"]*)\"\s*,	   # Description
-         \s*.*\s*		   # Driver data
-       \)/x ) {
-    ( my $vendor, my $device, my $image, my $desc ) = ( lc $1, lc $2, $3, $4 );
-    rom ( "pci", lc "${vendor}${device}", $desc, $vendor, $device );
-    rom ( "pci", $image, $desc, $vendor, $device, 1 );
-  } elsif ( /^\s*ISA_ROM\s*\(
-	      \s*\"([^\"]*)\"\s*,  # Image
-	      \s*\"([^\"]*)\"\s*   # Description
-	    \)/x ) {
-    ( my $image, my $desc ) = ( $1, $2 );
-    rom ( "isa", $image, $desc );
-  } else {
-    warn "Malformed PCI_ROM or ISA_ROM macro on line $. of $source\n";
-  }
+# Extract a PCI vendor/device ID e.g. 0x8086, possibly followed by a comma
+# Should always be 4-digit lower-case hex number
+sub extract_pci_id {
+    my ($str, $label) = @_;
+    return "", $str unless defined $str;
+    return "", $str unless length $str;
+    if ( $str =~ m/$RE{extract_pci_id}/ ) {
+        my $id = lc $1;
+        log_debug($label, $id);
+        return $id, $2;
+    }
+    return "", $str;
 }
 
-close DRV;
+# Extract a double-quoted string, possibly followed by a comma
+sub extract_quoted_string {
+    my ($str, $label) = @_;
+    return "", $str unless defined $str;
+    return "", $str unless length $str;
+    if ( $str =~ m/$RE{extract_quoted_string}/ ) {
+        log_debug($label, $1);
+        return $1, $2;
+    }
+    return "", $str;
+}
+
+# Output debug info to STDERR (off by default)
+sub log_debug {
+    my ($label, $str) = @_;
+    return unless $debug;
+    return unless defined $str;
+    print STDERR "\n" if $label eq 'SOURCE_FILE';
+    print STDERR "=";
+    if ( defined $label ) {
+        my $pad_count = 16 - length $label;
+        print STDERR $label . ":" . ( " " x $pad_count );
+    }
+    print STDERR $str . "\n";
+    return;
+}
diff --git a/roms/ipxe/src/util/relicense.pl b/roms/ipxe/src/util/relicense.pl
new file mode 100755
index 000000000..41954c1b3
--- /dev/null
+++ b/roms/ipxe/src/util/relicense.pl
@@ -0,0 +1,169 @@
+#!/usr/bin/perl -w
+
+=head1 NAME
+
+relicense.pl
+
+=head1 SYNOPSIS
+
+relicense.pl [options] -p <permissions file> <file> [<file>...]
+
+Option:
+
+    -p,--permitted=FILE	Specify file of emails with relicensing permission
+    -f,--force		Manually force relicensing
+    -h,--help		Display brief help message
+    -v,--verbose	Increase verbosity
+    -q,--quiet		Decrease verbosity
+
+=cut
+
+use File::Slurp;
+use IPC::Run qw ( run );
+use Getopt::Long;
+use Pod::Usage;
+use strict;
+use warnings;
+
+# Parse command-line options
+my $verbosity = 0;
+my $permfile;
+my $force;
+Getopt::Long::Configure ( "bundling", "auto_abbrev" );
+GetOptions (
+  'permitted|p=s' => \$permfile,
+  'force|f' => \$force,
+  'verbose|v+' => sub { $verbosity++; },
+  'quiet|q+' => sub { $verbosity--; },
+  'help|h' => sub { pod2usage ( 1 ); },
+) or die "Could not parse command-line options";
+pod2usage ( 1 ) unless @ARGV;
+
+# Read permitted emails file
+my @emails = ( $permfile ? read_file ( $permfile ) : () );
+chomp @emails;
+my $permitted = { map { /^.*<(\S+)>$/; ( $1 || $_ ) => 1 } @emails };
+
+# Define list of relicensable licences
+my $relicensable = {
+  GPL2_OR_LATER => 1,
+};
+
+# Define blurb to be added to copyright notice
+my $blurb = '
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.';
+
+# Process files
+my @succeeded;
+my @failed;
+while ( my $filename = shift @ARGV ) {
+
+  # Read file to determine existing licence
+  my $file = read_file ( $filename );
+  my @licences = ( $file =~ /^\s*FILE_LICENCE\s*\(\s*(\S+)\s*\)\s*;?$/mg );
+  die "No licence declaration in $filename\n" unless @licences;
+  die "Multiple licence declarations in $filename\n" if @licences > 1;
+  my $licence = $licences[0];
+
+  # Skip if file is already UBDL-licensed
+  next if $licence =~ /_OR_UBDL$/;
+
+  # Fail immediately if file is not a candidate for relicensing
+  if ( ! exists $relicensable->{$licence} ) {
+    print "Non-relicensable licence $licence in $filename\n";
+    push @failed, $filename;
+    next;
+  }
+
+  # Run git-blame
+  my $stdout;
+  my $stderr;
+  run [ "git", "blame", "-M", "-C", "-p", "-w", $filename ],
+      \undef, \$stdout, \$stderr
+      or die "git-blame $filename: $?";
+  die $stderr if $stderr;
+
+  # Process output
+  my @stdout = split ( /\n/, $stdout );
+  chomp @stdout;
+  my $details = {};
+  my $failures = 0;
+  while ( @stdout ) {
+
+    # Parse output
+    my $commit_line = shift @stdout;
+    ( my $commit, undef, my $lineno, undef, my $count ) =
+	( $commit_line =~
+	  /^([0-9a-f]{40})\s+([0-9]+)\s+([0-9]+)(\s+([0-9]+))?$/ )
+	or die "Malformed commit line \"$commit_line\"\n";
+    if ( $count ) {
+      $details->{$commit} ||= {};
+      while ( ! ( $stdout[0] =~ /^\t/ ) ) {
+	my $detail_line = shift @stdout;
+	( my $key, undef, my $value ) =
+	    ( $detail_line =~ /^([a-z-]+)(\s+(.+))?$/ )
+	    or die "Malformed detail line \"$detail_line\" for $commit_line\n";
+	$details->{$commit}->{$key} = $value;
+      }
+    }
+    die "Missing commit details for $commit_line\n"
+	unless %{$details->{$commit}};
+    my $code_line = shift @stdout;
+    ( my $line ) = ( $code_line =~ /^\t(.*)$/ )
+	or die "Malformed code line \"$code_line\" for $commit_line\n";
+
+    # Skip trivial lines and lines so common that they are likely to
+    # be misattributed by git-blame
+    next if $line =~ /^\s*$/;		# Empty lines
+    next if $line =~ /^\s*\/\*/;	# Start of comments
+    next if $line =~ /^\s*\*/;		# Middle (or end) of comments
+    next if $line =~ /^\s*\{\s*$/;	# Standalone opening braces
+    next if $line =~ /^\s*\};?\s*$/;	# Standalone closing braces
+    next if $line =~ /^\#include/;	# Header inclusions
+    next if $line =~ /^\s*return\s+0;/;	# return 0;
+    next if $line =~ /^\s*return\s+rc;/; # return rc;
+    next if $line =~ /^\s*PCI_ROM\s*\(.*\)\s*,\s*$/;	# PCI IDs
+    next if $line =~ /^\s*FILE_LICENCE\s*\(.*\)\s*;$/; # Licence declarations
+
+    # Identify author
+    my $author_mail = $details->{$commit}->{"author-mail"}
+    or die "Missing author email for $commit_line\n";
+    ( my $email ) = ( $author_mail =~ /^<(\S+)>$/ )
+	or die "Malformed author email \"$author_mail\" for $commit_line\n";
+    undef $email if exists $details->{$commit}->{boundary};
+
+    # Check for relicensing permission
+    next if defined $email && exists $permitted->{$email};
+
+    # Print out lines lacking permission
+    printf $filename."\n" unless $failures;
+    printf "%4d %-30s %s\n", $lineno, ( $email || "<root>" ), $line;
+    $failures++;
+  }
+
+  # Fail if there are any non-trivial lines lacking relicensing permission
+  if ( $failures && ! $force ) {
+    push @failed, $filename;
+    next;
+  }
+
+  # Modify FILE_LICENCE() line
+  $file =~ s/(^\s*FILE_LICENCE\s*\(\s*${licence})(\s*\)\s*;?$)/$1_OR_UBDL$2/m
+      or die "Could not modify FILE_LICENCE() in $filename\n";
+
+  # Modify copyright notice, if present
+  if ( $file =~ /GNU General Public License/i ) {
+    $file =~ s/(02110-1301, USA.$)/$1${blurb}/m
+	or die "Could not modify copyright notice in $filename\n";
+  }
+
+  # Write out modified file
+  write_file ( $filename, { atomic => 1 }, $file );
+  push @succeeded, $filename;
+}
+
+print "Relicensed: ".join ( " ", @succeeded )."\n" if @succeeded;
+die "Cannot relicense: ".join ( " ", @failed )."\n" if @failed;
diff --git a/roms/ipxe/src/util/zbin.c b/roms/ipxe/src/util/zbin.c
index 3b7cf95b3..1862a3827 100644
--- a/roms/ipxe/src/util/zbin.c
+++ b/roms/ipxe/src/util/zbin.c
@@ -1,13 +1,21 @@
+#include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
 #include <sys/stat.h>
-
-#define ENCODE
-#define VERBOSE
-#include "nrv2b.c"
-FILE *infile, *outfile;
+#include <lzma.h>
 
 #define DEBUG 0
 
+/* LZMA filter choices.  Must match those used by unlzma.S */
+#define LZMA_LC 2
+#define LZMA_LP 0
+#define LZMA_PB 0
+
+/* LZMA preset choice.  This is a policy decision */
+#define LZMA_PRESET ( LZMA_PRESET_DEFAULT | LZMA_PRESET_EXTREME )
+
 struct input_file {
 	void *buf;
 	size_t len;
@@ -177,13 +185,75 @@ static int process_zinfo_copy ( struct input_file *input,
 	return 0;
 }
 
+#define OPCODE_CALL 0xe8
+#define OPCODE_JMP 0xe9
+
+static void bcj_filter ( void *data, size_t len ) {
+	struct {
+		uint8_t opcode;
+		int32_t target;
+	} __attribute__ (( packed )) *jump;
+	ssize_t limit = ( len - sizeof ( *jump ) );
+	ssize_t offset;
+
+	/* liblzma does include an x86 BCJ filter, but it's hideously
+	 * convoluted and undocumented.  This BCJ filter is
+	 * substantially simpler and achieves the same compression (at
+	 * the cost of requiring the decompressor to know the size of
+	 * the decompressed data, which we already have in iPXE).
+	 */
+	for ( offset = 0 ; offset <= limit ; offset++ ) {
+		jump = ( data + offset );
+
+		/* Skip instructions that are not followed by a rel32 address */
+		if ( ( jump->opcode != OPCODE_CALL ) &&
+		     ( jump->opcode != OPCODE_JMP ) )
+			continue;
+
+		/* Convert rel32 address to an absolute address.  To
+		 * avoid false positives (which damage the compression
+		 * ratio), we should check that the jump target is
+		 * within the range [0,limit).
+		 *
+		 * Some output values would then end up being mapped
+		 * from two distinct input values, making the
+		 * transformation irreversible.  To solve this, we
+		 * transform such values back into the part of the
+		 * range which would otherwise correspond to no input
+		 * values.
+		 */
+		if ( ( jump->target >= -offset ) &&
+		     ( jump->target < ( limit - offset ) ) ) {
+			/* Convert relative addresses in the range
+			 * [-offset,limit-offset) to absolute
+			 * addresses in the range [0,limit).
+			 */
+			jump->target += offset;
+		} else if ( ( jump->target >= ( limit - offset ) ) &&
+			    ( jump->target < limit ) ) {
+			/* Convert positive numbers in the range
+			 * [limit-offset,limit) to negative numbers in
+			 * the range [-offset,0).
+			 */
+			jump->target -= limit;
+		}
+		offset += sizeof ( jump->target );
+	};
+}
+
 static int process_zinfo_pack ( struct input_file *input,
 				struct output_file *output,
 				union zinfo_record *zinfo ) {
 	struct zinfo_pack *pack = &zinfo->pack;
 	size_t offset = pack->offset;
 	size_t len = pack->len;
-	unsigned long packed_len;
+	size_t packed_len = 0;
+	size_t remaining = ( output->max_len - output->len );
+	lzma_options_lzma options;
+	const lzma_filter filters[] = {
+		{ .id = LZMA_FILTER_LZMA1, .options = &options },
+		{ .id = LZMA_VLI_UNKNOWN }
+	};
 
 	if ( ( offset + len ) > input->len ) {
 		fprintf ( stderr, "Input buffer overrun on pack\n" );
@@ -196,9 +266,15 @@ static int process_zinfo_pack ( struct input_file *input,
 		return -1;
 	}
 
-	if ( ucl_nrv2b_99_compress ( ( input->buf + offset ), len,
-				     ( output->buf + output->len ),
-				     &packed_len, 0 ) != UCL_E_OK ) {
+	bcj_filter ( ( input->buf + offset ), len );
+
+	lzma_lzma_preset ( &options, LZMA_PRESET );
+	options.lc = LZMA_LC;
+	options.lp = LZMA_LP;
+	options.pb = LZMA_PB;
+	if ( lzma_raw_buffer_encode ( filters, NULL, ( input->buf + offset ),
+				      len, ( output->buf + output->len ),
+				      &packed_len, remaining ) != LZMA_OK ) {
 		fprintf ( stderr, "Compression failure\n" );
 		return -1;
 	}
@@ -206,7 +282,7 @@ static int process_zinfo_pack ( struct input_file *input,
 	if ( DEBUG ) {
 		fprintf ( stderr, "PACK [%#zx,%#zx) to [%#zx,%#zx)\n",
 			  offset, ( offset + len ), output->len,
-			  ( size_t )( output->len + packed_len ) );
+			  ( output->len + packed_len ) );
 	}
 
 	output->len += packed_len;
diff --git a/roms/openbios/arch/ppc/qemu/init.c b/roms/openbios/arch/ppc/qemu/init.c
index 4fe8b7220..2b5b8e1a9 100644
--- a/roms/openbios/arch/ppc/qemu/init.c
+++ b/roms/openbios/arch/ppc/qemu/init.c
@@ -302,6 +302,11 @@ cpu_generic_init(const struct cpudef *cpu)
     fword("encode-string");
     push_str("state");
     fword("property");
+
+    PUSH(0x20);
+    fword("encode-int");
+    push_str("reservation-granule-size");
+    fword("property");
 }
 
 static void
@@ -680,6 +685,60 @@ static void ffilll(void)
     }   
 }
 
+/*
+ * adler32        ( adler buf len -- checksum )
+ *
+ * Adapted from Mark Adler's original implementation (zlib license)
+ *
+ * Both OS 9 and BootX require this word for payload validation.
+ */
+
+#define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
+#define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
+#define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
+#define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
+#define DO16(buf)   DO8(buf,0); DO8(buf,8);
+
+static void adler32(void)
+{
+    uint32_t len = (uint32_t)POP();
+    char *buf = (char *)POP();
+    uint32_t adler = (uint32_t)POP();
+
+    if (buf == NULL) {
+        RET(-1);
+    }
+
+    uint32_t base = 65521;
+    uint32_t nmax = 5552;
+
+    uint32_t s1 = adler & 0xffff;
+    uint32_t s2 = (adler >> 16) & 0xffff;
+
+    uint32_t k;
+    while (len > 0) {
+        k = (len < nmax ? len : nmax);
+        len -= k;
+
+        while (k >= 16) {
+            DO16(buf);
+            buf += 16;
+            k -= 16;
+        }
+        if (k != 0) {
+            do {
+                s1 += *buf++;
+                s2 += s1;
+            } while (--k);
+        }
+
+        s1 %= base;
+        s2 %= base;
+    }
+
+    RET(s2 << 16 | s1);
+}
+
 void
 arch_of_init(void)
 {
@@ -945,6 +1004,9 @@ arch_of_init(void)
 
     /* Implementation of filll word (required by BootX) */
     bind_func("filll", ffilll);
+
+    /* Implementation of adler32 word (required by OS 9, BootX) */
+    bind_func("(adler32)", adler32);
     
     bind_func("platform-boot", boot);
     bind_func("(go)", go);
diff --git a/roms/openbios/arch/ppc/qemu/methods.c b/roms/openbios/arch/ppc/qemu/methods.c
index fd993daa9..930b47c4e 100644
--- a/roms/openbios/arch/ppc/qemu/methods.c
+++ b/roms/openbios/arch/ppc/qemu/methods.c
@@ -114,6 +114,8 @@ static void
 ciface_quiesce( unsigned long args[], unsigned long ret[] )
 {
 	usb_exit();
+
+	ob_ide_quiesce();
 #if 0
 	unsigned long msr;
 	/* This seems to be the correct thing to do - but I'm not sure */
@@ -164,21 +166,21 @@ DECLARE_UNNAMED_NODE( mmu, INSTALL_OPEN, 0 );
 DECLARE_NODE( mmu_ciface, 0, 0, "+/openprom/client-services" );
 
 
-/* ( phys size align --- base ) */
+/* ( [phys] size align --- base ) */
 static void
 mem_claim( void )
 {
 	ucell align = POP();
 	ucell size = POP();
-	ucell phys = POP();
-	ucell ret = ofmem_claim_phys( phys, size, align );
+	phys_addr_t phys = -1;
 
-	if( ret == -1 ) {
-		printk("MEM: claim failure\n");
-		throw( -13 );
-		return;
+	if (!align) {
+		phys = POP();
 	}
-	PUSH( ret );
+
+	phys = ofmem_claim_phys(phys, size, align);
+
+	PUSH(phys);
 }
 
 /* ( phys size --- ) */
@@ -188,24 +190,24 @@ mem_release( void )
 	POP(); POP();
 }
 
-/* ( phys size align --- base ) */
+/* ( [virt] size align --- base ) */
 static void
 mmu_claim( void )
 {
 	ucell align = POP();
 	ucell size = POP();
-	ucell phys = POP();
-	ucell ret = ofmem_claim_virt( phys, size, align );
+	ucell virt = -1;
 
-	if( ret == -1 ) {
-		printk("MMU: CLAIM failure\n");
-		throw( -13 );
-		return;
+	if (!align) {
+		virt = POP();
 	}
-	PUSH( ret );
+
+	virt = ofmem_claim_virt(virt, size, align);
+
+	PUSH(virt);
 }
 
-/* ( phys size --- ) */
+/* ( virt size --- ) */
 static void
 mmu_release( void )
 {
diff --git a/roms/openbios/arch/ppc/qemu/qemu.fs b/roms/openbios/arch/ppc/qemu/qemu.fs
index 458af1bc7..11e344a59 100644
--- a/roms/openbios/arch/ppc/qemu/qemu.fs
+++ b/roms/openbios/arch/ppc/qemu/qemu.fs
@@ -93,3 +93,31 @@ variable keyboard-phandle 0 keyboard-phandle !
 :noname
   set-defaults
 ; PREPOST-initializer
+
+\ -------------------------------------------------------------------------
+\ Adler-32 wrapper
+\ -------------------------------------------------------------------------
+
+: adler32 ( adler buf len -- checksum )
+  \ Since Mac OS 9 is the only system using this word, we take this
+  \ opportunity to inject a copyright message that is necessary for the
+  \ system to boot.
+  " Copyright 1983-2001 Apple Computer, Inc. THIS MESSAGE FOR COMPATIBILITY ONLY"
+  encode-string " copyright"
+  " /" find-package if
+    " set-property" $find if
+      execute
+    else
+      3drop drop
+    then
+  then
+
+  ( adler buf len )
+
+  " (adler32)" $find if
+    execute
+  else
+    ." Can't find " ( adler32-name ) type cr
+    3drop 0
+  then
+;
diff --git a/roms/openbios/arch/ppc/qemu/tree.fs b/roms/openbios/arch/ppc/qemu/tree.fs
index 1ed838397..5b6bbc6f7 100644
--- a/roms/openbios/arch/ppc/qemu/tree.fs
+++ b/roms/openbios/arch/ppc/qemu/tree.fs
@@ -42,6 +42,14 @@ new-device
 	: close ;
 finish-device
 
+new-device
+	" rom" device-name
+	h# ff800000 encode-int 0 encode-int encode+ " reg" property
+	1 encode-int " #address-cells" property
+	h# ff800000 encode-int h# 800000 encode-int encode+
+	h# ff800000 encode-int encode+ " ranges" property
+finish-device
+
 \ -------------------------------------------------------------
 \ /packages
 \ -------------------------------------------------------------
diff --git a/roms/openbios/config/scripts/switch-arch b/roms/openbios/config/scripts/switch-arch
index d5e2f7710..7b8b457f6 100755
--- a/roms/openbios/config/scripts/switch-arch
+++ b/roms/openbios/config/scripts/switch-arch
@@ -99,23 +99,25 @@ archname()
 
 select_prefix()
 {
-    TARGETS="${1}-unknown-linux-gnu- ${1}-linux-gnu- ${1}-linux- ${1}-elf- ${1}-eabi-"
+    for target_arch ; do
+        TARGETS="${target_arch}-unknown-linux-gnu- ${target_arch}-linux-gnu- ${target_arch}-linux- ${target_arch}-elf- ${target_arch}-eabi-"
 
-    if [ x"$CROSS_COMPILE" != "x" ]; then
-        TARGETS=$CROSS_COMPILE
-    fi
+        if [ x"$CROSS_COMPILE" != "x" ]; then
+            TARGETS=$CROSS_COMPILE
+        fi
 
-    for TARGET in $TARGETS
-    do
-        if type ${TARGET}gcc > /dev/null 2>&1
-        then
+        for TARGET in $TARGETS
+        do
+            if type ${TARGET}gcc > /dev/null 2>&1
+            then
+                return
+            fi
+        done
+        if [ "$ARCH" = "$HOSTARCH" ]; then
             return
         fi
     done
-    if [ "$ARCH" = "$HOSTARCH" ]; then
-        return
-    fi
-    echo "ERROR: no ${1} cross-compiler found !" 1>&2
+    echo "ERROR: no $* cross-compiler found !" 1>&2
     exit 1
 }
 
@@ -251,7 +253,7 @@ for ARCH in $arch_list; do
         ;;
 
         ppc)
-        select_prefix powerpc
+        select_prefix powerpc powerpc64
         if [ "$unix" = "no" ]; then
             CFLAGS="-m32 -msoft-float -fno-builtin-bcopy -fno-builtin-log2"
             AS_FLAGS="-m32"
diff --git a/roms/openbios/drivers/cuda.c b/roms/openbios/drivers/cuda.c
index 9555dea49..ff5d22de2 100644
--- a/roms/openbios/drivers/cuda.c
+++ b/roms/openbios/drivers/cuda.c
@@ -144,8 +144,22 @@ static int cuda_adb_req (void *host, const uint8_t *snd_buf, int len,
  //   CUDA_DPRINTF("len: %d %02x\n", len, snd_buf[0]);
     len = cuda_request(host, ADB_PACKET, snd_buf, len, buffer);
     if (len > 1 && buffer[0] == ADB_PACKET) {
-        pos = buffer + 2;
-        len -= 2;
+        /* We handle 2 types of ADB packet here:
+               Normal: <type> <status> <data> ...
+               Error : <type> <status> <cmd> (<data> ...)
+           Ideally we should use buffer[1] (status) to determine whether this
+           is a normal or error packet but this requires a corresponding fix
+           in QEMU <= 2.4. Hence we temporarily handle it this way to ease
+           the transition. */
+        if (len > 2 && buffer[2] == snd_buf[0]) {
+            /* Error */
+            pos = buffer + 3;
+            len -= 3;
+        } else {
+            /* Normal */
+            pos = buffer + 2;
+            len -= 2;
+        }
     } else {
         pos = buffer + 1;
         len = -1;
@@ -380,7 +394,8 @@ powermgt_init(char *path)
 
 	ph = find_dev(buf);
 	set_property(ph, "device_type", "power-mgt", 10);
-	set_property(ph, "compatible", "power-mgt", 10);
+	set_property(ph, "mgt-kind", "min-consumption-pwm-led", strlen("min-consumption-pwm-led") + 1);
+	set_property(ph, "compatible", "cuda", strlen("cuda") + 1);
 }
 
 cuda_t *cuda_init (const char *path, phys_addr_t base)
diff --git a/roms/openbios/drivers/escc.c b/roms/openbios/drivers/escc.c
index 240043be3..afb97fa8a 100644
--- a/roms/openbios/drivers/escc.c
+++ b/roms/openbios/drivers/escc.c
@@ -380,12 +380,22 @@ ob_zs_init(phys_addr_t base, uint64_t offset, int intr, int slave, int keyboard)
 
 static void
 escc_add_channel(const char *path, const char *node, phys_addr_t addr,
-                 uint32_t offset)
+                 int esnum)
 {
     char buf[64], tty[32];
     phandle_t dnode, aliases;
-    int len;
-    cell props[2];
+
+    cell props[10];
+    int offset;
+    int legacy;
+
+    switch (esnum) {
+    case 2: offset = 1; legacy = 0; break;
+    case 3: offset = 0; legacy = 0; break;
+    case 4: offset = 1; legacy = 1; break;
+    case 5: offset = 0; legacy = 1; break;
+    default: return;
+    }
 
     /* add device */
 
@@ -411,16 +421,28 @@ escc_add_channel(const char *path, const char *node, phys_addr_t addr,
     set_property(dnode, "device_type", "serial",
                  strlen("serial") + 1);
 
-    snprintf(buf, sizeof(buf), "ch-%s", node);
-    len = strlen(buf) + 1;
-    snprintf(buf + len, sizeof(buf) - len, "CHRP,es2");
-    set_property(dnode, "compatible", buf, len + 9);
-
-    props[0] = IO_ESCC_OFFSET + offset * 0x20;
-    props[1] = 0x00000020;
-    set_property(dnode, "reg", (char *)&props, 2 * sizeof(cell));
+    snprintf(buf, sizeof(buf), "chrp,es%d", esnum);
+    set_property(dnode, "compatible", buf, 9);
+
+    if (legacy) {
+        props[0] = IO_ESCC_LEGACY_OFFSET + offset * 0x4;
+        props[1] = 0x00000001;
+        props[2] = IO_ESCC_LEGACY_OFFSET + offset * 0x4 + 2;
+        props[3] = 0x00000001;
+        props[4] = IO_ESCC_LEGACY_OFFSET + offset * 0x4 + 6;
+        props[5] = 0x00000001;
+        set_property(dnode, "reg", (char *)&props, 6 * sizeof(cell));
+    } else {
+        props[0] = IO_ESCC_OFFSET + offset * 0x20;
+        props[1] = 0x00000020;
+        set_property(dnode, "reg", (char *)&props, 2 * sizeof(cell));
+    }
 
-    props[0] = addr + IO_ESCC_OFFSET + offset * 0x20;
+    if (legacy) {
+        props[0] = addr + IO_ESCC_LEGACY_OFFSET + offset * 0x4;
+    } else {
+        props[0] = addr + IO_ESCC_OFFSET + offset * 0x20;
+    }
     OLDWORLD(set_property(dnode, "AAPL,address",
             (char *)&props, 1 * sizeof(cell)));
 
@@ -430,13 +452,21 @@ escc_add_channel(const char *path, const char *node, phys_addr_t addr,
 
     props[0] = (0x24) + offset;
     props[1] = 0;
+    props[2] = 0;
     NEWWORLD(set_property(dnode, "interrupts",
-             (char *)&props, 2 * sizeof(cell)));
+             (char *)&props, 3 * sizeof(cell)));
 
     device_end();
 
-    uart_init_line((unsigned char*)addr + IO_ESCC_OFFSET + offset * 0x20,
-                   CONFIG_SERIAL_SPEED);
+    if (legacy) {
+        uart_init_line(
+                (unsigned char*)addr + IO_ESCC_LEGACY_OFFSET + offset * 0x4,
+                CONFIG_SERIAL_SPEED);
+    } else {
+        uart_init_line(
+                (unsigned char*)addr + IO_ESCC_OFFSET + offset * 0x20,
+                CONFIG_SERIAL_SPEED);
+    }
 }
 
 void
@@ -467,10 +497,34 @@ escc_init(const char *path, phys_addr_t addr)
 
     fword("finish-device");
 
-    escc_add_channel(buf, "a", addr, 1);
-    escc_add_channel(buf, "b", addr, 0);
+    escc_add_channel(buf, "a", addr, 2);
+    escc_add_channel(buf, "b", addr, 3);
 
     escc_serial_dev = (unsigned char *)addr + IO_ESCC_OFFSET +
                  (CONFIG_SERIAL_PORT ? 0 : 0x20);
+
+    push_str(path);
+    fword("find-device");
+    fword("new-device");
+
+    push_str("escc-legacy");
+    fword("device-name");
+
+    snprintf(buf, sizeof(buf), "%s/escc-legacy", path);
+
+    dnode = find_dev(buf);
+
+    set_int_property(dnode, "#address-cells", 1);
+    props[0] = __cpu_to_be32(IO_ESCC_LEGACY_OFFSET);
+    props[1] = __cpu_to_be32(IO_ESCC_LEGACY_SIZE);
+    set_property(dnode, "reg", (char *)&props, sizeof(props));
+    set_property(dnode, "device_type", "escc-legacy",
+                 strlen("escc-legacy") + 1);
+    set_property(dnode, "compatible", "chrp,es1", 9);
+
+    fword("finish-device");
+
+    escc_add_channel(buf, "a", addr, 4);
+    escc_add_channel(buf, "b", addr, 5);
 }
 #endif
diff --git a/roms/openbios/drivers/escc.h b/roms/openbios/drivers/escc.h
index caaf00d40..e73f267b2 100644
--- a/roms/openbios/drivers/escc.h
+++ b/roms/openbios/drivers/escc.h
@@ -1,6 +1,8 @@
 
 #define IO_ESCC_SIZE    0x00001000
 #define IO_ESCC_OFFSET  0x00013000
+#define IO_ESCC_LEGACY_SIZE    0x00001000
+#define IO_ESCC_LEGACY_OFFSET  0x00012000
 
 #define ZS_REGS         8
 
diff --git a/roms/openbios/drivers/ide.c b/roms/openbios/drivers/ide.c
index 327c64a40..5125b78b9 100644
--- a/roms/openbios/drivers/ide.c
+++ b/roms/openbios/drivers/ide.c
@@ -1488,6 +1488,28 @@ int ob_ide_init(const char *path, uint32_t io_port0, uint32_t ctl_port0,
 	return 0;
 }
 
+void ob_ide_quiesce(void)
+{
+	struct ide_channel *channel;
+	int i;
+
+	channel = channels;
+	while (channel) {
+		for (i = 0; i < 2; i++) {
+			struct ide_drive *drive = &channel->drives[i];
+
+			if (!drive->present)
+				continue;
+
+			ob_ide_select_drive(drive);
+			ob_ide_software_reset(drive);
+			ob_ide_device_type_check(drive);
+		}
+
+		channel = channel->next;
+	}
+}
+
 #if defined(CONFIG_DRIVER_MACIO)
 static unsigned char
 macio_ide_inb(struct ide_channel *chan, unsigned int port)
@@ -1581,6 +1603,12 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 		set_property(dnode, "compatible", (is_oldworld() ?
 			     "heathrow-ata" : "keylargo-ata"), 13);
 
+		set_property(dnode, "model", ((current_channel == 3) ?
+			     "ata-3" : "ata-4"), strlen("ata-*") + 1);
+
+		set_property(dnode, "AAPL,connector", "ata",
+                             strlen("ata") + 1);
+
 		props[0] = 0x00000526;
 		props[1] = 0x00000085;
 		props[2] = 0x00000025;
@@ -1589,8 +1617,8 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 		props[5] = 0x00000000;
 		props[6] = 0x00000000;
 		props[7] = 0x00000000;
-		OLDWORLD(set_property(dnode, "AAPL,pio-timing",
-				      (char *)&props, 8*sizeof(props[0])));
+		set_property(dnode, "AAPL,pio-timing",
+				      (char *)&props, 8*sizeof(props[0]));
 
 		/* The first interrupt entry is the ide interrupt, the second
 		   the dbdma interrupt */
@@ -1634,8 +1662,8 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 				      (char *)&props, 2*sizeof(props[0])));
 
 		props[0] = 0;
-		OLDWORLD(set_property(dnode, "AAPL,bus-id", (char*)props,
-			 1 * sizeof(props[0])));
+		set_property(dnode, "AAPL,bus-id", (char*)props,
+			 1 * sizeof(props[0]));
 		IDE_DPRINTF(DEV_NAME": [io ports 0x%lx]\n",
 		            current_channel, chan->mmio);
 
diff --git a/roms/openbios/drivers/pci.c b/roms/openbios/drivers/pci.c
index 366f4a17f..935ecb8f8 100644
--- a/roms/openbios/drivers/pci.c
+++ b/roms/openbios/drivers/pci.c
@@ -824,7 +824,7 @@ int ebus_config_cb(const pci_config_t *config)
         ncells += pci_encode_phys_addr(props + ncells,
                                        flags, space_code, config->dev,
                                        PCI_BASE_ADDR_0 + (i * sizeof(uint32_t)),
-                                       0);
+                                       config->assigned[i] & ~mask);
 
         props[ncells++] = config->sizes[i];
     }
@@ -1420,6 +1420,12 @@ static void ob_pci_host_set_interrupt_map(phandle_t host)
         target_node = find_dev("/pci/mac-io/escc/ch-b");
         set_int_property(target_node, "interrupt-parent", dnode);
 
+        target_node = find_dev("/pci/mac-io/escc-legacy/ch-a");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        target_node = find_dev("/pci/mac-io/escc-legacy/ch-b");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
         /* QEMU only emulates 2 of the 3 ata buses currently */
         /* On a new world Mac these are not numbered but named by the
          * ATA version they support. Thus we have: ata-3, ata-3, ata-4
diff --git a/roms/openbios/include/drivers/drivers.h b/roms/openbios/include/drivers/drivers.h
index 3b83b12d1..48f81a870 100644
--- a/roms/openbios/include/drivers/drivers.h
+++ b/roms/openbios/include/drivers/drivers.h
@@ -52,6 +52,7 @@ void kbd_init(uint64_t base);
 /* drivers/ide.c */
 int ob_ide_init(const char *path, uint32_t io_port0, uint32_t ctl_port0,
                 uint32_t io_port1, uint32_t ctl_port1);
+void ob_ide_quiesce(void);
 int macio_ide_init(const char *path, uint32_t addr, int nb_channels);
 #endif
 #ifdef CONFIG_DRIVER_ESP
diff --git a/roms/openbios/libopenbios/bootinfo_load.c b/roms/openbios/libopenbios/bootinfo_load.c
index fa9e36bd4..f33678185 100644
--- a/roms/openbios/libopenbios/bootinfo_load.c
+++ b/roms/openbios/libopenbios/bootinfo_load.c
@@ -161,6 +161,12 @@ bootinfo_init_program(void)
 	feval("load-size");
 	size = POP();
 
+	/* Some bootinfo scripts contain a binary payload after the
+	   NULL-terminated Forth string such as OS 9. Restrict our
+	   size to just the Forth section, otherwise we end up trying
+	   to allocate memory for the entire binary which might fail. */
+	size = strnlen(base, size);
+
 	bootscript = malloc(size);
 	if (bootscript == NULL) {
 		DPRINTF("Can't malloc %d bytes\n", size);
diff --git a/rules.mak b/rules.mak
index aec27f81d..4551b9ea7 100644
--- a/rules.mak
+++ b/rules.mak
@@ -17,7 +17,7 @@ MAKEFLAGS += -rR
 QEMU_CXXFLAGS = -D__STDC_LIMIT_MACROS $(filter-out -Wstrict-prototypes -Wmissing-prototypes -Wnested-externs -Wold-style-declaration -Wold-style-definition -Wredundant-decls, $(QEMU_CFLAGS))
 
 # Flags for dependency generation
-QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(*D)/$(*F).d
+QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(@D)/$(*F).d
 
 # Same as -I$(SRC_PATH) -I., but for the nested source/object directories
 QEMU_INCLUDES += -I$(<D) -I$(@D)
@@ -368,6 +368,6 @@ define unnest-vars
                 $(error $o added in $v but $o-objs is not set)))
         $(shell mkdir -p ./ $(sort $(dir $($v))))
         # Include all the .d files
-        $(eval -include $(addsuffix *.d, $(sort $(dir $($v)))))
+        $(eval -include $(patsubst %.o,%.d,$(patsubst %.mo,%.d,$($v))))
         $(eval $v := $(filter-out %/,$($v))))
 endef
diff --git a/scripts/analyze-migration.py b/scripts/analyze-migration.py
index f6894bece..14553876a 100755
--- a/scripts/analyze-migration.py
+++ b/scripts/analyze-migration.py
@@ -252,6 +252,15 @@ class HTABSection(object):
     def getDict(self):
         return ""
 
+
+class ConfigurationSection(object):
+    def __init__(self, file):
+        self.file = file
+
+    def read(self):
+        name_len = self.file.read32()
+        name = self.file.readstr(len = name_len)
+
 class VMSDFieldGeneric(object):
     def __init__(self, desc, file):
         self.file = file
@@ -474,6 +483,7 @@ class MigrationDump(object):
     QEMU_VM_SECTION_FULL  = 0x04
     QEMU_VM_SUBSECTION    = 0x05
     QEMU_VM_VMDESCRIPTION = 0x06
+    QEMU_VM_CONFIGURATION = 0x07
     QEMU_VM_SECTION_FOOTER= 0x7e
 
     def __init__(self, filename):
@@ -514,6 +524,9 @@ class MigrationDump(object):
             section_type = file.read8()
             if section_type == self.QEMU_VM_EOF:
                 break
+            elif section_type == self.QEMU_VM_CONFIGURATION:
+                section = ConfigurationSection(file)
+                section.read()
             elif section_type == self.QEMU_VM_SECTION_START or section_type == self.QEMU_VM_SECTION_FULL:
                 section_id = file.read32()
                 name = file.readstr()
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 7f0aae977..b0f6e113c 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -141,44 +141,22 @@ our $Ident	= qr{
 		}x;
 our $Storage	= qr{extern|static|asmlinkage};
 our $Sparse	= qr{
-			__user|
-			__kernel|
-			__force|
-			__iomem|
-			__must_check|
-			__init_refok|
-			__kprobes|
-			__ref
+			__force
 		}x;
 
 # Notes to $Attribute:
-# We need \b after 'init' otherwise 'initconst' will cause a false positive in a check
 our $Attribute	= qr{
 			const|
-			__percpu|
-			__nocast|
-			__safe|
-			__bitwise__|
-			__packed__|
-			__packed2__|
-			__naked|
-			__maybe_unused|
-			__always_unused|
-			__noreturn|
-			__used|
-			__cold|
-			__noclone|
-			__deprecated|
-			__read_mostly|
-			__kprobes|
-			__(?:mem|cpu|dev|)(?:initdata|initconst|init\b)|
-			____cacheline_aligned|
-			____cacheline_aligned_in_smp|
-			____cacheline_internodealigned_in_smp|
-			__weak
+			volatile|
+			QEMU_NORETURN|
+			QEMU_WARN_UNUSED_RESULT|
+			QEMU_SENTINEL|
+			QEMU_ARTIFICIAL|
+			QEMU_PACKED|
+			GCC_FMT_ATTR
 		  }x;
 our $Modifier;
-our $Inline	= qr{inline|__always_inline|noinline};
+our $Inline	= qr{inline};
 our $Member	= qr{->$Ident|\.$Ident|\[[^]]*\]};
 our $Lval	= qr{$Ident(?:$Member)*};
 
@@ -215,14 +193,6 @@ our $typeTypedefs = qr{(?x:
         | QEMUBH                            # all uppercase
 )};
 
-our $logFunctions = qr{(?x:
-	printk|
-	pr_(debug|dbg|vdbg|devel|info|warning|err|notice|alert|crit|emerg|cont)|
-	(dev|netdev|netif)_(printk|dbg|vdbg|info|warn|err|notice|alert|crit|emerg|WARN)|
-	WARN|
-	panic
-)};
-
 our @typeList = (
 	qr{void},
 	qr{(?:unsigned\s+)?char},
@@ -243,20 +213,20 @@ our @typeList = (
 	qr{${Ident}_handler},
 	qr{${Ident}_handler_fn},
 );
+
+# This can be modified by sub possible.  Since it can be empty, be careful
+# about regexes that always match, because they can cause infinite loops.
 our @modifierList = (
-	qr{fastcall},
 );
 
-our $allowed_asm_includes = qr{(?x:
-	irq|
-	memory
-)};
-# memory.h: ARM has a custom one
-
 sub build_types {
-	my $mods = "(?x:  \n" . join("|\n  ", @modifierList) . "\n)";
 	my $all = "(?x:  \n" . join("|\n  ", @typeList) . "\n)";
-	$Modifier	= qr{(?:$Attribute|$Sparse|$mods)};
+	if (@modifierList > 0) {
+		my $mods = "(?x:  \n" . join("|\n  ", @modifierList) . "\n)";
+		$Modifier = qr{(?:$Attribute|$Sparse|$mods)};
+	} else {
+		$Modifier = qr{(?:$Attribute|$Sparse)};
+	}
 	$NonptrType	= qr{
 			(?:$Modifier\s+|const\s+)*
 			(?:
@@ -277,27 +247,6 @@ build_types();
 
 $chk_signoff = 0 if ($file);
 
-my @dep_includes = ();
-my @dep_functions = ();
-my $removal = "Documentation/feature-removal-schedule.txt";
-if ($tree && -f "$root/$removal") {
-	open(my $REMOVE, '<', "$root/$removal") ||
-				die "$P: $removal: open failed - $!\n";
-	while (<$REMOVE>) {
-		if (/^Check:\s+(.*\S)/) {
-			for my $entry (split(/[, ]+/, $1)) {
-				if ($entry =~ m@include/(.*)@) {
-					push(@dep_includes, $1);
-
-				} elsif ($entry !~ m@/@) {
-					push(@dep_functions, $entry);
-				}
-			}
-		}
-	}
-	close($REMOVE);
-}
-
 my @rawlines = ();
 my @lines = ();
 my $vname;
@@ -633,7 +582,7 @@ sub statement_block_size {
 	my ($stmt) = @_;
 
 	$stmt =~ s/(^|\n)./$1/g;
-	$stmt =~ s/^\s*{//;
+	$stmt =~ s/^\s*\{//;
 	$stmt =~ s/}\s*$//;
 	$stmt =~ s/^\s*//;
 	$stmt =~ s/\s*$//;
@@ -1061,7 +1010,9 @@ sub possible {
 			case|
 			else|
 			asm|__asm__|
-			do
+			do|
+			\#|
+			\#\#
 		)(?:\s|$)|
 		^(?:typedef|struct|enum)\b
 	    )}x;
@@ -1127,33 +1078,6 @@ sub CHK {
 	}
 }
 
-sub check_absolute_file {
-	my ($absolute, $herecurr) = @_;
-	my $file = $absolute;
-
-	##print "absolute<$absolute>\n";
-
-	# See if any suffix of this path is a path within the tree.
-	while ($file =~ s@^[^/]*/@@) {
-		if (-f "$root/$file") {
-			##print "file<$file>\n";
-			last;
-		}
-	}
-	if (! -f _)  {
-		return 0;
-	}
-
-	# It is, so see if the prefix is acceptable.
-	my $prefix = $absolute;
-	substr($prefix, -length($file)) = '';
-
-	##print "prefix<$prefix>\n";
-	if ($prefix ne ".../") {
-		WARN("use relative pathname instead of absolute in changelog text\n" . $herecurr);
-	}
-}
-
 sub process {
 	my $filename = shift;
 
@@ -1196,10 +1120,6 @@ sub process {
 	my %suppress_export;
 
 	# Pre-scan the patch sanitizing the lines.
-	# Pre-scan the patch looking for any __setup documentation.
-	#
-	my @setup_docs = ();
-	my $setup_docs = 0;
 
 	sanitise_line_reset();
 	my $line;
@@ -1207,13 +1127,6 @@ sub process {
 		$linenr++;
 		$line = $rawline;
 
-		if ($rawline=~/^\+\+\+\s+(\S+)/) {
-			$setup_docs = 0;
-			if ($1 =~ m@Documentation/kernel-parameters.txt$@) {
-				$setup_docs = 1;
-			}
-			#next;
-		}
 		if ($rawline=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
 			$realline=$1-1;
 			if (defined $2) {
@@ -1272,10 +1185,6 @@ sub process {
 
 		#print "==>$rawline\n";
 		#print "-->$line\n";
-
-		if ($setup_docs && $line =~ /^\+/) {
-			push(@setup_docs, $line);
-		}
 	}
 
 	$prefix = '';
@@ -1350,9 +1259,6 @@ sub process {
 				WARN("patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n");
 			}
 
-			if ($realfile =~ m@^include/asm/@) {
-				ERROR("do not modify files in include/asm, change architecture specific files in include/asm-<architecture>\n" . "$here$rawline\n");
-			}
 			next;
 		}
 
@@ -1367,7 +1273,7 @@ sub process {
 # Check for incorrect file permissions
 		if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) {
 			my $permhere = $here . "FILE: $realfile\n";
-			if ($realfile =~ /(Makefile|Kconfig|\.c|\.cpp|\.h|\.S|\.tmpl)$/) {
+			if ($realfile =~ /(\bMakefile(?:\.objs)?|\.c|\.cc|\.cpp|\.h|\.mak|\.[sS])$/) {
 				ERROR("do not set execute permissions for source files\n" . $permhere);
 			}
 		}
@@ -1392,20 +1298,6 @@ sub process {
 				$herecurr) if (!$emitted_corrupt++);
 		}
 
-# Check for absolute kernel paths.
-		if ($tree) {
-			while ($line =~ m{(?:^|\s)(/\S*)}g) {
-				my $file = $1;
-
-				if ($file =~ m{^(.*?)(?::\d+)+:?$} &&
-				    check_absolute_file($1, $herecurr)) {
-					#
-				} else {
-					check_absolute_file($file, $herecurr);
-				}
-			}
-		}
-
 # UTF-8 regex found at http://www.w3.org/International/questions/qa-forms-utf-8.en.php
 		if (($realfile =~ /^$/ || $line =~ /^\+/) &&
 		    $rawline !~ m/^$UTF8*$/) {
@@ -1432,45 +1324,12 @@ sub process {
 			$rpt_cleaners = 1;
 		}
 
-# check for Kconfig help text having a real description
-# Only applies when adding the entry originally, after that we do not have
-# sufficient context to determine whether it is indeed long enough.
-		if ($realfile =~ /Kconfig/ &&
-		    $line =~ /\+\s*(?:---)?help(?:---)?$/) {
-			my $length = 0;
-			my $cnt = $realcnt;
-			my $ln = $linenr + 1;
-			my $f;
-			my $is_end = 0;
-			while ($cnt > 0 && defined $lines[$ln - 1]) {
-				$f = $lines[$ln - 1];
-				$cnt-- if ($lines[$ln - 1] !~ /^-/);
-				$is_end = $lines[$ln - 1] =~ /^\+/;
-				$ln++;
-
-				next if ($f =~ /^-/);
-				$f =~ s/^.//;
-				$f =~ s/#.*//;
-				$f =~ s/^\s+//;
-				next if ($f =~ /^$/);
-				if ($f =~ /^\s*config\s/) {
-					$is_end = 1;
-					last;
-				}
-				$length++;
-			}
-			WARN("please write a paragraph that describes the config symbol fully\n" . $herecurr) if ($is_end && $length < 4);
-			#print "is_end<$is_end> length<$length>\n";
-		}
-
 # check we are in a valid source file if not then ignore this hunk
 		next if ($realfile !~ /\.(h|c|cpp|s|S|pl|sh)$/);
 
 #80 column limit
-		if ($line =~ /^\+/ && $prevrawline !~ /\/\*\*/ &&
-		    $rawline !~ /^.\s*\*\s*\@$Ident\s/ &&
-		    !($line =~ /^\+\s*$logFunctions\s*\(\s*(?:(KERN_\S+\s*|[^"]*))?"[X\t]*"\s*(?:,|\)\s*;)\s*$/ ||
-		    $line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
+		if ($line =~ /^\+/ &&
+		    !($line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
 		    $length > 80)
 		{
 			WARN("line over 80 characters\n" . $herecurr);
@@ -1486,18 +1345,6 @@ sub process {
 			WARN("adding a line without newline at end of file\n" . $herecurr);
 		}
 
-# Blackfin: use hi/lo macros
-		if ($realfile =~ m@arch/blackfin/.*\.S$@) {
-			if ($line =~ /\.[lL][[:space:]]*=.*&[[:space:]]*0x[fF][fF][fF][fF]/) {
-				my $herevet = "$here\n" . cat_vet($line) . "\n";
-				ERROR("use the LO() macro, not (... & 0xFFFF)\n" . $herevet);
-			}
-			if ($line =~ /\.[hH][[:space:]]*=.*>>[[:space:]]*16/) {
-				my $herevet = "$here\n" . cat_vet($line) . "\n";
-				ERROR("use the HI() macro, not (... >> 16)\n" . $herevet);
-			}
-		}
-
 # check we are in a valid source file C or perl if not then ignore this hunk
 		next if ($realfile !~ /\.(h|c|cpp|pl)$/);
 
@@ -1516,16 +1363,6 @@ sub process {
 			WARN("CVS style keyword markers, these will _not_ be updated\n". $herecurr);
 		}
 
-# Blackfin: don't use __builtin_bfin_[cs]sync
-		if ($line =~ /__builtin_bfin_csync/) {
-			my $herevet = "$here\n" . cat_vet($line) . "\n";
-			ERROR("use the CSYNC() macro in asm/blackfin.h\n" . $herevet);
-		}
-		if ($line =~ /__builtin_bfin_ssync/) {
-			my $herevet = "$here\n" . cat_vet($line) . "\n";
-			ERROR("use the SSYNC() macro in asm/blackfin.h\n" . $herevet);
-		}
-
 # Check for potential 'bare' types
 		my ($stat, $cond, $line_nr_next, $remain_next, $off_next,
 		    $realline_next);
@@ -1644,7 +1481,7 @@ sub process {
 			# 79 or 80 characters, it is no longer possible to add a space and an
 			# opening brace there)
 			if ($#ctx == 0 && $ctx !~ /{\s*/ &&
-			    defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/ &&
+			    defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*\{/ &&
 			    defined($lines[$ctx_ln - 2]) && length($lines[$ctx_ln - 2]) < 80) {
 				ERROR("that open brace { should be on the previous line\n" .
 					"$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
@@ -1684,7 +1521,7 @@ sub process {
 			my $continuation = 0;
 			my $check = 0;
 			$s =~ s/^.*\bdo\b//;
-			$s =~ s/^\s*{//;
+			$s =~ s/^\s*\{//;
 			if ($s =~ s/^\s*\\//) {
 				$continuation = 1;
 			}
@@ -1783,7 +1620,7 @@ sub process {
 		}
 
 # check for initialisation to aggregates open brace on the next line
-		if ($line =~ /^.\s*{/ &&
+		if ($line =~ /^.\s*\{/ &&
 		    $prevline =~ /(?:^|[^=])=\s*$/) {
 			ERROR("that open brace { should be on the previous line\n" . $hereprev);
 		}
@@ -1809,50 +1646,6 @@ sub process {
 		$line =~ s@//.*@@;
 		$opline =~ s@//.*@@;
 
-# EXPORT_SYMBOL should immediately follow the thing it is exporting, consider
-# the whole statement.
-#print "APW <$lines[$realline_next - 1]>\n";
-		if (defined $realline_next &&
-		    exists $lines[$realline_next - 1] &&
-		    !defined $suppress_export{$realline_next} &&
-		    ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
-		     $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
-			# Handle definitions which produce identifiers with
-			# a prefix:
-			#   XXX(foo);
-			#   EXPORT_SYMBOL(something_foo);
-			my $name = $1;
-			if ($stat =~ /^.([A-Z_]+)\s*\(\s*($Ident)/ &&
-			    $name =~ /^${Ident}_$2/) {
-#print "FOO C name<$name>\n";
-				$suppress_export{$realline_next} = 1;
-
-			} elsif ($stat !~ /(?:
-				\n.}\s*$|
-				^.DEFINE_$Ident\(\Q$name\E\)|
-				^.DECLARE_$Ident\(\Q$name\E\)|
-				^.LIST_HEAD\(\Q$name\E\)|
-				^.(?:$Storage\s+)?$Type\s*\(\s*\*\s*\Q$name\E\s*\)\s*\(|
-				\b\Q$name\E(?:\s+$Attribute)*\s*(?:;|=|\[|\()
-			    )/x) {
-#print "FOO A<$lines[$realline_next - 1]> stat<$stat> name<$name>\n";
-				$suppress_export{$realline_next} = 2;
-			} else {
-				$suppress_export{$realline_next} = 1;
-			}
-		}
-		if (!defined $suppress_export{$linenr} &&
-		    $prevline =~ /^.\s*$/ &&
-		    ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
-		     $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
-#print "FOO B <$lines[$linenr - 1]>\n";
-			$suppress_export{$linenr} = 2;
-		}
-		if (defined $suppress_export{$linenr} &&
-		    $suppress_export{$linenr} == 2) {
-			WARN("EXPORT_SYMBOL(foo); should immediately follow its function/variable\n" . $herecurr);
-		}
-
 # check for global initialisers.
 		if ($line =~ /^.$Type\s*$Ident\s*(?:\s+$Modifier)*\s*=\s*(0|NULL|false)\s*;/) {
 			ERROR("do not initialise globals to 0 or NULL\n" .
@@ -1900,56 +1693,22 @@ sub process {
 			}
 		}
 
-# # no BUG() or BUG_ON()
-# 		if ($line =~ /\b(BUG|BUG_ON)\b/) {
-# 			print "Try to use WARN_ON & Recovery code rather than BUG() or BUG_ON()\n";
-# 			print "$herecurr";
-# 			$clean = 0;
-# 		}
-
-		if ($line =~ /\bLINUX_VERSION_CODE\b/) {
-			WARN("LINUX_VERSION_CODE should be avoided, code should be for the version to which it is merged\n" . $herecurr);
-		}
-
-# printk should use KERN_* levels.  Note that follow on printk's on the
-# same line do not need a level, so we use the current block context
-# to try and find and validate the current printk.  In summary the current
-# printk includes all preceding printk's which have no newline on the end.
-# we assume the first bad printk is the one to report.
-		if ($line =~ /\bprintk\((?!KERN_)\s*"/) {
-			my $ok = 0;
-			for (my $ln = $linenr - 1; $ln >= $first_line; $ln--) {
-				#print "CHECK<$lines[$ln - 1]\n";
-				# we have a preceding printk if it ends
-				# with "\n" ignore it, else it is to blame
-				if ($lines[$ln - 1] =~ m{\bprintk\(}) {
-					if ($rawlines[$ln - 1] !~ m{\\n"}) {
-						$ok = 1;
-					}
-					last;
-				}
-			}
-			if ($ok == 0) {
-				WARN("printk() should include KERN_ facility level\n" . $herecurr);
-			}
-		}
-
 # function brace can't be on same line, except for #defines of do while,
 # or if closed on same line
-		if (($line=~/$Type\s*$Ident\(.*\).*\s{/) and
-		    !($line=~/\#\s*define.*do\s{/) and !($line=~/}/)) {
+		if (($line=~/$Type\s*$Ident\(.*\).*\s\{/) and
+		    !($line=~/\#\s*define.*do\s\{/) and !($line=~/}/)) {
 			ERROR("open brace '{' following function declarations go on the next line\n" . $herecurr);
 		}
 
 # open braces for enum, union and struct go on the same line.
-		if ($line =~ /^.\s*{/ &&
+		if ($line =~ /^.\s*\{/ &&
 		    $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) {
 			ERROR("open brace '{' following $1 go on the same line\n" . $hereprev);
 		}
 
 # missing space after union, struct or enum definition
 		if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?(?:\s+$Ident)?[=\{]/) {
-		    WARN("missing space after $1 definition\n" . $herecurr);
+		    ERROR("missing space after $1 definition\n" . $herecurr);
 		}
 
 # check for spacing round square brackets; allowed:
@@ -2091,7 +1850,7 @@ sub process {
                                 # not required when having a single },{ on one line
 				} elsif ($op eq ',') {
 					if ($ctx !~ /.x[WEC]/ && $cc !~ /^}/ &&
-                                            ($elements[$n] . $elements[$n + 2]) !~ " *}{") {
+                                            ($elements[$n] . $elements[$n + 2]) !~ " *}\\{") {
 						ERROR("space required after that '$op' $at\n" . $hereptr);
 					}
 
@@ -2190,29 +1949,9 @@ sub process {
 			}
 		}
 
-# check for multiple assignments
-		if ($line =~ /^.\s*$Lval\s*=\s*$Lval\s*=(?!=)/) {
-			CHK("multiple assignments should be avoided\n" . $herecurr);
-		}
-
-## # check for multiple declarations, allowing for a function declaration
-## # continuation.
-## 		if ($line =~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Ident.*/ &&
-## 		    $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) {
-##
-## 			# Remove any bracketed sections to ensure we do not
-## 			# falsly report the parameters of functions.
-## 			my $ln = $line;
-## 			while ($ln =~ s/\([^\(\)]*\)//g) {
-## 			}
-## 			if ($ln =~ /,/) {
-## 				WARN("declaring multiple variables together should be avoided\n" . $herecurr);
-## 			}
-## 		}
-
 #need space before brace following if, while, etc
-		if (($line =~ /\(.*\){/ && $line !~ /\($Type\){/) ||
-		    $line =~ /do{/) {
+		if (($line =~ /\(.*\)\{/ && $line !~ /\($Type\)\{/) ||
+		    $line =~ /do\{/) {
 			ERROR("space required before the open brace '{'\n" . $herecurr);
 		}
 
@@ -2267,7 +2006,7 @@ sub process {
 		if ($line =~ /^.\s*return\s*(E[A-Z]*)\s*;/) {
 			my $name = $1;
 			if ($name ne 'EOF' && $name ne 'ERROR') {
-				CHK("return of an errno should typically be -ve (return -$1)\n" . $herecurr);
+				WARN("return of an errno should typically be -ve (return -$1)\n" . $herecurr);
 			}
 		}
 
@@ -2398,22 +2137,6 @@ sub process {
 			WARN("Whitepspace after \\ makes next lines useless\n" . $herecurr);
 		}
 
-#warn if <asm/foo.h> is #included and <linux/foo.h> is available (uses RAW line)
-		if ($tree && $rawline =~ m{^.\s*\#\s*include\s*\<asm\/(.*)\.h\>}) {
-			my $file = "$1.h";
-			my $checkfile = "include/linux/$file";
-			if (-f "$root/$checkfile" &&
-			    $realfile ne $checkfile &&
-			    $1 !~ /$allowed_asm_includes/)
-			{
-				if ($realfile =~ m{^arch/}) {
-					CHK("Consider using #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
-				} else {
-					WARN("Use #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
-				}
-			}
-		}
-
 # multi-statement macros should be enclosed in a do while loop, grab the
 # first statement and ensure its the whole macro if its not enclosed
 # in a known good container
@@ -2508,15 +2231,6 @@ sub process {
 			}
 		}
 
-# make sure symbols are always wrapped with VMLINUX_SYMBOL() ...
-# all assignments may have only one of the following with an assignment:
-#	.
-#	ALIGN(...)
-#	VMLINUX_SYMBOL(...)
-		if ($realfile eq 'vmlinux.lds.h' && $line =~ /(?:(?:^|\s)$Ident\s*=|=\s*$Ident(?:\s|$))/) {
-			WARN("vmlinux.lds.h needs VMLINUX_SYMBOL() around C-visible symbols\n" . $herecurr);
-		}
-
 # check for missing bracing round if etc
 		if ($line =~ /(^.*)\bif\b/ && $line !~ /\#\s*if/) {
 			my ($level, $endln, @chunks) =
@@ -2551,7 +2265,7 @@ sub process {
 					my $spaced_block = $block;
 					$spaced_block =~ s/\n\+/ /g;
 
-					$seen++ if ($spaced_block =~ /^\s*{/);
+					$seen++ if ($spaced_block =~ /^\s*\{/);
 
                                         print "APW: cond<$cond> block<$block> allowed<$allowed>\n"
                                             if $dbg_adv_apw;
@@ -2644,64 +2358,23 @@ sub process {
 			}
 		}
 
-# don't include deprecated include files (uses RAW line)
-		for my $inc (@dep_includes) {
-			if ($rawline =~ m@^.\s*\#\s*include\s*\<$inc>@) {
-				ERROR("Don't use <$inc>: see Documentation/feature-removal-schedule.txt\n" . $herecurr);
-			}
-		}
-
-# don't use deprecated functions
-		for my $func (@dep_functions) {
-			if ($line =~ /\b$func\b/) {
-				ERROR("Don't use $func(): see Documentation/feature-removal-schedule.txt\n" . $herecurr);
-			}
-		}
-
 # no volatiles please
 		my $asm_volatile = qr{\b(__asm__|asm)\s+(__volatile__|volatile)\b};
 		if ($line =~ /\bvolatile\b/ && $line !~ /$asm_volatile/) {
 			WARN("Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr);
 		}
 
-# SPIN_LOCK_UNLOCKED & RW_LOCK_UNLOCKED are deprecated
-		if ($line =~ /\b(SPIN_LOCK_UNLOCKED|RW_LOCK_UNLOCKED)/) {
-			ERROR("Use of $1 is deprecated: see Documentation/spinlocks.txt\n" . $herecurr);
-		}
-
 # warn about #if 0
 		if ($line =~ /^.\s*\#\s*if\s+0\b/) {
-			CHK("if this code is redundant consider removing it\n" .
+			WARN("if this code is redundant consider removing it\n" .
 				$herecurr);
 		}
 
-# check for needless kfree() checks
+# check for needless g_free() checks
 		if ($prevline =~ /\bif\s*\(([^\)]*)\)/) {
 			my $expr = $1;
-			if ($line =~ /\bkfree\(\Q$expr\E\);/) {
-				WARN("kfree(NULL) is safe this check is probably not required\n" . $hereprev);
-			}
-		}
-# check for needless usb_free_urb() checks
-		if ($prevline =~ /\bif\s*\(([^\)]*)\)/) {
-			my $expr = $1;
-			if ($line =~ /\busb_free_urb\(\Q$expr\E\);/) {
-				WARN("usb_free_urb(NULL) is safe this check is probably not required\n" . $hereprev);
-			}
-		}
-
-# prefer usleep_range over udelay
-		if ($line =~ /\budelay\s*\(\s*(\w+)\s*\)/) {
-			# ignore udelay's < 10, however
-			if (! (($1 =~ /(\d+)/) && ($1 < 10)) ) {
-				CHK("usleep_range is preferred over udelay; see Documentation/timers/timers-howto.txt\n" . $line);
-			}
-		}
-
-# warn about unexpectedly long msleep's
-		if ($line =~ /\bmsleep\s*\((\d+)\);/) {
-			if ($1 < 20) {
-				WARN("msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.txt\n" . $line);
+			if ($line =~ /\bg_free\(\Q$expr\E\);/) {
+				WARN("g_free(NULL) is safe this check is probably not required\n" . $hereprev);
 			}
 		}
 
@@ -2716,24 +2389,17 @@ sub process {
 		if ($line =~ /^.\s*\#\s*(ifdef|ifndef|elif)\s\s+/) {
 			ERROR("exactly one space required after that #$1\n" . $herecurr);
 		}
-
-# check for spinlock_t definitions without a comment.
-		if ($line =~ /^.\s*(struct\s+mutex|spinlock_t)\s+\S+;/ ||
-		    $line =~ /^.\s*(DEFINE_MUTEX)\s*\(/) {
-			my $which = $1;
-			if (!ctx_has_comment($first_line, $linenr)) {
-				CHK("$1 definition without comment\n" . $herecurr);
-			}
-		}
 # check for memory barriers without a comment.
-		if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
+		if ($line =~ /\b(smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
 			if (!ctx_has_comment($first_line, $linenr)) {
-				CHK("memory barrier without comment\n" . $herecurr);
+				WARN("memory barrier without comment\n" . $herecurr);
 			}
 		}
 # check of hardware specific defines
-		if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) {
-			CHK("architecture specific defines should be avoided\n" .  $herecurr);
+# we have e.g. CONFIG_LINUX and CONFIG_WIN32 for common cases
+# where they might be necessary.
+		if ($line =~ m@^.\s*\#\s*if.*\b__@) {
+			WARN("architecture specific defines should be avoided\n" .  $herecurr);
 		}
 
 # Check that the storage class is at the beginning of a declaration
@@ -2748,11 +2414,6 @@ sub process {
 			ERROR("inline keyword should sit between storage class and type\n" . $herecurr);
 		}
 
-# Check for __inline__ and __inline, prefer inline
-		if ($line =~ /\b(__inline__|__inline)\b/) {
-			WARN("plain inline is preferred over $1\n" . $herecurr);
-		}
-
 # check for sizeof(&)
 		if ($line =~ /\bsizeof\s*\(\s*\&/) {
 			WARN("sizeof(& should be avoided\n" . $herecurr);
@@ -2785,98 +2446,55 @@ sub process {
 			WARN("externs should be avoided in .c files\n" .  $herecurr);
 		}
 
-# checks for new __setup's
-		if ($rawline =~ /\b__setup\("([^"]*)"/) {
-			my $name = $1;
-
-			if (!grep(/$name/, @setup_docs)) {
-				CHK("__setup appears un-documented -- check Documentation/kernel-parameters.txt\n" . $herecurr);
+# check for pointless casting of g_malloc return
+		if ($line =~ /\*\s*\)\s*g_(try)?(m|re)alloc(0?)(_n)?\b/) {
+			if ($2 == 'm') {
+				WARN("unnecessary cast may hide bugs, use g_$1new$3 instead\n" . $herecurr);
+			} else {
+				WARN("unnecessary cast may hide bugs, use g_$1renew$3 instead\n" . $herecurr);
 			}
 		}
 
-# check for pointless casting of kmalloc return
-		if ($line =~ /\*\s*\)\s*k[czm]alloc\b/) {
-			WARN("unnecessary cast may hide bugs, see http://c-faq.com/malloc/mallocnocast.html\n" . $herecurr);
-		}
-
 # check for gcc specific __FUNCTION__
 		if ($line =~ /__FUNCTION__/) {
 			WARN("__func__ should be used instead of gcc specific __FUNCTION__\n"  . $herecurr);
 		}
 
-# check for semaphores used as mutexes
-		if ($line =~ /^.\s*(DECLARE_MUTEX|init_MUTEX)\s*\(/) {
-			WARN("mutexes are preferred for single holder semaphores\n" . $herecurr);
-		}
-# check for semaphores used as mutexes
-		if ($line =~ /^.\s*init_MUTEX_LOCKED\s*\(/) {
-			WARN("consider using a completion\n" . $herecurr);
-
-		}
-# recommend strict_strto* over simple_strto*
-		if ($line =~ /\bsimple_(strto.*?)\s*\(/) {
-			WARN("consider using strict_$1 in preference to simple_$1\n" . $herecurr);
+# recommend qemu_strto* over strto* for numeric conversions
+		if ($line =~ /\b(strto[^k].*?)\s*\(/) {
+			WARN("consider using qemu_$1 in preference to $1\n" . $herecurr);
 		}
-# check for __initcall(), use device_initcall() explicitly please
-		if ($line =~ /^.\s*__initcall\s*\(/) {
-			WARN("please use device_initcall() instead of __initcall()\n" . $herecurr);
+# check for module_init(), use category-specific init macros explicitly please
+		if ($line =~ /^module_init\s*\(/) {
+			WARN("please use block_init(), type_init() etc. instead of module_init()\n" . $herecurr);
 		}
 # check for various ops structs, ensure they are const.
-		my $struct_ops = qr{acpi_dock_ops|
-				address_space_operations|
-				backlight_ops|
-				block_device_operations|
-				dentry_operations|
-				dev_pm_ops|
-				dma_map_ops|
-				extent_io_ops|
-				file_lock_operations|
-				file_operations|
-				hv_ops|
-				ide_dma_ops|
-				intel_dvo_dev_ops|
-				item_operations|
-				iwl_ops|
-				kgdb_arch|
-				kgdb_io|
-				kset_uevent_ops|
-				lock_manager_operations|
-				microcode_ops|
-				mtrr_ops|
-				neigh_ops|
-				nlmsvc_binding|
-				pci_raw_ops|
-				pipe_buf_operations|
-				platform_hibernation_ops|
-				platform_suspend_ops|
-				proto_ops|
-				rpc_pipe_ops|
-				seq_operations|
-				snd_ac97_build_ops|
-				soc_pcmcia_socket_ops|
-				stacktrace_ops|
-				sysfs_ops|
-				tty_operations|
-				usb_mon_operations|
-				wd_ops}x;
+		my $struct_ops = qr{AIOCBInfo|
+				BdrvActionOps|
+				BlockDevOps|
+				BlockJobDriver|
+				DisplayChangeListenerOps|
+				GraphicHwOps|
+				IDEDMAOps|
+				KVMCapabilityInfo|
+				MemoryRegionIOMMUOps|
+				MemoryRegionOps|
+				MemoryRegionPortio|
+				QEMUFileOps|
+				SCSIBusInfo|
+				SCSIReqOps|
+				Spice[A-Z][a-zA-Z0-9]*Interface|
+				TPMDriverOps|
+				USBDesc[A-Z][a-zA-Z0-9]*|
+				VhostOps|
+				VMStateDescription|
+				VMStateInfo}x;
 		if ($line !~ /\bconst\b/ &&
-		    $line =~ /\bstruct\s+($struct_ops)\b/) {
+		    $line =~ /\b($struct_ops)\b/) {
 			WARN("struct $1 should normally be const\n" .
 				$herecurr);
 		}
 
-# use of NR_CPUS is usually wrong
-# ignore definitions of NR_CPUS and usage to define arrays as likely right
-		if ($line =~ /\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*\s*#\s*if\b.*\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*\s*#\s*define\b.*\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*$Declare\s.*\[[^\]]*NR_CPUS[^\]]*\]/ &&
-		    $line !~ /\[[^\]]*\.\.\.[^\]]*NR_CPUS[^\]]*\]/ &&
-		    $line !~ /\[[^\]]*NR_CPUS[^\]]*\.\.\.[^\]]*\]/)
-		{
-			WARN("usage of NR_CPUS is often wrong - consider using cpu_possible(), num_possible_cpus(), for_each_possible_cpu(), etc\n" . $herecurr);
-		}
-
 # check for %L{u,d,i} in strings
 		my $string;
 		while ($line =~ /(?:^|")([X\t]*)(?:"|$)/g) {
@@ -2888,25 +2506,6 @@ sub process {
 			}
 		}
 
-# whine mightly about in_atomic
-		if ($line =~ /\bin_atomic\s*\(/) {
-			if ($realfile =~ m@^drivers/@) {
-				ERROR("do not use in_atomic in drivers\n" . $herecurr);
-			} elsif ($realfile !~ m@^kernel/@) {
-				WARN("use of in_atomic() is incorrect outside core kernel code\n" . $herecurr);
-			}
-		}
-
-# check for lockdep_set_novalidate_class
-		if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ ||
-		    $line =~ /__lockdep_no_validate__\s*\)/ ) {
-			if ($realfile !~ m@^kernel/lockdep@ &&
-			    $realfile !~ m@^include/linux/lockdep@ &&
-			    $realfile !~ m@^drivers/base/core@) {
-				ERROR("lockdep_no_validate class is reserved for device->mutex.\n" . $herecurr);
-			}
-		}
-
 # QEMU specific tests
 		if ($rawline =~ /\b(?:Qemu|QEmu)\b/) {
 			WARN("use QEMU instead of Qemu or QEmu\n" . $herecurr);
diff --git a/scripts/cocci-macro-file.h b/scripts/cocci-macro-file.h
new file mode 100644
index 000000000..eceb4be73
--- /dev/null
+++ b/scripts/cocci-macro-file.h
@@ -0,0 +1,119 @@
+/* Macro file for Coccinelle
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or, at your
+ * option, any later version.  See the COPYING file in the top-level directory.
+ */
+
+/* Coccinelle only does limited parsing of headers, and chokes on some idioms
+ * defined in compiler.h and queue.h.  Macros that Coccinelle must know about
+ * in order to parse .c files must be in a separate macro file---which is
+ * exactly what you're staring at now.
+ *
+ * To use this file, add the "--macro-file scripts/cocci-macro-file.h" to the
+ * Coccinelle command line.
+ */
+
+/* From qemu/compiler.h */
+#define QEMU_GNUC_PREREQ(maj, min) 1
+#define QEMU_NORETURN __attribute__ ((__noreturn__))
+#define QEMU_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#define QEMU_SENTINEL __attribute__((sentinel))
+#define QEMU_ARTIFICIAL __attribute__((always_inline, artificial))
+#define QEMU_PACKED __attribute__((gcc_struct, packed))
+
+#define cat(x,y) x ## y
+#define cat2(x,y) cat(x,y)
+#define QEMU_BUILD_BUG_ON(x) \
+    typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
+
+#define GCC_FMT_ATTR(n, m) __attribute__((format(gnu_printf, n, m)))
+
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)	tostring(s)
+#define tostring(s)	#s
+
+#define typeof_field(type, field) typeof(((type *)0)->field)
+#define type_check(t1,t2) ((t1*)0 - (t2*)0)
+
+/* From qemu/queue.h */
+
+#define QLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *lh_first;  /* first element */                     \
+}
+
+#define QLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *le_next;   /* next element */                      \
+        struct type **le_prev;  /* address of previous next element */  \
+}
+
+/*
+ * Singly-linked List definitions.
+ */
+#define QSLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *slh_first; /* first element */                     \
+}
+
+#define QSLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QSLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *sle_next;  /* next element */                      \
+}
+
+/*
+ * Simple queue definitions.
+ */
+#define QSIMPLEQ_HEAD(name, type)                                       \
+struct name {                                                           \
+    struct type *sqh_first;    /* first element */                      \
+    struct type **sqh_last;    /* addr of last next element */          \
+}
+
+#define QSIMPLEQ_HEAD_INITIALIZER(head)                                 \
+    { NULL, &(head).sqh_first }
+
+#define QSIMPLEQ_ENTRY(type)                                            \
+struct {                                                                \
+    struct type *sqe_next;    /* next element */                        \
+}
+
+/*
+ * Tail queue definitions.
+ */
+#define Q_TAILQ_HEAD(name, type, qual)                                  \
+struct name {                                                           \
+        qual type *tqh_first;           /* first element */             \
+        qual type *qual *tqh_last;      /* addr of last next element */ \
+}
+#define QTAILQ_HEAD(name, type)                                         \
+struct name {                                                           \
+        type *tqh_first;      /* first element */                       \
+        type **tqh_last;      /* addr of last next element */           \
+}
+
+#define QTAILQ_HEAD_INITIALIZER(head)                                   \
+        { NULL, &(head).tqh_first }
+
+#define Q_TAILQ_ENTRY(type, qual)                                       \
+struct {                                                                \
+        qual type *tqe_next;            /* next element */              \
+        qual type *qual *tqe_prev;      /* address of previous next element */\
+}
+#define QTAILQ_ENTRY(type)                                              \
+struct {                                                                \
+        type *tqe_next;       /* next element */                        \
+        type **tqe_prev;      /* address of previous next element */    \
+}
diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl
index f39630eb3..7dacf32f4 100755
--- a/scripts/get_maintainer.pl
+++ b/scripts/get_maintainer.pl
@@ -258,7 +258,7 @@ open (my $maint, '<', "${lk_path}MAINTAINERS")
 while (<$maint>) {
     my $line = $_;
 
-    if ($line =~ m/^(\C):\s*(.*)/) {
+    if ($line =~ m/^(.):\s*(.*)/) {
 	my $type = $1;
 	my $value = $2;
 
@@ -493,7 +493,7 @@ sub range_is_maintained {
 
     for (my $i = $start; $i < $end; $i++) {
 	my $line = $typevalue[$i];
-	if ($line =~ m/^(\C):\s*(.*)/) {
+	if ($line =~ m/^(.):\s*(.*)/) {
 	    my $type = $1;
 	    my $value = $2;
 	    if ($type eq 'S') {
@@ -511,7 +511,7 @@ sub range_has_maintainer {
 
     for (my $i = $start; $i < $end; $i++) {
 	my $line = $typevalue[$i];
-	if ($line =~ m/^(\C):\s*(.*)/) {
+	if ($line =~ m/^(.):\s*(.*)/) {
 	    my $type = $1;
 	    my $value = $2;
 	    if ($type eq 'M') {
@@ -560,7 +560,7 @@ sub get_maintainers {
 
 	    for ($i = $start; $i < $end; $i++) {
 		my $line = $typevalue[$i];
-		if ($line =~ m/^(\C):\s*(.*)/) {
+		if ($line =~ m/^(.):\s*(.*)/) {
 		    my $type = $1;
 		    my $value = $2;
 		    if ($type eq 'X') {
@@ -575,7 +575,7 @@ sub get_maintainers {
 	    if (!$exclude) {
 		for ($i = $start; $i < $end; $i++) {
 		    my $line = $typevalue[$i];
-		    if ($line =~ m/^(\C):\s*(.*)/) {
+		    if ($line =~ m/^(.):\s*(.*)/) {
 			my $type = $1;
 			my $value = $2;
 			if ($type eq 'F') {
@@ -855,7 +855,7 @@ sub find_first_section {
 
     while ($index < @typevalue) {
 	my $tv = $typevalue[$index];
-	if (($tv =~ m/^(\C):\s*(.*)/)) {
+	if (($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index++;
@@ -869,7 +869,7 @@ sub find_starting_index {
 
     while ($index > 0) {
 	my $tv = $typevalue[$index];
-	if (!($tv =~ m/^(\C):\s*(.*)/)) {
+	if (!($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index--;
@@ -883,7 +883,7 @@ sub find_ending_index {
 
     while ($index < @typevalue) {
 	my $tv = $typevalue[$index];
-	if (!($tv =~ m/^(\C):\s*(.*)/)) {
+	if (!($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index++;
@@ -909,7 +909,7 @@ sub get_maintainer_role {
 
     for ($i = $start + 1; $i < $end; $i++) {
 	my $tv = $typevalue[$i];
-	if ($tv =~ m/^(\C):\s*(.*)/) {
+	if ($tv =~ m/^(.):\s*(.*)/) {
 	    my $ptype = $1;
 	    my $pvalue = $2;
 	    if ($ptype eq "S") {
@@ -968,7 +968,7 @@ sub add_categories {
 
     for ($i = $start + 1; $i < $end; $i++) {
 	my $tv = $typevalue[$i];
-	if ($tv =~ m/^(\C):\s*(.*)/) {
+	if ($tv =~ m/^(.):\s*(.*)/) {
 	    my $ptype = $1;
 	    my $pvalue = $2;
 	    if ($ptype eq "L") {
@@ -1010,7 +1010,7 @@ sub add_categories {
 		if ($name eq "") {
 		    if ($i > 0) {
 			my $tv = $typevalue[$i - 1];
-			if ($tv =~ m/^(\C):\s*(.*)/) {
+			if ($tv =~ m/^(.):\s*(.*)/) {
 			    if ($1 eq "P") {
 				$name = $2;
 				$pvalue = format_email($name, $address, $email_usename);
diff --git a/scripts/ordereddict.py b/scripts/ordereddict.py
index 7242b5060..2d1d81370 100644
--- a/scripts/ordereddict.py
+++ b/scripts/ordereddict.py
@@ -22,6 +22,7 @@
 
 from UserDict import DictMixin
 
+
 class OrderedDict(dict, DictMixin):
 
     def __init__(self, *args, **kwds):
@@ -117,7 +118,7 @@ class OrderedDict(dict, DictMixin):
         if isinstance(other, OrderedDict):
             if len(self) != len(other):
                 return False
-            for p, q in  zip(self.items(), other.items()):
+            for p, q in zip(self.items(), other.items()):
                 if p != q:
                     return False
             return True
diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py
index ca22acc1d..561e47a42 100644
--- a/scripts/qapi-commands.py
+++ b/scripts/qapi-commands.py
@@ -12,274 +12,248 @@
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 import re
 
-def generate_command_decl(name, args, ret_type):
-    arglist=""
-    for argname, argtype, optional in parse_args(args):
-        argtype = c_type(argtype, is_param=True)
-        if optional:
-            arglist += "bool has_%s, " % c_name(argname)
-        arglist += "%s %s, " % (argtype, c_name(argname))
+
+def gen_command_decl(name, arg_type, ret_type):
     return mcgen('''
-%(ret_type)s qmp_%(name)s(%(args)sError **errp);
+%(c_type)s qmp_%(c_name)s(%(params)s);
 ''',
-                 ret_type=c_type(ret_type), name=c_name(name),
-                 args=arglist).strip()
-
-def gen_err_check(errvar):
-    if errvar:
-        return mcgen('''
-if (local_err) {
-    goto out;
-}
-''')
-    return ''
+                 c_type=(ret_type and ret_type.c_type()) or 'void',
+                 c_name=c_name(name),
+                 params=gen_params(arg_type, 'Error **errp'))
+
+
+def gen_call(name, arg_type, ret_type):
+    ret = ''
+
+    argstr = ''
+    if arg_type:
+        for memb in arg_type.members:
+            if memb.optional:
+                argstr += 'has_%s, ' % c_name(memb.name)
+            argstr += '%s, ' % c_name(memb.name)
 
-def gen_sync_call(name, args, ret_type, indent=0):
-    ret = ""
-    arglist=""
-    retval=""
+    lhs = ''
     if ret_type:
-        retval = "retval = "
-    for argname, argtype, optional in parse_args(args):
-        if optional:
-            arglist += "has_%s, " % c_name(argname)
-        arglist += "%s, " % (c_name(argname))
-    push_indent(indent)
+        lhs = 'retval = '
+
     ret = mcgen('''
-%(retval)sqmp_%(name)s(%(args)s&local_err);
 
+    %(lhs)sqmp_%(c_name)s(%(args)s&err);
 ''',
-                name=c_name(name), args=arglist, retval=retval).rstrip()
+                c_name=c_name(name), args=argstr, lhs=lhs)
     if ret_type:
-        ret += "\n" + gen_err_check('local_err')
-        ret += "\n" + mcgen(''''
-%(marshal_output_call)s
-''',
-                            marshal_output_call=gen_marshal_output_call(name, ret_type)).rstrip()
-    pop_indent(indent)
-    return ret.rstrip()
+        ret += gen_err_check()
+        ret += mcgen('''
 
+    qmp_marshal_output_%(c_name)s(retval, ret, &err);
+''',
+                     c_name=ret_type.c_name())
+    return ret
 
-def gen_marshal_output_call(name, ret_type):
-    if not ret_type:
-        return ""
-    return "qmp_marshal_output_%s(retval, ret, &local_err);" % c_name(name)
 
-def gen_visitor_input_containers_decl(args, obj):
-    ret = ""
+def gen_marshal_vars(arg_type, ret_type):
+    ret = mcgen('''
+    Error *err = NULL;
+''')
 
-    push_indent()
-    if len(args) > 0:
+    if ret_type:
         ret += mcgen('''
-QmpInputVisitor *mi = qmp_input_visitor_new_strict(%(obj)s);
-QapiDeallocVisitor *md;
-Visitor *v;
+    %(c_type)s retval;
 ''',
-                     obj=obj)
-    pop_indent()
+                     c_type=ret_type.c_type())
 
-    return ret.rstrip()
+    if arg_type:
+        ret += mcgen('''
+    QmpInputVisitor *qiv = qmp_input_visitor_new_strict(QOBJECT(args));
+    QapiDeallocVisitor *qdv;
+    Visitor *v;
+''')
 
-def gen_visitor_input_vars_decl(args):
-    ret = ""
-    push_indent()
-    for argname, argtype, optional in parse_args(args):
-        if optional:
-            ret += mcgen('''
-bool has_%(argname)s = false;
+        for memb in arg_type.members:
+            if memb.optional:
+                ret += mcgen('''
+    bool has_%(c_name)s = false;
 ''',
-                         argname=c_name(argname))
-        if is_c_ptr(argtype):
+                             c_name=c_name(memb.name))
             ret += mcgen('''
-%(argtype)s %(argname)s = NULL;
+    %(c_type)s %(c_name)s = %(c_null)s;
 ''',
-                         argname=c_name(argname), argtype=c_type(argtype))
-        else:
-            ret += mcgen('''
-%(argtype)s %(argname)s = {0};
-''',
-                         argname=c_name(argname), argtype=c_type(argtype))
+                         c_name=c_name(memb.name),
+                         c_type=memb.type.c_type(),
+                         c_null=memb.type.c_null())
+        ret += '\n'
+    else:
+        ret += mcgen('''
 
-    pop_indent()
-    return ret.rstrip()
+    (void)args;
+''')
 
-def gen_visitor_input_block(args, dealloc=False):
-    ret = ""
-    errparg = '&local_err'
-    errarg = 'local_err'
+    return ret
 
-    if len(args) == 0:
-        return ret
 
-    push_indent()
+def gen_marshal_input_visit(arg_type, dealloc=False):
+    ret = ''
+
+    if not arg_type:
+        return ret
 
     if dealloc:
-        errparg = 'NULL'
-        errarg = None;
         ret += mcgen('''
-qmp_input_visitor_cleanup(mi);
-md = qapi_dealloc_visitor_new();
-v = qapi_dealloc_get_visitor(md);
+    qmp_input_visitor_cleanup(qiv);
+    qdv = qapi_dealloc_visitor_new();
+    v = qapi_dealloc_get_visitor(qdv);
 ''')
     else:
         ret += mcgen('''
-v = qmp_input_get_visitor(mi);
+    v = qmp_input_get_visitor(qiv);
 ''')
 
-    for argname, argtype, optional in parse_args(args):
-        if optional:
-            ret += mcgen('''
-visit_optional(v, &has_%(c_name)s, "%(name)s", %(errp)s);
-''',
-                         c_name=c_name(argname), name=argname, errp=errparg)
-            ret += gen_err_check(errarg)
-            ret += mcgen('''
-if (has_%(c_name)s) {
-''',
-                         c_name=c_name(argname))
-            push_indent()
-        ret += mcgen('''
-visit_type_%(visitor)s(v, &%(c_name)s, "%(name)s", %(errp)s);
-''',
-                     c_name=c_name(argname), name=argname, argtype=argtype,
-                     visitor=type_name(argtype), errp=errparg)
-        ret += gen_err_check(errarg)
-        if optional:
-            pop_indent()
-            ret += mcgen('''
-}
-''')
+    ret += gen_visit_fields(arg_type.members, skiperr=dealloc)
 
     if dealloc:
         ret += mcgen('''
-qapi_dealloc_visitor_cleanup(md);
+    qapi_dealloc_visitor_cleanup(qdv);
 ''')
-    pop_indent()
-    return ret.rstrip()
+    return ret
 
-def gen_marshal_output(name, args, ret_type, middle_mode):
-    if not ret_type:
-        return ""
 
-    ret = mcgen('''
-static void qmp_marshal_output_%(c_name)s(%(c_ret_type)s ret_in, QObject **ret_out, Error **errp)
+def gen_marshal_output(ret_type):
+    return mcgen('''
+
+static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, QObject **ret_out, Error **errp)
 {
-    Error *local_err = NULL;
-    QmpOutputVisitor *mo = qmp_output_visitor_new();
-    QapiDeallocVisitor *md;
+    Error *err = NULL;
+    QmpOutputVisitor *qov = qmp_output_visitor_new();
+    QapiDeallocVisitor *qdv;
     Visitor *v;
 
-    v = qmp_output_get_visitor(mo);
-    visit_type_%(visitor)s(v, &ret_in, "unused", &local_err);
-    if (local_err) {
+    v = qmp_output_get_visitor(qov);
+    visit_type_%(c_name)s(v, &ret_in, "unused", &err);
+    if (err) {
         goto out;
     }
-    *ret_out = qmp_output_get_qobject(mo);
+    *ret_out = qmp_output_get_qobject(qov);
 
 out:
-    error_propagate(errp, local_err);
-    qmp_output_visitor_cleanup(mo);
-    md = qapi_dealloc_visitor_new();
-    v = qapi_dealloc_get_visitor(md);
-    visit_type_%(visitor)s(v, &ret_in, "unused", NULL);
-    qapi_dealloc_visitor_cleanup(md);
+    error_propagate(errp, err);
+    qmp_output_visitor_cleanup(qov);
+    qdv = qapi_dealloc_visitor_new();
+    v = qapi_dealloc_get_visitor(qdv);
+    visit_type_%(c_name)s(v, &ret_in, "unused", NULL);
+    qapi_dealloc_visitor_cleanup(qdv);
 }
 ''',
-                c_ret_type=c_type(ret_type), c_name=c_name(name),
-                visitor=type_name(ret_type))
+                 c_type=ret_type.c_type(), c_name=ret_type.c_name())
 
-    return ret
 
-def gen_marshal_input_decl(name, args, ret_type, middle_mode):
-    ret = 'void qmp_marshal_input_%s(QDict *args, QObject **ret, Error **errp)' % c_name(name)
+def gen_marshal_proto(name):
+    ret = 'void qmp_marshal_%s(QDict *args, QObject **ret, Error **errp)' % c_name(name)
     if not middle_mode:
-        ret = "static " + ret
+        ret = 'static ' + ret
     return ret
 
-def gen_marshal_input(name, args, ret_type, middle_mode):
-    hdr = gen_marshal_input_decl(name, args, ret_type, middle_mode)
-
-    ret = mcgen('''
-%(header)s
-{
-    Error *local_err = NULL;
-''',
-                header=hdr)
 
-    if ret_type:
-        if is_c_ptr(ret_type):
-            retval = "    %s retval = NULL;" % c_type(ret_type)
-        else:
-            retval = "    %s retval;" % c_type(ret_type)
-        ret += mcgen('''
-%(retval)s
+def gen_marshal_decl(name):
+    return mcgen('''
+%(proto)s;
 ''',
-                     retval=retval)
+                 proto=gen_marshal_proto(name))
 
-    if len(args) > 0:
-        ret += mcgen('''
-%(visitor_input_containers_decl)s
-%(visitor_input_vars_decl)s
 
-%(visitor_input_block)s
+def gen_marshal(name, arg_type, ret_type):
+    ret = mcgen('''
 
+%(proto)s
+{
 ''',
-                     visitor_input_containers_decl=gen_visitor_input_containers_decl(args, "QOBJECT(args)"),
-                     visitor_input_vars_decl=gen_visitor_input_vars_decl(args),
-                     visitor_input_block=gen_visitor_input_block(args))
-    else:
-        ret += mcgen('''
+                proto=gen_marshal_proto(name))
 
-    (void)args;
-''')
+    ret += gen_marshal_vars(arg_type, ret_type)
+    ret += gen_marshal_input_visit(arg_type)
+    ret += gen_call(name, arg_type, ret_type)
 
-    ret += mcgen('''
-%(sync_call)s
-''',
-                 sync_call=gen_sync_call(name, args, ret_type, indent=4))
-    if re.search('^ *goto out\\;', ret, re.MULTILINE):
+    # 'goto out' produced by gen_marshal_input_visit->gen_visit_fields()
+    # for each arg_type member, and by gen_call() for ret_type
+    if (arg_type and arg_type.members) or ret_type:
         ret += mcgen('''
 
 out:
 ''')
     ret += mcgen('''
-    error_propagate(errp, local_err);
-%(visitor_input_block_cleanup)s
+    error_propagate(errp, err);
+''')
+    ret += gen_marshal_input_visit(arg_type, dealloc=True)
+    ret += mcgen('''
 }
-''',
-                 visitor_input_block_cleanup=gen_visitor_input_block(args,
-                                                                     dealloc=True))
+''')
     return ret
 
-def gen_registry(commands):
-    registry=""
-    push_indent()
-    for cmd in commands:
-        options = 'QCO_NO_OPTIONS'
-        if not cmd.get('success-response', True):
-            options = 'QCO_NO_SUCCESS_RESP'
 
-        registry += mcgen('''
-qmp_register_command("%(name)s", qmp_marshal_input_%(c_name)s, %(opts)s);
+def gen_register_command(name, success_response):
+    options = 'QCO_NO_OPTIONS'
+    if not success_response:
+        options = 'QCO_NO_SUCCESS_RESP'
+
+    ret = mcgen('''
+    qmp_register_command("%(name)s", qmp_marshal_%(c_name)s, %(opts)s);
 ''',
-                     name=cmd['command'], c_name=c_name(cmd['command']),
-                     opts=options)
-    pop_indent()
+                name=name, c_name=c_name(name),
+                opts=options)
+    return ret
+
+
+def gen_registry(registry):
     ret = mcgen('''
+
 static void qmp_init_marshal(void)
 {
-%(registry)s
+''')
+    ret += registry
+    ret += mcgen('''
 }
 
 qapi_init(qmp_init_marshal);
-''',
-                registry=registry.rstrip())
+''')
     return ret
 
+
+class QAPISchemaGenCommandVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._regy = None
+        self._visited_ret_types = None
+
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._regy = ''
+        self._visited_ret_types = set()
+
+    def visit_end(self):
+        if not middle_mode:
+            self.defn += gen_registry(self._regy)
+        self._regy = None
+        self._visited_ret_types = None
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        if not gen:
+            return
+        self.decl += gen_command_decl(name, arg_type, ret_type)
+        if ret_type and ret_type not in self._visited_ret_types:
+            self._visited_ret_types.add(ret_type)
+            self.defn += gen_marshal_output(ret_type)
+        if middle_mode:
+            self.decl += gen_marshal_decl(name)
+        self.defn += gen_marshal(name, arg_type, ret_type)
+        if not middle_mode:
+            self._regy += gen_register_command(name, success_response)
+
+
 middle_mode = False
 
 (input_file, output_dir, do_c, do_h, prefix, opts) = \
@@ -289,10 +263,6 @@ for o, a in opts:
     if o in ("-m", "--middle"):
         middle_mode = True
 
-exprs = parse_schema(input_file)
-commands = filter(lambda expr: expr.has_key('command'), exprs)
-commands = filter(lambda expr: not expr.has_key('gen'), commands)
-
 c_comment = '''
 /*
  * schema-defined QMP->QAPI command dispatch
@@ -340,7 +310,7 @@ fdef.write(mcgen('''
 #include "%(prefix)sqmp-commands.h"
 
 ''',
-                prefix=prefix))
+                 prefix=prefix))
 
 fdecl.write(mcgen('''
 #include "%(prefix)sqapi-types.h"
@@ -348,29 +318,12 @@ fdecl.write(mcgen('''
 #include "qapi/error.h"
 
 ''',
-                 prefix=prefix))
-
-for cmd in commands:
-    arglist = []
-    ret_type = None
-    if cmd.has_key('data'):
-        arglist = cmd['data']
-    if cmd.has_key('returns'):
-        ret_type = cmd['returns']
-    ret = generate_command_decl(cmd['command'], arglist, ret_type) + "\n"
-    fdecl.write(ret)
-    if ret_type:
-        ret = gen_marshal_output(cmd['command'], arglist, ret_type, middle_mode) + "\n"
-        fdef.write(ret)
-
-    if middle_mode:
-        fdecl.write('%s;\n' % gen_marshal_input_decl(cmd['command'], arglist, ret_type, middle_mode))
-
-    ret = gen_marshal_input(cmd['command'], arglist, ret_type, middle_mode) + "\n"
-    fdef.write(ret)
+                  prefix=prefix))
 
-if not middle_mode:
-    ret = gen_registry(commands)
-    fdef.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenCommandVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/scripts/qapi-event.py b/scripts/qapi-event.py
index 56bc602a6..720486f06 100644
--- a/scripts/qapi-event.py
+++ b/scripts/qapi-event.py
@@ -2,78 +2,64 @@
 # QAPI event generator
 #
 # Copyright (c) 2014 Wenchao Xia
+# Copyright (c) 2015 Red Hat Inc.
 #
 # Authors:
 #  Wenchao Xia <wenchaoqemu@gmail.com>
+#  Markus Armbruster <armbru@redhat.com>
 #
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 
-def _generate_event_api_name(event_name, params):
-    api_name = "void qapi_event_send_%s(" % c_name(event_name).lower();
-    l = len(api_name)
 
-    if params:
-        for argname, argentry, optional in parse_args(params):
-            if optional:
-                api_name += "bool has_%s,\n" % c_name(argname)
-                api_name += "".ljust(l)
+def gen_event_send_proto(name, arg_type):
+    return 'void qapi_event_send_%(c_name)s(%(param)s)' % {
+        'c_name': c_name(name.lower()),
+        'param': gen_params(arg_type, 'Error **errp')}
 
-            api_name += "%s %s,\n" % (c_type(argentry, is_param=True),
-                                      c_name(argname))
-            api_name += "".ljust(l)
 
-    api_name += "Error **errp)"
-    return api_name;
-
-
-# Following are the core functions that generate C APIs to emit event.
-
-def generate_event_declaration(api_name):
+def gen_event_send_decl(name, arg_type):
     return mcgen('''
 
-%(api_name)s;
+%(proto)s;
 ''',
-                 api_name = api_name)
+                 proto=gen_event_send_proto(name, arg_type))
 
-def generate_event_implement(api_name, event_name, params):
-    # step 1: declare any variables
-    ret = mcgen("""
 
-%(api_name)s
+def gen_event_send(name, arg_type):
+    ret = mcgen('''
+
+%(proto)s
 {
     QDict *qmp;
-    Error *local_err = NULL;
+    Error *err = NULL;
     QMPEventFuncEmit emit;
-""",
-                api_name = api_name)
+''',
+                proto=gen_event_send_proto(name, arg_type))
 
-    if params:
-        ret += mcgen("""
+    if arg_type and arg_type.members:
+        ret += mcgen('''
     QmpOutputVisitor *qov;
     Visitor *v;
     QObject *obj;
 
-""")
+''')
 
-    # step 2: check emit function, create a dict
-    ret += mcgen("""
+    ret += mcgen('''
     emit = qmp_event_get_func_emit();
     if (!emit) {
         return;
     }
 
-    qmp = qmp_event_build_dict("%(event_name)s");
+    qmp = qmp_event_build_dict("%(name)s");
 
-""",
-                 event_name = event_name)
+''',
+                 name=name)
 
-    # step 3: visit the params if params != None
-    if params:
-        ret += mcgen("""
+    if arg_type and arg_type.members:
+        ret += mcgen('''
     qov = qmp_output_visitor_new();
     g_assert(qov);
 
@@ -81,136 +67,63 @@ def generate_event_implement(api_name, event_name, params):
     g_assert(v);
 
     /* Fake visit, as if all members are under a structure */
-    visit_start_struct(v, NULL, "", "%(event_name)s", 0, &local_err);
-    if (local_err) {
-        goto clean;
-    }
-
-""",
-                event_name = event_name)
-
-        for argname, argentry, optional in parse_args(params):
-            if optional:
-                ret += mcgen("""
-    if (has_%(var)s) {
-""",
-                             var = c_name(argname))
-                push_indent()
-
-            if argentry == "str":
-                var_type = "(char **)"
-            else:
-                var_type = ""
-
-            ret += mcgen("""
-    visit_type_%(type)s(v, %(var_type)s&%(var)s, "%(name)s", &local_err);
-    if (local_err) {
-        goto clean;
-    }
-""",
-                         var_type = var_type,
-                         var = c_name(argname),
-                         type = type_name(argentry),
-                         name = argname)
-
-            if optional:
-                pop_indent()
-                ret += mcgen("""
-    }
-""")
-
-        ret += mcgen("""
-
-    visit_end_struct(v, &local_err);
-    if (local_err) {
-        goto clean;
+    visit_start_struct(v, NULL, "", "%(name)s", 0, &err);
+''',
+                     name=name)
+        ret += gen_err_check()
+        ret += gen_visit_fields(arg_type.members, need_cast=True)
+        ret += mcgen('''
+    visit_end_struct(v, &err);
+    if (err) {
+        goto out;
     }
 
     obj = qmp_output_get_qobject(qov);
     g_assert(obj != NULL);
 
     qdict_put_obj(qmp, "data", obj);
-""")
+''')
 
-    # step 4: call qmp event api
-    ret += mcgen("""
-    emit(%(event_enum_value)s, qmp, &local_err);
+    ret += mcgen('''
+    emit(%(c_enum)s, qmp, &err);
 
-""",
-                 event_enum_value = event_enum_value)
+''',
+                 c_enum=c_enum_const(event_enum_name, name))
 
-    # step 5: clean up
-    if params:
-        ret += mcgen("""
- clean:
+    if arg_type and arg_type.members:
+        ret += mcgen('''
+out:
     qmp_output_visitor_cleanup(qov);
-""")
-    ret += mcgen("""
-    error_propagate(errp, local_err);
+''')
+    ret += mcgen('''
+    error_propagate(errp, err);
     QDECREF(qmp);
 }
-""")
-
+''')
     return ret
 
 
-# Following are the functions that generate an enum type for all defined
-# events, similar to qapi-types.py. Here we already have enum name and
-# values which were generated before and recorded in event_enum_*. It also
-# works around the issue that "import qapi-types" can't work.
+class QAPISchemaGenEventVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._event_names = None
 
-def generate_event_enum_decl(event_enum_name, event_enum_values):
-    lookup_decl = mcgen('''
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._event_names = []
 
-extern const char *%(event_enum_name)s_lookup[];
-''',
-                        event_enum_name = event_enum_name)
+    def visit_end(self):
+        self.decl += gen_enum(event_enum_name, self._event_names)
+        self.defn += gen_enum_lookup(event_enum_name, self._event_names)
+        self._event_names = None
 
-    enum_decl = mcgen('''
-typedef enum %(event_enum_name)s
-{
-''',
-                      event_enum_name = event_enum_name)
-
-    # append automatically generated _MAX value
-    enum_max_value = c_enum_const(event_enum_name, "MAX")
-    enum_values = event_enum_values + [ enum_max_value ]
-
-    i = 0
-    for value in enum_values:
-        enum_decl += mcgen('''
-    %(value)s = %(i)d,
-''',
-                     value = value,
-                     i = i)
-        i += 1
+    def visit_event(self, name, info, arg_type):
+        self.decl += gen_event_send_decl(name, arg_type)
+        self.defn += gen_event_send(name, arg_type)
+        self._event_names.append(name)
 
-    enum_decl += mcgen('''
-} %(event_enum_name)s;
-''',
-                       event_enum_name = event_enum_name)
-
-    return lookup_decl + enum_decl
-
-def generate_event_enum_lookup(event_enum_name, event_enum_strings):
-    ret = mcgen('''
-
-const char *%(event_enum_name)s_lookup[] = {
-''',
-                event_enum_name = event_enum_name)
-
-    i = 0
-    for string in event_enum_strings:
-        ret += mcgen('''
-    "%(string)s",
-''',
-                     string = string)
-
-    ret += mcgen('''
-    NULL,
-};
-''')
-    return ret
 
 (input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line()
 
@@ -265,35 +178,12 @@ fdecl.write(mcgen('''
 ''',
                   prefix=prefix))
 
-exprs = parse_schema(input_file)
-
-event_enum_name = prefix.upper().replace('-', '_') + "QAPIEvent"
-event_enum_values = []
-event_enum_strings = []
-
-for expr in exprs:
-    if expr.has_key('event'):
-        event_name = expr['event']
-        params = expr.get('data')
-        if params and len(params) == 0:
-            params = None
-
-        api_name = _generate_event_api_name(event_name, params)
-        ret = generate_event_declaration(api_name)
-        fdecl.write(ret)
-
-        # We need an enum value per event
-        event_enum_value = c_enum_const(event_enum_name, event_name)
-        ret = generate_event_implement(api_name, event_name, params)
-        fdef.write(ret)
-
-        # Record it, and generate enum later
-        event_enum_values.append(event_enum_value)
-        event_enum_strings.append(event_name)
+event_enum_name = c_name(prefix + "QAPIEvent", protect=False)
 
-ret = generate_event_enum_decl(event_enum_name, event_enum_values)
-fdecl.write(ret)
-ret = generate_event_enum_lookup(event_enum_name, event_enum_strings)
-fdef.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenEventVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/scripts/qapi-introspect.py b/scripts/qapi-introspect.py
new file mode 100644
index 000000000..64f2cd063
--- /dev/null
+++ b/scripts/qapi-introspect.py
@@ -0,0 +1,218 @@
+#
+# QAPI introspection generator
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+from qapi import *
+
+
+# Caveman's json.dumps() replacement (we're stuck at Python 2.4)
+# TODO try to use json.dumps() once we get unstuck
+def to_json(obj, level=0):
+    if obj is None:
+        ret = 'null'
+    elif isinstance(obj, str):
+        ret = '"' + obj.replace('"', r'\"') + '"'
+    elif isinstance(obj, list):
+        elts = [to_json(elt, level + 1)
+                for elt in obj]
+        ret = '[' + ', '.join(elts) + ']'
+    elif isinstance(obj, dict):
+        elts = ['"%s": %s' % (key.replace('"', r'\"'),
+                              to_json(obj[key], level + 1))
+                for key in sorted(obj.keys())]
+        ret = '{' + ', '.join(elts) + '}'
+    else:
+        assert False                # not implemented
+    if level == 1:
+        ret = '\n' + ret
+    return ret
+
+
+def to_c_string(string):
+    return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
+
+
+class QAPISchemaGenIntrospectVisitor(QAPISchemaVisitor):
+    def __init__(self, unmask):
+        self._unmask = unmask
+        self.defn = None
+        self.decl = None
+        self._schema = None
+        self._jsons = None
+        self._used_types = None
+        self._name_map = None
+
+    def visit_begin(self, schema):
+        self._schema = schema
+        self._jsons = []
+        self._used_types = []
+        self._name_map = {}
+
+    def visit_end(self):
+        # visit the types that are actually used
+        jsons = self._jsons
+        self._jsons = []
+        for typ in self._used_types:
+            typ.visit(self)
+        # generate C
+        # TODO can generate awfully long lines
+        jsons.extend(self._jsons)
+        name = prefix + 'qmp_schema_json'
+        self.decl = mcgen('''
+extern const char %(c_name)s[];
+''',
+                          c_name=c_name(name))
+        lines = to_json(jsons).split('\n')
+        c_string = '\n    '.join([to_c_string(line) for line in lines])
+        self.defn = mcgen('''
+const char %(c_name)s[] = %(c_string)s;
+''',
+                          c_name=c_name(name),
+                          c_string=c_string)
+        self._schema = None
+        self._jsons = None
+        self._used_types = None
+        self._name_map = None
+
+    def visit_needed(self, entity):
+        # Ignore types on first pass; visit_end() will pick up used types
+        return not isinstance(entity, QAPISchemaType)
+
+    def _name(self, name):
+        if self._unmask:
+            return name
+        if name not in self._name_map:
+            self._name_map[name] = '%d' % len(self._name_map)
+        return self._name_map[name]
+
+    def _use_type(self, typ):
+        # Map the various integer types to plain int
+        if typ.json_type() == 'int':
+            typ = self._schema.lookup_type('int')
+        elif (isinstance(typ, QAPISchemaArrayType) and
+              typ.element_type.json_type() == 'int'):
+            typ = self._schema.lookup_type('intList')
+        # Add type to work queue if new
+        if typ not in self._used_types:
+            self._used_types.append(typ)
+        # Clients should examine commands and events, not types.  Hide
+        # type names to reduce the temptation.  Also saves a few
+        # characters.
+        if isinstance(typ, QAPISchemaBuiltinType):
+            return typ.name
+        if isinstance(typ, QAPISchemaArrayType):
+            return '[' + self._use_type(typ.element_type) + ']'
+        return self._name(typ.name)
+
+    def _gen_json(self, name, mtype, obj):
+        if mtype not in ('command', 'event', 'builtin', 'array'):
+            name = self._name(name)
+        obj['name'] = name
+        obj['meta-type'] = mtype
+        self._jsons.append(obj)
+
+    def _gen_member(self, member):
+        ret = {'name': member.name, 'type': self._use_type(member.type)}
+        if member.optional:
+            ret['default'] = None
+        return ret
+
+    def _gen_variants(self, tag_name, variants):
+        return {'tag': tag_name,
+                'variants': [self._gen_variant(v) for v in variants]}
+
+    def _gen_variant(self, variant):
+        return {'case': variant.name, 'type': self._use_type(variant.type)}
+
+    def visit_builtin_type(self, name, info, json_type):
+        self._gen_json(name, 'builtin', {'json-type': json_type})
+
+    def visit_enum_type(self, name, info, values, prefix):
+        self._gen_json(name, 'enum', {'values': values})
+
+    def visit_array_type(self, name, info, element_type):
+        element = self._use_type(element_type)
+        self._gen_json('[' + element + ']', 'array', {'element-type': element})
+
+    def visit_object_type_flat(self, name, info, members, variants):
+        obj = {'members': [self._gen_member(m) for m in members]}
+        if variants:
+            obj.update(self._gen_variants(variants.tag_member.name,
+                                          variants.variants))
+        self._gen_json(name, 'object', obj)
+
+    def visit_alternate_type(self, name, info, variants):
+        self._gen_json(name, 'alternate',
+                       {'members': [{'type': self._use_type(m.type)}
+                                    for m in variants.variants]})
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        arg_type = arg_type or self._schema.the_empty_object_type
+        ret_type = ret_type or self._schema.the_empty_object_type
+        self._gen_json(name, 'command',
+                       {'arg-type': self._use_type(arg_type),
+                        'ret-type': self._use_type(ret_type)})
+
+    def visit_event(self, name, info, arg_type):
+        arg_type = arg_type or self._schema.the_empty_object_type
+        self._gen_json(name, 'event', {'arg-type': self._use_type(arg_type)})
+
+# Debugging aid: unmask QAPI schema's type names
+# We normally mask them, because they're not QMP wire ABI
+opt_unmask = False
+
+(input_file, output_dir, do_c, do_h, prefix, opts) = \
+    parse_command_line("u", ["unmask-non-abi-names"])
+
+for o, a in opts:
+    if o in ("-u", "--unmask-non-abi-names"):
+        opt_unmask = True
+
+c_comment = '''
+/*
+ * QAPI/QMP schema introspection
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
+h_comment = '''
+/*
+ * QAPI/QMP schema introspection
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
+
+(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix,
+                            'qmp-introspect.c', 'qmp-introspect.h',
+                            c_comment, h_comment)
+
+fdef.write(mcgen('''
+#include "%(prefix)sqmp-introspect.h"
+
+''',
+                 prefix=prefix))
+
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenIntrospectVisitor(opt_unmask)
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
+
+close_output(fdef, fdecl)
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index e6eb4b613..b37900f6f 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -2,98 +2,89 @@
 # QAPI types generator
 #
 # Copyright IBM, Corp. 2011
+# Copyright (c) 2013-2015 Red Hat Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
+#  Markus Armbruster <armbru@redhat.com>
 #
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 
-def generate_fwd_builtin(name):
+
+def gen_fwd_object_or_array(name):
     return mcgen('''
 
-typedef struct %(name)sList
-{
-    union {
-        %(type)s value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
+typedef struct %(c_name)s %(c_name)s;
 ''',
-                 type=c_type(name),
-                 name=name)
+                 c_name=c_name(name))
 
-def generate_fwd_struct(name):
-    return mcgen('''
 
-typedef struct %(name)s %(name)s;
+def gen_array(name, element_type):
+    return mcgen('''
 
-typedef struct %(name)sList
-{
+struct %(c_name)s {
     union {
-        %(name)s *value;
+        %(c_type)s value;
         uint64_t padding;
     };
-    struct %(name)sList *next;
-} %(name)sList;
+    %(c_name)s *next;
+};
 ''',
-                 name=c_name(name))
+                 c_name=c_name(name), c_type=element_type.c_type())
 
-def generate_fwd_enum_struct(name):
-    return mcgen('''
-typedef struct %(name)sList
-{
-    union {
-        %(name)s value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
-''',
-                 name=c_name(name))
 
-def generate_struct_fields(members):
+def gen_struct_field(member):
     ret = ''
 
-    for argname, argentry, optional in parse_args(members):
-        if optional:
-            ret += mcgen('''
+    if member.optional:
+        ret += mcgen('''
     bool has_%(c_name)s;
 ''',
-                         c_name=c_name(argname))
-        ret += mcgen('''
+                     c_name=c_name(member.name))
+    ret += mcgen('''
     %(c_type)s %(c_name)s;
 ''',
-                     c_type=c_type(argentry), c_name=c_name(argname))
-
+                 c_type=member.type.c_type(), c_name=c_name(member.name))
     return ret
 
-def generate_struct(expr):
 
-    structname = expr.get('struct', "")
-    members = expr['data']
-    base = expr.get('base')
+def gen_struct_fields(local_members, base=None):
+    ret = ''
+
+    if base:
+        ret += mcgen('''
+    /* Members inherited from %(c_name)s: */
+''',
+                     c_name=base.c_name())
+        for memb in base.members:
+            ret += gen_struct_field(memb)
+        ret += mcgen('''
+    /* Own members: */
+''')
+
+    for memb in local_members:
+        ret += gen_struct_field(memb)
+    return ret
 
+
+def gen_struct(name, base, members):
     ret = mcgen('''
-struct %(name)s
-{
-''',
-          name=c_name(structname))
 
-    if base:
-        ret += generate_struct_fields({'base': base})
+struct %(c_name)s {
+''',
+                c_name=c_name(name))
 
-    ret += generate_struct_fields(members)
+    ret += gen_struct_fields(members, base)
 
     # Make sure that all structs have at least one field; this avoids
-    # potential issues with attempting to malloc space for zero-length structs
-    # in C, and also incompatibility with C++ (where an empty struct is size 1).
-    if not base and not members:
-            ret += mcgen('''
+    # potential issues with attempting to malloc space for zero-length
+    # structs in C, and also incompatibility with C++ (where an empty
+    # struct is size 1).
+    if not (base and base.members) and not members:
+        ret += mcgen('''
     char qapi_dummy_field_for_empty_struct;
 ''')
 
@@ -103,80 +94,45 @@ struct %(name)s
 
     return ret
 
-def generate_enum_lookup(name, values):
-    ret = mcgen('''
-const char * const %(name)s_lookup[] = {
-''',
-                name=c_name(name))
-    i = 0
-    for value in values:
-        index = c_enum_const(name, value)
-        ret += mcgen('''
-    [%(index)s] = "%(value)s",
-''',
-                     index = index, value = value)
-
-    max_index = c_enum_const(name, 'MAX')
-    ret += mcgen('''
-    [%(max_index)s] = NULL,
-};
-
-''',
-        max_index=max_index)
-    return ret
 
-def generate_enum(name, values):
-    name = c_name(name)
-    lookup_decl = mcgen('''
-extern const char * const %(name)s_lookup[];
-''',
-                name=name)
+def gen_upcast(name, base):
+    # C makes const-correctness ugly.  We have to cast away const to let
+    # this function work for both const and non-const obj.
+    return mcgen('''
 
-    enum_decl = mcgen('''
-typedef enum %(name)s
+static inline %(base)s *qapi_%(c_name)s_base(const %(c_name)s *obj)
 {
+    return (%(base)s *)obj;
+}
 ''',
-                name=name)
+                 c_name=c_name(name), base=base.c_name())
 
-    # append automatically generated _MAX value
-    enum_values = values + [ 'MAX' ]
 
-    i = 0
-    for value in enum_values:
-        enum_full_value = c_enum_const(name, value)
-        enum_decl += mcgen('''
-    %(enum_full_value)s = %(i)d,
-''',
-                     enum_full_value = enum_full_value,
-                     i=i)
-        i += 1
+def gen_alternate_qtypes_decl(name):
+    return mcgen('''
 
-    enum_decl += mcgen('''
-} %(name)s;
+extern const int %(c_name)s_qtypes[];
 ''',
-                 name=name)
-
-    return lookup_decl + enum_decl
+                 c_name=c_name(name))
 
-def generate_alternate_qtypes(expr):
-
-    name = expr['alternate']
-    members = expr['data']
 
+def gen_alternate_qtypes(name, variants):
     ret = mcgen('''
-const int %(name)s_qtypes[QTYPE_MAX] = {
+
+const int %(c_name)s_qtypes[QTYPE_MAX] = {
 ''',
-                name=c_name(name))
+                c_name=c_name(name))
 
-    for key in members:
-        qtype = find_alternate_member_qtype(members[key])
-        assert qtype, "Invalid alternate member"
+    for var in variants.variants:
+        qtype = var.type.alternate_qtype()
+        assert qtype
 
         ret += mcgen('''
     [%(qtype)s] = %(enum_const)s,
 ''',
-                     qtype = qtype,
-                     enum_const = c_enum_const(name + 'Kind', key))
+                     qtype=qtype,
+                     enum_const=c_enum_const(variants.tag_member.type.name,
+                                             var.name))
 
     ret += mcgen('''
 };
@@ -184,89 +140,154 @@ const int %(name)s_qtypes[QTYPE_MAX] = {
     return ret
 
 
-def generate_union(expr, meta):
-
-    name = c_name(expr[meta])
-    typeinfo = expr['data']
-
-    base = expr.get('base')
-    discriminator = expr.get('discriminator')
+def gen_union(name, base, variants):
+    ret = mcgen('''
 
-    enum_define = discriminator_find_enum_define(expr)
-    if enum_define:
-        discriminator_type_name = enum_define['enum_name']
+struct %(c_name)s {
+''',
+                c_name=c_name(name))
+    if base:
+        ret += gen_struct_fields([], base)
     else:
-        discriminator_type_name = '%sKind' % (name)
-
-    ret = mcgen('''
-struct %(name)s
-{
-    %(discriminator_type_name)s kind;
-    union {
+        ret += gen_struct_field(variants.tag_member)
+
+    # FIXME: What purpose does data serve, besides preventing a union that
+    # has a branch named 'data'? We use it in qapi-visit.py to decide
+    # whether to bypass the switch statement if visiting the discriminator
+    # failed; but since we 0-initialize structs, and cannot tell what
+    # branch of the union is in use if the discriminator is invalid, there
+    # should not be any data leaks even without a data pointer.  Or, if
+    # 'data' is merely added to guarantee we don't have an empty union,
+    # shouldn't we enforce that at .json parse time?
+    ret += mcgen('''
+    union { /* union tag is @%(c_name)s */
         void *data;
 ''',
-                name=name,
-                discriminator_type_name=c_name(discriminator_type_name))
+                 c_name=c_name(variants.tag_member.name))
 
-    for key in typeinfo:
+    for var in variants.variants:
+        # Ugly special case for simple union TODO get rid of it
+        typ = var.simple_union_type() or var.type
         ret += mcgen('''
         %(c_type)s %(c_name)s;
 ''',
-                     c_type=c_type(typeinfo[key]),
-                     c_name=c_name(key))
-
-    ret += mcgen('''
-    };
-''')
-
-    if base:
-        assert discriminator
-        base_fields = find_struct(base)['data'].copy()
-        del base_fields[discriminator]
-        ret += generate_struct_fields(base_fields)
-    else:
-        assert not discriminator
+                     c_type=typ.c_type(),
+                     c_name=c_name(var.name))
 
     ret += mcgen('''
+    } u;
 };
 ''')
-    if meta == 'alternate':
-        ret += mcgen('''
-extern const int %(name)s_qtypes[];
-''',
-            name=name)
-
 
     return ret
 
-def generate_type_cleanup_decl(name):
+
+def gen_type_cleanup_decl(name):
     ret = mcgen('''
-void qapi_free_%(name)s(%(c_type)s obj);
+
+void qapi_free_%(c_name)s(%(c_name)s *obj);
 ''',
-                c_type=c_type(name), name=c_name(name))
+                c_name=c_name(name))
     return ret
 
-def generate_type_cleanup(name):
+
+def gen_type_cleanup(name):
     ret = mcgen('''
 
-void qapi_free_%(name)s(%(c_type)s obj)
+void qapi_free_%(c_name)s(%(c_name)s *obj)
 {
-    QapiDeallocVisitor *md;
+    QapiDeallocVisitor *qdv;
     Visitor *v;
 
     if (!obj) {
         return;
     }
 
-    md = qapi_dealloc_visitor_new();
-    v = qapi_dealloc_get_visitor(md);
-    visit_type_%(name)s(v, &obj, NULL, NULL);
-    qapi_dealloc_visitor_cleanup(md);
+    qdv = qapi_dealloc_visitor_new();
+    v = qapi_dealloc_get_visitor(qdv);
+    visit_type_%(c_name)s(v, &obj, NULL, NULL);
+    qapi_dealloc_visitor_cleanup(qdv);
 }
 ''',
-                c_type=c_type(name), name=c_name(name))
+                c_name=c_name(name))
     return ret
 
+
+class QAPISchemaGenTypeVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._fwdecl = None
+        self._fwdefn = None
+        self._btin = None
+
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._fwdecl = ''
+        self._fwdefn = ''
+        self._btin = guardstart('QAPI_TYPES_BUILTIN')
+
+    def visit_end(self):
+        self.decl = self._fwdecl + self.decl
+        self._fwdecl = None
+        self.defn = self._fwdefn + self.defn
+        self._fwdefn = None
+        # To avoid header dependency hell, we always generate
+        # declarations for built-in types in our header files and
+        # simply guard them.  See also do_builtins (command line
+        # option -b).
+        self._btin += guardend('QAPI_TYPES_BUILTIN')
+        self.decl = self._btin + self.decl
+        self._btin = None
+
+    def visit_needed(self, entity):
+        # Visit everything except implicit objects
+        return not (entity.is_implicit() and
+                    isinstance(entity, QAPISchemaObjectType))
+
+    def _gen_type_cleanup(self, name):
+        self.decl += gen_type_cleanup_decl(name)
+        self.defn += gen_type_cleanup(name)
+
+    def visit_enum_type(self, name, info, values, prefix):
+        self._fwdecl += gen_enum(name, values, prefix)
+        self._fwdefn += gen_enum_lookup(name, values, prefix)
+
+    def visit_array_type(self, name, info, element_type):
+        if isinstance(element_type, QAPISchemaBuiltinType):
+            self._btin += gen_fwd_object_or_array(name)
+            self._btin += gen_array(name, element_type)
+            self._btin += gen_type_cleanup_decl(name)
+            if do_builtins:
+                self.defn += gen_type_cleanup(name)
+        else:
+            self._fwdecl += gen_fwd_object_or_array(name)
+            self.decl += gen_array(name, element_type)
+            self._gen_type_cleanup(name)
+
+    def visit_object_type(self, name, info, base, members, variants):
+        self._fwdecl += gen_fwd_object_or_array(name)
+        if variants:
+            assert not members      # not implemented
+            self.decl += gen_union(name, base, variants)
+        else:
+            self.decl += gen_struct(name, base, members)
+        if base:
+            self.decl += gen_upcast(name, base)
+        self._gen_type_cleanup(name)
+
+    def visit_alternate_type(self, name, info, variants):
+        self._fwdecl += gen_fwd_object_or_array(name)
+        self._fwdefn += gen_alternate_qtypes(name, variants)
+        self.decl += gen_union(name, None, variants)
+        self.decl += gen_alternate_qtypes_decl(name)
+        self._gen_type_cleanup(name)
+
+# If you link code generated from multiple schemata, you want only one
+# instance of the code for built-in types.  Generate it only when
+# do_builtins, enabled by command line option -b.  See also
+# QAPISchemaGenTypeVisitor.visit_end().
 do_builtins = False
 
 (input_file, output_dir, do_c, do_h, prefix, opts) = \
@@ -314,89 +335,19 @@ fdef.write(mcgen('''
 #include "qapi/dealloc-visitor.h"
 #include "%(prefix)sqapi-types.h"
 #include "%(prefix)sqapi-visit.h"
-
 ''',
                  prefix=prefix))
 
 fdecl.write(mcgen('''
 #include <stdbool.h>
 #include <stdint.h>
-
+#include "qapi/qmp/qobject.h"
 '''))
 
-exprs = parse_schema(input_file)
-
-fdecl.write(guardstart("QAPI_TYPES_BUILTIN_STRUCT_DECL"))
-for typename in builtin_types.keys():
-    fdecl.write(generate_fwd_builtin(typename))
-fdecl.write(guardend("QAPI_TYPES_BUILTIN_STRUCT_DECL"))
-
-for expr in exprs:
-    ret = "\n"
-    if expr.has_key('struct'):
-        ret += generate_fwd_struct(expr['struct'])
-    elif expr.has_key('enum'):
-        ret += generate_enum(expr['enum'], expr['data']) + "\n"
-        ret += generate_fwd_enum_struct(expr['enum'])
-        fdef.write(generate_enum_lookup(expr['enum'], expr['data']))
-    elif expr.has_key('union'):
-        ret += generate_fwd_struct(expr['union']) + "\n"
-        enum_define = discriminator_find_enum_define(expr)
-        if not enum_define:
-            ret += generate_enum('%sKind' % expr['union'], expr['data'].keys())
-            fdef.write(generate_enum_lookup('%sKind' % expr['union'],
-                                            expr['data'].keys()))
-    elif expr.has_key('alternate'):
-        ret += generate_fwd_struct(expr['alternate']) + "\n"
-        ret += generate_enum('%sKind' % expr['alternate'], expr['data'].keys())
-        fdef.write(generate_enum_lookup('%sKind' % expr['alternate'],
-                                        expr['data'].keys()))
-        fdef.write(generate_alternate_qtypes(expr))
-    else:
-        continue
-    fdecl.write(ret)
-
-# to avoid header dependency hell, we always generate declarations
-# for built-in types in our header files and simply guard them
-fdecl.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DECL"))
-for typename in builtin_types.keys():
-    fdecl.write(generate_type_cleanup_decl(typename + "List"))
-fdecl.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DECL"))
-
-# ...this doesn't work for cases where we link in multiple objects that
-# have the functions defined, so we use -b option to provide control
-# over these cases
-if do_builtins:
-    fdef.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DEF"))
-    for typename in builtin_types.keys():
-        fdef.write(generate_type_cleanup(typename + "List"))
-    fdef.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DEF"))
-
-for expr in exprs:
-    ret = "\n"
-    if expr.has_key('struct'):
-        ret += generate_struct(expr) + "\n"
-        ret += generate_type_cleanup_decl(expr['struct'] + "List")
-        fdef.write(generate_type_cleanup(expr['struct'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['struct'])
-        fdef.write(generate_type_cleanup(expr['struct']) + "\n")
-    elif expr.has_key('union'):
-        ret += generate_union(expr, 'union')
-        ret += generate_type_cleanup_decl(expr['union'] + "List")
-        fdef.write(generate_type_cleanup(expr['union'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['union'])
-        fdef.write(generate_type_cleanup(expr['union']) + "\n")
-    elif expr.has_key('alternate'):
-        ret += generate_union(expr, 'alternate')
-        ret += generate_type_cleanup_decl(expr['alternate'] + "List")
-        fdef.write(generate_type_cleanup(expr['alternate'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['alternate'])
-        fdef.write(generate_type_cleanup(expr['alternate']) + "\n")
-    elif expr.has_key('enum'):
-        ret += generate_type_cleanup_decl(expr['enum'] + "List")
-        fdef.write(generate_type_cleanup(expr['enum'] + "List") + "\n")
-    else:
-        continue
-    fdecl.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenTypeVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index 5b9933648..3ef5c16a6 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -12,86 +12,92 @@
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 import re
 
-implicit_structs = []
+# visit_type_FOO_implicit() is emitted as needed; track if it has already
+# been output.
+implicit_structs_seen = set()
 
-def generate_visit_implicit_struct(type):
-    global implicit_structs
-    if type in implicit_structs:
-        return ''
-    implicit_structs.append(type)
+# visit_type_FOO_fields() is always emitted; track if a forward declaration
+# or implementation has already been output.
+struct_fields_seen = set()
+
+
+def gen_visit_decl(name, scalar=False):
+    c_type = c_name(name) + ' *'
+    if not scalar:
+        c_type += '*'
     return mcgen('''
+void visit_type_%(c_name)s(Visitor *v, %(c_type)sobj, const char *name, Error **errp);
+''',
+                 c_name=c_name(name), c_type=c_type)
+
+
+def gen_visit_fields_decl(typ):
+    ret = ''
+    if typ.name not in struct_fields_seen:
+        ret += mcgen('''
+
+static void visit_type_%(c_type)s_fields(Visitor *v, %(c_type)s **obj, Error **errp);
+''',
+                     c_type=typ.c_name())
+        struct_fields_seen.add(typ.name)
+    return ret
+
+
+def gen_visit_implicit_struct(typ):
+    if typ in implicit_structs_seen:
+        return ''
+    implicit_structs_seen.add(typ)
 
-static void visit_type_implicit_%(c_type)s(Visitor *m, %(c_type)s **obj, Error **errp)
+    ret = gen_visit_fields_decl(typ)
+
+    ret += mcgen('''
+
+static void visit_type_implicit_%(c_type)s(Visitor *v, %(c_type)s **obj, Error **errp)
 {
     Error *err = NULL;
 
-    visit_start_implicit_struct(m, (void **)obj, sizeof(%(c_type)s), &err);
+    visit_start_implicit_struct(v, (void **)obj, sizeof(%(c_type)s), &err);
     if (!err) {
-        visit_type_%(c_type)s_fields(m, obj, errp);
-        visit_end_implicit_struct(m, &err);
+        visit_type_%(c_type)s_fields(v, obj, errp);
+        visit_end_implicit_struct(v, &err);
     }
     error_propagate(errp, err);
 }
 ''',
-                 c_type=type_name(type))
+                 c_type=typ.c_name())
+    return ret
 
-def generate_visit_struct_fields(name, members, base = None):
-    substructs = []
+
+def gen_visit_struct_fields(name, base, members):
     ret = ''
 
     if base:
-        ret += generate_visit_implicit_struct(base)
+        ret += gen_visit_fields_decl(base)
 
+    struct_fields_seen.add(name)
     ret += mcgen('''
 
-static void visit_type_%(name)s_fields(Visitor *m, %(name)s **obj, Error **errp)
+static void visit_type_%(c_name)s_fields(Visitor *v, %(c_name)s **obj, Error **errp)
 {
     Error *err = NULL;
-''',
-                 name=c_name(name))
-    push_indent()
 
-    if base:
-        ret += mcgen('''
-visit_type_implicit_%(type)s(m, &(*obj)->%(c_name)s, &err);
-if (err) {
-    goto out;
-}
-''',
-                     type=type_name(base), c_name=c_name('base'))
-
-    for argname, argentry, optional in parse_args(members):
-        if optional:
-            ret += mcgen('''
-visit_optional(m, &(*obj)->has_%(c_name)s, "%(name)s", &err);
-if (!err && (*obj)->has_%(c_name)s) {
 ''',
-                         c_name=c_name(argname), name=argname)
-            push_indent()
+                 c_name=c_name(name))
 
+    if base:
         ret += mcgen('''
-visit_type_%(type)s(m, &(*obj)->%(c_name)s, "%(name)s", &err);
+    visit_type_%(c_type)s_fields(v, (%(c_type)s **)obj, &err);
 ''',
-                     type=type_name(argentry), c_name=c_name(argname),
-                     name=argname)
+                     c_type=base.c_name())
+        ret += gen_err_check()
 
-        if optional:
-            pop_indent()
-            ret += mcgen('''
-}
-''')
-        ret += mcgen('''
-if (err) {
-    goto out;
-}
-''')
+    ret += gen_visit_fields(members, prefix='(*obj)->')
 
-    pop_indent()
-    if re.search('^ *goto out\\;', ret, re.MULTILINE):
+    # 'goto out' produced for base, and by gen_visit_fields() for each member
+    if base or members:
         ret += mcgen('''
 
 out:
@@ -103,131 +109,117 @@ out:
     return ret
 
 
-def generate_visit_struct_body(name, members):
-    ret = mcgen('''
+def gen_visit_struct(name, base, members):
+    ret = gen_visit_struct_fields(name, base, members)
+
+    # FIXME: if *obj is NULL on entry, and visit_start_struct() assigns to
+    # *obj, but then visit_type_FOO_fields() fails, we should clean up *obj
+    # rather than leaving it non-NULL. As currently written, the caller must
+    # call qapi_free_FOO() to avoid a memory leak of the partial FOO.
+    ret += mcgen('''
+
+void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp)
+{
     Error *err = NULL;
 
-    visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(c_name)s), &err);
+    visit_start_struct(v, (void **)obj, "%(name)s", name, sizeof(%(c_name)s), &err);
     if (!err) {
         if (*obj) {
-            visit_type_%(c_name)s_fields(m, obj, errp);
+            visit_type_%(c_name)s_fields(v, obj, errp);
         }
-        visit_end_struct(m, &err);
+        visit_end_struct(v, &err);
     }
     error_propagate(errp, err);
+}
 ''',
-                name=name, c_name=c_name(name))
+                 name=name, c_name=c_name(name))
 
     return ret
 
-def generate_visit_struct(expr):
-
-    name = expr['struct']
-    members = expr['data']
-    base = expr.get('base')
-
-    ret = generate_visit_struct_fields(name, members, base)
 
-    ret += mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
-{
-''',
-                 name=c_name(name))
-
-    ret += generate_visit_struct_body(name, members)
-
-    ret += mcgen('''
-}
-''')
-    return ret
-
-def generate_visit_list(name, members):
+def gen_visit_list(name, element_type):
+    # FIXME: if *obj is NULL on entry, and the first visit_next_list()
+    # assigns to *obj, while a later one fails, we should clean up *obj
+    # rather than leaving it non-NULL. As currently written, the caller must
+    # call qapi_free_FOOList() to avoid a memory leak of the partial FOOList.
     return mcgen('''
 
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp)
 {
     Error *err = NULL;
     GenericList *i, **prev;
 
-    visit_start_list(m, name, &err);
+    visit_start_list(v, name, &err);
     if (err) {
         goto out;
     }
 
     for (prev = (GenericList **)obj;
-         !err && (i = visit_next_list(m, prev, &err)) != NULL;
+         !err && (i = visit_next_list(v, prev, &err)) != NULL;
          prev = &i) {
-        %(name)sList *native_i = (%(name)sList *)i;
-        visit_type_%(name)s(m, &native_i->value, NULL, &err);
+        %(c_name)s *native_i = (%(c_name)s *)i;
+        visit_type_%(c_elt_type)s(v, &native_i->value, NULL, &err);
     }
 
     error_propagate(errp, err);
     err = NULL;
-    visit_end_list(m, &err);
+    visit_end_list(v, &err);
 out:
     error_propagate(errp, err);
 }
 ''',
-                name=type_name(name))
+                 c_name=c_name(name), c_elt_type=element_type.c_name())
+
 
-def generate_visit_enum(name, members):
+def gen_visit_enum(name):
     return mcgen('''
 
-void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, %(c_name)s *obj, const char *name, Error **errp)
 {
-    visit_type_enum(m, (int *)obj, %(name)s_lookup, "%(name)s", name, errp);
+    visit_type_enum(v, (int *)obj, %(c_name)s_lookup, "%(name)s", name, errp);
 }
 ''',
-                 name=c_name(name))
+                 c_name=c_name(name), name=name)
+
 
-def generate_visit_alternate(name, members):
+def gen_visit_alternate(name, variants):
     ret = mcgen('''
 
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp)
 {
     Error *err = NULL;
 
-    visit_start_implicit_struct(m, (void**) obj, sizeof(%(name)s), &err);
+    visit_start_implicit_struct(v, (void**) obj, sizeof(%(c_name)s), &err);
     if (err) {
         goto out;
     }
-    visit_get_next_type(m, (int*) &(*obj)->kind, %(name)s_qtypes, name, &err);
+    visit_get_next_type(v, (int*) &(*obj)->type, %(c_name)s_qtypes, name, &err);
     if (err) {
-        goto out_end;
+        goto out_obj;
     }
-    switch ((*obj)->kind) {
+    switch ((*obj)->type) {
 ''',
-                name=c_name(name))
+                c_name=c_name(name))
 
-    # For alternate, always use the default enum type automatically generated
-    # as name + 'Kind'
-    disc_type = c_name(name) + 'Kind'
-
-    for key in members:
-        assert (members[key] in builtin_types.keys()
-            or find_struct(members[key])
-            or find_union(members[key])
-            or find_enum(members[key])), "Invalid alternate member"
-
-        enum_full_value = c_enum_const(disc_type, key)
+    for var in variants.variants:
         ret += mcgen('''
-    case %(enum_full_value)s:
-        visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, name, &err);
+    case %(case)s:
+        visit_type_%(c_type)s(v, &(*obj)->u.%(c_name)s, name, &err);
         break;
 ''',
-                enum_full_value = enum_full_value,
-                c_type = type_name(members[key]),
-                c_name = c_name(key))
+                     case=c_enum_const(variants.tag_member.type.name,
+                                       var.name),
+                     c_type=var.type.c_name(),
+                     c_name=c_name(var.name))
 
     ret += mcgen('''
     default:
         abort();
     }
-out_end:
+out_obj:
     error_propagate(errp, err);
     err = NULL;
-    visit_end_implicit_struct(m, &err);
+    visit_end_implicit_struct(v, &err);
 out:
     error_propagate(errp, err);
 }
@@ -236,103 +228,91 @@ out:
     return ret
 
 
-def generate_visit_union(expr):
-
-    name = expr['union']
-    members = expr['data']
-
-    base = expr.get('base')
-    discriminator = expr.get('discriminator')
-
-    enum_define = discriminator_find_enum_define(expr)
-    if enum_define:
-        # Use the enum type as discriminator
-        ret = ""
-        disc_type = c_name(enum_define['enum_name'])
-    else:
-        # There will always be a discriminator in the C switch code, by default
-        # it is an enum type generated silently
-        ret = generate_visit_enum(name + 'Kind', members.keys())
-        disc_type = c_name(name) + 'Kind'
+def gen_visit_union(name, base, variants):
+    ret = ''
 
     if base:
-        assert discriminator
-        base_fields = find_struct(base)['data'].copy()
-        del base_fields[discriminator]
-        ret += generate_visit_struct_fields(name, base_fields)
+        ret += gen_visit_fields_decl(base)
 
-    if discriminator:
-        for key in members:
-            ret += generate_visit_implicit_struct(members[key])
+    for var in variants.variants:
+        # Ugly special case for simple union TODO get rid of it
+        if not var.simple_union_type():
+            ret += gen_visit_implicit_struct(var.type)
 
     ret += mcgen('''
 
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp)
 {
     Error *err = NULL;
 
-    visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(name)s), &err);
+    visit_start_struct(v, (void **)obj, "%(name)s", name, sizeof(%(c_name)s), &err);
     if (err) {
         goto out;
     }
-    if (*obj) {
+    if (!*obj) {
+        goto out_obj;
+    }
 ''',
-                 name=c_name(name))
+                 c_name=c_name(name), name=name)
 
     if base:
         ret += mcgen('''
-        visit_type_%(name)s_fields(m, obj, &err);
-        if (err) {
-            goto out_obj;
-        }
+    visit_type_%(c_name)s_fields(v, (%(c_name)s **)obj, &err);
 ''',
-                     name=c_name(name))
-
-    if not discriminator:
-        disc_key = "type"
+                     c_name=base.c_name())
     else:
-        disc_key = discriminator
+        ret += mcgen('''
+    visit_type_%(c_type)s(v, &(*obj)->%(c_name)s, "%(name)s", &err);
+''',
+                     c_type=variants.tag_member.type.c_name(),
+                     c_name=c_name(variants.tag_member.name),
+                     name=variants.tag_member.name)
+    ret += gen_err_check(label='out_obj')
     ret += mcgen('''
-        visit_type_%(disc_type)s(m, &(*obj)->kind, "%(disc_key)s", &err);
-        if (err) {
-            goto out_obj;
-        }
-        if (!visit_start_union(m, !!(*obj)->data, &err) || err) {
-            goto out_obj;
-        }
-        switch ((*obj)->kind) {
+    if (!visit_start_union(v, !!(*obj)->u.data, &err) || err) {
+        goto out_obj;
+    }
+    switch ((*obj)->%(c_name)s) {
 ''',
-                 disc_type = disc_type,
-                 disc_key = disc_key)
+                 c_name=c_name(variants.tag_member.name))
 
-    for key in members:
-        if not discriminator:
-            fmt = 'visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, "data", &err);'
-        else:
-            fmt = 'visit_type_implicit_%(c_type)s(m, &(*obj)->%(c_name)s, &err);'
-
-        enum_full_value = c_enum_const(disc_type, key)
+    for var in variants.variants:
+        # TODO ugly special case for simple union
+        simple_union_type = var.simple_union_type()
         ret += mcgen('''
-        case %(enum_full_value)s:
-            ''' + fmt + '''
-            break;
+    case %(case)s:
+''',
+                     case=c_enum_const(variants.tag_member.type.name,
+                                       var.name))
+        if simple_union_type:
+            ret += mcgen('''
+        visit_type_%(c_type)s(v, &(*obj)->u.%(c_name)s, "data", &err);
 ''',
-                enum_full_value = enum_full_value,
-                c_type=type_name(members[key]),
-                c_name=c_name(key))
+                         c_type=simple_union_type.c_name(),
+                         c_name=c_name(var.name))
+        else:
+            ret += mcgen('''
+        visit_type_implicit_%(c_type)s(v, &(*obj)->u.%(c_name)s, &err);
+''',
+                         c_type=var.type.c_name(),
+                         c_name=c_name(var.name))
+        ret += mcgen('''
+        break;
+''')
 
     ret += mcgen('''
-        default:
-            abort();
-        }
+    default:
+        abort();
+    }
 out_obj:
-        error_propagate(errp, err);
-        err = NULL;
-        visit_end_union(m, !!(*obj)->data, &err);
-        error_propagate(errp, err);
-        err = NULL;
+    error_propagate(errp, err);
+    err = NULL;
+    if (*obj) {
+        visit_end_union(v, !!(*obj)->u.data, &err);
     }
-    visit_end_struct(m, &err);
+    error_propagate(errp, err);
+    err = NULL;
+    visit_end_struct(v, &err);
 out:
     error_propagate(errp, err);
 }
@@ -340,38 +320,63 @@ out:
 
     return ret
 
-def generate_declaration(name, members, builtin_type=False):
-    ret = ""
-    if not builtin_type:
-        name = c_name(name)
-        ret += mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp);
-''',
-                     name=name)
-
-    ret += mcgen('''
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp);
-''',
-                 name=name)
-
-    return ret
-
-def generate_enum_declaration(name, members):
-    ret = mcgen('''
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp);
-''',
-                name=c_name(name))
 
-    return ret
-
-def generate_decl_enum(name, members):
-    return mcgen('''
+class QAPISchemaGenVisitVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._btin = None
+
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._btin = guardstart('QAPI_VISIT_BUILTIN')
+
+    def visit_end(self):
+        # To avoid header dependency hell, we always generate
+        # declarations for built-in types in our header files and
+        # simply guard them.  See also do_builtins (command line
+        # option -b).
+        self._btin += guardend('QAPI_VISIT_BUILTIN')
+        self.decl = self._btin + self.decl
+        self._btin = None
+
+    def visit_needed(self, entity):
+        # Visit everything except implicit objects
+        return not (entity.is_implicit() and
+                    isinstance(entity, QAPISchemaObjectType))
+
+    def visit_enum_type(self, name, info, values, prefix):
+        self.decl += gen_visit_decl(name, scalar=True)
+        self.defn += gen_visit_enum(name)
+
+    def visit_array_type(self, name, info, element_type):
+        decl = gen_visit_decl(name)
+        defn = gen_visit_list(name, element_type)
+        if isinstance(element_type, QAPISchemaBuiltinType):
+            self._btin += decl
+            if do_builtins:
+                self.defn += defn
+        else:
+            self.decl += decl
+            self.defn += defn
+
+    def visit_object_type(self, name, info, base, members, variants):
+        self.decl += gen_visit_decl(name)
+        if variants:
+            assert not members      # not implemented
+            self.defn += gen_visit_union(name, base, variants)
+        else:
+            self.defn += gen_visit_struct(name, base, members)
 
-void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp);
-''',
-                 name=c_name(name))
+    def visit_alternate_type(self, name, info, variants):
+        self.decl += gen_visit_decl(name)
+        self.defn += gen_visit_alternate(name, variants)
 
+# If you link code generated from multiple schemata, you want only one
+# instance of the code for built-in types.  Generate it only when
+# do_builtins, enabled by command line option -b.  See also
+# QAPISchemaGenVisitVisitor.visit_end().
 do_builtins = False
 
 (input_file, output_dir, do_c, do_h, prefix, opts) = \
@@ -418,7 +423,7 @@ fdef.write(mcgen('''
 #include "qemu-common.h"
 #include "%(prefix)sqapi-visit.h"
 ''',
-                 prefix = prefix))
+                 prefix=prefix))
 
 fdecl.write(mcgen('''
 #include "qapi/visitor.h"
@@ -427,58 +432,10 @@ fdecl.write(mcgen('''
 ''',
                   prefix=prefix))
 
-exprs = parse_schema(input_file)
-
-# to avoid header dependency hell, we always generate declarations
-# for built-in types in our header files and simply guard them
-fdecl.write(guardstart("QAPI_VISIT_BUILTIN_VISITOR_DECL"))
-for typename in builtin_types.keys():
-    fdecl.write(generate_declaration(typename, None, builtin_type=True))
-fdecl.write(guardend("QAPI_VISIT_BUILTIN_VISITOR_DECL"))
-
-# ...this doesn't work for cases where we link in multiple objects that
-# have the functions defined, so we use -b option to provide control
-# over these cases
-if do_builtins:
-    for typename in builtin_types.keys():
-        fdef.write(generate_visit_list(typename, None))
-
-for expr in exprs:
-    if expr.has_key('struct'):
-        ret = generate_visit_struct(expr)
-        ret += generate_visit_list(expr['struct'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_declaration(expr['struct'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('union'):
-        ret = generate_visit_union(expr)
-        ret += generate_visit_list(expr['union'], expr['data'])
-        fdef.write(ret)
-
-        enum_define = discriminator_find_enum_define(expr)
-        ret = ""
-        if not enum_define:
-            ret = generate_decl_enum('%sKind' % expr['union'],
-                                     expr['data'].keys())
-        ret += generate_declaration(expr['union'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('alternate'):
-        ret = generate_visit_alternate(expr['alternate'], expr['data'])
-        ret += generate_visit_list(expr['alternate'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_decl_enum('%sKind' % expr['alternate'],
-                                 expr['data'].keys())
-        ret += generate_declaration(expr['alternate'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('enum'):
-        ret = generate_visit_list(expr['enum'], expr['data'])
-        ret += generate_visit_enum(expr['enum'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_decl_enum(expr['enum'], expr['data'])
-        ret += generate_enum_declaration(expr['enum'], expr['data'])
-        fdecl.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenVisitVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 06d7fc284..7c50cc4c8 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -33,12 +33,14 @@ builtin_types = {
     'uint32':   'QTYPE_QINT',
     'uint64':   'QTYPE_QINT',
     'size':     'QTYPE_QINT',
+    'any':      None,           # any qtype_code possible, actually
 }
 
 # Whitelist of commands allowed to return a non-dictionary
 returns_whitelist = [
     # From QMP:
     'human-monitor-command',
+    'qom-get',
     'query-migrate-cache-size',
     'query-tpm-models',
     'query-tpm-types',
@@ -54,9 +56,6 @@ returns_whitelist = [
     'guest-set-vcpus',
     'guest-sync',
     'guest-sync-delimited',
-
-    # From qapi-schema-test:
-    'user_def_cmd3',
 ]
 
 enum_types = []
@@ -69,6 +68,7 @@ all_names = {}
 # Parsing the schema into expressions
 #
 
+
 def error_path(parent):
     res = ""
     while parent:
@@ -77,8 +77,10 @@ def error_path(parent):
         parent = parent['parent']
     return res
 
+
 class QAPISchemaError(Exception):
     def __init__(self, schema, msg):
+        Exception.__init__(self)
         self.fname = schema.fname
         self.msg = msg
         self.col = 1
@@ -94,8 +96,11 @@ class QAPISchemaError(Exception):
         return error_path(self.info) + \
             "%s:%d:%d: %s" % (self.fname, self.line, self.col, self.msg)
 
+
 class QAPIExprError(Exception):
     def __init__(self, expr_info, msg):
+        Exception.__init__(self)
+        assert expr_info
         self.info = expr_info
         self.msg = msg
 
@@ -103,9 +108,10 @@ class QAPIExprError(Exception):
         return error_path(self.info['parent']) + \
             "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
 
-class QAPISchema:
 
-    def __init__(self, fp, previously_included = [], incl_info = None):
+class QAPISchemaParser(object):
+
+    def __init__(self, fp, previously_included=[], incl_info=None):
         abs_fname = os.path.abspath(fp.name)
         fname = fp.name
         self.fname = fname
@@ -120,18 +126,18 @@ class QAPISchema:
         self.exprs = []
         self.accept()
 
-        while self.tok != None:
+        while self.tok is not None:
             expr_info = {'file': fname, 'line': self.line,
                          'parent': self.incl_info}
             expr = self.get_expr(False)
             if isinstance(expr, dict) and "include" in expr:
                 if len(expr) != 1:
-                    raise QAPIExprError(expr_info, "Invalid 'include' directive")
+                    raise QAPIExprError(expr_info,
+                                        "Invalid 'include' directive")
                 include = expr["include"]
                 if not isinstance(include, str):
                     raise QAPIExprError(expr_info,
-                                        'Expected a file name (string), got: %s'
-                                        % include)
+                                        "Value of 'include' must be a string")
                 incl_abs_fname = os.path.join(os.path.dirname(abs_fname),
                                               include)
                 # catch inclusion cycle
@@ -149,8 +155,8 @@ class QAPISchema:
                 except IOError, e:
                     raise QAPIExprError(expr_info,
                                         '%s: %s' % (e.strerror, include))
-                exprs_include = QAPISchema(fobj, previously_included,
-                                           expr_info)
+                exprs_include = QAPISchemaParser(fobj, previously_included,
+                                                 expr_info)
                 self.exprs.extend(exprs_include.exprs)
             else:
                 expr_elem = {'expr': expr,
@@ -166,7 +172,7 @@ class QAPISchema:
 
             if self.tok == '#':
                 self.cursor = self.src.find('\n', self.cursor)
-            elif self.tok in ['{', '}', ':', ',', '[', ']']:
+            elif self.tok in "{}:,[]":
                 return
             elif self.tok == "'":
                 string = ''
@@ -190,7 +196,7 @@ class QAPISchema:
                             string += '\t'
                         elif ch == 'u':
                             value = 0
-                            for x in range(0, 4):
+                            for _ in range(0, 4):
                                 ch = self.src[self.cursor]
                                 self.cursor += 1
                                 if ch not in "0123456789abcdefABCDEF":
@@ -212,7 +218,7 @@ class QAPISchema:
                             string += ch
                         else:
                             raise QAPISchemaError(self,
-                                                  "Unknown escape \\%s" %ch)
+                                                  "Unknown escape \\%s" % ch)
                         esc = False
                     elif ch == "\\":
                         esc = True
@@ -272,7 +278,7 @@ class QAPISchema:
         if self.tok == ']':
             self.accept()
             return expr
-        if not self.tok in "{['tfn":
+        if self.tok not in "{['tfn":
             raise QAPISchemaError(self, 'Expected "{", "[", "]", string, '
                                   'boolean or "null"')
         while True:
@@ -302,17 +308,21 @@ class QAPISchema:
 
 #
 # Semantic analysis of schema expressions
+# TODO fold into QAPISchema
+# TODO catching name collisions in generated code would be nice
 #
 
+
 def find_base_fields(base):
     base_struct_define = find_struct(base)
     if not base_struct_define:
         return None
     return base_struct_define['data']
 
+
 # Return the qtype of an alternate branch, or None on error.
 def find_alternate_member_qtype(qapi_type):
-    if builtin_types.has_key(qapi_type):
+    if qapi_type in builtin_types:
         return builtin_types[qapi_type]
     elif find_struct(qapi_type):
         return "QTYPE_QDICT"
@@ -322,6 +332,7 @@ def find_alternate_member_qtype(qapi_type):
         return "QTYPE_QDICT"
     return None
 
+
 # Return the discriminator enum define if discriminator is specified as an
 # enum type, otherwise return None.
 def discriminator_find_enum_define(expr):
@@ -341,9 +352,14 @@ def discriminator_find_enum_define(expr):
 
     return find_enum(discriminator_type)
 
+
+# FIXME should enforce "other than downstream extensions [...], all
+# names should begin with a letter".
 valid_name = re.compile('^[a-zA-Z_][a-zA-Z0-9_.-]*$')
-def check_name(expr_info, source, name, allow_optional = False,
-               enum_member = False):
+
+
+def check_name(expr_info, source, name, allow_optional=False,
+               enum_member=False):
     global valid_name
     membername = name
 
@@ -360,29 +376,36 @@ def check_name(expr_info, source, name, allow_optional = False,
     # code always prefixes it with the enum name
     if enum_member:
         membername = '_' + membername
-    if not valid_name.match(membername):
+    # Reserve the entire 'q_' namespace for c_name()
+    if not valid_name.match(membername) or \
+       c_name(membername, False).startswith('q_'):
         raise QAPIExprError(expr_info,
                             "%s uses invalid name '%s'" % (source, name))
 
-def add_name(name, info, meta, implicit = False):
+
+def add_name(name, info, meta, implicit=False):
     global all_names
     check_name(info, "'%s'" % meta, name)
+    # FIXME should reject names that differ only in '_' vs. '.'
+    # vs. '-', because they're liable to clash in generated C.
     if name in all_names:
         raise QAPIExprError(info,
                             "%s '%s' is already defined"
                             % (all_names[name], name))
-    if not implicit and name[-4:] == 'Kind':
+    if not implicit and (name.endswith('Kind') or name.endswith('List')):
         raise QAPIExprError(info,
-                            "%s '%s' should not end in 'Kind'"
-                            % (meta, name))
+                            "%s '%s' should not end in '%s'"
+                            % (meta, name, name[-4:]))
     all_names[name] = meta
 
+
 def add_struct(definition, info):
     global struct_types
     name = definition['struct']
     add_name(name, info, 'struct')
     struct_types.append(definition)
 
+
 def find_struct(name):
     global struct_types
     for struct in struct_types:
@@ -390,12 +413,14 @@ def find_struct(name):
             return struct
     return None
 
+
 def add_union(definition, info):
     global union_types
     name = definition['union']
     add_name(name, info, 'union')
     union_types.append(definition)
 
+
 def find_union(name):
     global union_types
     for union in union_types:
@@ -403,11 +428,13 @@ def find_union(name):
             return union
     return None
 
-def add_enum(name, info, enum_values = None, implicit = False):
+
+def add_enum(name, info, enum_values=None, implicit=False):
     global enum_types
     add_name(name, info, 'enum', implicit)
     enum_types.append({"enum_name": name, "enum_values": enum_values})
 
+
 def find_enum(name):
     global enum_types
     for enum in enum_types:
@@ -415,21 +442,19 @@ def find_enum(name):
             return enum
     return None
 
+
 def is_enum(name):
-    return find_enum(name) != None
+    return find_enum(name) is not None
 
-def check_type(expr_info, source, value, allow_array = False,
-               allow_dict = False, allow_optional = False,
-               allow_star = False, allow_metas = []):
+
+def check_type(expr_info, source, value, allow_array=False,
+               allow_dict=False, allow_optional=False,
+               allow_metas=[]):
     global all_names
-    orig_value = value
 
     if value is None:
         return
 
-    if allow_star and value == '**':
-        return
-
     # Check if array type for value is okay
     if isinstance(value, list):
         if not allow_array:
@@ -440,42 +465,44 @@ def check_type(expr_info, source, value, allow_array = False,
                                 "%s: array type must contain single type name"
                                 % source)
         value = value[0]
-        orig_value = "array of %s" %value
 
     # Check if type name for value is okay
     if isinstance(value, str):
-        if value == '**':
-            raise QAPIExprError(expr_info,
-                                "%s uses '**' but did not request 'gen':false"
-                                % source)
-        if not value in all_names:
+        if value not in all_names:
             raise QAPIExprError(expr_info,
                                 "%s uses unknown type '%s'"
-                                % (source, orig_value))
+                                % (source, value))
         if not all_names[value] in allow_metas:
             raise QAPIExprError(expr_info,
                                 "%s cannot use %s type '%s'"
-                                % (source, all_names[value], orig_value))
+                                % (source, all_names[value], value))
         return
 
-    # value is a dictionary, check that each member is okay
-    if not isinstance(value, OrderedDict):
-        raise QAPIExprError(expr_info,
-                            "%s should be a dictionary" % source)
     if not allow_dict:
         raise QAPIExprError(expr_info,
                             "%s should be a type name" % source)
+
+    if not isinstance(value, OrderedDict):
+        raise QAPIExprError(expr_info,
+                            "%s should be a dictionary or type name" % source)
+
+    # value is a dictionary, check that each member is okay
     for (key, arg) in value.items():
         check_name(expr_info, "Member of %s" % source, key,
                    allow_optional=allow_optional)
+        if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
+            raise QAPIExprError(expr_info,
+                                "Member of %s uses reserved name '%s'"
+                                % (source, key))
         # Todo: allow dictionaries to represent default values of
         # an optional argument.
         check_type(expr_info, "Member '%s' of %s" % (key, source), arg,
-                   allow_array=True, allow_star=allow_star,
+                   allow_array=True,
                    allow_metas=['built-in', 'union', 'alternate', 'struct',
                                 'enum'])
 
-def check_member_clash(expr_info, base_name, data, source = ""):
+
+def check_member_clash(expr_info, base_name, data, source=""):
     base = find_struct(base_name)
     assert base
     base_members = base['data']
@@ -489,54 +516,47 @@ def check_member_clash(expr_info, base_name, data, source = ""):
     if base.get('base'):
         check_member_clash(expr_info, base['base'], data, source)
 
+
 def check_command(expr, expr_info):
     name = expr['command']
-    allow_star = expr.has_key('gen')
 
     check_type(expr_info, "'data' for command '%s'" % name,
                expr.get('data'), allow_dict=True, allow_optional=True,
-               allow_metas=['union', 'struct'], allow_star=allow_star)
+               allow_metas=['struct'])
     returns_meta = ['union', 'struct']
     if name in returns_whitelist:
         returns_meta += ['built-in', 'alternate', 'enum']
     check_type(expr_info, "'returns' for command '%s'" % name,
-               expr.get('returns'), allow_array=True, allow_dict=True,
-               allow_optional=True, allow_metas=returns_meta,
-               allow_star=allow_star)
+               expr.get('returns'), allow_array=True,
+               allow_optional=True, allow_metas=returns_meta)
+
 
 def check_event(expr, expr_info):
     global events
     name = expr['event']
-    params = expr.get('data')
 
     if name.upper() == 'MAX':
         raise QAPIExprError(expr_info, "Event name 'MAX' cannot be created")
     events.append(name)
     check_type(expr_info, "'data' for event '%s'" % name,
                expr.get('data'), allow_dict=True, allow_optional=True,
-               allow_metas=['union', 'struct'])
+               allow_metas=['struct'])
+
 
 def check_union(expr, expr_info):
     name = expr['union']
     base = expr.get('base')
     discriminator = expr.get('discriminator')
     members = expr['data']
-    values = { 'MAX': '(automatic)' }
-
-    # If the object has a member 'base', its value must name a struct,
-    # and there must be a discriminator.
-    if base is not None:
-        if discriminator is None:
-            raise QAPIExprError(expr_info,
-                                "Union '%s' requires a discriminator to go "
-                                "along with base" %name)
+    values = {'MAX': '(automatic)', 'KIND': '(automatic)',
+              'TYPE': '(automatic)'}
 
     # Two types of unions, determined by discriminator.
 
     # With no discriminator it is a simple union.
     if discriminator is None:
         enum_define = None
-        allow_metas=['built-in', 'union', 'alternate', 'struct', 'enum']
+        allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
         if base is not None:
             raise QAPIExprError(expr_info,
                                 "Simple union '%s' must not have a base"
@@ -545,15 +565,14 @@ def check_union(expr, expr_info):
     # Else, it's a flat union.
     else:
         # The object must have a string member 'base'.
-        if not isinstance(base, str):
+        check_type(expr_info, "'base' for union '%s'" % name,
+                   base, allow_metas=['struct'])
+        if not base:
             raise QAPIExprError(expr_info,
-                                "Flat union '%s' must have a string base field"
+                                "Flat union '%s' must have a base"
                                 % name)
         base_fields = find_base_fields(base)
-        if not base_fields:
-            raise QAPIExprError(expr_info,
-                                "Base '%s' is not a valid struct"
-                                % base)
+        assert base_fields
 
         # The value of member 'discriminator' must name a non-optional
         # member of the base struct.
@@ -566,7 +585,7 @@ def check_union(expr, expr_info):
                                 "struct '%s'"
                                 % (discriminator, base))
         enum_define = find_enum(discriminator_type)
-        allow_metas=['struct']
+        allow_metas = ['struct']
         # Do not allow string discriminator
         if not enum_define:
             raise QAPIExprError(expr_info,
@@ -588,13 +607,19 @@ def check_union(expr, expr_info):
                                " of branch '%s'" % key)
 
         # If the discriminator names an enum type, then all members
-        # of 'data' must also be members of the enum type.
+        # of 'data' must also be members of the enum type, which in turn
+        # must not collide with the discriminator name.
         if enum_define:
-            if not key in enum_define['enum_values']:
+            if key not in enum_define['enum_values']:
                 raise QAPIExprError(expr_info,
                                     "Discriminator value '%s' is not found in "
                                     "enum '%s'" %
                                     (key, enum_define["enum_name"]))
+            if discriminator in enum_define['enum_values']:
+                raise QAPIExprError(expr_info,
+                                    "Discriminator name '%s' collides with "
+                                    "enum value in '%s'" %
+                                    (discriminator, enum_define["enum_name"]))
 
         # Otherwise, check for conflicts in the generated enum
         else:
@@ -605,10 +630,11 @@ def check_union(expr, expr_info):
                                     % (name, key, values[c_key]))
             values[c_key] = key
 
+
 def check_alternate(expr, expr_info):
     name = expr['alternate']
     members = expr['data']
-    values = { 'MAX': '(automatic)' }
+    values = {'MAX': '(automatic)'}
     types_seen = {}
 
     # Check every branch
@@ -636,16 +662,21 @@ def check_alternate(expr, expr_info):
                                 % (name, key, types_seen[qtype]))
         types_seen[qtype] = key
 
+
 def check_enum(expr, expr_info):
     name = expr['enum']
     members = expr.get('data')
-    values = { 'MAX': '(automatic)' }
+    prefix = expr.get('prefix')
+    values = {'MAX': '(automatic)'}
 
     if not isinstance(members, list):
         raise QAPIExprError(expr_info,
                             "Enum '%s' requires an array for 'data'" % name)
+    if prefix is not None and not isinstance(prefix, str):
+        raise QAPIExprError(expr_info,
+                            "Enum '%s' requires a string for 'prefix'" % name)
     for member in members:
-        check_name(expr_info, "Member of enum '%s'" %name, member,
+        check_name(expr_info, "Member of enum '%s'" % name, member,
                    enum_member=True)
         key = camel_to_upper(member)
         if key in values:
@@ -654,6 +685,7 @@ def check_enum(expr, expr_info):
                                 % (name, member, values[key]))
         values[key] = member
 
+
 def check_struct(expr, expr_info):
     name = expr['struct']
     members = expr['data']
@@ -665,6 +697,7 @@ def check_struct(expr, expr_info):
     if expr.get('base'):
         check_member_clash(expr_info, expr['base'], expr['data'])
 
+
 def check_keys(expr_elem, meta, required, optional=[]):
     expr = expr_elem['expr']
     info = expr_elem['info']
@@ -672,22 +705,23 @@ def check_keys(expr_elem, meta, required, optional=[]):
     if not isinstance(name, str):
         raise QAPIExprError(info,
                             "'%s' key must have a string value" % meta)
-    required = required + [ meta ]
+    required = required + [meta]
     for (key, value) in expr.items():
-        if not key in required and not key in optional:
+        if key not in required and key not in optional:
             raise QAPIExprError(info,
                                 "Unknown key '%s' in %s '%s'"
                                 % (key, meta, name))
-        if (key == 'gen' or key == 'success-response') and value != False:
+        if (key == 'gen' or key == 'success-response') and value is not False:
             raise QAPIExprError(info,
                                 "'%s' of %s '%s' should only use false value"
                                 % (key, meta, name))
     for key in required:
-        if not expr.has_key(key):
+        if key not in expr:
             raise QAPIExprError(info,
                                 "Key '%s' is missing from %s '%s'"
                                 % (key, meta, name))
 
+
 def check_exprs(exprs):
     global all_names
 
@@ -697,24 +731,24 @@ def check_exprs(exprs):
     for expr_elem in exprs:
         expr = expr_elem['expr']
         info = expr_elem['info']
-        if expr.has_key('enum'):
-            check_keys(expr_elem, 'enum', ['data'])
+        if 'enum' in expr:
+            check_keys(expr_elem, 'enum', ['data'], ['prefix'])
             add_enum(expr['enum'], info, expr['data'])
-        elif expr.has_key('union'):
+        elif 'union' in expr:
             check_keys(expr_elem, 'union', ['data'],
                        ['base', 'discriminator'])
             add_union(expr, info)
-        elif expr.has_key('alternate'):
+        elif 'alternate' in expr:
             check_keys(expr_elem, 'alternate', ['data'])
             add_name(expr['alternate'], info, 'alternate')
-        elif expr.has_key('struct'):
+        elif 'struct' in expr:
             check_keys(expr_elem, 'struct', ['data'], ['base'])
             add_struct(expr, info)
-        elif expr.has_key('command'):
+        elif 'command' in expr:
             check_keys(expr_elem, 'command', [],
                        ['data', 'returns', 'gen', 'success-response'])
             add_name(expr['command'], info, 'command')
-        elif expr.has_key('event'):
+        elif 'event' in expr:
             check_keys(expr_elem, 'event', [], ['data'])
             add_name(expr['event'], info, 'event')
         else:
@@ -724,11 +758,11 @@ def check_exprs(exprs):
     # Try again for hidden UnionKind enum
     for expr_elem in exprs:
         expr = expr_elem['expr']
-        if expr.has_key('union'):
+        if 'union' in expr:
             if not discriminator_find_enum_define(expr):
                 add_enum('%sKind' % expr['union'], expr_elem['info'],
                          implicit=True)
-        elif expr.has_key('alternate'):
+        elif 'alternate' in expr:
             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
                      implicit=True)
 
@@ -737,51 +771,585 @@ def check_exprs(exprs):
         expr = expr_elem['expr']
         info = expr_elem['info']
 
-        if expr.has_key('enum'):
+        if 'enum' in expr:
             check_enum(expr, info)
-        elif expr.has_key('union'):
+        elif 'union' in expr:
             check_union(expr, info)
-        elif expr.has_key('alternate'):
+        elif 'alternate' in expr:
             check_alternate(expr, info)
-        elif expr.has_key('struct'):
+        elif 'struct' in expr:
             check_struct(expr, info)
-        elif expr.has_key('command'):
+        elif 'command' in expr:
             check_command(expr, info)
-        elif expr.has_key('event'):
+        elif 'event' in expr:
             check_event(expr, info)
         else:
             assert False, 'unexpected meta type'
 
-    return map(lambda expr_elem: expr_elem['expr'], exprs)
+    return exprs
 
-def parse_schema(fname):
-    try:
-        schema = QAPISchema(open(fname, "r"))
-        return check_exprs(schema.exprs)
-    except (QAPISchemaError, QAPIExprError), e:
-        print >>sys.stderr, e
-        exit(1)
 
 #
-# Code generation helpers
+# Schema compiler frontend
 #
 
-def parse_args(typeinfo):
-    if isinstance(typeinfo, str):
-        struct = find_struct(typeinfo)
-        assert struct != None
-        typeinfo = struct['data']
+class QAPISchemaEntity(object):
+    def __init__(self, name, info):
+        assert isinstance(name, str)
+        self.name = name
+        # For explicitly defined entities, info points to the (explicit)
+        # definition.  For builtins (and their arrays), info is None.
+        # For implicitly defined entities, info points to a place that
+        # triggered the implicit definition (there may be more than one
+        # such place).
+        self.info = info
+
+    def c_name(self):
+        return c_name(self.name)
+
+    def check(self, schema):
+        pass
+
+    def is_implicit(self):
+        return not self.info
+
+    def visit(self, visitor):
+        pass
+
+
+class QAPISchemaVisitor(object):
+    def visit_begin(self, schema):
+        pass
+
+    def visit_end(self):
+        pass
+
+    def visit_needed(self, entity):
+        # Default to visiting everything
+        return True
+
+    def visit_builtin_type(self, name, info, json_type):
+        pass
+
+    def visit_enum_type(self, name, info, values, prefix):
+        pass
+
+    def visit_array_type(self, name, info, element_type):
+        pass
+
+    def visit_object_type(self, name, info, base, members, variants):
+        pass
+
+    def visit_object_type_flat(self, name, info, members, variants):
+        pass
+
+    def visit_alternate_type(self, name, info, variants):
+        pass
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        pass
+
+    def visit_event(self, name, info, arg_type):
+        pass
+
+
+class QAPISchemaType(QAPISchemaEntity):
+    def c_type(self, is_param=False):
+        return c_name(self.name) + pointer_suffix
+
+    def c_null(self):
+        return 'NULL'
+
+    def json_type(self):
+        pass
+
+    def alternate_qtype(self):
+        json2qtype = {
+            'string':  'QTYPE_QSTRING',
+            'number':  'QTYPE_QFLOAT',
+            'int':     'QTYPE_QINT',
+            'boolean': 'QTYPE_QBOOL',
+            'object':  'QTYPE_QDICT'
+        }
+        return json2qtype.get(self.json_type())
+
+
+class QAPISchemaBuiltinType(QAPISchemaType):
+    def __init__(self, name, json_type, c_type, c_null):
+        QAPISchemaType.__init__(self, name, None)
+        assert not c_type or isinstance(c_type, str)
+        assert json_type in ('string', 'number', 'int', 'boolean', 'null',
+                             'value')
+        self._json_type_name = json_type
+        self._c_type_name = c_type
+        self._c_null_val = c_null
+
+    def c_name(self):
+        return self.name
+
+    def c_type(self, is_param=False):
+        if is_param and self.name == 'str':
+            return 'const ' + self._c_type_name
+        return self._c_type_name
+
+    def c_null(self):
+        return self._c_null_val
+
+    def json_type(self):
+        return self._json_type_name
+
+    def visit(self, visitor):
+        visitor.visit_builtin_type(self.name, self.info, self.json_type())
+
+
+class QAPISchemaEnumType(QAPISchemaType):
+    def __init__(self, name, info, values, prefix):
+        QAPISchemaType.__init__(self, name, info)
+        for v in values:
+            assert isinstance(v, str)
+        assert prefix is None or isinstance(prefix, str)
+        self.values = values
+        self.prefix = prefix
+
+    def check(self, schema):
+        assert len(set(self.values)) == len(self.values)
 
-    for member in typeinfo:
-        argname = member
-        argentry = typeinfo[member]
+    def is_implicit(self):
+        # See QAPISchema._make_implicit_enum_type()
+        return self.name.endswith('Kind')
+
+    def c_type(self, is_param=False):
+        return c_name(self.name)
+
+    def c_null(self):
+        return c_enum_const(self.name, (self.values + ['MAX'])[0],
+                            self.prefix)
+
+    def json_type(self):
+        return 'string'
+
+    def visit(self, visitor):
+        visitor.visit_enum_type(self.name, self.info,
+                                self.values, self.prefix)
+
+
+class QAPISchemaArrayType(QAPISchemaType):
+    def __init__(self, name, info, element_type):
+        QAPISchemaType.__init__(self, name, info)
+        assert isinstance(element_type, str)
+        self._element_type_name = element_type
+        self.element_type = None
+
+    def check(self, schema):
+        self.element_type = schema.lookup_type(self._element_type_name)
+        assert self.element_type
+
+    def is_implicit(self):
+        return True
+
+    def json_type(self):
+        return 'array'
+
+    def visit(self, visitor):
+        visitor.visit_array_type(self.name, self.info, self.element_type)
+
+
+class QAPISchemaObjectType(QAPISchemaType):
+    def __init__(self, name, info, base, local_members, variants):
+        QAPISchemaType.__init__(self, name, info)
+        assert base is None or isinstance(base, str)
+        for m in local_members:
+            assert isinstance(m, QAPISchemaObjectTypeMember)
+        assert (variants is None or
+                isinstance(variants, QAPISchemaObjectTypeVariants))
+        self._base_name = base
+        self.base = None
+        self.local_members = local_members
+        self.variants = variants
+        self.members = None
+
+    def check(self, schema):
+        assert self.members is not False        # not running in cycles
+        if self.members:
+            return
+        self.members = False                    # mark as being checked
+        if self._base_name:
+            self.base = schema.lookup_type(self._base_name)
+            assert isinstance(self.base, QAPISchemaObjectType)
+            assert not self.base.variants       # not implemented
+            self.base.check(schema)
+            members = list(self.base.members)
+        else:
+            members = []
+        seen = {}
+        for m in members:
+            assert c_name(m.name) not in seen
+            seen[m.name] = m
+        for m in self.local_members:
+            m.check(schema, members, seen)
+        if self.variants:
+            self.variants.check(schema, members, seen)
+        self.members = members
+
+    def is_implicit(self):
+        # See QAPISchema._make_implicit_object_type()
+        return self.name[0] == ':'
+
+    def c_name(self):
+        assert not self.is_implicit()
+        return QAPISchemaType.c_name(self)
+
+    def c_type(self, is_param=False):
+        assert not self.is_implicit()
+        return QAPISchemaType.c_type(self)
+
+    def json_type(self):
+        return 'object'
+
+    def visit(self, visitor):
+        visitor.visit_object_type(self.name, self.info,
+                                  self.base, self.local_members, self.variants)
+        visitor.visit_object_type_flat(self.name, self.info,
+                                       self.members, self.variants)
+
+
+class QAPISchemaObjectTypeMember(object):
+    def __init__(self, name, typ, optional):
+        assert isinstance(name, str)
+        assert isinstance(typ, str)
+        assert isinstance(optional, bool)
+        self.name = name
+        self._type_name = typ
+        self.type = None
+        self.optional = optional
+
+    def check(self, schema, all_members, seen):
+        assert self.name not in seen
+        self.type = schema.lookup_type(self._type_name)
+        assert self.type
+        all_members.append(self)
+        seen[self.name] = self
+
+
+class QAPISchemaObjectTypeVariants(object):
+    def __init__(self, tag_name, tag_member, variants):
+        # Flat unions pass tag_name but not tag_member.
+        # Simple unions and alternates pass tag_member but not tag_name.
+        # After check(), tag_member is always set, and tag_name remains
+        # a reliable witness of being used by a flat union.
+        assert bool(tag_member) != bool(tag_name)
+        assert (isinstance(tag_name, str) or
+                isinstance(tag_member, QAPISchemaObjectTypeMember))
+        for v in variants:
+            assert isinstance(v, QAPISchemaObjectTypeVariant)
+        self.tag_name = tag_name
+        self.tag_member = tag_member
+        self.variants = variants
+
+    def check(self, schema, members, seen):
+        if self.tag_name:
+            self.tag_member = seen[self.tag_name]
+        else:
+            self.tag_member.check(schema, members, seen)
+        assert isinstance(self.tag_member.type, QAPISchemaEnumType)
+        for v in self.variants:
+            vseen = dict(seen)
+            v.check(schema, self.tag_member.type, vseen)
+
+
+class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
+    def __init__(self, name, typ):
+        QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
+
+    def check(self, schema, tag_type, seen):
+        QAPISchemaObjectTypeMember.check(self, schema, [], seen)
+        assert self.name in tag_type.values
+
+    # This function exists to support ugly simple union special cases
+    # TODO get rid of them, and drop the function
+    def simple_union_type(self):
+        if (self.type.is_implicit() and
+                isinstance(self.type, QAPISchemaObjectType)):
+            assert len(self.type.members) == 1
+            assert not self.type.variants
+            return self.type.members[0].type
+        return None
+
+
+class QAPISchemaAlternateType(QAPISchemaType):
+    def __init__(self, name, info, variants):
+        QAPISchemaType.__init__(self, name, info)
+        assert isinstance(variants, QAPISchemaObjectTypeVariants)
+        assert not variants.tag_name
+        self.variants = variants
+
+    def check(self, schema):
+        self.variants.check(schema, [], {})
+
+    def json_type(self):
+        return 'value'
+
+    def visit(self, visitor):
+        visitor.visit_alternate_type(self.name, self.info, self.variants)
+
+
+class QAPISchemaCommand(QAPISchemaEntity):
+    def __init__(self, name, info, arg_type, ret_type, gen, success_response):
+        QAPISchemaEntity.__init__(self, name, info)
+        assert not arg_type or isinstance(arg_type, str)
+        assert not ret_type or isinstance(ret_type, str)
+        self._arg_type_name = arg_type
+        self.arg_type = None
+        self._ret_type_name = ret_type
+        self.ret_type = None
+        self.gen = gen
+        self.success_response = success_response
+
+    def check(self, schema):
+        if self._arg_type_name:
+            self.arg_type = schema.lookup_type(self._arg_type_name)
+            assert isinstance(self.arg_type, QAPISchemaObjectType)
+            assert not self.arg_type.variants   # not implemented
+        if self._ret_type_name:
+            self.ret_type = schema.lookup_type(self._ret_type_name)
+            assert isinstance(self.ret_type, QAPISchemaType)
+
+    def visit(self, visitor):
+        visitor.visit_command(self.name, self.info,
+                              self.arg_type, self.ret_type,
+                              self.gen, self.success_response)
+
+
+class QAPISchemaEvent(QAPISchemaEntity):
+    def __init__(self, name, info, arg_type):
+        QAPISchemaEntity.__init__(self, name, info)
+        assert not arg_type or isinstance(arg_type, str)
+        self._arg_type_name = arg_type
+        self.arg_type = None
+
+    def check(self, schema):
+        if self._arg_type_name:
+            self.arg_type = schema.lookup_type(self._arg_type_name)
+            assert isinstance(self.arg_type, QAPISchemaObjectType)
+            assert not self.arg_type.variants   # not implemented
+
+    def visit(self, visitor):
+        visitor.visit_event(self.name, self.info, self.arg_type)
+
+
+class QAPISchema(object):
+    def __init__(self, fname):
+        try:
+            self.exprs = check_exprs(QAPISchemaParser(open(fname, "r")).exprs)
+            self._entity_dict = {}
+            self._predefining = True
+            self._def_predefineds()
+            self._predefining = False
+            self._def_exprs()
+            self.check()
+        except (QAPISchemaError, QAPIExprError), err:
+            print >>sys.stderr, err
+            exit(1)
+
+    def _def_entity(self, ent):
+        # Only the predefined types are allowed to not have info
+        assert ent.info or self._predefining
+        assert ent.name not in self._entity_dict
+        self._entity_dict[ent.name] = ent
+
+    def lookup_entity(self, name, typ=None):
+        ent = self._entity_dict.get(name)
+        if typ and not isinstance(ent, typ):
+            return None
+        return ent
+
+    def lookup_type(self, name):
+        return self.lookup_entity(name, QAPISchemaType)
+
+    def _def_builtin_type(self, name, json_type, c_type, c_null):
+        self._def_entity(QAPISchemaBuiltinType(name, json_type,
+                                               c_type, c_null))
+        # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
+        # qapi-types.h from a single .c, all arrays of builtins must be
+        # declared in the first file whether or not they are used.  Nicer
+        # would be to use lazy instantiation, while figuring out how to
+        # avoid compilation issues with multiple qapi-types.h.
+        self._make_array_type(name, None)
+
+    def _def_predefineds(self):
+        for t in [('str',    'string',  'char' + pointer_suffix, 'NULL'),
+                  ('number', 'number',  'double',   '0'),
+                  ('int',    'int',     'int64_t',  '0'),
+                  ('int8',   'int',     'int8_t',   '0'),
+                  ('int16',  'int',     'int16_t',  '0'),
+                  ('int32',  'int',     'int32_t',  '0'),
+                  ('int64',  'int',     'int64_t',  '0'),
+                  ('uint8',  'int',     'uint8_t',  '0'),
+                  ('uint16', 'int',     'uint16_t', '0'),
+                  ('uint32', 'int',     'uint32_t', '0'),
+                  ('uint64', 'int',     'uint64_t', '0'),
+                  ('size',   'int',     'uint64_t', '0'),
+                  ('bool',   'boolean', 'bool',     'false'),
+                  ('any',    'value',   'QObject' + pointer_suffix, 'NULL')]:
+            self._def_builtin_type(*t)
+        self.the_empty_object_type = QAPISchemaObjectType(':empty', None, None,
+                                                          [], None)
+        self._def_entity(self.the_empty_object_type)
+
+    def _make_implicit_enum_type(self, name, info, values):
+        name = name + 'Kind'   # Use namespace reserved by add_name()
+        self._def_entity(QAPISchemaEnumType(name, info, values, None))
+        return name
+
+    def _make_array_type(self, element_type, info):
+        name = element_type + 'List'   # Use namespace reserved by add_name()
+        if not self.lookup_type(name):
+            self._def_entity(QAPISchemaArrayType(name, info, element_type))
+        return name
+
+    def _make_implicit_object_type(self, name, info, role, members):
+        if not members:
+            return None
+        name = ':obj-%s-%s' % (name, role)
+        if not self.lookup_entity(name, QAPISchemaObjectType):
+            self._def_entity(QAPISchemaObjectType(name, info, None,
+                                                  members, None))
+        return name
+
+    def _def_enum_type(self, expr, info):
+        name = expr['enum']
+        data = expr['data']
+        prefix = expr.get('prefix')
+        self._def_entity(QAPISchemaEnumType(name, info, data, prefix))
+
+    def _make_member(self, name, typ, info):
         optional = False
-        if member.startswith('*'):
-            argname = member[1:]
+        if name.startswith('*'):
+            name = name[1:]
             optional = True
-        # Todo: allow argentry to be OrderedDict, for providing the
-        # value of an optional argument.
-        yield (argname, argentry, optional)
+        if isinstance(typ, list):
+            assert len(typ) == 1
+            typ = self._make_array_type(typ[0], info)
+        return QAPISchemaObjectTypeMember(name, typ, optional)
+
+    def _make_members(self, data, info):
+        return [self._make_member(key, value, info)
+                for (key, value) in data.iteritems()]
+
+    def _def_struct_type(self, expr, info):
+        name = expr['struct']
+        base = expr.get('base')
+        data = expr['data']
+        self._def_entity(QAPISchemaObjectType(name, info, base,
+                                              self._make_members(data, info),
+                                              None))
+
+    def _make_variant(self, case, typ):
+        return QAPISchemaObjectTypeVariant(case, typ)
+
+    def _make_simple_variant(self, case, typ, info):
+        if isinstance(typ, list):
+            assert len(typ) == 1
+            typ = self._make_array_type(typ[0], info)
+        typ = self._make_implicit_object_type(
+            typ, info, 'wrapper', [self._make_member('data', typ, info)])
+        return QAPISchemaObjectTypeVariant(case, typ)
+
+    def _make_implicit_tag(self, type_name, info, variants):
+        typ = self._make_implicit_enum_type(type_name, info,
+                                            [v.name for v in variants])
+        return QAPISchemaObjectTypeMember('type', typ, False)
+
+    def _def_union_type(self, expr, info):
+        name = expr['union']
+        data = expr['data']
+        base = expr.get('base')
+        tag_name = expr.get('discriminator')
+        tag_member = None
+        if tag_name:
+            variants = [self._make_variant(key, value)
+                        for (key, value) in data.iteritems()]
+        else:
+            variants = [self._make_simple_variant(key, value, info)
+                        for (key, value) in data.iteritems()]
+            tag_member = self._make_implicit_tag(name, info, variants)
+        self._def_entity(
+            QAPISchemaObjectType(name, info, base,
+                                 self._make_members(OrderedDict(), info),
+                                 QAPISchemaObjectTypeVariants(tag_name,
+                                                              tag_member,
+                                                              variants)))
+
+    def _def_alternate_type(self, expr, info):
+        name = expr['alternate']
+        data = expr['data']
+        variants = [self._make_variant(key, value)
+                    for (key, value) in data.iteritems()]
+        tag_member = self._make_implicit_tag(name, info, variants)
+        self._def_entity(
+            QAPISchemaAlternateType(name, info,
+                                    QAPISchemaObjectTypeVariants(None,
+                                                                 tag_member,
+                                                                 variants)))
+
+    def _def_command(self, expr, info):
+        name = expr['command']
+        data = expr.get('data')
+        rets = expr.get('returns')
+        gen = expr.get('gen', True)
+        success_response = expr.get('success-response', True)
+        if isinstance(data, OrderedDict):
+            data = self._make_implicit_object_type(
+                name, info, 'arg', self._make_members(data, info))
+        if isinstance(rets, list):
+            assert len(rets) == 1
+            rets = self._make_array_type(rets[0], info)
+        self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
+                                           success_response))
+
+    def _def_event(self, expr, info):
+        name = expr['event']
+        data = expr.get('data')
+        if isinstance(data, OrderedDict):
+            data = self._make_implicit_object_type(
+                name, info, 'arg', self._make_members(data, info))
+        self._def_entity(QAPISchemaEvent(name, info, data))
+
+    def _def_exprs(self):
+        for expr_elem in self.exprs:
+            expr = expr_elem['expr']
+            info = expr_elem['info']
+            if 'enum' in expr:
+                self._def_enum_type(expr, info)
+            elif 'struct' in expr:
+                self._def_struct_type(expr, info)
+            elif 'union' in expr:
+                self._def_union_type(expr, info)
+            elif 'alternate' in expr:
+                self._def_alternate_type(expr, info)
+            elif 'command' in expr:
+                self._def_command(expr, info)
+            elif 'event' in expr:
+                self._def_event(expr, info)
+            else:
+                assert False
+
+    def check(self):
+        for ent in self._entity_dict.values():
+            ent.check(self)
+
+    def visit(self, visitor):
+        visitor.visit_begin(self)
+        for (name, entity) in sorted(self._entity_dict.items()):
+            if visitor.visit_needed(entity):
+                entity.visit(visitor)
+        visitor.visit_end()
+
+
+#
+# Code generation helpers
+#
 
 def camel_case(name):
     new_name = ''
@@ -796,6 +1364,7 @@ def camel_case(name):
             new_name += ch.lower()
     return new_name
 
+
 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
 # ENUM24_Name -> ENUM24_NAME
@@ -810,19 +1379,22 @@ def camel_to_upper(value):
         c = c_fun_str[i]
         # When c is upper and no "_" appears before, do more checks
         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
-            # Case 1: next string is lower
-            # Case 2: previous string is digit
-            if (i < (l - 1) and c_fun_str[i + 1].islower()) or \
-            c_fun_str[i - 1].isdigit():
+            if i < l - 1 and c_fun_str[i + 1].islower():
+                new_name += '_'
+            elif c_fun_str[i - 1].isdigit():
                 new_name += '_'
         new_name += c
     return new_name.lstrip('_').upper()
 
-def c_enum_const(type_name, const_name):
+
+def c_enum_const(type_name, const_name, prefix=None):
+    if prefix is not None:
+        type_name = prefix
     return camel_to_upper(type_name + '_' + const_name)
 
 c_name_trans = string.maketrans('.-', '__')
 
+
 # Map @name to a valid C identifier.
 # If @protect, avoid returning certain ticklish identifiers (like
 # C keywords) by prepending "q_".
@@ -835,15 +1407,16 @@ c_name_trans = string.maketrans('.-', '__')
 def c_name(name, protect=True):
     # ANSI X3J11/88-090, 3.1.1
     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
-                     'default', 'do', 'double', 'else', 'enum', 'extern', 'float',
-                     'for', 'goto', 'if', 'int', 'long', 'register', 'return',
-                     'short', 'signed', 'sizeof', 'static', 'struct', 'switch',
-                     'typedef', 'union', 'unsigned', 'void', 'volatile', 'while'])
+                     'default', 'do', 'double', 'else', 'enum', 'extern',
+                     'float', 'for', 'goto', 'if', 'int', 'long', 'register',
+                     'return', 'short', 'signed', 'sizeof', 'static',
+                     'struct', 'switch', 'typedef', 'union', 'unsigned',
+                     'void', 'volatile', 'while'])
     # ISO/IEC 9899:1999, 6.4.1
     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
     # ISO/IEC 9899:2011, 6.4.1
-    c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic', '_Noreturn',
-                     '_Static_assert', '_Thread_local'])
+    c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
+                     '_Noreturn', '_Static_assert', '_Thread_local'])
     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
     # excluding _.*
     gcc_words = set(['asm', 'typeof'])
@@ -859,108 +1432,56 @@ def c_name(name, protect=True):
                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
     # namespace pollution:
     polluted_words = set(['unix', 'errno'])
-    if protect and (name in c89_words | c99_words | c11_words | gcc_words | cpp_words | polluted_words):
+    if protect and (name in c89_words | c99_words | c11_words | gcc_words
+                    | cpp_words | polluted_words):
         return "q_" + name
     return name.translate(c_name_trans)
 
-# Map type @name to the C typedef name for the list form.
-#
-# ['Name'] -> 'NameList', ['x-Foo'] -> 'x_FooList', ['int'] -> 'intList'
-def c_list_type(name):
-    return type_name(name) + 'List'
-
-# Map type @value to the C typedef form.
-#
-# Used for converting 'type' from a 'member':'type' qapi definition
-# into the alphanumeric portion of the type for a generated C parameter,
-# as well as generated C function names.  See c_type() for the rest of
-# the conversion such as adding '*' on pointer types.
-# 'int' -> 'int', '[x-Foo]' -> 'x_FooList', '__a.b_c' -> '__a_b_c'
-def type_name(value):
-    if type(value) == list:
-        return c_list_type(value[0])
-    if value in builtin_types.keys():
-        return value
-    return c_name(value)
-
 eatspace = '\033EATSPACE.'
 pointer_suffix = ' *' + eatspace
 
-# Map type @name to its C type expression.
-# If @is_param, const-qualify the string type.
-#
-# This function is used for computing the full C type of 'member':'name'.
-# A special suffix is added in c_type() for pointer types, and it's
-# stripped in mcgen(). So please notice this when you check the return
-# value of c_type() outside mcgen().
-def c_type(value, is_param=False):
-    if value == 'str':
-        if is_param:
-            return 'const char' + pointer_suffix
-        return 'char' + pointer_suffix
-
-    elif value == 'int':
-        return 'int64_t'
-    elif (value == 'int8' or value == 'int16' or value == 'int32' or
-          value == 'int64' or value == 'uint8' or value == 'uint16' or
-          value == 'uint32' or value == 'uint64'):
-        return value + '_t'
-    elif value == 'size':
-        return 'uint64_t'
-    elif value == 'bool':
-        return 'bool'
-    elif value == 'number':
-        return 'double'
-    elif type(value) == list:
-        return c_list_type(value[0]) + pointer_suffix
-    elif is_enum(value):
-        return c_name(value)
-    elif value == None:
-        return 'void'
-    elif value in events:
-        return camel_case(value) + 'Event' + pointer_suffix
-    else:
-        # complex type name
-        assert isinstance(value, str) and value != ""
-        return c_name(value) + pointer_suffix
-
-def is_c_ptr(value):
-    return c_type(value).endswith(pointer_suffix)
 
 def genindent(count):
     ret = ""
-    for i in range(count):
+    for _ in range(count):
         ret += " "
     return ret
 
 indent_level = 0
 
+
 def push_indent(indent_amount=4):
     global indent_level
     indent_level += indent_amount
 
+
 def pop_indent(indent_amount=4):
     global indent_level
     indent_level -= indent_amount
 
+
+# Generate @code with @kwds interpolated.
+# Obey indent_level, and strip eatspace.
 def cgen(code, **kwds):
-    indent = genindent(indent_level)
-    lines = code.split('\n')
-    lines = map(lambda x: indent + x, lines)
-    return '\n'.join(lines) % kwds + '\n'
+    raw = code % kwds
+    if indent_level:
+        indent = genindent(indent_level)
+        # re.subn() lacks flags support before Python 2.7, use re.compile()
+        raw = re.subn(re.compile("^.", re.MULTILINE),
+                      indent + r'\g<0>', raw)
+        raw = raw[0]
+    return re.sub(re.escape(eatspace) + ' *', '', raw)
+
 
 def mcgen(code, **kwds):
-    raw = cgen('\n'.join(code.split('\n')[1:-1]), **kwds)
-    return re.sub(re.escape(eatspace) + ' *', '', raw)
+    if code[0] == '\n':
+        code = code[1:]
+    return cgen(code, **kwds)
 
-def basename(filename):
-    return filename.split("/")[-1]
 
 def guardname(filename):
-    guard = basename(filename).rsplit(".", 1)[0]
-    for substr in [".", " ", "-"]:
-        guard = guard.replace(substr, "_")
-    return guard.upper() + '_H'
+    return c_name(filename, protect=False).upper()
+
 
 def guardstart(name):
     return mcgen('''
@@ -971,6 +1492,7 @@ def guardstart(name):
 ''',
                  name=guardname(name))
 
+
 def guardend(name):
     return mcgen('''
 
@@ -979,11 +1501,138 @@ def guardend(name):
 ''',
                  name=guardname(name))
 
+
+def gen_enum_lookup(name, values, prefix=None):
+    ret = mcgen('''
+
+const char *const %(c_name)s_lookup[] = {
+''',
+                c_name=c_name(name))
+    for value in values:
+        index = c_enum_const(name, value, prefix)
+        ret += mcgen('''
+    [%(index)s] = "%(value)s",
+''',
+                     index=index, value=value)
+
+    max_index = c_enum_const(name, 'MAX', prefix)
+    ret += mcgen('''
+    [%(max_index)s] = NULL,
+};
+''',
+                 max_index=max_index)
+    return ret
+
+
+def gen_enum(name, values, prefix=None):
+    # append automatically generated _MAX value
+    enum_values = values + ['MAX']
+
+    ret = mcgen('''
+
+typedef enum %(c_name)s {
+''',
+                c_name=c_name(name))
+
+    i = 0
+    for value in enum_values:
+        ret += mcgen('''
+    %(c_enum)s = %(i)d,
+''',
+                     c_enum=c_enum_const(name, value, prefix),
+                     i=i)
+        i += 1
+
+    ret += mcgen('''
+} %(c_name)s;
+''',
+                 c_name=c_name(name))
+
+    ret += mcgen('''
+
+extern const char *const %(c_name)s_lookup[];
+''',
+                 c_name=c_name(name))
+    return ret
+
+
+def gen_params(arg_type, extra):
+    if not arg_type:
+        return extra
+    assert not arg_type.variants
+    ret = ''
+    sep = ''
+    for memb in arg_type.members:
+        ret += sep
+        sep = ', '
+        if memb.optional:
+            ret += 'bool has_%s, ' % c_name(memb.name)
+        ret += '%s %s' % (memb.type.c_type(is_param=True), c_name(memb.name))
+    if extra:
+        ret += sep + extra
+    return ret
+
+
+def gen_err_check(label='out', skiperr=False):
+    if skiperr:
+        return ''
+    return mcgen('''
+    if (err) {
+        goto %(label)s;
+    }
+''',
+                 label=label)
+
+
+def gen_visit_fields(members, prefix='', need_cast=False, skiperr=False):
+    ret = ''
+    if skiperr:
+        errparg = 'NULL'
+    else:
+        errparg = '&err'
+
+    for memb in members:
+        if memb.optional:
+            ret += mcgen('''
+    visit_optional(v, &%(prefix)shas_%(c_name)s, "%(name)s", %(errp)s);
+''',
+                         prefix=prefix, c_name=c_name(memb.name),
+                         name=memb.name, errp=errparg)
+            ret += gen_err_check(skiperr=skiperr)
+            ret += mcgen('''
+    if (%(prefix)shas_%(c_name)s) {
+''',
+                         prefix=prefix, c_name=c_name(memb.name))
+            push_indent()
+
+        # Ugly: sometimes we need to cast away const
+        if need_cast and memb.type.name == 'str':
+            cast = '(char **)'
+        else:
+            cast = ''
+
+        ret += mcgen('''
+    visit_type_%(c_type)s(v, %(cast)s&%(prefix)s%(c_name)s, "%(name)s", %(errp)s);
+''',
+                     c_type=memb.type.c_name(), prefix=prefix, cast=cast,
+                     c_name=c_name(memb.name), name=memb.name,
+                     errp=errparg)
+        ret += gen_err_check(skiperr=skiperr)
+
+        if memb.optional:
+            pop_indent()
+            ret += mcgen('''
+    }
+''')
+    return ret
+
+
 #
 # Common command line parsing
 #
 
-def parse_command_line(extra_options = "", extra_long_options = []):
+
+def parse_command_line(extra_options="", extra_long_options=[]):
 
     try:
         opts, args = getopt.gnu_getopt(sys.argv[1:],
@@ -1003,6 +1652,12 @@ def parse_command_line(extra_options = "", extra_long_options = []):
     for oa in opts:
         o, a = oa
         if o in ("-p", "--prefix"):
+            match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
+            if match.end() != len(a):
+                print >>sys.stderr, \
+                    "%s: 'funny character '%s' in argument of --prefix" \
+                    % (sys.argv[0], a[match.end()])
+                sys.exit(1)
             prefix = a
         elif o in ("-o", "--output-dir"):
             output_dir = a + "/"
@@ -1028,16 +1683,19 @@ def parse_command_line(extra_options = "", extra_long_options = []):
 # Generate output files with boilerplate
 #
 
+
 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
                 c_comment, h_comment):
+    guard = guardname(prefix + h_file)
     c_file = output_dir + prefix + c_file
     h_file = output_dir + prefix + h_file
 
-    try:
-        os.makedirs(output_dir)
-    except os.error, e:
-        if e.errno != errno.EEXIST:
-            raise
+    if output_dir:
+        try:
+            os.makedirs(output_dir)
+        except os.error, e:
+            if e.errno != errno.EEXIST:
+                raise
 
     def maybe_open(really, name, opt):
         if really:
@@ -1053,7 +1711,7 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
 %(comment)s
 ''',
-                     comment = c_comment))
+                     comment=c_comment))
 
     fdecl.write(mcgen('''
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
@@ -1062,10 +1720,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
 #define %(guard)s
 
 ''',
-                      comment = h_comment, guard = guardname(h_file)))
+                      comment=h_comment, guard=guard))
 
     return (fdef, fdecl)
 
+
 def close_output(fdef, fdecl):
     fdecl.write('''
 #endif
diff --git a/scripts/qemu-gdb.py b/scripts/qemu-gdb.py
index 6c7f4fbe5..b3f8e04f7 100644
--- a/scripts/qemu-gdb.py
+++ b/scripts/qemu-gdb.py
@@ -13,73 +13,20 @@
 # Contributions after 2012-01-13 are licensed under the terms of the
 # GNU GPL, version 2 or (at your option) any later version.
 
+# Usage:
+# At the (gdb) prompt, type "source scripts/qemu-gdb.py".
+# "help qemu" should then list the supported QEMU debug support commands.
 
 import gdb
 
-def isnull(ptr):
-    return ptr == gdb.Value(0).cast(ptr.type)
+import os, sys
 
-def int128(p):
-    return long(p['lo']) + (long(p['hi']) << 64)
+# Annoyingly, gdb doesn't put the directory of scripts onto the
+# module search path. Do it manually.
 
-def get_fs_base():
-    '''Fetch %fs base value using arch_prctl(ARCH_GET_FS)'''
-    # %rsp - 120 is scratch space according to the SystemV ABI
-    old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
-    gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True)
-    fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
-    gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True)
-    return fs_base
+sys.path.append(os.path.dirname(__file__))
 
-def get_glibc_pointer_guard():
-    '''Fetch glibc pointer guard value'''
-    fs_base = get_fs_base()
-    return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base)
-
-def glibc_ptr_demangle(val, pointer_guard):
-    '''Undo effect of glibc's PTR_MANGLE()'''
-    return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard))
-
-def bt_jmpbuf(jmpbuf):
-    '''Backtrace a jmpbuf'''
-    JB_RBX  = 0
-    JB_RBP  = 1
-    JB_R12  = 2
-    JB_R13  = 3
-    JB_R14  = 4
-    JB_R15  = 5
-    JB_RSP  = 6
-    JB_PC   = 7
-
-    old_rbx = gdb.parse_and_eval('(uint64_t)$rbx')
-    old_rbp = gdb.parse_and_eval('(uint64_t)$rbp')
-    old_rsp = gdb.parse_and_eval('(uint64_t)$rsp')
-    old_r12 = gdb.parse_and_eval('(uint64_t)$r12')
-    old_r13 = gdb.parse_and_eval('(uint64_t)$r13')
-    old_r14 = gdb.parse_and_eval('(uint64_t)$r14')
-    old_r15 = gdb.parse_and_eval('(uint64_t)$r15')
-    old_rip = gdb.parse_and_eval('(uint64_t)$rip')
-
-    pointer_guard = get_glibc_pointer_guard()
-    gdb.execute('set $rbx = %s' % jmpbuf[JB_RBX])
-    gdb.execute('set $rbp = %s' % glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard))
-    gdb.execute('set $rsp = %s' % glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard))
-    gdb.execute('set $r12 = %s' % jmpbuf[JB_R12])
-    gdb.execute('set $r13 = %s' % jmpbuf[JB_R13])
-    gdb.execute('set $r14 = %s' % jmpbuf[JB_R14])
-    gdb.execute('set $r15 = %s' % jmpbuf[JB_R15])
-    gdb.execute('set $rip = %s' % glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard))
-
-    gdb.execute('bt')
-
-    gdb.execute('set $rbx = %s' % old_rbx)
-    gdb.execute('set $rbp = %s' % old_rbp)
-    gdb.execute('set $rsp = %s' % old_rsp)
-    gdb.execute('set $r12 = %s' % old_r12)
-    gdb.execute('set $r13 = %s' % old_r13)
-    gdb.execute('set $r14 = %s' % old_r14)
-    gdb.execute('set $r15 = %s' % old_r15)
-    gdb.execute('set $rip = %s' % old_rip)
+from qemugdb import aio, mtree, coroutine
 
 class QemuCommand(gdb.Command):
     '''Prefix for QEMU debug support commands'''
@@ -87,78 +34,14 @@ class QemuCommand(gdb.Command):
         gdb.Command.__init__(self, 'qemu', gdb.COMMAND_DATA,
                              gdb.COMPLETE_NONE, True)
 
-class CoroutineCommand(gdb.Command):
-    '''Display coroutine backtrace'''
-    def __init__(self):
-        gdb.Command.__init__(self, 'qemu coroutine', gdb.COMMAND_DATA,
-                             gdb.COMPLETE_NONE)
-
-    def invoke(self, arg, from_tty):
-        argv = gdb.string_to_argv(arg)
-        if len(argv) != 1:
-            gdb.write('usage: qemu coroutine <coroutine-pointer>\n')
-            return
-
-        coroutine_pointer = gdb.parse_and_eval(argv[0]).cast(gdb.lookup_type('CoroutineUContext').pointer())
-        bt_jmpbuf(coroutine_pointer['env']['__jmpbuf'])
+QemuCommand()
+coroutine.CoroutineCommand()
+mtree.MtreeCommand()
+aio.HandlersCommand()
 
-class MtreeCommand(gdb.Command):
-    '''Display the memory tree hierarchy'''
-    def __init__(self):
-        gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA,
-                             gdb.COMPLETE_NONE)
-        self.queue = []
-    def invoke(self, arg, from_tty):
-        self.seen = set()
-        self.queue_root('address_space_memory')
-        self.queue_root('address_space_io')
-        self.process_queue()
-    def queue_root(self, varname):
-        ptr = gdb.parse_and_eval(varname)['root']
-        self.queue.append(ptr)
-    def process_queue(self):
-        while self.queue:
-            ptr = self.queue.pop(0)
-            if long(ptr) in self.seen:
-                continue
-            self.print_item(ptr)
-    def print_item(self, ptr, offset = gdb.Value(0), level = 0):
-        self.seen.add(long(ptr))
-        addr = ptr['addr']
-        addr += offset
-        size = int128(ptr['size'])
-        alias = ptr['alias']
-        klass = ''
-        if not isnull(alias):
-            klass = ' (alias)'
-        elif not isnull(ptr['ops']):
-            klass = ' (I/O)'
-        elif bool(ptr['ram']):
-            klass = ' (RAM)'
-        gdb.write('%s%016x-%016x %s%s (@ %s)\n'
-                  % ('  ' * level,
-                     long(addr),
-                     long(addr + (size - 1)),
-                     ptr['name'].string(),
-                     klass,
-                     ptr,
-                     ),
-                  gdb.STDOUT)
-        if not isnull(alias):
-            gdb.write('%s    alias: %s@%016x (@ %s)\n' %
-                      ('  ' * level,
-                       alias['name'].string(),
-                       ptr['alias_offset'],
-                       alias,
-                       ),
-                      gdb.STDOUT)
-            self.queue.append(alias)
-        subregion = ptr['subregions']['tqh_first']
-        level += 1
-        while not isnull(subregion):
-            self.print_item(subregion, addr, level)
-            subregion = subregion['subregions_link']['tqe_next']
+coroutine.CoroutineSPFunction()
+coroutine.CoroutinePCFunction()
 
-QemuCommand()
-CoroutineCommand()
-MtreeCommand()
+# Default to silently passing through SIGUSR1, because QEMU sends it
+# to itself a lot.
+gdb.execute('handle SIGUSR1 pass noprint nostop')
diff --git a/scripts/qemugdb/__init__.py b/scripts/qemugdb/__init__.py
new file mode 100644
index 000000000..969f552b2
--- /dev/null
+++ b/scripts/qemugdb/__init__.py
@@ -0,0 +1,28 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright (c) 2015 Linaro Ltd
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see
+# <http://www.gnu.org/licenses/gpl-2.0.html>
+#
+
+# We don't need to do anything in our init file currently.
+
+"""
+Support routines for debugging QEMU under GDB
+"""
+
+__license__    = "GPL version 2 or (at your option) any later version"
diff --git a/scripts/qemugdb/aio.py b/scripts/qemugdb/aio.py
new file mode 100644
index 000000000..2ba00c444
--- /dev/null
+++ b/scripts/qemugdb/aio.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+
+# GDB debugging support: aio/iohandler debug
+#
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Author: Dr. David Alan Gilbert <dgilbert@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+#
+
+import gdb
+from qemugdb import coroutine
+
+def isnull(ptr):
+    return ptr == gdb.Value(0).cast(ptr.type)
+
+def dump_aiocontext(context, verbose):
+    '''Display a dump and backtrace for an aiocontext'''
+    cur = context['aio_handlers']['lh_first']
+    # Get pointers to functions we're going to process specially
+    sym_fd_coroutine_enter = gdb.parse_and_eval('fd_coroutine_enter')
+
+    while not isnull(cur):
+        entry = cur.dereference()
+        gdb.write('----\n%s\n' % entry)
+        if verbose and cur['io_read'] == sym_fd_coroutine_enter:
+            coptr = (cur['opaque'].cast(gdb.lookup_type('FDYieldUntilData').pointer()))['co']
+            coptr = coptr.cast(gdb.lookup_type('CoroutineUContext').pointer())
+            coroutine.bt_jmpbuf(coptr['env']['__jmpbuf'])
+        cur = cur['node']['le_next'];
+
+    gdb.write('----\n')
+
+class HandlersCommand(gdb.Command):
+    '''Display aio handlers'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu handlers', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+
+    def invoke(self, arg, from_tty):
+        verbose = False
+        argv = gdb.string_to_argv(arg)
+
+        if len(argv) > 0 and argv[0] == '--verbose':
+            verbose = True
+            argv.pop(0)
+
+        if len(argv) > 1:
+            gdb.write('usage: qemu handlers [--verbose] [handler]\n')
+            return
+
+        if len(argv) == 1:
+            handlers_name = argv[0]
+        else:
+            handlers_name = 'qemu_aio_context'
+        dump_aiocontext(gdb.parse_and_eval(handlers_name), verbose)
diff --git a/scripts/qemugdb/coroutine.py b/scripts/qemugdb/coroutine.py
new file mode 100644
index 000000000..ab699794a
--- /dev/null
+++ b/scripts/qemugdb/coroutine.py
@@ -0,0 +1,119 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright 2012 Red Hat, Inc. and/or its affiliates
+#
+# Authors:
+#  Avi Kivity <avi@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Contributions after 2012-01-13 are licensed under the terms of the
+# GNU GPL, version 2 or (at your option) any later version.
+
+import gdb
+
+VOID_PTR = gdb.lookup_type('void').pointer()
+
+def get_fs_base():
+    '''Fetch %fs base value using arch_prctl(ARCH_GET_FS).  This is
+       pthread_self().'''
+    # %rsp - 120 is scratch space according to the SystemV ABI
+    old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
+    gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True)
+    fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
+    gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True)
+    return fs_base
+
+def pthread_self():
+    '''Fetch pthread_self() from the glibc start_thread function.'''
+    f = gdb.newest_frame()
+    while f.name() != 'start_thread':
+        f = f.older()
+        if f is None:
+            return get_fs_base()
+
+    try:
+        return f.read_var("arg")
+    except ValueError:
+        return get_fs_base()
+
+def get_glibc_pointer_guard():
+    '''Fetch glibc pointer guard value'''
+    fs_base = pthread_self()
+    return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base)
+
+def glibc_ptr_demangle(val, pointer_guard):
+    '''Undo effect of glibc's PTR_MANGLE()'''
+    return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard))
+
+def get_jmpbuf_regs(jmpbuf):
+    JB_RBX  = 0
+    JB_RBP  = 1
+    JB_R12  = 2
+    JB_R13  = 3
+    JB_R14  = 4
+    JB_R15  = 5
+    JB_RSP  = 6
+    JB_PC   = 7
+
+    pointer_guard = get_glibc_pointer_guard()
+    return {'rbx': jmpbuf[JB_RBX],
+        'rbp': glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard),
+        'rsp': glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard),
+        'r12': jmpbuf[JB_R12],
+        'r13': jmpbuf[JB_R13],
+        'r14': jmpbuf[JB_R14],
+        'r15': jmpbuf[JB_R15],
+        'rip': glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard) }
+
+def bt_jmpbuf(jmpbuf):
+    '''Backtrace a jmpbuf'''
+    regs = get_jmpbuf_regs(jmpbuf)
+    old = dict()
+
+    for i in regs:
+        old[i] = gdb.parse_and_eval('(uint64_t)$%s' % i)
+
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, regs[i]))
+
+    gdb.execute('bt')
+
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, old[i]))
+
+def coroutine_to_jmpbuf(co):
+    coroutine_pointer = co.cast(gdb.lookup_type('CoroutineUContext').pointer())
+    return coroutine_pointer['env']['__jmpbuf']
+
+
+class CoroutineCommand(gdb.Command):
+    '''Display coroutine backtrace'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu coroutine', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+
+    def invoke(self, arg, from_tty):
+        argv = gdb.string_to_argv(arg)
+        if len(argv) != 1:
+            gdb.write('usage: qemu coroutine <coroutine-pointer>\n')
+            return
+
+        bt_jmpbuf(coroutine_to_jmpbuf(gdb.parse_and_eval(argv[0])))
+
+class CoroutineSPFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_sp')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rsp'].cast(VOID_PTR)
+
+class CoroutinePCFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_pc')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rip'].cast(VOID_PTR)
diff --git a/scripts/qemugdb/mtree.py b/scripts/qemugdb/mtree.py
new file mode 100644
index 000000000..06011c30c
--- /dev/null
+++ b/scripts/qemugdb/mtree.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright 2012 Red Hat, Inc. and/or its affiliates
+#
+# Authors:
+#  Avi Kivity <avi@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Contributions after 2012-01-13 are licensed under the terms of the
+# GNU GPL, version 2 or (at your option) any later version.
+
+# 'qemu mtree' -- display the memory hierarchy
+
+import gdb
+
+def isnull(ptr):
+    return ptr == gdb.Value(0).cast(ptr.type)
+
+def int128(p):
+    return long(p['lo']) + (long(p['hi']) << 64)
+
+class MtreeCommand(gdb.Command):
+    '''Display the memory tree hierarchy'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+        self.queue = []
+    def invoke(self, arg, from_tty):
+        self.seen = set()
+        self.queue_root('address_space_memory')
+        self.queue_root('address_space_io')
+        self.process_queue()
+    def queue_root(self, varname):
+        ptr = gdb.parse_and_eval(varname)['root']
+        self.queue.append(ptr)
+    def process_queue(self):
+        while self.queue:
+            ptr = self.queue.pop(0)
+            if long(ptr) in self.seen:
+                continue
+            self.print_item(ptr)
+    def print_item(self, ptr, offset = gdb.Value(0), level = 0):
+        self.seen.add(long(ptr))
+        addr = ptr['addr']
+        addr += offset
+        size = int128(ptr['size'])
+        alias = ptr['alias']
+        klass = ''
+        if not isnull(alias):
+            klass = ' (alias)'
+        elif not isnull(ptr['ops']):
+            klass = ' (I/O)'
+        elif bool(ptr['ram']):
+            klass = ' (RAM)'
+        gdb.write('%s%016x-%016x %s%s (@ %s)\n'
+                  % ('  ' * level,
+                     long(addr),
+                     long(addr + (size - 1)),
+                     ptr['name'].string(),
+                     klass,
+                     ptr,
+                     ),
+                  gdb.STDOUT)
+        if not isnull(alias):
+            gdb.write('%s    alias: %s@%016x (@ %s)\n' %
+                      ('  ' * level,
+                       alias['name'].string(),
+                       ptr['alias_offset'],
+                       alias,
+                       ),
+                      gdb.STDOUT)
+            self.queue.append(alias)
+        subregion = ptr['subregions']['tqh_first']
+        level += 1
+        while not isnull(subregion):
+            self.print_item(subregion, addr, level)
+            subregion = subregion['subregions_link']['tqe_next']
+
diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell
index 65280d29d..fa39bf0d7 100755
--- a/scripts/qmp/qmp-shell
+++ b/scripts/qmp/qmp-shell
@@ -29,6 +29,41 @@
 # (QEMU) device_add driver=e1000 id=net1
 # {u'return': {}}
 # (QEMU)
+#
+# key=value pairs also support Python or JSON object literal subset notations,
+# without spaces. Dictionaries/objects {} are supported as are arrays [].
+#
+#    example-command arg-name1={'key':'value','obj'={'prop':"value"}}
+#
+# Both JSON and Python formatting should work, including both styles of
+# string literal quotes. Both paradigms of literal values should work,
+# including null/true/false for JSON and None/True/False for Python.
+#
+#
+# Transactions have the following multi-line format:
+#
+#    transaction(
+#    action-name1 [ arg-name1=arg1 ] ... [arg-nameN=argN ]
+#    ...
+#    action-nameN [ arg-name1=arg1 ] ... [arg-nameN=argN ]
+#    )
+#
+# One line transactions are also supported:
+#
+#    transaction( action-name1 ... )
+#
+# For example:
+#
+#     (QEMU) transaction(
+#     TRANS> block-dirty-bitmap-add node=drive0 name=bitmap1
+#     TRANS> block-dirty-bitmap-clear node=drive0 name=bitmap0
+#     TRANS> )
+#     {"return": {}}
+#     (QEMU)
+#
+# Use the -v and -p options to activate the verbose and pretty-print options,
+# which will echo back the properly formatted JSON-compliant QMP that is being
+# sent to QEMU, which is useful for debugging and documentation generation.
 
 import qmp
 import json
diff --git a/scripts/texi2pod.pl b/scripts/texi2pod.pl
index 94097fb06..8767662d3 100755
--- a/scripts/texi2pod.pl
+++ b/scripts/texi2pod.pl
@@ -317,7 +317,7 @@ while(<$inf>) {
 	@columns = ();
 	for $column (split (/\s*\@tab\s*/, $1)) {
 	    # @strong{...} is used a @headitem work-alike
-	    $column =~ s/^\@strong{(.*)}$/$1/;
+	    $column =~ s/^\@strong\{(.*)\}$/$1/;
 	    push @columns, $column;
 	}
 	$_ = "\n=item ".join (" : ", @columns)."\n";
diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh
index 201032846..096d0900c 100755
--- a/scripts/update-linux-headers.sh
+++ b/scripts/update-linux-headers.sh
@@ -28,39 +28,33 @@ if [ -z "$output" ]; then
     output="$PWD"
 fi
 
-cp_virtio() {
-    from=$1
+cp_portable() {
+    f=$1
     to=$2
-    virtio=$(find "$from" -name '*virtio*h' -o -name "input.h" -o -name "pci_regs.h")
-    if [ "$virtio" ]; then
-        rm -rf "$to"
-        mkdir -p "$to"
-        for f in $virtio; do
-            if
-                grep '#include' "$f" | grep -v -e 'linux/virtio' \
-                                             -e 'linux/types' \
-                                             -e 'linux/if_ether' \
-                                             -e 'sys/' \
-                                             > /dev/null
-            then
-                echo "Unexpected #include in input file $f".
-                exit 2
-            fi
-
-            header=$(basename "$f");
-            sed -e 's/__u\([0-9][0-9]*\)/uint\1_t/g' \
-                -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \
-                -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \
-                -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \
-                -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \
-                -e 's/__bitwise__//' \
-                -e 's/__attribute__((packed))/QEMU_PACKED/' \
-                -e 's/__inline__/inline/' \
-                -e '/sys\/ioctl.h/d' \
-                -e 's/SW_MAX/SW_MAX_/' \
-                "$f" > "$to/$header";
-        done
+    if
+        grep '#include' "$f" | grep -v -e 'linux/virtio' \
+                                     -e 'linux/types' \
+                                     -e 'stdint' \
+                                     -e 'linux/if_ether' \
+                                     -e 'sys/' \
+                                     > /dev/null
+    then
+        echo "Unexpected #include in input file $f".
+        exit 2
     fi
+
+    header=$(basename "$f");
+    sed -e 's/__u\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \
+        -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \
+        -e 's/__bitwise__//' \
+        -e 's/__attribute__((packed))/QEMU_PACKED/' \
+        -e 's/__inline__/inline/' \
+        -e '/sys\/ioctl.h/d' \
+        -e 's/SW_MAX/SW_MAX_/' \
+        "$f" > "$to/$header";
 }
 
 # This will pick up non-directories too (eg "Kconfig") but we will
@@ -75,7 +69,7 @@ for arch in $ARCHLIST; do
     fi
 
     # Blacklist architectures which have KVM headers but are actually dead
-    if [ "$arch" = "ia64" ]; then
+    if [ "$arch" = "ia64" -o "$arch" = "mips" ]; then
         continue
     fi
 
@@ -83,17 +77,25 @@ for arch in $ARCHLIST; do
 
     rm -rf "$output/linux-headers/asm-$arch"
     mkdir -p "$output/linux-headers/asm-$arch"
-    for header in kvm.h kvm_para.h; do
+    for header in kvm.h kvm_para.h unistd.h; do
         cp "$tmpdir/include/asm/$header" "$output/linux-headers/asm-$arch"
     done
-    if [ $arch = x86 ]; then
-        cp "$tmpdir/include/asm/hyperv.h" "$output/linux-headers/asm-x86"
-    fi
     if [ $arch = powerpc ]; then
         cp "$tmpdir/include/asm/epapr_hcalls.h" "$output/linux-headers/asm-powerpc/"
     fi
 
-    cp_virtio "$tmpdir/include/asm" "$output/include/standard-headers/asm-$arch"
+    rm -rf "$output/include/standard-headers/asm-$arch"
+    mkdir -p "$output/include/standard-headers/asm-$arch"
+    if [ $arch = s390 ]; then
+        cp_portable "$tmpdir/include/asm/kvm_virtio.h" "$output/include/standard-headers/asm-s390/"
+        cp_portable "$tmpdir/include/asm/virtio-ccw.h" "$output/include/standard-headers/asm-s390/"
+    fi
+    if [ $arch = x86 ]; then
+        cp_portable "$tmpdir/include/asm/hyperv.h" "$output/include/standard-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_32.h" "$output/linux-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_x32.h" "$output/linux-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_64.h" "$output/linux-headers/asm-x86/"
+    fi
 done
 
 rm -rf "$output/linux-headers/linux"
@@ -113,6 +115,9 @@ else
     cp "$linux/COPYING" "$output/linux-headers"
 fi
 
+cat <<EOF >$output/linux-headers/asm-x86/hyperv.h
+#include "standard-headers/asm-x86/hyperv.h"
+EOF
 cat <<EOF >$output/linux-headers/linux/virtio_config.h
 #include "standard-headers/linux/virtio_config.h"
 EOF
@@ -120,7 +125,12 @@ cat <<EOF >$output/linux-headers/linux/virtio_ring.h
 #include "standard-headers/linux/virtio_ring.h"
 EOF
 
-cp_virtio "$tmpdir/include/linux/" "$output/include/standard-headers/linux"
+rm -rf "$output/include/standard-headers/linux"
+mkdir -p "$output/include/standard-headers/linux"
+for i in "$tmpdir"/include/linux/*virtio*.h "$tmpdir/include/linux/input.h" \
+         "$tmpdir/include/linux/pci_regs.h"; do
+    cp_portable "$i" "$output/include/standard-headers/linux"
+done
 
 cat <<EOF >$output/include/standard-headers/linux/types.h
 #include <stdint.h>
diff --git a/slirp/bootp.c b/slirp/bootp.c
index b7db9fa33..1baaab1ab 100644
--- a/slirp/bootp.c
+++ b/slirp/bootp.c
@@ -23,6 +23,12 @@
  */
 #include <slirp.h>
 
+#if defined(_WIN32)
+/* Windows ntohl() returns an u_long value.
+ * Add a type cast to match the format strings. */
+# define ntohl(n) ((uint32_t)ntohl(n))
+#endif
+
 /* XXX: only DHCP is supported */
 
 #define LEASE_TIME (24 * 3600)
@@ -155,7 +161,7 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
     dhcp_decode(bp, &dhcp_msg_type, &preq_addr);
     DPRINTF("bootp packet op=%d msgtype=%d", bp->bp_op, dhcp_msg_type);
     if (preq_addr.s_addr != htonl(0L))
-        DPRINTF(" req_addr=%08x\n", ntohl(preq_addr.s_addr));
+        DPRINTF(" req_addr=%08" PRIx32 "\n", ntohl(preq_addr.s_addr));
     else
         DPRINTF("\n");
 
@@ -234,7 +240,7 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
     q += 4;
 
     if (bc) {
-        DPRINTF("%s addr=%08x\n",
+        DPRINTF("%s addr=%08" PRIx32 "\n",
                 (dhcp_msg_type == DHCPDISCOVER) ? "offered" : "ack'ed",
                 ntohl(daddr.sin_addr.s_addr));
 
@@ -302,7 +308,7 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
     } else {
         static const char nak_msg[] = "requested address not available";
 
-        DPRINTF("nak'ed addr=%08x\n", ntohl(preq_addr.s_addr));
+        DPRINTF("nak'ed addr=%08" PRIx32 "\n", ntohl(preq_addr.s_addr));
 
         *q++ = RFC2132_MSG_TYPE;
         *q++ = 1;
diff --git a/slirp/if.c b/slirp/if.c
index fb7acf87d..8325a2afb 100644
--- a/slirp/if.c
+++ b/slirp/if.c
@@ -53,8 +53,8 @@ if_output(struct socket *so, struct mbuf *ifm)
 	int on_fastq = 1;
 
 	DEBUG_CALL("if_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("ifm = %lx", (long)ifm);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("ifm = %p", ifm);
 
 	/*
 	 * First remove the mbuf from m_usedlist,
diff --git a/slirp/ip_icmp.c b/slirp/ip_icmp.c
index 9f1cb08a1..23b9f0fa4 100644
--- a/slirp/ip_icmp.c
+++ b/slirp/ip_icmp.c
@@ -125,7 +125,7 @@ icmp_input(struct mbuf *m, int hlen)
   Slirp *slirp = m->slirp;
 
   DEBUG_CALL("icmp_input");
-  DEBUG_ARG("m = %lx", (long )m);
+  DEBUG_ARG("m = %p", m);
   DEBUG_ARG("m_len = %d", m->m_len);
 
   /*
@@ -252,7 +252,7 @@ icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
   register struct mbuf *m;
 
   DEBUG_CALL("icmp_error");
-  DEBUG_ARG("msrc = %lx", (long )msrc);
+  DEBUG_ARG("msrc = %p", msrc);
   DEBUG_ARG("msrc_len = %d", msrc->m_len);
 
   if(type!=ICMP_UNREACH && type!=ICMP_TIMXCEED) goto end_error;
diff --git a/slirp/ip_input.c b/slirp/ip_input.c
index 880bdfd3c..7d436e6ec 100644
--- a/slirp/ip_input.c
+++ b/slirp/ip_input.c
@@ -80,7 +80,7 @@ ip_input(struct mbuf *m)
 	int hlen;
 
 	DEBUG_CALL("ip_input");
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("m_len = %d", m->m_len);
 
 	if (m->m_len < sizeof (struct ip)) {
@@ -232,9 +232,9 @@ ip_reass(Slirp *slirp, struct ip *ip, struct ipq *fp)
 	int i, next;
 
 	DEBUG_CALL("ip_reass");
-	DEBUG_ARG("ip = %lx", (long)ip);
-	DEBUG_ARG("fp = %lx", (long)fp);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("ip = %p", ip);
+	DEBUG_ARG("fp = %p", fp);
+	DEBUG_ARG("m = %p", m);
 
 	/*
 	 * Presence of header sizes in mbufs
@@ -400,7 +400,7 @@ static void
 ip_enq(register struct ipasfrag *p, register struct ipasfrag *prev)
 {
 	DEBUG_CALL("ip_enq");
-	DEBUG_ARG("prev = %lx", (long)prev);
+	DEBUG_ARG("prev = %p", prev);
 	p->ipf_prev =  prev;
 	p->ipf_next = prev->ipf_next;
 	((struct ipasfrag *)(prev->ipf_next))->ipf_prev = p;
diff --git a/slirp/ip_output.c b/slirp/ip_output.c
index c82830fe7..1254d0d58 100644
--- a/slirp/ip_output.c
+++ b/slirp/ip_output.c
@@ -60,8 +60,8 @@ ip_output(struct socket *so, struct mbuf *m0)
 	int len, off, error = 0;
 
 	DEBUG_CALL("ip_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m0 = %lx", (long)m0);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m0 = %p", m0);
 
 	ip = mtod(m, struct ip *);
 	/*
diff --git a/slirp/mbuf.c b/slirp/mbuf.c
index 4fefb043b..795fc29f9 100644
--- a/slirp/mbuf.c
+++ b/slirp/mbuf.c
@@ -94,7 +94,7 @@ m_get(Slirp *slirp)
         m->arp_requested = false;
         m->expiration_date = (uint64_t)-1;
 end_error:
-	DEBUG_ARG("m = %lx", (long )m);
+	DEBUG_ARG("m = %p", m);
 	return m;
 }
 
@@ -103,7 +103,7 @@ m_free(struct mbuf *m)
 {
 
   DEBUG_CALL("m_free");
-  DEBUG_ARG("m = %lx", (long )m);
+  DEBUG_ARG("m = %p", m);
 
   if(m) {
 	/* Remove from m_usedlist */
@@ -221,7 +221,7 @@ dtom(Slirp *slirp, void *dat)
 	struct mbuf *m;
 
 	DEBUG_CALL("dtom");
-	DEBUG_ARG("dat = %lx", (long )dat);
+	DEBUG_ARG("dat = %p", dat);
 
 	/* bug corrected for M_EXT buffers */
 	for (m = slirp->m_usedlist.m_next; m != &slirp->m_usedlist;
diff --git a/slirp/misc.c b/slirp/misc.c
index 578e8b2c1..5497161f1 100644
--- a/slirp/misc.c
+++ b/slirp/misc.c
@@ -123,9 +123,9 @@ fork_exec(struct socket *so, const char *ex, int do_pty)
 	pid_t pid;
 
 	DEBUG_CALL("fork_exec");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("ex = %lx", (long)ex);
-	DEBUG_ARG("do_pty = %lx", (long)do_pty);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("ex = %p", ex);
+	DEBUG_ARG("do_pty = %x", do_pty);
 
 	if (do_pty == 2) {
                 return 0;
diff --git a/slirp/sbuf.c b/slirp/sbuf.c
index 08ec2b4f4..b8c3db744 100644
--- a/slirp/sbuf.c
+++ b/slirp/sbuf.c
@@ -72,8 +72,8 @@ sbappend(struct socket *so, struct mbuf *m)
 	int ret = 0;
 
 	DEBUG_CALL("sbappend");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("m->m_len = %d", m->m_len);
 
 	/* Shouldn't happen, but...  e.g. foreign host closes connection */
diff --git a/slirp/socket.c b/slirp/socket.c
index 37ac5cf2f..1673e3afc 100644
--- a/slirp/socket.c
+++ b/slirp/socket.c
@@ -91,7 +91,7 @@ size_t sopreprbuf(struct socket *so, struct iovec *iov, int *np)
 	int mss = so->so_tcpcb->t_maxseg;
 
 	DEBUG_CALL("sopreprbuf");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	if (len <= 0)
 		return 0;
@@ -155,7 +155,7 @@ soread(struct socket *so)
 	struct iovec iov[2];
 
 	DEBUG_CALL("soread");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * No need to check if there's enough room to read.
@@ -215,7 +215,7 @@ int soreadbuf(struct socket *so, const char *buf, int size)
 	struct iovec iov[2];
 
 	DEBUG_CALL("soreadbuf");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * No need to check if there's enough room to read.
@@ -263,7 +263,7 @@ sorecvoob(struct socket *so)
 	struct tcpcb *tp = sototcpcb(so);
 
 	DEBUG_CALL("sorecvoob");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * We take a guess at how much urgent data has arrived.
@@ -293,7 +293,7 @@ sosendoob(struct socket *so)
 	int n, len;
 
 	DEBUG_CALL("sosendoob");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 	DEBUG_ARG("sb->sb_cc = %d", sb->sb_cc);
 
 	if (so->so_urgc > 2048)
@@ -351,7 +351,7 @@ sowrite(struct socket *so)
 	struct iovec iov[2];
 
 	DEBUG_CALL("sowrite");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	if (so->so_urgc) {
 		sosendoob(so);
@@ -441,7 +441,7 @@ sorecvfrom(struct socket *so)
 	socklen_t addrlen = sizeof(struct sockaddr_in);
 
 	DEBUG_CALL("sorecvfrom");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	if (so->so_type == IPPROTO_ICMP) {   /* This is a "ping" reply */
 	  char buff[256];
@@ -543,8 +543,8 @@ sosendto(struct socket *so, struct mbuf *m)
 	struct sockaddr_in addr;
 
 	DEBUG_CALL("sosendto");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 
         addr.sin_family = AF_INET;
 	if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
diff --git a/slirp/tcp_input.c b/slirp/tcp_input.c
index 00a77b4a5..6b096ecb3 100644
--- a/slirp/tcp_input.c
+++ b/slirp/tcp_input.c
@@ -231,8 +231,8 @@ tcp_input(struct mbuf *m, int iphlen, struct socket *inso)
     Slirp *slirp;
 
 	DEBUG_CALL("tcp_input");
-	DEBUG_ARGS((dfd, " m = %8lx  iphlen = %2d  inso = %lx\n",
-		    (long )m, iphlen, (long )inso ));
+	DEBUG_ARGS((dfd, " m = %p  iphlen = %2d  inso = %p\n",
+		    m, iphlen, inso));
 
 	/*
 	 * If called with m == 0, then we're continuing the connect
@@ -923,8 +923,8 @@ trimthenstep6:
 
 		if (SEQ_LEQ(ti->ti_ack, tp->snd_una)) {
 			if (ti->ti_len == 0 && tiwin == tp->snd_wnd) {
-			  DEBUG_MISC((dfd, " dup ack  m = %lx  so = %lx\n",
-				      (long )m, (long )so));
+			  DEBUG_MISC((dfd, " dup ack  m = %p  so = %p\n",
+				      m, so));
 				/*
 				 * If we have outstanding data (other than
 				 * a window probe), this is a completely
@@ -1302,7 +1302,7 @@ tcp_dooptions(struct tcpcb *tp, u_char *cp, int cnt, struct tcpiphdr *ti)
 	int opt, optlen;
 
 	DEBUG_CALL("tcp_dooptions");
-	DEBUG_ARGS((dfd, " tp = %lx  cnt=%i\n", (long)tp, cnt));
+	DEBUG_ARGS((dfd, " tp = %p  cnt=%i\n", tp, cnt));
 
 	for (; cnt > 0; cnt -= optlen, cp += optlen) {
 		opt = cp[0];
@@ -1383,7 +1383,7 @@ tcp_xmit_timer(register struct tcpcb *tp, int rtt)
 	register short delta;
 
 	DEBUG_CALL("tcp_xmit_timer");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("rtt = %d", rtt);
 
 	if (tp->t_srtt != 0) {
@@ -1471,7 +1471,7 @@ tcp_mss(struct tcpcb *tp, u_int offer)
 	int mss;
 
 	DEBUG_CALL("tcp_mss");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("offer = %d", offer);
 
 	mss = min(IF_MTU, IF_MRU) - sizeof(struct tcpiphdr);
diff --git a/slirp/tcp_output.c b/slirp/tcp_output.c
index 8aa3d9047..fafca58a0 100644
--- a/slirp/tcp_output.c
+++ b/slirp/tcp_output.c
@@ -66,7 +66,7 @@ tcp_output(struct tcpcb *tp)
 	int idle, sendalot;
 
 	DEBUG_CALL("tcp_output");
-	DEBUG_ARG("tp = %lx", (long )tp);
+	DEBUG_ARG("tp = %p", tp);
 
 	/*
 	 * Determine length of data that should be transmitted,
diff --git a/slirp/tcp_subr.c b/slirp/tcp_subr.c
index 7571c5a28..e161ed2a9 100644
--- a/slirp/tcp_subr.c
+++ b/slirp/tcp_subr.c
@@ -224,7 +224,7 @@ tcp_newtcpcb(struct socket *so)
 struct tcpcb *tcp_drop(struct tcpcb *tp, int err)
 {
 	DEBUG_CALL("tcp_drop");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("errno = %d", errno);
 
 	if (TCPS_HAVERCVDSYN(tp->t_state)) {
@@ -249,7 +249,7 @@ tcp_close(struct tcpcb *tp)
 	register struct mbuf *m;
 
 	DEBUG_CALL("tcp_close");
-	DEBUG_ARG("tp = %lx", (long )tp);
+	DEBUG_ARG("tp = %p", tp);
 
 	/* free the reassembly queue, if any */
 	t = tcpfrag_list_first(tp);
@@ -290,7 +290,7 @@ tcp_sockclosed(struct tcpcb *tp)
 {
 
 	DEBUG_CALL("tcp_sockclosed");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 
 	switch (tp->t_state) {
 
@@ -330,7 +330,7 @@ int tcp_fconnect(struct socket *so)
   int ret=0;
 
   DEBUG_CALL("tcp_fconnect");
-  DEBUG_ARG("so = %lx", (long )so);
+  DEBUG_ARG("so = %p", so);
 
   if( (ret = so->s = qemu_socket(AF_INET,SOCK_STREAM,0)) >= 0) {
     int opt, s=so->s;
@@ -393,7 +393,7 @@ void tcp_connect(struct socket *inso)
     int s, opt;
 
     DEBUG_CALL("tcp_connect");
-    DEBUG_ARG("inso = %lx", (long)inso);
+    DEBUG_ARG("inso = %p", inso);
 
     /*
      * If it's an SS_ACCEPTONCE socket, no need to socreate()
@@ -564,8 +564,8 @@ tcp_emu(struct socket *so, struct mbuf *m)
 	char *bptr;
 
 	DEBUG_CALL("tcp_emu");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 
 	switch(so->so_emu) {
 		int x, i;
@@ -900,7 +900,7 @@ int tcp_ctl(struct socket *so)
     int do_pty;
 
     DEBUG_CALL("tcp_ctl");
-    DEBUG_ARG("so = %lx", (long )so);
+    DEBUG_ARG("so = %p", so);
 
     if (so->so_faddr.s_addr != slirp->vhost_addr.s_addr) {
         /* Check if it's pty_exec */
diff --git a/slirp/udp.c b/slirp/udp.c
index f77e00f5a..fee13b4db 100644
--- a/slirp/udp.c
+++ b/slirp/udp.c
@@ -72,7 +72,7 @@ udp_input(register struct mbuf *m, int iphlen)
 	struct socket *so;
 
 	DEBUG_CALL("udp_input");
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("iphlen = %d", iphlen);
 
 	/*
@@ -241,8 +241,8 @@ int udp_output2(struct socket *so, struct mbuf *m,
 	int error = 0;
 
 	DEBUG_CALL("udp_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("saddr = %lx", (long)saddr->sin_addr.s_addr);
 	DEBUG_ARG("daddr = %lx", (long)daddr->sin_addr.s_addr);
 
diff --git a/softmmu_template.h b/softmmu_template.h
index d42d89d54..6803890e4 100644
--- a/softmmu_template.h
+++ b/softmmu_template.h
@@ -154,7 +154,7 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
 
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
     cpu->mem_io_pc = retaddr;
-    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
+    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
         cpu_io_recompile(cpu, retaddr);
     }
 
@@ -165,9 +165,6 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
 }
 #endif
 
-#ifdef SOFTMMU_CODE_ACCESS
-static __attribute__((unused))
-#endif
 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
@@ -252,9 +249,6 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
 }
 
 #if DATA_SIZE > 1
-#ifdef SOFTMMU_CODE_ACCESS
-static __attribute__((unused))
-#endif
 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
@@ -335,14 +329,6 @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
 }
 #endif /* DATA_SIZE > 1 */
 
-DATA_TYPE
-glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
-                                         int mmu_idx)
-{
-    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
-    return helper_te_ld_name (env, addr, oi, GETRA());
-}
-
 #ifndef SOFTMMU_CODE_ACCESS
 
 /* Provide signed versions of the load routines as well.  We can of course
@@ -374,7 +360,7 @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env,
     MemoryRegion *mr = iotlb_to_region(cpu, physaddr);
 
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
-    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
+    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
         cpu_io_recompile(cpu, retaddr);
     }
 
@@ -540,14 +526,6 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 }
 #endif /* DATA_SIZE > 1 */
 
-void
-glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
-                                         DATA_TYPE val, int mmu_idx)
-{
-    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
-    helper_te_st_name(env, addr, val, oi, GETRA());
-}
-
 #if DATA_SIZE == 1
 /* Probe for whether the specified guest write access is permitted.
  * If it is not permitted then an exception will be taken in the same
diff --git a/spice-qemu-char.c b/spice-qemu-char.c
index d41bb745a..e70e0f736 100644
--- a/spice-qemu-char.c
+++ b/spice-qemu-char.c
@@ -296,15 +296,14 @@ static CharDriverState *chr_open(const char *subtype,
     return chr;
 }
 
-CharDriverState *qemu_chr_open_spice_vmc(const char *type)
+static CharDriverState *qemu_chr_open_spice_vmc(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
+                                                Error **errp)
 {
+    const char *type = backend->u.spicevmc->type;
     const char **psubtype = spice_server_char_device_recognized_subtypes();
 
-    if (type == NULL) {
-        fprintf(stderr, "spice-qemu-char: missing name parameter\n");
-        print_allowed_subtypes();
-        return NULL;
-    }
     for (; *psubtype != NULL; ++psubtype) {
         if (strcmp(type, *psubtype) == 0) {
             break;
@@ -320,8 +319,12 @@ CharDriverState *qemu_chr_open_spice_vmc(const char *type)
 }
 
 #if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name)
+static CharDriverState *qemu_chr_open_spice_port(const char *id,
+                                                 ChardevBackend *backend,
+                                                 ChardevReturn *ret,
+                                                 Error **errp)
 {
+    const char *name = backend->u.spiceport->fqdn;
     CharDriverState *chr;
     SpiceCharDriver *s;
 
@@ -359,8 +362,8 @@ static void qemu_chr_parse_spice_vmc(QemuOpts *opts, ChardevBackend *backend,
         error_setg(errp, "chardev: spice channel: no name given");
         return;
     }
-    backend->spicevmc = g_new0(ChardevSpiceChannel, 1);
-    backend->spicevmc->type = g_strdup(name);
+    backend->u.spicevmc = g_new0(ChardevSpiceChannel, 1);
+    backend->u.spicevmc->type = g_strdup(name);
 }
 
 static void qemu_chr_parse_spice_port(QemuOpts *opts, ChardevBackend *backend,
@@ -372,16 +375,16 @@ static void qemu_chr_parse_spice_port(QemuOpts *opts, ChardevBackend *backend,
         error_setg(errp, "chardev: spice port: no name given");
         return;
     }
-    backend->spiceport = g_new0(ChardevSpicePort, 1);
-    backend->spiceport->fqdn = g_strdup(name);
+    backend->u.spiceport = g_new0(ChardevSpicePort, 1);
+    backend->u.spiceport->fqdn = g_strdup(name);
 }
 
 static void register_types(void)
 {
     register_char_driver("spicevmc", CHARDEV_BACKEND_KIND_SPICEVMC,
-                         qemu_chr_parse_spice_vmc);
+                         qemu_chr_parse_spice_vmc, qemu_chr_open_spice_vmc);
     register_char_driver("spiceport", CHARDEV_BACKEND_KIND_SPICEPORT,
-                         qemu_chr_parse_spice_port);
+                         qemu_chr_parse_spice_port, qemu_chr_open_spice_port);
 }
 
 type_init(register_types);
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
index 9937a1295..d7898a0c4 100644
--- a/stubs/Makefile.objs
+++ b/stubs/Makefile.objs
@@ -1,8 +1,5 @@
 stub-obj-y += arch-query-cpu-def.o
 stub-obj-y += bdrv-commit-all.o
-stub-obj-y += chr-baum-init.o
-stub-obj-y += chr-msmouse.o
-stub-obj-y += chr-testdev.o
 stub-obj-y += clock-warp.o
 stub-obj-y += cpu-get-clock.o
 stub-obj-y += cpu-get-icount.o
@@ -23,18 +20,21 @@ stub-obj-y += mon-is-qmp.o
 stub-obj-y += mon-printf.o
 stub-obj-y += monitor-init.o
 stub-obj-y += notify-event.o
-stub-obj-$(CONFIG_SPICE) += qemu-chr-open-spice.o
 stub-obj-y += qtest.o
+stub-obj-y += replay.o
+stub-obj-y += replay-user.o
 stub-obj-y += reset.o
 stub-obj-y += runstate-check.o
 stub-obj-y += set-fd-handler.o
 stub-obj-y += slirp.o
 stub-obj-y += sysbus.o
 stub-obj-y += uuid.o
-stub-obj-y += vc-init.o
 stub-obj-y += vm-stop.o
 stub-obj-y += vmstate.o
 stub-obj-$(CONFIG_WIN32) += fd-register.o
 stub-obj-y += cpus.o
 stub-obj-y += kvm.o
 stub-obj-y += qmp_pc_dimm_device_list.o
+stub-obj-y += target-monitor-defs.o
+stub-obj-y += target-get-monitor-def.o
+stub-obj-y += vhost.o
diff --git a/stubs/chr-baum-init.c b/stubs/chr-baum-init.c
deleted file mode 100644
index f5cc6ce1f..000000000
--- a/stubs/chr-baum-init.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *chr_baum_init(void)
-{
-    return NULL;
-}
diff --git a/stubs/chr-msmouse.c b/stubs/chr-msmouse.c
deleted file mode 100644
index 812f8b0ab..000000000
--- a/stubs/chr-msmouse.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *qemu_chr_open_msmouse(void)
-{
-    return 0;
-}
diff --git a/stubs/chr-testdev.c b/stubs/chr-testdev.c
deleted file mode 100644
index 23112a2c0..000000000
--- a/stubs/chr-testdev.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *chr_testdev_init(void)
-{
-    return 0;
-}
diff --git a/stubs/qemu-chr-open-spice.c b/stubs/qemu-chr-open-spice.c
deleted file mode 100644
index f1c4849d9..000000000
--- a/stubs/qemu-chr-open-spice.c
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "qemu-common.h"
-#include "ui/qemu-spice.h"
-
-CharDriverState *qemu_chr_open_spice_vmc(const char *type)
-{
-    return NULL;
-}
-
-#if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name)
-{
-    return NULL;
-}
-#endif
diff --git a/stubs/qtest.c b/stubs/qtest.c
index dc17594bb..4dfde6104 100644
--- a/stubs/qtest.c
+++ b/stubs/qtest.c
@@ -12,3 +12,8 @@
 
 /* Needed for qtest_allowed() */
 bool qtest_allowed;
+
+bool qtest_driver(void)
+{
+    return false;
+}
diff --git a/stubs/replay-user.c b/stubs/replay-user.c
new file mode 100644
index 000000000..cf330721c
--- /dev/null
+++ b/stubs/replay-user.c
@@ -0,0 +1,32 @@
+/*
+ * replay.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "sysemu/replay.h"
+
+bool replay_exception(void)
+{
+    return true;
+}
+
+bool replay_has_exception(void)
+{
+    return false;
+}
+
+bool replay_interrupt(void)
+{
+    return true;
+}
+
+bool replay_has_interrupt(void)
+{
+    return false;
+}
diff --git a/stubs/replay.c b/stubs/replay.c
new file mode 100644
index 000000000..8f9879025
--- /dev/null
+++ b/stubs/replay.c
@@ -0,0 +1,31 @@
+#include "sysemu/replay.h"
+#include <stdlib.h>
+#include "sysemu/sysemu.h"
+
+ReplayMode replay_mode;
+
+int64_t replay_save_clock(unsigned int kind, int64_t clock)
+{
+    abort();
+    return 0;
+}
+
+int64_t replay_read_clock(unsigned int kind)
+{
+    abort();
+    return 0;
+}
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    return true;
+}
+
+bool replay_events_enabled(void)
+{
+    return false;
+}
+
+void replay_finish(void)
+{
+}
diff --git a/stubs/target-get-monitor-def.c b/stubs/target-get-monitor-def.c
new file mode 100644
index 000000000..711a9ae46
--- /dev/null
+++ b/stubs/target-get-monitor-def.c
@@ -0,0 +1,31 @@
+/*
+ *  Stub for target_get_monitor_def.
+ *
+ *  Copyright IBM Corp., 2015
+ *
+ *  Author: Alexey Kardashevskiy <aik@ozlabs.ru>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License,
+ *  or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "stdint.h"
+
+typedef struct CPUState CPUState;
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval);
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval)
+{
+    return -1;
+}
diff --git a/stubs/target-monitor-defs.c b/stubs/target-monitor-defs.c
new file mode 100644
index 000000000..7d8d182b2
--- /dev/null
+++ b/stubs/target-monitor-defs.c
@@ -0,0 +1,9 @@
+#include "stddef.h"
+#include "qemu/typedefs.h"
+
+const MonitorDef *target_monitor_defs(void);
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return NULL;
+}
diff --git a/stubs/vc-init.c b/stubs/vc-init.c
deleted file mode 100644
index 308dfa080..000000000
--- a/stubs/vc-init.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *vc_init(ChardevVC *vc)
-{
-    return 0;
-}
diff --git a/stubs/vhost.c b/stubs/vhost.c
new file mode 100644
index 000000000..d346b856f
--- /dev/null
+++ b/stubs/vhost.c
@@ -0,0 +1,6 @@
+#include "hw/virtio/vhost.h"
+
+bool vhost_has_free_slot(void)
+{
+    return true;
+}
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index ff1926a5d..e5bdfa8ca 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -46,6 +46,12 @@ static bool alpha_cpu_has_work(CPUState *cs)
                                     | CPU_INTERRUPT_MCHK);
 }
 
+static void alpha_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_alpha_ev6;
+    info->print_insn = print_insn_alpha;
+}
+
 static void alpha_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -297,6 +303,8 @@ static void alpha_cpu_class_init(ObjectClass *oc, void *data)
     cc->get_phys_page_debug = alpha_cpu_get_phys_page_debug;
     dc->vmsd = &vmstate_alpha_cpu;
 #endif
+    cc->disas_set_info = alpha_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = 67;
 
     /*
diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h
index 91c56d6bc..bcd8076ab 100644
--- a/target-alpha/cpu.h
+++ b/target-alpha/cpu.h
@@ -32,8 +32,6 @@
 
 #include "fpu/softfloat.h"
 
-#define ELF_MACHINE     EM_ALPHA
-
 #define ICACHE_LINE_SIZE 32
 #define DCACHE_LINE_SIZE 32
 
@@ -289,7 +287,6 @@ struct CPUAlphaState {
 
 #define cpu_list alpha_cpu_list
 #define cpu_exec cpu_alpha_exec
-#define cpu_gen_code cpu_alpha_gen_code
 #define cpu_signal_handler cpu_alpha_signal_handler
 
 #include "exec/cpu-all.h"
@@ -376,7 +373,7 @@ enum {
     PS_USER_MODE = 8
 };
 
-static inline int cpu_mmu_index(CPUAlphaState *env)
+static inline int cpu_mmu_index(CPUAlphaState *env, bool ifetch)
 {
     if (env->pal_mode) {
         return MMU_KERNEL_IDX;
@@ -445,8 +442,9 @@ void QEMU_NORETURN arith_excp(CPUAlphaState *, uintptr_t, int, uint64_t);
 
 uint64_t cpu_alpha_load_fpcr (CPUAlphaState *env);
 void cpu_alpha_store_fpcr (CPUAlphaState *env, uint64_t val);
+uint64_t cpu_alpha_load_gr(CPUAlphaState *env, unsigned reg);
+void cpu_alpha_store_gr(CPUAlphaState *env, unsigned reg, uint64_t val);
 #ifndef CONFIG_USER_ONLY
-void swap_shadow_regs(CPUAlphaState *env);
 QEMU_NORETURN void alpha_cpu_unassigned_access(CPUState *cpu, hwaddr addr,
                                                bool is_write, bool is_exec,
                                                int unused, unsigned size);
diff --git a/target-alpha/gdbstub.c b/target-alpha/gdbstub.c
index 980f140e7..99a4051b3 100644
--- a/target-alpha/gdbstub.c
+++ b/target-alpha/gdbstub.c
@@ -30,7 +30,7 @@ int alpha_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
 
     switch (n) {
     case 0 ... 30:
-        val = env->ir[n];
+        val = cpu_alpha_load_gr(env, n);
         break;
     case 32 ... 62:
         d.d = env->fir[n - 32];
@@ -66,7 +66,7 @@ int alpha_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
 
     switch (n) {
     case 0 ... 30:
-        env->ir[n] = tmp;
+        cpu_alpha_store_gr(env, n, tmp);
         break;
     case 32 ... 62:
         d.ll = tmp;
diff --git a/target-alpha/helper.c b/target-alpha/helper.c
index 46b8ef914..5a8533583 100644
--- a/target-alpha/helper.c
+++ b/target-alpha/helper.c
@@ -79,6 +79,30 @@ void helper_store_fpcr(CPUAlphaState *env, uint64_t val)
     cpu_alpha_store_fpcr(env, val);
 }
 
+static uint64_t *cpu_alpha_addr_gr(CPUAlphaState *env, unsigned reg)
+{
+#ifndef CONFIG_USER_ONLY
+    if (env->pal_mode) {
+        if (reg >= 8 && reg <= 14) {
+            return &env->shadow[reg - 8];
+        } else if (reg == 25) {
+            return &env->shadow[7];
+        }
+    }
+#endif
+    return &env->ir[reg];
+}
+
+uint64_t cpu_alpha_load_gr(CPUAlphaState *env, unsigned reg)
+{
+    return *cpu_alpha_addr_gr(env, reg);
+}
+
+void cpu_alpha_store_gr(CPUAlphaState *env, unsigned reg, uint64_t val)
+{
+    *cpu_alpha_addr_gr(env, reg) = val;
+}
+
 #if defined(CONFIG_USER_ONLY)
 int alpha_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
                                int rw, int mmu_idx)
@@ -90,38 +114,6 @@ int alpha_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
     return 1;
 }
 #else
-void swap_shadow_regs(CPUAlphaState *env)
-{
-    uint64_t i0, i1, i2, i3, i4, i5, i6, i7;
-
-    i0 = env->ir[8];
-    i1 = env->ir[9];
-    i2 = env->ir[10];
-    i3 = env->ir[11];
-    i4 = env->ir[12];
-    i5 = env->ir[13];
-    i6 = env->ir[14];
-    i7 = env->ir[25];
-
-    env->ir[8]  = env->shadow[0];
-    env->ir[9]  = env->shadow[1];
-    env->ir[10] = env->shadow[2];
-    env->ir[11] = env->shadow[3];
-    env->ir[12] = env->shadow[4];
-    env->ir[13] = env->shadow[5];
-    env->ir[14] = env->shadow[6];
-    env->ir[25] = env->shadow[7];
-
-    env->shadow[0] = i0;
-    env->shadow[1] = i1;
-    env->shadow[2] = i2;
-    env->shadow[3] = i3;
-    env->shadow[4] = i4;
-    env->shadow[5] = i5;
-    env->shadow[6] = i6;
-    env->shadow[7] = i7;
-}
-
 /* Returns the OSF/1 entMM failure indication, or -1 on success.  */
 static int get_physical_address(CPUAlphaState *env, target_ulong addr,
                                 int prot_need, int mmu_idx,
@@ -375,10 +367,7 @@ void alpha_cpu_do_interrupt(CPUState *cs)
     env->pc = env->palbr + i;
 
     /* Switch to PALmode.  */
-    if (!env->pal_mode) {
-        env->pal_mode = 1;
-        swap_shadow_regs(env);
-    }
+    env->pal_mode = 1;
 #endif /* !USER_ONLY */
 }
 
@@ -443,7 +432,7 @@ void alpha_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                 env->pc, env->ps);
     for (i = 0; i < 31; i++) {
         cpu_fprintf(f, "IR%02d %s " TARGET_FMT_lx " ", i,
-                    linux_reg_names[i], env->ir[i]);
+                    linux_reg_names[i], cpu_alpha_load_gr(env, i));
         if ((i % 3) == 2)
             cpu_fprintf(f, "\n");
     }
diff --git a/target-alpha/helper.h b/target-alpha/helper.h
index d221f0d7d..c3d8a3ee4 100644
--- a/target-alpha/helper.h
+++ b/target-alpha/helper.h
@@ -10,6 +10,7 @@ DEF_HELPER_FLAGS_1(cttz, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_FLAGS_2(zap, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 DEF_HELPER_FLAGS_2(zapnot, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
+DEF_HELPER_FLAGS_1(cmpbe0, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_FLAGS_2(cmpbge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
 DEF_HELPER_FLAGS_2(minub8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
@@ -91,9 +92,6 @@ DEF_HELPER_FLAGS_2(ieee_input_cmp, TCG_CALL_NO_WG, void, env, i64)
 DEF_HELPER_FLAGS_2(ieee_input_s, TCG_CALL_NO_WG, void, env, i64)
 
 #if !defined (CONFIG_USER_ONLY)
-DEF_HELPER_2(hw_ret, void, env, i64)
-DEF_HELPER_3(call_pal, void, env, i64, i64)
-
 DEF_HELPER_2(ldl_phys, i64, env, i64)
 DEF_HELPER_2(ldq_phys, i64, env, i64)
 DEF_HELPER_2(ldl_l_phys, i64, env, i64)
diff --git a/target-alpha/int_helper.c b/target-alpha/int_helper.c
index 74f38cbe7..d7f477412 100644
--- a/target-alpha/int_helper.c
+++ b/target-alpha/int_helper.c
@@ -58,20 +58,47 @@ uint64_t helper_zap(uint64_t val, uint64_t mask)
     return helper_zapnot(val, ~mask);
 }
 
-uint64_t helper_cmpbge(uint64_t op1, uint64_t op2)
+uint64_t helper_cmpbe0(uint64_t a)
 {
-    uint8_t opa, opb, res;
-    int i;
+    uint64_t m = 0x7f7f7f7f7f7f7f7fULL;
+    uint64_t c = ~(((a & m) + m) | a | m);
+    /* a.......b.......c.......d.......e.......f.......g.......h....... */
+    c |= c << 7;
+    /* ab......bc......cd......de......ef......fg......gh......h....... */
+    c |= c << 14;
+    /* abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... */
+    c |= c << 28;
+    /* abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... */
+    return c >> 56;
+}
 
-    res = 0;
-    for (i = 0; i < 8; i++) {
-        opa = op1 >> (i * 8);
-        opb = op2 >> (i * 8);
-        if (opa >= opb) {
-            res |= 1 << i;
-        }
-    }
-    return res;
+uint64_t helper_cmpbge(uint64_t a, uint64_t b)
+{
+    uint64_t mask = 0x00ff00ff00ff00ffULL;
+    uint64_t test = 0x0100010001000100ULL;
+    uint64_t al, ah, bl, bh, cl, ch;
+
+    /* Separate the bytes to avoid false positives.  */
+    al = a & mask;
+    bl = b & mask;
+    ah = (a >> 8) & mask;
+    bh = (b >> 8) & mask;
+
+    /* "Compare".  If a byte in B is greater than a byte in A,
+       it will clear the test bit.  */
+    cl = ((al | test) - bl) & test;
+    ch = ((ah | test) - bh) & test;
+
+    /* Fold all of the test bits into a contiguous set.  */
+    /* ch=.......a...............c...............e...............g........ */
+    /* cl=.......b...............d...............f...............h........ */
+    cl += ch << 1;
+    /* cl=......ab..............cd..............ef..............gh........ */
+    cl |= cl << 14;
+    /* cl=......abcd............cdef............efgh............gh........ */
+    cl |= cl << 28;
+    /* cl=......abcdefgh........cdefgh..........efgh............gh........ */
+    return cl >> 50;
 }
 
 uint64_t helper_minub8(uint64_t op1, uint64_t op2)
diff --git a/target-alpha/machine.c b/target-alpha/machine.c
index e796bbe27..d9bf977fc 100644
--- a/target-alpha/machine.c
+++ b/target-alpha/machine.c
@@ -70,8 +70,8 @@ static VMStateField vmstate_env_fields[] = {
 
 static const VMStateDescription vmstate_env = {
     .name = "env",
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
     .fields = vmstate_env_fields,
 };
 
diff --git a/target-alpha/mem_helper.c b/target-alpha/mem_helper.c
index 7b5e30ddb..c9258f5aa 100644
--- a/target-alpha/mem_helper.c
+++ b/target-alpha/mem_helper.c
@@ -132,7 +132,7 @@ void alpha_cpu_unassigned_access(CPUState *cs, hwaddr addr,
     env->error_code = 0;
 
     /* ??? We should cpu_restore_state to the faulting insn, but this hook
-       does not have access to the retaddr value from the orignal helper.
+       does not have access to the retaddr value from the original helper.
        It's all moot until the QEMU PALcode grows an MCHK handler.  */
 
     cpu_loop_exit(cs);
diff --git a/target-alpha/sys_helper.c b/target-alpha/sys_helper.c
index 1c59e108b..75c96c1c2 100644
--- a/target-alpha/sys_helper.c
+++ b/target-alpha/sys_helper.c
@@ -34,34 +34,12 @@ uint64_t helper_load_pcc(CPUAlphaState *env)
 #else
     /* In user-mode, QEMU_CLOCK_VIRTUAL doesn't exist.  Just pass through the host cpu
        clock ticks.  Also, don't bother taking PCC_OFS into account.  */
-    return (uint32_t)cpu_get_real_ticks();
+    return (uint32_t)cpu_get_host_ticks();
 #endif
 }
 
 /* PALcode support special instructions */
 #ifndef CONFIG_USER_ONLY
-void helper_hw_ret(CPUAlphaState *env, uint64_t a)
-{
-    env->pc = a & ~3;
-    env->intr_flag = 0;
-    env->lock_addr = -1;
-    if ((a & 1) == 0) {
-        env->pal_mode = 0;
-        swap_shadow_regs(env);
-    }
-}
-
-void helper_call_pal(CPUAlphaState *env, uint64_t pc, uint64_t entry_ofs)
-{
-    int pal_mode = env->pal_mode;
-    env->exc_addr = pc | pal_mode;
-    env->pc = env->palbr + entry_ofs;
-    if (!pal_mode) {
-        env->pal_mode = 1;
-        swap_shadow_regs(env);
-    }
-}
-
 void helper_tbia(CPUAlphaState *env)
 {
     tlb_flush(CPU(alpha_env_get_cpu(env)), 1);
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 81d4ff827..9909c70b1 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -42,6 +42,9 @@ typedef struct DisasContext DisasContext;
 struct DisasContext {
     struct TranslationBlock *tb;
     uint64_t pc;
+#ifndef CONFIG_USER_ONLY
+    uint64_t palbr;
+#endif
     int mem_idx;
 
     /* Current rounding mode for this TB.  */
@@ -52,6 +55,9 @@ struct DisasContext {
     /* implver value for this CPU.  */
     int implver;
 
+    /* The set of registers active in the current context.  */
+    TCGv *ir;
+
     /* Temporaries for $31 and $f31 as source and destination.  */
     TCGv zero;
     TCGv sink;
@@ -86,13 +92,17 @@ typedef enum {
 
 /* global register indexes */
 static TCGv_ptr cpu_env;
-static TCGv cpu_ir[31];
+static TCGv cpu_std_ir[31];
 static TCGv cpu_fir[31];
 static TCGv cpu_pc;
 static TCGv cpu_lock_addr;
 static TCGv cpu_lock_st_addr;
 static TCGv cpu_lock_value;
 
+#ifndef CONFIG_USER_ONLY
+static TCGv cpu_pal_ir[31];
+#endif
+
 #include "exec/gen-icount.h"
 
 void alpha_translate_init(void)
@@ -122,6 +132,12 @@ void alpha_translate_init(void)
         "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
         "f24", "f25", "f26", "f27", "f28", "f29", "f30"
     };
+#ifndef CONFIG_USER_ONLY
+    static const char shadow_names[8][8] = {
+        "pal_t7", "pal_s0", "pal_s1", "pal_s2",
+        "pal_s3", "pal_s4", "pal_s5", "pal_t11"
+    };
+#endif
 
     static bool done_init = 0;
     int i;
@@ -134,9 +150,9 @@ void alpha_translate_init(void)
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
 
     for (i = 0; i < 31; i++) {
-        cpu_ir[i] = tcg_global_mem_new_i64(TCG_AREG0,
-                                           offsetof(CPUAlphaState, ir[i]),
-                                           greg_names[i]);
+        cpu_std_ir[i] = tcg_global_mem_new_i64(TCG_AREG0,
+                                               offsetof(CPUAlphaState, ir[i]),
+                                               greg_names[i]);
     }
 
     for (i = 0; i < 31; i++) {
@@ -145,6 +161,17 @@ void alpha_translate_init(void)
                                             freg_names[i]);
     }
 
+#ifndef CONFIG_USER_ONLY
+    memcpy(cpu_pal_ir, cpu_std_ir, sizeof(cpu_pal_ir));
+    for (i = 0; i < 8; i++) {
+        int r = (i == 7 ? 25 : i + 8);
+        cpu_pal_ir[r] = tcg_global_mem_new_i64(TCG_AREG0,
+                                               offsetof(CPUAlphaState,
+                                                        shadow[i]),
+                                               shadow_names[i]);
+    }
+#endif
+
     for (i = 0; i < ARRAY_SIZE(vars); ++i) {
         const GlobalVar *v = &vars[i];
         *v->var = tcg_global_mem_new_i64(TCG_AREG0, v->ofs, v->name);
@@ -170,7 +197,7 @@ static TCGv dest_sink(DisasContext *ctx)
 static TCGv load_gpr(DisasContext *ctx, unsigned reg)
 {
     if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return load_zero(ctx);
     }
@@ -183,7 +210,7 @@ static TCGv load_gpr_lit(DisasContext *ctx, unsigned reg,
         ctx->lit = tcg_const_i64(lit);
         return ctx->lit;
     } else if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return load_zero(ctx);
     }
@@ -192,7 +219,7 @@ static TCGv load_gpr_lit(DisasContext *ctx, unsigned reg,
 static TCGv dest_gpr(DisasContext *ctx, unsigned reg)
 {
     if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return dest_sink(ctx);
     }
@@ -304,7 +331,7 @@ static inline void gen_load_mem(DisasContext *ctx,
         addr = tmp;
     }
 
-    va = (fp ? cpu_fir[ra] : cpu_ir[ra]);
+    va = (fp ? cpu_fir[ra] : ctx->ir[ra]);
     tcg_gen_qemu_load(va, addr, ctx->mem_idx);
 
     tcg_temp_free(tmp);
@@ -399,13 +426,13 @@ static ExitStatus gen_store_conditional(DisasContext *ctx, int ra, int rb,
         tcg_gen_qemu_ld_i64(val, addr, ctx->mem_idx, quad ? MO_LEQ : MO_LESL);
         tcg_gen_brcond_i64(TCG_COND_NE, val, cpu_lock_value, lab_fail);
 
-        tcg_gen_qemu_st_i64(cpu_ir[ra], addr, ctx->mem_idx,
+        tcg_gen_qemu_st_i64(ctx->ir[ra], addr, ctx->mem_idx,
                             quad ? MO_LEQ : MO_LEUL);
-        tcg_gen_movi_i64(cpu_ir[ra], 1);
+        tcg_gen_movi_i64(ctx->ir[ra], 1);
         tcg_gen_br(lab_done);
 
         gen_set_label(lab_fail);
-        tcg_gen_movi_i64(cpu_ir[ra], 0);
+        tcg_gen_movi_i64(ctx->ir[ra], 0);
 
         gen_set_label(lab_done);
         tcg_gen_movi_i64(cpu_lock_addr, -1);
@@ -444,7 +471,7 @@ static ExitStatus gen_bdirect(DisasContext *ctx, int ra, int32_t disp)
     uint64_t dest = ctx->pc + (disp << 2);
 
     if (ra != 31) {
-        tcg_gen_movi_i64(cpu_ir[ra], ctx->pc);
+        tcg_gen_movi_i64(ctx->ir[ra], ctx->pc);
     }
 
     /* Notice branch-to-next; used to initialize RA with the PC.  */
@@ -1059,12 +1086,13 @@ static void gen_msk_l(DisasContext *ctx, TCGv vc, TCGv va, int rb, bool islit,
     }
 }
 
-static void gen_rx(int ra, int set)
+static void gen_rx(DisasContext *ctx, int ra, int set)
 {
     TCGv_i32 tmp;
 
     if (ra != 31) {
-        tcg_gen_ld8u_i64(cpu_ir[ra], cpu_env, offsetof(CPUAlphaState, intr_flag));
+        tcg_gen_ld8u_i64(ctx->ir[ra], cpu_env,
+                         offsetof(CPUAlphaState, intr_flag));
     }
 
     tmp = tcg_const_i32(set);
@@ -1086,12 +1114,12 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
             break;
         case 0x9E:
             /* RDUNIQUE */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, unique));
             break;
         case 0x9F:
             /* WRUNIQUE */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, unique));
             break;
         default:
@@ -1115,17 +1143,17 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
             break;
         case 0x2D:
             /* WRVPTPTR */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, vptptr));
             break;
         case 0x31:
             /* WRVAL */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, sysval));
             break;
         case 0x32:
             /* RDVAL */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, sysval));
             break;
 
@@ -1135,12 +1163,12 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
 
             /* Note that we already know we're in kernel mode, so we know
                that PS only contains the 3 IPL bits.  */
-            tcg_gen_ld8u_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld8u_i64(ctx->ir[IR_V0], cpu_env,
                              offsetof(CPUAlphaState, ps));
 
             /* But make sure and store only the 3 IPL bits from the user.  */
             tmp = tcg_temp_new();
-            tcg_gen_andi_i64(tmp, cpu_ir[IR_A0], PS_INT_MASK);
+            tcg_gen_andi_i64(tmp, ctx->ir[IR_A0], PS_INT_MASK);
             tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, ps));
             tcg_temp_free(tmp);
             break;
@@ -1148,22 +1176,22 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
 
         case 0x36:
             /* RDPS */
-            tcg_gen_ld8u_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld8u_i64(ctx->ir[IR_V0], cpu_env,
                              offsetof(CPUAlphaState, ps));
             break;
         case 0x38:
             /* WRUSP */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, usp));
             break;
         case 0x3A:
             /* RDUSP */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, usp));
             break;
         case 0x3C:
             /* WHAMI */
-            tcg_gen_ld32s_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld32s_i64(ctx->ir[IR_V0], cpu_env,
                 -offsetof(AlphaCPU, env) + offsetof(CPUState, cpu_index));
             break;
 
@@ -1181,15 +1209,24 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
     return gen_excp(ctx, EXCP_CALL_PAL, palcode);
 #else
     {
-        TCGv pc = tcg_const_i64(ctx->pc);
-        TCGv entry = tcg_const_i64(palcode & 0x80
-                                   ? 0x2000 + (palcode - 0x80) * 64
-                                   : 0x1000 + palcode * 64);
+        TCGv tmp = tcg_temp_new();
+        uint64_t exc_addr = ctx->pc;
+        uint64_t entry = ctx->palbr;
 
-        gen_helper_call_pal(cpu_env, pc, entry);
+        if (ctx->tb->flags & TB_FLAGS_PAL_MODE) {
+            exc_addr |= 1;
+        } else {
+            tcg_gen_movi_i64(tmp, 1);
+            tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, pal_mode));
+        }
+
+        tcg_gen_movi_i64(tmp, exc_addr);
+        tcg_gen_st_i64(tmp, cpu_env, offsetof(CPUAlphaState, exc_addr));
+        tcg_temp_free(tmp);
 
-        tcg_temp_free(entry);
-        tcg_temp_free(pc);
+        entry += (palcode & 0x80
+                  ? 0x2000 + (palcode - 0x80) * 64
+                  : 0x1000 + palcode * 64);
 
         /* Since the destination is running in PALmode, we don't really
            need the page permissions check.  We'll see the existence of
@@ -1197,11 +1234,13 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
            we change the PAL base register.  */
         if (!ctx->singlestep_enabled && !(ctx->tb->cflags & CF_LAST_IO)) {
             tcg_gen_goto_tb(0);
+            tcg_gen_movi_i64(cpu_pc, entry);
             tcg_gen_exit_tb((uintptr_t)ctx->tb);
             return EXIT_GOTO_TB;
+        } else {
+            tcg_gen_movi_i64(cpu_pc, entry);
+            return EXIT_PC_UPDATED;
         }
-
-        return EXIT_PC_UPDATED;
     }
 #endif
 }
@@ -1228,8 +1267,6 @@ static int cpu_pr_data(int pr)
     case 11: return offsetof(CPUAlphaState, sysval);
     case 12: return offsetof(CPUAlphaState, usp);
 
-    case 32 ... 39:
-        return offsetof(CPUAlphaState, shadow[pr - 32]);
     case 40 ... 63:
         return offsetof(CPUAlphaState, scratch[pr - 40]);
 
@@ -1241,36 +1278,48 @@ static int cpu_pr_data(int pr)
 
 static ExitStatus gen_mfpr(DisasContext *ctx, TCGv va, int regno)
 {
-    int data = cpu_pr_data(regno);
-
-    /* Special help for VMTIME and WALLTIME.  */
-    if (regno == 250 || regno == 249) {
-	void (*helper)(TCGv) = gen_helper_get_walltime;
-	if (regno == 249) {
-		helper = gen_helper_get_vmtime;
-	}
-        if (ctx->tb->cflags & CF_USE_ICOUNT) {
+    void (*helper)(TCGv);
+    int data;
+
+    switch (regno) {
+    case 32 ... 39:
+        /* Accessing the "non-shadow" general registers.  */
+        regno = regno == 39 ? 25 : regno - 32 + 8;
+        tcg_gen_mov_i64(va, cpu_std_ir[regno]);
+        break;
+
+    case 250: /* WALLTIME */
+        helper = gen_helper_get_walltime;
+        goto do_helper;
+    case 249: /* VMTIME */
+        helper = gen_helper_get_vmtime;
+    do_helper:
+        if (use_icount) {
             gen_io_start();
             helper(va);
             gen_io_end();
             return EXIT_PC_STALE;
         } else {
             helper(va);
-            return NO_EXIT;
         }
-    }
+        break;
 
-    /* The basic registers are data only, and unknown registers
-       are read-zero, write-ignore.  */
-    if (data == 0) {
-        tcg_gen_movi_i64(va, 0);
-    } else if (data & PR_BYTE) {
-        tcg_gen_ld8u_i64(va, cpu_env, data & ~PR_BYTE);
-    } else if (data & PR_LONG) {
-        tcg_gen_ld32s_i64(va, cpu_env, data & ~PR_LONG);
-    } else {
-        tcg_gen_ld_i64(va, cpu_env, data);
+    default:
+        /* The basic registers are data only, and unknown registers
+           are read-zero, write-ignore.  */
+        data = cpu_pr_data(regno);
+        if (data == 0) {
+            tcg_gen_movi_i64(va, 0);
+        } else if (data & PR_BYTE) {
+            tcg_gen_ld8u_i64(va, cpu_env, data & ~PR_BYTE);
+        } else if (data & PR_LONG) {
+            tcg_gen_ld32s_i64(va, cpu_env, data & ~PR_LONG);
+        } else {
+            tcg_gen_ld_i64(va, cpu_env, data);
+        }
+        break;
     }
+
     return NO_EXIT;
 }
 
@@ -1316,6 +1365,12 @@ static ExitStatus gen_mtpr(DisasContext *ctx, TCGv vb, int regno)
         gen_helper_tb_flush(cpu_env);
         return EXIT_PC_STALE;
 
+    case 32 ... 39:
+        /* Accessing the "non-shadow" general registers.  */
+        regno = regno == 39 ? 25 : regno - 32 + 8;
+        tcg_gen_mov_i64(cpu_std_ir[regno], vb);
+        break;
+
     default:
         /* The basic registers are data only, and unknown registers
            are read-zero, write-ignore.  */
@@ -1507,7 +1562,12 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             break;
         case 0x0F:
             /* CMPBGE */
-            gen_helper_cmpbge(vc, va, vb);
+            if (ra == 31) {
+                /* Special case 0 >= X as X == 0.  */
+                gen_helper_cmpbe0(vc, vb);
+            } else {
+                gen_helper_cmpbge(vc, va, vb);
+            }
             break;
         case 0x12:
             /* S8ADDL */
@@ -1952,7 +2012,7 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             REQUIRE_REG_31(rb);
             t32 = tcg_temp_new_i32();
             va = load_gpr(ctx, ra);
-            tcg_gen_trunc_i64_i32(t32, va);
+            tcg_gen_extrl_i64_i32(t32, va);
             gen_helper_memory_to_s(vc, t32);
             tcg_temp_free_i32(t32);
             break;
@@ -1972,7 +2032,7 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             REQUIRE_REG_31(rb);
             t32 = tcg_temp_new_i32();
             va = load_gpr(ctx, ra);
-            tcg_gen_trunc_i64_i32(t32, va);
+            tcg_gen_extrl_i64_i32(t32, va);
             gen_helper_memory_to_f(vc, t32);
             tcg_temp_free_i32(t32);
             break;
@@ -2295,14 +2355,14 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             break;
         case 0xE000:
             /* RC */
-            gen_rx(ra, 0);
+            gen_rx(ctx, ra, 0);
             break;
         case 0xE800:
             /* ECB */
             break;
         case 0xF000:
             /* RS */
-            gen_rx(ra, 1);
+            gen_rx(ctx, ra, 1);
             break;
         case 0xF800:
             /* WH64 */
@@ -2334,7 +2394,7 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
         vb = load_gpr(ctx, rb);
         tcg_gen_andi_i64(cpu_pc, vb, ~3);
         if (ra != 31) {
-            tcg_gen_movi_i64(cpu_ir[ra], ctx->pc);
+            tcg_gen_movi_i64(ctx->ir[ra], ctx->pc);
         }
         ret = EXIT_PC_UPDATED;
         break;
@@ -2374,10 +2434,10 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
                 goto invalid_opc;
                 break;
             case 0x6:
-                /* Incpu_ir[ra]id */
+                /* Invalid */
                 goto invalid_opc;
             case 0x7:
-                /* Incpu_ir[ra]id */
+                /* Invaliid */
                 goto invalid_opc;
             case 0x8:
                 /* Longword virtual access (hw_ldl) */
@@ -2580,13 +2640,18 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             /* Pre-EV6 CPUs interpreted this as HW_REI, loading the return
                address from EXC_ADDR.  This turns out to be useful for our
                emulation PALcode, so continue to accept it.  */
-            tmp = tcg_temp_new();
-            tcg_gen_ld_i64(tmp, cpu_env, offsetof(CPUAlphaState, exc_addr));
-            gen_helper_hw_ret(cpu_env, tmp);
-            tcg_temp_free(tmp);
+            ctx->lit = vb = tcg_temp_new();
+            tcg_gen_ld_i64(vb, cpu_env, offsetof(CPUAlphaState, exc_addr));
         } else {
-            gen_helper_hw_ret(cpu_env, load_gpr(ctx, rb));
+            vb = load_gpr(ctx, rb);
         }
+        tmp = tcg_temp_new();
+        tcg_gen_movi_i64(tmp, 0);
+        tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, intr_flag));
+        tcg_gen_movi_i64(cpu_lock_addr, -1);
+        tcg_gen_andi_i64(tmp, vb, 1);
+        tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, pal_mode));
+        tcg_gen_andi_i64(cpu_pc, vb, ~3);
         ret = EXIT_PC_UPDATED;
         break;
 #else
@@ -2793,18 +2858,14 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
     return ret;
 }
 
-static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUAlphaState *env, struct TranslationBlock *tb)
 {
+    AlphaCPU *cpu = alpha_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUAlphaState *env = &cpu->env;
     DisasContext ctx, *ctxp = &ctx;
     target_ulong pc_start;
     target_ulong pc_mask;
     uint32_t insn;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     ExitStatus ret;
     int num_insns;
     int max_insns;
@@ -2813,10 +2874,17 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 
     ctx.tb = tb;
     ctx.pc = pc_start;
-    ctx.mem_idx = cpu_mmu_index(env);
+    ctx.mem_idx = cpu_mmu_index(env, false);
     ctx.implver = env->implver;
     ctx.singlestep_enabled = cs->singlestep_enabled;
 
+#ifdef CONFIG_USER_ONLY
+    ctx.ir = cpu_std_ir;
+#else
+    ctx.palbr = env->palbr;
+    ctx.ir = (tb->flags & TB_FLAGS_PAL_MODE ? cpu_pal_ir : cpu_std_ir);
+#endif
+
     /* ??? Every TB begins with unset rounding mode, to be initialized on
        the first fp insn of the TB.  Alternately we could define a proper
        default for every TB (e.g. QUAL_RM_N or QUAL_RM_D) and make sure
@@ -2832,6 +2900,9 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     if (in_superpage(&ctx, pc_start)) {
         pc_mask = (1ULL << 41) - 1;
@@ -2841,35 +2912,22 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.pc) {
-                    gen_excp(&ctx, EXCP_DEBUG, 0);
-                    break;
-                }
-            }
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            ret = gen_excp(&ctx, EXCP_DEBUG, 0);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 4;
+            break;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         insn = cpu_ldl_code(env, ctx.pc);
-        num_insns++;
-
-	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(ctx.pc);
-        }
 
         TCGV_UNUSED_I64(ctx.zero);
         TCGV_UNUSED_I64(ctx.sink);
@@ -2925,16 +2983,8 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -2945,17 +2995,8 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code (CPUAlphaState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(alpha_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUAlphaState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(alpha_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-arm/arm-semi.c b/target-arm/arm-semi.c
index 42522a70d..d7cff3db2 100644
--- a/target-arm/arm-semi.c
+++ b/target-arm/arm-semi.c
@@ -58,6 +58,7 @@
 #define TARGET_SYS_GET_CMDLINE 0x15
 #define TARGET_SYS_HEAPINFO    0x16
 #define TARGET_SYS_EXIT        0x18
+#define TARGET_SYS_SYNCCACHE   0x19
 
 /* ADP_Stopped_ApplicationExit is used for exit(0),
  * anything else is implemented as exit(1) */
@@ -134,6 +135,7 @@ static void arm_semi_cb(CPUState *cs, target_ulong ret, target_ulong err)
 #ifdef CONFIG_USER_ONLY
     TaskState *ts = cs->opaque;
 #endif
+    target_ulong reg0 = is_a64(env) ? env->xregs[0] : env->regs[0];
 
     if (ret == (target_ulong)-1) {
 #ifdef CONFIG_USER_ONLY
@@ -141,22 +143,46 @@ static void arm_semi_cb(CPUState *cs, target_ulong ret, target_ulong err)
 #else
 	syscall_err = err;
 #endif
-        env->regs[0] = ret;
+        reg0 = ret;
     } else {
         /* Fixup syscalls that use nonstardard return conventions.  */
-        switch (env->regs[0]) {
+        switch (reg0) {
         case TARGET_SYS_WRITE:
         case TARGET_SYS_READ:
-            env->regs[0] = arm_semi_syscall_len - ret;
+            reg0 = arm_semi_syscall_len - ret;
             break;
         case TARGET_SYS_SEEK:
-            env->regs[0] = 0;
+            reg0 = 0;
             break;
         default:
-            env->regs[0] = ret;
+            reg0 = ret;
             break;
         }
     }
+    if (is_a64(env)) {
+        env->xregs[0] = reg0;
+    } else {
+        env->regs[0] = reg0;
+    }
+}
+
+static target_ulong arm_flen_buf(ARMCPU *cpu)
+{
+    /* Return an address in target memory of 64 bytes where the remote
+     * gdb should write its stat struct. (The format of this structure
+     * is defined by GDB's remote protocol and is not target-specific.)
+     * We put this on the guest's stack just below SP.
+     */
+    CPUARMState *env = &cpu->env;
+    target_ulong sp;
+
+    if (is_a64(env)) {
+        sp = env->xregs[31];
+    } else {
+        sp = env->regs[13];
+    }
+
+    return sp - 64;
 }
 
 static void arm_semi_flen_cb(CPUState *cs, target_ulong ret, target_ulong err)
@@ -166,8 +192,13 @@ static void arm_semi_flen_cb(CPUState *cs, target_ulong ret, target_ulong err)
     /* The size is always stored in big-endian order, extract
        the value. We assume the size always fit in 32 bits.  */
     uint32_t size;
-    cpu_memory_rw_debug(cs, env->regs[13]-64+32, (uint8_t *)&size, 4, 0);
-    env->regs[0] = be32_to_cpu(size);
+    cpu_memory_rw_debug(cs, arm_flen_buf(cpu) + 32, (uint8_t *)&size, 4, 0);
+    size = be32_to_cpu(size);
+    if (is_a64(env)) {
+        env->xregs[0] = size;
+    } else {
+        env->regs[0] = size;
+    }
 #ifdef CONFIG_USER_ONLY
     ((TaskState *)cs->opaque)->swi_errno = err;
 #else
@@ -175,17 +206,46 @@ static void arm_semi_flen_cb(CPUState *cs, target_ulong ret, target_ulong err)
 #endif
 }
 
+static target_ulong arm_gdb_syscall(ARMCPU *cpu, gdb_syscall_complete_cb cb,
+                                    const char *fmt, ...)
+{
+    va_list va;
+    CPUARMState *env = &cpu->env;
+
+    va_start(va, fmt);
+    gdb_do_syscallv(cb, fmt, va);
+    va_end(va);
+
+    /* FIXME: we are implicitly relying on the syscall completing
+     * before this point, which is not guaranteed. We should
+     * put in an explicit synchronization between this and
+     * the callback function.
+     */
+
+    return is_a64(env) ? env->xregs[0] : env->regs[0];
+}
+
 /* Read the input value from the argument block; fail the semihosting
  * call if the memory read fails.
  */
 #define GET_ARG(n) do {                                 \
-    if (get_user_ual(arg ## n, args + (n) * 4)) {       \
-        return (uint32_t)-1;                            \
+    if (is_a64(env)) {                                  \
+        if (get_user_u64(arg ## n, args + (n) * 8)) {   \
+            return -1;                                  \
+        }                                               \
+    } else {                                            \
+        if (get_user_u32(arg ## n, args + (n) * 4)) {   \
+            return -1;                                  \
+        }                                               \
     }                                                   \
 } while (0)
 
-#define SET_ARG(n, val) put_user_ual(val, args + (n) * 4)
-uint32_t do_arm_semihosting(CPUARMState *env)
+#define SET_ARG(n, val)                                 \
+    (is_a64(env) ?                                      \
+     put_user_u64(val, args + (n) * 8) :                \
+     put_user_u32(val, args + (n) * 4))
+
+target_ulong do_arm_semihosting(CPUARMState *env)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
@@ -201,8 +261,15 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     CPUARMState *ts = env;
 #endif
 
-    nr = env->regs[0];
-    args = env->regs[1];
+    if (is_a64(env)) {
+        /* Note that the syscall number is in W0, not X0 */
+        nr = env->xregs[0] & 0xffffffffU;
+        args = env->xregs[1];
+    } else {
+        nr = env->regs[0];
+        args = env->regs[1];
+    }
+
     switch (nr) {
     case TARGET_SYS_OPEN:
         GET_ARG(0);
@@ -223,9 +290,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             return result_fileno;
         }
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "open,%s,%x,1a4", arg0,
-                           (int)arg2+1, gdb_open_modeflags[arg1]);
-            ret = env->regs[0];
+            ret = arm_gdb_syscall(cpu, arm_semi_cb, "open,%s,%x,1a4", arg0,
+                                  (int)arg2+1, gdb_open_modeflags[arg1]);
         } else {
             ret = set_swi_errno(ts, open(s, open_modeflags[arg1], 0644));
         }
@@ -234,8 +300,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_CLOSE:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "close,%x", arg0);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "close,%x", arg0);
         } else {
             return set_swi_errno(ts, close(arg0));
         }
@@ -248,8 +313,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
               return (uint32_t)-1;
           /* Write to debug console.  stderr is near enough.  */
           if (use_gdb_syscalls()) {
-                gdb_do_syscall(arm_semi_cb, "write,2,%x,1", args);
-                return env->regs[0];
+                return arm_gdb_syscall(cpu, arm_semi_cb, "write,2,%x,1", args);
           } else {
                 return write(STDERR_FILENO, &c, 1);
           }
@@ -260,8 +324,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             return (uint32_t)-1;
         len = strlen(s);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "write,2,%x,%x", args, len);
-            ret = env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "write,2,%x,%x",
+                                   args, len);
         } else {
             ret = write(STDERR_FILENO, s, len);
         }
@@ -274,8 +338,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "write,%x,%x,%x", arg0, arg1, len);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "write,%x,%x,%x",
+                                   arg0, arg1, len);
         } else {
             s = lock_user(VERIFY_READ, arg1, len, 1);
             if (!s) {
@@ -295,8 +359,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "read,%x,%x,%x", arg0, arg1, len);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "read,%x,%x,%x",
+                                   arg0, arg1, len);
         } else {
             s = lock_user(VERIFY_WRITE, arg1, len, 0);
             if (!s) {
@@ -317,8 +381,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_ISTTY:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "isatty,%x", arg0);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "isatty,%x", arg0);
         } else {
             return isatty(arg0);
         }
@@ -326,8 +389,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "lseek,%x,%x,0", arg0, arg1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "lseek,%x,%x,0",
+                                   arg0, arg1);
         } else {
             ret = set_swi_errno(ts, lseek(arg0, arg1, SEEK_SET));
             if (ret == (uint32_t)-1)
@@ -337,9 +400,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_FLEN:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_flen_cb, "fstat,%x,%x",
-                           arg0, env->regs[13]-64);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_flen_cb, "fstat,%x,%x",
+                                   arg0, arm_flen_buf(cpu));
         } else {
             struct stat buf;
             ret = set_swi_errno(ts, fstat(arg0, &buf));
@@ -354,8 +416,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "unlink,%s", arg0, (int)arg1+1);
-            ret = env->regs[0];
+            ret = arm_gdb_syscall(cpu, arm_semi_cb, "unlink,%s",
+                                  arg0, (int)arg1+1);
         } else {
             s = lock_user_string(arg0);
             if (!s) {
@@ -372,9 +434,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(2);
         GET_ARG(3);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "rename,%s,%s",
-                           arg0, (int)arg1+1, arg2, (int)arg3+1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "rename,%s,%s",
+                                   arg0, (int)arg1+1, arg2, (int)arg3+1);
         } else {
             char *s2;
             s = lock_user_string(arg0);
@@ -398,8 +459,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "system,%s", arg0, (int)arg1+1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "system,%s",
+                                   arg0, (int)arg1+1);
         } else {
             s = lock_user_string(arg0);
             if (!s) {
@@ -558,11 +619,35 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             return 0;
         }
     case TARGET_SYS_EXIT:
-        /* ARM specifies only Stopped_ApplicationExit as normal
-         * exit, everything else is considered an error */
-        ret = (args == ADP_Stopped_ApplicationExit) ? 0 : 1;
+        if (is_a64(env)) {
+            /* The A64 version of this call takes a parameter block,
+             * so the application-exit type can return a subcode which
+             * is the exit status code from the application.
+             */
+            GET_ARG(0);
+            GET_ARG(1);
+
+            if (arg0 == ADP_Stopped_ApplicationExit) {
+                ret = arg1;
+            } else {
+                ret = 1;
+            }
+        } else {
+            /* ARM specifies only Stopped_ApplicationExit as normal
+             * exit, everything else is considered an error */
+            ret = (args == ADP_Stopped_ApplicationExit) ? 0 : 1;
+        }
         gdb_exit(env, ret);
         exit(ret);
+    case TARGET_SYS_SYNCCACHE:
+        /* Clean the D-cache and invalidate the I-cache for the specified
+         * virtual address range. This is a nop for us since we don't
+         * implement caches. This is only present on A64.
+         */
+        if (is_a64(env)) {
+            return 0;
+        }
+        /* fall through -- invalid for A32/T32 */
     default:
         fprintf(stderr, "qemu: Unsupported SemiHosting SWI 0x%02x\n", nr);
         cpu_dump_state(cs, stderr, fprintf, 0);
diff --git a/target-arm/cpu-qom.h b/target-arm/cpu-qom.h
index 3cbc4a006..9376eafcb 100644
--- a/target-arm/cpu-qom.h
+++ b/target-arm/cpu-qom.h
@@ -150,6 +150,7 @@ typedef struct ARMCPU {
     uint32_t id_mmfr1;
     uint32_t id_mmfr2;
     uint32_t id_mmfr3;
+    uint32_t id_mmfr4;
     uint32_t id_isar0;
     uint32_t id_isar1;
     uint32_t id_isar2;
@@ -224,6 +225,21 @@ int arm_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 /* Callback functions for the generic timer's timers. */
 void arm_gt_ptimer_cb(void *opaque);
 void arm_gt_vtimer_cb(void *opaque);
+void arm_gt_htimer_cb(void *opaque);
+void arm_gt_stimer_cb(void *opaque);
+
+#define ARM_AFF0_SHIFT 0
+#define ARM_AFF0_MASK  (0xFFULL << ARM_AFF0_SHIFT)
+#define ARM_AFF1_SHIFT 8
+#define ARM_AFF1_MASK  (0xFFULL << ARM_AFF1_SHIFT)
+#define ARM_AFF2_SHIFT 16
+#define ARM_AFF2_MASK  (0xFFULL << ARM_AFF2_SHIFT)
+#define ARM_AFF3_SHIFT 32
+#define ARM_AFF3_MASK  (0xFFULL << ARM_AFF3_SHIFT)
+
+#define ARM32_AFFINITY_MASK (ARM_AFF0_MASK|ARM_AFF1_MASK|ARM_AFF2_MASK)
+#define ARM64_AFFINITY_MASK \
+    (ARM_AFF0_MASK|ARM_AFF1_MASK|ARM_AFF2_MASK|ARM_AFF3_MASK)
 
 #ifdef TARGET_AARCH64
 int aarch64_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
diff --git a/target-arm/cpu.c b/target-arm/cpu.c
index ce0b8c09c..30739fc0d 100644
--- a/target-arm/cpu.c
+++ b/target-arm/cpu.c
@@ -79,6 +79,27 @@ static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque)
     }
 }
 
+static void cp_reg_check_reset(gpointer key, gpointer value,  gpointer opaque)
+{
+    /* Purely an assertion check: we've already done reset once,
+     * so now check that running the reset for the cpreg doesn't
+     * change its value. This traps bugs where two different cpregs
+     * both try to reset the same state field but to different values.
+     */
+    ARMCPRegInfo *ri = value;
+    ARMCPU *cpu = opaque;
+    uint64_t oldvalue, newvalue;
+
+    if (ri->type & (ARM_CP_SPECIAL | ARM_CP_ALIAS | ARM_CP_NO_RAW)) {
+        return;
+    }
+
+    oldvalue = read_raw_cp_reg(&cpu->env, ri);
+    cp_reg_reset(key, value, opaque);
+    newvalue = read_raw_cp_reg(&cpu->env, ri);
+    assert(oldvalue == newvalue);
+}
+
 /* CPUClass::reset() */
 static void arm_cpu_reset(CPUState *s)
 {
@@ -90,6 +111,8 @@ static void arm_cpu_reset(CPUState *s)
 
     memset(env, 0, offsetof(CPUARMState, features));
     g_hash_table_foreach(cpu->cp_regs, cp_reg_reset, cpu);
+    g_hash_table_foreach(cpu->cp_regs, cp_reg_check_reset, cpu);
+
     env->vfp.xregs[ARM_VFP_FPSID] = cpu->reset_fpsid;
     env->vfp.xregs[ARM_VFP_MVFR0] = cpu->mvfr0;
     env->vfp.xregs[ARM_VFP_MVFR1] = cpu->mvfr1;
@@ -308,10 +331,7 @@ static void arm_cpu_set_irq(void *opaque, int irq, int level)
     switch (irq) {
     case ARM_CPU_VIRQ:
     case ARM_CPU_VFIQ:
-        if (!arm_feature(env, ARM_FEATURE_EL2)) {
-            hw_error("%s: Virtual interrupt line %d with no EL2 support\n",
-                     __func__, irq);
-        }
+        assert(arm_feature(env, ARM_FEATURE_EL2));
         /* fall through */
     case ARM_CPU_IRQ:
     case ARM_CPU_FIQ:
@@ -322,7 +342,7 @@ static void arm_cpu_set_irq(void *opaque, int irq, int level)
         }
         break;
     default:
-        hw_error("arm_cpu_set_irq: Bad interrupt line %d\n", irq);
+        g_assert_not_reached();
     }
 }
 
@@ -341,7 +361,7 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq, int level)
         kvm_irq |= KVM_ARM_IRQ_CPU_FIQ;
         break;
     default:
-        hw_error("arm_cpu_kvm_set_irq: Bad interrupt line %d\n", irq);
+        g_assert_not_reached();
     }
     kvm_irq |= cs->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
     kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
@@ -436,7 +456,7 @@ static void arm_cpu_initfn(Object *obj)
      */
     Aff1 = cs->cpu_index / ARM_CPUS_PER_CLUSTER;
     Aff0 = cs->cpu_index % ARM_CPUS_PER_CLUSTER;
-    cpu->mp_affinity = (Aff1 << 8) | Aff0;
+    cpu->mp_affinity = (Aff1 << ARM_AFF1_SHIFT) | Aff0;
 
 #ifndef CONFIG_USER_ONLY
     /* Our inbound IRQ and FIQ lines */
@@ -453,6 +473,10 @@ static void arm_cpu_initfn(Object *obj)
                                                 arm_gt_ptimer_cb, cpu);
     cpu->gt_timer[GTIMER_VIRT] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
                                                 arm_gt_vtimer_cb, cpu);
+    cpu->gt_timer[GTIMER_HYP] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
+                                                arm_gt_htimer_cb, cpu);
+    cpu->gt_timer[GTIMER_SEC] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
+                                                arm_gt_stimer_cb, cpu);
     qdev_init_gpio_out(DEVICE(cpu), cpu->gt_timer_outputs,
                        ARRAY_SIZE(cpu->gt_timer_outputs));
 #endif
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index 7e89152bd..815fef8a3 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -26,10 +26,8 @@
 #if defined(TARGET_AARCH64)
   /* AArch64 definitions */
 #  define TARGET_LONG_BITS 64
-#  define ELF_MACHINE EM_AARCH64
 #else
 #  define TARGET_LONG_BITS 32
-#  define ELF_MACHINE EM_ARM
 #endif
 
 #define TARGET_IS_BIENDIAN 1
@@ -56,6 +54,7 @@
 #define EXCP_SMC            13   /* Secure Monitor Call */
 #define EXCP_VIRQ           14
 #define EXCP_VFIQ           15
+#define EXCP_SEMIHOST       16   /* semihosting call (A64 only) */
 
 #define ARMV7M_EXCP_RESET   1
 #define ARMV7M_EXCP_NMI     2
@@ -96,6 +95,7 @@
 struct arm_boot_info;
 
 #define NB_MMU_MODES 7
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 /* We currently assume float and double are IEEE single and double
    precision respectively.
@@ -113,7 +113,9 @@ typedef struct ARMGenericTimer {
 
 #define GTIMER_PHYS 0
 #define GTIMER_VIRT 1
-#define NUM_GTIMERS 2
+#define GTIMER_HYP  2
+#define GTIMER_SEC  3
+#define NUM_GTIMERS 4
 
 typedef struct {
     uint64_t raw_tcr;
@@ -170,7 +172,7 @@ typedef struct CPUARMState {
     uint32_t GE; /* cpsr[19:16] */
     uint32_t thumb; /* cpsr[5]. 0 = arm mode, 1 = thumb mode. */
     uint32_t condexec_bits; /* IT bits.  cpsr[15:10,26:25].  */
-    uint64_t daif; /* exception masks, in the bits they are in in PSTATE */
+    uint64_t daif; /* exception masks, in the bits they are in PSTATE */
 
     uint64_t elr_el[4]; /* AArch64 exception link regs  */
     uint64_t sp_el[4]; /* AArch64 banked stack pointers */
@@ -219,10 +221,12 @@ typedef struct CPUARMState {
             };
             uint64_t ttbr1_el[4];
         };
+        uint64_t vttbr_el2; /* Virtualization Translation Table Base.  */
         /* MMU translation table base control. */
         TCR tcr_el[4];
-        uint32_t c2_data; /* MPU data cachable bits.  */
-        uint32_t c2_insn; /* MPU instruction cachable bits.  */
+        TCR vtcr_el2; /* Virtualization Translation Control.  */
+        uint32_t c2_data; /* MPU data cacheable bits.  */
+        uint32_t c2_insn; /* MPU instruction cacheable bits.  */
         union { /* MMU domain access control register
                  * MPU write buffer control.
                  */
@@ -275,6 +279,7 @@ typedef struct CPUARMState {
             };
             uint64_t far_el[4];
         };
+        uint64_t hpfar_el2;
         union { /* Translation result. */
             struct {
                 uint64_t _unused_par_0;
@@ -358,6 +363,8 @@ typedef struct CPUARMState {
         };
         uint64_t c14_cntfrq; /* Counter Frequency register */
         uint64_t c14_cntkctl; /* Timer Control register */
+        uint32_t cnthctl_el2; /* Counter/Timer Hyp Control register */
+        uint64_t cntvoff_el2; /* Counter Virtual Offset register */
         ARMGenericTimer c14_timer[NUM_GTIMERS];
         uint32_t c15_cpar; /* XScale Coprocessor Access Register */
         uint32_t c15_ticonfig; /* TI925T configuration byte.  */
@@ -373,11 +380,15 @@ typedef struct CPUARMState {
         uint64_t dbgwvr[16]; /* watchpoint value registers */
         uint64_t dbgwcr[16]; /* watchpoint control registers */
         uint64_t mdscr_el1;
+        uint64_t oslsr_el1; /* OS Lock Status */
+        uint64_t mdcr_el2;
         /* If the counter is enabled, this stores the last time the counter
          * was reset. Otherwise it stores the counter value
          */
         uint64_t c15_ccnt;
         uint64_t pmccfiltr_el0; /* Performance Monitor Filter Register */
+        uint64_t vpidr_el2; /* Virtualization Processor ID Register */
+        uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */
     } cp15;
 
     struct {
@@ -500,7 +511,7 @@ typedef struct CPUARMState {
 
 ARMCPU *cpu_arm_init(const char *cpu_model);
 int cpu_arm_exec(CPUState *cpu);
-uint32_t do_arm_semihosting(CPUARMState *env);
+target_ulong do_arm_semihosting(CPUARMState *env);
 void aarch64_sync_32_to_64(CPUARMState *env);
 void aarch64_sync_64_to_32(CPUARMState *env);
 
@@ -1008,11 +1019,11 @@ static inline bool access_secure_reg(CPUARMState *env)
  */
 #define A32_BANKED_CURRENT_REG_GET(_env, _regname)        \
     A32_BANKED_REG_GET((_env), _regname,                \
-                       ((!arm_el_is_aa64((_env), 3) && arm_is_secure(_env))))
+                       (arm_is_secure(_env) && !arm_el_is_aa64((_env), 3)))
 
 #define A32_BANKED_CURRENT_REG_SET(_env, _regname, _val)                       \
     A32_BANKED_REG_SET((_env), _regname,                                    \
-                       ((!arm_el_is_aa64((_env), 3) && arm_is_secure(_env))),  \
+                       (arm_is_secure(_env) && !arm_el_is_aa64((_env), 3)), \
                        (_val))
 
 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf);
@@ -1280,6 +1291,9 @@ typedef enum CPAccessResult {
     /* As CP_ACCESS_TRAP, but for traps directly to EL2 or EL3 */
     CP_ACCESS_TRAP_EL2 = 3,
     CP_ACCESS_TRAP_EL3 = 4,
+    /* As CP_ACCESS_UNCATEGORIZED, but for traps directly to EL2 or EL3 */
+    CP_ACCESS_TRAP_UNCATEGORIZED_EL2 = 5,
+    CP_ACCESS_TRAP_UNCATEGORIZED_EL3 = 6,
 } CPAccessResult;
 
 /* Access functions for coprocessor registers. These cannot fail and
@@ -1445,6 +1459,9 @@ static inline bool cp_access_ok(int current_el,
     return (ri->access >> ((current_el * 2) + isread)) & 1;
 }
 
+/* Raw read of a coprocessor register (as needed for migration, etc) */
+uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri);
+
 /**
  * write_list_to_cpustate
  * @cpu: ARMCPU
@@ -1477,7 +1494,7 @@ bool write_list_to_cpustate(ARMCPU *cpu);
  */
 bool write_cpustate_to_list(ARMCPU *cpu);
 
-/* Does the core conform to the the "MicroController" profile. e.g. Cortex-M3.
+/* Does the core conform to the "MicroController" profile. e.g. Cortex-M3.
    Note the M in older cores (eg. ARM7TDMI) stands for Multiply. These are
    conventional cores (ie. Application or Realtime profile).  */
 
@@ -1509,8 +1526,6 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
     CPUARMState *env = cs->env_ptr;
     unsigned int cur_el = arm_current_el(env);
     bool secure = arm_is_secure(env);
-    uint32_t scr;
-    uint32_t hcr;
     bool pstate_unmasked;
     int8_t unmasked = 0;
 
@@ -1524,31 +1539,10 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
 
     switch (excp_idx) {
     case EXCP_FIQ:
-        /* If FIQs are routed to EL3 or EL2 then there are cases where we
-         * override the CPSR.F in determining if the exception is masked or
-         * not.  If neither of these are set then we fall back to the CPSR.F
-         * setting otherwise we further assess the state below.
-         */
-        hcr = (env->cp15.hcr_el2 & HCR_FMO);
-        scr = (env->cp15.scr_el3 & SCR_FIQ);
-
-        /* When EL3 is 32-bit, the SCR.FW bit controls whether the CPSR.F bit
-         * masks FIQ interrupts when taken in non-secure state.  If SCR.FW is
-         * set then FIQs can be masked by CPSR.F when non-secure but only
-         * when FIQs are only routed to EL3.
-         */
-        scr &= !((env->cp15.scr_el3 & SCR_FW) && !hcr);
         pstate_unmasked = !(env->daif & PSTATE_F);
         break;
 
     case EXCP_IRQ:
-        /* When EL3 execution state is 32-bit, if HCR.IMO is set then we may
-         * override the CPSR.I masking when in non-secure state.  The SCR.IRQ
-         * setting has already been taken into consideration when setting the
-         * target EL, so it does not have a further affect here.
-         */
-        hcr = (env->cp15.hcr_el2 & HCR_IMO);
-        scr = false;
         pstate_unmasked = !(env->daif & PSTATE_I);
         break;
 
@@ -1573,8 +1567,58 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
      * interrupt.
      */
     if ((target_el > cur_el) && (target_el != 1)) {
-        if (arm_el_is_aa64(env, 3) || ((scr || hcr) && (!secure))) {
-            unmasked = 1;
+        /* Exceptions targeting a higher EL may not be maskable */
+        if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+            /* 64-bit masking rules are simple: exceptions to EL3
+             * can't be masked, and exceptions to EL2 can only be
+             * masked from Secure state. The HCR and SCR settings
+             * don't affect the masking logic, only the interrupt routing.
+             */
+            if (target_el == 3 || !secure) {
+                unmasked = 1;
+            }
+        } else {
+            /* The old 32-bit-only environment has a more complicated
+             * masking setup. HCR and SCR bits not only affect interrupt
+             * routing but also change the behaviour of masking.
+             */
+            bool hcr, scr;
+
+            switch (excp_idx) {
+            case EXCP_FIQ:
+                /* If FIQs are routed to EL3 or EL2 then there are cases where
+                 * we override the CPSR.F in determining if the exception is
+                 * masked or not. If neither of these are set then we fall back
+                 * to the CPSR.F setting otherwise we further assess the state
+                 * below.
+                 */
+                hcr = (env->cp15.hcr_el2 & HCR_FMO);
+                scr = (env->cp15.scr_el3 & SCR_FIQ);
+
+                /* When EL3 is 32-bit, the SCR.FW bit controls whether the
+                 * CPSR.F bit masks FIQ interrupts when taken in non-secure
+                 * state. If SCR.FW is set then FIQs can be masked by CPSR.F
+                 * when non-secure but only when FIQs are only routed to EL3.
+                 */
+                scr = scr && !((env->cp15.scr_el3 & SCR_FW) && !hcr);
+                break;
+            case EXCP_IRQ:
+                /* When EL3 execution state is 32-bit, if HCR.IMO is set then
+                 * we may override the CPSR.I masking when in non-secure state.
+                 * The SCR.IRQ setting has already been taken into consideration
+                 * when setting the target EL, so it does not have a further
+                 * affect here.
+                 */
+                hcr = (env->cp15.hcr_el2 & HCR_IMO);
+                scr = false;
+                break;
+            default:
+                g_assert_not_reached();
+            }
+
+            if ((scr || hcr) && !secure) {
+                unmasked = 1;
+            }
         }
     }
 
@@ -1587,7 +1631,6 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
 #define cpu_init(cpu_model) CPU(cpu_arm_init(cpu_model))
 
 #define cpu_exec cpu_arm_exec
-#define cpu_gen_code cpu_arm_gen_code
 #define cpu_signal_handler cpu_arm_signal_handler
 #define cpu_list arm_cpu_list
 
@@ -1667,7 +1710,7 @@ static inline int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx)
 }
 
 /* Determine the current mmu_idx to use for normal loads/stores */
-static inline int cpu_mmu_index(CPUARMState *env)
+static inline int cpu_mmu_index(CPUARMState *env, bool ifetch)
 {
     int el = arm_current_el(env);
 
@@ -1682,7 +1725,22 @@ static inline int cpu_mmu_index(CPUARMState *env)
  */
 static inline int arm_debug_target_el(CPUARMState *env)
 {
-    return 1;
+    bool secure = arm_is_secure(env);
+    bool route_to_el2 = false;
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !secure) {
+        route_to_el2 = env->cp15.hcr_el2 & HCR_TGE ||
+                       env->cp15.mdcr_el2 & (1 << 8);
+    }
+
+    if (route_to_el2) {
+        return 2;
+    } else if (arm_feature(env, ARM_FEATURE_EL3) &&
+               !arm_el_is_aa64(env, 3) && secure) {
+        return 3;
+    } else {
+        return 1;
+    }
 }
 
 static inline bool aa64_generate_debug_exceptions(CPUARMState *env)
@@ -1900,7 +1958,7 @@ static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
                    << ARM_TBFLAG_XSCALE_CPAR_SHIFT);
     }
 
-    *flags |= (cpu_mmu_index(env) << ARM_TBFLAG_MMUIDX_SHIFT);
+    *flags |= (cpu_mmu_index(env, false) << ARM_TBFLAG_MMUIDX_SHIFT);
     /* The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
      * states defined in the ARM ARM for software singlestep:
      *  SS_ACTIVE   PSTATE.SS   State
diff --git a/target-arm/helper-a64.c b/target-arm/helper-a64.c
index 08c95a3f5..deb8dbe48 100644
--- a/target-arm/helper-a64.c
+++ b/target-arm/helper-a64.c
@@ -70,20 +70,7 @@ uint32_t HELPER(clz32)(uint32_t x)
 
 uint64_t HELPER(rbit64)(uint64_t x)
 {
-    /* assign the correct byte position */
-    x = bswap64(x);
-
-    /* assign the correct nibble position */
-    x = ((x & 0xf0f0f0f0f0f0f0f0ULL) >> 4)
-        | ((x & 0x0f0f0f0f0f0f0f0fULL) << 4);
-
-    /* assign the correct bit position */
-    x = ((x & 0x8888888888888888ULL) >> 3)
-        | ((x & 0x4444444444444444ULL) >> 1)
-        | ((x & 0x2222222222222222ULL) << 1)
-        | ((x & 0x1111111111111111ULL) << 3);
-
-    return x;
+    return revbit64(x);
 }
 
 /* Convert a softfloat float_relation_ (as returned by
@@ -478,7 +465,8 @@ void aarch64_cpu_do_interrupt(CPUState *cs)
     }
 
     arm_log_exception(cs->exception_index);
-    qemu_log_mask(CPU_LOG_INT, "...from EL%d\n", arm_current_el(env));
+    qemu_log_mask(CPU_LOG_INT, "...from EL%d to EL%d\n", arm_current_el(env),
+                  new_el);
     if (qemu_loglevel_mask(CPU_LOG_INT)
         && !excp_is_internal(cs->exception_index)) {
         qemu_log_mask(CPU_LOG_INT, "...with ESR 0x%" PRIx32 "\n",
@@ -514,6 +502,12 @@ void aarch64_cpu_do_interrupt(CPUState *cs)
     case EXCP_VFIQ:
         addr += 0x100;
         break;
+    case EXCP_SEMIHOST:
+        qemu_log_mask(CPU_LOG_INT,
+                      "...handling as semihosting call 0x%" PRIx64 "\n",
+                      env->xregs[0]);
+        env->xregs[0] = do_arm_semihosting(env);
+        return;
     default:
         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
     }
diff --git a/target-arm/helper.c b/target-arm/helper.c
index 01f0d0dac..1743e37ab 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -12,11 +12,20 @@
 #include <zlib.h> /* For crc32 */
 #include "exec/semihost.h"
 
+#define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */
+
 #ifndef CONFIG_USER_ONLY
-static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
-                                 int access_type, ARMMMUIdx mmu_idx,
-                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
-                                 target_ulong *page_size, uint32_t *fsr);
+static bool get_phys_addr(CPUARMState *env, target_ulong address,
+                          int access_type, ARMMMUIdx mmu_idx,
+                          hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
+                          target_ulong *page_size, uint32_t *fsr,
+                          ARMMMUFaultInfo *fi);
+
+static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
+                               int access_type, ARMMMUIdx mmu_idx,
+                               hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
+                               target_ulong *page_size_ptr, uint32_t *fsr,
+                               ARMMMUFaultInfo *fi);
 
 /* Definitions for the PMCCNTR and PMCR registers */
 #define PMCRD   0x8
@@ -144,7 +153,7 @@ static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
     return (char *)env + ri->fieldoffset;
 }
 
-static uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
+uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     /* Raw read of a coprocessor register (as needed for migration, etc). */
     if (ri->type & ARM_CP_CONST) {
@@ -325,6 +334,34 @@ void init_cpreg_list(ARMCPU *cpu)
     g_list_free(keys);
 }
 
+/*
+ * Some registers are not accessible if EL3.NS=0 and EL3 is using AArch32 but
+ * they are accessible when EL3 is using AArch64 regardless of EL3.NS.
+ *
+ * access_el3_aa32ns: Used to check AArch32 register views.
+ * access_el3_aa32ns_aa64any: Used to check both AArch32/64 register views.
+ */
+static CPAccessResult access_el3_aa32ns(CPUARMState *env,
+                                        const ARMCPRegInfo *ri)
+{
+    bool secure = arm_is_secure_below_el3(env);
+
+    assert(!arm_el_is_aa64(env, 3));
+    if (secure) {
+        return CP_ACCESS_TRAP_UNCATEGORIZED;
+    }
+    return CP_ACCESS_OK;
+}
+
+static CPAccessResult access_el3_aa32ns_aa64any(CPUARMState *env,
+                                                const ARMCPRegInfo *ri)
+{
+    if (!arm_el_is_aa64(env, 3)) {
+        return access_el3_aa32ns(env, ri);
+    }
+    return CP_ACCESS_OK;
+}
+
 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
@@ -627,8 +664,12 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
     { .name = "MVA_prefetch",
       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NOP },
+    /* We need to break the TB after ISB to execute self-modifying code
+     * correctly and also to take any pending interrupts immediately.
+     * So use arm_cp_write_ignore() function instead of ARM_CP_NOP flag.
+     */
     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
-      .access = PL0_W, .type = ARM_CP_NOP },
+      .access = PL0_W, .type = ARM_CP_NO_RAW, .writefn = arm_cp_write_ignore },
     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
       .access = PL0_W, .type = ARM_CP_NOP },
     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
@@ -678,8 +719,8 @@ void pmccntr_sync(CPUARMState *env)
 {
     uint64_t temp_ticks;
 
-    temp_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                          get_ticks_per_sec(), 1000000);
+    temp_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                          ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -717,8 +758,8 @@ static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
         return env->cp15.c15_ccnt;
     }
 
-    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                           get_ticks_per_sec(), 1000000);
+    total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -738,8 +779,8 @@ static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
         return;
     }
 
-    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                           get_ticks_per_sec(), 1000000);
+    total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -1022,6 +1063,10 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
       .resetvalue = 0 },
+    { .name = "MAIR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 2, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[3]),
+      .resetvalue = 0 },
     /* For non-long-descriptor page tables these are PRRR and NMRR;
      * regardless they still act as reads-as-written for QEMU.
      */
@@ -1154,23 +1199,41 @@ static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
 
 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx)
 {
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
-    if (arm_current_el(env) == 0 &&
+    if (cur_el == 0 &&
         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) &&
+        timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
+        !extract32(env->cp15.cnthctl_el2, 0, 1)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
     return CP_ACCESS_OK;
 }
 
 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx)
 {
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
      * EL0[PV]TEN is zero.
      */
-    if (arm_current_el(env) == 0 &&
+    if (cur_el == 0 &&
         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) &&
+        timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
+        !extract32(env->cp15.cnthctl_el2, 1, 1)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
     return CP_ACCESS_OK;
 }
 
@@ -1196,6 +1259,32 @@ static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
     return gt_timer_access(env, GTIMER_VIRT);
 }
 
+static CPAccessResult gt_stimer_access(CPUARMState *env,
+                                       const ARMCPRegInfo *ri)
+{
+    /* The AArch64 register view of the secure physical timer is
+     * always accessible from EL3, and configurably accessible from
+     * Secure EL1.
+     */
+    switch (arm_current_el(env)) {
+    case 1:
+        if (!arm_is_secure(env)) {
+            return CP_ACCESS_TRAP;
+        }
+        if (!(env->cp15.scr_el3 & SCR_ST)) {
+            return CP_ACCESS_TRAP_EL3;
+        }
+        return CP_ACCESS_OK;
+    case 0:
+    case 2:
+        return CP_ACCESS_TRAP;
+    case 3:
+        return CP_ACCESS_OK;
+    default:
+        g_assert_not_reached();
+    }
+}
+
 static uint64_t gt_get_countervalue(CPUARMState *env)
 {
     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / GTIMER_SCALE;
@@ -1209,9 +1298,11 @@ static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
         /* Timer enabled: calculate and set current ISTATUS, irq, and
          * reset timer to when ISTATUS next has to change
          */
+        uint64_t offset = timeridx == GTIMER_VIRT ?
+                                      cpu->env.cp15.cntvoff_el2 : 0;
         uint64_t count = gt_get_countervalue(&cpu->env);
         /* Note that this must be unsigned 64 bit arithmetic: */
-        int istatus = count >= gt->cval;
+        int istatus = count - offset >= gt->cval;
         uint64_t nexttick;
 
         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
@@ -1222,7 +1313,7 @@ static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
             nexttick = UINT64_MAX;
         } else {
             /* Next transition is when we hit cval */
-            nexttick = gt->cval;
+            nexttick = gt->cval + offset;
         }
         /* Note that the desired next expiry time might be beyond the
          * signed-64-bit range of a QEMUTimer -- in this case we just
@@ -1241,10 +1332,10 @@ static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
     }
 }
 
-static void gt_cnt_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
+                           int timeridx)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int timeridx = ri->opc1 & 1;
 
     timer_del(cpu->gt_timer[timeridx]);
 }
@@ -1254,38 +1345,44 @@ static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
     return gt_get_countervalue(env);
 }
 
+static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_get_countervalue(env) - env->cp15.cntvoff_el2;
+}
+
 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          int timeridx,
                           uint64_t value)
 {
-    int timeridx = ri->opc1 & 1;
-
     env->cp15.c14_timer[timeridx].cval = value;
     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
 }
 
-static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
+                             int timeridx)
 {
-    int timeridx = ri->crm & 1;
+    uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
 
     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
-                      gt_get_countervalue(env));
+                      (gt_get_countervalue(env) - offset));
 }
 
 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          int timeridx,
                           uint64_t value)
 {
-    int timeridx = ri->crm & 1;
+    uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
 
-    env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) +
+    env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset +
                                          sextract64(value, 0, 32);
     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
 }
 
 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         int timeridx,
                          uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int timeridx = ri->crm & 1;
     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
 
     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
@@ -1301,6 +1398,127 @@ static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
     }
 }
 
+static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_PHYS);
+}
+
+static void gt_phys_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_PHYS, value);
+}
+
+static uint64_t gt_phys_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_PHYS);
+}
+
+static void gt_phys_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_PHYS, value);
+}
+
+static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_PHYS, value);
+}
+
+static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_VIRT);
+}
+
+static void gt_virt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_VIRT, value);
+}
+
+static uint64_t gt_virt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_VIRT);
+}
+
+static void gt_virt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_VIRT, value);
+}
+
+static void gt_virt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_VIRT, value);
+}
+
+static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+
+    raw_write(env, ri, value);
+    gt_recalc_timer(cpu, GTIMER_VIRT);
+}
+
+static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_HYP);
+}
+
+static void gt_hyp_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_HYP, value);
+}
+
+static uint64_t gt_hyp_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_HYP);
+}
+
+static void gt_hyp_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_HYP, value);
+}
+
+static void gt_hyp_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_HYP, value);
+}
+
+static void gt_sec_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_SEC);
+}
+
+static void gt_sec_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_SEC, value);
+}
+
+static uint64_t gt_sec_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_SEC);
+}
+
+static void gt_sec_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_SEC, value);
+}
+
+static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_SEC, value);
+}
+
 void arm_gt_ptimer_cb(void *opaque)
 {
     ARMCPU *cpu = opaque;
@@ -1315,6 +1533,20 @@ void arm_gt_vtimer_cb(void *opaque)
     gt_recalc_timer(cpu, GTIMER_VIRT);
 }
 
+void arm_gt_htimer_cb(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+
+    gt_recalc_timer(cpu, GTIMER_HYP);
+}
+
+void arm_gt_stimer_cb(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+
+    gt_recalc_timer(cpu, GTIMER_SEC);
+}
+
 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     /* Note that CNTFRQ is purely reads-as-written for the benefit
      * of software; writing it doesn't actually change the timer frequency.
@@ -1340,11 +1572,21 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     },
     /* per-timer control */
     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
+      .secure = ARM_CP_SECSTATE_NS,
       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
       .accessfn = gt_ptimer_access,
       .fieldoffset = offsetoflow32(CPUARMState,
                                    cp15.c14_timer[GTIMER_PHYS].ctl),
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTP_CTL(S)",
+      .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
+      .secure = ARM_CP_SECSTATE_S,
+      .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
+      .accessfn = gt_ptimer_access,
+      .fieldoffset = offsetoflow32(CPUARMState,
+                                   cp15.c14_timer[GTIMER_SEC].ctl),
+      .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
@@ -1352,14 +1594,14 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .accessfn = gt_ptimer_access,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
       .resetvalue = 0,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
       .accessfn = gt_vtimer_access,
       .fieldoffset = offsetoflow32(CPUARMState,
                                    cp15.c14_timer[GTIMER_VIRT].ctl),
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
@@ -1367,30 +1609,38 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .accessfn = gt_vtimer_access,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
       .resetvalue = 0,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
     },
     /* TimerValue views: a 32 bit downcounting view of the underlying state */
     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
+      .secure = ARM_CP_SECSTATE_NS,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
       .accessfn = gt_ptimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
+    },
+    { .name = "CNTP_TVAL(S)",
+      .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
+      .secure = ARM_CP_SECSTATE_S,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
+      .accessfn = gt_ptimer_access,
+      .readfn = gt_sec_tval_read, .writefn = gt_sec_tval_write,
     },
     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
-      .accessfn = gt_ptimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset,
+      .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
     },
     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
       .accessfn = gt_vtimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
     },
     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
-      .accessfn = gt_vtimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset,
+      .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
     },
     /* The counter itself */
     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
@@ -1401,27 +1651,34 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
-      .accessfn = gt_pct_access,
-      .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
+      .accessfn = gt_pct_access, .readfn = gt_cnt_read,
     },
     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
       .accessfn = gt_vct_access,
-      .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
+      .readfn = gt_virt_cnt_read, .resetfn = arm_cp_reset_ignore,
     },
     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
-      .accessfn = gt_vct_access,
-      .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
+      .accessfn = gt_vct_access, .readfn = gt_virt_cnt_read,
     },
     /* Comparison value, indicating when the timer goes off */
     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
+      .secure = ARM_CP_SECSTATE_NS,
       .access = PL1_RW | PL0_R,
       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
       .accessfn = gt_ptimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTP_CVAL(S)", .cp = 15, .crm = 14, .opc1 = 2,
+      .secure = ARM_CP_SECSTATE_S,
+      .access = PL1_RW | PL0_R,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
+      .accessfn = gt_ptimer_access,
+      .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
@@ -1429,14 +1686,14 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .type = ARM_CP_IO,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
       .resetvalue = 0, .accessfn = gt_ptimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
       .access = PL1_RW | PL0_R,
       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
       .accessfn = gt_vtimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
@@ -1444,7 +1701,33 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .type = ARM_CP_IO,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
       .resetvalue = 0, .accessfn = gt_vtimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
+    },
+    /* Secure timer -- this is actually restricted to only EL3
+     * and configurably Secure-EL1 via the accessfn.
+     */
+    { .name = "CNTPS_TVAL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 0,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .readfn = gt_sec_tval_read,
+      .writefn = gt_sec_tval_write,
+      .resetfn = gt_sec_timer_reset,
+    },
+    { .name = "CNTPS_CTL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 1,
+      .type = ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].ctl),
+      .resetvalue = 0,
+      .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTPS_CVAL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 2,
+      .type = ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
+      .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
     },
     REGINFO_SENTINEL
 };
@@ -1477,12 +1760,17 @@ static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     if (ri->opc2 & 4) {
-        /* Other states are only available with TrustZone; in
-         * a non-TZ implementation these registers don't exist
-         * at all, which is an Uncategorized trap. This underdecoding
-         * is safe because the reginfo is NO_RAW.
+        /* The ATS12NSO* operations must trap to EL3 if executed in
+         * Secure EL1 (which can only happen if EL3 is AArch64).
+         * They are simply UNDEF if executed from NS EL1.
+         * They function normally from EL2 or EL3.
          */
-        return CP_ACCESS_TRAP_UNCATEGORIZED;
+        if (arm_current_el(env) == 1) {
+            if (arm_is_secure_below_el3(env)) {
+                return CP_ACCESS_TRAP_UNCATEGORIZED_EL3;
+            }
+            return CP_ACCESS_TRAP_UNCATEGORIZED;
+        }
     }
     return CP_ACCESS_OK;
 }
@@ -1497,9 +1785,10 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
     bool ret;
     uint64_t par64;
     MemTxAttrs attrs = {};
+    ARMMMUFaultInfo fi = {};
 
     ret = get_phys_addr(env, value, access_type, mmu_idx,
-                        &phys_addr, &attrs, &prot, &page_size, &fsr);
+                        &phys_addr, &attrs, &prot, &page_size, &fsr, &fi);
     if (extended_addresses_enabled(env)) {
         /* fsr is a DFSR/IFSR value for the long descriptor
          * translation table format, but with WnR always clear.
@@ -1602,6 +1891,25 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
     A32_BANKED_CURRENT_REG_SET(env, par, par64);
 }
 
+static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    int access_type = ri->opc2 & 1;
+    uint64_t par64;
+
+    par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S2NS);
+
+    A32_BANKED_CURRENT_REG_SET(env, par, par64);
+}
+
+static CPAccessResult at_s1e2_access(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    if (arm_current_el(env) == 3 && !(env->cp15.scr_el3 & SCR_NS)) {
+        return CP_ACCESS_TRAP;
+    }
+    return CP_ACCESS_OK;
+}
+
 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
                         uint64_t value)
 {
@@ -1629,10 +1937,10 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
         break;
     case 4: /* AT S12E1R, AT S12E1W */
-        mmu_idx = ARMMMUIdx_S12NSE1;
+        mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1;
         break;
     case 6: /* AT S12E0R, AT S12E0W */
-        mmu_idx = ARMMMUIdx_S12NSE0;
+        mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0;
         break;
     default:
         g_assert_not_reached();
@@ -1649,6 +1957,7 @@ static const ARMCPRegInfo vapa_cp_reginfo[] = {
                              offsetoflow32(CPUARMState, cp15.par_ns) },
       .writefn = par_write },
 #ifndef CONFIG_USER_ONLY
+    /* This underdecoding is safe because the reginfo is NO_RAW. */
     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
       .access = PL1_W, .accessfn = ats_access,
       .writefn = ats_write, .type = ARM_CP_NO_RAW },
@@ -1856,7 +2165,7 @@ static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
         }
     }
 
-    /* Update the masks corresponding to the the TCR bank being written
+    /* Update the masks corresponding to the TCR bank being written
      * Note that we always calculate mask and base_mask, but
      * they are only used for short-descriptor tables (ie if EAE is 0);
      * for long-descriptor tables the TCR fields are used differently
@@ -1918,6 +2227,20 @@ static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     raw_write(env, ri, value);
 }
 
+static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    /* Accesses to VTTBR may change the VMID so we must flush the TLB.  */
+    if (raw_read(env, ri) != value) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
+                            ARMMMUIdx_S2NS, -1);
+        raw_write(env, ri, value);
+    }
+}
+
 static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_ALIAS,
@@ -2136,7 +2459,19 @@ static const ARMCPRegInfo strongarm_cp_reginfo[] = {
     REGINFO_SENTINEL
 };
 
-static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+static uint64_t midr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
+    if (arm_feature(&cpu->env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
+        return env->cp15.vpidr_el2;
+    }
+    return raw_read(env, ri);
+}
+
+static uint64_t mpidr_read_val(CPUARMState *env)
 {
     ARMCPU *cpu = ARM_CPU(arm_env_get_cpu(env));
     uint64_t mpidr = cpu->mp_affinity;
@@ -2154,6 +2489,17 @@ static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
     return mpidr;
 }
 
+static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
+        return env->cp15.vmpidr_el2;
+    }
+    return mpidr_read_val(env);
+}
+
 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
@@ -2240,65 +2586,244 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env,
  * Page D4-1736 (DDI0487A.b)
  */
 
-static void tlbi_aa64_va_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                               uint64_t value)
+static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+    } else {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
+    }
+}
+
+static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t value)
+{
+    bool sec = arm_is_secure_below_el3(env);
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        if (sec) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+        } else {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    /* Note that the 'ALL' scope must invalidate both stage 1 and
+     * stage 2 translations, whereas most other scopes only invalidate
+     * stage 1 translations.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+    } else {
+        if (arm_feature(env, ARM_FEATURE_EL2)) {
+            tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
+                                ARMMMUIdx_S2NS, -1);
+        } else {
+            tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E2, -1);
+}
+
+static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E3, -1);
+}
+
+static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    /* Note that the 'ALL' scope must invalidate both stage 1 and
+     * stage 2 translations, whereas most other scopes only invalidate
+     * stage 1 translations.
+     */
+    bool sec = arm_is_secure_below_el3(env);
+    bool has_el2 = arm_feature(env, ARM_FEATURE_EL2);
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        if (sec) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+        } else if (has_el2) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, ARMMMUIdx_S2NS, -1);
+        } else {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E2, -1);
+    }
+}
+
+static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E3, -1);
+    }
+}
+
+static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
 {
-    /* Invalidate by VA (AArch64 version) */
+    /* Invalidate by VA, EL1&0 (AArch64 version).
+     * Currently handles all of VAE1, VAAE1, VAALE1 and VALE1,
+     * since we don't support flush-for-specific-ASID-only or
+     * flush-last-level-only.
+     */
     ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
-    tlb_flush_page(CPU(cpu), pageaddr);
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1SE1,
+                                 ARMMMUIdx_S1SE0, -1);
+    } else {
+        tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S12NSE1,
+                                 ARMMMUIdx_S12NSE0, -1);
+    }
 }
 
-static void tlbi_aa64_vaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                uint64_t value)
+static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
 {
-    /* Invalidate by VA, all ASIDs (AArch64 version) */
+    /* Invalidate by VA, EL2
+     * Currently handles both VAE2 and VALE2, since we don't support
+     * flush-last-level-only.
+     */
     ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
-    tlb_flush_page(CPU(cpu), pageaddr);
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E2, -1);
 }
 
-static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
+static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                  uint64_t value)
 {
-    /* Invalidate by ASID (AArch64 version) */
+    /* Invalidate by VA, EL3
+     * Currently handles both VAE3 and VALE3, since we don't support
+     * flush-last-level-only.
+     */
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int asid = extract64(value, 48, 16);
-    tlb_flush(CPU(cpu), asid == 0);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E3, -1);
 }
 
-static void tlbi_aa64_va_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
 {
+    bool sec = arm_is_secure_below_el3(env);
     CPUState *other_cs;
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush_page(other_cs, pageaddr);
+        if (sec) {
+            tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1SE1,
+                                     ARMMMUIdx_S1SE0, -1);
+        } else {
+            tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S12NSE1,
+                                     ARMMMUIdx_S12NSE0, -1);
+        }
     }
 }
 
-static void tlbi_aa64_vaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
 {
     CPUState *other_cs;
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush_page(other_cs, pageaddr);
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E2, -1);
     }
 }
 
-static void tlbi_aa64_asid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
 {
     CPUState *other_cs;
-    int asid = extract64(value, 48, 16);
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush(other_cs, asid == 0);
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E3, -1);
+    }
+}
+
+static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    /* Invalidate by IPA. This has to invalidate any structures that
+     * contain only stage 2 translation information, but does not need
+     * to apply to structures that contain combined stage 1 and stage 2
+     * translation information.
+     * This must NOP if EL2 isn't implemented or SCR_EL3.NS is zero.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 48);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S2NS, -1);
+}
+
+static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t value)
+{
+    CPUState *other_cs;
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 48);
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S2NS, -1);
     }
 }
 
@@ -2434,62 +2959,86 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
       .access = PL1_W, .type = ARM_CP_NOP },
     /* TLBI operations */
-    { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
-      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
-      .access = PL2_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_write },
-    { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
-      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
-      .access = PL2_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_is_write },
     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_is_write },
+      .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_asid_is_write },
+      .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_write },
+      .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_write },
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_asid_write },
+      .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_write },
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae1_write },
+    { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1is_write },
+    { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1is_write },
+    { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
+    { .name = "TLBI_VMALLS12E1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
+    { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1_write },
+    { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1_write },
+    { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1_write },
+    { .name = "TLBI_VMALLS12E1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
 #ifndef CONFIG_USER_ONLY
     /* 64 bit address translation operations */
     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
@@ -2504,6 +3053,31 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E1R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E1W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E0R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E0W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 7,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    /* AT S1E2* are elsewhere as they UNDEF from EL3 if EL2 is not present */
+    { .name = "AT_S1E3R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S1E3W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
+      .access = PL1_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
+      .writefn = par_write },
 #endif
     /* TLB invalidate last level of translation table walk */
     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
@@ -2556,7 +3130,8 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[1]) },
+      .access = PL1_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_SVC]) },
     /* We rely on the access checks not allowing the guest to write to the
      * state field when SPSel indicates that it's being used as the stack
      * pointer.
@@ -2598,9 +3173,36 @@ static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "VTCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 6, .crm = 2,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .type = ARM_CP_CONST | ARM_CP_64BIT, .resetvalue = 0 },
+    { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
@@ -2613,6 +3215,34 @@ static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
       .resetvalue = 0 },
+    { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "HPFAR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
@@ -2670,7 +3300,28 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[6]) },
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_HYP]) },
+    { .name = "SPSR_IRQ", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 0,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_IRQ]) },
+    { .name = "SPSR_ABT", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 1,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_ABT]) },
+    { .name = "SPSR_UND", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 2,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_UND]) },
+    { .name = "SPSR_FIQ", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 3,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_FIQ]) },
     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .writefn = vbar_write,
@@ -2692,11 +3343,46 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
       .access = PL2_RW, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
+    { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    /* HAMAIR1 is mapped to AMAIR_EL2[63:32] */
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
       .access = PL2_RW, .writefn = vmsa_tcr_el1_write,
       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
+    { .name = "VTCR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
+    { .name = "VTCR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .access = PL2_RW, .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
+    { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 6, .crm = 2,
+      .type = ARM_CP_64BIT | ARM_CP_ALIAS,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
+      .writefn = vttbr_write },
+    { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .writefn = vttbr_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
@@ -2715,15 +3401,107 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
     { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
       .type = ARM_CP_NO_RAW, .access = PL2_W,
-      .writefn = tlbiall_write },
+      .writefn = tlbi_aa64_alle2_write },
     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
       .type = ARM_CP_NO_RAW, .access = PL2_W,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae2_write },
+    { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae2_write },
+    { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle2is_write },
     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
       .type = ARM_CP_NO_RAW, .access = PL2_W,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae2is_write },
+    { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae2is_write },
+#ifndef CONFIG_USER_ONLY
+    /* Unlike the other EL2-related AT operations, these must
+     * UNDEF from EL3 if EL2 is not implemented, which is why we
+     * define them here rather than with the rest of the AT ops.
+     */
+    { .name = "AT_S1E2R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL2_W, .accessfn = at_s1e2_access,
+      .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S1E2W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL2_W, .accessfn = at_s1e2_access,
+      .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    /* The AArch32 ATS1H* operations are CONSTRAINED UNPREDICTABLE
+     * if EL2 is not implemented; we choose to UNDEF. Behaviour at EL3
+     * with SCR.NS == 0 outside Monitor mode is UNPREDICTABLE; we choose
+     * to behave as if SCR.NS was 1.
+     */
+    { .name = "ATS1HR", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL2_W,
+      .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
+    { .name = "ATS1HW", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL2_W,
+      .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
+    { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
+      /* ARMv7 requires bit 0 and 1 to reset to 1. ARMv8 defines the
+       * reset values as IMPDEF. We choose to reset to 3 to comply with
+       * both ARMv7 and ARMv8.
+       */
+      .access = PL2_RW, .resetvalue = 3,
+      .fieldoffset = offsetof(CPUARMState, cp15.cnthctl_el2) },
+    { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
+      .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 0,
+      .writefn = gt_cntvoff_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
+    { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS | ARM_CP_IO,
+      .writefn = gt_cntvoff_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
+    { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
+      .type = ARM_CP_IO, .access = PL2_RW,
+      .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
+    { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_IO,
+      .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
+    { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
+      .type = ARM_CP_IO, .access = PL2_RW,
+      .resetfn = gt_hyp_timer_reset,
+      .readfn = gt_hyp_tval_read, .writefn = gt_hyp_tval_write },
+    { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .type = ARM_CP_IO,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
+      .resetvalue = 0,
+      .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
+#endif
+    /* The only field of MDCR_EL2 that has a defined architectural reset value
+     * is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N; but we
+     * don't impelment any PMU event counters, so using zero as a reset
+     * value for MDCR_EL2 is okay
+     */
+    { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2), },
+    { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
+    { .name = "HPFAR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
     REGINFO_SENTINEL
 };
 
@@ -2780,7 +3558,8 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[7]) },
+      .access = PL3_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_MON]) },
     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL3_RW, .writefn = vbar_write,
@@ -2790,6 +3569,46 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
       .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
+    { .name = "TPIDR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 2,
+      .access = PL3_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[3]) },
+    { .name = "AMAIR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "TLBI_ALLE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle3is_write },
+    { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3is_write },
+    { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3is_write },
+    { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle3_write },
+    { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3_write },
+    { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3_write },
     REGINFO_SENTINEL
 };
 
@@ -2804,6 +3623,23 @@ static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
     return CP_ACCESS_OK;
 }
 
+static void oslar_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    /* Writes to OSLAR_EL1 may update the OS lock status, which can be
+     * read via a bit in OSLSR_EL1.
+     */
+    int oslock;
+
+    if (ri->state == ARM_CP_STATE_AA32) {
+        oslock = (value == 0xC5ACCE55);
+    } else {
+        oslock = value & 1;
+    }
+
+    env->cp15.oslsr_el1 = deposit32(env->cp15.oslsr_el1, 1, 1, oslock);
+}
+
 static const ARMCPRegInfo debug_cp_reginfo[] = {
     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
@@ -2832,10 +3668,14 @@ static const ARMCPRegInfo debug_cp_reginfo[] = {
       .type = ARM_CP_ALIAS,
       .access = PL1_R,
       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1), },
-    /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
-      .access = PL1_W, .type = ARM_CP_NOP },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = oslar_write },
+    { .name = "OSLSR_EL1", .state = ARM_CP_STATE_BOTH,
+      .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 4,
+      .access = PL1_R, .resetvalue = 10,
+      .fieldoffset = offsetof(CPUARMState, cp15.oslsr_el1) },
     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
@@ -3252,12 +4092,14 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_isar5 },
-            /* 6..7 are as yet unallocated and must RAZ */
-            { .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
-              .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
-              .resetvalue = 0 },
-            { .name = "ID_ISAR7", .cp = 15, .crn = 0, .crm = 2,
-              .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
+            { .name = "ID_MMFR4", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = cpu->id_mmfr4 },
+            /* 7 is as yet unallocated and must RAZ */
+            { .name = "ID_ISAR7_RESERVED", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = 0 },
             REGINFO_SENTINEL
         };
@@ -3311,7 +4153,11 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_V8)) {
-        /* AArch64 ID registers, which all have impdef reset values */
+        /* AArch64 ID registers, which all have impdef reset values.
+         * Note that within the ID register ranges the unused slots
+         * must all RAZ, not UNDEF; future architecture versions may
+         * define new registers here.
+         */
         ARMCPRegInfo v8_idregs[] = {
             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
@@ -3321,6 +4167,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64pfr1},
+            { .name = "ID_AA64PFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3334,6 +4204,14 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64dfr1 },
+            { .name = "ID_AA64DFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64DFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3342,6 +4220,14 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64afr1 },
+            { .name = "ID_AA64AFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64AFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3350,6 +4236,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64isar1 },
+            { .name = "ID_AA64ISAR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3358,6 +4268,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64mmfr1 },
+            { .name = "ID_AA64MMFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3370,6 +4304,26 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->mvfr2 },
+            { .name = "MVFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             REGINFO_SENTINEL
         };
         /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
@@ -3386,6 +4340,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         define_arm_cp_regs(cpu, v8_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_EL2)) {
+        uint64_t vmpidr_def = mpidr_read_val(env);
+        ARMCPRegInfo vpidr_regs[] = {
+            { .name = "VPIDR", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+              .access = PL2_RW, .accessfn = access_el3_aa32ns,
+              .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+            { .name = "VPIDR_EL2", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+              .access = PL2_RW, .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+            { .name = "VMPIDR", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+              .access = PL2_RW, .accessfn = access_el3_aa32ns,
+              .resetvalue = vmpidr_def,
+              .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
+            { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+              .access = PL2_RW,
+              .resetvalue = vmpidr_def,
+              .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
+            REGINFO_SENTINEL
+        };
+        define_arm_cp_regs(cpu, vpidr_regs);
         define_arm_cp_regs(cpu, el2_cp_reginfo);
         /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
         if (!arm_feature(env, ARM_FEATURE_EL3)) {
@@ -3401,6 +4379,23 @@ void register_cp_regs_for_features(ARMCPU *cpu)
          * register the no_el2 reginfos.
          */
         if (arm_feature(env, ARM_FEATURE_EL3)) {
+            /* When EL3 exists but not EL2, VPIDR and VMPIDR take the value
+             * of MIDR_EL1 and MPIDR_EL1.
+             */
+            ARMCPRegInfo vpidr_regs[] = {
+                { .name = "VPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+                  .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+                  .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+                  .type = ARM_CP_CONST, .resetvalue = cpu->midr,
+                  .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+                { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+                  .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+                  .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+                  .type = ARM_CP_NO_RAW,
+                  .writefn = arm_cp_write_ignore, .readfn = mpidr_read },
+                REGINFO_SENTINEL
+            };
+            define_arm_cp_regs(cpu, vpidr_regs);
             define_arm_cp_regs(cpu, el3_no_el2_cp_reginfo);
         }
     }
@@ -3478,6 +4473,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
               .access = PL1_R, .resetvalue = cpu->midr,
               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
+              .readfn = midr_read,
               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
               .type = ARM_CP_OVERRIDE },
             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
@@ -3501,7 +4497,9 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
-              .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
+              .access = PL1_R, .type = ARM_CP_NO_RAW, .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
+              .readfn = midr_read },
             /* crn = 0 op1 = 0 crm = 0 op2 = 4,7 : AArch32 aliases of MIDR */
             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
@@ -3584,13 +4582,22 @@ void register_cp_regs_for_features(ARMCPU *cpu)
     }
 
     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
-        ARMCPRegInfo auxcr = {
-            .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
-            .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
-            .access = PL1_RW, .type = ARM_CP_CONST,
-            .resetvalue = cpu->reset_auxcr
+        ARMCPRegInfo auxcr_reginfo[] = {
+            { .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL1_RW, .type = ARM_CP_CONST,
+              .resetvalue = cpu->reset_auxcr },
+            { .name = "ACTLR_EL2", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL2_RW, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ACTLR_EL3", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL3_RW, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            REGINFO_SENTINEL
         };
-        define_one_arm_cp_reg(cpu, &auxcr);
+        define_arm_cp_regs(cpu, auxcr_reginfo);
     }
 
     if (arm_feature(env, ARM_FEATURE_CBAR)) {
@@ -4233,17 +5240,7 @@ uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
 
 uint32_t HELPER(rbit)(uint32_t x)
 {
-    x =  ((x & 0xff000000) >> 24)
-       | ((x & 0x00ff0000) >> 8)
-       | ((x & 0x0000ff00) << 8)
-       | ((x & 0x000000ff) << 24);
-    x =  ((x & 0xf0f0f0f0) >> 4)
-       | ((x & 0x0f0f0f0f) << 4);
-    x =  ((x & 0x88888888) >> 3)
-       | ((x & 0x44444444) >> 1)
-       | ((x & 0x22222222) << 1)
-       | ((x & 0x11111111) << 3);
-    return x;
+    return revbit32(x);
 }
 
 #if defined(CONFIG_USER_ONLY)
@@ -4307,23 +5304,23 @@ int bank_number(int mode)
     switch (mode) {
     case ARM_CPU_MODE_USR:
     case ARM_CPU_MODE_SYS:
-        return 0;
+        return BANK_USRSYS;
     case ARM_CPU_MODE_SVC:
-        return 1;
+        return BANK_SVC;
     case ARM_CPU_MODE_ABT:
-        return 2;
+        return BANK_ABT;
     case ARM_CPU_MODE_UND:
-        return 3;
+        return BANK_UND;
     case ARM_CPU_MODE_IRQ:
-        return 4;
+        return BANK_IRQ;
     case ARM_CPU_MODE_FIQ:
-        return 5;
+        return BANK_FIQ;
     case ARM_CPU_MODE_HYP:
-        return 6;
+        return BANK_HYP;
     case ARM_CPU_MODE_MON:
-        return 7;
+        return BANK_MON;
     }
-    hw_error("bank number requested for bad CPSR mode value 0x%x\n", mode);
+    g_assert_not_reached();
 }
 
 void switch_mode(CPUARMState *env, int mode)
@@ -4391,7 +5388,7 @@ void switch_mode(CPUARMState *env, int mode)
  *        BIT IRQ     IMO      Non-secure         Secure
  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
  */
-const int8_t target_el_table[2][2][2][2][2][4] = {
+static const int8_t target_el_table[2][2][2][2][2][4] = {
     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
@@ -4417,11 +5414,22 @@ uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
                                  uint32_t cur_el, bool secure)
 {
     CPUARMState *env = cs->env_ptr;
-    int rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
+    int rw;
     int scr;
     int hcr;
     int target_el;
-    int is64 = arm_el_is_aa64(env, 3);
+    /* Is the highest EL AArch64? */
+    int is64 = arm_feature(env, ARM_FEATURE_AARCH64);
+
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
+    } else {
+        /* Either EL2 is the highest EL (and so the EL2 register width
+         * is given by is64); or there is no EL2 or EL3, in which case
+         * the value of 'rw' does not affect the table lookup anyway.
+         */
+        rw = is64;
+    }
 
     switch (excp_idx) {
     case EXCP_IRQ:
@@ -4561,8 +5569,10 @@ void arm_v7m_cpu_do_interrupt(CPUState *cs)
             nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
             if (nr == 0xab) {
                 env->regs[15] += 2;
+                qemu_log_mask(CPU_LOG_INT,
+                              "...handling as semihosting call 0x%x\n",
+                              env->regs[0]);
                 env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
                 return;
             }
         }
@@ -4655,35 +5665,35 @@ void aarch64_sync_32_to_64(CPUARMState *env)
     }
 
     if (mode == ARM_CPU_MODE_IRQ) {
-        env->xregs[16] = env->regs[13];
-        env->xregs[17] = env->regs[14];
+        env->xregs[16] = env->regs[14];
+        env->xregs[17] = env->regs[13];
     } else {
-        env->xregs[16] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
-        env->xregs[17] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
+        env->xregs[16] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
+        env->xregs[17] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
     }
 
     if (mode == ARM_CPU_MODE_SVC) {
-        env->xregs[18] = env->regs[13];
-        env->xregs[19] = env->regs[14];
+        env->xregs[18] = env->regs[14];
+        env->xregs[19] = env->regs[13];
     } else {
-        env->xregs[18] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
-        env->xregs[19] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
+        env->xregs[18] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
+        env->xregs[19] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
     }
 
     if (mode == ARM_CPU_MODE_ABT) {
-        env->xregs[20] = env->regs[13];
-        env->xregs[21] = env->regs[14];
+        env->xregs[20] = env->regs[14];
+        env->xregs[21] = env->regs[13];
     } else {
-        env->xregs[20] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
-        env->xregs[21] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
+        env->xregs[20] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
+        env->xregs[21] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
     }
 
     if (mode == ARM_CPU_MODE_UND) {
-        env->xregs[22] = env->regs[13];
-        env->xregs[23] = env->regs[14];
+        env->xregs[22] = env->regs[14];
+        env->xregs[23] = env->regs[13];
     } else {
-        env->xregs[22] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
-        env->xregs[23] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
+        env->xregs[22] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
+        env->xregs[23] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
     }
 
     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
@@ -4760,35 +5770,35 @@ void aarch64_sync_64_to_32(CPUARMState *env)
     }
 
     if (mode == ARM_CPU_MODE_IRQ) {
-        env->regs[13] = env->xregs[16];
-        env->regs[14] = env->xregs[17];
+        env->regs[14] = env->xregs[16];
+        env->regs[13] = env->xregs[17];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
-        env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
+        env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
+        env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
     }
 
     if (mode == ARM_CPU_MODE_SVC) {
-        env->regs[13] = env->xregs[18];
-        env->regs[14] = env->xregs[19];
+        env->regs[14] = env->xregs[18];
+        env->regs[13] = env->xregs[19];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
-        env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
+        env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
+        env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
     }
 
     if (mode == ARM_CPU_MODE_ABT) {
-        env->regs[13] = env->xregs[20];
-        env->regs[14] = env->xregs[21];
+        env->regs[14] = env->xregs[20];
+        env->regs[13] = env->xregs[21];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
-        env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
+        env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
+        env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
     }
 
     if (mode == ARM_CPU_MODE_UND) {
-        env->regs[13] = env->xregs[22];
-        env->regs[14] = env->xregs[23];
+        env->regs[14] = env->xregs[22];
+        env->regs[13] = env->xregs[23];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
-        env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
+        env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
+        env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
     }
 
     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
@@ -4882,8 +5892,10 @@ void arm_cpu_do_interrupt(CPUState *cs)
             if (((mask == 0x123456 && !env->thumb)
                     || (mask == 0xab && env->thumb))
                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
+                qemu_log_mask(CPU_LOG_INT,
+                              "...handling as semihosting call 0x%x\n",
+                              env->regs[0]);
                 env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
                 return;
             }
         }
@@ -4900,8 +5912,10 @@ void arm_cpu_do_interrupt(CPUState *cs)
             if (mask == 0xab
                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
                 env->regs[15] += 2;
+                qemu_log_mask(CPU_LOG_INT,
+                              "...handling as semihosting call 0x%x\n",
+                              env->regs[0]);
                 env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
                 return;
             }
         }
@@ -5062,8 +6076,7 @@ static inline bool regime_translation_disabled(CPUARMState *env,
 static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx)
 {
     if (mmu_idx == ARMMMUIdx_S2NS) {
-        /* TODO: return VTCR_EL2 */
-        g_assert_not_reached();
+        return &env->cp15.vtcr_el2;
     }
     return &env->cp15.tcr_el[regime_el(env, mmu_idx)];
 }
@@ -5073,8 +6086,7 @@ static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx,
                                    int ttbrn)
 {
     if (mmu_idx == ARMMMUIdx_S2NS) {
-        /* TODO: return VTTBR_EL2 */
-        g_assert_not_reached();
+        return env->cp15.vttbr_el2;
     }
     if (ttbrn == 0) {
         return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
@@ -5196,6 +6208,28 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
     return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
 }
 
+/* Translate S2 section/page access permissions to protection flags
+ *
+ * @env:     CPUARMState
+ * @s2ap:    The 2-bit stage2 access permissions (S2AP)
+ * @xn:      XN (execute-never) bit
+ */
+static int get_S2prot(CPUARMState *env, int s2ap, int xn)
+{
+    int prot = 0;
+
+    if (s2ap & 1) {
+        prot |= PAGE_READ;
+    }
+    if (s2ap & 2) {
+        prot |= PAGE_WRITE;
+    }
+    if (!xn) {
+        prot |= PAGE_EXEC;
+    }
+    return prot;
+}
+
 /* Translate section/page access permissions to protection flags
  *
  * @env:     CPUARMState
@@ -5300,6 +6334,32 @@ static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
     return true;
 }
 
+/* Translate a S1 pagetable walk through S2 if needed.  */
+static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx,
+                               hwaddr addr, MemTxAttrs txattrs,
+                               uint32_t *fsr,
+                               ARMMMUFaultInfo *fi)
+{
+    if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) &&
+        !regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
+        target_ulong s2size;
+        hwaddr s2pa;
+        int s2prot;
+        int ret;
+
+        ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa,
+                                 &txattrs, &s2prot, &s2size, fsr, fi);
+        if (ret) {
+            fi->s2addr = addr;
+            fi->stage2 = true;
+            fi->s1ptw = true;
+            return ~0;
+        }
+        addr = s2pa;
+    }
+    return addr;
+}
+
 /* All loads done in the course of a page table walk go through here.
  * TODO: rather than ignoring errors from physical memory reads (which
  * are external aborts in ARM terminology) we should propagate this
@@ -5307,26 +6367,43 @@ static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
  * was being done for a CPU load/store or an address translation instruction
  * (but not if it was for a debug access).
  */
-static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure)
+static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure,
+                            ARMMMUIdx mmu_idx, uint32_t *fsr,
+                            ARMMMUFaultInfo *fi)
 {
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
     MemTxAttrs attrs = {};
 
     attrs.secure = is_secure;
+    addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
+    if (fi->s1ptw) {
+        return 0;
+    }
     return address_space_ldl(cs->as, addr, attrs, NULL);
 }
 
-static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure)
+static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure,
+                            ARMMMUIdx mmu_idx, uint32_t *fsr,
+                            ARMMMUFaultInfo *fi)
 {
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
     MemTxAttrs attrs = {};
 
     attrs.secure = is_secure;
+    addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
+    if (fi->s1ptw) {
+        return 0;
+    }
     return address_space_ldq(cs->as, addr, attrs, NULL);
 }
 
 static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
                              int access_type, ARMMMUIdx mmu_idx,
                              hwaddr *phys_ptr, int *prot,
-                             target_ulong *page_size, uint32_t *fsr)
+                             target_ulong *page_size, uint32_t *fsr,
+                             ARMMMUFaultInfo *fi)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int code;
@@ -5346,7 +6423,8 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
         code = 5;
         goto do_fault;
     }
-    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                       mmu_idx, fsr, fi);
     type = (desc & 3);
     domain = (desc >> 5) & 0x0f;
     if (regime_el(env, mmu_idx) == 1) {
@@ -5382,7 +6460,8 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
             /* Fine pagetable.  */
             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
         }
-        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                           mmu_idx, fsr, fi);
         switch (desc & 3) {
         case 0: /* Page translation fault.  */
             code = 7;
@@ -5439,7 +6518,8 @@ do_fault:
 static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
                              int access_type, ARMMMUIdx mmu_idx,
                              hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
-                             target_ulong *page_size, uint32_t *fsr)
+                             target_ulong *page_size, uint32_t *fsr,
+                             ARMMMUFaultInfo *fi)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int code;
@@ -5462,7 +6542,8 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
         code = 5;
         goto do_fault;
     }
-    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                       mmu_idx, fsr, fi);
     type = (desc & 3);
     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
         /* Section translation fault, or attempt to use the encoding
@@ -5513,7 +6594,8 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
         ns = extract32(desc, 3, 1);
         /* Lookup l2 entry.  */
         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
-        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                           mmu_idx, fsr, fi);
         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
         switch (desc & 3) {
         case 0: /* Page translation fault.  */
@@ -5587,17 +6669,78 @@ typedef enum {
     permission_fault = 3,
 } MMUFaultType;
 
+/*
+ * check_s2_startlevel
+ * @cpu:        ARMCPU
+ * @is_aa64:    True if the translation regime is in AArch64 state
+ * @startlevel: Suggested starting level
+ * @inputsize:  Bitsize of IPAs
+ * @stride:     Page-table stride (See the ARM ARM)
+ *
+ * Returns true if the suggested starting level is OK and false otherwise.
+ */
+static bool check_s2_startlevel(ARMCPU *cpu, bool is_aa64, int level,
+                                int inputsize, int stride)
+{
+    /* Negative levels are never allowed.  */
+    if (level < 0) {
+        return false;
+    }
+
+    if (is_aa64) {
+        unsigned int pamax = arm_pamax(cpu);
+
+        switch (stride) {
+        case 13: /* 64KB Pages.  */
+            if (level == 0 || (level == 1 && pamax <= 42)) {
+                return false;
+            }
+            break;
+        case 11: /* 16KB Pages.  */
+            if (level == 0 || (level == 1 && pamax <= 40)) {
+                return false;
+            }
+            break;
+        case 9: /* 4KB Pages.  */
+            if (level == 0 && pamax <= 42) {
+                return false;
+            }
+            break;
+        default:
+            g_assert_not_reached();
+        }
+    } else {
+        const int grainsize = stride + 3;
+        int startsizecheck;
+
+        /* AArch32 only supports 4KB pages. Assert on that.  */
+        assert(stride == 9);
+
+        if (level == 0) {
+            return false;
+        }
+
+        startsizecheck = inputsize - ((3 - level) * stride + grainsize);
+        if (startsizecheck < 1 || startsizecheck > stride + 4) {
+            return false;
+        }
+    }
+    return true;
+}
+
 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
                                int access_type, ARMMMUIdx mmu_idx,
                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
-                               target_ulong *page_size_ptr, uint32_t *fsr)
+                               target_ulong *page_size_ptr, uint32_t *fsr,
+                               ARMMMUFaultInfo *fi)
 {
-    CPUState *cs = CPU(arm_env_get_cpu(env));
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     /* Read an LPAE long-descriptor translation table. */
     MMUFaultType fault_type = translation_fault;
     uint32_t level = 1;
-    uint32_t epd;
-    int32_t tsz;
+    uint32_t epd = 0;
+    int32_t t0sz, t1sz;
     uint32_t tg;
     uint64_t ttbr;
     int ttbr_select;
@@ -5605,13 +6748,15 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
     uint32_t tableattrs;
     target_ulong page_size;
     uint32_t attrs;
-    int32_t granule_sz = 9;
+    int32_t stride = 9;
     int32_t va_size = 32;
+    int inputsize;
     int32_t tbi = 0;
     TCR *tcr = regime_tcr(env, mmu_idx);
     int ap, ns, xn, pxn;
     uint32_t el = regime_el(env, mmu_idx);
     bool ttbr1_valid = true;
+    uint64_t descaddrmask;
 
     /* TODO:
      * This code does not handle the different format TCR for VTCR_EL2.
@@ -5622,7 +6767,9 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
     if (arm_el_is_aa64(env, el)) {
         va_size = 64;
         if (el > 1) {
-            tbi = extract64(tcr->raw_tcr, 20, 1);
+            if (mmu_idx != ARMMMUIdx_S2NS) {
+                tbi = extract64(tcr->raw_tcr, 20, 1);
+            }
         } else {
             if (extract64(address, 55, 1)) {
                 tbi = extract64(tcr->raw_tcr, 38, 1);
@@ -5638,6 +6785,11 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
         if (el > 1) {
             ttbr1_valid = false;
         }
+    } else {
+        /* There is no TTBR1 for EL2 */
+        if (el == 2) {
+            ttbr1_valid = false;
+        }
     }
 
     /* Determine whether this address is in the region controlled by
@@ -5645,12 +6797,28 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
      */
-    uint32_t t0sz = extract32(tcr->raw_tcr, 0, 6);
     if (va_size == 64) {
+        /* AArch64 translation.  */
+        t0sz = extract32(tcr->raw_tcr, 0, 6);
         t0sz = MIN(t0sz, 39);
         t0sz = MAX(t0sz, 16);
+    } else if (mmu_idx != ARMMMUIdx_S2NS) {
+        /* AArch32 stage 1 translation.  */
+        t0sz = extract32(tcr->raw_tcr, 0, 3);
+    } else {
+        /* AArch32 stage 2 translation.  */
+        bool sext = extract32(tcr->raw_tcr, 4, 1);
+        bool sign = extract32(tcr->raw_tcr, 3, 1);
+        t0sz = sextract32(tcr->raw_tcr, 0, 4);
+
+        /* If the sign-extend bit is not the same as t0sz[3], the result
+         * is unpredictable. Flag this as a guest error.  */
+        if (sign != sext) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "AArch32: VTCR.S / VTCR.T0SZ[3] missmatch\n");
+        }
     }
-    uint32_t t1sz = extract32(tcr->raw_tcr, 16, 6);
+    t1sz = extract32(tcr->raw_tcr, 16, 6);
     if (va_size == 64) {
         t1sz = MIN(t1sz, 39);
         t1sz = MAX(t1sz, 16);
@@ -5683,15 +6851,17 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
      */
     if (ttbr_select == 0) {
         ttbr = regime_ttbr(env, mmu_idx, 0);
-        epd = extract32(tcr->raw_tcr, 7, 1);
-        tsz = t0sz;
+        if (el < 2) {
+            epd = extract32(tcr->raw_tcr, 7, 1);
+        }
+        inputsize = va_size - t0sz;
 
         tg = extract32(tcr->raw_tcr, 14, 2);
         if (tg == 1) { /* 64KB pages */
-            granule_sz = 13;
+            stride = 13;
         }
         if (tg == 2) { /* 16KB pages */
-            granule_sz = 11;
+            stride = 11;
         }
     } else {
         /* We should only be here if TTBR1 is valid */
@@ -5699,19 +6869,19 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
 
         ttbr = regime_ttbr(env, mmu_idx, 1);
         epd = extract32(tcr->raw_tcr, 23, 1);
-        tsz = t1sz;
+        inputsize = va_size - t1sz;
 
         tg = extract32(tcr->raw_tcr, 30, 2);
         if (tg == 3)  { /* 64KB pages */
-            granule_sz = 13;
+            stride = 13;
         }
         if (tg == 1) { /* 16KB pages */
-            granule_sz = 11;
+            stride = 11;
         }
     }
 
     /* Here we should have set up all the parameters for the translation:
-     * va_size, ttbr, epd, tsz, granule_sz, tbi
+     * va_size, inputsize, ttbr, epd, stride, tbi
      */
 
     if (epd) {
@@ -5721,32 +6891,69 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
         goto do_fault;
     }
 
-    /* The starting level depends on the virtual address size (which can be
-     * up to 48 bits) and the translation granule size. It indicates the number
-     * of strides (granule_sz bits at a time) needed to consume the bits
-     * of the input address. In the pseudocode this is:
-     *  level = 4 - RoundUp((inputsize - grainsize) / stride)
-     * where their 'inputsize' is our 'va_size - tsz', 'grainsize' is
-     * our 'granule_sz + 3' and 'stride' is our 'granule_sz'.
-     * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
-     *     = 4 - (va_size - tsz - granule_sz - 3 + granule_sz - 1) / granule_sz
-     *     = 4 - (va_size - tsz - 4) / granule_sz;
-     */
-    level = 4 - (va_size - tsz - 4) / granule_sz;
+    if (mmu_idx != ARMMMUIdx_S2NS) {
+        /* The starting level depends on the virtual address size (which can
+         * be up to 48 bits) and the translation granule size. It indicates
+         * the number of strides (stride bits at a time) needed to
+         * consume the bits of the input address. In the pseudocode this is:
+         *  level = 4 - RoundUp((inputsize - grainsize) / stride)
+         * where their 'inputsize' is our 'inputsize', 'grainsize' is
+         * our 'stride + 3' and 'stride' is our 'stride'.
+         * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
+         * = 4 - (inputsize - stride - 3 + stride - 1) / stride
+         * = 4 - (inputsize - 4) / stride;
+         */
+        level = 4 - (inputsize - 4) / stride;
+    } else {
+        /* For stage 2 translations the starting level is specified by the
+         * VTCR_EL2.SL0 field (whose interpretation depends on the page size)
+         */
+        int startlevel = extract32(tcr->raw_tcr, 6, 2);
+        bool ok;
+
+        if (va_size == 32 || stride == 9) {
+            /* AArch32 or 4KB pages */
+            level = 2 - startlevel;
+        } else {
+            /* 16KB or 64KB pages */
+            level = 3 - startlevel;
+        }
+
+        /* Check that the starting level is valid. */
+        ok = check_s2_startlevel(cpu, va_size == 64, level,
+                                 inputsize, stride);
+        if (!ok) {
+            /* AArch64 reports these as level 0 faults.
+             * AArch32 reports these as level 1 faults.
+             */
+            level = va_size == 64 ? 0 : 1;
+            fault_type = translation_fault;
+            goto do_fault;
+        }
+    }
 
     /* Clear the vaddr bits which aren't part of the within-region address,
      * so that we don't have to special case things when calculating the
      * first descriptor address.
      */
-    if (tsz) {
-        address &= (1ULL << (va_size - tsz)) - 1;
+    if (va_size != inputsize) {
+        address &= (1ULL << inputsize) - 1;
     }
 
-    descmask = (1ULL << (granule_sz + 3)) - 1;
+    descmask = (1ULL << (stride + 3)) - 1;
 
     /* Now we can extract the actual base address from the TTBR */
     descaddr = extract64(ttbr, 0, 48);
-    descaddr &= ~((1ULL << (va_size - tsz - (granule_sz * (4 - level)))) - 1);
+    descaddr &= ~((1ULL << (inputsize - (stride * (4 - level)))) - 1);
+
+    /* The address field in the descriptor goes up to bit 39 for ARMv7
+     * but up to bit 47 for ARMv8.
+     */
+    if (arm_feature(env, ARM_FEATURE_V8)) {
+        descaddrmask = 0xfffffffff000ULL;
+    } else {
+        descaddrmask = 0xfffffff000ULL;
+    }
 
     /* Secure accesses start with the page table in secure memory and
      * can be downgraded to non-secure at any step. Non-secure accesses
@@ -5758,16 +6965,20 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
         uint64_t descriptor;
         bool nstable;
 
-        descaddr |= (address >> (granule_sz * (4 - level))) & descmask;
+        descaddr |= (address >> (stride * (4 - level))) & descmask;
         descaddr &= ~7ULL;
         nstable = extract32(tableattrs, 4, 1);
-        descriptor = arm_ldq_ptw(cs, descaddr, !nstable);
+        descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fsr, fi);
+        if (fi->s1ptw) {
+            goto do_fault;
+        }
+
         if (!(descriptor & 1) ||
             (!(descriptor & 2) && (level == 3))) {
             /* Invalid, or the Reserved level 3 encoding */
             goto do_fault;
         }
-        descaddr = descriptor & 0xfffffff000ULL;
+        descaddr = descriptor & descaddrmask;
 
         if ((descriptor & 2) && (level < 3)) {
             /* Table entry. The top five bits are attributes which  may
@@ -5783,11 +6994,17 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
          * These are basically the same thing, although the number
          * of bits we pull in from the vaddr varies.
          */
-        page_size = (1ULL << ((granule_sz * (4 - level)) + 3));
+        page_size = (1ULL << ((stride * (4 - level)) + 3));
         descaddr |= (address & (page_size - 1));
-        /* Extract attributes from the descriptor and merge with table attrs */
+        /* Extract attributes from the descriptor */
         attrs = extract64(descriptor, 2, 10)
             | (extract64(descriptor, 52, 12) << 10);
+
+        if (mmu_idx == ARMMMUIdx_S2NS) {
+            /* Stage 2 table descriptors do not include any attribute fields */
+            break;
+        }
+        /* Merge in attributes from table descriptors */
         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
@@ -5809,11 +7026,16 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
     }
 
     ap = extract32(attrs, 4, 2);
-    ns = extract32(attrs, 3, 1);
     xn = extract32(attrs, 12, 1);
-    pxn = extract32(attrs, 11, 1);
 
-    *prot = get_S1prot(env, mmu_idx, va_size == 64, ap, ns, xn, pxn);
+    if (mmu_idx == ARMMMUIdx_S2NS) {
+        ns = true;
+        *prot = get_S2prot(env, ap, xn);
+    } else {
+        ns = extract32(attrs, 3, 1);
+        pxn = extract32(attrs, 11, 1);
+        *prot = get_S1prot(env, mmu_idx, va_size == 64, ap, ns, xn, pxn);
+    }
 
     fault_type = permission_fault;
     if (!(*prot & (1 << access_type))) {
@@ -5834,6 +7056,8 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
 do_fault:
     /* Long-descriptor format IFSR/DFSR value */
     *fsr = (1 << 9) | (fault_type << 2) | level;
+    /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2.  */
+    fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS);
     return true;
 }
 
@@ -6096,20 +7320,45 @@ static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address,
  * @page_size: set to the size of the page containing phys_ptr
  * @fsr: set to the DFSR/IFSR value on failure
  */
-static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
-                                 int access_type, ARMMMUIdx mmu_idx,
-                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
-                                 target_ulong *page_size, uint32_t *fsr)
+static bool get_phys_addr(CPUARMState *env, target_ulong address,
+                          int access_type, ARMMMUIdx mmu_idx,
+                          hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
+                          target_ulong *page_size, uint32_t *fsr,
+                          ARMMMUFaultInfo *fi)
 {
     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
-        /* TODO: when we support EL2 we should here call ourselves recursively
-         * to do the stage 1 and then stage 2 translations. The arm_ld*_ptw
-         * functions will also need changing to perform ARMMMUIdx_S2NS loads
-         * rather than direct physical memory loads when appropriate.
-         * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
+        /* Call ourselves recursively to do the stage 1 and then stage 2
+         * translations.
          */
-        assert(!arm_feature(env, ARM_FEATURE_EL2));
-        mmu_idx += ARMMMUIdx_S1NSE0;
+        if (arm_feature(env, ARM_FEATURE_EL2)) {
+            hwaddr ipa;
+            int s2_prot;
+            int ret;
+
+            ret = get_phys_addr(env, address, access_type,
+                                mmu_idx + ARMMMUIdx_S1NSE0, &ipa, attrs,
+                                prot, page_size, fsr, fi);
+
+            /* If S1 fails or S2 is disabled, return early.  */
+            if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
+                *phys_ptr = ipa;
+                return ret;
+            }
+
+            /* S1 is done. Now do S2 translation.  */
+            ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS,
+                                     phys_ptr, attrs, &s2_prot,
+                                     page_size, fsr, fi);
+            fi->s2addr = ipa;
+            /* Combine the S1 and S2 perms.  */
+            *prot &= s2_prot;
+            return ret;
+        } else {
+            /*
+             * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
+             */
+            mmu_idx += ARMMMUIdx_S1NSE0;
+        }
     }
 
     /* The page table entries may downgrade secure to non-secure, but
@@ -6158,13 +7407,13 @@ static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
 
     if (regime_using_lpae_format(env, mmu_idx)) {
         return get_phys_addr_lpae(env, address, access_type, mmu_idx, phys_ptr,
-                                  attrs, prot, page_size, fsr);
+                                  attrs, prot, page_size, fsr, fi);
     } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
         return get_phys_addr_v6(env, address, access_type, mmu_idx, phys_ptr,
-                                attrs, prot, page_size, fsr);
+                                attrs, prot, page_size, fsr, fi);
     } else {
         return get_phys_addr_v5(env, address, access_type, mmu_idx, phys_ptr,
-                                prot, page_size, fsr);
+                                prot, page_size, fsr, fi);
     }
 }
 
@@ -6173,7 +7422,8 @@ static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
  * fsr with ARM DFSR/IFSR fault register format value on failure.
  */
 bool arm_tlb_fill(CPUState *cs, vaddr address,
-                  int access_type, int mmu_idx, uint32_t *fsr)
+                  int access_type, int mmu_idx, uint32_t *fsr,
+                  ARMMMUFaultInfo *fi)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
@@ -6184,7 +7434,7 @@ bool arm_tlb_fill(CPUState *cs, vaddr address,
     MemTxAttrs attrs = {};
 
     ret = get_phys_addr(env, address, access_type, mmu_idx, &phys_addr,
-                        &attrs, &prot, &page_size, fsr);
+                        &attrs, &prot, &page_size, fsr, fi);
     if (!ret) {
         /* Map a single [sub]page.  */
         phys_addr &= TARGET_PAGE_MASK;
@@ -6207,9 +7457,10 @@ hwaddr arm_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     bool ret;
     uint32_t fsr;
     MemTxAttrs attrs = {};
+    ARMMMUFaultInfo fi = {};
 
-    ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env), &phys_addr,
-                        &attrs, &prot, &page_size, &fsr);
+    ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env, false), &phys_addr,
+                        &attrs, &prot, &page_size, &fsr, &fi);
 
     if (ret) {
         return -1;
@@ -6373,7 +7624,7 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
         void *hostaddr[maxidx];
         int try, i;
-        unsigned mmu_idx = cpu_mmu_index(env);
+        unsigned mmu_idx = cpu_mmu_index(env, false);
         TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
 
         for (try = 0; try < 2; try++) {
diff --git a/target-arm/helper.h b/target-arm/helper.h
index 827b33dfe..c2a85c722 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -54,6 +54,8 @@ DEF_HELPER_1(yield, void, env)
 DEF_HELPER_1(pre_hvc, void, env)
 DEF_HELPER_2(pre_smc, void, env, i32)
 
+DEF_HELPER_1(check_breakpoints, void, env)
+
 DEF_HELPER_3(cpsr_write, void, env, i32, i32)
 DEF_HELPER_1(cpsr_read, i32, env)
 
diff --git a/target-arm/internals.h b/target-arm/internals.h
index 924aff9d0..347998c8b 100644
--- a/target-arm/internals.h
+++ b/target-arm/internals.h
@@ -25,6 +25,16 @@
 #ifndef TARGET_ARM_INTERNALS_H
 #define TARGET_ARM_INTERNALS_H
 
+/* register banks for CPU modes */
+#define BANK_USRSYS 0
+#define BANK_SVC    1
+#define BANK_ABT    2
+#define BANK_UND    3
+#define BANK_IRQ    4
+#define BANK_FIQ    5
+#define BANK_HYP    6
+#define BANK_MON    7
+
 static inline bool excp_is_internal(int excp)
 {
     /* Return true if this exception number represents a QEMU-internal
@@ -36,6 +46,7 @@ static inline bool excp_is_internal(int excp)
         || excp == EXCP_HALTED
         || excp == EXCP_EXCEPTION_EXIT
         || excp == EXCP_KERNEL_TRAP
+        || excp == EXCP_SEMIHOST
         || excp == EXCP_STREX;
 }
 
@@ -58,6 +69,7 @@ static const char * const excnames[] = {
     [EXCP_SMC] = "Secure Monitor Call",
     [EXCP_VIRQ] = "Virtual IRQ",
     [EXCP_VFIQ] = "Virtual FIQ",
+    [EXCP_SEMIHOST] = "Semihosting call",
 };
 
 static inline void arm_log_exception(int idx)
@@ -89,9 +101,9 @@ static inline void arm_log_exception(int idx)
 static inline unsigned int aarch64_banked_spsr_index(unsigned int el)
 {
     static const unsigned int map[4] = {
-        [1] = 1, /* EL1.  */
-        [2] = 6, /* EL2.  */
-        [3] = 7, /* EL3.  */
+        [1] = BANK_SVC, /* EL1.  */
+        [2] = BANK_HYP, /* EL2.  */
+        [3] = BANK_MON, /* EL3.  */
     };
     assert(el >= 1 && el <= 3);
     return map[el];
@@ -150,6 +162,31 @@ static inline void update_spsel(CPUARMState *env, uint32_t imm)
     aarch64_restore_sp(env, cur_el);
 }
 
+/*
+ * arm_pamax
+ * @cpu: ARMCPU
+ *
+ * Returns the implementation defined bit-width of physical addresses.
+ * The ARMv8 reference manuals refer to this as PAMax().
+ */
+static inline unsigned int arm_pamax(ARMCPU *cpu)
+{
+    static const unsigned int pamax_map[] = {
+        [0] = 32,
+        [1] = 36,
+        [2] = 40,
+        [3] = 42,
+        [4] = 44,
+        [5] = 48,
+    };
+    unsigned int parange = extract32(cpu->id_aa64mmfr0, 0, 4);
+
+    /* id_aa64mmfr0 is a read-only register so values outside of the
+     * supported mappings can be considered an implementation error.  */
+    assert(parange < ARRAY_SIZE(pamax_map));
+    return pamax_map[parange];
+}
+
 /* Return true if extended addresses are enabled.
  * This is always the case if our translation regime is 64 bit,
  * but depends on TTBCR.EAE for 32 bit.
@@ -387,8 +424,21 @@ bool arm_is_psci_call(ARMCPU *cpu, int excp_type);
 void arm_handle_psci_call(ARMCPU *cpu);
 #endif
 
+/**
+ * ARMMMUFaultInfo: Information describing an ARM MMU Fault
+ * @s2addr: Address that caused a fault at stage 2
+ * @stage2: True if we faulted at stage 2
+ * @s1ptw: True if we faulted at stage 2 while doing a stage 1 page-table walk
+ */
+typedef struct ARMMMUFaultInfo ARMMMUFaultInfo;
+struct ARMMMUFaultInfo {
+    target_ulong s2addr;
+    bool stage2;
+    bool s1ptw;
+};
+
 /* Do a page table walk and add page to TLB if possible */
 bool arm_tlb_fill(CPUState *cpu, vaddr address, int rw, int mmu_idx,
-                  uint32_t *fsr);
+                  uint32_t *fsr, ARMMMUFaultInfo *fi);
 
 #endif
diff --git a/target-arm/kvm.c b/target-arm/kvm.c
index b27854208..79ef4c61d 100644
--- a/target-arm/kvm.c
+++ b/target-arm/kvm.c
@@ -585,22 +585,27 @@ void kvm_arch_init_irq_routing(KVMState *s)
 
 int kvm_arch_irqchip_create(KVMState *s)
 {
-    int ret;
-
     /* If we can create the VGIC using the newer device control API, we
      * let the device do this when it initializes itself, otherwise we
      * fall back to the old API */
+    return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
+}
 
-    ret = kvm_create_device(s, KVM_DEV_TYPE_ARM_VGIC_V2, true);
-    if (ret == 0) {
-        return 1;
+int kvm_arm_vgic_probe(void)
+{
+    if (kvm_create_device(kvm_state,
+                          KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
+        return 3;
+    } else if (kvm_create_device(kvm_state,
+                                 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
+        return 2;
+    } else {
+        return 0;
     }
-
-    return 0;
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
diff --git a/target-arm/kvm32.c b/target-arm/kvm32.c
index 421ce0ea0..df1e2b0eb 100644
--- a/target-arm/kvm32.c
+++ b/target-arm/kvm32.c
@@ -181,7 +181,6 @@ int kvm_arm_cpreg_level(uint64_t regidx)
     return KVM_PUT_RUNTIME_STATE;
 }
 
-#define ARM_MPIDR_HWID_BITMASK 0xFFFFFF
 #define ARM_CPU_ID_MPIDR       0, 0, 0, 5
 
 int kvm_arch_init_vcpu(CPUState *cs)
@@ -234,7 +233,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
     if (ret) {
         return ret;
     }
-    cpu->mp_affinity = mpidr & ARM_MPIDR_HWID_BITMASK;
+    cpu->mp_affinity = mpidr & ARM32_AFFINITY_MASK;
 
     return kvm_arm_init_cpreg_list(cpu);
 }
@@ -281,30 +280,30 @@ static const Reg regs[] = {
     COREREG(usr_regs.uregs[10], usr_regs[2]),
     COREREG(usr_regs.uregs[11], usr_regs[3]),
     COREREG(usr_regs.uregs[12], usr_regs[4]),
-    COREREG(usr_regs.uregs[13], banked_r13[0]),
-    COREREG(usr_regs.uregs[14], banked_r14[0]),
+    COREREG(usr_regs.uregs[13], banked_r13[BANK_USRSYS]),
+    COREREG(usr_regs.uregs[14], banked_r14[BANK_USRSYS]),
     /* R13, R14, SPSR for SVC, ABT, UND, IRQ banks */
-    COREREG(svc_regs[0], banked_r13[1]),
-    COREREG(svc_regs[1], banked_r14[1]),
-    COREREG64(svc_regs[2], banked_spsr[1]),
-    COREREG(abt_regs[0], banked_r13[2]),
-    COREREG(abt_regs[1], banked_r14[2]),
-    COREREG64(abt_regs[2], banked_spsr[2]),
-    COREREG(und_regs[0], banked_r13[3]),
-    COREREG(und_regs[1], banked_r14[3]),
-    COREREG64(und_regs[2], banked_spsr[3]),
-    COREREG(irq_regs[0], banked_r13[4]),
-    COREREG(irq_regs[1], banked_r14[4]),
-    COREREG64(irq_regs[2], banked_spsr[4]),
+    COREREG(svc_regs[0], banked_r13[BANK_SVC]),
+    COREREG(svc_regs[1], banked_r14[BANK_SVC]),
+    COREREG64(svc_regs[2], banked_spsr[BANK_SVC]),
+    COREREG(abt_regs[0], banked_r13[BANK_ABT]),
+    COREREG(abt_regs[1], banked_r14[BANK_ABT]),
+    COREREG64(abt_regs[2], banked_spsr[BANK_ABT]),
+    COREREG(und_regs[0], banked_r13[BANK_UND]),
+    COREREG(und_regs[1], banked_r14[BANK_UND]),
+    COREREG64(und_regs[2], banked_spsr[BANK_UND]),
+    COREREG(irq_regs[0], banked_r13[BANK_IRQ]),
+    COREREG(irq_regs[1], banked_r14[BANK_IRQ]),
+    COREREG64(irq_regs[2], banked_spsr[BANK_IRQ]),
     /* R8_fiq .. R14_fiq and SPSR_fiq */
     COREREG(fiq_regs[0], fiq_regs[0]),
     COREREG(fiq_regs[1], fiq_regs[1]),
     COREREG(fiq_regs[2], fiq_regs[2]),
     COREREG(fiq_regs[3], fiq_regs[3]),
     COREREG(fiq_regs[4], fiq_regs[4]),
-    COREREG(fiq_regs[5], banked_r13[5]),
-    COREREG(fiq_regs[6], banked_r14[5]),
-    COREREG64(fiq_regs[7], banked_spsr[5]),
+    COREREG(fiq_regs[5], banked_r13[BANK_FIQ]),
+    COREREG(fiq_regs[6], banked_r14[BANK_FIQ]),
+    COREREG64(fiq_regs[7], banked_spsr[BANK_FIQ]),
     /* R15 */
     COREREG(usr_regs.uregs[15], regs[15]),
     /* VFP system registers */
diff --git a/target-arm/kvm64.c b/target-arm/kvm64.c
index bd60889d1..ceebfeb77 100644
--- a/target-arm/kvm64.c
+++ b/target-arm/kvm64.c
@@ -77,7 +77,6 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUClass *ahcc)
     return true;
 }
 
-#define ARM_MPIDR_HWID_BITMASK 0xFF00FFFFFFULL
 #define ARM_CPU_ID_MPIDR       3, 0, 0, 0, 5
 
 int kvm_arch_init_vcpu(CPUState *cs)
@@ -120,7 +119,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
     if (ret) {
         return ret;
     }
-    cpu->mp_affinity = mpidr & ARM_MPIDR_HWID_BITMASK;
+    cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
 
     return kvm_arm_init_cpreg_list(cpu);
 }
diff --git a/target-arm/kvm_arm.h b/target-arm/kvm_arm.h
index 7912d7433..b51604177 100644
--- a/target-arm/kvm_arm.h
+++ b/target-arm/kvm_arm.h
@@ -189,6 +189,30 @@ int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu);
  */
 int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu);
 
+int kvm_arm_vgic_probe(void);
+
+#else
+
+static inline int kvm_arm_vgic_probe(void)
+{
+    return 0;
+}
+
 #endif
 
+static inline const char *gic_class_name(void)
+{
+    return kvm_irqchip_in_kernel() ? "kvm-arm-gic" : "arm_gic";
+}
+
+/**
+ * gicv3_class_name
+ *
+ * Return name of GICv3 class to use depending on whether KVM acceleration is
+ * in use. May throw an error if the chosen implementation is not available.
+ *
+ * Returns: class name to use
+ */
+const char *gicv3_class_name(void);
+
 #endif
diff --git a/target-arm/machine.c b/target-arm/machine.c
index 32adfe792..36a0d159c 100644
--- a/target-arm/machine.c
+++ b/target-arm/machine.c
@@ -1,5 +1,6 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "internals.h"
@@ -328,3 +329,20 @@ const VMStateDescription vmstate_arm_cpu = {
         NULL
     }
 };
+
+const char *gicv3_class_name(void)
+{
+    if (kvm_irqchip_in_kernel()) {
+#ifdef TARGET_AARCH64
+        return "kvm-arm-gicv3";
+#else
+        error_report("KVM GICv3 acceleration is not supported on this "
+                     "platform\n");
+#endif
+    } else {
+        /* TODO: Software emulation is not implemented yet */
+        error_report("KVM is currently required for GICv3 emulation\n");
+    }
+
+    exit(1);
+}
diff --git a/target-arm/op_helper.c b/target-arm/op_helper.c
index 663c05d1d..6cd54c8f7 100644
--- a/target-arm/op_helper.c
+++ b/target-arm/op_helper.c
@@ -83,19 +83,27 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
 {
     bool ret;
     uint32_t fsr = 0;
+    ARMMMUFaultInfo fi = {};
 
-    ret = arm_tlb_fill(cs, addr, is_write, mmu_idx, &fsr);
+    ret = arm_tlb_fill(cs, addr, is_write, mmu_idx, &fsr, &fi);
     if (unlikely(ret)) {
         ARMCPU *cpu = ARM_CPU(cs);
         CPUARMState *env = &cpu->env;
         uint32_t syn, exc;
-        bool same_el = (arm_current_el(env) != 0);
+        unsigned int target_el;
+        bool same_el;
 
         if (retaddr) {
             /* now we have a real cpu fault */
             cpu_restore_state(cs, retaddr);
         }
 
+        target_el = exception_target_el(env);
+        if (fi.stage2) {
+            target_el = 2;
+            env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
+        }
+        same_el = arm_current_el(env) == target_el;
         /* AArch64 syndrome does not have an LPAE bit */
         syn = fsr & ~(1 << 9);
 
@@ -103,10 +111,10 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
          * information; this is always true for exceptions reported to EL1.
          */
         if (is_write == 2) {
-            syn = syn_insn_abort(same_el, 0, 0, syn);
+            syn = syn_insn_abort(same_el, 0, fi.s1ptw, syn);
             exc = EXCP_PREFETCH_ABORT;
         } else {
-            syn = syn_data_abort(same_el, 0, 0, 0, is_write == 1, syn);
+            syn = syn_data_abort(same_el, 0, 0, fi.s1ptw, is_write == 1, syn);
             if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
                 fsr |= (1 << 11);
             }
@@ -115,7 +123,7 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
 
         env->exception.vaddress = addr;
         env->exception.fsr = fsr;
-        raise_exception(env, exc, syn, exception_target_el(env));
+        raise_exception(env, exc, syn, target_el);
     }
 }
 #endif
@@ -384,9 +392,9 @@ uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
     uint32_t val;
 
     if (regno == 13) {
-        val = env->banked_r13[0];
+        val = env->banked_r13[BANK_USRSYS];
     } else if (regno == 14) {
-        val = env->banked_r14[0];
+        val = env->banked_r14[BANK_USRSYS];
     } else if (regno >= 8
                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
         val = env->usr_regs[regno - 8];
@@ -399,9 +407,9 @@ uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 {
     if (regno == 13) {
-        env->banked_r13[0] = val;
+        env->banked_r13[BANK_USRSYS] = val;
     } else if (regno == 14) {
-        env->banked_r14[0] = val;
+        env->banked_r14[BANK_USRSYS] = val;
     } else if (regno >= 8
                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
         env->usr_regs[regno - 8] = val;
@@ -444,6 +452,14 @@ void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome)
         target_el = exception_target_el(env);
         syndrome = syn_uncategorized();
         break;
+    case CP_ACCESS_TRAP_UNCATEGORIZED_EL2:
+        target_el = 2;
+        syndrome = syn_uncategorized();
+        break;
+    case CP_ACCESS_TRAP_UNCATEGORIZED_EL3:
+        target_el = 3;
+        syndrome = syn_uncategorized();
+        break;
     default:
         g_assert_not_reached();
     }
@@ -859,6 +875,15 @@ static bool check_breakpoints(ARMCPU *cpu)
     return false;
 }
 
+void HELPER(check_breakpoints)(CPUARMState *env)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+
+    if (check_breakpoints(cpu)) {
+        HELPER(exception_internal(env, EXCP_DEBUG));
+    }
+}
+
 void arm_debug_excp_handler(CPUState *cs)
 {
     /* Called by core code when a watchpoint or breakpoint fires;
@@ -889,18 +914,28 @@ void arm_debug_excp_handler(CPUState *cs)
             }
         }
     } else {
-        if (check_breakpoints(cpu)) {
-            bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
-            if (extended_addresses_enabled(env)) {
-                env->exception.fsr = (1 << 9) | 0x22;
-            } else {
-                env->exception.fsr = 0x2;
-            }
-            /* FAR is UNKNOWN, so doesn't need setting */
-            raise_exception(env, EXCP_PREFETCH_ABORT,
-                            syn_breakpoint(same_el),
-                            arm_debug_target_el(env));
+        uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
+        bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
+
+        /* (1) GDB breakpoints should be handled first.
+         * (2) Do not raise a CPU exception if no CPU breakpoint has fired,
+         * since singlestep is also done by generating a debug internal
+         * exception.
+         */
+        if (cpu_breakpoint_test(cs, pc, BP_GDB)
+            || !cpu_breakpoint_test(cs, pc, BP_CPU)) {
+            return;
+        }
+
+        if (extended_addresses_enabled(env)) {
+            env->exception.fsr = (1 << 9) | 0x22;
+        } else {
+            env->exception.fsr = 0x2;
         }
+        /* FAR is UNKNOWN, so doesn't need setting */
+        raise_exception(env, EXCP_PREFETCH_ABORT,
+                        syn_breakpoint(same_el),
+                        arm_debug_target_el(env));
     }
 }
 
diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c
index 0f923d36c..14e8131b0 100644
--- a/target-arm/translate-a64.c
+++ b/target-arm/translate-a64.c
@@ -30,6 +30,7 @@
 #include "internals.h"
 #include "qemu/host-utils.h"
 
+#include "exec/semihost.h"
 #include "exec/gen-icount.h"
 
 #include "exec/helper-proto.h"
@@ -125,6 +126,8 @@ void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
     CPUARMState *env = &cpu->env;
     uint32_t psr = pstate_read(env);
     int i;
+    int el = arm_current_el(env);
+    const char *ns_status;
 
     cpu_fprintf(f, "PC=%016"PRIx64"  SP=%016"PRIx64"\n",
             env->pc, env->xregs[31]);
@@ -136,13 +139,22 @@ void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
             cpu_fprintf(f, " ");
         }
     }
-    cpu_fprintf(f, "PSTATE=%08x (flags %c%c%c%c)\n",
+
+    if (arm_feature(env, ARM_FEATURE_EL3) && el != 3) {
+        ns_status = env->cp15.scr_el3 & SCR_NS ? "NS " : "S ";
+    } else {
+        ns_status = "";
+    }
+
+    cpu_fprintf(f, "\nPSTATE=%08x %c%c%c%c %sEL%d%c\n",
                 psr,
                 psr & PSTATE_N ? 'N' : '-',
                 psr & PSTATE_Z ? 'Z' : '-',
                 psr & PSTATE_C ? 'C' : '-',
-                psr & PSTATE_V ? 'V' : '-');
-    cpu_fprintf(f, "\n");
+                psr & PSTATE_V ? 'V' : '-',
+                ns_status,
+                el,
+                psr & PSTATE_SP ? 'h' : 't');
 
     if (flags & CPU_DUMP_FPU) {
         int numvfpregs = 32;
@@ -166,6 +178,31 @@ void gen_a64_set_pc_im(uint64_t val)
     tcg_gen_movi_i64(cpu_pc, val);
 }
 
+typedef struct DisasCompare64 {
+    TCGCond cond;
+    TCGv_i64 value;
+} DisasCompare64;
+
+static void a64_test_cc(DisasCompare64 *c64, int cc)
+{
+    DisasCompare c32;
+
+    arm_test_cc(&c32, cc);
+
+    /* Sign-extend the 32-bit value so that the GE/LT comparisons work
+       * properly.  The NE/EQ comparisons are also fine with this choice.  */
+    c64->cond = c32.cond;
+    c64->value = tcg_temp_new_i64();
+    tcg_gen_ext_i32_i64(c64->value, c32.value);
+
+    arm_free_cc(&c32);
+}
+
+static void a64_free_cc(DisasCompare64 *c64)
+{
+    tcg_temp_free_i64(c64->value);
+}
+
 static void gen_exception_internal(int excp)
 {
     TCGv_i32 tcg_excp = tcg_const_i32(excp);
@@ -503,13 +540,8 @@ static TCGv_ptr get_fpstatus_ptr(void)
  */
 static inline void gen_set_NZ64(TCGv_i64 result)
 {
-    TCGv_i64 flag = tcg_temp_new_i64();
-
-    tcg_gen_setcondi_i64(TCG_COND_NE, flag, result, 0);
-    tcg_gen_trunc_i64_i32(cpu_ZF, flag);
-    tcg_gen_shri_i64(flag, result, 32);
-    tcg_gen_trunc_i64_i32(cpu_NF, flag);
-    tcg_temp_free_i64(flag);
+    tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
+    tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
 }
 
 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
@@ -518,8 +550,8 @@ static inline void gen_logic_CC(int sf, TCGv_i64 result)
     if (sf) {
         gen_set_NZ64(result);
     } else {
-        tcg_gen_trunc_i64_i32(cpu_ZF, result);
-        tcg_gen_trunc_i64_i32(cpu_NF, result);
+        tcg_gen_extrl_i64_i32(cpu_ZF, result);
+        tcg_gen_mov_i32(cpu_NF, cpu_ZF);
     }
     tcg_gen_movi_i32(cpu_CF, 0);
     tcg_gen_movi_i32(cpu_VF, 0);
@@ -537,7 +569,7 @@ static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         tcg_gen_movi_i64(tmp, 0);
         tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
 
-        tcg_gen_trunc_i64_i32(cpu_CF, flag);
+        tcg_gen_extrl_i64_i32(cpu_CF, flag);
 
         gen_set_NZ64(result);
 
@@ -545,8 +577,7 @@ static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_andc_i64(flag, flag, tmp);
         tcg_temp_free_i64(tmp);
-        tcg_gen_shri_i64(flag, flag, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, flag);
+        tcg_gen_extrh_i64_i32(cpu_VF, flag);
 
         tcg_gen_mov_i64(dest, result);
         tcg_temp_free_i64(result);
@@ -558,8 +589,8 @@ static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         TCGv_i32 tmp = tcg_temp_new_i32();
 
         tcg_gen_movi_i32(tmp, 0);
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
         tcg_gen_mov_i32(cpu_ZF, cpu_NF);
         tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
@@ -587,15 +618,14 @@ static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         gen_set_NZ64(result);
 
         tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
-        tcg_gen_trunc_i64_i32(cpu_CF, flag);
+        tcg_gen_extrl_i64_i32(cpu_CF, flag);
 
         tcg_gen_xor_i64(flag, result, t0);
         tmp = tcg_temp_new_i64();
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_and_i64(flag, flag, tmp);
         tcg_temp_free_i64(tmp);
-        tcg_gen_shri_i64(flag, flag, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, flag);
+        tcg_gen_extrh_i64_i32(cpu_VF, flag);
         tcg_gen_mov_i64(dest, result);
         tcg_temp_free_i64(flag);
         tcg_temp_free_i64(result);
@@ -605,8 +635,8 @@ static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         TCGv_i32 t1_32 = tcg_temp_new_i32();
         TCGv_i32 tmp;
 
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
         tcg_gen_mov_i32(cpu_ZF, cpu_NF);
         tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
@@ -648,14 +678,13 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         tcg_gen_extu_i32_i64(cf_64, cpu_CF);
         tcg_gen_add2_i64(result, cf_64, t0, tmp, cf_64, tmp);
         tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, tmp);
-        tcg_gen_trunc_i64_i32(cpu_CF, cf_64);
+        tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
         gen_set_NZ64(result);
 
         tcg_gen_xor_i64(vf_64, result, t0);
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_andc_i64(vf_64, vf_64, tmp);
-        tcg_gen_shri_i64(vf_64, vf_64, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, vf_64);
+        tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
 
         tcg_gen_mov_i64(dest, result);
 
@@ -669,8 +698,8 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         t1_32 = tcg_temp_new_i32();
         tmp = tcg_const_i32(0);
 
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, cpu_CF, tmp);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, tmp);
 
@@ -1212,9 +1241,15 @@ static void handle_sync(DisasContext *s, uint32_t insn,
         return;
     case 4: /* DSB */
     case 5: /* DMB */
-    case 6: /* ISB */
         /* We don't emulate caches so barriers are no-ops */
         return;
+    case 6: /* ISB */
+        /* We need to break the TB after this insn to execute
+         * a self-modified code correctly and also to take
+         * any pending interrupts immediately.
+         */
+        s->is_jmp = DISAS_UPDATE;
+        return;
     default:
         unallocated_encoding(s);
         return;
@@ -1279,7 +1314,7 @@ static void gen_set_nzcv(TCGv_i64 tcg_rt)
     TCGv_i32 nzcv = tcg_temp_new_i32();
 
     /* take NZCV from R[t] */
-    tcg_gen_trunc_i64_i32(nzcv, tcg_rt);
+    tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
 
     /* bit 31, N */
     tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
@@ -1531,8 +1566,27 @@ static void disas_exc(DisasContext *s, uint32_t insn)
             unallocated_encoding(s);
             break;
         }
-        /* HLT */
-        unsupported_encoding(s, insn);
+        /* HLT. This has two purposes.
+         * Architecturally, it is an external halting debug instruction.
+         * Since QEMU doesn't implement external debug, we treat this as
+         * it is required for halting debug disabled: it will UNDEF.
+         * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
+         */
+        if (semihosting_enabled() && imm16 == 0xf000) {
+#ifndef CONFIG_USER_ONLY
+            /* In system mode, don't allow userspace access to semihosting,
+             * to provide some semblance of security (and for consistency
+             * with our 32-bit semihosting).
+             */
+            if (s->current_el == 0) {
+                unsupported_encoding(s, insn);
+                break;
+            }
+#endif
+            gen_exception_internal_insn(s, 0, EXCP_SEMIHOST);
+        } else {
+            unsupported_encoding(s, insn);
+        }
         break;
     case 5:
         if (op2_ll < 1 || op2_ll > 3) {
@@ -1762,9 +1816,6 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
  *  o2: 0 -> exclusive, 1 -> not
  *  o1: 0 -> single register, 1 -> register pair
  *  o0: 1 -> load-acquire/store-release, 0 -> not
- *
- *  o0 == 0 AND o2 == 1 is un-allocated
- *  o1 == 1 is un-allocated except for 32 and 64 bit sizes
  */
 static void disas_ldst_excl(DisasContext *s, uint32_t insn)
 {
@@ -1779,7 +1830,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
     int size = extract32(insn, 30, 2);
     TCGv_i64 tcg_addr;
 
-    if ((!is_excl && !is_lasr) ||
+    if ((!is_excl && !is_pair && !is_lasr) ||
+        (!is_excl && is_pair) ||
         (is_pair && size < 2)) {
         unallocated_encoding(s);
         return;
@@ -1808,15 +1860,6 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
         } else {
             do_gpr_ld(s, tcg_rt, tcg_addr, size, false, false);
         }
-        if (is_pair) {
-            TCGv_i64 tcg_rt2 = cpu_reg(s, rt);
-            tcg_gen_addi_i64(tcg_addr, tcg_addr, 1 << size);
-            if (is_store) {
-                do_gpr_st(s, tcg_rt2, tcg_addr, size);
-            } else {
-                do_gpr_ld(s, tcg_rt2, tcg_addr, size, false, false);
-            }
-        }
     }
 }
 
@@ -2970,9 +3013,51 @@ static void disas_bitfield(DisasContext *s, uint32_t insn)
     }
 
     tcg_rd = cpu_reg(s, rd);
-    tcg_tmp = read_cpu_reg(s, rn, sf);
 
-    /* OPTME: probably worth recognizing common cases of ext{8,16,32}{u,s} */
+    /* Suppress the zero-extend for !sf.  Since RI and SI are constrained
+       to be smaller than bitsize, we'll never reference data outside the
+       low 32-bits anyway.  */
+    tcg_tmp = read_cpu_reg(s, rn, 1);
+
+    /* Recognize the common aliases.  */
+    if (opc == 0) { /* SBFM */
+        if (ri == 0) {
+            if (si == 7) { /* SXTB */
+                tcg_gen_ext8s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            } else if (si == 15) { /* SXTH */
+                tcg_gen_ext16s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            } else if (si == 31) { /* SXTW */
+                tcg_gen_ext32s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            }
+        }
+        if (si == 63 || (si == 31 && ri <= si)) { /* ASR */
+            if (si == 31) {
+                tcg_gen_ext32s_i64(tcg_tmp, tcg_tmp);
+            }
+            tcg_gen_sari_i64(tcg_rd, tcg_tmp, ri);
+            goto done;
+        }
+    } else if (opc == 2) { /* UBFM */
+        if (ri == 0) { /* UXTB, UXTH, plus non-canonical AND */
+            tcg_gen_andi_i64(tcg_rd, tcg_tmp, bitmask64(si + 1));
+            return;
+        }
+        if (si == 63 || (si == 31 && ri <= si)) { /* LSR */
+            if (si == 31) {
+                tcg_gen_ext32u_i64(tcg_tmp, tcg_tmp);
+            }
+            tcg_gen_shri_i64(tcg_rd, tcg_tmp, ri);
+            return;
+        }
+        if (si + 1 == ri && si != bitsize - 1) { /* LSL */
+            int shift = bitsize - 1 - si;
+            tcg_gen_shli_i64(tcg_rd, tcg_tmp, shift);
+            goto done;
+        }
+    }
 
     if (opc != 1) { /* SBFM or UBFM */
         tcg_gen_movi_i64(tcg_rd, 0);
@@ -2997,6 +3082,7 @@ static void disas_bitfield(DisasContext *s, uint32_t insn)
         tcg_gen_sari_i64(tcg_rd, tcg_rd, 64 - (pos + len));
     }
 
+ done:
     if (!sf) { /* zero extend final result */
         tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
     }
@@ -3029,17 +3115,7 @@ static void disas_extract(DisasContext *s, uint32_t insn)
 
         tcg_rd = cpu_reg(s, rd);
 
-        if (imm) {
-            /* OPTME: we can special case rm==rn as a rotate */
-            tcg_rm = read_cpu_reg(s, rm, sf);
-            tcg_rn = read_cpu_reg(s, rn, sf);
-            tcg_gen_shri_i64(tcg_rm, tcg_rm, imm);
-            tcg_gen_shli_i64(tcg_rn, tcg_rn, bitsize - imm);
-            tcg_gen_or_i64(tcg_rd, tcg_rm, tcg_rn);
-            if (!sf) {
-                tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
-            }
-        } else {
+        if (unlikely(imm == 0)) {
             /* tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
              * so an extract from bit 0 is a special case.
              */
@@ -3048,8 +3124,27 @@ static void disas_extract(DisasContext *s, uint32_t insn)
             } else {
                 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, rm));
             }
+        } else if (rm == rn) { /* ROR */
+            tcg_rm = cpu_reg(s, rm);
+            if (sf) {
+                tcg_gen_rotri_i64(tcg_rd, tcg_rm, imm);
+            } else {
+                TCGv_i32 tmp = tcg_temp_new_i32();
+                tcg_gen_extrl_i64_i32(tmp, tcg_rm);
+                tcg_gen_rotri_i32(tmp, tmp, imm);
+                tcg_gen_extu_i32_i64(tcg_rd, tmp);
+                tcg_temp_free_i32(tmp);
+            }
+        } else {
+            tcg_rm = read_cpu_reg(s, rm, sf);
+            tcg_rn = read_cpu_reg(s, rn, sf);
+            tcg_gen_shri_i64(tcg_rm, tcg_rm, imm);
+            tcg_gen_shli_i64(tcg_rn, tcg_rn, bitsize - imm);
+            tcg_gen_or_i64(tcg_rd, tcg_rm, tcg_rn);
+            if (!sf) {
+                tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
+            }
         }
-
     }
 }
 
@@ -3109,8 +3204,8 @@ static void shift_reg(TCGv_i64 dst, TCGv_i64 src, int sf,
             TCGv_i32 t0, t1;
             t0 = tcg_temp_new_i32();
             t1 = tcg_temp_new_i32();
-            tcg_gen_trunc_i64_i32(t0, src);
-            tcg_gen_trunc_i64_i32(t1, shift_amount);
+            tcg_gen_extrl_i64_i32(t0, src);
+            tcg_gen_extrl_i64_i32(t1, shift_amount);
             tcg_gen_rotr_i32(t0, t0, t1);
             tcg_gen_extu_i32_i64(dst, t0);
             tcg_temp_free_i32(t0);
@@ -3525,8 +3620,9 @@ static void disas_adc_sbc(DisasContext *s, uint32_t insn)
 static void disas_cc(DisasContext *s, uint32_t insn)
 {
     unsigned int sf, op, y, cond, rn, nzcv, is_imm;
-    TCGLabel *label_continue = NULL;
+    TCGv_i32 tcg_t0, tcg_t1, tcg_t2;
     TCGv_i64 tcg_tmp, tcg_y, tcg_rn;
+    DisasCompare c;
 
     if (!extract32(insn, 29, 1)) {
         unallocated_encoding(s);
@@ -3544,19 +3640,13 @@ static void disas_cc(DisasContext *s, uint32_t insn)
     rn = extract32(insn, 5, 5);
     nzcv = extract32(insn, 0, 4);
 
-    if (cond < 0x0e) { /* not always */
-        TCGLabel *label_match = gen_new_label();
-        label_continue = gen_new_label();
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        tcg_tmp = tcg_temp_new_i64();
-        tcg_gen_movi_i64(tcg_tmp, nzcv << 28);
-        gen_set_nzcv(tcg_tmp);
-        tcg_temp_free_i64(tcg_tmp);
-        tcg_gen_br(label_continue);
-        gen_set_label(label_match);
-    }
-    /* match, or condition is always */
+    /* Set T0 = !COND.  */
+    tcg_t0 = tcg_temp_new_i32();
+    arm_test_cc(&c, cond);
+    tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0);
+    arm_free_cc(&c);
+
+    /* Load the arguments for the new comparison.  */
     if (is_imm) {
         tcg_y = new_tmp_a64(s);
         tcg_gen_movi_i64(tcg_y, y);
@@ -3565,6 +3655,7 @@ static void disas_cc(DisasContext *s, uint32_t insn)
     }
     tcg_rn = cpu_reg(s, rn);
 
+    /* Set the flags for the new comparison.  */
     tcg_tmp = tcg_temp_new_i64();
     if (op) {
         gen_sub_CC(sf, tcg_tmp, tcg_rn, tcg_y);
@@ -3573,9 +3664,55 @@ static void disas_cc(DisasContext *s, uint32_t insn)
     }
     tcg_temp_free_i64(tcg_tmp);
 
-    if (cond < 0x0e) { /* continue */
-        gen_set_label(label_continue);
+    /* If COND was false, force the flags to #nzcv.  Compute two masks
+     * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
+     * For tcg hosts that support ANDC, we can make do with just T1.
+     * In either case, allow the tcg optimizer to delete any unused mask.
+     */
+    tcg_t1 = tcg_temp_new_i32();
+    tcg_t2 = tcg_temp_new_i32();
+    tcg_gen_neg_i32(tcg_t1, tcg_t0);
+    tcg_gen_subi_i32(tcg_t2, tcg_t0, 1);
+
+    if (nzcv & 8) { /* N */
+        tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2);
+        }
+    }
+    if (nzcv & 4) { /* Z */
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2);
+        }
+    } else {
+        tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0);
+    }
+    if (nzcv & 2) { /* C */
+        tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2);
+        }
+    }
+    if (nzcv & 1) { /* V */
+        tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2);
+        }
     }
+    tcg_temp_free_i32(tcg_t0);
+    tcg_temp_free_i32(tcg_t1);
+    tcg_temp_free_i32(tcg_t2);
 }
 
 /* C3.5.6 Conditional select
@@ -3587,7 +3724,8 @@ static void disas_cc(DisasContext *s, uint32_t insn)
 static void disas_cond_select(DisasContext *s, uint32_t insn)
 {
     unsigned int sf, else_inv, rm, cond, else_inc, rn, rd;
-    TCGv_i64 tcg_rd, tcg_src;
+    TCGv_i64 tcg_rd, zero;
+    DisasCompare64 c;
 
     if (extract32(insn, 29, 1) || extract32(insn, 11, 1)) {
         /* S == 1 or op2<1> == 1 */
@@ -3602,48 +3740,35 @@ static void disas_cond_select(DisasContext *s, uint32_t insn)
     rn = extract32(insn, 5, 5);
     rd = extract32(insn, 0, 5);
 
-    if (rd == 31) {
-        /* silly no-op write; until we use movcond we must special-case
-         * this to avoid a dead temporary across basic blocks.
-         */
-        return;
-    }
-
     tcg_rd = cpu_reg(s, rd);
 
-    if (cond >= 0x0e) { /* condition "always" */
-        tcg_src = read_cpu_reg(s, rn, sf);
-        tcg_gen_mov_i64(tcg_rd, tcg_src);
-    } else {
-        /* OPTME: we could use movcond here, at the cost of duplicating
-         * a lot of the arm_gen_test_cc() logic.
-         */
-        TCGLabel *label_match = gen_new_label();
-        TCGLabel *label_continue = gen_new_label();
-
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        tcg_src = cpu_reg(s, rm);
+    a64_test_cc(&c, cond);
+    zero = tcg_const_i64(0);
 
+    if (rn == 31 && rm == 31 && (else_inc ^ else_inv)) {
+        /* CSET & CSETM.  */
+        tcg_gen_setcond_i64(tcg_invert_cond(c.cond), tcg_rd, c.value, zero);
+        if (else_inv) {
+            tcg_gen_neg_i64(tcg_rd, tcg_rd);
+        }
+    } else {
+        TCGv_i64 t_true = cpu_reg(s, rn);
+        TCGv_i64 t_false = read_cpu_reg(s, rm, 1);
         if (else_inv && else_inc) {
-            tcg_gen_neg_i64(tcg_rd, tcg_src);
+            tcg_gen_neg_i64(t_false, t_false);
         } else if (else_inv) {
-            tcg_gen_not_i64(tcg_rd, tcg_src);
+            tcg_gen_not_i64(t_false, t_false);
         } else if (else_inc) {
-            tcg_gen_addi_i64(tcg_rd, tcg_src, 1);
-        } else {
-            tcg_gen_mov_i64(tcg_rd, tcg_src);
+            tcg_gen_addi_i64(t_false, t_false, 1);
         }
-        if (!sf) {
-            tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
-        }
-        tcg_gen_br(label_continue);
-        /* match: */
-        gen_set_label(label_match);
-        tcg_src = read_cpu_reg(s, rn, sf);
-        tcg_gen_mov_i64(tcg_rd, tcg_src);
-        /* continue: */
-        gen_set_label(label_continue);
+        tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false);
+    }
+
+    tcg_temp_free_i64(zero);
+    a64_free_cc(&c);
+
+    if (!sf) {
+        tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
     }
 }
 
@@ -3658,7 +3783,7 @@ static void handle_clz(DisasContext *s, unsigned int sf,
         gen_helper_clz64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_clz(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -3676,7 +3801,7 @@ static void handle_cls(DisasContext *s, unsigned int sf,
         gen_helper_cls64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_cls32(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -3694,7 +3819,7 @@ static void handle_rbit(DisasContext *s, unsigned int sf,
         gen_helper_rbit64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_rbit(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -4130,20 +4255,6 @@ static void disas_fp_ccomp(DisasContext *s, uint32_t insn)
     }
 }
 
-/* copy src FP register to dst FP register; type specifies single or double */
-static void gen_mov_fp2fp(DisasContext *s, int type, int dst, int src)
-{
-    if (type) {
-        TCGv_i64 v = read_fp_dreg(s, src);
-        write_fp_dreg(s, dst, v);
-        tcg_temp_free_i64(v);
-    } else {
-        TCGv_i32 v = read_fp_sreg(s, src);
-        write_fp_sreg(s, dst, v);
-        tcg_temp_free_i32(v);
-    }
-}
-
 /* C3.6.24 Floating point conditional select
  *   31  30  29 28       24 23  22  21 20  16 15  12 11 10 9    5 4    0
  * +---+---+---+-----------+------+---+------+------+-----+------+------+
@@ -4153,7 +4264,8 @@ static void gen_mov_fp2fp(DisasContext *s, int type, int dst, int src)
 static void disas_fp_csel(DisasContext *s, uint32_t insn)
 {
     unsigned int mos, type, rm, cond, rn, rd;
-    TCGLabel *label_continue = NULL;
+    TCGv_i64 t_true, t_false, t_zero;
+    DisasCompare64 c;
 
     mos = extract32(insn, 29, 3);
     type = extract32(insn, 22, 2); /* 0 = single, 1 = double */
@@ -4171,21 +4283,23 @@ static void disas_fp_csel(DisasContext *s, uint32_t insn)
         return;
     }
 
-    if (cond < 0x0e) { /* not always */
-        TCGLabel *label_match = gen_new_label();
-        label_continue = gen_new_label();
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        gen_mov_fp2fp(s, type, rd, rm);
-        tcg_gen_br(label_continue);
-        gen_set_label(label_match);
-    }
+    /* Zero extend sreg inputs to 64 bits now.  */
+    t_true = tcg_temp_new_i64();
+    t_false = tcg_temp_new_i64();
+    read_vec_element(s, t_true, rn, 0, type ? MO_64 : MO_32);
+    read_vec_element(s, t_false, rm, 0, type ? MO_64 : MO_32);
 
-    gen_mov_fp2fp(s, type, rd, rn);
+    a64_test_cc(&c, cond);
+    t_zero = tcg_const_i64(0);
+    tcg_gen_movcond_i64(c.cond, t_true, c.value, t_zero, t_true, t_false);
+    tcg_temp_free_i64(t_zero);
+    tcg_temp_free_i64(t_false);
+    a64_free_cc(&c);
 
-    if (cond < 0x0e) { /* continue */
-        gen_set_label(label_continue);
-    }
+    /* Note that sregs write back zeros to the high bits,
+       and we've already done the zero-extension.  */
+    write_fp_dreg(s, rd, t_true);
+    tcg_temp_free_i64(t_true);
 }
 
 /* C3.6.25 Floating-point data-processing (1 source) - single precision */
@@ -5453,16 +5567,16 @@ static void disas_simd_across_lanes(DisasContext *s, uint32_t insn)
         assert(elements == 4);
 
         read_vec_element(s, tcg_elt, rn, 0, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt1, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt1, tcg_elt);
         read_vec_element(s, tcg_elt, rn, 1, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt2, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt2, tcg_elt);
 
         do_minmaxop(s, tcg_elt1, tcg_elt2, opcode, is_min, fpst);
 
         read_vec_element(s, tcg_elt, rn, 2, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt2, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt2, tcg_elt);
         read_vec_element(s, tcg_elt, rn, 3, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt3, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt3, tcg_elt);
 
         do_minmaxop(s, tcg_elt2, tcg_elt3, opcode, is_min, fpst);
 
@@ -7625,7 +7739,7 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar,
             static NeonGenNarrowFn * const xtnfns[3] = {
                 gen_helper_neon_narrow_u8,
                 gen_helper_neon_narrow_u16,
-                tcg_gen_trunc_i64_i32,
+                tcg_gen_extrl_i64_i32,
             };
             static NeonGenNarrowEnvFn * const sqxtunfns[3] = {
                 gen_helper_neon_unarrow_sat8,
@@ -7659,10 +7773,8 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar,
             } else {
                 TCGv_i32 tcg_lo = tcg_temp_new_i32();
                 TCGv_i32 tcg_hi = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tcg_lo, tcg_op);
+                tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op);
                 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, cpu_env);
-                tcg_gen_shri_i64(tcg_op, tcg_op, 32);
-                tcg_gen_trunc_i64_i32(tcg_hi, tcg_op);
                 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, cpu_env);
                 tcg_gen_deposit_i32(tcg_res[pass], tcg_lo, tcg_hi, 16, 16);
                 tcg_temp_free_i32(tcg_lo);
@@ -8568,16 +8680,10 @@ static void handle_3rd_wide(DisasContext *s, int is_q, int is_u, int size,
     }
 }
 
-static void do_narrow_high_u32(TCGv_i32 res, TCGv_i64 in)
-{
-    tcg_gen_shri_i64(in, in, 32);
-    tcg_gen_trunc_i64_i32(res, in);
-}
-
 static void do_narrow_round_high_u32(TCGv_i32 res, TCGv_i64 in)
 {
     tcg_gen_addi_i64(in, in, 1U << 31);
-    do_narrow_high_u32(res, in);
+    tcg_gen_extrh_i64_i32(res, in);
 }
 
 static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size,
@@ -8596,7 +8702,7 @@ static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size,
               gen_helper_neon_narrow_round_high_u8 },
             { gen_helper_neon_narrow_high_u16,
               gen_helper_neon_narrow_round_high_u16 },
-            { do_narrow_high_u32, do_narrow_round_high_u32 },
+            { tcg_gen_extrh_i64_i32, do_narrow_round_high_u32 },
         };
         NeonGenNarrowFn *gennarrow = narrowfns[size][is_u];
 
@@ -10900,15 +11006,11 @@ static void disas_a64_insn(CPUARMState *env, DisasContext *s)
     free_tmp_a64(s);
 }
 
-void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                        TranslationBlock *tb,
-                                        bool search_pc)
+void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb)
 {
     CPUState *cs = CPU(cpu);
     CPUARMState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     target_ulong next_page_start;
     int num_insns;
@@ -10924,7 +11026,11 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
     dc->condjmp = 0;
 
     dc->aarch64 = 1;
-    dc->el3_is_aa64 = arm_el_is_aa64(env, 3);
+    /* If we are coming from secure EL0 in a system with a 32-bit EL3, then
+     * there is no secure EL1, so we route exceptions to EL3.
+     */
+    dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) &&
+                               !arm_el_is_aa64(env, 3);
     dc->thumb = 0;
     dc->bswap_code = 0;
     dc->condexec_mask = 0;
@@ -10963,51 +11069,51 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
     init_tmp_a64_array(dc);
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
     tcg_clear_temp_count();
 
     do {
+        tcg_gen_insn_start(dc->pc, 0);
+        num_insns++;
+
         if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
+            CPUBreakpoint *bp;
             QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
-                    gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2;
-                    goto done_generating;
-                }
-            }
-        }
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
+                    if (bp->flags & BP_CPU) {
+                        gen_a64_set_pc_im(dc->pc);
+                        gen_helper_check_breakpoints(cpu_env);
+                        /* End the TB early; it likely won't be executed */
+                        dc->is_jmp = DISAS_UPDATE;
+                    } else {
+                        gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
+                        /* The address covered by the breakpoint must be
+                           included in [tb->pc, tb->pc + tb->size) in order
+                           to for it to be properly cleared -- thus we
+                           increment the PC here so that the logic setting
+                           tb->size below does the right thing.  */
+                        dc->pc += 4;
+                        goto done_generating;
+                    }
+                    break;
                 }
             }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
-        }
-
         if (dc->ss_active && !dc->pstate_ss) {
             /* Singlestep state is Active-pending.
              * If we're in this state at the start of a TB then either
@@ -11019,7 +11125,7 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
              * "did not step an insn" case, and so the syndrome ISV and EX
              * bits should be zero.
              */
-            assert(num_insns == 0);
+            assert(num_insns == 1);
             gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0),
                           default_exception_el(dc));
             dc->is_jmp = DISAS_EXC;
@@ -11038,7 +11144,6 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.
          */
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -11117,14 +11222,6 @@ done_generating:
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
diff --git a/target-arm/translate.c b/target-arm/translate.c
index 43853221f..5d2287975 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -52,7 +52,6 @@
 #define ARCH(x) do { if (!ENABLE_ARCH_##x) goto illegal_op; } while(0)
 
 #include "translate.h"
-static uint32_t gen_opc_condexec_bits[OPC_BUF_SIZE];
 
 #if defined(CONFIG_USER_ONLY)
 #define IS_USER(s) 1
@@ -738,81 +737,113 @@ static void gen_thumb2_parallel_addsub(int op1, int op2, TCGv_i32 a, TCGv_i32 b)
 #undef PAS_OP
 
 /*
- * generate a conditional branch based on ARM condition code cc.
+ * Generate a conditional based on ARM condition code cc.
  * This is common between ARM and Aarch64 targets.
  */
-void arm_gen_test_cc(int cc, TCGLabel *label)
+void arm_test_cc(DisasCompare *cmp, int cc)
 {
-    TCGv_i32 tmp;
-    TCGLabel *inv;
+    TCGv_i32 value;
+    TCGCond cond;
+    bool global = true;
 
     switch (cc) {
     case 0: /* eq: Z */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
-        break;
     case 1: /* ne: !Z */
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
+        cond = TCG_COND_EQ;
+        value = cpu_ZF;
         break;
+
     case 2: /* cs: C */
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_CF, 0, label);
-        break;
     case 3: /* cc: !C */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
+        cond = TCG_COND_NE;
+        value = cpu_CF;
         break;
+
     case 4: /* mi: N */
-        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_NF, 0, label);
-        break;
     case 5: /* pl: !N */
-        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_NF, 0, label);
+        cond = TCG_COND_LT;
+        value = cpu_NF;
         break;
+
     case 6: /* vs: V */
-        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_VF, 0, label);
-        break;
     case 7: /* vc: !V */
-        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_VF, 0, label);
+        cond = TCG_COND_LT;
+        value = cpu_VF;
         break;
+
     case 8: /* hi: C && !Z */
-        inv = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, inv);
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
-        gen_set_label(inv);
-        break;
-    case 9: /* ls: !C || Z */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
+    case 9: /* ls: !C || Z -> !(C && !Z) */
+        cond = TCG_COND_NE;
+        value = tcg_temp_new_i32();
+        global = false;
+        /* CF is 1 for C, so -CF is an all-bits-set mask for C;
+           ZF is non-zero for !Z; so AND the two subexpressions.  */
+        tcg_gen_neg_i32(value, cpu_CF);
+        tcg_gen_and_i32(value, value, cpu_ZF);
         break;
+
     case 10: /* ge: N == V -> N ^ V == 0 */
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        break;
     case 11: /* lt: N != V -> N ^ V != 0 */
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
+        /* Since we're only interested in the sign bit, == 0 is >= 0.  */
+        cond = TCG_COND_GE;
+        value = tcg_temp_new_i32();
+        global = false;
+        tcg_gen_xor_i32(value, cpu_VF, cpu_NF);
         break;
+
     case 12: /* gt: !Z && N == V */
-        inv = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, inv);
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        gen_set_label(inv);
-        break;
     case 13: /* le: Z || N != V */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
+        cond = TCG_COND_NE;
+        value = tcg_temp_new_i32();
+        global = false;
+        /* (N == V) is equal to the sign bit of ~(NF ^ VF).  Propagate
+         * the sign bit then AND with ZF to yield the result.  */
+        tcg_gen_xor_i32(value, cpu_VF, cpu_NF);
+        tcg_gen_sari_i32(value, value, 31);
+        tcg_gen_andc_i32(value, cpu_ZF, value);
         break;
+
+    case 14: /* always */
+    case 15: /* always */
+        /* Use the ALWAYS condition, which will fold early.
+         * It doesn't matter what we use for the value.  */
+        cond = TCG_COND_ALWAYS;
+        value = cpu_ZF;
+        goto no_invert;
+
     default:
         fprintf(stderr, "Bad condition code 0x%x\n", cc);
         abort();
     }
+
+    if (cc & 1) {
+        cond = tcg_invert_cond(cond);
+    }
+
+ no_invert:
+    cmp->cond = cond;
+    cmp->value = value;
+    cmp->value_global = global;
+}
+
+void arm_free_cc(DisasCompare *cmp)
+{
+    if (!cmp->value_global) {
+        tcg_temp_free_i32(cmp->value);
+    }
+}
+
+void arm_jump_cc(DisasCompare *cmp, TCGLabel *label)
+{
+    tcg_gen_brcondi_i32(cmp->cond, cmp->value, 0, label);
+}
+
+void arm_gen_test_cc(int cc, TCGLabel *label)
+{
+    DisasCompare cmp;
+    arm_test_cc(&cmp, cc);
+    arm_jump_cc(&cmp, label);
+    arm_free_cc(&cmp);
 }
 
 static const uint8_t table_logic_cc[16] = {
@@ -839,7 +870,7 @@ static inline void gen_bx_im(DisasContext *s, uint32_t addr)
 {
     TCGv_i32 tmp;
 
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
     if (s->thumb != (addr & 1)) {
         tmp = tcg_temp_new_i32();
         tcg_gen_movi_i32(tmp, addr & 1);
@@ -852,7 +883,7 @@ static inline void gen_bx_im(DisasContext *s, uint32_t addr)
 /* Set PC and Thumb state from var.  var is marked as dead.  */
 static inline void gen_bx(DisasContext *s, TCGv_i32 var)
 {
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
     tcg_gen_andi_i32(cpu_R[15], var, ~1);
     tcg_gen_andi_i32(var, var, 1);
     store_cpu_field(var, thumb);
@@ -1031,7 +1062,7 @@ static void gen_exception_insn(DisasContext *s, int offset, int excp,
 static inline void gen_lookup_tb(DisasContext *s)
 {
     tcg_gen_movi_i32(cpu_R[15], s->pc & ~1);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 static inline void gen_add_data_offset(DisasContext *s, unsigned int insn,
@@ -1557,7 +1588,7 @@ static inline int gen_iwmmxt_shift(uint32_t insn, uint32_t mask, TCGv_i32 dest)
     } else {
         tmp = tcg_temp_new_i32();
         iwmmxt_load_reg(cpu_V0, rd);
-        tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+        tcg_gen_extrl_i64_i32(tmp, cpu_V0);
     }
     tcg_gen_andi_i32(tmp, tmp, mask);
     tcg_gen_mov_i32(dest, tmp);
@@ -1581,9 +1612,9 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
             rdhi = (insn >> 16) & 0xf;
             if (insn & ARM_CP_RW_BIT) {			/* TMRRC */
                 iwmmxt_load_reg(cpu_V0, wrd);
-                tcg_gen_trunc_i64_i32(cpu_R[rdlo], cpu_V0);
+                tcg_gen_extrl_i64_i32(cpu_R[rdlo], cpu_V0);
                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                tcg_gen_trunc_i64_i32(cpu_R[rdhi], cpu_V0);
+                tcg_gen_extrl_i64_i32(cpu_R[rdhi], cpu_V0);
             } else {					/* TMCRR */
                 tcg_gen_concat_i32_i64(cpu_V0, cpu_R[rdlo], cpu_R[rdhi]);
                 iwmmxt_store_reg(cpu_V0, wrd);
@@ -1638,15 +1669,15 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
                     if (insn & (1 << 22)) {		/* WSTRD */
                         gen_aa32_st64(cpu_M0, addr, get_mem_index(s));
                     } else {				/* WSTRW wRd */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
                         gen_aa32_st32(tmp, addr, get_mem_index(s));
                     }
                 } else {
                     if (insn & (1 << 22)) {		/* WSTRH */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
                         gen_aa32_st16(tmp, addr, get_mem_index(s));
                     } else {				/* WSTRB */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
                         gen_aa32_st8(tmp, addr, get_mem_index(s));
                     }
                 }
@@ -1946,7 +1977,7 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
         switch ((insn >> 22) & 3) {
         case 0:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 7) << 3);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             if (insn & 8) {
                 tcg_gen_ext8s_i32(tmp, tmp);
             } else {
@@ -1955,7 +1986,7 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
             break;
         case 1:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 3) << 4);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             if (insn & 8) {
                 tcg_gen_ext16s_i32(tmp, tmp);
             } else {
@@ -1964,7 +1995,7 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
             break;
         case 2:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 1) << 5);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             break;
         }
         store_reg(s, rd, tmp);
@@ -2627,9 +2658,9 @@ static int disas_dsp_insn(DisasContext *s, uint32_t insn)
 
         if (insn & ARM_CP_RW_BIT) {			/* MRA */
             iwmmxt_load_reg(cpu_V0, acc);
-            tcg_gen_trunc_i64_i32(cpu_R[rdlo], cpu_V0);
+            tcg_gen_extrl_i64_i32(cpu_R[rdlo], cpu_V0);
             tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-            tcg_gen_trunc_i64_i32(cpu_R[rdhi], cpu_V0);
+            tcg_gen_extrl_i64_i32(cpu_R[rdhi], cpu_V0);
             tcg_gen_andi_i32(cpu_R[rdhi], cpu_R[rdhi], (1 << (40 - 32)) - 1);
         } else {					/* MAR */
             tcg_gen_concat_i32_i64(cpu_V0, cpu_R[rdlo], cpu_R[rdhi]);
@@ -2951,7 +2982,7 @@ static int handle_vcvt(uint32_t insn, uint32_t rd, uint32_t rm, uint32_t dp,
         } else {
             gen_helper_vfp_tould(tcg_res, tcg_double, tcg_shift, fpst);
         }
-        tcg_gen_trunc_i64_i32(tcg_tmp, tcg_res);
+        tcg_gen_extrl_i64_i32(tcg_tmp, tcg_res);
         tcg_gen_st_f32(tcg_tmp, cpu_env, vfp_reg_offset(0, rd));
         tcg_temp_free_i32(tcg_tmp);
         tcg_temp_free_i64(tcg_res);
@@ -4065,7 +4096,7 @@ static void gen_exception_return(DisasContext *s, TCGv_i32 pc)
     tmp = load_cpu_field(spsr);
     gen_set_cpsr(tmp, CPSR_ERET_MASK);
     tcg_temp_free_i32(tmp);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 /* Generate a v6 exception return.  Marks both values as dead.  */
@@ -4074,7 +4105,7 @@ static void gen_rfe(DisasContext *s, TCGv_i32 pc, TCGv_i32 cpsr)
     gen_set_cpsr(cpsr, CPSR_ERET_MASK);
     tcg_temp_free_i32(cpsr);
     store_reg(s, 15, pc);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 static void gen_nop_hint(DisasContext *s, int val)
@@ -4683,7 +4714,7 @@ static inline void gen_neon_narrow(int size, TCGv_i32 dest, TCGv_i64 src)
     switch (size) {
     case 0: gen_helper_neon_narrow_u8(dest, src); break;
     case 1: gen_helper_neon_narrow_u16(dest, src); break;
-    case 2: tcg_gen_trunc_i64_i32(dest, src); break;
+    case 2: tcg_gen_extrl_i64_i32(dest, src); break;
     default: abort();
     }
 }
@@ -6254,7 +6285,7 @@ static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
                                 break;
                             case 2:
                                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                                tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+                                tcg_gen_extrl_i64_i32(tmp, cpu_V0);
                                 break;
                             default: abort();
                             }
@@ -6269,7 +6300,7 @@ static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
                             case 2:
                                 tcg_gen_addi_i64(cpu_V0, cpu_V0, 1u << 31);
                                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                                tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+                                tcg_gen_extrl_i64_i32(tmp, cpu_V0);
                                 break;
                             default: abort();
                             }
@@ -7179,6 +7210,7 @@ static int disas_coproc_insn(DisasContext *s, uint32_t insn)
                 break;
             }
 
+            gen_set_condexec(s);
             gen_set_pc_im(s, s->pc - 4);
             tmpptr = tcg_const_ptr(ri);
             tcg_syn = tcg_const_i32(syndrome);
@@ -7224,11 +7256,11 @@ static int disas_coproc_insn(DisasContext *s, uint32_t insn)
                     tcg_gen_ld_i64(tmp64, cpu_env, ri->fieldoffset);
                 }
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 store_reg(s, rt, tmp);
                 tcg_gen_shri_i64(tmp64, tmp64, 32);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 store_reg(s, rt2, tmp);
             } else {
@@ -7334,11 +7366,11 @@ static void gen_storeq_reg(DisasContext *s, int rlow, int rhigh, TCGv_i64 val)
 {
     TCGv_i32 tmp;
     tmp = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(tmp, val);
+    tcg_gen_extrl_i64_i32(tmp, val);
     store_reg(s, rlow, tmp);
     tmp = tcg_temp_new_i32();
     tcg_gen_shri_i64(val, val, 32);
-    tcg_gen_trunc_i64_i32(tmp, val);
+    tcg_gen_extrl_i64_i32(tmp, val);
     store_reg(s, rhigh, tmp);
 }
 
@@ -7689,10 +7721,16 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 return;
             case 4: /* dsb */
             case 5: /* dmb */
-            case 6: /* isb */
                 ARCH(7);
                 /* We don't emulate caches so these are a no-op.  */
                 return;
+            case 6: /* isb */
+                /* We need to break the TB after this insn to execute
+                 * self-modifying code correctly and also to take
+                 * any pending interrupts immediately.
+                 */
+                gen_lookup_tb(s);
+                return;
             default:
                 goto illegal_op;
             }
@@ -8013,7 +8051,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 tmp64 = gen_muls_i64_i32(tmp, tmp2);
                 tcg_gen_shri_i64(tmp64, tmp64, 16);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 if ((sh & 2) == 0) {
                     tmp2 = load_reg(s, rn);
@@ -8500,7 +8538,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 }
                 /* Perform base writeback before the loaded value to
                    ensure correct behavior with overlapping index registers.
-                   ldrd with base writeback is is undefined if the
+                   ldrd with base writeback is undefined if the
                    destination and index registers overlap.  */
                 if (!(insn & (1 << 24))) {
                     gen_add_datah_offset(s, insn, address_offset, addr);
@@ -8679,7 +8717,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                         }
                         tcg_gen_shri_i64(tmp64, tmp64, 32);
                         tmp = tcg_temp_new_i32();
-                        tcg_gen_trunc_i64_i32(tmp, tmp64);
+                        tcg_gen_extrl_i64_i32(tmp, tmp64);
                         tcg_temp_free_i64(tmp64);
                         store_reg(s, rn, tmp);
                         break;
@@ -8998,7 +9036,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                     tmp = load_cpu_field(spsr);
                     gen_set_cpsr(tmp, CPSR_ERET_MASK);
                     tcg_temp_free_i32(tmp);
-                    s->is_jmp = DISAS_UPDATE;
+                    s->is_jmp = DISAS_JUMP;
                 }
             }
             break;
@@ -9749,7 +9787,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 tmp64 = gen_muls_i64_i32(tmp, tmp2);
                 tcg_gen_shri_i64(tmp64, tmp64, 16);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 if (rs != 15)
                   {
@@ -9773,7 +9811,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 }
                 tcg_gen_shri_i64(tmp64, tmp64, 32);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 break;
             case 7: /* Unsigned sum of absolute differences.  */
@@ -9999,9 +10037,16 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                             break;
                         case 4: /* dsb */
                         case 5: /* dmb */
-                        case 6: /* isb */
                             /* These execute as NOPs.  */
                             break;
+                        case 6: /* isb */
+                            /* We need to break the TB after this insn
+                             * to execute self-modifying code correctly
+                             * and also to take any pending interrupts
+                             * immediately.
+                             */
+                            gen_lookup_tb(s);
+                            break;
                         default:
                             goto illegal_op;
                         }
@@ -11135,22 +11180,47 @@ undef:
                        default_exception_el(s));
 }
 
+static bool insn_crosses_page(CPUARMState *env, DisasContext *s)
+{
+    /* Return true if the insn at dc->pc might cross a page boundary.
+     * (False positives are OK, false negatives are not.)
+     */
+    uint16_t insn;
+
+    if ((s->pc & 3) == 0) {
+        /* At a 4-aligned address we can't be crossing a page */
+        return false;
+    }
+
+    /* This must be a Thumb insn */
+    insn = arm_lduw_code(env, s->pc, s->bswap_code);
+
+    if ((insn >> 11) >= 0x1d) {
+        /* Top five bits 0b11101 / 0b11110 / 0b11111 : this is the
+         * First half of a 32-bit Thumb insn. Thumb-1 cores might
+         * end up actually treating this as two 16-bit insns (see the
+         * code at the start of disas_thumb2_insn()) but we don't bother
+         * to check for that as it is unlikely, and false positives here
+         * are harmless.
+         */
+        return true;
+    }
+    /* Definitely a 16-bit insn, can't be crossing a page. */
+    return false;
+}
+
 /* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(ARMCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+   basic block 'tb'.  */
+void gen_intermediate_code(CPUARMState *env, TranslationBlock *tb)
 {
+    ARMCPU *cpu = arm_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUARMState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     target_ulong next_page_start;
     int num_insns;
     int max_insns;
+    bool end_of_page;
 
     /* generate intermediate code */
 
@@ -11158,7 +11228,7 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
      * the A32/T32 complexity to do with conditional execution/IT blocks/etc.
      */
     if (ARM_TBFLAG_AARCH64_STATE(tb->flags)) {
-        gen_intermediate_code_internal_a64(cpu, tb, search_pc);
+        gen_intermediate_code_a64(cpu, tb);
         return;
     }
 
@@ -11172,7 +11242,11 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
     dc->condjmp = 0;
 
     dc->aarch64 = 0;
-    dc->el3_is_aa64 = arm_el_is_aa64(env, 3);
+    /* If we are coming from secure EL0 in a system with a 32-bit EL3, then
+     * there is no secure EL1, so we route exceptions to EL3.
+     */
+    dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) &&
+                               !arm_el_is_aa64(env, 3);
     dc->thumb = ARM_TBFLAG_THUMB(tb->flags);
     dc->bswap_code = ARM_TBFLAG_BSWAP_CODE(tb->flags);
     dc->condexec_mask = (ARM_TBFLAG_CONDEXEC(tb->flags) & 0xf) << 1;
@@ -11220,11 +11294,14 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
     /* FIXME: cpu_M0 can probably be the same as cpu_V0.  */
     cpu_M0 = tcg_temp_new_i64();
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
@@ -11250,10 +11327,9 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
      * (3) if we leave the TB unexpectedly (eg a data abort on a load)
      * then the CPUARMState will be wrong and we need to reset it.
      * This is handled in the same way as restoration of the
-     * PC in these situations: we will be called again with search_pc=1
-     * and generate a mapping of the condexec bits for each PC in
-     * gen_opc_condexec_bits[]. restore_state_to_opc() then uses
-     * this to restore the condexec bits.
+     * PC in these situations; we save the value of the condexec bits
+     * for each PC via tcg_gen_insn_start(), and restore_state_to_opc()
+     * then uses this to restore them after an exception.
      *
      * Note that there are no instructions which can read the condexec
      * bits, and none which can write non-static values to them, so
@@ -11270,13 +11346,17 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
         store_cpu_field(tmp, condexec_bits);
       }
     do {
+        tcg_gen_insn_start(dc->pc,
+                           (dc->condexec_cond << 4) | (dc->condexec_mask >> 1));
+        num_insns++;
+
 #ifdef CONFIG_USER_ONLY
         /* Intercept jump to the magic kernel page.  */
         if (dc->pc >= 0xffff0000) {
             /* We always get here via a jump, so know we are not in a
                conditional execution block.  */
             gen_exception_internal(EXCP_KERNEL_TRAP);
-            dc->is_jmp = DISAS_UPDATE;
+            dc->is_jmp = DISAS_EXC;
             break;
         }
 #else
@@ -11284,40 +11364,40 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
             /* We always get here via a jump, so know we are not in a
                conditional execution block.  */
             gen_exception_internal(EXCP_EXCEPTION_EXIT);
-            dc->is_jmp = DISAS_UPDATE;
+            dc->is_jmp = DISAS_EXC;
             break;
         }
 #endif
 
         if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
+            CPUBreakpoint *bp;
             QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
-                    gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2;
-                    goto done_generating;
+                    if (bp->flags & BP_CPU) {
+                        gen_set_condexec(dc);
+                        gen_set_pc_im(dc, dc->pc);
+                        gen_helper_check_breakpoints(cpu_env);
+                        /* End the TB early; it's likely not going to be executed */
+                        dc->is_jmp = DISAS_UPDATE;
+                    } else {
+                        gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
+                        /* The address covered by the breakpoint must be
+                           included in [tb->pc, tb->pc + tb->size) in order
+                           to for it to be properly cleared -- thus we
+                           increment the PC here so that the logic setting
+                           tb->size below does the right thing.  */
+                        /* TODO: Advance PC by correct instruction length to
+                         * avoid disassembler error messages */
+                        dc->pc += 2;
+                        goto done_generating;
+                    }
+                    break;
                 }
             }
         }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            gen_opc_condexec_bits[lj] = (dc->condexec_cond << 4) | (dc->condexec_mask >> 1);
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
         }
 
         if (dc->ss_active && !dc->pstate_ss) {
@@ -11331,7 +11411,7 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
              * "did not step an insn" case, and so the syndrome ISV and EX
              * bits should be zero.
              */
-            assert(num_insns == 0);
+            assert(num_insns == 1);
             gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0),
                           default_exception_el(dc));
             goto done_generating;
@@ -11367,12 +11447,24 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
          * Otherwise the subsequent code could get translated several times.
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
-        num_insns ++;
+
+        /* We want to stop the TB if the next insn starts in a new page,
+         * or if it spans between this page and the next. This means that
+         * if we're looking at the last halfword in the page we need to
+         * see if it's a 16-bit Thumb insn (which will fit in this TB)
+         * or a 32-bit Thumb insn (which won't).
+         * This is to avoid generating a silly TB with a single 16-bit insn
+         * in it at the end of this page (which would execute correctly
+         * but isn't very efficient).
+         */
+        end_of_page = (dc->pc >= next_page_start) ||
+            ((dc->pc >= next_page_start - 3) && insn_crosses_page(env, dc));
+
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
              !dc->ss_active &&
-             dc->pc < next_page_start &&
+             !end_of_page &&
              num_insns < max_insns);
 
     if (tb->cflags & CF_LAST_IO) {
@@ -11408,7 +11500,8 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
             }
             gen_set_label(dc->condlabel);
         }
-        if (dc->condjmp || !dc->is_jmp) {
+        if (dc->condjmp || dc->is_jmp == DISAS_NEXT ||
+            dc->is_jmp == DISAS_UPDATE) {
             gen_set_pc_im(dc, dc->pc);
             dc->condjmp = 0;
         }
@@ -11444,9 +11537,11 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
         case DISAS_NEXT:
             gen_goto_tb(dc, 1, dc->pc);
             break;
-        default:
-        case DISAS_JUMP:
         case DISAS_UPDATE:
+            gen_set_pc_im(dc, dc->pc);
+            /* fall through */
+        case DISAS_JUMP:
+        default:
             /* indicate that the hash table must be used to find the next TB */
             tcg_gen_exit_tb(0);
             break;
@@ -11497,25 +11592,8 @@ done_generating:
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUARMState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(arm_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUARMState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(arm_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 static const char *cpu_mode_names[16] = {
@@ -11530,6 +11608,7 @@ void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     CPUARMState *env = &cpu->env;
     int i;
     uint32_t psr;
+    const char *ns_status;
 
     if (is_a64(env)) {
         aarch64_cpu_dump_state(cs, f, cpu_fprintf, flags);
@@ -11544,13 +11623,22 @@ void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
             cpu_fprintf(f, " ");
     }
     psr = cpsr_read(env);
-    cpu_fprintf(f, "PSR=%08x %c%c%c%c %c %s%d\n",
+
+    if (arm_feature(env, ARM_FEATURE_EL3) &&
+        (psr & CPSR_M) != ARM_CPU_MODE_MON) {
+        ns_status = env->cp15.scr_el3 & SCR_NS ? "NS " : "S ";
+    } else {
+        ns_status = "";
+    }
+
+    cpu_fprintf(f, "PSR=%08x %c%c%c%c %c %s%s%d\n",
                 psr,
                 psr & (1 << 31) ? 'N' : '-',
                 psr & (1 << 30) ? 'Z' : '-',
                 psr & (1 << 29) ? 'C' : '-',
                 psr & (1 << 28) ? 'V' : '-',
                 psr & CPSR_T ? 'T' : 'A',
+                ns_status,
                 cpu_mode_names[psr & 0xf], (psr & 0x10) ? 32 : 26);
 
     if (flags & CPU_DUMP_FPU) {
@@ -11572,13 +11660,14 @@ void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     }
 }
 
-void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
     if (is_a64(env)) {
-        env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+        env->pc = data[0];
         env->condexec_bits = 0;
     } else {
-        env->regs[15] = tcg_ctx.gen_opc_pc[pc_pos];
-        env->condexec_bits = gen_opc_condexec_bits[pc_pos];
+        env->regs[15] = data[0];
+        env->condexec_bits = data[1];
     }
 }
diff --git a/target-arm/translate.h b/target-arm/translate.h
index 679bdbcbc..53ef97105 100644
--- a/target-arm/translate.h
+++ b/target-arm/translate.h
@@ -23,7 +23,8 @@ typedef struct DisasContext {
     ARMMMUIdx mmu_idx; /* MMU index to use for normal loads/stores */
     bool ns;        /* Use non-secure CPREG bank on access */
     int fp_excp_el; /* FP exception EL or 0 if enabled */
-    bool el3_is_aa64;  /* Flag indicating whether EL3 is AArch64 or not */
+    /* Flag indicating that exceptions from secure mode are routed to EL3. */
+    bool secure_routed_to_el3;
     bool vfp_enabled; /* FP enabled via FPSCR.EN */
     int vec_len;
     int vec_stride;
@@ -62,6 +63,12 @@ typedef struct DisasContext {
     TCGv_i64 tmp_a64[TMP_A64_MAX];
 } DisasContext;
 
+typedef struct DisasCompare {
+    TCGCond cond;
+    TCGv_i32 value;
+    bool value_global;
+} DisasCompare;
+
 /* Share the TCG temporaries common between 32 and 64 bit modes.  */
 extern TCGv_ptr cpu_env;
 extern TCGv_i32 cpu_NF, cpu_ZF, cpu_CF, cpu_VF;
@@ -92,7 +99,7 @@ static inline int default_exception_el(DisasContext *s)
      * exceptions can only be routed to ELs above 1, so we target the higher of
      * 1 or the current EL.
      */
-    return (s->mmu_idx == ARMMMUIdx_S1SE0 && !s->el3_is_aa64)
+    return (s->mmu_idx == ARMMMUIdx_S1SE0 && s->secure_routed_to_el3)
             ? 3 : MAX(1, s->current_el);
 }
 
@@ -115,9 +122,7 @@ static inline int default_exception_el(DisasContext *s)
 
 #ifdef TARGET_AARCH64
 void a64_translate_init(void);
-void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                        TranslationBlock *tb,
-                                        bool search_pc);
+void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb);
 void gen_a64_set_pc_im(uint64_t val);
 void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
                             fprintf_function cpu_fprintf, int flags);
@@ -126,9 +131,7 @@ static inline void a64_translate_init(void)
 {
 }
 
-static inline void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                                      TranslationBlock *tb,
-                                                      bool search_pc)
+static inline void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb)
 {
 }
 
@@ -143,6 +146,9 @@ static inline void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
 }
 #endif
 
+void arm_test_cc(DisasCompare *cmp, int cc);
+void arm_free_cc(DisasCompare *cmp);
+void arm_jump_cc(DisasCompare *cmp, TCGLabel *label);
 void arm_gen_test_cc(int cc, TCGLabel *label);
 
 #endif /* TARGET_ARM_TRANSLATE_H */
diff --git a/target-cris/cpu-qom.h b/target-cris/cpu-qom.h
index 6fc30c208..df4c0b50a 100644
--- a/target-cris/cpu-qom.h
+++ b/target-cris/cpu-qom.h
@@ -73,6 +73,10 @@ static inline CRISCPU *cris_env_get_cpu(CPUCRISState *env)
 
 #define ENV_OFFSET offsetof(CRISCPU, env)
 
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_cris_cpu;
+#endif
+
 void cris_cpu_do_interrupt(CPUState *cpu);
 void crisv10_cpu_do_interrupt(CPUState *cpu);
 bool cris_cpu_exec_interrupt(CPUState *cpu, int int_req);
diff --git a/target-cris/cpu.c b/target-cris/cpu.c
index aa5e1c57c..8eaf5a5a3 100644
--- a/target-cris/cpu.c
+++ b/target-cris/cpu.c
@@ -302,6 +302,7 @@ static void cris_cpu_class_init(ObjectClass *oc, void *data)
     cc->handle_mmu_fault = cris_cpu_handle_mmu_fault;
 #else
     cc->get_phys_page_debug = cris_cpu_get_phys_page_debug;
+    dc->vmsd = &vmstate_cris_cpu;
 #endif
 
     cc->gdb_num_core_regs = 49;
diff --git a/target-cris/cpu.h b/target-cris/cpu.h
index d422e3571..32204607c 100644
--- a/target-cris/cpu.h
+++ b/target-cris/cpu.h
@@ -29,8 +29,6 @@
 
 #include "exec/cpu-defs.h"
 
-#define ELF_MACHINE	EM_CRIS
-
 #define EXCP_NMI        1
 #define EXCP_GURU       2
 #define EXCP_BUSFAULT   3
@@ -108,6 +106,11 @@
 
 #define NB_MMU_MODES 2
 
+typedef struct {
+    uint32_t hi;
+    uint32_t lo;
+} TLBSet;
+
 typedef struct CPUCRISState {
 	uint32_t regs[16];
 	/* P0 - P15 are referred to as special registers in the docs.  */
@@ -151,7 +154,7 @@ typedef struct CPUCRISState {
 	uint32_t sregs[4][16];
 
 	/* Linear feedback shift reg in the mmu. Used to provide pseudo
-	   randomness for the 'hint' the mmu gives to sw for chosing valid
+	   randomness for the 'hint' the mmu gives to sw for choosing valid
 	   sets on TLB refills.  */
 	uint32_t mmu_rand_lfsr;
 
@@ -161,11 +164,7 @@ typedef struct CPUCRISState {
 	 *
 	 * One for I and another for D.
 	 */
-	struct
-	{
-		uint32_t hi;
-		uint32_t lo;
-	} tlbsets[2][4][16];
+        TLBSet tlbsets[2][4][16];
 
 	CPU_COMMON
 
@@ -224,16 +223,13 @@ enum {
 #define cpu_init(cpu_model) CPU(cpu_cris_init(cpu_model))
 
 #define cpu_exec cpu_cris_exec
-#define cpu_gen_code cpu_cris_gen_code
 #define cpu_signal_handler cpu_cris_signal_handler
 
-#define CPU_SAVE_VERSION 1
-
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUCRISState *env)
+static inline int cpu_mmu_index (CPUCRISState *env, bool ifetch)
 {
 	return !!(env->pregs[PR_CCS] & U_FLAG);
 }
diff --git a/target-cris/machine.c b/target-cris/machine.c
index 8f9c0dd59..983b67c47 100644
--- a/target-cris/machine.c
+++ b/target-cris/machine.c
@@ -1,90 +1,91 @@
-#include "hw/hw.h"
-#include "hw/boards.h"
-
-void cpu_save(QEMUFile *f, void *opaque)
-{
-    CPUCRISState *env = opaque;
-    int i;
-    int s;
-    int mmu;
-
-    for (i = 0; i < 16; i++)
-        qemu_put_be32(f, env->regs[i]);
-    for (i = 0; i < 16; i++)
-        qemu_put_be32(f, env->pregs[i]);
-
-    qemu_put_be32(f, env->pc);
-    qemu_put_be32(f, env->ksp);
+/*
+ *  CRIS virtual CPU state save/load support
+ *
+ *  Copyright (c) 2012 Red Hat, Inc.
+ *  Written by Juan Quintela <quintela@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
 
-    qemu_put_be32(f, env->dslot);
-    qemu_put_be32(f, env->btaken);
-    qemu_put_be32(f, env->btarget);
-
-    qemu_put_be32(f, env->cc_op);
-    qemu_put_be32(f, env->cc_mask);
-    qemu_put_be32(f, env->cc_dest);
-    qemu_put_be32(f, env->cc_src);
-    qemu_put_be32(f, env->cc_result);
-    qemu_put_be32(f, env->cc_size);
-    qemu_put_be32(f, env->cc_x);
-
-    for (s = 0; s < 4; s++) {
-        for (i = 0; i < 16; i++)
-            qemu_put_be32(f, env->sregs[s][i]);
-    }
+#include "hw/hw.h"
 
-    qemu_put_be32(f, env->mmu_rand_lfsr);
-    for (mmu = 0; mmu < 2; mmu++) {
-        for (s = 0; s < 4; s++) {
-            for (i = 0; i < 16; i++) {
-                qemu_put_be32(f, env->tlbsets[mmu][s][i].lo);
-                qemu_put_be32(f, env->tlbsets[mmu][s][i].hi);
-            }
-        }
+static const VMStateDescription vmstate_tlbset = {
+    .name = "cpu/tlbset",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(lo, TLBSet),
+        VMSTATE_UINT32(hi, TLBSet),
+        VMSTATE_END_OF_LIST()
     }
-}
-
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-	CPUCRISState *env = opaque;
-    int i;
-    int s;
-    int mmu;
-
-    for (i = 0; i < 16; i++)
-        env->regs[i] = qemu_get_be32(f);
-    for (i = 0; i < 16; i++)
-        env->pregs[i] = qemu_get_be32(f);
-
-    env->pc = qemu_get_be32(f);
-    env->ksp = qemu_get_be32(f);
+};
 
-    env->dslot = qemu_get_be32(f);
-    env->btaken = qemu_get_be32(f);
-    env->btarget = qemu_get_be32(f);
-
-    env->cc_op = qemu_get_be32(f);
-    env->cc_mask = qemu_get_be32(f);
-    env->cc_dest = qemu_get_be32(f);
-    env->cc_src = qemu_get_be32(f);
-    env->cc_result = qemu_get_be32(f);
-    env->cc_size = qemu_get_be32(f);
-    env->cc_x = qemu_get_be32(f);
-
-    for (s = 0; s < 4; s++) {
-        for (i = 0; i < 16; i++)
-            env->sregs[s][i] = qemu_get_be32(f);
+static const VMStateDescription vmstate_cris_env = {
+    .name = "env",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(pregs, CPUCRISState, 16),
+        VMSTATE_UINT32(pc, CPUCRISState),
+        VMSTATE_UINT32(ksp, CPUCRISState),
+        VMSTATE_INT32(dslot, CPUCRISState),
+        VMSTATE_INT32(btaken, CPUCRISState),
+        VMSTATE_UINT32(btarget, CPUCRISState),
+        VMSTATE_UINT32(cc_op, CPUCRISState),
+        VMSTATE_UINT32(cc_mask, CPUCRISState),
+        VMSTATE_UINT32(cc_dest, CPUCRISState),
+        VMSTATE_UINT32(cc_src, CPUCRISState),
+        VMSTATE_UINT32(cc_result, CPUCRISState),
+        VMSTATE_INT32(cc_size, CPUCRISState),
+        VMSTATE_INT32(cc_x, CPUCRISState),
+        VMSTATE_INT32(locked_irq, CPUCRISState),
+        VMSTATE_INT32(interrupt_vector, CPUCRISState),
+        VMSTATE_INT32(fault_vector, CPUCRISState),
+        VMSTATE_INT32(trap_vector, CPUCRISState),
+        VMSTATE_UINT32_ARRAY(sregs[0], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[1], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[2], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[3], CPUCRISState, 16),
+        VMSTATE_UINT32(mmu_rand_lfsr, CPUCRISState),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][0], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][1], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][2], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][3], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][0], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][1], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][2], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][3], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_END_OF_LIST()
     }
+};
 
-    env->mmu_rand_lfsr = qemu_get_be32(f);
-    for (mmu = 0; mmu < 2; mmu++) {
-        for (s = 0; s < 4; s++) {
-            for (i = 0; i < 16; i++) {
-                env->tlbsets[mmu][s][i].lo = qemu_get_be32(f);
-                env->tlbsets[mmu][s][i].hi = qemu_get_be32(f);
-            }
-        }
+const VMStateDescription vmstate_cris_cpu = {
+    .name = "cpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_CPU(),
+        VMSTATE_STRUCT(env, CRISCPU, 1, vmstate_cris_env, CPUCRISState),
+        VMSTATE_END_OF_LIST()
     }
-
-    return 0;
-}
+};
diff --git a/target-cris/translate.c b/target-cris/translate.c
index 3e59601eb..2d710cc10 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -102,9 +102,9 @@ typedef struct DisasContext {
 
     int cc_size_uptodate; /* -1 invalid or last written value.  */
 
-    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not uptodate.  */
-    int flags_uptodate; /* Wether or not $ccs is uptodate.  */
-    int flagx_known; /* Wether or not flags_x has the x flag known at
+    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not up-to-date.  */
+    int flags_uptodate; /* Whether or not $ccs is up-to-date.  */
+    int flagx_known; /* Whether or not flags_x has the x flag known at
                 translation time.  */
     int flags_x;
 
@@ -311,7 +311,7 @@ static void t_gen_asr(TCGv d, TCGv a, TCGv b)
 
 static void t_gen_cris_dstep(TCGv d, TCGv a, TCGv b)
 {
-    TCGLabel *l1 = gen_new_label();
+    TCGv t = tcg_temp_new();
 
     /*
      * d <<= 1
@@ -319,9 +319,9 @@ static void t_gen_cris_dstep(TCGv d, TCGv a, TCGv b)
      *    d -= s;
      */
     tcg_gen_shli_tl(d, a, 1);
-    tcg_gen_brcond_tl(TCG_COND_LTU, d, b, l1);
-    tcg_gen_sub_tl(d, d, b);
-    gen_set_label(l1);
+    tcg_gen_sub_tl(t, d, b);
+    tcg_gen_movcond_tl(TCG_COND_GEU, d, d, b, t, d);
+    tcg_temp_free(t);
 }
 
 static void t_gen_cris_mstep(TCGv d, TCGv a, TCGv b, TCGv ccs)
@@ -769,13 +769,7 @@ static void cris_alu_op_exec(DisasContext *dc, int op,
         t_gen_cris_mstep(dst, a, b, cpu_PR[PR_CCS]);
         break;
     case CC_OP_BOUND:
-    {
-        TCGLabel *l1 = gen_new_label();
-        tcg_gen_mov_tl(dst, a);
-        tcg_gen_brcond_tl(TCG_COND_LEU, a, b, l1);
-        tcg_gen_mov_tl(dst, b);
-        gen_set_label(l1);
-    }
+        tcg_gen_movcond_tl(TCG_COND_LEU, dst, a, b, a, b);
         break;
     case CC_OP_CMP:
         tcg_gen_sub_tl(dst, a, b);
@@ -1089,7 +1083,7 @@ static inline void cris_prepare_jmp (DisasContext *dc, unsigned int type)
 
 static void gen_load64(DisasContext *dc, TCGv_i64 dst, TCGv addr)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1103,7 +1097,7 @@ static void gen_load64(DisasContext *dc, TCGv_i64 dst, TCGv addr)
 static void gen_load(DisasContext *dc, TCGv dst, TCGv addr, 
              unsigned int size, int sign)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1118,7 +1112,7 @@ static void gen_load(DisasContext *dc, TCGv dst, TCGv addr,
 static void gen_store (DisasContext *dc, TCGv addr, TCGv val,
                unsigned int size)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1482,15 +1476,8 @@ static int dec_scc_r(CPUCRISState *env, DisasContext *dc)
     LOG_DIS("s%s $r%u\n",
             cc_name(cond), dc->op1);
 
-    if (cond != CC_A) {
-        TCGLabel *l1 = gen_new_label();
-        gen_tst_cc(dc, cpu_R[dc->op1], cond);
-        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->op1], 0, l1);
-        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-        gen_set_label(l1);
-    } else {
-        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-    }
+    gen_tst_cc(dc, cpu_R[dc->op1], cond);
+    tcg_gen_setcondi_tl(TCG_COND_NE, cpu_R[dc->op1], cpu_R[dc->op1], 0);
 
     cris_cc_mask(dc, 0);
     return 2;
@@ -2604,9 +2591,9 @@ static int dec_movem_mr(CPUCRISState *env, DisasContext *dc)
     tcg_temp_free(addr);
 
     for (i = 0; i < (nr >> 1); i++) {
-        tcg_gen_trunc_i64_i32(cpu_R[i * 2], tmp[i]);
+        tcg_gen_extrl_i64_i32(cpu_R[i * 2], tmp[i]);
         tcg_gen_shri_i64(tmp[i], tmp[i], 32);
-        tcg_gen_trunc_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
+        tcg_gen_extrl_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
         tcg_temp_free_i64(tmp[i]);
     }
     if (nr & 1) {
@@ -3007,10 +2994,6 @@ static unsigned int crisv32_decoder(CPUCRISState *env, DisasContext *dc)
     int insn_len = 2;
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-        }
-
     /* Load a halfword onto the instruction register.  */
         dc->ir = cris_fetch(env, dc, dc->pc, 2, 0);
 
@@ -3047,23 +3030,6 @@ static unsigned int crisv32_decoder(CPUCRISState *env, DisasContext *dc)
     return insn_len;
 }
 
-static void check_breakpoint(CPUCRISState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(cris_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                cris_evaluate_flags(dc);
-                tcg_gen_movi_tl(env_pc, dc->pc);
-                t_gen_raise_exception(EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-            }
-        }
-    }
-}
-
 #include "translate_v10.c"
 
 /*
@@ -3101,15 +3067,12 @@ static void check_breakpoint(CPUCRISState *env, DisasContext *dc)
  */
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUCRISState *env, struct TranslationBlock *tb)
 {
+    CRISCPU *cpu = cris_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUCRISState *env = &cpu->env;
     uint32_t pc_start;
     unsigned int insn_len;
-    int j, lj;
     struct DisasContext ctx;
     struct DisasContext *dc = &ctx;
     uint32_t next_page_start;
@@ -3161,13 +3124,13 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
 
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log(
-                "srch=%d pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
+                "pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
                 "pid=%x usp=%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n",
-                search_pc, dc->pc, dc->ppc,
+                dc->pc, dc->ppc,
                 (uint64_t)tb->flags,
                 env->btarget, (unsigned)tb->flags & 7,
                 env->pregs[PR_CCS],
@@ -3183,38 +3146,38 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
-        check_breakpoint(env, dc);
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            if (dc->delayed_branch == 1) {
-                tcg_ctx.gen_opc_pc[lj] = dc->ppc | 1;
-            } else {
-                tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            }
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->delayed_branch == 1
+                           ? dc->ppc | 1 : dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            cris_evaluate_flags(dc);
+            tcg_gen_movi_tl(env_pc, dc->pc);
+            t_gen_raise_exception(EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 2;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         dc->clear_x = 1;
@@ -3226,7 +3189,6 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
             cris_clear_x_flag(dc);
         }
 
-        num_insns++;
         /* Check for delayed branches here. If we do it before
            actually generating any host code, the simulator will just
            loop doing nothing for on this program location.  */
@@ -3331,16 +3293,8 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
     }
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
 #if !DISAS_CRIS
@@ -3354,16 +3308,6 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
 #endif
 }
 
-void gen_intermediate_code (CPUCRISState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(cris_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUCRISState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(cris_env_get_cpu(env), tb, true);
-}
-
 void cris_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -3456,7 +3400,8 @@ void cris_initialize_tcg(void)
     }
 }
 
-void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-cris/translate_v10.c b/target-cris/translate_v10.c
index b742c4cd0..3ab1c398e 100644
--- a/target-cris/translate_v10.c
+++ b/target-cris/translate_v10.c
@@ -96,7 +96,7 @@ static void gen_store_v10_conditional(DisasContext *dc, TCGv addr, TCGv val,
 static void gen_store_v10(DisasContext *dc, TCGv addr, TCGv val,
                        unsigned int size)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -535,16 +535,8 @@ static void dec10_reg_scc(DisasContext *dc)
 
     LOG_DIS("s%s $r%u\n", cc_name(cond), dc->src);
 
-    if (cond != CC_A)
-    {
-        TCGLabel *l1 = gen_new_label();
-        gen_tst_cc (dc, cpu_R[dc->src], cond);
-        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->src], 0, l1);
-        tcg_gen_movi_tl(cpu_R[dc->src], 1);
-        gen_set_label(l1);
-    } else {
-        tcg_gen_movi_tl(cpu_R[dc->src], 1);
-    }
+    gen_tst_cc(dc, cpu_R[dc->src], cond);
+    tcg_gen_setcondi_tl(TCG_COND_NE, cpu_R[dc->src], cpu_R[dc->src], 0);
 
     cris_cc_mask(dc, 0);
 }
@@ -1207,9 +1199,6 @@ static unsigned int crisv10_decoder(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
-        tcg_gen_debug_insn_start(dc->pc);
-
     /* Load a halfword onto the instruction register.  */
     dc->ir = cpu_lduw_code(env, dc->pc);
 
diff --git a/target-i386/Makefile.objs b/target-i386/Makefile.objs
index 7a1df2c98..437d9975b 100644
--- a/target-i386/Makefile.objs
+++ b/target-i386/Makefile.objs
@@ -1,7 +1,7 @@
-obj-y += translate.o helper.o cpu.o
+obj-y += translate.o helper.o cpu.o bpt_helper.o
 obj-y += excp_helper.o fpu_helper.o cc_helper.o int_helper.o svm_helper.o
 obj-y += smm_helper.o misc_helper.o mem_helper.o seg_helper.o
 obj-y += gdbstub.o
-obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o
+obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o monitor.o
 obj-$(CONFIG_KVM) += kvm.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
diff --git a/target-i386/bpt_helper.c b/target-i386/bpt_helper.c
new file mode 100644
index 000000000..dac1b1a36
--- /dev/null
+++ b/target-i386/bpt_helper.c
@@ -0,0 +1,326 @@
+/*
+ *  i386 breakpoint helpers
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "exec/helper-proto.h"
+
+
+#ifndef CONFIG_USER_ONLY
+static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 1;
+}
+
+static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 2;
+
+}
+static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return hw_global_breakpoint_enabled(dr7, index) ||
+           hw_local_breakpoint_enabled(dr7, index);
+}
+
+static inline int hw_breakpoint_type(unsigned long dr7, int index)
+{
+    return (dr7 >> (DR7_TYPE_SHIFT + (index * 4))) & 3;
+}
+
+static inline int hw_breakpoint_len(unsigned long dr7, int index)
+{
+    int len = ((dr7 >> (DR7_LEN_SHIFT + (index * 4))) & 3);
+    return (len == 2) ? 8 : len + 1;
+}
+
+static int hw_breakpoint_insert(CPUX86State *env, int index)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+    target_ulong dr7 = env->dr[7];
+    target_ulong drN = env->dr[index];
+    int err = 0;
+
+    switch (hw_breakpoint_type(dr7, index)) {
+    case DR7_TYPE_BP_INST:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_breakpoint_insert(cs, drN, BP_CPU,
+                                        &env->cpu_breakpoint[index]);
+        }
+        break;
+
+    case DR7_TYPE_IO_RW:
+        /* Notice when we should enable calls to bpt_io.  */
+        return hw_breakpoint_enabled(env->dr[7], index)
+               ? HF_IOBPT_MASK : 0;
+
+    case DR7_TYPE_DATA_WR:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_watchpoint_insert(cs, drN,
+                                        hw_breakpoint_len(dr7, index),
+                                        BP_CPU | BP_MEM_WRITE,
+                                        &env->cpu_watchpoint[index]);
+        }
+        break;
+
+    case DR7_TYPE_DATA_RW:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_watchpoint_insert(cs, drN,
+                                        hw_breakpoint_len(dr7, index),
+                                        BP_CPU | BP_MEM_ACCESS,
+                                        &env->cpu_watchpoint[index]);
+        }
+        break;
+    }
+    if (err) {
+        env->cpu_breakpoint[index] = NULL;
+    }
+    return 0;
+}
+
+static void hw_breakpoint_remove(CPUX86State *env, int index)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+
+    switch (hw_breakpoint_type(env->dr[7], index)) {
+    case DR7_TYPE_BP_INST:
+        if (env->cpu_breakpoint[index]) {
+            cpu_breakpoint_remove_by_ref(cs, env->cpu_breakpoint[index]);
+            env->cpu_breakpoint[index] = NULL;
+        }
+        break;
+
+    case DR7_TYPE_DATA_WR:
+    case DR7_TYPE_DATA_RW:
+        if (env->cpu_breakpoint[index]) {
+            cpu_watchpoint_remove_by_ref(cs, env->cpu_watchpoint[index]);
+            env->cpu_breakpoint[index] = NULL;
+        }
+        break;
+
+    case DR7_TYPE_IO_RW:
+        /* HF_IOBPT_MASK cleared elsewhere.  */
+        break;
+    }
+}
+
+void cpu_x86_update_dr7(CPUX86State *env, uint32_t new_dr7)
+{
+    target_ulong old_dr7 = env->dr[7];
+    int iobpt = 0;
+    int i;
+
+    new_dr7 |= DR7_FIXED_1;
+
+    /* If nothing is changing except the global/local enable bits,
+       then we can make the change more efficient.  */
+    if (((old_dr7 ^ new_dr7) & ~0xff) == 0) {
+        /* Fold the global and local enable bits together into the
+           global fields, then xor to show which registers have
+           changed collective enable state.  */
+        int mod = ((old_dr7 | old_dr7 * 2) ^ (new_dr7 | new_dr7 * 2)) & 0xff;
+
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if ((mod & (2 << i * 2)) && !hw_breakpoint_enabled(new_dr7, i)) {
+                hw_breakpoint_remove(env, i);
+            }
+        }
+        env->dr[7] = new_dr7;
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if (mod & (2 << i * 2) && hw_breakpoint_enabled(new_dr7, i)) {
+                iobpt |= hw_breakpoint_insert(env, i);
+            } else if (hw_breakpoint_type(new_dr7, i) == DR7_TYPE_IO_RW
+                       && hw_breakpoint_enabled(new_dr7, i)) {
+                iobpt |= HF_IOBPT_MASK;
+            }
+        }
+    } else {
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            hw_breakpoint_remove(env, i);
+        }
+        env->dr[7] = new_dr7;
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            iobpt |= hw_breakpoint_insert(env, i);
+        }
+    }
+
+    env->hflags = (env->hflags & ~HF_IOBPT_MASK) | iobpt;
+}
+
+static bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
+{
+    target_ulong dr6;
+    int reg;
+    bool hit_enabled = false;
+
+    dr6 = env->dr[6] & ~0xf;
+    for (reg = 0; reg < DR7_MAX_BP; reg++) {
+        bool bp_match = false;
+        bool wp_match = false;
+
+        switch (hw_breakpoint_type(env->dr[7], reg)) {
+        case DR7_TYPE_BP_INST:
+            if (env->dr[reg] == env->eip) {
+                bp_match = true;
+            }
+            break;
+        case DR7_TYPE_DATA_WR:
+        case DR7_TYPE_DATA_RW:
+            if (env->cpu_watchpoint[reg] &&
+                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
+                wp_match = true;
+            }
+            break;
+        case DR7_TYPE_IO_RW:
+            break;
+        }
+        if (bp_match || wp_match) {
+            dr6 |= 1 << reg;
+            if (hw_breakpoint_enabled(env->dr[7], reg)) {
+                hit_enabled = true;
+            }
+        }
+    }
+
+    if (hit_enabled || force_dr6_update) {
+        env->dr[6] = dr6;
+    }
+
+    return hit_enabled;
+}
+
+void breakpoint_handler(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    CPUBreakpoint *bp;
+
+    if (cs->watchpoint_hit) {
+        if (cs->watchpoint_hit->flags & BP_CPU) {
+            cs->watchpoint_hit = NULL;
+            if (check_hw_breakpoints(env, false)) {
+                raise_exception(env, EXCP01_DB);
+            } else {
+                cpu_resume_from_signal(cs, NULL);
+            }
+        }
+    } else {
+        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
+            if (bp->pc == env->eip) {
+                if (bp->flags & BP_CPU) {
+                    check_hw_breakpoints(env, true);
+                    raise_exception(env, EXCP01_DB);
+                }
+                break;
+            }
+        }
+    }
+}
+#endif
+
+void helper_single_step(CPUX86State *env)
+{
+#ifndef CONFIG_USER_ONLY
+    check_hw_breakpoints(env, true);
+    env->dr[6] |= DR6_BS;
+#endif
+    raise_exception(env, EXCP01_DB);
+}
+
+void helper_set_dr(CPUX86State *env, int reg, target_ulong t0)
+{
+#ifndef CONFIG_USER_ONLY
+    switch (reg) {
+    case 0: case 1: case 2: case 3:
+        if (hw_breakpoint_enabled(env->dr[7], reg)
+            && hw_breakpoint_type(env->dr[7], reg) != DR7_TYPE_IO_RW) {
+            hw_breakpoint_remove(env, reg);
+            env->dr[reg] = t0;
+            hw_breakpoint_insert(env, reg);
+        } else {
+            env->dr[reg] = t0;
+        }
+        return;
+    case 4:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        }
+        /* fallthru */
+    case 6:
+        env->dr[6] = t0 | DR6_FIXED_1;
+        return;
+    case 5:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        }
+        /* fallthru */
+    case 7:
+        cpu_x86_update_dr7(env, t0);
+        return;
+    }
+    raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+#endif
+}
+
+target_ulong helper_get_dr(CPUX86State *env, int reg)
+{
+    switch (reg) {
+    case 0: case 1: case 2: case 3: case 6: case 7:
+        return env->dr[reg];
+    case 4:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        } else {
+            return env->dr[6];
+        }
+    case 5:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        } else {
+            return env->dr[7];
+        }
+    }
+    raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+}
+
+/* Check if Port I/O is trapped by a breakpoint.  */
+void helper_bpt_io(CPUX86State *env, uint32_t port,
+                   uint32_t size, target_ulong next_eip)
+{
+#ifndef CONFIG_USER_ONLY
+    target_ulong dr7 = env->dr[7];
+    int i, hit = 0;
+
+    for (i = 0; i < DR7_MAX_BP; ++i) {
+        if (hw_breakpoint_type(dr7, i) == DR7_TYPE_IO_RW
+            && hw_breakpoint_enabled(dr7, i)) {
+            int bpt_len = hw_breakpoint_len(dr7, i);
+            if (port + size - 1 >= env->dr[i]
+                && port <= env->dr[i] + bpt_len - 1) {
+                hit |= 1 << i;
+            }
+        }
+    }
+
+    if (hit) {
+        env->dr[6] = (env->dr[6] & ~0xf) | hit;
+        env->eip = next_eip;
+        raise_exception(env, EXCP01_DB);
+    }
+#endif
+}
diff --git a/target-i386/cc_helper.c b/target-i386/cc_helper.c
index ecbf0ec09..d5b7c7b17 100644
--- a/target-i386/cc_helper.c
+++ b/target-i386/cc_helper.c
@@ -378,7 +378,7 @@ void helper_sti_vm(CPUX86State *env)
 {
     env->eflags |= VIF_MASK;
     if (env->eflags & VIP_MASK) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
 }
 #endif
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index 7a4fddd85..e3bfe9d07 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -88,7 +88,12 @@ typedef struct X86CPU {
     bool hyperv_vapic;
     bool hyperv_relaxed_timing;
     int hyperv_spinlock_attempts;
+    char *hyperv_vendor_id;
     bool hyperv_time;
+    bool hyperv_crash;
+    bool hyperv_reset;
+    bool hyperv_vpindex;
+    bool hyperv_runtime;
     bool check_cpuid;
     bool enforce_cpuid;
     bool expose_kvm;
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 4d1b08533..11e5e39a7 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -43,7 +43,6 @@
 
 #include "sysemu/sysemu.h"
 #include "hw/qdev-properties.h"
-#include "hw/cpu/icc_bus.h"
 #ifndef CONFIG_USER_ONLY
 #include "exec/address-spaces.h"
 #include "hw/xen/xen.h"
@@ -260,8 +259,8 @@ static const char *svm_feature_name[] = {
 static const char *cpuid_7_0_ebx_feature_name[] = {
     "fsgsbase", "tsc_adjust", NULL, "bmi1", "hle", "avx2", NULL, "smep",
     "bmi2", "erms", "invpcid", "rtm", NULL, NULL, "mpx", NULL,
-    "avx512f", NULL, "rdseed", "adx", "smap", NULL, NULL, NULL,
-    NULL, NULL, "avx512pf", "avx512er", "avx512cd", NULL, NULL, NULL,
+    "avx512f", NULL, "rdseed", "adx", "smap", NULL, "pcommit", "clflushopt",
+    "clwb", NULL, "avx512pf", "avx512er", "avx512cd", NULL, NULL, NULL,
 };
 
 static const char *cpuid_apm_edx_feature_name[] = {
@@ -313,7 +312,7 @@ static const char *cpuid_6_feature_name[] = {
           CPUID_PAE | CPUID_MCE | CPUID_CX8 | CPUID_APIC | CPUID_SEP | \
           CPUID_MTRR | CPUID_PGE | CPUID_MCA | CPUID_CMOV | CPUID_PAT | \
           CPUID_PSE36 | CPUID_CLFLUSH | CPUID_ACPI | CPUID_MMX | \
-          CPUID_FXSR | CPUID_SSE | CPUID_SSE2 | CPUID_SS)
+          CPUID_FXSR | CPUID_SSE | CPUID_SSE2 | CPUID_SS | CPUID_DE)
           /* partly implemented:
           CPUID_MTRR, CPUID_MCA, CPUID_CLFLUSH (needed for Win64) */
           /* missing:
@@ -346,7 +345,9 @@ static const char *cpuid_6_feature_name[] = {
 #define TCG_SVM_FEATURES 0
 #define TCG_KVM_FEATURES 0
 #define TCG_7_0_EBX_FEATURES (CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_SMAP | \
-          CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ADX)
+          CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ADX | \
+          CPUID_7_0_EBX_PCOMMIT | CPUID_7_0_EBX_CLFLUSHOPT |            \
+          CPUID_7_0_EBX_CLWB)
           /* missing:
           CPUID_7_0_EBX_FSGSBASE, CPUID_7_0_EBX_HLE, CPUID_7_0_EBX_AVX2,
           CPUID_7_0_EBX_ERMS, CPUID_7_0_EBX_INVPCID, CPUID_7_0_EBX_RTM,
@@ -478,38 +479,6 @@ const char *get_register_name_32(unsigned int reg)
     return x86_reg_info_32[reg].name;
 }
 
-/* KVM-specific features that are automatically added to all CPU models
- * when KVM is enabled.
- */
-static uint32_t kvm_default_features[FEATURE_WORDS] = {
-    [FEAT_KVM] = (1 << KVM_FEATURE_CLOCKSOURCE) |
-        (1 << KVM_FEATURE_NOP_IO_DELAY) |
-        (1 << KVM_FEATURE_CLOCKSOURCE2) |
-        (1 << KVM_FEATURE_ASYNC_PF) |
-        (1 << KVM_FEATURE_STEAL_TIME) |
-        (1 << KVM_FEATURE_PV_EOI) |
-        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT),
-    [FEAT_1_ECX] = CPUID_EXT_X2APIC,
-};
-
-/* Features that are not added by default to any CPU model when KVM is enabled.
- */
-static uint32_t kvm_default_unset_features[FEATURE_WORDS] = {
-    [FEAT_1_EDX] = CPUID_ACPI,
-    [FEAT_1_ECX] = CPUID_EXT_MONITOR,
-    [FEAT_8000_0001_ECX] = CPUID_EXT3_SVM,
-};
-
-void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features)
-{
-    kvm_default_features[w] &= ~features;
-}
-
-void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features)
-{
-    kvm_default_unset_features[w] &= ~features;
-}
-
 /*
  * Returns the set of feature flags that are supported and migratable by
  * QEMU, for a given FeatureWord.
@@ -689,7 +658,6 @@ struct X86CPUDefinition {
     int stepping;
     FeatureWordArray features;
     char model_id[48];
-    bool cache_info_passthrough;
 };
 
 static X86CPUDefinition builtin_x86_defs[] = {
@@ -705,12 +673,11 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
             CPUID_PSE36,
         .features[FEAT_1_ECX] =
-            CPUID_EXT_SSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT,
+            CPUID_EXT_SSE3 | CPUID_EXT_CX16,
         .features[FEAT_8000_0001_EDX] =
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
-            CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
+            CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM,
         .xlevel = 0x8000000A,
     },
     {
@@ -806,7 +773,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .features[FEAT_1_EDX] =
             PPRO_FEATURES,
         .features[FEAT_1_ECX] =
-            CPUID_EXT_SSE3 | CPUID_EXT_POPCNT,
+            CPUID_EXT_SSE3,
         .xlevel = 0x80000004,
     },
     {
@@ -1113,7 +1080,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1148,7 +1115,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1185,7 +1152,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1223,7 +1190,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1277,8 +1244,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_DE | CPUID_FP87,
         .features[FEAT_1_ECX] =
             CPUID_EXT_CX16 | CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+            CPUID_EXT2_LM | CPUID_EXT2_FXSR |
             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
@@ -1306,8 +1274,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .features[FEAT_1_ECX] =
             CPUID_EXT_POPCNT | CPUID_EXT_CX16 | CPUID_EXT_MONITOR |
             CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+            CPUID_EXT2_LM | CPUID_EXT2_FXSR |
             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
@@ -1338,8 +1307,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT_POPCNT | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
             CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+            CPUID_EXT2_LM |
             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
@@ -1373,8 +1343,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT_AES | CPUID_EXT_POPCNT | CPUID_EXT_SSE42 |
             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_FMA |
             CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+            CPUID_EXT2_LM |
             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
@@ -1392,30 +1363,41 @@ static X86CPUDefinition builtin_x86_defs[] = {
     },
 };
 
-/**
- * x86_cpu_compat_set_features:
- * @cpu_model: CPU model name to be changed. If NULL, all CPU models are changed
- * @w: Identifies the feature word to be changed.
- * @feat_add: Feature bits to be added to feature word
- * @feat_remove: Feature bits to be removed from feature word
- *
- * Change CPU model feature bits for compatibility.
- *
- * This function may be used by machine-type compatibility functions
- * to enable or disable feature bits on specific CPU models.
+typedef struct PropValue {
+    const char *prop, *value;
+} PropValue;
+
+/* KVM-specific features that are automatically added/removed
+ * from all CPU models when KVM is enabled.
  */
-void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
-                                 uint32_t feat_add, uint32_t feat_remove)
+static PropValue kvm_default_props[] = {
+    { "kvmclock", "on" },
+    { "kvm-nopiodelay", "on" },
+    { "kvm-asyncpf", "on" },
+    { "kvm-steal-time", "on" },
+    { "kvm-pv-eoi", "on" },
+    { "kvmclock-stable-bit", "on" },
+    { "x2apic", "on" },
+    { "acpi", "off" },
+    { "monitor", "off" },
+    { "svm", "off" },
+    { NULL, NULL },
+};
+
+void x86_cpu_change_kvm_default(const char *prop, const char *value)
 {
-    X86CPUDefinition *def;
-    int i;
-    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
-        def = &builtin_x86_defs[i];
-        if (!cpu_model || !strcmp(cpu_model, def->name)) {
-            def->features[w] |= feat_add;
-            def->features[w] &= ~feat_remove;
+    PropValue *pv;
+    for (pv = kvm_default_props; pv->prop; pv++) {
+        if (!strcmp(pv->prop, prop)) {
+            pv->value = value;
+            break;
         }
     }
+
+    /* It is valid to call this function only for properties that
+     * are already present in the kvm_default_props table.
+     */
+    assert(pv->prop);
 }
 
 static uint32_t x86_cpu_get_supported_feature_word(FeatureWord w,
@@ -1442,6 +1424,7 @@ static X86CPUDefinition host_cpudef;
 
 static Property host_x86_cpu_properties[] = {
     DEFINE_PROP_BOOL("migratable", X86CPU, migratable, true),
+    DEFINE_PROP_BOOL("host-cache-info", X86CPU, cache_info_passthrough, false),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -1468,7 +1451,6 @@ static void host_x86_cpu_class_init(ObjectClass *oc, void *data)
     cpu_x86_fill_model_id(host_cpudef.model_id);
 
     xcc->cpu_def = &host_cpudef;
-    host_cpudef.cache_info_passthrough = true;
 
     /* level, xlevel, xlevel2, and the feature words are initialized on
      * instance_init, because they require KVM to be initialized.
@@ -1514,7 +1496,7 @@ static void report_unavailable_features(FeatureWord w, uint32_t mask)
     int i;
 
     for (i = 0; i < 32; ++i) {
-        if (1 << i & mask) {
+        if ((1UL << i) & mask) {
             const char *reg = get_register_name_32(f->cpuid_reg);
             assert(reg);
             fprintf(stderr, "warning: %s doesn't support requested feature: "
@@ -1921,8 +1903,8 @@ static void x86_cpu_parse_featurestr(CPUState *cs, char *features,
                 char *err;
                 char num[32];
 
-                tsc_freq = strtosz_suffix_unit(val, &err,
-                                               STRTOSZ_DEFSUFFIX_B, 1000);
+                tsc_freq = qemu_strtosz_suffix_unit(val, &err,
+                                               QEMU_STRTOSZ_DEFSUFFIX_B, 1000);
                 if (tsc_freq < 0 || *err) {
                     error_setg(errp, "bad numerical value %s", val);
                     return;
@@ -2089,6 +2071,18 @@ static int x86_cpu_filter_features(X86CPU *cpu)
     return rv;
 }
 
+static void x86_cpu_apply_props(X86CPU *cpu, PropValue *props)
+{
+    PropValue *pv;
+    for (pv = props; pv->prop; pv++) {
+        if (!pv->value) {
+            continue;
+        }
+        object_property_parse(OBJECT(cpu), pv->value, pv->prop,
+                              &error_abort);
+    }
+}
+
 /* Load data from X86CPUDefinition
  */
 static void x86_cpu_load_def(X86CPU *cpu, X86CPUDefinition *def, Error **errp)
@@ -2104,7 +2098,6 @@ static void x86_cpu_load_def(X86CPU *cpu, X86CPUDefinition *def, Error **errp)
     object_property_set_int(OBJECT(cpu), def->stepping, "stepping", errp);
     object_property_set_int(OBJECT(cpu), def->xlevel, "xlevel", errp);
     object_property_set_int(OBJECT(cpu), def->xlevel2, "xlevel2", errp);
-    cpu->cache_info_passthrough = def->cache_info_passthrough;
     object_property_set_str(OBJECT(cpu), def->model_id, "model-id", errp);
     for (w = 0; w < FEATURE_WORDS; w++) {
         env->features[w] = def->features[w];
@@ -2112,11 +2105,7 @@ static void x86_cpu_load_def(X86CPU *cpu, X86CPUDefinition *def, Error **errp)
 
     /* Special cases not set in the X86CPUDefinition structs: */
     if (kvm_enabled()) {
-        FeatureWord w;
-        for (w = 0; w < FEATURE_WORDS; w++) {
-            env->features[w] |= kvm_default_features[w];
-            env->features[w] &= ~kvm_default_unset_features[w];
-        }
+        x86_cpu_apply_props(cpu, kvm_default_props);
     }
 
     env->features[FEAT_1_ECX] |= CPUID_EXT_HYPERVISOR;
@@ -2259,7 +2248,7 @@ void x86_cpudef_setup(void)
                 pstrcpy(def->model_id, sizeof(def->model_id),
                         "QEMU Virtual CPU version ");
                 pstrcat(def->model_id, sizeof(def->model_id),
-                        qemu_get_version());
+                        qemu_hw_version());
                 break;
             }
         }
@@ -2751,7 +2740,6 @@ static void mce_init(X86CPU *cpu)
 #ifndef CONFIG_USER_ONLY
 static void x86_cpu_apic_create(X86CPU *cpu, Error **errp)
 {
-    DeviceState *dev = DEVICE(cpu);
     APICCommonState *apic;
     const char *apic_type = "apic";
 
@@ -2761,11 +2749,7 @@ static void x86_cpu_apic_create(X86CPU *cpu, Error **errp)
         apic_type = "xen-apic";
     }
 
-    cpu->apic_state = qdev_try_create(qdev_get_parent_bus(dev), apic_type);
-    if (cpu->apic_state == NULL) {
-        error_setg(errp, "APIC device '%s' could not be created", apic_type);
-        return;
-    }
+    cpu->apic_state = DEVICE(object_new(apic_type));
 
     object_property_add_child(OBJECT(cpu), "apic",
                               OBJECT(cpu->apic_state), NULL);
@@ -2773,15 +2757,30 @@ static void x86_cpu_apic_create(X86CPU *cpu, Error **errp)
     /* TODO: convert to link<> */
     apic = APIC_COMMON(cpu->apic_state);
     apic->cpu = cpu;
+    apic->apicbase = APIC_DEFAULT_ADDRESS | MSR_IA32_APICBASE_ENABLE;
 }
 
 static void x86_cpu_apic_realize(X86CPU *cpu, Error **errp)
 {
+    APICCommonState *apic;
+    static bool apic_mmio_map_once;
+
     if (cpu->apic_state == NULL) {
         return;
     }
     object_property_set_bool(OBJECT(cpu->apic_state), true, "realized",
                              errp);
+
+    /* Map APIC MMIO area */
+    apic = APIC_COMMON(cpu->apic_state);
+    if (!apic_mmio_map_once) {
+        memory_region_add_subregion_overlap(get_system_memory(),
+                                            apic->apicbase &
+                                            MSR_IA32_APICBASE_BASE,
+                                            &apic->io_memory,
+                                            0x1000);
+        apic_mmio_map_once = true;
+     }
 }
 
 static void x86_cpu_machine_done(Notifier *n, void *unused)
@@ -3126,14 +3125,8 @@ static bool x86_cpu_has_work(CPUState *cs)
     X86CPU *cpu = X86_CPU(cs);
     CPUX86State *env = &cpu->env;
 
-#if !defined(CONFIG_USER_ONLY)
-    if (cs->interrupt_request & CPU_INTERRUPT_POLL) {
-        apic_poll_irq(cpu->apic_state);
-        cpu_reset_interrupt(cs, CPU_INTERRUPT_POLL);
-    }
-#endif
-
-    return ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
+    return ((cs->interrupt_request & (CPU_INTERRUPT_HARD |
+                                      CPU_INTERRUPT_POLL)) &&
             (env->eflags & IF_MASK)) ||
            (cs->interrupt_request & (CPU_INTERRUPT_NMI |
                                      CPU_INTERRUPT_INIT |
@@ -3149,12 +3142,17 @@ static Property x86_cpu_properties[] = {
     DEFINE_PROP_BOOL("hv-relaxed", X86CPU, hyperv_relaxed_timing, false),
     DEFINE_PROP_BOOL("hv-vapic", X86CPU, hyperv_vapic, false),
     DEFINE_PROP_BOOL("hv-time", X86CPU, hyperv_time, false),
-    DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, false),
+    DEFINE_PROP_BOOL("hv-crash", X86CPU, hyperv_crash, false),
+    DEFINE_PROP_BOOL("hv-reset", X86CPU, hyperv_reset, false),
+    DEFINE_PROP_BOOL("hv-vpindex", X86CPU, hyperv_vpindex, false),
+    DEFINE_PROP_BOOL("hv-runtime", X86CPU, hyperv_runtime, false),
+    DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, true),
     DEFINE_PROP_BOOL("enforce", X86CPU, enforce_cpuid, false),
     DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),
     DEFINE_PROP_UINT32("level", X86CPU, env.cpuid_level, 0),
     DEFINE_PROP_UINT32("xlevel", X86CPU, env.cpuid_xlevel, 0),
     DEFINE_PROP_UINT32("xlevel2", X86CPU, env.cpuid_xlevel2, 0),
+    DEFINE_PROP_STRING("hv-vendor-id", X86CPU, hyperv_vendor_id),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -3166,7 +3164,6 @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
 
     xcc->parent_realize = dc->realize;
     dc->realize = x86_cpu_realizefn;
-    dc->bus_type = TYPE_ICC_BUS;
     dc->props = x86_cpu_properties;
 
     xcc->parent_reset = cc->reset;
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index ead28325b..84edfd0d8 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -21,6 +21,7 @@
 
 #include "config.h"
 #include "qemu-common.h"
+#include "standard-headers/asm-x86/hyperv.h"
 
 #ifdef TARGET_X86_64
 #define TARGET_LONG_BITS 64
@@ -36,10 +37,10 @@
 #define TARGET_HAS_PRECISE_SMC
 
 #ifdef TARGET_X86_64
-#define ELF_MACHINE     EM_X86_64
+#define I386_ELF_MACHINE  EM_X86_64
 #define ELF_MACHINE_UNAME "x86_64"
 #else
-#define ELF_MACHINE     EM_386
+#define I386_ELF_MACHINE  EM_386
 #define ELF_MACHINE_UNAME "i686"
 #endif
 
@@ -154,6 +155,7 @@
 #define HF_SVMI_SHIFT       21 /* SVM intercepts are active */
 #define HF_OSFXSR_SHIFT     22 /* CR4.OSFXSR */
 #define HF_SMAP_SHIFT       23 /* CR4.SMAP */
+#define HF_IOBPT_SHIFT      24 /* an io breakpoint enabled */
 
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
 #define HF_SOFTMMU_MASK      (1 << HF_SOFTMMU_SHIFT)
@@ -177,6 +179,7 @@
 #define HF_SVMI_MASK         (1 << HF_SVMI_SHIFT)
 #define HF_OSFXSR_MASK       (1 << HF_OSFXSR_SHIFT)
 #define HF_SMAP_MASK         (1 << HF_SMAP_SHIFT)
+#define HF_IOBPT_MASK        (1 << HF_IOBPT_SHIFT)
 
 /* hflags2 */
 
@@ -234,6 +237,7 @@
 #define DR7_TYPE_SHIFT  16
 #define DR7_LEN_SHIFT   18
 #define DR7_FIXED_1     0x00000400
+#define DR7_GLOBAL_BP_MASK   0xaa
 #define DR7_LOCAL_BP_MASK    0x55
 #define DR7_MAX_BP           4
 #define DR7_TYPE_BP_INST     0x0
@@ -282,6 +286,8 @@
 #define MCE_CAP_DEF     (MCG_CTL_P|MCG_SER_P)
 #define MCE_BANKS_DEF   10
 
+#define MCG_CAP_BANKS_MASK 0xff
+
 #define MCG_STATUS_RIPV (1ULL<<0)   /* restart ip valid */
 #define MCG_STATUS_EIPV (1ULL<<1)   /* ip points to correct instruction */
 #define MCG_STATUS_MCIP (1ULL<<2)   /* machine check in progress */
@@ -572,6 +578,9 @@ typedef uint32_t FeatureWordArray[FEATURE_WORDS];
 #define CPUID_7_0_EBX_RDSEED   (1U << 18)
 #define CPUID_7_0_EBX_ADX      (1U << 19)
 #define CPUID_7_0_EBX_SMAP     (1U << 20)
+#define CPUID_7_0_EBX_PCOMMIT  (1U << 22) /* Persistent Commit */
+#define CPUID_7_0_EBX_CLFLUSHOPT (1U << 23) /* Flush a Cache Line Optimized */
+#define CPUID_7_0_EBX_CLWB     (1U << 24) /* Cache Line Write Back */
 #define CPUID_7_0_EBX_AVX512PF (1U << 26) /* AVX-512 Prefetch */
 #define CPUID_7_0_EBX_AVX512ER (1U << 27) /* AVX-512 Exponential and Reciprocal */
 #define CPUID_7_0_EBX_AVX512CD (1U << 28) /* AVX-512 Conflict Detection */
@@ -793,6 +802,7 @@ typedef struct {
 #define MAX_GP_COUNTERS    (MSR_IA32_PERF_STATUS - MSR_P6_EVNTSEL0)
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 #define NB_OPMASK_REGS 8
 
@@ -832,6 +842,7 @@ typedef struct CPUX86State {
     BNDReg bnd_regs[4];
     BNDCSReg bndcs_regs;
     uint64_t msr_bndcfgs;
+    uint64_t efer;
 
     /* Beginning of state preserved by INIT (dummy marker).  */
     struct {} start_init_save;
@@ -864,7 +875,6 @@ typedef struct CPUX86State {
     uint32_t sysenter_cs;
     target_ulong sysenter_esp;
     target_ulong sysenter_eip;
-    uint64_t efer;
     uint64_t star;
 
     uint64_t vm_hsave;
@@ -908,12 +918,14 @@ typedef struct CPUX86State {
     uint64_t msr_hv_guest_os_id;
     uint64_t msr_hv_vapic;
     uint64_t msr_hv_tsc;
+    uint64_t msr_hv_crash_params[HV_X64_MSR_CRASH_PARAMS];
+    uint64_t msr_hv_runtime;
 
     /* exception/interrupt handling */
     int error_code;
     int exception_is_int;
     target_ulong exception_next_eip;
-    target_ulong dr[8]; /* debug registers */
+    target_ulong dr[8]; /* debug registers; note dr4 and dr5 are unused */
     union {
         struct CPUBreakpoint *cpu_breakpoint[4];
         struct CPUWatchpoint *cpu_watchpoint[4];
@@ -1123,42 +1135,13 @@ void x86_stl_phys(CPUState *cs, hwaddr addr, uint32_t val);
 void x86_stq_phys(CPUState *cs, hwaddr addr, uint64_t val);
 #endif
 
-static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return (dr7 >> (index * 2)) & 1;
-}
-
-static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return (dr7 >> (index * 2)) & 2;
-
-}
-static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return hw_global_breakpoint_enabled(dr7, index) ||
-           hw_local_breakpoint_enabled(dr7, index);
-}
-
-static inline int hw_breakpoint_type(unsigned long dr7, int index)
-{
-    return (dr7 >> (DR7_TYPE_SHIFT + (index * 4))) & 3;
-}
-
-static inline int hw_breakpoint_len(unsigned long dr7, int index)
-{
-    int len = ((dr7 >> (DR7_LEN_SHIFT + (index * 4))) & 3);
-    return (len == 2) ? 8 : len + 1;
-}
-
-void hw_breakpoint_insert(CPUX86State *env, int index);
-void hw_breakpoint_remove(CPUX86State *env, int index);
-bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update);
 void breakpoint_handler(CPUState *cs);
 
 /* will be suppressed */
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
 void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
+void cpu_x86_update_dr7(CPUX86State *env, uint32_t new_dr7);
 
 /* hw/pc.c */
 uint64_t cpu_get_tsc(CPUX86State *env);
@@ -1187,7 +1170,6 @@ uint64_t cpu_get_tsc(CPUX86State *env);
 #define cpu_init(cpu_model) CPU(cpu_x86_init(cpu_model))
 
 #define cpu_exec cpu_x86_exec
-#define cpu_gen_code cpu_x86_gen_code
 #define cpu_signal_handler cpu_x86_signal_handler
 #define cpu_list x86_cpu_list
 #define cpudef_setup x86_cpudef_setup
@@ -1199,7 +1181,7 @@ uint64_t cpu_get_tsc(CPUX86State *env);
 #define MMU_KSMAP_IDX   0
 #define MMU_USER_IDX    1
 #define MMU_KNOSMAP_IDX 2
-static inline int cpu_mmu_index(CPUX86State *env)
+static inline int cpu_mmu_index(CPUX86State *env, bool ifetch)
 {
     return (env->hflags & HF_CPL_MASK) == 3 ? MMU_USER_IDX :
         (!(env->hflags & HF_SMAP_MASK) || (env->eflags & AC_MASK))
@@ -1267,8 +1249,12 @@ void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
 
 /* excp_helper.c */
 void QEMU_NORETURN raise_exception(CPUX86State *env, int exception_index);
+void QEMU_NORETURN raise_exception_ra(CPUX86State *env, int exception_index,
+                                      uintptr_t retaddr);
 void QEMU_NORETURN raise_exception_err(CPUX86State *env, int exception_index,
                                        int error_code);
+void QEMU_NORETURN raise_exception_err_ra(CPUX86State *env, int exception_index,
+                                          int error_code, uintptr_t retaddr);
 void QEMU_NORETURN raise_interrupt(CPUX86State *nenv, int intno, int is_int,
                                    int error_code, int next_eip_addend);
 
@@ -1318,6 +1304,9 @@ static inline MemTxAttrs cpu_get_mem_attrs(CPUX86State *env)
 void cpu_set_mxcsr(CPUX86State *env, uint32_t val);
 void cpu_set_fpuc(CPUX86State *env, uint16_t val);
 
+/* mem_helper.c */
+void helper_lock_init(void);
+
 /* svm_helper.c */
 void cpu_svm_check_intercept_param(CPUX86State *env1, uint32_t type,
                                    uint64_t param);
@@ -1332,11 +1321,15 @@ void cpu_smm_update(X86CPU *cpu);
 
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access);
 
-void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
-                                 uint32_t feat_add, uint32_t feat_remove);
-
-void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features);
-void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features);
+/* Change the value of a KVM-specific default
+ *
+ * If value is NULL, no default will be set and the original
+ * value from the CPU model table will be kept.
+ *
+ * It is valid to call this funciton only for properties that
+ * are already present in the kvm_default_props table.
+ */
+void x86_cpu_change_kvm_default(const char *prop, const char *value);
 
 
 /* Return name of 32-bit register, from a R_* constant */
@@ -1347,4 +1340,7 @@ void enable_compat_apic_id_mode(void);
 #define APIC_DEFAULT_ADDRESS 0xfee00000
 #define APIC_SPACE_SIZE      0x100000
 
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags);
+
 #endif /* CPU_I386_H */
diff --git a/target-i386/excp_helper.c b/target-i386/excp_helper.c
index 99fca847d..5e347bc46 100644
--- a/target-i386/excp_helper.c
+++ b/target-i386/excp_helper.c
@@ -22,14 +22,6 @@
 #include "sysemu/sysemu.h"
 #include "exec/helper-proto.h"
 
-#if 0
-#define raise_exception_err(env, a, b)                                  \
-    do {                                                                \
-        qemu_log("raise_exception line=%d\n", __LINE__);                \
-        (raise_exception_err)(env, a, b);                               \
-    } while (0)
-#endif
-
 void helper_raise_interrupt(CPUX86State *env, int intno, int next_eip_addend)
 {
     raise_interrupt(env, intno, 1, 0, next_eip_addend);
@@ -92,7 +84,8 @@ static int check_exception(CPUX86State *env, int intno, int *error_code)
  */
 static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
                                            int is_int, int error_code,
-                                           int next_eip_addend)
+                                           int next_eip_addend,
+                                           uintptr_t retaddr)
 {
     CPUState *cs = CPU(x86_env_get_cpu(env));
 
@@ -108,7 +101,7 @@ static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
     env->error_code = error_code;
     env->exception_is_int = is_int;
     env->exception_next_eip = env->eip + next_eip_addend;
-    cpu_loop_exit(cs);
+    cpu_loop_exit_restore(cs, retaddr);
 }
 
 /* shortcuts to generate exceptions */
@@ -116,16 +109,27 @@ static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
 void QEMU_NORETURN raise_interrupt(CPUX86State *env, int intno, int is_int,
                                    int error_code, int next_eip_addend)
 {
-    raise_interrupt2(env, intno, is_int, error_code, next_eip_addend);
+    raise_interrupt2(env, intno, is_int, error_code, next_eip_addend, 0);
 }
 
 void raise_exception_err(CPUX86State *env, int exception_index,
                          int error_code)
 {
-    raise_interrupt2(env, exception_index, 0, error_code, 0);
+    raise_interrupt2(env, exception_index, 0, error_code, 0, 0);
+}
+
+void raise_exception_err_ra(CPUX86State *env, int exception_index,
+                            int error_code, uintptr_t retaddr)
+{
+    raise_interrupt2(env, exception_index, 0, error_code, 0, retaddr);
 }
 
 void raise_exception(CPUX86State *env, int exception_index)
 {
-    raise_interrupt2(env, exception_index, 0, 0, 0);
+    raise_interrupt2(env, exception_index, 0, 0, 0, 0);
+}
+
+void raise_exception_ra(CPUX86State *env, int exception_index, uintptr_t retaddr)
+{
+    raise_interrupt2(env, exception_index, 0, 0, 0, retaddr);
 }
diff --git a/target-i386/fpu_helper.c b/target-i386/fpu_helper.c
index 1f954e0c5..d421a475f 100644
--- a/target-i386/fpu_helper.c
+++ b/target-i386/fpu_helper.c
@@ -67,22 +67,24 @@ static inline void fpop(CPUX86State *env)
     env->fpstt = (env->fpstt + 1) & 7;
 }
 
-static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr)
+static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr,
+                                   uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
-    temp.l.lower = cpu_ldq_data(env, ptr);
-    temp.l.upper = cpu_lduw_data(env, ptr + 8);
+    temp.l.lower = cpu_ldq_data_ra(env, ptr, retaddr);
+    temp.l.upper = cpu_lduw_data_ra(env, ptr + 8, retaddr);
     return temp.d;
 }
 
-static inline void helper_fstt(CPUX86State *env, floatx80 f, target_ulong ptr)
+static inline void helper_fstt(CPUX86State *env, floatx80 f, target_ulong ptr,
+                               uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
     temp.d = f;
-    cpu_stq_data(env, ptr, temp.l.lower);
-    cpu_stw_data(env, ptr + 8, temp.l.upper);
+    cpu_stq_data_ra(env, ptr, temp.l.lower, retaddr);
+    cpu_stw_data_ra(env, ptr + 8, temp.l.upper, retaddr);
 }
 
 /* x87 FPU helpers */
@@ -125,10 +127,10 @@ static inline floatx80 helper_fdiv(CPUX86State *env, floatx80 a, floatx80 b)
     return floatx80_div(a, b, &env->fp_status);
 }
 
-static void fpu_raise_exception(CPUX86State *env)
+static void fpu_raise_exception(CPUX86State *env, uintptr_t retaddr)
 {
     if (env->cr[0] & CR0_NE_MASK) {
-        raise_exception(env, EXCP10_COPR);
+        raise_exception_ra(env, EXCP10_COPR, retaddr);
     }
 #if !defined(CONFIG_USER_ONLY)
     else {
@@ -313,14 +315,14 @@ void helper_fldt_ST0(CPUX86State *env, target_ulong ptr)
     int new_fpstt;
 
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt].d = helper_fldt(env, ptr);
+    env->fpregs[new_fpstt].d = helper_fldt(env, ptr, GETPC());
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
 
 void helper_fstt_ST0(CPUX86State *env, target_ulong ptr)
 {
-    helper_fstt(env, ST0, ptr);
+    helper_fstt(env, ST0, ptr, GETPC());
 }
 
 void helper_fpush(CPUX86State *env)
@@ -603,7 +605,7 @@ void helper_fclex(CPUX86State *env)
 void helper_fwait(CPUX86State *env)
 {
     if (env->fpus & FPUS_SE) {
-        fpu_raise_exception(env);
+        fpu_raise_exception(env, GETPC());
     }
 }
 
@@ -633,11 +635,11 @@ void helper_fbld_ST0(CPUX86State *env, target_ulong ptr)
 
     val = 0;
     for (i = 8; i >= 0; i--) {
-        v = cpu_ldub_data(env, ptr + i);
+        v = cpu_ldub_data_ra(env, ptr + i, GETPC());
         val = (val * 100) + ((v >> 4) * 10) + (v & 0xf);
     }
     tmp = int64_to_floatx80(val, &env->fp_status);
-    if (cpu_ldub_data(env, ptr + 9) & 0x80) {
+    if (cpu_ldub_data_ra(env, ptr + 9, GETPC()) & 0x80) {
         tmp = floatx80_chs(tmp);
     }
     fpush(env);
@@ -654,10 +656,10 @@ void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
     mem_ref = ptr;
     mem_end = mem_ref + 9;
     if (val < 0) {
-        cpu_stb_data(env, mem_end, 0x80);
+        cpu_stb_data_ra(env, mem_end, 0x80, GETPC());
         val = -val;
     } else {
-        cpu_stb_data(env, mem_end, 0x00);
+        cpu_stb_data_ra(env, mem_end, 0x00, GETPC());
     }
     while (mem_ref < mem_end) {
         if (val == 0) {
@@ -666,10 +668,10 @@ void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
         v = val % 100;
         val = val / 100;
         v = ((v / 10) << 4) | (v % 10);
-        cpu_stb_data(env, mem_ref++, v);
+        cpu_stb_data_ra(env, mem_ref++, v, GETPC());
     }
     while (mem_ref < mem_end) {
-        cpu_stb_data(env, mem_ref++, 0);
+        cpu_stb_data_ra(env, mem_ref++, 0, GETPC());
     }
 }
 
@@ -977,7 +979,8 @@ void helper_fxam_ST0(CPUX86State *env)
     }
 }
 
-void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
+static void do_fstenv(CPUX86State *env, target_ulong ptr, int data32,
+                      uintptr_t retaddr)
 {
     int fpus, fptag, exp, i;
     uint64_t mant;
@@ -1005,37 +1008,43 @@ void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
     }
     if (data32) {
         /* 32 bit */
-        cpu_stl_data(env, ptr, env->fpuc);
-        cpu_stl_data(env, ptr + 4, fpus);
-        cpu_stl_data(env, ptr + 8, fptag);
-        cpu_stl_data(env, ptr + 12, 0); /* fpip */
-        cpu_stl_data(env, ptr + 16, 0); /* fpcs */
-        cpu_stl_data(env, ptr + 20, 0); /* fpoo */
-        cpu_stl_data(env, ptr + 24, 0); /* fpos */
+        cpu_stl_data_ra(env, ptr, env->fpuc, retaddr);
+        cpu_stl_data_ra(env, ptr + 4, fpus, retaddr);
+        cpu_stl_data_ra(env, ptr + 8, fptag, retaddr);
+        cpu_stl_data_ra(env, ptr + 12, 0, retaddr); /* fpip */
+        cpu_stl_data_ra(env, ptr + 16, 0, retaddr); /* fpcs */
+        cpu_stl_data_ra(env, ptr + 20, 0, retaddr); /* fpoo */
+        cpu_stl_data_ra(env, ptr + 24, 0, retaddr); /* fpos */
     } else {
         /* 16 bit */
-        cpu_stw_data(env, ptr, env->fpuc);
-        cpu_stw_data(env, ptr + 2, fpus);
-        cpu_stw_data(env, ptr + 4, fptag);
-        cpu_stw_data(env, ptr + 6, 0);
-        cpu_stw_data(env, ptr + 8, 0);
-        cpu_stw_data(env, ptr + 10, 0);
-        cpu_stw_data(env, ptr + 12, 0);
+        cpu_stw_data_ra(env, ptr, env->fpuc, retaddr);
+        cpu_stw_data_ra(env, ptr + 2, fpus, retaddr);
+        cpu_stw_data_ra(env, ptr + 4, fptag, retaddr);
+        cpu_stw_data_ra(env, ptr + 6, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 8, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 10, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 12, 0, retaddr);
     }
 }
 
-void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
+void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fstenv(env, ptr, data32, GETPC());
+}
+
+static void do_fldenv(CPUX86State *env, target_ulong ptr, int data32,
+                      uintptr_t retaddr)
 {
     int i, fpus, fptag;
 
     if (data32) {
-        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-        fpus = cpu_lduw_data(env, ptr + 4);
-        fptag = cpu_lduw_data(env, ptr + 8);
+        cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, retaddr));
+        fpus = cpu_lduw_data_ra(env, ptr + 4, retaddr);
+        fptag = cpu_lduw_data_ra(env, ptr + 8, retaddr);
     } else {
-        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-        fpus = cpu_lduw_data(env, ptr + 2);
-        fptag = cpu_lduw_data(env, ptr + 4);
+        cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, retaddr));
+        fpus = cpu_lduw_data_ra(env, ptr + 2, retaddr);
+        fptag = cpu_lduw_data_ra(env, ptr + 4, retaddr);
     }
     env->fpstt = (fpus >> 11) & 7;
     env->fpus = fpus & ~0x3800;
@@ -1045,17 +1054,22 @@ void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
     }
 }
 
+void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fldenv(env, ptr, data32, GETPC());
+}
+
 void helper_fsave(CPUX86State *env, target_ulong ptr, int data32)
 {
     floatx80 tmp;
     int i;
 
-    helper_fstenv(env, ptr, data32);
+    do_fstenv(env, ptr, data32, GETPC());
 
     ptr += (14 << data32);
     for (i = 0; i < 8; i++) {
         tmp = ST(i);
-        helper_fstt(env, tmp, ptr);
+        helper_fstt(env, tmp, ptr, GETPC());
         ptr += 10;
     }
 
@@ -1078,11 +1092,11 @@ void helper_frstor(CPUX86State *env, target_ulong ptr, int data32)
     floatx80 tmp;
     int i;
 
-    helper_fldenv(env, ptr, data32);
+    do_fldenv(env, ptr, data32, GETPC());
     ptr += (14 << data32);
 
     for (i = 0; i < 8; i++) {
-        tmp = helper_fldt(env, ptr);
+        tmp = helper_fldt(env, ptr, GETPC());
         ST(i) = tmp;
         ptr += 10;
     }
@@ -1100,7 +1114,8 @@ void cpu_x86_frstor(CPUX86State *env, target_ulong ptr, int data32)
 }
 #endif
 
-void helper_fxsave(CPUX86State *env, target_ulong ptr, int data64)
+static void do_fxsave(CPUX86State *env, target_ulong ptr, int data64,
+                      uintptr_t retaddr)
 {
     int fpus, fptag, i, nb_xmm_regs;
     floatx80 tmp;
@@ -1108,7 +1123,7 @@ void helper_fxsave(CPUX86State *env, target_ulong ptr, int data64)
 
     /* The operand must be 16 byte aligned */
     if (ptr & 0xf) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, retaddr);
     }
 
     fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
@@ -1116,33 +1131,33 @@ void helper_fxsave(CPUX86State *env, target_ulong ptr, int data64)
     for (i = 0; i < 8; i++) {
         fptag |= (env->fptags[i] << i);
     }
-    cpu_stw_data(env, ptr, env->fpuc);
-    cpu_stw_data(env, ptr + 2, fpus);
-    cpu_stw_data(env, ptr + 4, fptag ^ 0xff);
+    cpu_stw_data_ra(env, ptr, env->fpuc, retaddr);
+    cpu_stw_data_ra(env, ptr + 2, fpus, retaddr);
+    cpu_stw_data_ra(env, ptr + 4, fptag ^ 0xff, retaddr);
 #ifdef TARGET_X86_64
     if (data64) {
-        cpu_stq_data(env, ptr + 0x08, 0); /* rip */
-        cpu_stq_data(env, ptr + 0x10, 0); /* rdp */
+        cpu_stq_data_ra(env, ptr + 0x08, 0, retaddr); /* rip */
+        cpu_stq_data_ra(env, ptr + 0x10, 0, retaddr); /* rdp */
     } else
 #endif
     {
-        cpu_stl_data(env, ptr + 0x08, 0); /* eip */
-        cpu_stl_data(env, ptr + 0x0c, 0); /* sel  */
-        cpu_stl_data(env, ptr + 0x10, 0); /* dp */
-        cpu_stl_data(env, ptr + 0x14, 0); /* sel  */
+        cpu_stl_data_ra(env, ptr + 0x08, 0, retaddr); /* eip */
+        cpu_stl_data_ra(env, ptr + 0x0c, 0, retaddr); /* sel  */
+        cpu_stl_data_ra(env, ptr + 0x10, 0, retaddr); /* dp */
+        cpu_stl_data_ra(env, ptr + 0x14, 0, retaddr); /* sel  */
     }
 
     addr = ptr + 0x20;
     for (i = 0; i < 8; i++) {
         tmp = ST(i);
-        helper_fstt(env, tmp, addr);
+        helper_fstt(env, tmp, addr, retaddr);
         addr += 16;
     }
 
     if (env->cr[4] & CR4_OSFXSR_MASK) {
         /* XXX: finish it */
-        cpu_stl_data(env, ptr + 0x18, env->mxcsr); /* mxcsr */
-        cpu_stl_data(env, ptr + 0x1c, 0x0000ffff); /* mxcsr_mask */
+        cpu_stl_data_ra(env, ptr + 0x18, env->mxcsr, retaddr); /* mxcsr */
+        cpu_stl_data_ra(env, ptr + 0x1c, 0x0000ffff, retaddr); /* mxcsr_mask */
         if (env->hflags & HF_CS64_MASK) {
             nb_xmm_regs = 16;
         } else {
@@ -1154,15 +1169,21 @@ void helper_fxsave(CPUX86State *env, target_ulong ptr, int data64)
             || (env->hflags & HF_CPL_MASK)
             || !(env->hflags & HF_LMA_MASK)) {
             for (i = 0; i < nb_xmm_regs; i++) {
-                cpu_stq_data(env, addr, env->xmm_regs[i].XMM_Q(0));
-                cpu_stq_data(env, addr + 8, env->xmm_regs[i].XMM_Q(1));
+                cpu_stq_data_ra(env, addr, env->xmm_regs[i].XMM_Q(0), retaddr);
+                cpu_stq_data_ra(env, addr + 8, env->xmm_regs[i].XMM_Q(1), retaddr);
                 addr += 16;
             }
         }
     }
 }
 
-void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
+void helper_fxsave(CPUX86State *env, target_ulong ptr, int data64)
+{
+    do_fxsave(env, ptr, data64, GETPC());
+}
+
+static void do_fxrstor(CPUX86State *env, target_ulong ptr, int data64,
+                       uintptr_t retaddr)
 {
     int i, fpus, fptag, nb_xmm_regs;
     floatx80 tmp;
@@ -1170,12 +1191,12 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
 
     /* The operand must be 16 byte aligned */
     if (ptr & 0xf) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, retaddr);
     }
 
-    cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-    fpus = cpu_lduw_data(env, ptr + 2);
-    fptag = cpu_lduw_data(env, ptr + 4);
+    cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, retaddr));
+    fpus = cpu_lduw_data_ra(env, ptr + 2, retaddr);
+    fptag = cpu_lduw_data_ra(env, ptr + 4, retaddr);
     env->fpstt = (fpus >> 11) & 7;
     env->fpus = fpus & ~0x3800;
     fptag ^= 0xff;
@@ -1185,15 +1206,15 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
 
     addr = ptr + 0x20;
     for (i = 0; i < 8; i++) {
-        tmp = helper_fldt(env, addr);
+        tmp = helper_fldt(env, addr, retaddr);
         ST(i) = tmp;
         addr += 16;
     }
 
     if (env->cr[4] & CR4_OSFXSR_MASK) {
         /* XXX: finish it */
-        cpu_set_mxcsr(env, cpu_ldl_data(env, ptr + 0x18));
-        /* cpu_ldl_data(env, ptr + 0x1c); */
+        cpu_set_mxcsr(env, cpu_ldl_data_ra(env, ptr + 0x18, retaddr));
+        /* cpu_ldl_data_ra(env, ptr + 0x1c, retaddr); */
         if (env->hflags & HF_CS64_MASK) {
             nb_xmm_regs = 16;
         } else {
@@ -1205,14 +1226,19 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
             || (env->hflags & HF_CPL_MASK)
             || !(env->hflags & HF_LMA_MASK)) {
             for (i = 0; i < nb_xmm_regs; i++) {
-                env->xmm_regs[i].XMM_Q(0) = cpu_ldq_data(env, addr);
-                env->xmm_regs[i].XMM_Q(1) = cpu_ldq_data(env, addr + 8);
+                env->xmm_regs[i].XMM_Q(0) = cpu_ldq_data_ra(env, addr, retaddr);
+                env->xmm_regs[i].XMM_Q(1) = cpu_ldq_data_ra(env, addr + 8, retaddr);
                 addr += 16;
             }
         }
     }
 }
 
+void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
+{
+    do_fxrstor(env, ptr, data64, GETPC());
+}
+
 void cpu_get_fp80(uint64_t *pmant, uint16_t *pexp, floatx80 f)
 {
     CPU_LDoubleU temp;
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 5480a96a0..d18be95c3 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -23,6 +23,7 @@
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
+#include "hw/i386/apic_internal.h"
 #endif
 
 static void cpu_x86_version(CPUX86State *env, int *family, int *model)
@@ -177,6 +178,196 @@ done:
     cpu_fprintf(f, "\n");
 }
 
+#ifndef CONFIG_USER_ONLY
+
+/* ARRAY_SIZE check is not required because
+ * DeliveryMode(dm) has a size of 3 bit.
+ */
+static inline const char *dm2str(uint32_t dm)
+{
+    static const char *str[] = {
+        "Fixed",
+        "...",
+        "SMI",
+        "...",
+        "NMI",
+        "INIT",
+        "...",
+        "ExtINT"
+    };
+    return str[dm];
+}
+
+static void dump_apic_lvt(FILE *f, fprintf_function cpu_fprintf,
+                          const char *name, uint32_t lvt, bool is_timer)
+{
+    uint32_t dm = (lvt & APIC_LVT_DELIV_MOD) >> APIC_LVT_DELIV_MOD_SHIFT;
+    cpu_fprintf(f,
+                "%s\t 0x%08x %s %-5s %-6s %-7s %-12s %-6s",
+                name, lvt,
+                lvt & APIC_LVT_INT_POLARITY ? "active-lo" : "active-hi",
+                lvt & APIC_LVT_LEVEL_TRIGGER ? "level" : "edge",
+                lvt & APIC_LVT_MASKED ? "masked" : "",
+                lvt & APIC_LVT_DELIV_STS ? "pending" : "",
+                !is_timer ?
+                    "" : lvt & APIC_LVT_TIMER_PERIODIC ?
+                            "periodic" : lvt & APIC_LVT_TIMER_TSCDEADLINE ?
+                                            "tsc-deadline" : "one-shot",
+                dm2str(dm));
+    if (dm != APIC_DM_NMI) {
+        cpu_fprintf(f, " (vec %u)\n", lvt & APIC_VECTOR_MASK);
+    } else {
+        cpu_fprintf(f, "\n");
+    }
+}
+
+/* ARRAY_SIZE check is not required because
+ * destination shorthand has a size of 2 bit.
+ */
+static inline const char *shorthand2str(uint32_t shorthand)
+{
+    const char *str[] = {
+        "no-shorthand", "self", "all-self", "all"
+    };
+    return str[shorthand];
+}
+
+static inline uint8_t divider_conf(uint32_t divide_conf)
+{
+    uint8_t divide_val = ((divide_conf & 0x8) >> 1) | (divide_conf & 0x3);
+
+    return divide_val == 7 ? 1 : 2 << divide_val;
+}
+
+static inline void mask2str(char *str, uint32_t val, uint8_t size)
+{
+    while (size--) {
+        *str++ = (val >> size) & 1 ? '1' : '0';
+    }
+    *str = 0;
+}
+
+#define MAX_LOGICAL_APIC_ID_MASK_SIZE 16
+
+static void dump_apic_icr(FILE *f, fprintf_function cpu_fprintf,
+                          APICCommonState *s, CPUX86State *env)
+{
+    uint32_t icr = s->icr[0], icr2 = s->icr[1];
+    uint8_t dest_shorthand = \
+        (icr & APIC_ICR_DEST_SHORT) >> APIC_ICR_DEST_SHORT_SHIFT;
+    bool logical_mod = icr & APIC_ICR_DEST_MOD;
+    char apic_id_str[MAX_LOGICAL_APIC_ID_MASK_SIZE + 1];
+    uint32_t dest_field;
+    bool x2apic;
+
+    cpu_fprintf(f, "ICR\t 0x%08x %s %s %s %s\n",
+                icr,
+                logical_mod ? "logical" : "physical",
+                icr & APIC_ICR_TRIGGER_MOD ? "level" : "edge",
+                icr & APIC_ICR_LEVEL ? "assert" : "de-assert",
+                shorthand2str(dest_shorthand));
+
+    cpu_fprintf(f, "ICR2\t 0x%08x", icr2);
+    if (dest_shorthand != 0) {
+        cpu_fprintf(f, "\n");
+        return;
+    }
+    x2apic = env->features[FEAT_1_ECX] & CPUID_EXT_X2APIC;
+    dest_field = x2apic ? icr2 : icr2 >> APIC_ICR_DEST_SHIFT;
+
+    if (!logical_mod) {
+        if (x2apic) {
+            cpu_fprintf(f, " cpu %u (X2APIC ID)\n", dest_field);
+        } else {
+            cpu_fprintf(f, " cpu %u (APIC ID)\n",
+                        dest_field & APIC_LOGDEST_XAPIC_ID);
+        }
+        return;
+    }
+
+    if (s->dest_mode == 0xf) { /* flat mode */
+        mask2str(apic_id_str, icr2 >> APIC_ICR_DEST_SHIFT, 8);
+        cpu_fprintf(f, " mask %s (APIC ID)\n", apic_id_str);
+    } else if (s->dest_mode == 0) { /* cluster mode */
+        if (x2apic) {
+            mask2str(apic_id_str, dest_field & APIC_LOGDEST_X2APIC_ID, 16);
+            cpu_fprintf(f, " cluster %u mask %s (X2APIC ID)\n",
+                        dest_field >> APIC_LOGDEST_X2APIC_SHIFT, apic_id_str);
+        } else {
+            mask2str(apic_id_str, dest_field & APIC_LOGDEST_XAPIC_ID, 4);
+            cpu_fprintf(f, " cluster %u mask %s (APIC ID)\n",
+                        dest_field >> APIC_LOGDEST_XAPIC_SHIFT, apic_id_str);
+        }
+    }
+}
+
+static void dump_apic_interrupt(FILE *f, fprintf_function cpu_fprintf,
+                                const char *name, uint32_t *ireg_tab,
+                                uint32_t *tmr_tab)
+{
+    int i, empty = true;
+
+    cpu_fprintf(f, "%s\t ", name);
+    for (i = 0; i < 256; i++) {
+        if (apic_get_bit(ireg_tab, i)) {
+            cpu_fprintf(f, "%u%s ", i,
+                        apic_get_bit(tmr_tab, i) ? "(level)" : "");
+            empty = false;
+        }
+    }
+    cpu_fprintf(f, "%s\n", empty ? "(none)" : "");
+}
+
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    APICCommonState *s = APIC_COMMON(cpu->apic_state);
+    uint32_t *lvt = s->lvt;
+
+    cpu_fprintf(f, "dumping local APIC state for CPU %-2u\n\n",
+                CPU(cpu)->cpu_index);
+    dump_apic_lvt(f, cpu_fprintf, "LVT0", lvt[APIC_LVT_LINT0], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVT1", lvt[APIC_LVT_LINT1], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTPC", lvt[APIC_LVT_PERFORM], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTERR", lvt[APIC_LVT_ERROR], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTTHMR", lvt[APIC_LVT_THERMAL], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTT", lvt[APIC_LVT_TIMER], true);
+
+    cpu_fprintf(f, "Timer\t DCR=0x%x (divide by %u) initial_count = %u\n",
+                s->divide_conf & APIC_DCR_MASK,
+                divider_conf(s->divide_conf),
+                s->initial_count);
+
+    cpu_fprintf(f, "SPIV\t 0x%08x APIC %s, focus=%s, spurious vec %u\n",
+                s->spurious_vec,
+                s->spurious_vec & APIC_SPURIO_ENABLED ? "enabled" : "disabled",
+                s->spurious_vec & APIC_SPURIO_FOCUS ? "on" : "off",
+                s->spurious_vec & APIC_VECTOR_MASK);
+
+    dump_apic_icr(f, cpu_fprintf, s, &cpu->env);
+
+    cpu_fprintf(f, "ESR\t 0x%08x\n", s->esr);
+
+    dump_apic_interrupt(f, cpu_fprintf, "ISR", s->isr, s->tmr);
+    dump_apic_interrupt(f, cpu_fprintf, "IRR", s->irr, s->tmr);
+
+    cpu_fprintf(f, "\nAPR 0x%02x TPR 0x%02x DFR 0x%02x LDR 0x%02x",
+                s->arb_id, s->tpr, s->dest_mode, s->log_dest);
+    if (s->dest_mode == 0) {
+        cpu_fprintf(f, "(cluster %u: id %u)",
+                    s->log_dest >> APIC_LOGDEST_XAPIC_SHIFT,
+                    s->log_dest & APIC_LOGDEST_XAPIC_ID);
+    }
+    cpu_fprintf(f, " PPR 0x%02x\n", apic_get_ppr(s));
+}
+#else
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags)
+{
+}
+#endif /* !CONFIG_USER_ONLY */
+
 #define DUMP_CODE_BYTES_TOTAL    50
 #define DUMP_CODE_BYTES_BACKWARD 20
 
@@ -905,134 +1096,6 @@ out:
     return pte | page_offset;
 }
 
-void hw_breakpoint_insert(CPUX86State *env, int index)
-{
-    CPUState *cs = CPU(x86_env_get_cpu(env));
-    int type = 0, err = 0;
-
-    switch (hw_breakpoint_type(env->dr[7], index)) {
-    case DR7_TYPE_BP_INST:
-        if (hw_breakpoint_enabled(env->dr[7], index)) {
-            err = cpu_breakpoint_insert(cs, env->dr[index], BP_CPU,
-                                        &env->cpu_breakpoint[index]);
-        }
-        break;
-    case DR7_TYPE_DATA_WR:
-        type = BP_CPU | BP_MEM_WRITE;
-        break;
-    case DR7_TYPE_IO_RW:
-        /* No support for I/O watchpoints yet */
-        break;
-    case DR7_TYPE_DATA_RW:
-        type = BP_CPU | BP_MEM_ACCESS;
-        break;
-    }
-
-    if (type != 0) {
-        err = cpu_watchpoint_insert(cs, env->dr[index],
-                                    hw_breakpoint_len(env->dr[7], index),
-                                    type, &env->cpu_watchpoint[index]);
-    }
-
-    if (err) {
-        env->cpu_breakpoint[index] = NULL;
-    }
-}
-
-void hw_breakpoint_remove(CPUX86State *env, int index)
-{
-    CPUState *cs;
-
-    if (!env->cpu_breakpoint[index]) {
-        return;
-    }
-    cs = CPU(x86_env_get_cpu(env));
-    switch (hw_breakpoint_type(env->dr[7], index)) {
-    case DR7_TYPE_BP_INST:
-        if (hw_breakpoint_enabled(env->dr[7], index)) {
-            cpu_breakpoint_remove_by_ref(cs, env->cpu_breakpoint[index]);
-        }
-        break;
-    case DR7_TYPE_DATA_WR:
-    case DR7_TYPE_DATA_RW:
-        cpu_watchpoint_remove_by_ref(cs, env->cpu_watchpoint[index]);
-        break;
-    case DR7_TYPE_IO_RW:
-        /* No support for I/O watchpoints yet */
-        break;
-    }
-}
-
-bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
-{
-    target_ulong dr6;
-    int reg;
-    bool hit_enabled = false;
-
-    dr6 = env->dr[6] & ~0xf;
-    for (reg = 0; reg < DR7_MAX_BP; reg++) {
-        bool bp_match = false;
-        bool wp_match = false;
-
-        switch (hw_breakpoint_type(env->dr[7], reg)) {
-        case DR7_TYPE_BP_INST:
-            if (env->dr[reg] == env->eip) {
-                bp_match = true;
-            }
-            break;
-        case DR7_TYPE_DATA_WR:
-        case DR7_TYPE_DATA_RW:
-            if (env->cpu_watchpoint[reg] &&
-                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
-                wp_match = true;
-            }
-            break;
-        case DR7_TYPE_IO_RW:
-            break;
-        }
-        if (bp_match || wp_match) {
-            dr6 |= 1 << reg;
-            if (hw_breakpoint_enabled(env->dr[7], reg)) {
-                hit_enabled = true;
-            }
-        }
-    }
-
-    if (hit_enabled || force_dr6_update) {
-        env->dr[6] = dr6;
-    }
-
-    return hit_enabled;
-}
-
-void breakpoint_handler(CPUState *cs)
-{
-    X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
-    CPUBreakpoint *bp;
-
-    if (cs->watchpoint_hit) {
-        if (cs->watchpoint_hit->flags & BP_CPU) {
-            cs->watchpoint_hit = NULL;
-            if (check_hw_breakpoints(env, false)) {
-                raise_exception(env, EXCP01_DB);
-            } else {
-                cpu_resume_from_signal(cs, NULL);
-            }
-        }
-    } else {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == env->eip) {
-                if (bp->flags & BP_CPU) {
-                    check_hw_breakpoints(env, true);
-                    raise_exception(env, EXCP01_DB);
-                }
-                break;
-            }
-        }
-    }
-}
-
 typedef struct MCEInjectionParams {
     Monitor *mon;
     X86CPU *cpu;
diff --git a/target-i386/helper.h b/target-i386/helper.h
index 74308f442..ecfcfd1a9 100644
--- a/target-i386/helper.h
+++ b/target-i386/helper.h
@@ -30,9 +30,9 @@ DEF_HELPER_2(verw, void, env, tl)
 DEF_HELPER_2(lldt, void, env, int)
 DEF_HELPER_2(ltr, void, env, int)
 DEF_HELPER_3(load_seg, void, env, int, int)
-DEF_HELPER_4(ljmp_protected, void, env, int, tl, int)
+DEF_HELPER_4(ljmp_protected, void, env, int, tl, tl)
 DEF_HELPER_5(lcall_real, void, env, int, tl, int, int)
-DEF_HELPER_5(lcall_protected, void, env, int, tl, int, int)
+DEF_HELPER_5(lcall_protected, void, env, int, tl, int, tl)
 DEF_HELPER_2(iret_real, void, env, int)
 DEF_HELPER_3(iret_protected, void, env, int, int)
 DEF_HELPER_3(lret_protected, void, env, int, int)
@@ -40,7 +40,8 @@ DEF_HELPER_2(read_crN, tl, env, int)
 DEF_HELPER_3(write_crN, void, env, int, tl)
 DEF_HELPER_2(lmsw, void, env, tl)
 DEF_HELPER_1(clts, void, env)
-DEF_HELPER_3(movl_drN_T0, void, env, int, tl)
+DEF_HELPER_FLAGS_3(set_dr, TCG_CALL_NO_WG, void, env, int, tl)
+DEF_HELPER_FLAGS_2(get_dr, TCG_CALL_NO_WG, tl, env, int)
 DEF_HELPER_2(invlpg, void, env, tl)
 
 DEF_HELPER_4(enter_level, void, env, int, int, tl)
@@ -92,6 +93,7 @@ DEF_HELPER_3(outw, void, env, i32, i32)
 DEF_HELPER_2(inw, tl, env, i32)
 DEF_HELPER_3(outl, void, env, i32, i32)
 DEF_HELPER_2(inl, tl, env, i32)
+DEF_HELPER_FLAGS_4(bpt_io, TCG_CALL_NO_WG, void, env, i32, i32, tl)
 
 DEF_HELPER_3(svm_check_intercept_param, void, env, i32, i64)
 DEF_HELPER_3(vmexit, void, env, i32, i64)
diff --git a/target-i386/int_helper.c b/target-i386/int_helper.c
index b0d78e6ee..3dcd25fd6 100644
--- a/target-i386/int_helper.c
+++ b/target-i386/int_helper.c
@@ -48,11 +48,11 @@ void helper_divb_AL(CPUX86State *env, target_ulong t0)
     num = (env->regs[R_EAX] & 0xffff);
     den = (t0 & 0xff);
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q > 0xff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xff;
     r = (num % den) & 0xff;
@@ -66,11 +66,11 @@ void helper_idivb_AL(CPUX86State *env, target_ulong t0)
     num = (int16_t)env->regs[R_EAX];
     den = (int8_t)t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q != (int8_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xff;
     r = (num % den) & 0xff;
@@ -84,11 +84,11 @@ void helper_divw_AX(CPUX86State *env, target_ulong t0)
     num = (env->regs[R_EAX] & 0xffff) | ((env->regs[R_EDX] & 0xffff) << 16);
     den = (t0 & 0xffff);
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q > 0xffff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xffff;
     r = (num % den) & 0xffff;
@@ -103,11 +103,11 @@ void helper_idivw_AX(CPUX86State *env, target_ulong t0)
     num = (env->regs[R_EAX] & 0xffff) | ((env->regs[R_EDX] & 0xffff) << 16);
     den = (int16_t)t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q != (int16_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xffff;
     r = (num % den) & 0xffff;
@@ -123,12 +123,12 @@ void helper_divl_EAX(CPUX86State *env, target_ulong t0)
     num = ((uint32_t)env->regs[R_EAX]) | ((uint64_t)((uint32_t)env->regs[R_EDX]) << 32);
     den = t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     r = (num % den);
     if (q > 0xffffffff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = (uint32_t)q;
     env->regs[R_EDX] = (uint32_t)r;
@@ -142,12 +142,12 @@ void helper_idivl_EAX(CPUX86State *env, target_ulong t0)
     num = ((uint32_t)env->regs[R_EAX]) | ((uint64_t)((uint32_t)env->regs[R_EDX]) << 32);
     den = t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     r = (num % den);
     if (q != (int32_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = (uint32_t)q;
     env->regs[R_EDX] = (uint32_t)r;
@@ -379,12 +379,12 @@ void helper_divq_EAX(CPUX86State *env, target_ulong t0)
     uint64_t r0, r1;
 
     if (t0 == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     r0 = env->regs[R_EAX];
     r1 = env->regs[R_EDX];
     if (div64(&r0, &r1, t0)) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = r0;
     env->regs[R_EDX] = r1;
@@ -395,12 +395,12 @@ void helper_idivq_EAX(CPUX86State *env, target_ulong t0)
     uint64_t r0, r1;
 
     if (t0 == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     r0 = env->regs[R_EAX];
     r1 = env->regs[R_EDX];
     if (idiv64(&r0, &r1, t0)) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = r0;
     env->regs[R_EDX] = r1;
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index 066d03d99..36fa3f071 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -28,12 +28,13 @@
 #include "exec/gdbstub.h"
 #include "qemu/host-utils.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/apic.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/i386/apic-msidef.h"
 #include "exec/ioport.h"
-#include <asm/hyperv.h>
+#include "standard-headers/asm-x86/hyperv.h"
 #include "hw/pci/pci.h"
 #include "migration/migration.h"
 #include "exec/memattrs.h"
@@ -67,6 +68,7 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 
 static bool has_msr_star;
 static bool has_msr_hsave_pa;
+static bool has_msr_tsc_aux;
 static bool has_msr_tsc_adjust;
 static bool has_msr_tsc_deadline;
 static bool has_msr_feature_control;
@@ -80,12 +82,25 @@ static int lm_capable_kernel;
 static bool has_msr_hv_hypercall;
 static bool has_msr_hv_vapic;
 static bool has_msr_hv_tsc;
+static bool has_msr_hv_crash;
+static bool has_msr_hv_reset;
+static bool has_msr_hv_vpindex;
+static bool has_msr_hv_runtime;
 static bool has_msr_mtrr;
 static bool has_msr_xss;
 
 static bool has_msr_architectural_pmu;
 static uint32_t num_architectural_pmu_counters;
 
+static int has_xsave;
+static int has_xcrs;
+static int has_pit_state2;
+
+int kvm_has_pit_state2(void)
+{
+    return has_pit_state2;
+}
+
 bool kvm_has_smm(void)
 {
     return kvm_check_extension(kvm_state, KVM_CAP_X86_SMM);
@@ -96,6 +111,51 @@ bool kvm_allows_irq0_override(void)
     return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
 }
 
+static int kvm_get_tsc(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    struct {
+        struct kvm_msrs info;
+        struct kvm_msr_entry entries[1];
+    } msr_data;
+    int ret;
+
+    if (env->tsc_valid) {
+        return 0;
+    }
+
+    msr_data.info.nmsrs = 1;
+    msr_data.entries[0].index = MSR_IA32_TSC;
+    env->tsc_valid = !runstate_is_running();
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    env->tsc = msr_data.entries[0].data;
+    return 0;
+}
+
+static inline void do_kvm_synchronize_tsc(void *arg)
+{
+    CPUState *cpu = arg;
+
+    kvm_get_tsc(cpu);
+}
+
+void kvm_synchronize_all_tsc(void)
+{
+    CPUState *cpu;
+
+    if (kvm_enabled()) {
+        CPU_FOREACH(cpu) {
+            run_on_cpu(cpu, do_kvm_synchronize_tsc, cpu);
+        }
+    }
+}
+
 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
 {
     struct kvm_cpuid2 *cpuid;
@@ -457,7 +517,11 @@ static bool hyperv_enabled(X86CPU *cpu)
     return kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0 &&
            (hyperv_hypercall_available(cpu) ||
             cpu->hyperv_time  ||
-            cpu->hyperv_relaxed_timing);
+            cpu->hyperv_relaxed_timing ||
+            cpu->hyperv_crash ||
+            cpu->hyperv_reset ||
+            cpu->hyperv_vpindex ||
+            cpu->hyperv_runtime);
 }
 
 static Error *invtsc_mig_blocker;
@@ -487,7 +551,18 @@ int kvm_arch_init_vcpu(CPUState *cs)
     if (hyperv_enabled(cpu)) {
         c = &cpuid_data.entries[cpuid_i++];
         c->function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
-        memcpy(signature, "Microsoft Hv", 12);
+        if (!cpu->hyperv_vendor_id) {
+            memcpy(signature, "Microsoft Hv", 12);
+        } else {
+            size_t len = strlen(cpu->hyperv_vendor_id);
+
+            if (len > 12) {
+                error_report("hv-vendor-id truncated to 12 characters");
+                len = 12;
+            }
+            memset(signature, 0, 12);
+            memcpy(signature, cpu->hyperv_vendor_id, len);
+        }
         c->eax = HYPERV_CPUID_MIN;
         c->ebx = signature[0];
         c->ecx = signature[1];
@@ -523,6 +598,19 @@ int kvm_arch_init_vcpu(CPUState *cs)
             c->eax |= 0x200;
             has_msr_hv_tsc = true;
         }
+        if (cpu->hyperv_crash && has_msr_hv_crash) {
+            c->edx |= HV_X64_GUEST_CRASH_MSR_AVAILABLE;
+        }
+        c->edx |= HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
+        if (cpu->hyperv_reset && has_msr_hv_reset) {
+            c->eax |= HV_X64_MSR_RESET_AVAILABLE;
+        }
+        if (cpu->hyperv_vpindex && has_msr_hv_vpindex) {
+            c->eax |= HV_X64_MSR_VP_INDEX_AVAILABLE;
+        }
+        if (cpu->hyperv_runtime && has_msr_hv_runtime) {
+            c->eax |= HV_X64_MSR_VP_RUNTIME_AVAILABLE;
+        }
         c = &cpuid_data.entries[cpuid_i++];
         c->function = HYPERV_CPUID_ENLIGHTMENT_INFO;
         if (cpu->hyperv_relaxed_timing) {
@@ -687,7 +775,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
         && (env->features[FEAT_1_EDX] & (CPUID_MCE | CPUID_MCA)) ==
            (CPUID_MCE | CPUID_MCA)
         && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) {
-        uint64_t mcg_cap;
+        uint64_t mcg_cap, unsupported_caps;
         int banks;
         int ret;
 
@@ -697,18 +785,24 @@ int kvm_arch_init_vcpu(CPUState *cs)
             return ret;
         }
 
-        if (banks > MCE_BANKS_DEF) {
-            banks = MCE_BANKS_DEF;
+        if (banks < (env->mcg_cap & MCG_CAP_BANKS_MASK)) {
+            error_report("kvm: Unsupported MCE bank count (QEMU = %d, KVM = %d)",
+                         (int)(env->mcg_cap & MCG_CAP_BANKS_MASK), banks);
+            return -ENOTSUP;
         }
-        mcg_cap &= MCE_CAP_DEF;
-        mcg_cap |= banks;
-        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &mcg_cap);
+
+        unsupported_caps = env->mcg_cap & ~(mcg_cap | MCG_CAP_BANKS_MASK);
+        if (unsupported_caps) {
+            error_report("warning: Unsupported MCG_CAP bits: 0x%" PRIx64,
+                         unsupported_caps);
+        }
+
+        env->mcg_cap &= mcg_cap | MCG_CAP_BANKS_MASK;
+        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &env->mcg_cap);
         if (ret < 0) {
             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
             return ret;
         }
-
-        env->mcg_cap = mcg_cap;
     }
 
     qemu_add_vm_change_state_handler(cpu_update_state, env);
@@ -745,7 +839,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
         }
     }
 
-    if (kvm_has_xsave()) {
+    if (has_xsave) {
         env->kvm_xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
     }
 
@@ -819,6 +913,10 @@ static int kvm_get_supported_msrs(KVMState *s)
                     has_msr_hsave_pa = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == MSR_TSC_AUX) {
+                    has_msr_tsc_aux = true;
+                    continue;
+                }
                 if (kvm_msr_list->indices[i] == MSR_TSC_ADJUST) {
                     has_msr_tsc_adjust = true;
                     continue;
@@ -843,6 +941,22 @@ static int kvm_get_supported_msrs(KVMState *s)
                     has_msr_xss = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_CRASH_CTL) {
+                    has_msr_hv_crash = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_RESET) {
+                    has_msr_hv_reset = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_VP_INDEX) {
+                    has_msr_hv_vpindex = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_VP_RUNTIME) {
+                    has_msr_hv_runtime = true;
+                    continue;
+                }
             }
         }
 
@@ -893,6 +1007,18 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     int ret;
     struct utsname utsname;
 
+#ifdef KVM_CAP_XSAVE
+    has_xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
+#endif
+
+#ifdef KVM_CAP_XCRS
+    has_xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
+#endif
+
+#ifdef KVM_CAP_PIT_STATE2
+    has_pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
+#endif
+
     ret = kvm_get_supported_msrs(s);
     if (ret < 0) {
         return ret;
@@ -1101,7 +1227,7 @@ static int kvm_put_xsave(X86CPU *cpu)
     uint8_t *xmm, *ymmh, *zmmh;
     int i, r;
 
-    if (!kvm_has_xsave()) {
+    if (!has_xsave) {
         return kvm_put_fpu(cpu);
     }
 
@@ -1155,7 +1281,7 @@ static int kvm_put_xcrs(X86CPU *cpu)
     CPUX86State *env = &cpu->env;
     struct kvm_xcrs xcrs = {};
 
-    if (!kvm_has_xcrs()) {
+    if (!has_xcrs) {
         return 0;
     }
 
@@ -1289,6 +1415,9 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
     if (has_msr_hsave_pa) {
         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
     }
+    if (has_msr_tsc_aux) {
+        kvm_msr_entry_set(&msrs[n++], MSR_TSC_AUX, env->tsc_aux);
+    }
     if (has_msr_tsc_adjust) {
         kvm_msr_entry_set(&msrs[n++], MSR_TSC_ADJUST, env->tsc_adjust);
     }
@@ -1375,6 +1504,20 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_REFERENCE_TSC,
                               env->msr_hv_tsc);
         }
+        if (has_msr_hv_crash) {
+            int j;
+
+            for (j = 0; j < HV_X64_MSR_CRASH_PARAMS; j++)
+                kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_CRASH_P0 + j,
+                                  env->msr_hv_crash_params[j]);
+
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_CRASH_CTL,
+                              HV_X64_MSR_CRASH_CTL_NOTIFY);
+        }
+        if (has_msr_hv_runtime) {
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_VP_RUNTIME,
+                              env->msr_hv_runtime);
+        }
         if (has_msr_mtrr) {
             kvm_msr_entry_set(&msrs[n++], MSR_MTRRdefType, env->mtrr_deftype);
             kvm_msr_entry_set(&msrs[n++],
@@ -1467,7 +1610,7 @@ static int kvm_get_xsave(X86CPU *cpu)
     const uint8_t *xmm, *ymmh, *zmmh;
     uint16_t cwd, swd, twd;
 
-    if (!kvm_has_xsave()) {
+    if (!has_xsave) {
         return kvm_get_fpu(cpu);
     }
 
@@ -1526,7 +1669,7 @@ static int kvm_get_xcrs(X86CPU *cpu)
     int i, ret;
     struct kvm_xcrs xcrs;
 
-    if (!kvm_has_xcrs()) {
+    if (!has_xcrs) {
         return 0;
     }
 
@@ -1651,6 +1794,9 @@ static int kvm_get_msrs(X86CPU *cpu)
     if (has_msr_hsave_pa) {
         msrs[n++].index = MSR_VM_HSAVE_PA;
     }
+    if (has_msr_tsc_aux) {
+        msrs[n++].index = MSR_TSC_AUX;
+    }
     if (has_msr_tsc_adjust) {
         msrs[n++].index = MSR_TSC_ADJUST;
     }
@@ -1730,6 +1876,16 @@ static int kvm_get_msrs(X86CPU *cpu)
     if (has_msr_hv_tsc) {
         msrs[n++].index = HV_X64_MSR_REFERENCE_TSC;
     }
+    if (has_msr_hv_crash) {
+        int j;
+
+        for (j = 0; j < HV_X64_MSR_CRASH_PARAMS; j++) {
+            msrs[n++].index = HV_X64_MSR_CRASH_P0 + j;
+        }
+    }
+    if (has_msr_hv_runtime) {
+        msrs[n++].index = HV_X64_MSR_VP_RUNTIME;
+    }
     if (has_msr_mtrr) {
         msrs[n++].index = MSR_MTRRdefType;
         msrs[n++].index = MSR_MTRRfix64K_00000;
@@ -1793,6 +1949,9 @@ static int kvm_get_msrs(X86CPU *cpu)
         case MSR_IA32_TSC:
             env->tsc = msrs[i].data;
             break;
+        case MSR_TSC_AUX:
+            env->tsc_aux = msrs[i].data;
+            break;
         case MSR_TSC_ADJUST:
             env->tsc_adjust = msrs[i].data;
             break;
@@ -1877,6 +2036,12 @@ static int kvm_get_msrs(X86CPU *cpu)
         case HV_X64_MSR_REFERENCE_TSC:
             env->msr_hv_tsc = msrs[i].data;
             break;
+        case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
+            env->msr_hv_crash_params[index - HV_X64_MSR_CRASH_P0] = msrs[i].data;
+            break;
+        case HV_X64_MSR_VP_RUNTIME:
+            env->msr_hv_runtime = msrs[i].data;
+            break;
         case MSR_MTRRdefType:
             env->mtrr_deftype = msrs[i].data;
             break;
@@ -2912,7 +3077,7 @@ int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id)
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
diff --git a/target-i386/kvm_i386.h b/target-i386/kvm_i386.h
index e557e94f4..c1b312ba2 100644
--- a/target-i386/kvm_i386.h
+++ b/target-i386/kvm_i386.h
@@ -15,6 +15,7 @@
 
 bool kvm_allows_irq0_override(void);
 bool kvm_has_smm(void);
+void kvm_synchronize_all_tsc(void);
 void kvm_arch_reset_vcpu(X86CPU *cs);
 void kvm_arch_do_init_vcpu(X86CPU *cs);
 
diff --git a/target-i386/machine.c b/target-i386/machine.c
index a0df64b57..a18e16e0d 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -367,8 +367,12 @@ static int cpu_post_load(void *opaque, int version_id)
 
     cpu_breakpoint_remove_all(cs, BP_CPU);
     cpu_watchpoint_remove_all(cs, BP_CPU);
-    for (i = 0; i < DR7_MAX_BP; i++) {
-        hw_breakpoint_insert(env, i);
+    {
+        /* Indicate all breakpoints disabled, as they are, then
+           let the helper re-enable them.  */
+        target_ulong dr7 = env->dr[7];
+        env->dr[7] = dr7 & ~(DR7_GLOBAL_BP_MASK | DR7_LOCAL_BP_MASK);
+        cpu_x86_update_dr7(env, dr7);
     }
     tlb_flush(cs, 1);
 
@@ -661,6 +665,51 @@ static const VMStateDescription vmstate_msr_hyperv_time = {
     }
 };
 
+static bool hyperv_crash_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < HV_X64_MSR_CRASH_PARAMS; i++) {
+        if (env->msr_hv_crash_params[i]) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_crash = {
+    .name = "cpu/msr_hyperv_crash",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_crash_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64_ARRAY(env.msr_hv_crash_params,
+                             X86CPU, HV_X64_MSR_CRASH_PARAMS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool hyperv_runtime_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->msr_hv_runtime != 0;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_runtime = {
+    .name = "cpu/msr_hyperv_runtime",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_runtime_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.msr_hv_runtime, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static bool avx512_needed(void *opaque)
 {
     X86CPU *cpu = opaque;
@@ -842,6 +891,8 @@ VMStateDescription vmstate_x86_cpu = {
         &vmstate_msr_hypercall_hypercall,
         &vmstate_msr_hyperv_vapic,
         &vmstate_msr_hyperv_time,
+        &vmstate_msr_hyperv_crash,
+        &vmstate_msr_hyperv_runtime,
         &vmstate_avx512,
         &vmstate_xss,
         NULL
diff --git a/target-i386/mem_helper.c b/target-i386/mem_helper.c
index 1aec8a5f1..0e20df2f7 100644
--- a/target-i386/mem_helper.c
+++ b/target-i386/mem_helper.c
@@ -23,31 +23,51 @@
 
 /* broken thread support */
 
-static spinlock_t global_cpu_lock = SPIN_LOCK_UNLOCKED;
+#if defined(CONFIG_USER_ONLY)
+QemuMutex global_cpu_lock;
 
 void helper_lock(void)
 {
-    spin_lock(&global_cpu_lock);
+    qemu_mutex_lock(&global_cpu_lock);
 }
 
 void helper_unlock(void)
 {
-    spin_unlock(&global_cpu_lock);
+    qemu_mutex_unlock(&global_cpu_lock);
 }
 
+void helper_lock_init(void)
+{
+    qemu_mutex_init(&global_cpu_lock);
+}
+#else
+void helper_lock(void)
+{
+}
+
+void helper_unlock(void)
+{
+}
+
+void helper_lock_init(void)
+{
+}
+#endif
+
 void helper_cmpxchg8b(CPUX86State *env, target_ulong a0)
 {
     uint64_t d;
     int eflags;
 
     eflags = cpu_cc_compute_all(env, CC_OP);
-    d = cpu_ldq_data(env, a0);
+    d = cpu_ldq_data_ra(env, a0, GETPC());
     if (d == (((uint64_t)env->regs[R_EDX] << 32) | (uint32_t)env->regs[R_EAX])) {
-        cpu_stq_data(env, a0, ((uint64_t)env->regs[R_ECX] << 32) | (uint32_t)env->regs[R_EBX]);
+        cpu_stq_data_ra(env, a0, ((uint64_t)env->regs[R_ECX] << 32)
+                                  | (uint32_t)env->regs[R_EBX], GETPC());
         eflags |= CC_Z;
     } else {
         /* always do the store */
-        cpu_stq_data(env, a0, d);
+        cpu_stq_data_ra(env, a0, d, GETPC());
         env->regs[R_EDX] = (uint32_t)(d >> 32);
         env->regs[R_EAX] = (uint32_t)d;
         eflags &= ~CC_Z;
@@ -62,19 +82,19 @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
     int eflags;
 
     if ((a0 & 0xf) != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     eflags = cpu_cc_compute_all(env, CC_OP);
-    d0 = cpu_ldq_data(env, a0);
-    d1 = cpu_ldq_data(env, a0 + 8);
+    d0 = cpu_ldq_data_ra(env, a0, GETPC());
+    d1 = cpu_ldq_data_ra(env, a0 + 8, GETPC());
     if (d0 == env->regs[R_EAX] && d1 == env->regs[R_EDX]) {
-        cpu_stq_data(env, a0, env->regs[R_EBX]);
-        cpu_stq_data(env, a0 + 8, env->regs[R_ECX]);
+        cpu_stq_data_ra(env, a0, env->regs[R_EBX], GETPC());
+        cpu_stq_data_ra(env, a0 + 8, env->regs[R_ECX], GETPC());
         eflags |= CC_Z;
     } else {
         /* always do the store */
-        cpu_stq_data(env, a0, d0);
-        cpu_stq_data(env, a0 + 8, d1);
+        cpu_stq_data_ra(env, a0, d0, GETPC());
+        cpu_stq_data_ra(env, a0 + 8, d1, GETPC());
         env->regs[R_EDX] = d1;
         env->regs[R_EAX] = d0;
         eflags &= ~CC_Z;
@@ -87,11 +107,11 @@ void helper_boundw(CPUX86State *env, target_ulong a0, int v)
 {
     int low, high;
 
-    low = cpu_ldsw_data(env, a0);
-    high = cpu_ldsw_data(env, a0 + 2);
+    low = cpu_ldsw_data_ra(env, a0, GETPC());
+    high = cpu_ldsw_data_ra(env, a0 + 2, GETPC());
     v = (int16_t)v;
     if (v < low || v > high) {
-        raise_exception(env, EXCP05_BOUND);
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
     }
 }
 
@@ -99,10 +119,10 @@ void helper_boundl(CPUX86State *env, target_ulong a0, int v)
 {
     int low, high;
 
-    low = cpu_ldl_data(env, a0);
-    high = cpu_ldl_data(env, a0 + 4);
+    low = cpu_ldl_data_ra(env, a0, GETPC());
+    high = cpu_ldl_data_ra(env, a0 + 4, GETPC());
     if (v < low || v > high) {
-        raise_exception(env, EXCP05_BOUND);
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
     }
 }
 
@@ -122,11 +142,7 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
         X86CPU *cpu = X86_CPU(cs);
         CPUX86State *env = &cpu->env;
 
-        if (retaddr) {
-            /* now we have a real cpu fault */
-            cpu_restore_state(cs, retaddr);
-        }
-        raise_exception_err(env, cs->exception_index, env->error_code);
+        raise_exception_err_ra(env, cs->exception_index, env->error_code, retaddr);
     }
 }
 #endif
diff --git a/target-i386/misc_helper.c b/target-i386/misc_helper.c
index 52c5d65e9..13bd4f5ee 100644
--- a/target-i386/misc_helper.c
+++ b/target-i386/misc_helper.c
@@ -95,15 +95,6 @@ void helper_into(CPUX86State *env, int next_eip_addend)
     }
 }
 
-void helper_single_step(CPUX86State *env)
-{
-#ifndef CONFIG_USER_ONLY
-    check_hw_breakpoints(env, true);
-    env->dr[6] |= DR6_BS;
-#endif
-    raise_exception(env, EXCP01_DB);
-}
-
 void helper_cpuid(CPUX86State *env)
 {
     uint32_t eax, ebx, ecx, edx;
@@ -127,10 +118,6 @@ target_ulong helper_read_crN(CPUX86State *env, int reg)
 void helper_write_crN(CPUX86State *env, int reg, target_ulong t0)
 {
 }
-
-void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
-{
-}
 #else
 target_ulong helper_read_crN(CPUX86State *env, int reg)
 {
@@ -176,27 +163,6 @@ void helper_write_crN(CPUX86State *env, int reg, target_ulong t0)
         break;
     }
 }
-
-void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
-{
-    int i;
-
-    if (reg < 4) {
-        hw_breakpoint_remove(env, reg);
-        env->dr[reg] = t0;
-        hw_breakpoint_insert(env, reg);
-    } else if (reg == 7) {
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            hw_breakpoint_remove(env, i);
-        }
-        env->dr[7] = t0;
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            hw_breakpoint_insert(env, i);
-        }
-    } else {
-        env->dr[reg] = t0;
-    }
-}
 #endif
 
 void helper_lmsw(CPUX86State *env, target_ulong t0)
@@ -220,7 +186,7 @@ void helper_rdtsc(CPUX86State *env)
     uint64_t val;
 
     if ((env->cr[4] & CR4_TSD_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_RDTSC, 0);
 
@@ -238,7 +204,7 @@ void helper_rdtscp(CPUX86State *env)
 void helper_rdpmc(CPUX86State *env)
 {
     if ((env->cr[4] & CR4_PCE_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_RDPMC, 0);
 
@@ -589,7 +555,7 @@ void helper_hlt(CPUX86State *env, int next_eip_addend)
 void helper_monitor(CPUX86State *env, target_ulong ptr)
 {
     if ((uint32_t)env->regs[R_ECX] != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     /* XXX: store address? */
     cpu_svm_check_intercept_param(env, SVM_EXIT_MONITOR, 0);
@@ -601,7 +567,7 @@ void helper_mwait(CPUX86State *env, int next_eip_addend)
     X86CPU *cpu;
 
     if ((uint32_t)env->regs[R_ECX] != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_MWAIT, 0);
     env->eip += next_eip_addend;
diff --git a/target-i386/monitor.c b/target-i386/monitor.c
new file mode 100644
index 000000000..aac6b1ba8
--- /dev/null
+++ b/target-i386/monitor.c
@@ -0,0 +1,511 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hw/i386/pc.h"
+#include "sysemu/kvm.h"
+#include "hmp.h"
+
+
+static void print_pte(Monitor *mon, hwaddr addr,
+                      hwaddr pte,
+                      hwaddr mask)
+{
+#ifdef TARGET_X86_64
+    if (addr & (1ULL << 47)) {
+        addr |= -1LL << 48;
+    }
+#endif
+    monitor_printf(mon, TARGET_FMT_plx ": " TARGET_FMT_plx
+                   " %c%c%c%c%c%c%c%c%c\n",
+                   addr,
+                   pte & mask,
+                   pte & PG_NX_MASK ? 'X' : '-',
+                   pte & PG_GLOBAL_MASK ? 'G' : '-',
+                   pte & PG_PSE_MASK ? 'P' : '-',
+                   pte & PG_DIRTY_MASK ? 'D' : '-',
+                   pte & PG_ACCESSED_MASK ? 'A' : '-',
+                   pte & PG_PCD_MASK ? 'C' : '-',
+                   pte & PG_PWT_MASK ? 'T' : '-',
+                   pte & PG_USER_MASK ? 'U' : '-',
+                   pte & PG_RW_MASK ? 'W' : '-');
+}
+
+static void tlb_info_32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2;
+    uint32_t pgd, pde, pte;
+
+    pgd = env->cr[3] & ~0xfff;
+    for(l1 = 0; l1 < 1024; l1++) {
+        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
+        pde = le32_to_cpu(pde);
+        if (pde & PG_PRESENT_MASK) {
+            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+                /* 4M pages */
+                print_pte(mon, (l1 << 22), pde, ~((1 << 21) - 1));
+            } else {
+                for(l2 = 0; l2 < 1024; l2++) {
+                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
+                    pte = le32_to_cpu(pte);
+                    if (pte & PG_PRESENT_MASK) {
+                        print_pte(mon, (l1 << 22) + (l2 << 12),
+                                  pte & ~PG_PSE_MASK,
+                                  ~0xfff);
+                    }
+                }
+            }
+        }
+    }
+}
+
+static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2, l3;
+    uint64_t pdpe, pde, pte;
+    uint64_t pdp_addr, pd_addr, pt_addr;
+
+    pdp_addr = env->cr[3] & ~0x1f;
+    for (l1 = 0; l1 < 4; l1++) {
+        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
+        pdpe = le64_to_cpu(pdpe);
+        if (pdpe & PG_PRESENT_MASK) {
+            pd_addr = pdpe & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
+                pde = le64_to_cpu(pde);
+                if (pde & PG_PRESENT_MASK) {
+                    if (pde & PG_PSE_MASK) {
+                        /* 2M pages with PAE, CR4.PSE is ignored */
+                        print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
+                                  ~((hwaddr)(1 << 20) - 1));
+                    } else {
+                        pt_addr = pde & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
+                            pte = le64_to_cpu(pte);
+                            if (pte & PG_PRESENT_MASK) {
+                                print_pte(mon, (l1 << 30 ) + (l2 << 21)
+                                          + (l3 << 12),
+                                          pte & ~PG_PSE_MASK,
+                                          ~(hwaddr)0xfff);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#ifdef TARGET_X86_64
+static void tlb_info_64(Monitor *mon, CPUArchState *env)
+{
+    uint64_t l1, l2, l3, l4;
+    uint64_t pml4e, pdpe, pde, pte;
+    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr;
+
+    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
+    for (l1 = 0; l1 < 512; l1++) {
+        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
+        pml4e = le64_to_cpu(pml4e);
+        if (pml4e & PG_PRESENT_MASK) {
+            pdp_addr = pml4e & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
+                pdpe = le64_to_cpu(pdpe);
+                if (pdpe & PG_PRESENT_MASK) {
+                    if (pdpe & PG_PSE_MASK) {
+                        /* 1G pages, CR4.PSE is ignored */
+                        print_pte(mon, (l1 << 39) + (l2 << 30), pdpe,
+                                  0x3ffffc0000000ULL);
+                    } else {
+                        pd_addr = pdpe & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
+                            pde = le64_to_cpu(pde);
+                            if (pde & PG_PRESENT_MASK) {
+                                if (pde & PG_PSE_MASK) {
+                                    /* 2M pages, CR4.PSE is ignored */
+                                    print_pte(mon, (l1 << 39) + (l2 << 30) +
+                                              (l3 << 21), pde,
+                                              0x3ffffffe00000ULL);
+                                } else {
+                                    pt_addr = pde & 0x3fffffffff000ULL;
+                                    for (l4 = 0; l4 < 512; l4++) {
+                                        cpu_physical_memory_read(pt_addr
+                                                                 + l4 * 8,
+                                                                 &pte, 8);
+                                        pte = le64_to_cpu(pte);
+                                        if (pte & PG_PRESENT_MASK) {
+                                            print_pte(mon, (l1 << 39) +
+                                                      (l2 << 30) +
+                                                      (l3 << 21) + (l4 << 12),
+                                                      pte & ~PG_PSE_MASK,
+                                                      0x3fffffffff000ULL);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+#endif /* TARGET_X86_64 */
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env;
+
+    env = mon_get_cpu_env();
+
+    if (!(env->cr[0] & CR0_PG_MASK)) {
+        monitor_printf(mon, "PG disabled\n");
+        return;
+    }
+    if (env->cr[4] & CR4_PAE_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            tlb_info_64(mon, env);
+        } else
+#endif
+        {
+            tlb_info_pae32(mon, env);
+        }
+    } else {
+        tlb_info_32(mon, env);
+    }
+}
+
+static void mem_print(Monitor *mon, hwaddr *pstart,
+                      int *plast_prot,
+                      hwaddr end, int prot)
+{
+    int prot1;
+    prot1 = *plast_prot;
+    if (prot != prot1) {
+        if (*pstart != -1) {
+            monitor_printf(mon, TARGET_FMT_plx "-" TARGET_FMT_plx " "
+                           TARGET_FMT_plx " %c%c%c\n",
+                           *pstart, end, end - *pstart,
+                           prot1 & PG_USER_MASK ? 'u' : '-',
+                           'r',
+                           prot1 & PG_RW_MASK ? 'w' : '-');
+        }
+        if (prot != 0)
+            *pstart = end;
+        else
+            *pstart = -1;
+        *plast_prot = prot;
+    }
+}
+
+static void mem_info_32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2;
+    int prot, last_prot;
+    uint32_t pgd, pde, pte;
+    hwaddr start, end;
+
+    pgd = env->cr[3] & ~0xfff;
+    last_prot = 0;
+    start = -1;
+    for(l1 = 0; l1 < 1024; l1++) {
+        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
+        pde = le32_to_cpu(pde);
+        end = l1 << 22;
+        if (pde & PG_PRESENT_MASK) {
+            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+                prot = pde & (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
+                mem_print(mon, &start, &last_prot, end, prot);
+            } else {
+                for(l2 = 0; l2 < 1024; l2++) {
+                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
+                    pte = le32_to_cpu(pte);
+                    end = (l1 << 22) + (l2 << 12);
+                    if (pte & PG_PRESENT_MASK) {
+                        prot = pte & pde &
+                            (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
+                    } else {
+                        prot = 0;
+                    }
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
+}
+
+static void mem_info_pae32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2, l3;
+    int prot, last_prot;
+    uint64_t pdpe, pde, pte;
+    uint64_t pdp_addr, pd_addr, pt_addr;
+    hwaddr start, end;
+
+    pdp_addr = env->cr[3] & ~0x1f;
+    last_prot = 0;
+    start = -1;
+    for (l1 = 0; l1 < 4; l1++) {
+        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
+        pdpe = le64_to_cpu(pdpe);
+        end = l1 << 30;
+        if (pdpe & PG_PRESENT_MASK) {
+            pd_addr = pdpe & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
+                pde = le64_to_cpu(pde);
+                end = (l1 << 30) + (l2 << 21);
+                if (pde & PG_PRESENT_MASK) {
+                    if (pde & PG_PSE_MASK) {
+                        prot = pde & (PG_USER_MASK | PG_RW_MASK |
+                                      PG_PRESENT_MASK);
+                        mem_print(mon, &start, &last_prot, end, prot);
+                    } else {
+                        pt_addr = pde & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
+                            pte = le64_to_cpu(pte);
+                            end = (l1 << 30) + (l2 << 21) + (l3 << 12);
+                            if (pte & PG_PRESENT_MASK) {
+                                prot = pte & pde & (PG_USER_MASK | PG_RW_MASK |
+                                                    PG_PRESENT_MASK);
+                            } else {
+                                prot = 0;
+                            }
+                            mem_print(mon, &start, &last_prot, end, prot);
+                        }
+                    }
+                } else {
+                    prot = 0;
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
+}
+
+
+#ifdef TARGET_X86_64
+static void mem_info_64(Monitor *mon, CPUArchState *env)
+{
+    int prot, last_prot;
+    uint64_t l1, l2, l3, l4;
+    uint64_t pml4e, pdpe, pde, pte;
+    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr, start, end;
+
+    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
+    last_prot = 0;
+    start = -1;
+    for (l1 = 0; l1 < 512; l1++) {
+        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
+        pml4e = le64_to_cpu(pml4e);
+        end = l1 << 39;
+        if (pml4e & PG_PRESENT_MASK) {
+            pdp_addr = pml4e & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
+                pdpe = le64_to_cpu(pdpe);
+                end = (l1 << 39) + (l2 << 30);
+                if (pdpe & PG_PRESENT_MASK) {
+                    if (pdpe & PG_PSE_MASK) {
+                        prot = pdpe & (PG_USER_MASK | PG_RW_MASK |
+                                       PG_PRESENT_MASK);
+                        prot &= pml4e;
+                        mem_print(mon, &start, &last_prot, end, prot);
+                    } else {
+                        pd_addr = pdpe & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
+                            pde = le64_to_cpu(pde);
+                            end = (l1 << 39) + (l2 << 30) + (l3 << 21);
+                            if (pde & PG_PRESENT_MASK) {
+                                if (pde & PG_PSE_MASK) {
+                                    prot = pde & (PG_USER_MASK | PG_RW_MASK |
+                                                  PG_PRESENT_MASK);
+                                    prot &= pml4e & pdpe;
+                                    mem_print(mon, &start, &last_prot, end, prot);
+                                } else {
+                                    pt_addr = pde & 0x3fffffffff000ULL;
+                                    for (l4 = 0; l4 < 512; l4++) {
+                                        cpu_physical_memory_read(pt_addr
+                                                                 + l4 * 8,
+                                                                 &pte, 8);
+                                        pte = le64_to_cpu(pte);
+                                        end = (l1 << 39) + (l2 << 30) +
+                                            (l3 << 21) + (l4 << 12);
+                                        if (pte & PG_PRESENT_MASK) {
+                                            prot = pte & (PG_USER_MASK | PG_RW_MASK |
+                                                          PG_PRESENT_MASK);
+                                            prot &= pml4e & pdpe & pde;
+                                        } else {
+                                            prot = 0;
+                                        }
+                                        mem_print(mon, &start, &last_prot, end, prot);
+                                    }
+                                }
+                            } else {
+                                prot = 0;
+                                mem_print(mon, &start, &last_prot, end, prot);
+                            }
+                        }
+                    }
+                } else {
+                    prot = 0;
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
+}
+#endif /* TARGET_X86_64 */
+
+void hmp_info_mem(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env;
+
+    env = mon_get_cpu_env();
+
+    if (!(env->cr[0] & CR0_PG_MASK)) {
+        monitor_printf(mon, "PG disabled\n");
+        return;
+    }
+    if (env->cr[4] & CR4_PAE_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            mem_info_64(mon, env);
+        } else
+#endif
+        {
+            mem_info_pae32(mon, env);
+        }
+    } else {
+        mem_info_32(mon, env);
+    }
+}
+
+void hmp_mce(Monitor *mon, const QDict *qdict)
+{
+    X86CPU *cpu;
+    CPUState *cs;
+    int cpu_index = qdict_get_int(qdict, "cpu_index");
+    int bank = qdict_get_int(qdict, "bank");
+    uint64_t status = qdict_get_int(qdict, "status");
+    uint64_t mcg_status = qdict_get_int(qdict, "mcg_status");
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint64_t misc = qdict_get_int(qdict, "misc");
+    int flags = MCE_INJECT_UNCOND_AO;
+
+    if (qdict_get_try_bool(qdict, "broadcast", false)) {
+        flags |= MCE_INJECT_BROADCAST;
+    }
+    cs = qemu_get_cpu(cpu_index);
+    if (cs != NULL) {
+        cpu = X86_CPU(cs);
+        cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
+                           flags);
+    }
+}
+
+static target_long monitor_get_pc(const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return env->eip + env->segs[R_CS].base;
+}
+
+const MonitorDef monitor_defs[] = {
+#define SEG(name, seg) \
+    { name, offsetof(CPUX86State, segs[seg].selector), NULL, MD_I32 },\
+    { name ".base", offsetof(CPUX86State, segs[seg].base) },\
+    { name ".limit", offsetof(CPUX86State, segs[seg].limit), NULL, MD_I32 },
+
+    { "eax", offsetof(CPUX86State, regs[0]) },
+    { "ecx", offsetof(CPUX86State, regs[1]) },
+    { "edx", offsetof(CPUX86State, regs[2]) },
+    { "ebx", offsetof(CPUX86State, regs[3]) },
+    { "esp|sp", offsetof(CPUX86State, regs[4]) },
+    { "ebp|fp", offsetof(CPUX86State, regs[5]) },
+    { "esi", offsetof(CPUX86State, regs[6]) },
+    { "edi", offsetof(CPUX86State, regs[7]) },
+#ifdef TARGET_X86_64
+    { "r8", offsetof(CPUX86State, regs[8]) },
+    { "r9", offsetof(CPUX86State, regs[9]) },
+    { "r10", offsetof(CPUX86State, regs[10]) },
+    { "r11", offsetof(CPUX86State, regs[11]) },
+    { "r12", offsetof(CPUX86State, regs[12]) },
+    { "r13", offsetof(CPUX86State, regs[13]) },
+    { "r14", offsetof(CPUX86State, regs[14]) },
+    { "r15", offsetof(CPUX86State, regs[15]) },
+#endif
+    { "eflags", offsetof(CPUX86State, eflags) },
+    { "eip", offsetof(CPUX86State, eip) },
+    SEG("cs", R_CS)
+    SEG("ds", R_DS)
+    SEG("es", R_ES)
+    SEG("ss", R_SS)
+    SEG("fs", R_FS)
+    SEG("gs", R_GS)
+    { "pc", 0, monitor_get_pc, },
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
+
+void hmp_info_local_apic(Monitor *mon, const QDict *qdict)
+{
+    x86_cpu_dump_local_apic_state(mon_get_cpu(), (FILE *)mon, monitor_fprintf,
+                                  CPU_DUMP_FPU);
+}
+
+void hmp_info_io_apic(Monitor *mon, const QDict *qdict)
+{
+    if (kvm_irqchip_in_kernel()) {
+        kvm_ioapic_dump_state(mon, qdict);
+    } else {
+        ioapic_dump_state(mon, qdict);
+    }
+}
diff --git a/target-i386/ops_sse.h b/target-i386/ops_sse.h
index bee134bae..1780d1d79 100644
--- a/target-i386/ops_sse.h
+++ b/target-i386/ops_sse.h
@@ -483,7 +483,7 @@ void glue(helper_maskmov, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
 
     for (i = 0; i < (8 << SHIFT); i++) {
         if (s->B(i) & 0x80) {
-            cpu_stb_data(env, a0 + i, d->B(i));
+            cpu_stb_data_ra(env, a0 + i, d->B(i), GETPC());
         }
     }
 }
@@ -2037,10 +2037,10 @@ static inline unsigned pcmpxstrx(CPUX86State *env, Reg *d, Reg *s,
         }
         break;
     case 3:
-        for (j = valids - validd; j >= 0; j--) {
+        for (j = valids; j >= 0; j--) {
             res <<= 1;
             v = 1;
-            for (i = MIN(upper - j, validd); i >= 0; i--) {
+            for (i = MIN(valids - j, validd); i >= 0; i--) {
                 v &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i));
             }
             res |= v;
diff --git a/target-i386/seg_helper.c b/target-i386/seg_helper.c
index 8a4271ebe..20ee89222 100644
--- a/target-i386/seg_helper.c
+++ b/target-i386/seg_helper.c
@@ -67,8 +67,9 @@
 #endif
 
 /* return non zero if error */
-static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
-                               uint32_t *e2_ptr, int selector)
+static inline int load_segment_ra(CPUX86State *env, uint32_t *e1_ptr,
+                               uint32_t *e2_ptr, int selector,
+                               uintptr_t retaddr)
 {
     SegmentCache *dt;
     int index;
@@ -84,11 +85,17 @@ static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
         return -1;
     }
     ptr = dt->base + index;
-    *e1_ptr = cpu_ldl_kernel(env, ptr);
-    *e2_ptr = cpu_ldl_kernel(env, ptr + 4);
+    *e1_ptr = cpu_ldl_kernel_ra(env, ptr, retaddr);
+    *e2_ptr = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
     return 0;
 }
 
+static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
+                               uint32_t *e2_ptr, int selector)
+{
+    return load_segment_ra(env, e1_ptr, e2_ptr, selector, 0);
+}
+
 static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)
 {
     unsigned int limit;
@@ -124,7 +131,8 @@ static inline void load_seg_vm(CPUX86State *env, int seg, int selector)
 }
 
 static inline void get_ss_esp_from_tss(CPUX86State *env, uint32_t *ss_ptr,
-                                       uint32_t *esp_ptr, int dpl)
+                                       uint32_t *esp_ptr, int dpl,
+                                       uintptr_t retaddr)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
     int type, index, shift;
@@ -153,60 +161,61 @@ static inline void get_ss_esp_from_tss(CPUX86State *env, uint32_t *ss_ptr,
     shift = type >> 3;
     index = (dpl * 4 + 2) << shift;
     if (index + (4 << shift) - 1 > env->tr.limit) {
-        raise_exception_err(env, EXCP0A_TSS, env->tr.selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, env->tr.selector & 0xfffc, retaddr);
     }
     if (shift == 0) {
-        *esp_ptr = cpu_lduw_kernel(env, env->tr.base + index);
-        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 2);
+        *esp_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index, retaddr);
+        *ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 2, retaddr);
     } else {
-        *esp_ptr = cpu_ldl_kernel(env, env->tr.base + index);
-        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 4);
+        *esp_ptr = cpu_ldl_kernel_ra(env, env->tr.base + index, retaddr);
+        *ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 4, retaddr);
     }
 }
 
-static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
+static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl,
+                         uintptr_t retaddr)
 {
     uint32_t e1, e2;
     int rpl, dpl;
 
     if ((selector & 0xfffc) != 0) {
-        if (load_segment(env, &e1, &e2, selector) != 0) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, selector, retaddr) != 0) {
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
         if (!(e2 & DESC_S_MASK)) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
         rpl = selector & 3;
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (seg_reg == R_CS) {
             if (!(e2 & DESC_CS_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             if (dpl != rpl) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
         } else if (seg_reg == R_SS) {
             /* SS must be writable data */
             if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             if (dpl != cpl || dpl != rpl) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
         } else {
             /* not readable code */
             if ((e2 & DESC_CS_MASK) && !(e2 & DESC_R_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             /* if data or non conforming code, checks the rights */
             if (((e2 >> DESC_TYPE_SHIFT) & 0xf) < 12) {
                 if (dpl < cpl || dpl < rpl) {
-                    raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                    raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
                 }
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, retaddr);
         }
         cpu_x86_load_seg_cache(env, seg_reg, selector,
                                get_seg_base(e1, e2),
@@ -214,7 +223,7 @@ static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
                                e2);
     } else {
         if (seg_reg == R_SS || seg_reg == R_CS) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
     }
 }
@@ -224,9 +233,9 @@ static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
 #define SWITCH_TSS_CALL 2
 
 /* XXX: restore CPU state in registers (PowerPC case) */
-static void switch_tss(CPUX86State *env, int tss_selector,
-                       uint32_t e1, uint32_t e2, int source,
-                       uint32_t next_eip)
+static void switch_tss_ra(CPUX86State *env, int tss_selector,
+                          uint32_t e1, uint32_t e2, int source,
+                          uint32_t next_eip, uintptr_t retaddr)
 {
     int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, v1, v2, i;
     target_ulong tss_base;
@@ -244,26 +253,26 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* if task gate, we read the TSS segment and we load it */
     if (type == 5) {
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
         }
         tss_selector = e1 >> 16;
         if (tss_selector & 4) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
         }
-        if (load_segment(env, &e1, &e2, tss_selector) != 0) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, tss_selector, retaddr) != 0) {
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
         if (e2 & DESC_S_MASK) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
         type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
         if ((type & 7) != 1) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
     }
 
     if (!(e2 & DESC_P_MASK)) {
-        raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
     }
 
     if (type & 8) {
@@ -275,7 +284,7 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     tss_base = get_seg_base(e1, e2);
     if ((tss_selector & 4) != 0 ||
         tss_limit < tss_limit_max) {
-        raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
     }
     old_type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
     if (old_type & 8) {
@@ -287,30 +296,33 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* read all the registers from the new TSS */
     if (type & 8) {
         /* 32 bit */
-        new_cr3 = cpu_ldl_kernel(env, tss_base + 0x1c);
-        new_eip = cpu_ldl_kernel(env, tss_base + 0x20);
-        new_eflags = cpu_ldl_kernel(env, tss_base + 0x24);
+        new_cr3 = cpu_ldl_kernel_ra(env, tss_base + 0x1c, retaddr);
+        new_eip = cpu_ldl_kernel_ra(env, tss_base + 0x20, retaddr);
+        new_eflags = cpu_ldl_kernel_ra(env, tss_base + 0x24, retaddr);
         for (i = 0; i < 8; i++) {
-            new_regs[i] = cpu_ldl_kernel(env, tss_base + (0x28 + i * 4));
+            new_regs[i] = cpu_ldl_kernel_ra(env, tss_base + (0x28 + i * 4),
+                                            retaddr);
         }
         for (i = 0; i < 6; i++) {
-            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x48 + i * 4));
+            new_segs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x48 + i * 4),
+                                             retaddr);
         }
-        new_ldt = cpu_lduw_kernel(env, tss_base + 0x60);
-        new_trap = cpu_ldl_kernel(env, tss_base + 0x64);
+        new_ldt = cpu_lduw_kernel_ra(env, tss_base + 0x60, retaddr);
+        new_trap = cpu_ldl_kernel_ra(env, tss_base + 0x64, retaddr);
     } else {
         /* 16 bit */
         new_cr3 = 0;
-        new_eip = cpu_lduw_kernel(env, tss_base + 0x0e);
-        new_eflags = cpu_lduw_kernel(env, tss_base + 0x10);
+        new_eip = cpu_lduw_kernel_ra(env, tss_base + 0x0e, retaddr);
+        new_eflags = cpu_lduw_kernel_ra(env, tss_base + 0x10, retaddr);
         for (i = 0; i < 8; i++) {
-            new_regs[i] = cpu_lduw_kernel(env, tss_base + (0x12 + i * 2)) |
-                0xffff0000;
+            new_regs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x12 + i * 2),
+                                             retaddr) | 0xffff0000;
         }
         for (i = 0; i < 4; i++) {
-            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x22 + i * 4));
+            new_segs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x22 + i * 4),
+                                             retaddr);
         }
-        new_ldt = cpu_lduw_kernel(env, tss_base + 0x2a);
+        new_ldt = cpu_lduw_kernel_ra(env, tss_base + 0x2a, retaddr);
         new_segs[R_FS] = 0;
         new_segs[R_GS] = 0;
         new_trap = 0;
@@ -325,10 +337,10 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* XXX: it can still fail in some cases, so a bigger hack is
        necessary to valid the TLB after having done the accesses */
 
-    v1 = cpu_ldub_kernel(env, env->tr.base);
-    v2 = cpu_ldub_kernel(env, env->tr.base + old_tss_limit_max);
-    cpu_stb_kernel(env, env->tr.base, v1);
-    cpu_stb_kernel(env, env->tr.base + old_tss_limit_max, v2);
+    v1 = cpu_ldub_kernel_ra(env, env->tr.base, retaddr);
+    v2 = cpu_ldub_kernel_ra(env, env->tr.base + old_tss_limit_max, retaddr);
+    cpu_stb_kernel_ra(env, env->tr.base, v1, retaddr);
+    cpu_stb_kernel_ra(env, env->tr.base + old_tss_limit_max, v2, retaddr);
 
     /* clear busy bit (it is restartable) */
     if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_IRET) {
@@ -336,9 +348,9 @@ static void switch_tss(CPUX86State *env, int tss_selector,
         uint32_t e2;
 
         ptr = env->gdt.base + (env->tr.selector & ~7);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         e2 &= ~DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, retaddr);
     }
     old_eflags = cpu_compute_eflags(env);
     if (source == SWITCH_TSS_IRET) {
@@ -348,35 +360,35 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* save the current state in the old TSS */
     if (type & 8) {
         /* 32 bit */
-        cpu_stl_kernel(env, env->tr.base + 0x20, next_eip);
-        cpu_stl_kernel(env, env->tr.base + 0x24, old_eflags);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 0 * 4), env->regs[R_EAX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 1 * 4), env->regs[R_ECX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 2 * 4), env->regs[R_EDX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 3 * 4), env->regs[R_EBX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 4 * 4), env->regs[R_ESP]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 5 * 4), env->regs[R_EBP]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 6 * 4), env->regs[R_ESI]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 7 * 4), env->regs[R_EDI]);
+        cpu_stl_kernel_ra(env, env->tr.base + 0x20, next_eip, retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + 0x24, old_eflags, retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 0 * 4), env->regs[R_EAX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 1 * 4), env->regs[R_ECX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 2 * 4), env->regs[R_EDX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 3 * 4), env->regs[R_EBX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 4 * 4), env->regs[R_ESP], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 5 * 4), env->regs[R_EBP], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 6 * 4), env->regs[R_ESI], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 7 * 4), env->regs[R_EDI], retaddr);
         for (i = 0; i < 6; i++) {
-            cpu_stw_kernel(env, env->tr.base + (0x48 + i * 4),
-                           env->segs[i].selector);
+            cpu_stw_kernel_ra(env, env->tr.base + (0x48 + i * 4),
+                              env->segs[i].selector, retaddr);
         }
     } else {
         /* 16 bit */
-        cpu_stw_kernel(env, env->tr.base + 0x0e, next_eip);
-        cpu_stw_kernel(env, env->tr.base + 0x10, old_eflags);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 0 * 2), env->regs[R_EAX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 1 * 2), env->regs[R_ECX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 2 * 2), env->regs[R_EDX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 3 * 2), env->regs[R_EBX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 4 * 2), env->regs[R_ESP]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 5 * 2), env->regs[R_EBP]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 6 * 2), env->regs[R_ESI]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 7 * 2), env->regs[R_EDI]);
+        cpu_stw_kernel_ra(env, env->tr.base + 0x0e, next_eip, retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + 0x10, old_eflags, retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 0 * 2), env->regs[R_EAX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 1 * 2), env->regs[R_ECX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 2 * 2), env->regs[R_EDX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 3 * 2), env->regs[R_EBX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 4 * 2), env->regs[R_ESP], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 5 * 2), env->regs[R_EBP], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 6 * 2), env->regs[R_ESI], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 7 * 2), env->regs[R_EDI], retaddr);
         for (i = 0; i < 4; i++) {
-            cpu_stw_kernel(env, env->tr.base + (0x22 + i * 4),
-                           env->segs[i].selector);
+            cpu_stw_kernel_ra(env, env->tr.base + (0x22 + i * 4),
+                              env->segs[i].selector, retaddr);
         }
     }
 
@@ -384,7 +396,7 @@ static void switch_tss(CPUX86State *env, int tss_selector,
        context */
 
     if (source == SWITCH_TSS_CALL) {
-        cpu_stw_kernel(env, tss_base, env->tr.selector);
+        cpu_stw_kernel_ra(env, tss_base, env->tr.selector, retaddr);
         new_eflags |= NT_MASK;
     }
 
@@ -394,9 +406,9 @@ static void switch_tss(CPUX86State *env, int tss_selector,
         uint32_t e2;
 
         ptr = env->gdt.base + (tss_selector & ~7);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         e2 |= DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, retaddr);
     }
 
     /* set the new CPU state */
@@ -448,23 +460,23 @@ static void switch_tss(CPUX86State *env, int tss_selector,
 
     /* load the LDT */
     if (new_ldt & 4) {
-        raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
     }
 
     if ((new_ldt & 0xfffc) != 0) {
         dt = &env->gdt;
         index = new_ldt & ~7;
         if ((index + 7) > dt->limit) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, retaddr);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         if ((e2 & DESC_S_MASK) || ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         load_seg_cache_raw_dt(&env->ldt, e1, e2);
     }
@@ -472,34 +484,35 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* load the segments */
     if (!(new_eflags & VM_MASK)) {
         int cpl = new_segs[R_CS] & 3;
-        tss_load_seg(env, R_CS, new_segs[R_CS], cpl);
-        tss_load_seg(env, R_SS, new_segs[R_SS], cpl);
-        tss_load_seg(env, R_ES, new_segs[R_ES], cpl);
-        tss_load_seg(env, R_DS, new_segs[R_DS], cpl);
-        tss_load_seg(env, R_FS, new_segs[R_FS], cpl);
-        tss_load_seg(env, R_GS, new_segs[R_GS], cpl);
+        tss_load_seg(env, R_CS, new_segs[R_CS], cpl, retaddr);
+        tss_load_seg(env, R_SS, new_segs[R_SS], cpl, retaddr);
+        tss_load_seg(env, R_ES, new_segs[R_ES], cpl, retaddr);
+        tss_load_seg(env, R_DS, new_segs[R_DS], cpl, retaddr);
+        tss_load_seg(env, R_FS, new_segs[R_FS], cpl, retaddr);
+        tss_load_seg(env, R_GS, new_segs[R_GS], cpl, retaddr);
     }
 
     /* check that env->eip is in the CS segment limits */
     if (new_eip > env->segs[R_CS].limit) {
         /* XXX: different exception if CALL? */
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
     }
 
 #ifndef CONFIG_USER_ONLY
     /* reset local breakpoints */
     if (env->dr[7] & DR7_LOCAL_BP_MASK) {
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            if (hw_local_breakpoint_enabled(env->dr[7], i) &&
-                !hw_global_breakpoint_enabled(env->dr[7], i)) {
-                hw_breakpoint_remove(env, i);
-            }
-        }
-        env->dr[7] &= ~DR7_LOCAL_BP_MASK;
+        cpu_x86_update_dr7(env, env->dr[7] & ~DR7_LOCAL_BP_MASK);
     }
 #endif
 }
 
+static void switch_tss(CPUX86State *env, int tss_selector,
+                       uint32_t e1, uint32_t e2, int source,
+                        uint32_t next_eip)
+{
+    switch_tss_ra(env, tss_selector, e1, e2, source, next_eip, 0);
+}
+
 static inline unsigned int get_sp_mask(unsigned int e2)
 {
     if (e2 & DESC_B_MASK) {
@@ -549,30 +562,35 @@ static int exception_has_error_code(int intno)
 #define SEG_ADDL(ssp, sp, sp_mask) ((uint32_t)((ssp) + (sp & (sp_mask))))
 
 /* XXX: add a is_user flag to have proper security support */
-#define PUSHW(ssp, sp, sp_mask, val)                             \
+#define PUSHW_RA(ssp, sp, sp_mask, val, ra)                      \
     {                                                            \
         sp -= 2;                                                 \
-        cpu_stw_kernel(env, (ssp) + (sp & (sp_mask)), (val));    \
+        cpu_stw_kernel_ra(env, (ssp) + (sp & (sp_mask)), (val), ra); \
     }
 
-#define PUSHL(ssp, sp, sp_mask, val)                                    \
+#define PUSHL_RA(ssp, sp, sp_mask, val, ra)                             \
     {                                                                   \
         sp -= 4;                                                        \
-        cpu_stl_kernel(env, SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val)); \
+        cpu_stl_kernel_ra(env, SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val), ra); \
     }
 
-#define POPW(ssp, sp, sp_mask, val)                              \
+#define POPW_RA(ssp, sp, sp_mask, val, ra)                       \
     {                                                            \
-        val = cpu_lduw_kernel(env, (ssp) + (sp & (sp_mask)));    \
+        val = cpu_lduw_kernel_ra(env, (ssp) + (sp & (sp_mask)), ra); \
         sp += 2;                                                 \
     }
 
-#define POPL(ssp, sp, sp_mask, val)                                     \
+#define POPL_RA(ssp, sp, sp_mask, val, ra)                              \
     {                                                                   \
-        val = (uint32_t)cpu_ldl_kernel(env, SEG_ADDL(ssp, sp, sp_mask)); \
+        val = (uint32_t)cpu_ldl_kernel_ra(env, SEG_ADDL(ssp, sp, sp_mask), ra); \
         sp += 4;                                                        \
     }
 
+#define PUSHW(ssp, sp, sp_mask, val) PUSHW_RA(ssp, sp, sp_mask, val, 0)
+#define PUSHL(ssp, sp, sp_mask, val) PUSHL_RA(ssp, sp, sp_mask, val, 0)
+#define POPW(ssp, sp, sp_mask, val) POPW_RA(ssp, sp, sp_mask, val, 0)
+#define POPL(ssp, sp, sp_mask, val) POPL_RA(ssp, sp, sp_mask, val, 0)
+
 /* protected mode interrupt */
 static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
                                    int error_code, unsigned int next_eip,
@@ -673,7 +691,7 @@ static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
     }
     if (!(e2 & DESC_C_MASK) && dpl < cpl) {
         /* to inner privilege */
-        get_ss_esp_from_tss(env, &ss, &esp, dpl);
+        get_ss_esp_from_tss(env, &ss, &esp, dpl, 0);
         if ((ss & 0xfffc) == 0) {
             raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
         }
@@ -791,18 +809,21 @@ static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
 
 #ifdef TARGET_X86_64
 
-#define PUSHQ(sp, val)                          \
+#define PUSHQ_RA(sp, val, ra)                   \
     {                                           \
         sp -= 8;                                \
-        cpu_stq_kernel(env, sp, (val));         \
+        cpu_stq_kernel_ra(env, sp, (val), ra);  \
     }
 
-#define POPQ(sp, val)                           \
+#define POPQ_RA(sp, val, ra)                    \
     {                                           \
-        val = cpu_ldq_kernel(env, sp);          \
+        val = cpu_ldq_kernel_ra(env, sp, ra);   \
         sp += 8;                                \
     }
 
+#define PUSHQ(sp, val) PUSHQ_RA(sp, val, 0)
+#define POPQ(sp, val) POPQ_RA(sp, val, 0)
+
 static inline target_ulong get_rsp_from_tss(CPUX86State *env, int level)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
@@ -961,7 +982,7 @@ void helper_syscall(CPUX86State *env, int next_eip_addend)
     int selector;
 
     if (!(env->efer & MSR_EFER_SCE)) {
-        raise_exception_err(env, EXCP06_ILLOP, 0);
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
     }
     selector = (env->star >> 32) & 0xffff;
     if (env->hflags & HF_LMA_MASK) {
@@ -1016,11 +1037,11 @@ void helper_sysret(CPUX86State *env, int dflag)
     int cpl, selector;
 
     if (!(env->efer & MSR_EFER_SCE)) {
-        raise_exception_err(env, EXCP06_ILLOP, 0);
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     if (!(env->cr[0] & CR0_PE_MASK) || cpl != 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
     selector = (env->star >> 48) & 0xffff;
     if (env->hflags & HF_LMA_MASK) {
@@ -1294,6 +1315,9 @@ bool x86_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
     if (interrupt_request & CPU_INTERRUPT_POLL) {
         cs->interrupt_request &= ~CPU_INTERRUPT_POLL;
         apic_poll_irq(cpu->apic_state);
+        /* Don't process multiple interrupt requests in a single call.
+           This is required to make icount-driven execution deterministic. */
+        return true;
     }
 #endif
     if (interrupt_request & CPU_INTERRUPT_SIPI) {
@@ -1369,22 +1393,26 @@ void helper_enter_level(CPUX86State *env, int level, int data32,
         while (--level) {
             esp -= 4;
             ebp -= 4;
-            cpu_stl_data(env, ssp + (esp & esp_mask),
-                         cpu_ldl_data(env, ssp + (ebp & esp_mask)));
+            cpu_stl_data_ra(env, ssp + (esp & esp_mask),
+                            cpu_ldl_data_ra(env, ssp + (ebp & esp_mask),
+                                            GETPC()),
+                            GETPC());
         }
         esp -= 4;
-        cpu_stl_data(env, ssp + (esp & esp_mask), t1);
+        cpu_stl_data_ra(env, ssp + (esp & esp_mask), t1, GETPC());
     } else {
         /* 16 bit */
         esp -= 2;
         while (--level) {
             esp -= 2;
             ebp -= 2;
-            cpu_stw_data(env, ssp + (esp & esp_mask),
-                         cpu_lduw_data(env, ssp + (ebp & esp_mask)));
+            cpu_stw_data_ra(env, ssp + (esp & esp_mask),
+                            cpu_lduw_data_ra(env, ssp + (ebp & esp_mask),
+                                             GETPC()),
+                            GETPC());
         }
         esp -= 2;
-        cpu_stw_data(env, ssp + (esp & esp_mask), t1);
+        cpu_stw_data_ra(env, ssp + (esp & esp_mask), t1, GETPC());
     }
 }
 
@@ -1403,20 +1431,22 @@ void helper_enter64_level(CPUX86State *env, int level, int data64,
         while (--level) {
             esp -= 8;
             ebp -= 8;
-            cpu_stq_data(env, esp, cpu_ldq_data(env, ebp));
+            cpu_stq_data_ra(env, esp, cpu_ldq_data_ra(env, ebp, GETPC()),
+                            GETPC());
         }
         esp -= 8;
-        cpu_stq_data(env, esp, t1);
+        cpu_stq_data_ra(env, esp, t1, GETPC());
     } else {
         /* 16 bit */
         esp -= 2;
         while (--level) {
             esp -= 2;
             ebp -= 2;
-            cpu_stw_data(env, esp, cpu_lduw_data(env, ebp));
+            cpu_stw_data_ra(env, esp, cpu_lduw_data_ra(env, ebp, GETPC()),
+                            GETPC());
         }
         esp -= 2;
-        cpu_stw_data(env, esp, t1);
+        cpu_stw_data_ra(env, esp, t1, GETPC());
     }
 }
 #endif
@@ -1435,7 +1465,7 @@ void helper_lldt(CPUX86State *env, int selector)
         env->ldt.limit = 0;
     } else {
         if (selector & 0x4) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dt = &env->gdt;
         index = selector & ~7;
@@ -1448,22 +1478,22 @@ void helper_lldt(CPUX86State *env, int selector)
             entry_limit = 7;
         }
         if ((index + entry_limit) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
         if ((e2 & DESC_S_MASK) || ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
             uint32_t e3;
 
-            e3 = cpu_ldl_kernel(env, ptr + 8);
+            e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
             load_seg_cache_raw_dt(&env->ldt, e1, e2);
             env->ldt.base |= (target_ulong)e3 << 32;
         } else
@@ -1490,7 +1520,7 @@ void helper_ltr(CPUX86State *env, int selector)
         env->tr.flags = 0;
     } else {
         if (selector & 0x4) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dt = &env->gdt;
         index = selector & ~7;
@@ -1503,27 +1533,27 @@ void helper_ltr(CPUX86State *env, int selector)
             entry_limit = 7;
         }
         if ((index + entry_limit) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
         type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
         if ((e2 & DESC_S_MASK) ||
             (type != 1 && type != 9)) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
             uint32_t e3, e4;
 
-            e3 = cpu_ldl_kernel(env, ptr + 8);
-            e4 = cpu_ldl_kernel(env, ptr + 12);
+            e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
+            e4 = cpu_ldl_kernel_ra(env, ptr + 12, GETPC());
             if ((e4 >> DESC_TYPE_SHIFT) & 0xf) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
             load_seg_cache_raw_dt(&env->tr, e1, e2);
             env->tr.base |= (target_ulong)e3 << 32;
@@ -1533,7 +1563,7 @@ void helper_ltr(CPUX86State *env, int selector)
             load_seg_cache_raw_dt(&env->tr, e1, e2);
         }
         e2 |= DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
     }
     env->tr.selector = selector;
 }
@@ -1556,7 +1586,7 @@ void helper_load_seg(CPUX86State *env, int seg_reg, int selector)
             && (!(env->hflags & HF_CS64_MASK) || cpl == 3)
 #endif
             ) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
         cpu_x86_load_seg_cache(env, seg_reg, selector, 0, 0, 0);
     } else {
@@ -1568,51 +1598,51 @@ void helper_load_seg(CPUX86State *env, int seg_reg, int selector)
         }
         index = selector & ~7;
         if ((index + 7) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
 
         if (!(e2 & DESC_S_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         rpl = selector & 3;
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (seg_reg == R_SS) {
             /* must be writable segment */
             if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
             if (rpl != cpl || dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
         } else {
             /* must be readable segment */
             if ((e2 & (DESC_CS_MASK | DESC_R_MASK)) == DESC_CS_MASK) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
 
             if (!(e2 & DESC_CS_MASK) || !(e2 & DESC_C_MASK)) {
                 /* if not conforming code, test rights */
                 if (dpl < cpl || dpl < rpl) {
-                    raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                    raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
                 }
             }
         }
 
         if (!(e2 & DESC_P_MASK)) {
             if (seg_reg == R_SS) {
-                raise_exception_err(env, EXCP0C_STACK, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0C_STACK, selector & 0xfffc, GETPC());
             } else {
-                raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
             }
         }
 
         /* set the access bit if not already set */
         if (!(e2 & DESC_A_MASK)) {
             e2 |= DESC_A_MASK;
-            cpu_stl_kernel(env, ptr + 4, e2);
+            cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
         }
 
         cpu_x86_load_seg_cache(env, seg_reg, selector,
@@ -1628,46 +1658,45 @@ void helper_load_seg(CPUX86State *env, int seg_reg, int selector)
 
 /* protected mode jump */
 void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
-                           int next_eip_addend)
+                           target_ulong next_eip)
 {
     int gate_cs, type;
     uint32_t e1, e2, cpl, dpl, rpl, limit;
-    target_ulong next_eip;
 
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     if (e2 & DESC_S_MASK) {
         if (!(e2 & DESC_CS_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (e2 & DESC_C_MASK) {
             /* conforming code segment */
             if (dpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         } else {
             /* non conforming code segment */
             rpl = new_cs & 3;
             if (rpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
         }
         limit = get_seg_limit(e1, e2);
         if (new_eip > limit &&
             !(env->hflags & HF_LMA_MASK) && !(e2 & DESC_L_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) | cpl,
                        get_seg_base(e1, e2), limit, e2);
@@ -1683,50 +1712,49 @@ void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
         case 9: /* 386 TSS */
         case 5: /* task gate */
             if (dpl < cpl || dpl < rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
-            next_eip = env->eip + next_eip_addend;
-            switch_tss(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip);
+            switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip, GETPC());
             break;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
             if ((dpl < cpl) || (dpl < rpl)) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (!(e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
             }
             gate_cs = e1 >> 16;
             new_eip = (e1 & 0xffff);
             if (type == 12) {
                 new_eip |= (e2 & 0xffff0000);
             }
-            if (load_segment(env, &e1, &e2, gate_cs) != 0) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+            if (load_segment_ra(env, &e1, &e2, gate_cs, GETPC()) != 0) {
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             dpl = (e2 >> DESC_DPL_SHIFT) & 3;
             /* must be code segment */
             if (((e2 & (DESC_S_MASK | DESC_CS_MASK)) !=
                  (DESC_S_MASK | DESC_CS_MASK))) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             if (((e2 & DESC_C_MASK) && (dpl > cpl)) ||
                 (!(e2 & DESC_C_MASK) && (dpl != cpl))) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             if (!(e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             limit = get_seg_limit(e1, e2);
             if (new_eip > limit) {
-                raise_exception_err(env, EXCP0D_GPF, 0);
+                raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
             }
             cpu_x86_load_seg_cache(env, R_CS, (gate_cs & 0xfffc) | cpl,
                                    get_seg_base(e1, e2), limit, e2);
             env->eip = new_eip;
             break;
         default:
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             break;
         }
     }
@@ -1745,11 +1773,11 @@ void helper_lcall_real(CPUX86State *env, int new_cs, target_ulong new_eip1,
     esp_mask = get_sp_mask(env->segs[R_SS].flags);
     ssp = env->segs[R_SS].base;
     if (shift) {
-        PUSHL(ssp, esp, esp_mask, env->segs[R_CS].selector);
-        PUSHL(ssp, esp, esp_mask, next_eip);
+        PUSHL_RA(ssp, esp, esp_mask, env->segs[R_CS].selector, GETPC());
+        PUSHL_RA(ssp, esp, esp_mask, next_eip, GETPC());
     } else {
-        PUSHW(ssp, esp, esp_mask, env->segs[R_CS].selector);
-        PUSHW(ssp, esp, esp_mask, next_eip);
+        PUSHW_RA(ssp, esp, esp_mask, env->segs[R_CS].selector, GETPC());
+        PUSHW_RA(ssp, esp, esp_mask, next_eip, GETPC());
     }
 
     SET_ESP(esp, esp_mask);
@@ -1760,47 +1788,46 @@ void helper_lcall_real(CPUX86State *env, int new_cs, target_ulong new_eip1,
 
 /* protected mode call */
 void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
-                            int shift, int next_eip_addend)
+                            int shift, target_ulong next_eip)
 {
     int new_stack, i;
     uint32_t e1, e2, cpl, dpl, rpl, selector, offset, param_count;
     uint32_t ss = 0, ss_e1 = 0, ss_e2 = 0, sp, type, ss_dpl, sp_mask;
     uint32_t val, limit, old_sp_mask;
-    target_ulong ssp, old_ssp, next_eip;
+    target_ulong ssp, old_ssp;
 
-    next_eip = env->eip + next_eip_addend;
     LOG_PCALL("lcall %04x:%08x s=%d\n", new_cs, (uint32_t)new_eip, shift);
     LOG_PCALL_STATE(CPU(x86_env_get_cpu(env)));
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     LOG_PCALL("desc=%08x:%08x\n", e1, e2);
     if (e2 & DESC_S_MASK) {
         if (!(e2 & DESC_CS_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (e2 & DESC_C_MASK) {
             /* conforming code segment */
             if (dpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         } else {
             /* non conforming code segment */
             rpl = new_cs & 3;
             if (rpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
         }
 
 #ifdef TARGET_X86_64
@@ -1810,8 +1837,8 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
 
             /* 64 bit case */
             rsp = env->regs[R_ESP];
-            PUSHQ(rsp, env->segs[R_CS].selector);
-            PUSHQ(rsp, next_eip);
+            PUSHQ_RA(rsp, env->segs[R_CS].selector, GETPC());
+            PUSHQ_RA(rsp, next_eip, GETPC());
             /* from this point, not restartable */
             env->regs[R_ESP] = rsp;
             cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) | cpl,
@@ -1825,16 +1852,16 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
             sp_mask = get_sp_mask(env->segs[R_SS].flags);
             ssp = env->segs[R_SS].base;
             if (shift) {
-                PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
-                PUSHL(ssp, sp, sp_mask, next_eip);
+                PUSHL_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+                PUSHL_RA(ssp, sp, sp_mask, next_eip, GETPC());
             } else {
-                PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
-                PUSHW(ssp, sp, sp_mask, next_eip);
+                PUSHW_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+                PUSHW_RA(ssp, sp, sp_mask, next_eip, GETPC());
             }
 
             limit = get_seg_limit(e1, e2);
             if (new_eip > limit) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             /* from this point, not restartable */
             SET_ESP(sp, sp_mask);
@@ -1852,73 +1879,73 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
         case 9: /* available 386 TSS */
         case 5: /* task gate */
             if (dpl < cpl || dpl < rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
-            switch_tss(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip);
+            switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip, GETPC());
             return;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
             break;
         default:
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             break;
         }
         shift = type >> 3;
 
         if (dpl < cpl || dpl < rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         /* check valid bit */
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG,  new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG,  new_cs & 0xfffc, GETPC());
         }
         selector = e1 >> 16;
         offset = (e2 & 0xffff0000) | (e1 & 0x0000ffff);
         param_count = e2 & 0x1f;
         if ((selector & 0xfffc) == 0) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
 
-        if (load_segment(env, &e1, &e2, selector) != 0) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_S_MASK) || !(e2 & (DESC_CS_MASK))) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (dpl > cpl) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 
         if (!(e2 & DESC_C_MASK) && dpl < cpl) {
             /* to inner privilege */
-            get_ss_esp_from_tss(env, &ss, &sp, dpl);
+            get_ss_esp_from_tss(env, &ss, &sp, dpl, GETPC());
             LOG_PCALL("new ss:esp=%04x:%08x param_count=%d env->regs[R_ESP]="
                       TARGET_FMT_lx "\n", ss, sp, param_count,
                       env->regs[R_ESP]);
             if ((ss & 0xfffc) == 0) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if ((ss & 3) != dpl) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
-            if (load_segment(env, &ss_e1, &ss_e2, ss) != 0) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+            if (load_segment_ra(env, &ss_e1, &ss_e2, ss, GETPC()) != 0) {
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
             if (ss_dpl != dpl) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if (!(ss_e2 & DESC_S_MASK) ||
                 (ss_e2 & DESC_CS_MASK) ||
                 !(ss_e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if (!(ss_e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
 
             /* push_size = ((param_count * 2) + 8) << shift; */
@@ -1929,22 +1956,22 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
             sp_mask = get_sp_mask(ss_e2);
             ssp = get_seg_base(ss_e1, ss_e2);
             if (shift) {
-                PUSHL(ssp, sp, sp_mask, env->segs[R_SS].selector);
-                PUSHL(ssp, sp, sp_mask, env->regs[R_ESP]);
+                PUSHL_RA(ssp, sp, sp_mask, env->segs[R_SS].selector, GETPC());
+                PUSHL_RA(ssp, sp, sp_mask, env->regs[R_ESP], GETPC());
                 for (i = param_count - 1; i >= 0; i--) {
-                    val = cpu_ldl_kernel(env, old_ssp +
-                                         ((env->regs[R_ESP] + i * 4) &
-                                          old_sp_mask));
-                    PUSHL(ssp, sp, sp_mask, val);
+                    val = cpu_ldl_kernel_ra(env, old_ssp +
+                                            ((env->regs[R_ESP] + i * 4) &
+                                             old_sp_mask), GETPC());
+                    PUSHL_RA(ssp, sp, sp_mask, val, GETPC());
                 }
             } else {
-                PUSHW(ssp, sp, sp_mask, env->segs[R_SS].selector);
-                PUSHW(ssp, sp, sp_mask, env->regs[R_ESP]);
+                PUSHW_RA(ssp, sp, sp_mask, env->segs[R_SS].selector, GETPC());
+                PUSHW_RA(ssp, sp, sp_mask, env->regs[R_ESP], GETPC());
                 for (i = param_count - 1; i >= 0; i--) {
-                    val = cpu_lduw_kernel(env, old_ssp +
-                                          ((env->regs[R_ESP] + i * 2) &
-                                           old_sp_mask));
-                    PUSHW(ssp, sp, sp_mask, val);
+                    val = cpu_lduw_kernel_ra(env, old_ssp +
+                                             ((env->regs[R_ESP] + i * 2) &
+                                              old_sp_mask), GETPC());
+                    PUSHW_RA(ssp, sp, sp_mask, val, GETPC());
                 }
             }
             new_stack = 1;
@@ -1958,11 +1985,11 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
         }
 
         if (shift) {
-            PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
-            PUSHL(ssp, sp, sp_mask, next_eip);
+            PUSHL_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+            PUSHL_RA(ssp, sp, sp_mask, next_eip, GETPC());
         } else {
-            PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
-            PUSHW(ssp, sp, sp_mask, next_eip);
+            PUSHW_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+            PUSHW_RA(ssp, sp, sp_mask, next_eip, GETPC());
         }
 
         /* from this point, not restartable */
@@ -1997,15 +2024,15 @@ void helper_iret_real(CPUX86State *env, int shift)
     ssp = env->segs[R_SS].base;
     if (shift == 1) {
         /* 32 bits */
-        POPL(ssp, sp, sp_mask, new_eip);
-        POPL(ssp, sp, sp_mask, new_cs);
+        POPL_RA(ssp, sp, sp_mask, new_eip, GETPC());
+        POPL_RA(ssp, sp, sp_mask, new_cs, GETPC());
         new_cs &= 0xffff;
-        POPL(ssp, sp, sp_mask, new_eflags);
+        POPL_RA(ssp, sp, sp_mask, new_eflags, GETPC());
     } else {
         /* 16 bits */
-        POPW(ssp, sp, sp_mask, new_eip);
-        POPW(ssp, sp, sp_mask, new_cs);
-        POPW(ssp, sp, sp_mask, new_eflags);
+        POPW_RA(ssp, sp, sp_mask, new_eip, GETPC());
+        POPW_RA(ssp, sp, sp_mask, new_cs, GETPC());
+        POPW_RA(ssp, sp, sp_mask, new_eflags, GETPC());
     }
     env->regs[R_ESP] = (env->regs[R_ESP] & ~sp_mask) | (sp & sp_mask);
     env->segs[R_CS].selector = new_cs;
@@ -2050,7 +2077,8 @@ static inline void validate_seg(CPUX86State *env, int seg_reg, int cpl)
 
 /* protected mode iret */
 static inline void helper_ret_protected(CPUX86State *env, int shift,
-                                        int is_iret, int addend)
+                                        int is_iret, int addend,
+                                        uintptr_t retaddr)
 {
     uint32_t new_cs, new_eflags, new_ss;
     uint32_t new_es, new_ds, new_fs, new_gs;
@@ -2071,32 +2099,32 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
     new_eflags = 0; /* avoid warning */
 #ifdef TARGET_X86_64
     if (shift == 2) {
-        POPQ(sp, new_eip);
-        POPQ(sp, new_cs);
+        POPQ_RA(sp, new_eip, retaddr);
+        POPQ_RA(sp, new_cs, retaddr);
         new_cs &= 0xffff;
         if (is_iret) {
-            POPQ(sp, new_eflags);
+            POPQ_RA(sp, new_eflags, retaddr);
         }
     } else
 #endif
     {
         if (shift == 1) {
             /* 32 bits */
-            POPL(ssp, sp, sp_mask, new_eip);
-            POPL(ssp, sp, sp_mask, new_cs);
+            POPL_RA(ssp, sp, sp_mask, new_eip, retaddr);
+            POPL_RA(ssp, sp, sp_mask, new_cs, retaddr);
             new_cs &= 0xffff;
             if (is_iret) {
-                POPL(ssp, sp, sp_mask, new_eflags);
+                POPL_RA(ssp, sp, sp_mask, new_eflags, retaddr);
                 if (new_eflags & VM_MASK) {
                     goto return_to_vm86;
                 }
             }
         } else {
             /* 16 bits */
-            POPW(ssp, sp, sp_mask, new_eip);
-            POPW(ssp, sp, sp_mask, new_cs);
+            POPW_RA(ssp, sp, sp_mask, new_eip, retaddr);
+            POPW_RA(ssp, sp, sp_mask, new_cs, retaddr);
             if (is_iret) {
-                POPW(ssp, sp, sp_mask, new_eflags);
+                POPW_RA(ssp, sp, sp_mask, new_eflags, retaddr);
             }
         }
     }
@@ -2104,32 +2132,32 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
               new_cs, new_eip, shift, addend);
     LOG_PCALL_STATE(CPU(x86_env_get_cpu(env)));
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, retaddr) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     if (!(e2 & DESC_S_MASK) ||
         !(e2 & DESC_CS_MASK)) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     cpl = env->hflags & HF_CPL_MASK;
     rpl = new_cs & 3;
     if (rpl < cpl) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     dpl = (e2 >> DESC_DPL_SHIFT) & 3;
     if (e2 & DESC_C_MASK) {
         if (dpl > rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
         }
     } else {
         if (dpl != rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
         }
     }
     if (!(e2 & DESC_P_MASK)) {
-        raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, retaddr);
     }
 
     sp += addend;
@@ -2144,21 +2172,21 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
         /* return to different privilege level */
 #ifdef TARGET_X86_64
         if (shift == 2) {
-            POPQ(sp, new_esp);
-            POPQ(sp, new_ss);
+            POPQ_RA(sp, new_esp, retaddr);
+            POPQ_RA(sp, new_ss, retaddr);
             new_ss &= 0xffff;
         } else
 #endif
         {
             if (shift == 1) {
                 /* 32 bits */
-                POPL(ssp, sp, sp_mask, new_esp);
-                POPL(ssp, sp, sp_mask, new_ss);
+                POPL_RA(ssp, sp, sp_mask, new_esp, retaddr);
+                POPL_RA(ssp, sp, sp_mask, new_ss, retaddr);
                 new_ss &= 0xffff;
             } else {
                 /* 16 bits */
-                POPW(ssp, sp, sp_mask, new_esp);
-                POPW(ssp, sp, sp_mask, new_ss);
+                POPW_RA(ssp, sp, sp_mask, new_esp, retaddr);
+                POPW_RA(ssp, sp, sp_mask, new_ss, retaddr);
             }
         }
         LOG_PCALL("new ss:esp=%04x:" TARGET_FMT_lx "\n",
@@ -2177,26 +2205,26 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
             } else
 #endif
             {
-                raise_exception_err(env, EXCP0D_GPF, 0);
+                raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
             }
         } else {
             if ((new_ss & 3) != rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
-            if (load_segment(env, &ss_e1, &ss_e2, new_ss) != 0) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+            if (load_segment_ra(env, &ss_e1, &ss_e2, new_ss, retaddr) != 0) {
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             if (!(ss_e2 & DESC_S_MASK) ||
                 (ss_e2 & DESC_CS_MASK) ||
                 !(ss_e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
             if (dpl != rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             if (!(ss_e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0B_NOSEG, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, new_ss & 0xfffc, retaddr);
             }
             cpu_x86_load_seg_cache(env, R_SS, new_ss,
                                    get_seg_base(ss_e1, ss_e2),
@@ -2246,12 +2274,12 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
     return;
 
  return_to_vm86:
-    POPL(ssp, sp, sp_mask, new_esp);
-    POPL(ssp, sp, sp_mask, new_ss);
-    POPL(ssp, sp, sp_mask, new_es);
-    POPL(ssp, sp, sp_mask, new_ds);
-    POPL(ssp, sp, sp_mask, new_fs);
-    POPL(ssp, sp, sp_mask, new_gs);
+    POPL_RA(ssp, sp, sp_mask, new_esp, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_ss, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_es, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_ds, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_fs, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_gs, retaddr);
 
     /* modify processor state */
     cpu_load_eflags(env, new_eflags, TF_MASK | AC_MASK | ID_MASK |
@@ -2277,37 +2305,37 @@ void helper_iret_protected(CPUX86State *env, int shift, int next_eip)
     if (env->eflags & NT_MASK) {
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
 #endif
-        tss_selector = cpu_lduw_kernel(env, env->tr.base + 0);
+        tss_selector = cpu_lduw_kernel_ra(env, env->tr.base + 0, GETPC());
         if (tss_selector & 4) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
-        if (load_segment(env, &e1, &e2, tss_selector) != 0) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, tss_selector, GETPC()) != 0) {
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
         type = (e2 >> DESC_TYPE_SHIFT) & 0x17;
         /* NOTE: we check both segment and busy TSS */
         if (type != 3) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
-        switch_tss(env, tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip);
+        switch_tss_ra(env, tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip, GETPC());
     } else {
-        helper_ret_protected(env, shift, 1, 0);
+        helper_ret_protected(env, shift, 1, 0, GETPC());
     }
     env->hflags2 &= ~HF2_NMI_MASK;
 }
 
 void helper_lret_protected(CPUX86State *env, int shift, int addend)
 {
-    helper_ret_protected(env, shift, 0, addend);
+    helper_ret_protected(env, shift, 0, addend, GETPC());
 }
 
 void helper_sysenter(CPUX86State *env)
 {
     if (env->sysenter_cs == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
     env->eflags &= ~(VM_MASK | IF_MASK | RF_MASK);
 
@@ -2343,7 +2371,7 @@ void helper_sysexit(CPUX86State *env, int dflag)
 
     cpl = env->hflags & HF_CPL_MASK;
     if (env->sysenter_cs == 0 || cpl != 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
 #ifdef TARGET_X86_64
     if (dflag == 2) {
@@ -2387,7 +2415,7 @@ target_ulong helper_lsl(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     rpl = selector & 3;
@@ -2434,7 +2462,7 @@ target_ulong helper_lar(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     rpl = selector & 3;
@@ -2483,7 +2511,7 @@ void helper_verr(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     if (!(e2 & DESC_S_MASK)) {
@@ -2521,7 +2549,7 @@ void helper_verw(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     if (!(e2 & DESC_S_MASK)) {
@@ -2562,7 +2590,8 @@ void cpu_x86_load_seg(CPUX86State *env, int seg_reg, int selector)
 #endif
 
 /* check if Port I/O is allowed in TSS */
-static inline void check_io(CPUX86State *env, int addr, int size)
+static inline void check_io(CPUX86State *env, int addr, int size,
+                            uintptr_t retaddr)
 {
     int io_offset, val, mask;
 
@@ -2572,33 +2601,33 @@ static inline void check_io(CPUX86State *env, int addr, int size)
         env->tr.limit < 103) {
         goto fail;
     }
-    io_offset = cpu_lduw_kernel(env, env->tr.base + 0x66);
+    io_offset = cpu_lduw_kernel_ra(env, env->tr.base + 0x66, retaddr);
     io_offset += (addr >> 3);
     /* Note: the check needs two bytes */
     if ((io_offset + 1) > env->tr.limit) {
         goto fail;
     }
-    val = cpu_lduw_kernel(env, env->tr.base + io_offset);
+    val = cpu_lduw_kernel_ra(env, env->tr.base + io_offset, retaddr);
     val >>= (addr & 7);
     mask = (1 << size) - 1;
     /* all bits must be zero to allow the I/O */
     if ((val & mask) != 0) {
     fail:
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
     }
 }
 
 void helper_check_iob(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 1);
+    check_io(env, t0, 1, GETPC());
 }
 
 void helper_check_iow(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 2);
+    check_io(env, t0, 2, GETPC());
 }
 
 void helper_check_iol(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 4);
+    check_io(env, t0, 4, GETPC());
 }
diff --git a/target-i386/smm_helper.c b/target-i386/smm_helper.c
index 02e24b923..c272a9840 100644
--- a/target-i386/smm_helper.c
+++ b/target-i386/smm_helper.c
@@ -266,7 +266,7 @@ void helper_rsm(CPUX86State *env)
 
     val = x86_ldl_phys(cs, sm_state + 0x7efc); /* revision ID */
     if (val & 0x20000) {
-        env->smbase = x86_ldl_phys(cs, sm_state + 0x7f00) & ~0x7fff;
+        env->smbase = x86_ldl_phys(cs, sm_state + 0x7f00);
     }
 #else
     cpu_x86_update_cr0(env, x86_ldl_phys(cs, sm_state + 0x7ffc));
@@ -319,7 +319,7 @@ void helper_rsm(CPUX86State *env)
 
     val = x86_ldl_phys(cs, sm_state + 0x7efc); /* revision ID */
     if (val & 0x20000) {
-        env->smbase = x86_ldl_phys(cs, sm_state + 0x7ef8) & ~0x7fff;
+        env->smbase = x86_ldl_phys(cs, sm_state + 0x7ef8);
     }
 #endif
     if ((env->hflags2 & HF2_SMM_INSIDE_NMI_MASK) == 0) {
diff --git a/target-i386/translate.c b/target-i386/translate.c
index 82e2245bf..a3dd167a9 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -21,7 +21,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <inttypes.h>
-#include <signal.h>
 
 #include "qemu/host-utils.h"
 #include "cpu.h"
@@ -76,8 +75,6 @@ static TCGv_ptr cpu_ptr0, cpu_ptr1;
 static TCGv_i32 cpu_tmp2_i32, cpu_tmp3_i32;
 static TCGv_i64 cpu_tmp1_i64;
 
-static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 #ifdef TARGET_X86_64
@@ -664,14 +661,9 @@ static void gen_helper_out_func(TCGMemOp ot, TCGv_i32 v, TCGv_i32 n)
 static void gen_check_io(DisasContext *s, TCGMemOp ot, target_ulong cur_eip,
                          uint32_t svm_flags)
 {
-    int state_saved;
     target_ulong next_eip;
 
-    state_saved = 0;
     if (s->pe && (s->cpl > s->iopl || s->vm86)) {
-        gen_update_cc_op(s);
-        gen_jmp_im(cur_eip);
-        state_saved = 1;
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         switch (ot) {
         case MO_8:
@@ -688,10 +680,8 @@ static void gen_check_io(DisasContext *s, TCGMemOp ot, target_ulong cur_eip,
         }
     }
     if(s->flags & HF_SVMI_MASK) {
-        if (!state_saved) {
-            gen_update_cc_op(s);
-            gen_jmp_im(cur_eip);
-        }
+        gen_update_cc_op(s);
+        gen_jmp_im(cur_eip);
         svm_flags |= (1 << (4 + ot));
         next_eip = s->pc - s->cs_base;
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -1164,6 +1154,19 @@ static inline void gen_cmps(DisasContext *s, TCGMemOp ot)
     gen_op_add_reg_T0(s->aflag, R_EDI);
 }
 
+static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot)
+{
+    if (s->flags & HF_IOBPT_MASK) {
+        TCGv_i32 t_size = tcg_const_i32(1 << ot);
+        TCGv t_next = tcg_const_tl(s->pc - s->cs_base);
+
+        gen_helper_bpt_io(cpu_env, t_port, t_size, t_next);
+        tcg_temp_free_i32(t_size);
+        tcg_temp_free(t_next);
+    }
+}
+
+
 static inline void gen_ins(DisasContext *s, TCGMemOp ot)
 {
     if (s->tb->cflags & CF_USE_ICOUNT) {
@@ -1180,6 +1183,7 @@ static inline void gen_ins(DisasContext *s, TCGMemOp ot)
     gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_EDI);
+    gen_bpt_io(s, cpu_tmp2_i32, ot);
     if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
     }
@@ -1197,9 +1201,9 @@ static inline void gen_outs(DisasContext *s, TCGMemOp ot)
     tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
     tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[0]);
     gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
-
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_ESI);
+    gen_bpt_io(s, cpu_tmp2_i32, ot);
     if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
     }
@@ -2298,12 +2302,9 @@ static inline void gen_op_movl_seg_T0_vm(int seg_reg)
 
 /* move T0 to seg_reg and compute if the CPU state may change. Never
    call this function with seg_reg == R_CS */
-static void gen_movl_seg_T0(DisasContext *s, int seg_reg, target_ulong cur_eip)
+static void gen_movl_seg_T0(DisasContext *s, int seg_reg)
 {
     if (s->pe && !s->vm86) {
-        /* XXX: optimize by finding processor state dynamically */
-        gen_update_cc_op(s);
-        gen_jmp_im(cur_eip);
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         gen_helper_load_seg(cpu_env, tcg_const_i32(seg_reg), cpu_tmp2_i32);
         /* abort translation because the addseg value may change or
@@ -2431,7 +2432,7 @@ static void gen_pusha(DisasContext *s)
 {
     int i;
     gen_op_movl_A0_reg(R_ESP);
-    gen_op_addl_A0_im(-8 << s->dflag);
+    gen_op_addl_A0_im(-(8 << s->dflag));
     if (!s->ss32)
         tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
     tcg_gen_mov_tl(cpu_T[1], cpu_A0);
@@ -3847,8 +3848,10 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     break;
 #ifdef TARGET_X86_64
                 case MO_64:
-                    tcg_gen_mulu2_i64(cpu_regs[s->vex_v], cpu_regs[reg],
+                    tcg_gen_mulu2_i64(cpu_T[0], cpu_T[1],
                                       cpu_T[0], cpu_regs[R_EDX]);
+                    tcg_gen_mov_i64(cpu_regs[s->vex_v], cpu_T[0]);
+                    tcg_gen_mov_i64(cpu_regs[reg], cpu_T[1]);
                     break;
 #endif
                 }
@@ -4412,9 +4415,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     target_ulong next_eip, tval;
     int rex_w, rex_r;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(pc_start);
-    }
     s->pc = pc_start;
     prefixes = 0;
     s->override = -1;
@@ -4842,21 +4842,17 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 6: /* div */
             switch(ot) {
             case MO_8:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_divb_AL(cpu_env, cpu_T[0]);
                 break;
             case MO_16:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_divw_AX(cpu_env, cpu_T[0]);
                 break;
             default:
             case MO_32:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_divl_EAX(cpu_env, cpu_T[0]);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_divq_EAX(cpu_env, cpu_T[0]);
                 break;
 #endif
@@ -4865,21 +4861,17 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 7: /* idiv */
             switch(ot) {
             case MO_8:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_idivb_AL(cpu_env, cpu_T[0]);
                 break;
             case MO_16:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_idivw_AX(cpu_env, cpu_T[0]);
                 break;
             default:
             case MO_32:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_idivl_EAX(cpu_env, cpu_T[0]);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_idivq_EAX(cpu_env, cpu_T[0]);
                 break;
 #endif
@@ -4952,12 +4944,10 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
         do_lcall:
             if (s->pe && !s->vm86) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_lcall_protected(cpu_env, cpu_tmp2_i32, cpu_T[1],
                                            tcg_const_i32(dflag - 1),
-                                           tcg_const_i32(s->pc - pc_start));
+                                           tcg_const_tl(s->pc - s->cs_base));
             } else {
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_lcall_real(cpu_env, cpu_tmp2_i32, cpu_T[1],
@@ -4979,11 +4969,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
         do_ljmp:
             if (s->pe && !s->vm86) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_ljmp_protected(cpu_env, cpu_tmp2_i32, cpu_T[1],
-                                          tcg_const_i32(s->pc - pc_start));
+                                          tcg_const_tl(s->pc - s->cs_base));
             } else {
                 gen_op_movl_seg_T0_vm(R_CS);
                 gen_op_jmp_v(cpu_T[1]);
@@ -5212,8 +5200,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (dflag == MO_64) {
             if (!(s->cpuid_ext_features & CPUID_EXT_CX16))
                 goto illegal_op;
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_update_cc_op(s);
             gen_lea_modrm(env, s, modrm);
             gen_helper_cmpxchg16b(cpu_env, cpu_A0);
         } else
@@ -5221,8 +5207,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         {
             if (!(s->cpuid_features & CPUID_CX8))
                 goto illegal_op;
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_update_cc_op(s);
             gen_lea_modrm(env, s, modrm);
             gen_helper_cmpxchg8b(cpu_env, cpu_A0);
         }
@@ -5324,7 +5308,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             goto illegal_op;
         reg = b >> 3;
         ot = gen_pop_T0(s);
-        gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, reg);
         gen_pop_update(s, ot);
         if (reg == R_SS) {
             /* if reg == SS, inhibit interrupts/trace. */
@@ -5342,7 +5326,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0x1a1: /* pop fs */
     case 0x1a9: /* pop gs */
         ot = gen_pop_T0(s);
-        gen_movl_seg_T0(s, (b >> 3) & 7, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, (b >> 3) & 7);
         gen_pop_update(s, ot);
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
@@ -5393,7 +5377,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (reg >= 6 || reg == R_CS)
             goto illegal_op;
         gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-        gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, reg);
         if (reg == R_SS) {
             /* if reg == SS, inhibit interrupts/trace */
             /* If several instructions disable interrupts, only the
@@ -5605,7 +5589,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         gen_add_A0_im(s, 1 << ot);
         /* load the segment first to handle exceptions properly */
         gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
-        gen_movl_seg_T0(s, op, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, op);
         /* then put the data */
         gen_op_mov_reg_v(ot, reg, cpu_T[1]);
         if (s->is_jmp) {
@@ -5837,8 +5821,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 }
                 break;
             case 0x0c: /* fldenv mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fldenv(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x0d: /* fldcw mem */
@@ -5847,8 +5829,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 gen_helper_fldcw(cpu_env, cpu_tmp2_i32);
                 break;
             case 0x0e: /* fnstenv mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fstenv(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x0f: /* fnstcw mem */
@@ -5857,24 +5837,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                                     s->mem_index, MO_LEUW);
                 break;
             case 0x1d: /* fldt mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fldt_ST0(cpu_env, cpu_A0);
                 break;
             case 0x1f: /* fstpt mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fstt_ST0(cpu_env, cpu_A0);
                 gen_helper_fpop(cpu_env);
                 break;
             case 0x2c: /* frstor mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_frstor(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x2e: /* fnsave mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fsave(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x2f: /* fnstsw mem */
@@ -5883,13 +5855,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                                     s->mem_index, MO_LEUW);
                 break;
             case 0x3c: /* fbld */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fbld_ST0(cpu_env, cpu_A0);
                 break;
             case 0x3e: /* fbstp */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fbst_ST0(cpu_env, cpu_A0);
                 gen_helper_fpop(cpu_env);
                 break;
@@ -5924,8 +5892,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 switch(rm) {
                 case 0: /* fnop */
                     /* check exceptions (FreeBSD FPU probe) */
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
                     gen_helper_fwait(cpu_env);
                     break;
                 default:
@@ -6319,6 +6285,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         tcg_gen_movi_i32(cpu_tmp2_i32, val);
         gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
         gen_op_mov_reg_v(ot, R_EAX, cpu_T[1]);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6339,6 +6306,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         tcg_gen_movi_i32(cpu_tmp2_i32, val);
         tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
         gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6356,6 +6324,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
         gen_op_mov_reg_v(ot, R_EAX, cpu_T[1]);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6375,6 +6344,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
         gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6441,8 +6411,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 set_cc_op(s, CC_OP_EFLAGS);
             }
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_iret_protected(cpu_env, tcg_const_i32(dflag - 1),
                                       tcg_const_i32(s->pc - s->cs_base));
             set_cc_op(s, CC_OP_EFLAGS);
@@ -6895,8 +6863,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             (HF_MP_MASK | HF_TS_MASK)) {
             gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_fwait(cpu_env);
         }
         break;
@@ -6980,7 +6946,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             goto illegal_op;
         gen_op_mov_v_reg(ot, cpu_T[0], reg);
         gen_lea_modrm(env, s, modrm);
-        gen_jmp_im(pc_start - s->cs_base);
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         if (ot == MO_16) {
             gen_helper_boundw(cpu_env, cpu_A0, cpu_tmp2_i32);
@@ -7094,8 +7059,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysenter(cpu_env);
             gen_eob(s);
         }
@@ -7107,8 +7070,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysexit(cpu_env, tcg_const_i32(dflag - 1));
             gen_eob(s);
         }
@@ -7125,8 +7086,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysret(cpu_env, tcg_const_i32(dflag - 1));
             /* condition codes are modified only in long mode */
             if (s->lma) {
@@ -7172,7 +7131,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_lldt(cpu_env, cpu_tmp2_i32);
             }
@@ -7193,7 +7151,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_ltr(cpu_env, cpu_tmp2_i32);
             }
@@ -7672,18 +7629,20 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 ot = MO_64;
             else
                 ot = MO_32;
-            /* XXX: do it dynamically with CR4.DE bit */
-            if (reg == 4 || reg == 5 || reg >= 8)
+            if (reg >= 8) {
                 goto illegal_op;
+            }
             if (b & 2) {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_DR0 + reg);
                 gen_op_mov_v_reg(ot, cpu_T[0], rm);
-                gen_helper_movl_drN_T0(cpu_env, tcg_const_i32(reg), cpu_T[0]);
+                tcg_gen_movi_i32(cpu_tmp2_i32, reg);
+                gen_helper_set_dr(cpu_env, cpu_tmp2_i32, cpu_T[0]);
                 gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_DR0 + reg);
-                tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,dr[reg]));
+                tcg_gen_movi_i32(cpu_tmp2_i32, reg);
+                gen_helper_get_dr(cpu_T[0], cpu_env, cpu_tmp2_i32);
                 gen_op_mov_reg_v(ot, rm, cpu_T[0]);
             }
         }
@@ -7726,8 +7685,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_fxsave(cpu_env, cpu_A0, tcg_const_i32(dflag == MO_64));
             break;
         case 1: /* fxrstor */
@@ -7739,8 +7696,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_fxrstor(cpu_env, cpu_A0, tcg_const_i32(dflag == MO_64));
             break;
         case 2: /* ldmxcsr */
@@ -7763,20 +7718,43 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             }
             break;
         case 5: /* lfence */
-        case 6: /* mfence */
             if ((modrm & 0xc7) != 0xc0 || !(s->cpuid_features & CPUID_SSE2))
                 goto illegal_op;
             break;
-        case 7: /* sfence / clflush */
-            if ((modrm & 0xc7) == 0xc0) {
-                /* sfence */
-                /* XXX: also check for cpuid_ext2_features & CPUID_EXT2_EMMX */
-                if (!(s->cpuid_features & CPUID_SSE))
+        case 6: /* mfence/clwb */
+            if (s->prefix & PREFIX_DATA) {
+                /* clwb */
+                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_CLWB))
                     goto illegal_op;
+                gen_nop_modrm(env, s, modrm);
             } else {
-                /* clflush */
-                if (!(s->cpuid_features & CPUID_CLFLUSH))
+                /* mfence */
+                if ((modrm & 0xc7) != 0xc0 || !(s->cpuid_features & CPUID_SSE2))
                     goto illegal_op;
+            }
+            break;
+        case 7: /* sfence / clflush / clflushopt / pcommit */
+            if ((modrm & 0xc7) == 0xc0) {
+                if (s->prefix & PREFIX_DATA) {
+                    /* pcommit */
+                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_PCOMMIT))
+                        goto illegal_op;
+                } else {
+                    /* sfence */
+                    /* XXX: also check for cpuid_ext2_features & CPUID_EXT2_EMMX */
+                    if (!(s->cpuid_features & CPUID_SSE))
+                        goto illegal_op;
+                }
+            } else {
+                if (s->prefix & PREFIX_DATA) {
+                    /* clflushopt */
+                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_CLFLUSHOPT))
+                        goto illegal_op;
+                } else {
+                    /* clflush */
+                    if (!(s->cpuid_features & CPUID_CLFLUSH))
+                        goto illegal_op;
+                }
                 gen_lea_modrm(env, s, modrm);
             }
             break;
@@ -7899,21 +7877,18 @@ void optimize_flags_init(void)
                                          offsetof(CPUX86State, regs[i]),
                                          reg_names[i]);
     }
+
+    helper_lock_init();
 }
 
 /* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(X86CPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+   basic block 'tb'.  */
+void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUX86State *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
     target_ulong pc_ptr;
-    CPUBreakpoint *bp;
-    int j, lj;
     uint64_t flags;
     target_ulong pc_start;
     target_ulong cs_base;
@@ -7943,7 +7918,7 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
     /* select memory access functions */
     dc->mem_index = 0;
     if (flags & HF_SOFTMMU_MASK) {
-        dc->mem_index = cpu_mmu_index(env);
+	dc->mem_index = cpu_mmu_index(env, false);
     }
     dc->cpuid_features = env->features[FEAT_1_EDX];
     dc->cpuid_ext_features = env->features[FEAT_1_ECX];
@@ -7993,40 +7968,37 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
 
     dc->is_jmp = DISAS_NEXT;
     pc_ptr = pc_start;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     for(;;) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == pc_ptr &&
-                    !((bp->flags & BP_CPU) && (tb->flags & HF_RF_MASK))) {
-                    gen_debug(dc, pc_ptr - dc->cs_base);
-                    goto done_generating;
-                }
-            }
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = pc_ptr;
-            gen_opc_cc_op[lj] = dc->cc_op;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        tcg_gen_insn_start(pc_ptr, dc->cc_op);
+        num_insns++;
+
+        /* If RF is set, suppress an internally generated breakpoint.  */
+        if (unlikely(cpu_breakpoint_test(cs, pc_ptr,
+                                         tb->flags & HF_RF_MASK
+                                         ? BP_GDB : BP_ANY))) {
+            gen_debug(dc, pc_ptr - dc->cs_base);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            pc_ptr += 1;
+            goto done_generating;
+        }
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         pc_ptr = disas_insn(env, dc, pc_ptr);
-        num_insns++;
         /* stop translation if indicated */
         if (dc->is_jmp)
             break;
@@ -8074,14 +8046,6 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
 done_generating:
     gen_tb_end(tb, num_insns);
 
-    /* we don't forget to fill the last values */
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    }
-
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         int disas_flags;
@@ -8098,42 +8062,16 @@ done_generating:
     }
 #endif
 
-    if (!search_pc) {
-        tb->size = pc_ptr - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = pc_ptr - pc_start;
+    tb->icount = num_insns;
 }
 
-void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb)
+void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(x86_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUX86State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(x86_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, int pc_pos)
-{
-    int cc_op;
-#ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_OP)) {
-        int i;
-        qemu_log("RESTORE:\n");
-        for(i = 0;i <= pc_pos; i++) {
-            if (tcg_ctx.gen_opc_instr_start[i]) {
-                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i,
-                        tcg_ctx.gen_opc_pc[i]);
-            }
-        }
-        qemu_log("pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
-                pc_pos, tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base,
-                (uint32_t)tb->cs_base);
-    }
-#endif
-    env->eip = tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base;
-    cc_op = gen_opc_cc_op[pc_pos];
-    if (cc_op != CC_OP_DYNAMIC)
+    int cc_op = data[1];
+    env->eip = data[0] - tb->cs_base;
+    if (cc_op != CC_OP_DYNAMIC) {
         env->cc_op = cc_op;
+    }
 }
diff --git a/target-lm32/cpu.c b/target-lm32/cpu.c
index d0ab2786a..0bc544c1e 100644
--- a/target-lm32/cpu.c
+++ b/target-lm32/cpu.c
@@ -131,6 +131,12 @@ static void lm32_cpu_reset(CPUState *s)
     tlb_flush(s, 1);
 }
 
+static void lm32_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_lm32;
+    info->print_insn = print_insn_lm32;
+}
+
 static void lm32_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -275,6 +281,7 @@ static void lm32_cpu_class_init(ObjectClass *oc, void *data)
     cc->gdb_num_core_regs = 32 + 7;
     cc->gdb_stop_before_watchpoint = true;
     cc->debug_excp_handler = lm32_debug_excp_handler;
+    cc->disas_set_info = lm32_cpu_disas_set_info;
 
     /*
      * Reason: lm32_cpu_initfn() calls cpu_exec_init(), which saves
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index 944777d05..2b7620c7d 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -30,11 +30,9 @@
 struct CPULM32State;
 typedef struct CPULM32State CPULM32State;
 
-#define ELF_MACHINE EM_LATTICEMICO32
-
 #define NB_MMU_MODES 1
 #define TARGET_PAGE_BITS 12
-static inline int cpu_mmu_index(CPULM32State *env)
+static inline int cpu_mmu_index(CPULM32State *env, bool ifetch)
 {
     return 0;
 }
@@ -221,7 +219,6 @@ bool lm32_cpu_do_semihosting(CPUState *cs);
 
 #define cpu_list lm32_cpu_list
 #define cpu_exec cpu_lm32_exec
-#define cpu_gen_code cpu_lm32_gen_code
 #define cpu_signal_handler cpu_lm32_signal_handler
 
 int lm32_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
diff --git a/target-lm32/helper.c b/target-lm32/helper.c
index a88aa5ad1..e26c13357 100644
--- a/target-lm32/helper.c
+++ b/target-lm32/helper.c
@@ -81,7 +81,7 @@ void lm32_watchpoint_insert(CPULM32State *env, int idx, target_ulong address,
 
     switch (wp_type) {
     case LM32_WP_DISABLED:
-        /* nothing to to */
+        /* nothing to do */
         break;
     case LM32_WP_READ:
         flags = BP_CPU | BP_STOP_BEFORE_ACCESS | BP_MEM_READ;
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index cf7042e3e..fa5b0b93a 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -1005,10 +1005,6 @@ static const DecoderInfo decinfo[] = {
 
 static inline void decode(DisasContext *dc, uint32_t ir)
 {
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
@@ -1036,32 +1032,13 @@ static inline void decode(DisasContext *dc, uint32_t ir)
     decinfo[dc->opcode](dc);
 }
 
-static void check_breakpoint(CPULM32State *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(lm32_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_tl(cpu_pc, dc->pc);
-                t_gen_raise_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 /* generate intermediate code for basic block 'tb'.  */
-static inline
-void gen_intermediate_code_internal(LM32CPU *cpu,
-                                    TranslationBlock *tb, bool search_pc)
+void gen_intermediate_code(CPULM32State *env, struct TranslationBlock *tb)
 {
+    LM32CPU *cpu = lm32_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPULM32State *env = &cpu->env;
     struct DisasContext ctx, *dc = &ctx;
     uint32_t pc_start;
-    int j, lj;
     uint32_t next_page_start;
     int num_insns;
     int max_insns;
@@ -1083,41 +1060,41 @@ void gen_intermediate_code_internal(LM32CPU *cpu,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
-        check_breakpoint(env, dc);
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            t_gen_raise_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
         decode(dc, cpu_ldl_code(env, dc->pc));
         dc->pc += 4;
-        num_insns++;
-
     } while (!dc->is_jmp
          && !tcg_op_buf_full()
          && !cs->singlestep_enabled
@@ -1154,16 +1131,8 @@ void gen_intermediate_code_internal(LM32CPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -1175,16 +1144,6 @@ void gen_intermediate_code_internal(LM32CPU *cpu,
 #endif
 }
 
-void gen_intermediate_code(CPULM32State *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(lm32_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPULM32State *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(lm32_env_get_cpu(env), tb, true);
-}
-
 void lm32_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -1219,9 +1178,10 @@ void lm32_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "\n\n");
 }
 
-void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
 
 void lm32_translate_init(void)
diff --git a/target-m68k/cpu.c b/target-m68k/cpu.c
index 97527ef32..e8a4eed1f 100644
--- a/target-m68k/cpu.c
+++ b/target-m68k/cpu.c
@@ -61,6 +61,11 @@ static void m68k_cpu_reset(CPUState *s)
     tlb_flush(s, 1);
 }
 
+static void m68k_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->print_insn = print_insn_m68k;
+}
+
 /* CPU models */
 
 static ObjectClass *m68k_cpu_class_by_name(const char *cpu_model)
@@ -208,11 +213,13 @@ static void m68k_cpu_class_init(ObjectClass *c, void *data)
 #endif
     cc->cpu_exec_enter = m68k_cpu_exec_enter;
     cc->cpu_exec_exit = m68k_cpu_exec_exit;
+    cc->disas_set_info = m68k_cpu_disas_set_info;
 
-    dc->vmsd = &vmstate_m68k_cpu;
     cc->gdb_num_core_regs = 18;
     cc->gdb_core_xml_file = "cf-core.xml";
 
+    dc->vmsd = &vmstate_m68k_cpu;
+
     /*
      * Reason: m68k_cpu_initfn() calls cpu_exec_init(), which saves
      * the object in cpus -> dangling pointer after final
diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h
index 9a62f6cb5..224c16967 100644
--- a/target-m68k/cpu.h
+++ b/target-m68k/cpu.h
@@ -32,8 +32,6 @@
 
 #define MAX_QREGS 32
 
-#define ELF_MACHINE	EM_68K
-
 #define EXCP_ACCESS         2   /* Access (MMU) error.  */
 #define EXCP_ADDRESS        3   /* Address error.  */
 #define EXCP_ILLEGAL        4   /* Illegal instruction.  */
@@ -215,7 +213,6 @@ void register_m68k_insns (CPUM68KState *env);
 #define cpu_init(cpu_model) CPU(cpu_m68k_init(cpu_model))
 
 #define cpu_exec cpu_m68k_exec
-#define cpu_gen_code cpu_m68k_gen_code
 #define cpu_signal_handler cpu_m68k_signal_handler
 #define cpu_list m68k_cpu_list
 
@@ -223,7 +220,7 @@ void register_m68k_insns (CPUM68KState *env);
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUM68KState *env)
+static inline int cpu_mmu_index (CPUM68KState *env, bool ifetch)
 {
     return (env->sr & SR_S) == 0 ? 1 : 0;
 }
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index a57d2415c..41ae2c605 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -2680,7 +2680,7 @@ DISAS_INSN(from_mac)
     if (s->env->macsr & MACSR_FI) {
         gen_helper_get_macf(rx, cpu_env, acc);
     } else if ((s->env->macsr & MACSR_OMC) == 0) {
-        tcg_gen_trunc_i64_i32(rx, acc);
+        tcg_gen_extrl_i64_i32(rx, acc);
     } else if (s->env->macsr & MACSR_SU) {
         gen_helper_get_macs(rx, acc);
     } else {
@@ -2955,10 +2955,6 @@ static void disas_m68k_insn(CPUM68KState * env, DisasContext *s)
 {
     uint16_t insn;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(s->pc);
-    }
-
     insn = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
@@ -2966,15 +2962,11 @@ static void disas_m68k_insn(CPUM68KState * env, DisasContext *s)
 }
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(M68kCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUM68KState *env, TranslationBlock *tb)
 {
+    M68kCPU *cpu = m68k_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUM68KState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     int pc_offset;
     int num_insns;
@@ -2993,43 +2985,39 @@ gen_intermediate_code_internal(M68kCPU *cpu, TranslationBlock *tb,
     dc->fpcr = env->fpcr;
     dc->user = (env->sr & SR_S) == 0;
     dc->done_mac = 0;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
         pc_offset = dc->pc - pc_start;
         gen_throws_exception = NULL;
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    gen_exception(dc, dc->pc, EXCP_DEBUG);
-                    dc->is_jmp = DISAS_JUMP;
-                    break;
-                }
-            }
-            if (dc->is_jmp)
-                break;
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            gen_exception(dc, dc->pc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_JUMP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 2;
+            break;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
+
         dc->insn_pc = dc->pc;
 	disas_m68k_insn(env, dc);
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -3073,28 +3061,8 @@ gen_intermediate_code_internal(M68kCPU *cpu, TranslationBlock *tb,
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-
-    //optimize_flags();
-    //expand_target_qops();
-}
-
-void gen_intermediate_code(CPUM68KState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(m68k_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUM68KState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(m68k_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 void m68k_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
@@ -3120,7 +3088,8 @@ void m68k_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf (f, "FPRESULT = %12g\n", *(double *)&env->fp_result);
 }
 
-void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-microblaze/cpu.c b/target-microblaze/cpu.c
index 15b46f768..52959e13b 100644
--- a/target-microblaze/cpu.c
+++ b/target-microblaze/cpu.c
@@ -107,6 +107,8 @@ static void mb_cpu_reset(CPUState *s)
     /* Disable stack protector.  */
     env->shr = ~0;
 
+    env->sregs[SR_PC] = cpu->cfg.base_vectors;
+
 #if defined(CONFIG_USER_ONLY)
     /* start in user mode with interrupts enabled.  */
     env->sregs[SR_MSR] = MSR_EE | MSR_IE | MSR_VM | MSR_UM;
@@ -183,8 +185,6 @@ static void mb_cpu_realizefn(DeviceState *dev, Error **errp)
     env->pvr.regs[10] = 0x0c000000; /* Default to spartan 3a dsp family.  */
     env->pvr.regs[11] = PVR11_USE_MMU | (16 << 17);
 
-    env->sregs[SR_PC] = cpu->cfg.base_vectors;
-
     mcc->parent_realize(dev, errp);
 }
 
diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h
index 7e20e59b8..6b212ab7f 100644
--- a/target-microblaze/cpu.h
+++ b/target-microblaze/cpu.h
@@ -34,8 +34,6 @@ typedef struct CPUMBState CPUMBState;
 #include "mmu.h"
 #endif
 
-#define ELF_MACHINE	EM_MICROBLAZE
-
 #define EXCP_MMU        1
 #define EXCP_IRQ        2
 #define EXCP_BREAK      3
@@ -297,7 +295,6 @@ int cpu_mb_signal_handler(int host_signum, void *pinfo,
 #define cpu_init(cpu_model) CPU(cpu_mb_init(cpu_model))
 
 #define cpu_exec cpu_mb_exec
-#define cpu_gen_code cpu_mb_gen_code
 #define cpu_signal_handler cpu_mb_signal_handler
 
 /* MMU modes definitions */
@@ -309,7 +306,7 @@ int cpu_mb_signal_handler(int host_signum, void *pinfo,
 #define MMU_USER_IDX    2
 /* See NB_MMU_MODES further up the file.  */
 
-static inline int cpu_mmu_index (CPUMBState *env)
+static inline int cpu_mmu_index (CPUMBState *env, bool ifetch)
 {
         /* Are we in nommu mode?.  */
         if (!(env->sregs[SR_MSR] & MSR_VM))
diff --git a/target-microblaze/mmu.c b/target-microblaze/mmu.c
index 728da133d..2ef1dc21a 100644
--- a/target-microblaze/mmu.c
+++ b/target-microblaze/mmu.c
@@ -279,7 +279,7 @@ void mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
             }
 
             hit = mmu_translate(&env->mmu, &lu,
-                                v & TLB_EPN_MASK, 0, cpu_mmu_index(env));
+                                v & TLB_EPN_MASK, 0, cpu_mmu_index(env, false));
             if (hit) {
                 env->mmu.regs[MMU_R_TLBX] = lu.idx;
             } else
diff --git a/target-microblaze/op_helper.c b/target-microblaze/op_helper.c
index d2b362451..d32434770 100644
--- a/target-microblaze/op_helper.c
+++ b/target-microblaze/op_helper.c
@@ -18,7 +18,6 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <assert.h>
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
@@ -151,9 +150,7 @@ uint32_t helper_clz(uint32_t t0)
 
 uint32_t helper_carry(uint32_t a, uint32_t b, uint32_t cf)
 {
-    uint32_t ncf;
-    ncf = compute_carry(a, b, cf);
-    return ncf;
+    return compute_carry(a, b, cf);
 }
 
 static inline int div_prepare(CPUMBState *env, uint32_t a, uint32_t b)
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index f4e969b29..154b9d683 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -313,7 +313,6 @@ static void dec_sub(DisasContext *dc)
 static void dec_pattern(DisasContext *dc)
 {
     unsigned int mode;
-    TCGLabel *l1;
 
     if ((dc->tb_flags & MSR_EE_FLAG)
           && (dc->cpu->env.pvr.regs[2] & PVR2_ILL_OPCODE_EXC_MASK)
@@ -333,29 +332,15 @@ static void dec_pattern(DisasContext *dc)
         case 2:
             LOG_DIS("pcmpeq r%d r%d r%d\n", dc->rd, dc->ra, dc->rb);
             if (dc->rd) {
-                TCGv t0 = tcg_temp_local_new();
-                l1 = gen_new_label();
-                tcg_gen_movi_tl(t0, 1);
-                tcg_gen_brcond_tl(TCG_COND_EQ,
-                                  cpu_R[dc->ra], cpu_R[dc->rb], l1);
-                tcg_gen_movi_tl(t0, 0);
-                gen_set_label(l1);
-                tcg_gen_mov_tl(cpu_R[dc->rd], t0);
-                tcg_temp_free(t0);
+                tcg_gen_setcond_tl(TCG_COND_EQ, cpu_R[dc->rd],
+                                   cpu_R[dc->ra], cpu_R[dc->rb]);
             }
             break;
         case 3:
             LOG_DIS("pcmpne r%d r%d r%d\n", dc->rd, dc->ra, dc->rb);
-            l1 = gen_new_label();
             if (dc->rd) {
-                TCGv t0 = tcg_temp_local_new();
-                tcg_gen_movi_tl(t0, 1);
-                tcg_gen_brcond_tl(TCG_COND_NE,
-                                  cpu_R[dc->ra], cpu_R[dc->rb], l1);
-                tcg_gen_movi_tl(t0, 0);
-                gen_set_label(l1);
-                tcg_gen_mov_tl(cpu_R[dc->rd], t0);
-                tcg_temp_free(t0);
+                tcg_gen_setcond_tl(TCG_COND_NE, cpu_R[dc->rd],
+                                   cpu_R[dc->ra], cpu_R[dc->rb]);
             }
             break;
         default:
@@ -433,7 +418,7 @@ static void dec_msr(DisasContext *dc)
     CPUState *cs = CPU(dc->cpu);
     TCGv t0, t1;
     unsigned int sr, to, rn;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     sr = dc->imm & ((1 << 14) - 1);
     to = dc->imm & (1 << 14);
@@ -598,9 +583,9 @@ static void t_gen_muls(TCGv d, TCGv d2, TCGv a, TCGv b)
     tcg_gen_ext_i32_i64(t1, b);
     tcg_gen_mul_i64(t0, t0, t1);
 
-    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_extrl_i64_i32(d, t0);
     tcg_gen_shri_i64(t0, t0, 32);
-    tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_extrl_i64_i32(d2, t0);
 
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
@@ -618,9 +603,9 @@ static void t_gen_mulu(TCGv d, TCGv d2, TCGv a, TCGv b)
     tcg_gen_extu_i32_i64(t1, b);
     tcg_gen_mul_i64(t0, t0, t1);
 
-    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_extrl_i64_i32(d, t0);
     tcg_gen_shri_i64(t0, t0, 32);
-    tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_extrl_i64_i32(d2, t0);
 
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
@@ -745,7 +730,7 @@ static void dec_bit(DisasContext *dc)
     CPUState *cs = CPU(dc->cpu);
     TCGv t0;
     unsigned int op;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     op = dc->ir & ((1 << 9) - 1);
     switch (op) {
@@ -1009,7 +994,7 @@ static void dec_load(DisasContext *dc)
      * address and if that succeeds we write into the destination reg.
      */
     v = tcg_temp_new();
-    tcg_gen_qemu_ld_tl(v, *addr, cpu_mmu_index(&dc->cpu->env), mop);
+    tcg_gen_qemu_ld_tl(v, *addr, cpu_mmu_index(&dc->cpu->env, false), mop);
 
     if ((dc->cpu->env.pvr.regs[2] & PVR2_UNALIGNED_EXC_MASK) && size > 1) {
         tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
@@ -1027,7 +1012,7 @@ static void dec_load(DisasContext *dc)
     tcg_temp_free(v);
 
     if (ex) { /* lwx */
-        /* no support for for AXI exclusive so always clear C */
+        /* no support for AXI exclusive so always clear C */
         write_carryi(dc, 0);
     }
 
@@ -1087,7 +1072,7 @@ static void dec_store(DisasContext *dc)
            this compare and the following write to be atomic. For user
            emulation we need to add atomicity between threads.  */
         tval = tcg_temp_new();
-        tcg_gen_qemu_ld_tl(tval, swx_addr, cpu_mmu_index(&dc->cpu->env),
+        tcg_gen_qemu_ld_tl(tval, swx_addr, cpu_mmu_index(&dc->cpu->env, false),
                            MO_TEUL);
         tcg_gen_brcond_tl(TCG_COND_NE, env_res_val, tval, swx_skip);
         write_carryi(dc, 0);
@@ -1138,7 +1123,7 @@ static void dec_store(DisasContext *dc)
                 break;
         }
     }
-    tcg_gen_qemu_st_tl(cpu_R[dc->rd], *addr, cpu_mmu_index(&dc->cpu->env), mop);
+    tcg_gen_qemu_st_tl(cpu_R[dc->rd], *addr, cpu_mmu_index(&dc->cpu->env, false), mop);
 
     /* Verify alignment if needed.  */
     if ((dc->cpu->env.pvr.regs[2] & PVR2_UNALIGNED_EXC_MASK) && size > 1) {
@@ -1234,7 +1219,7 @@ static void dec_bcc(DisasContext *dc)
 static void dec_br(DisasContext *dc)
 {
     unsigned int dslot, link, abs, mbar;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     dslot = dc->ir & (1 << 20);
     abs = dc->ir & (1 << 19);
@@ -1366,7 +1351,7 @@ static inline void do_rte(DisasContext *dc)
 static void dec_rts(DisasContext *dc)
 {
     unsigned int b_bit, i_bit, e_bit;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     i_bit = dc->ir & (1 << 21);
     b_bit = dc->ir & (1 << 22);
@@ -1538,7 +1523,7 @@ static void dec_null(DisasContext *dc)
 /* Insns connected to FSL or AXI stream attached devices.  */
 static void dec_stream(DisasContext *dc)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
     TCGv_i32 t_id, t_ctrl;
     int ctrl;
 
@@ -1603,10 +1588,6 @@ static inline void decode(DisasContext *dc, uint32_t ir)
 {
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
@@ -1645,30 +1626,12 @@ static inline void decode(DisasContext *dc, uint32_t ir)
     }
 }
 
-static void check_breakpoint(CPUMBState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(mb_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                t_gen_raise_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMBState *env, struct TranslationBlock *tb)
 {
+    MicroBlazeCPU *cpu = mb_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUMBState *env = &cpu->env;
     uint32_t pc_start;
-    int j, lj;
     struct DisasContext ctx;
     struct DisasContext *dc = &ctx;
     uint32_t next_page_start, org_flags;
@@ -1705,47 +1668,51 @@ gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do
     {
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
 #if SIM_COMPAT
         if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
             tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
             gen_helper_debug();
         }
 #endif
-        check_breakpoint(env, dc);
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-                        tcg_ctx.gen_opc_icount[lj] = num_insns;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            t_gen_raise_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         dc->clear_imm = 1;
         decode(dc, cpu_ldl_code(env, dc->pc));
         if (dc->clear_imm)
             dc->tb_flags &= ~IMM_FLAG;
         dc->pc += 4;
-        num_insns++;
 
         if (dc->delayed_branch) {
             dc->delayed_branch--;
@@ -1836,15 +1803,8 @@ gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
     }
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-                tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
 #if !SIM_COMPAT
@@ -1861,16 +1821,6 @@ gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
     assert(!dc->abort_at_next_insn);
 }
 
-void gen_intermediate_code (CPUMBState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mb_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUMBState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mb_env_get_cpu(env), tb, true);
-}
-
 void mb_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                        int flags)
 {
@@ -1951,7 +1901,8 @@ void mb_tcg_init(void)
     }
 }
 
-void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->sregs[SR_PC] = tcg_ctx.gen_opc_pc[pc_pos];
+    env->sregs[SR_PC] = data[0];
 }
diff --git a/target-mips/cpu.c b/target-mips/cpu.c
index 7fe1f0407..639a24b36 100644
--- a/target-mips/cpu.c
+++ b/target-mips/cpu.c
@@ -53,12 +53,15 @@ static bool mips_cpu_has_work(CPUState *cs)
     CPUMIPSState *env = &cpu->env;
     bool has_work = false;
 
-    /* It is implementation dependent if non-enabled interrupts
-       wake-up the CPU, however most of the implementations only
+    /* Prior to MIPS Release 6 it is implementation dependent if non-enabled
+       interrupts wake-up the CPU, however most of the implementations only
        check for interrupts that can be taken. */
     if ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
         cpu_mips_hw_interrupts_pending(env)) {
-        has_work = true;
+        if (cpu_mips_hw_interrupts_enabled(env) ||
+            (env->insn_flags & ISA_MIPS32R6)) {
+            has_work = true;
+        }
     }
 
     /* MIPS-MT has the ability to halt the CPU.  */
@@ -97,6 +100,14 @@ static void mips_cpu_reset(CPUState *s)
 #endif
 }
 
+static void mips_cpu_disas_set_info(CPUState *s, disassemble_info *info) {
+#ifdef TARGET_WORDS_BIGENDIAN
+    info->print_insn = print_insn_big_mips;
+#else
+    info->print_insn = print_insn_little_mips;
+#endif
+}
+
 static void mips_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -150,6 +161,7 @@ static void mips_cpu_class_init(ObjectClass *c, void *data)
     cc->get_phys_page_debug = mips_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_mips_cpu;
 #endif
+    cc->disas_set_info = mips_cpu_disas_set_info;
 
     cc->gdb_num_core_regs = 73;
     cc->gdb_stop_before_watchpoint = true;
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index 075c561c8..89c01f7a3 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -5,8 +5,6 @@
 
 #define ALIGNED_ONLY
 
-#define ELF_MACHINE	EM_MIPS
-
 #define CPUArchState struct CPUMIPSState
 
 #include "config.h"
@@ -132,6 +130,7 @@ struct CPUMIPSFPUContext {
 };
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 2
 
 typedef struct CPUMIPSMVPContext CPUMIPSMVPContext;
 struct CPUMIPSMVPContext {
@@ -395,7 +394,7 @@ struct CPUMIPSState {
 #define CP0C0_K23  28
 #define CP0C0_KU   25
 #define CP0C0_MDU  20
-#define CP0C0_MM   17
+#define CP0C0_MM   18
 #define CP0C0_BM   16
 #define CP0C0_BE   15
 #define CP0C0_AT   13
@@ -470,6 +469,7 @@ struct CPUMIPSState {
 #define CP0C5_CV         29
 #define CP0C5_EVA        28
 #define CP0C5_MSAEn      27
+#define CP0C5_XNP        13
 #define CP0C5_UFE        9
 #define CP0C5_FRE        8
 #define CP0C5_SBRI       6
@@ -621,7 +621,6 @@ void mips_cpu_unassigned_access(CPUState *cpu, hwaddr addr,
 void mips_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 
 #define cpu_exec cpu_mips_exec
-#define cpu_gen_code cpu_mips_gen_code
 #define cpu_signal_handler cpu_mips_signal_handler
 #define cpu_list mips_cpu_list
 
@@ -634,28 +633,29 @@ extern uint32_t cpu_rddsp(uint32_t mask_num, CPUMIPSState *env);
 #define MMU_MODE1_SUFFIX _super
 #define MMU_MODE2_SUFFIX _user
 #define MMU_USER_IDX 2
-static inline int cpu_mmu_index (CPUMIPSState *env)
+static inline int cpu_mmu_index (CPUMIPSState *env, bool ifetch)
 {
     return env->hflags & MIPS_HFLAG_KSU;
 }
 
-static inline int cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
+static inline bool cpu_mips_hw_interrupts_enabled(CPUMIPSState *env)
 {
-    int32_t pending;
-    int32_t status;
-    int r;
-
-    if (!(env->CP0_Status & (1 << CP0St_IE)) ||
-        (env->CP0_Status & (1 << CP0St_EXL)) ||
-        (env->CP0_Status & (1 << CP0St_ERL)) ||
+    return (env->CP0_Status & (1 << CP0St_IE)) &&
+        !(env->CP0_Status & (1 << CP0St_EXL)) &&
+        !(env->CP0_Status & (1 << CP0St_ERL)) &&
+        !(env->hflags & MIPS_HFLAG_DM) &&
         /* Note that the TCStatus IXMT field is initialized to zero,
            and only MT capable cores can set it to one. So we don't
            need to check for MT capabilities here.  */
-        (env->active_tc.CP0_TCStatus & (1 << CP0TCSt_IXMT)) ||
-        (env->hflags & MIPS_HFLAG_DM)) {
-        /* Interrupts are disabled */
-        return 0;
-    }
+        !(env->active_tc.CP0_TCStatus & (1 << CP0TCSt_IXMT));
+}
+
+/* Check if there is pending and not masked out interrupt */
+static inline bool cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
+{
+    int32_t pending;
+    int32_t status;
+    bool r;
 
     pending = env->CP0_Cause & CP0Ca_IP_mask;
     status = env->CP0_Status & CP0Ca_IP_mask;
@@ -669,7 +669,7 @@ static inline int cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
         /* A MIPS configured with compatibility or VInt (Vectored Interrupts)
            treats the pending lines as individual interrupt lines, the status
            lines are individual masks.  */
-        r = pending & status;
+        r = (pending & status) != 0;
     }
     return r;
 }
@@ -961,6 +961,15 @@ static inline void compute_hflags(CPUMIPSState *env)
 }
 
 #ifndef CONFIG_USER_ONLY
+static inline void cpu_mips_tlb_flush(CPUMIPSState *env, int flush_global)
+{
+    MIPSCPU *cpu = mips_env_get_cpu(env);
+
+    /* Flush qemu's TLB and discard all shadowed entries.  */
+    tlb_flush(CPU(cpu), flush_global);
+    env->tlb->tlb_in_use = env->tlb->nb_tlb;
+}
+
 /* Called for updates to CP0_Status.  */
 static inline void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
 {
@@ -999,17 +1008,29 @@ static inline void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
 static inline void cpu_mips_store_status(CPUMIPSState *env, target_ulong val)
 {
     uint32_t mask = env->CP0_Status_rw_bitmask;
+    target_ulong old = env->CP0_Status;
 
     if (env->insn_flags & ISA_MIPS32R6) {
         bool has_supervisor = extract32(mask, CP0St_KSU, 2) == 0x3;
-
+#if defined(TARGET_MIPS64)
+        uint32_t ksux = (1 << CP0St_KX) & val;
+        ksux |= (ksux >> 1) & val; /* KX = 0 forces SX to be 0 */
+        ksux |= (ksux >> 1) & val; /* SX = 0 forces UX to be 0 */
+        val = (val & ~(7 << CP0St_UX)) | ksux;
+#endif
         if (has_supervisor && extract32(val, CP0St_KSU, 2) == 0x3) {
             mask &= ~(3 << CP0St_KSU);
         }
         mask &= ~(((1 << CP0St_SR) | (1 << CP0St_NMI)) & val);
     }
 
-    env->CP0_Status = (env->CP0_Status & ~mask) | (val & mask);
+    env->CP0_Status = (old & ~mask) | (val & mask);
+#if defined(TARGET_MIPS64)
+    if ((env->CP0_Status ^ old) & (old & (7 << CP0St_UX))) {
+        /* Access to at least one of the 64-bit segments has been disabled */
+        cpu_mips_tlb_flush(env, 1);
+    }
+#endif
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
         sync_c0_status(env, env, env->current_tc);
     } else {
@@ -1049,4 +1070,28 @@ static inline void cpu_mips_store_cause(CPUMIPSState *env, target_ulong val)
 }
 #endif
 
+static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
+                                                        uint32_t exception,
+                                                        int error_code,
+                                                        uintptr_t pc)
+{
+    CPUState *cs = CPU(mips_env_get_cpu(env));
+
+    if (exception < EXCP_SC) {
+        qemu_log_mask(CPU_LOG_INT, "%s: %d %d\n",
+                      __func__, exception, error_code);
+    }
+    cs->exception_index = exception;
+    env->error_code = error_code;
+
+    cpu_loop_exit_restore(cs, pc);
+}
+
+static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
+                                                    uint32_t exception,
+                                                    uintptr_t pc)
+{
+    do_raise_exception_err(env, exception, 0, pc);
+}
+
 #endif /* !defined (__MIPS_CPU_H__) */
diff --git a/target-mips/helper.c b/target-mips/helper.c
index 04ba19fd4..118072a9e 100644
--- a/target-mips/helper.c
+++ b/target-mips/helper.c
@@ -21,7 +21,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <inttypes.h>
-#include <signal.h>
 
 #include "cpu.h"
 #include "sysemu/kvm.h"
@@ -127,10 +126,6 @@ static int get_physical_address (CPUMIPSState *env, hwaddr *physical,
     /* effective address (modified for KVM T&E kernel segments) */
     target_ulong address = real_address;
 
-#if 0
-    qemu_log("user mode %d h %08x\n", user_mode, env->hflags);
-#endif
-
 #define USEG_LIMIT      0x7FFFFFFFUL
 #define KSEG0_BASE      0x80000000UL
 #define KSEG1_BASE      0xA0000000UL
@@ -227,11 +222,6 @@ static int get_physical_address (CPUMIPSState *env, hwaddr *physical,
             ret = TLBRET_BADADDR;
         }
     }
-#if 0
-    qemu_log(TARGET_FMT_lx " %d %d => %" HWADDR_PRIx " %d (%d)\n",
-            address, rw, access_type, *physical, *prot, ret);
-#endif
-
     return ret;
 }
 #endif
@@ -303,9 +293,10 @@ static void raise_mmu_exception(CPUMIPSState *env, target_ulong address,
         (env->CP0_EntryHi & 0xFF) | (address & (TARGET_PAGE_MASK << 1));
 #if defined(TARGET_MIPS64)
     env->CP0_EntryHi &= env->SEGMask;
-    env->CP0_XContext = (env->CP0_XContext & ((~0ULL) << (env->SEGBITS - 7))) |
-                        ((address & 0xC00000000000ULL) >> (55 - env->SEGBITS)) |
-                        ((address & ((1ULL << env->SEGBITS) - 1) & 0xFFFFFFFFFFFFE000ULL) >> 9);
+    env->CP0_XContext =
+        /* PTEBase */   (env->CP0_XContext & ((~0ULL) << (env->SEGBITS - 7))) |
+        /* R */         (extract64(address, 62, 2) << (env->SEGBITS - 9)) |
+        /* BadVPN2 */   (extract64(address, 13, env->SEGBITS - 13) << 4);
 #endif
     cs->exception_index = exception;
     env->error_code = error_code;
@@ -487,14 +478,16 @@ void mips_cpu_do_interrupt(CPUState *cs)
     int cause = -1;
     const char *name;
 
-    if (qemu_log_enabled() && cs->exception_index != EXCP_EXT_INTERRUPT) {
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && cs->exception_index != EXCP_EXT_INTERRUPT) {
         if (cs->exception_index < 0 || cs->exception_index > EXCP_LAST) {
             name = "unknown";
         } else {
             name = excp_names[cs->exception_index];
         }
 
-        qemu_log("%s enter: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx " %s exception\n",
+        qemu_log("%s enter: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx
+                 " %s exception\n",
                  __func__, env->active_tc.PC, env->CP0_EPC, name);
     }
     if (cs->exception_index == EXCP_EXT_INTERRUPT &&
@@ -531,6 +524,10 @@ void mips_cpu_do_interrupt(CPUState *cs)
  enter_debug_mode:
         if (env->insn_flags & ISA_MIPS3) {
             env->hflags |= MIPS_HFLAG_64;
+            if (!(env->insn_flags & ISA_MIPS64R6) ||
+                env->CP0_Status & (1 << CP0St_KX)) {
+                env->hflags &= ~MIPS_HFLAG_AWRAP;
+            }
         }
         env->hflags |= MIPS_HFLAG_DM | MIPS_HFLAG_CP0;
         env->hflags &= ~(MIPS_HFLAG_KSU);
@@ -555,6 +552,10 @@ void mips_cpu_do_interrupt(CPUState *cs)
         env->CP0_Status |= (1 << CP0St_ERL) | (1 << CP0St_BEV);
         if (env->insn_flags & ISA_MIPS3) {
             env->hflags |= MIPS_HFLAG_64;
+            if (!(env->insn_flags & ISA_MIPS64R6) ||
+                env->CP0_Status & (1 << CP0St_KX)) {
+                env->hflags &= ~MIPS_HFLAG_AWRAP;
+            }
         }
         env->hflags |= MIPS_HFLAG_CP0;
         env->hflags &= ~(MIPS_HFLAG_KSU);
@@ -700,7 +701,7 @@ void mips_cpu_do_interrupt(CPUState *cs)
         goto set_EPC;
     case EXCP_DWATCH:
         cause = 23;
-        /* XXX: TODO: manage defered watch exceptions */
+        /* XXX: TODO: manage deferred watch exceptions */
         goto set_EPC;
     case EXCP_MCHECK:
         cause = 24;
@@ -732,6 +733,10 @@ void mips_cpu_do_interrupt(CPUState *cs)
             env->CP0_Status |= (1 << CP0St_EXL);
             if (env->insn_flags & ISA_MIPS3) {
                 env->hflags |= MIPS_HFLAG_64;
+                if (!(env->insn_flags & ISA_MIPS64R6) ||
+                    env->CP0_Status & (1 << CP0St_KX)) {
+                    env->hflags &= ~MIPS_HFLAG_AWRAP;
+                }
             }
             env->hflags |= MIPS_HFLAG_CP0;
             env->hflags &= ~(MIPS_HFLAG_KSU);
@@ -747,16 +752,15 @@ void mips_cpu_do_interrupt(CPUState *cs)
         env->CP0_Cause = (env->CP0_Cause & ~(0x1f << CP0Ca_EC)) | (cause << CP0Ca_EC);
         break;
     default:
-        qemu_log("Invalid MIPS exception %d. Exiting\n", cs->exception_index);
-        printf("Invalid MIPS exception %d. Exiting\n", cs->exception_index);
-        exit(1);
+        abort();
     }
-    if (qemu_log_enabled() && cs->exception_index != EXCP_EXT_INTERRUPT) {
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && cs->exception_index != EXCP_EXT_INTERRUPT) {
         qemu_log("%s: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx " cause %d\n"
-                "    S %08x C %08x A " TARGET_FMT_lx " D " TARGET_FMT_lx "\n",
-                __func__, env->active_tc.PC, env->CP0_EPC, cause,
-                env->CP0_Status, env->CP0_Cause, env->CP0_BadVAddr,
-                env->CP0_DEPC);
+                 "    S %08x C %08x A " TARGET_FMT_lx " D " TARGET_FMT_lx "\n",
+                 __func__, env->active_tc.PC, env->CP0_EPC, cause,
+                 env->CP0_Status, env->CP0_Cause, env->CP0_BadVAddr,
+                 env->CP0_DEPC);
     }
 #endif
     cs->exception_index = EXCP_NONE;
@@ -768,7 +772,8 @@ bool mips_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
         MIPSCPU *cpu = MIPS_CPU(cs);
         CPUMIPSState *env = &cpu->env;
 
-        if (cpu_mips_hw_interrupts_pending(env)) {
+        if (cpu_mips_hw_interrupts_enabled(env) &&
+            cpu_mips_hw_interrupts_pending(env)) {
             /* Raise it */
             cs->exception_index = EXCP_EXT_INTERRUPT;
             env->error_code = 0;
diff --git a/target-mips/helper.h b/target-mips/helper.h
index 2b28e875c..95b9149d8 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -1,5 +1,6 @@
 DEF_HELPER_3(raise_exception_err, noreturn, env, i32, int)
 DEF_HELPER_2(raise_exception, noreturn, env, i32)
+DEF_HELPER_1(raise_exception_debug, noreturn, env)
 
 DEF_HELPER_1(do_semihosting, void, env)
 
@@ -357,6 +358,8 @@ DEF_HELPER_1(rdhwr_cpunum, tl, env)
 DEF_HELPER_1(rdhwr_synci_step, tl, env)
 DEF_HELPER_1(rdhwr_cc, tl, env)
 DEF_HELPER_1(rdhwr_ccres, tl, env)
+DEF_HELPER_1(rdhwr_performance, tl, env)
+DEF_HELPER_1(rdhwr_xnp, tl, env)
 DEF_HELPER_2(pmon, void, env, int)
 DEF_HELPER_1(wait, void, env)
 
diff --git a/target-mips/kvm.c b/target-mips/kvm.c
index d287d4288..12d7db311 100644
--- a/target-mips/kvm.c
+++ b/target-mips/kvm.c
@@ -678,7 +678,7 @@ int kvm_arch_get_registers(CPUState *cs)
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
diff --git a/target-mips/msa_helper.c b/target-mips/msa_helper.c
index a1cb48f2a..5dd3da67a 100644
--- a/target-mips/msa_helper.c
+++ b/target-mips/msa_helper.c
@@ -1352,7 +1352,7 @@ void helper_msa_ctcmsa(CPUMIPSState *env, target_ulong elm, uint32_t cd)
         /* check exception */
         if ((GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)
             & GET_FP_CAUSE(env->active_tc.msacsr)) {
-            helper_raise_exception(env, EXCP_MSAFPE);
+            do_raise_exception(env, EXCP_MSAFPE, GETPC());
         }
         break;
     }
@@ -1505,14 +1505,14 @@ static inline void clear_msacsr_cause(CPUMIPSState *env)
     SET_FP_CAUSE(env->active_tc.msacsr, 0);
 }
 
-static inline void check_msacsr_cause(CPUMIPSState *env)
+static inline void check_msacsr_cause(CPUMIPSState *env, uintptr_t retaddr)
 {
     if ((GET_FP_CAUSE(env->active_tc.msacsr) &
             (GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)) == 0) {
         UPDATE_FP_FLAGS(env->active_tc.msacsr,
                 GET_FP_CAUSE(env->active_tc.msacsr));
     } else {
-        helper_raise_exception(env, EXCP_MSAFPE);
+        do_raise_exception(env, EXCP_MSAFPE, retaddr);
     }
 }
 
@@ -1851,7 +1851,8 @@ static inline int32 float64_to_q32(float64 a, float_status *status)
     } while (0)
 
 static inline void compare_af(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1873,13 +1874,14 @@ static inline void compare_af(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_un(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1903,13 +1905,14 @@ static inline void compare_un(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_eq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1931,13 +1934,14 @@ static inline void compare_eq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ueq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1959,13 +1963,14 @@ static inline void compare_ueq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_lt(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1987,13 +1992,14 @@ static inline void compare_lt(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ult(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2015,13 +2021,14 @@ static inline void compare_ult(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_le(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2043,13 +2050,14 @@ static inline void compare_le(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ule(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2071,13 +2079,14 @@ static inline void compare_ule(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_or(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2099,13 +2108,14 @@ static inline void compare_or(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_une(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2127,13 +2137,15 @@ static inline void compare_une(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ne(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet) {
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
+{
     wr_t wx, *pwx = &wx;
     uint32_t i;
 
@@ -2154,7 +2166,7 @@ static inline void compare_ne(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
@@ -2165,7 +2177,7 @@ void helper_msa_fcaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_af(env, pwd, pws, pwt, df, 1);
+    compare_af(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2174,7 +2186,7 @@ void helper_msa_fcun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_un(env, pwd, pws, pwt, df, 1);
+    compare_un(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fceq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2183,7 +2195,7 @@ void helper_msa_fceq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_eq(env, pwd, pws, pwt, df, 1);
+    compare_eq(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2192,7 +2204,7 @@ void helper_msa_fcueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ueq(env, pwd, pws, pwt, df, 1);
+    compare_ueq(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fclt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2201,7 +2213,7 @@ void helper_msa_fclt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_lt(env, pwd, pws, pwt, df, 1);
+    compare_lt(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2210,7 +2222,7 @@ void helper_msa_fcult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ult(env, pwd, pws, pwt, df, 1);
+    compare_ult(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2219,7 +2231,7 @@ void helper_msa_fcle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_le(env, pwd, pws, pwt, df, 1);
+    compare_le(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2228,7 +2240,7 @@ void helper_msa_fcule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ule(env, pwd, pws, pwt, df, 1);
+    compare_ule(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fsaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2237,7 +2249,7 @@ void helper_msa_fsaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_af(env, pwd, pws, pwt, df, 0);
+    compare_af(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2246,7 +2258,7 @@ void helper_msa_fsun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_un(env, pwd, pws, pwt, df, 0);
+    compare_un(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fseq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2255,7 +2267,7 @@ void helper_msa_fseq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_eq(env, pwd, pws, pwt, df, 0);
+    compare_eq(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2264,7 +2276,7 @@ void helper_msa_fsueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ueq(env, pwd, pws, pwt, df, 0);
+    compare_ueq(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fslt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2273,7 +2285,7 @@ void helper_msa_fslt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_lt(env, pwd, pws, pwt, df, 0);
+    compare_lt(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2282,7 +2294,7 @@ void helper_msa_fsult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ult(env, pwd, pws, pwt, df, 0);
+    compare_ult(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2291,7 +2303,7 @@ void helper_msa_fsle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_le(env, pwd, pws, pwt, df, 0);
+    compare_le(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2300,7 +2312,7 @@ void helper_msa_fsule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ule(env, pwd, pws, pwt, df, 0);
+    compare_ule(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fcor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2309,7 +2321,7 @@ void helper_msa_fcor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_or(env, pwd, pws, pwt, df, 1);
+    compare_or(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2318,7 +2330,7 @@ void helper_msa_fcune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_une(env, pwd, pws, pwt, df, 1);
+    compare_une(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2327,7 +2339,7 @@ void helper_msa_fcne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ne(env, pwd, pws, pwt, df, 1);
+    compare_ne(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fsor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2336,7 +2348,7 @@ void helper_msa_fsor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_or(env, pwd, pws, pwt, df, 0);
+    compare_or(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2345,7 +2357,7 @@ void helper_msa_fsune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_une(env, pwd, pws, pwt, df, 0);
+    compare_une(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2354,7 +2366,7 @@ void helper_msa_fsne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ne(env, pwd, pws, pwt, df, 0);
+    compare_ne(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 #define float16_is_zero(ARG) 0
@@ -2404,7 +2416,7 @@ void helper_msa_fadd_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2434,7 +2446,7 @@ void helper_msa_fsub_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2464,7 +2476,7 @@ void helper_msa_fmul_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2495,7 +2507,7 @@ void helper_msa_fdiv_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2542,7 +2554,7 @@ void helper_msa_fmadd_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2577,7 +2589,7 @@ void helper_msa_fmsub_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2614,7 +2626,7 @@ void helper_msa_fexp2_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2666,7 +2678,7 @@ void helper_msa_fexdo_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2712,7 +2724,7 @@ void helper_msa_ftq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2791,7 +2803,7 @@ void helper_msa_fmin_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2822,7 +2834,7 @@ void helper_msa_fmin_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2865,7 +2877,7 @@ void helper_msa_fmax_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2896,7 +2908,7 @@ void helper_msa_fmax_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2958,7 +2970,7 @@ void helper_msa_ftrunc_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2988,7 +3000,7 @@ void helper_msa_ftrunc_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3018,7 +3030,7 @@ void helper_msa_fsqrt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3067,7 +3079,7 @@ void helper_msa_frsqrt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3097,7 +3109,7 @@ void helper_msa_frcp_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3127,7 +3139,7 @@ void helper_msa_frint_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3181,7 +3193,7 @@ void helper_msa_flog2_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3216,7 +3228,7 @@ void helper_msa_fexupl_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -3250,7 +3262,7 @@ void helper_msa_fexupr_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -3331,7 +3343,7 @@ void helper_msa_ftint_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3361,7 +3373,7 @@ void helper_msa_ftint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3397,7 +3409,7 @@ void helper_msa_ffint_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3427,7 +3439,7 @@ void helper_msa_ffint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
diff --git a/target-mips/op_helper.c b/target-mips/op_helper.c
index db4f6b946..d2c98c968 100644
--- a/target-mips/op_helper.c
+++ b/target-mips/op_helper.c
@@ -23,48 +23,26 @@
 #include "exec/cpu_ldst.h"
 #include "sysemu/kvm.h"
 
-#ifndef CONFIG_USER_ONLY
-static inline void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global);
-#endif
-
 /*****************************************************************************/
 /* Exceptions processing helpers */
 
-static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
-                                                        uint32_t exception,
-                                                        int error_code,
-                                                        uintptr_t pc)
+void helper_raise_exception_err(CPUMIPSState *env, uint32_t exception,
+                                int error_code)
 {
-    CPUState *cs = CPU(mips_env_get_cpu(env));
-
-    if (exception < EXCP_SC) {
-        qemu_log("%s: %d %d\n", __func__, exception, error_code);
-    }
-    cs->exception_index = exception;
-    env->error_code = error_code;
-
-    if (pc) {
-        /* now we have a real cpu fault */
-        cpu_restore_state(cs, pc);
-    }
-
-    cpu_loop_exit(cs);
+    do_raise_exception_err(env, exception, error_code, 0);
 }
 
-static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
-                                                    uint32_t exception,
-                                                    uintptr_t pc)
+void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
 {
-    do_raise_exception_err(env, exception, 0, pc);
+    do_raise_exception(env, exception, GETPC());
 }
 
-void helper_raise_exception_err(CPUMIPSState *env, uint32_t exception,
-                                int error_code)
+void helper_raise_exception_debug(CPUMIPSState *env)
 {
-    do_raise_exception_err(env, exception, error_code, 0);
+    do_raise_exception(env, EXCP_DEBUG, 0);
 }
 
-void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
+static void raise_exception(CPUMIPSState *env, uint32_t exception)
 {
     do_raise_exception(env, exception, 0);
 }
@@ -72,21 +50,21 @@ void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_LD(name, insn, type)                                     \
 static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             int mem_idx)                               \
+                             int mem_idx, uintptr_t retaddr)            \
 {                                                                       \
-    return (type) cpu_##insn##_data(env, addr);                         \
+    return (type) cpu_##insn##_data_ra(env, addr, retaddr);             \
 }
 #else
 #define HELPER_LD(name, insn, type)                                     \
 static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             int mem_idx)                               \
+                             int mem_idx, uintptr_t retaddr)            \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: return (type) cpu_##insn##_kernel(env, addr); break;        \
-    case 1: return (type) cpu_##insn##_super(env, addr); break;         \
+    case 0: return (type) cpu_##insn##_kernel_ra(env, addr, retaddr);   \
+    case 1: return (type) cpu_##insn##_super_ra(env, addr, retaddr);    \
     default:                                                            \
-    case 2: return (type) cpu_##insn##_user(env, addr); break;          \
+    case 2: return (type) cpu_##insn##_user_ra(env, addr, retaddr);     \
     }                                                                   \
 }
 #endif
@@ -99,21 +77,21 @@ HELPER_LD(ld, ldq, int64_t)
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_ST(name, insn, type)                                     \
 static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             type val, int mem_idx)                     \
+                             type val, int mem_idx, uintptr_t retaddr)  \
 {                                                                       \
-    cpu_##insn##_data(env, addr, val);                                  \
+    cpu_##insn##_data_ra(env, addr, val, retaddr);                      \
 }
 #else
 #define HELPER_ST(name, insn, type)                                     \
 static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             type val, int mem_idx)                     \
+                             type val, int mem_idx, uintptr_t retaddr)  \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: cpu_##insn##_kernel(env, addr, val); break;                 \
-    case 1: cpu_##insn##_super(env, addr, val); break;                  \
+    case 0: cpu_##insn##_kernel_ra(env, addr, val, retaddr); break;     \
+    case 1: cpu_##insn##_super_ra(env, addr, val, retaddr); break;      \
     default:                                                            \
-    case 2: cpu_##insn##_user(env, addr, val); break;                   \
+    case 2: cpu_##insn##_user_ra(env, addr, val, retaddr); break;       \
     }                                                                   \
 }
 #endif
@@ -292,14 +270,15 @@ target_ulong helper_bitswap(target_ulong rt)
 
 static inline hwaddr do_translate_address(CPUMIPSState *env,
                                                       target_ulong address,
-                                                      int rw)
+                                                      int rw, uintptr_t retaddr)
 {
     hwaddr lladdr;
+    CPUState *cs = CPU(mips_env_get_cpu(env));
 
     lladdr = cpu_mips_translate_address(env, address, rw);
 
     if (lladdr == -1LL) {
-        cpu_loop_exit(CPU(mips_env_get_cpu(env)));
+        cpu_loop_exit_restore(cs, retaddr);
     } else {
         return lladdr;
     }
@@ -310,10 +289,10 @@ target_ulong helper_##name(CPUMIPSState *env, target_ulong arg, int mem_idx)  \
 {                                                                             \
     if (arg & almask) {                                                       \
         env->CP0_BadVAddr = arg;                                              \
-        helper_raise_exception(env, EXCP_AdEL);                               \
+        do_raise_exception(env, EXCP_AdEL, GETPC());                          \
     }                                                                         \
-    env->lladdr = do_translate_address(env, arg, 0);                          \
-    env->llval = do_##insn(env, arg, mem_idx);                                \
+    env->lladdr = do_translate_address(env, arg, 0, GETPC());                 \
+    env->llval = do_##insn(env, arg, mem_idx, GETPC());                       \
     return env->llval;                                                        \
 }
 HELPER_LD_ATOMIC(ll, lw, 0x3)
@@ -330,12 +309,12 @@ target_ulong helper_##name(CPUMIPSState *env, target_ulong arg1,              \
                                                                               \
     if (arg2 & almask) {                                                      \
         env->CP0_BadVAddr = arg2;                                             \
-        helper_raise_exception(env, EXCP_AdES);                               \
+        do_raise_exception(env, EXCP_AdES, GETPC());                          \
     }                                                                         \
-    if (do_translate_address(env, arg2, 1) == env->lladdr) {                  \
-        tmp = do_##ld_insn(env, arg2, mem_idx);                               \
+    if (do_translate_address(env, arg2, 1, GETPC()) == env->lladdr) {         \
+        tmp = do_##ld_insn(env, arg2, mem_idx, GETPC());                      \
         if (tmp == env->llval) {                                              \
-            do_##st_insn(env, arg2, arg1, mem_idx);                           \
+            do_##st_insn(env, arg2, arg1, mem_idx, GETPC());                  \
             return 1;                                                         \
         }                                                                     \
     }                                                                         \
@@ -359,31 +338,43 @@ HELPER_ST_ATOMIC(scd, ld, sd, 0x7)
 void helper_swl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) <= 2)
-        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK(arg2) <= 2) {
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) <= 1)
-        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK(arg2) <= 1) {
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) == 0)
-        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx);
+    if (GET_LMASK(arg2) == 0) {
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx,
+              GETPC());
+    }
 }
 
 void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) >= 1)
-        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK(arg2) >= 1) {
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) >= 2)
-        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK(arg2) >= 2) {
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) == 3)
-        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK(arg2) == 3) {
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 }
 
 #if defined(TARGET_MIPS64)
@@ -399,55 +390,83 @@ void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
 void helper_sdl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) <= 6)
-        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx);
+    if (GET_LMASK64(arg2) <= 6) {
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 5)
-        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx);
+    if (GET_LMASK64(arg2) <= 5) {
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 4)
-        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx);
+    if (GET_LMASK64(arg2) <= 4) {
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 3)
-        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK64(arg2) <= 3) {
+        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 2)
-        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK64(arg2) <= 2) {
+        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 1)
-        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK64(arg2) <= 1) {
+        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 0)
-        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx);
+    if (GET_LMASK64(arg2) <= 0) {
+        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx,
+              GETPC());
+    }
 }
 
 void helper_sdr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) >= 1)
-        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK64(arg2) >= 1) {
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 2)
-        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK64(arg2) >= 2) {
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 3)
-        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK64(arg2) >= 3) {
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 4)
-        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx);
+    if (GET_LMASK64(arg2) >= 4) {
+        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 5)
-        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx);
+    if (GET_LMASK64(arg2) >= 5) {
+        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 6)
-        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx);
+    if (GET_LMASK64(arg2) >= 6) {
+        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) == 7)
-        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx);
+    if (GET_LMASK64(arg2) == 7) {
+        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx,
+              GETPC());
+    }
 }
 #endif /* TARGET_MIPS64 */
 
@@ -464,13 +483,14 @@ void helper_lwm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
 
         for (i = 0; i < base_reglist; i++) {
             env->active_tc.gpr[multiple_regs[i]] =
-                (target_long)do_lw(env, addr, mem_idx);
+                (target_long)do_lw(env, addr, mem_idx, GETPC());
             addr += 4;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx);
+        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx,
+                                                    GETPC());
     }
 }
 
@@ -484,13 +504,14 @@ void helper_swm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
+            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx,
+                  GETPC());
             addr += 4;
         }
     }
 
     if (do_r31) {
-        do_sw(env, addr, env->active_tc.gpr[31], mem_idx);
+        do_sw(env, addr, env->active_tc.gpr[31], mem_idx, GETPC());
     }
 }
 
@@ -505,13 +526,14 @@ void helper_ldm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx);
+            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx,
+                                                         GETPC());
             addr += 8;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx);
+        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx, GETPC());
     }
 }
 
@@ -525,13 +547,14 @@ void helper_sdm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
+            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx,
+                  GETPC());
             addr += 8;
         }
     }
 
     if (do_r31) {
-        do_sd(env, addr, env->active_tc.gpr[31], mem_idx);
+        do_sd(env, addr, env->active_tc.gpr[31], mem_idx, GETPC());
     }
 }
 #endif
@@ -1330,6 +1353,13 @@ void helper_mtc0_hwrena(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0x0000000F;
 
+    if ((env->CP0_Config1 & (1 << CP0C1_PC)) &&
+        (env->insn_flags & ISA_MIPS32R6)) {
+        mask |= (1 << 4);
+    }
+    if (env->insn_flags & ISA_MIPS32R6) {
+        mask |= (1 << 5);
+    }
     if (env->CP0_Config3 & (1 << CP0C3_ULRI)) {
         mask |= (1 << 29);
 
@@ -1791,13 +1821,13 @@ target_ulong helper_yield(CPUMIPSState *env, target_ulong arg)
                 env->active_tc.CP0_TCStatus & (1 << CP0TCSt_DT)) {
                 env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
                 env->CP0_VPEControl |= 4 << CP0VPECo_EXCPT;
-                helper_raise_exception(env, EXCP_THREAD);
+                do_raise_exception(env, EXCP_THREAD, GETPC());
             }
         }
     } else if (arg1 == 0) {
         if (0 /* TODO: TC underflow */) {
             env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
-            helper_raise_exception(env, EXCP_THREAD);
+            do_raise_exception(env, EXCP_THREAD, GETPC());
         } else {
             // TODO: Deallocate TC
         }
@@ -1805,22 +1835,13 @@ target_ulong helper_yield(CPUMIPSState *env, target_ulong arg)
         /* Yield qualifier inputs not implemented. */
         env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
         env->CP0_VPEControl |= 2 << CP0VPECo_EXCPT;
-        helper_raise_exception(env, EXCP_THREAD);
+        do_raise_exception(env, EXCP_THREAD, GETPC());
     }
     return env->CP0_YQMask;
 }
 
 #ifndef CONFIG_USER_ONLY
 /* TLB management */
-static void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global)
-{
-    MIPSCPU *cpu = mips_env_get_cpu(env);
-
-    /* Flush qemu's TLB and discard all shadowed entries.  */
-    tlb_flush(CPU(cpu), flush_global);
-    env->tlb->tlb_in_use = env->tlb->nb_tlb;
-}
-
 static void r4k_mips_tlb_flush_extra (CPUMIPSState *env, int first)
 {
     /* Discard entries from env->tlb[first] onwards.  */
@@ -2158,48 +2179,52 @@ void helper_deret(CPUMIPSState *env)
 }
 #endif /* !CONFIG_USER_ONLY */
 
-target_ulong helper_rdhwr_cpunum(CPUMIPSState *env)
+static inline void check_hwrena(CPUMIPSState *env, int reg)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 0)))
-        return env->CP0_EBase & 0x3ff;
-    else
-        helper_raise_exception(env, EXCP_RI);
+    if ((env->hflags & MIPS_HFLAG_CP0) || (env->CP0_HWREna & (1 << reg))) {
+        return;
+    }
+    do_raise_exception(env, EXCP_RI, GETPC());
+}
 
-    return 0;
+target_ulong helper_rdhwr_cpunum(CPUMIPSState *env)
+{
+    check_hwrena(env, 0);
+    return env->CP0_EBase & 0x3ff;
 }
 
 target_ulong helper_rdhwr_synci_step(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 1)))
-        return env->SYNCI_Step;
-    else
-        helper_raise_exception(env, EXCP_RI);
-
-    return 0;
+    check_hwrena(env, 1);
+    return env->SYNCI_Step;
 }
 
 target_ulong helper_rdhwr_cc(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 2)))
-        return env->CP0_Count;
-    else
-        helper_raise_exception(env, EXCP_RI);
-
-    return 0;
+    check_hwrena(env, 2);
+#ifdef CONFIG_USER_ONLY
+    return env->CP0_Count;
+#else
+    return (int32_t)cpu_mips_get_count(env);
+#endif
 }
 
 target_ulong helper_rdhwr_ccres(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 3)))
-        return env->CCRes;
-    else
-        helper_raise_exception(env, EXCP_RI);
+    check_hwrena(env, 3);
+    return env->CCRes;
+}
 
-    return 0;
+target_ulong helper_rdhwr_performance(CPUMIPSState *env)
+{
+    check_hwrena(env, 4);
+    return env->CP0_Performance0;
+}
+
+target_ulong helper_rdhwr_xnp(CPUMIPSState *env)
+{
+    check_hwrena(env, 5);
+    return (env->CP0_Config5 >> CP0C5_XNP) & 1;
 }
 
 void helper_pmon(CPUMIPSState *env, int function)
@@ -2234,7 +2259,9 @@ void helper_wait(CPUMIPSState *env)
 
     cs->halted = 1;
     cpu_reset_interrupt(cs, CPU_INTERRUPT_WAKE);
-    helper_raise_exception(env, EXCP_HLT);
+    /* Last instruction in the block, PC was updated before
+       - no need to recover PC and icount */
+    raise_exception(env, EXCP_HLT);
 }
 
 #if !defined(CONFIG_USER_ONLY)
@@ -2295,9 +2322,9 @@ void mips_cpu_unassigned_access(CPUState *cs, hwaddr addr,
     }
 
     if (is_exec) {
-        helper_raise_exception(env, EXCP_IBE);
+        raise_exception(env, EXCP_IBE);
     } else {
-        helper_raise_exception(env, EXCP_DBE);
+        raise_exception(env, EXCP_DBE);
     }
 }
 #endif /* !CONFIG_USER_ONLY */
@@ -2332,7 +2359,7 @@ target_ulong helper_cfc1(CPUMIPSState *env, uint32_t reg)
                 arg1 = (int32_t)
                        ((env->CP0_Status & (1  << CP0St_FR)) >> CP0St_FR);
             } else {
-                helper_raise_exception(env, EXCP_RI);
+                do_raise_exception(env, EXCP_RI, GETPC());
             }
         }
         break;
@@ -2375,7 +2402,7 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t fs, uint32_t rt)
             env->CP0_Status &= ~(1 << CP0St_FR);
             compute_hflags(env);
         } else {
-            helper_raise_exception(env, EXCP_RI);
+            do_raise_exception(env, EXCP_RI, GETPC());
         }
         break;
     case 4:
@@ -2387,7 +2414,7 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t fs, uint32_t rt)
             env->CP0_Status |= (1 << CP0St_FR);
             compute_hflags(env);
         } else {
-            helper_raise_exception(env, EXCP_RI);
+            do_raise_exception(env, EXCP_RI, GETPC());
         }
         break;
     case 5:
@@ -3628,7 +3655,7 @@ FOP_CONDN_S(sne,  (float32_lt(fst1, fst0, &env->active_fpu.fp_status)
 #if !defined(CONFIG_USER_ONLY)
 #define MEMOP_IDX(DF)                                           \
         TCGMemOpIdx oi = make_memop_idx(MO_TE | DF | MO_UNALN,  \
-                                        cpu_mmu_index(env));
+                                        cpu_mmu_index(env, false));
 #else
 #define MEMOP_IDX(DF)
 #endif
@@ -3684,7 +3711,7 @@ void helper_msa_st_ ## TYPE(CPUMIPSState *env, uint32_t wd,             \
                             target_ulong addr)                          \
 {                                                                       \
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
-    int mmu_idx = cpu_mmu_index(env);                                   \
+    int mmu_idx = cpu_mmu_index(env, false);				\
     int i;                                                              \
     MEMOP_IDX(DF)                                                       \
     ensure_writable_pages(env, addr, mmu_idx, GETRA());                 \
diff --git a/target-mips/translate.c b/target-mips/translate.c
index 22ef84df9..56266471c 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -33,9 +33,7 @@
 
 #include "trace-tcg.h"
 
-
 #define MIPS_DEBUG_DISAS 0
-//#define MIPS_DEBUG_SIGN_EXTENSIONS
 
 /* MIPS major opcodes */
 #define MASK_OP_MAJOR(op)  (op & (0x3F << 26))
@@ -325,6 +323,7 @@ enum {
     OPC_TLTIU    = (0x0B << 16) | OPC_REGIMM,
     OPC_TEQI     = (0x0C << 16) | OPC_REGIMM,
     OPC_TNEI     = (0x0E << 16) | OPC_REGIMM,
+    OPC_SIGRIE   = (0x17 << 16) | OPC_REGIMM,
     OPC_SYNCI    = (0x1F << 16) | OPC_REGIMM,
 
     OPC_DAHI     = (0x06 << 16) | OPC_REGIMM,
@@ -1361,9 +1360,6 @@ static TCGv_i32 fpu_fcr0, fpu_fcr31;
 static TCGv_i64 fpu_f64[32];
 static TCGv_i64 msa_wr_d[64];
 
-static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-static target_ulong gen_opc_btarget[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 #define gen_helper_0e0i(name, arg) do {                           \
@@ -1482,26 +1478,23 @@ static const char * const msaregnames[] = {
     "w30.d0", "w30.d1", "w31.d0", "w31.d1",
 };
 
-#define MIPS_DEBUG(fmt, ...)                                                  \
+#define LOG_DISAS(...)                                                        \
     do {                                                                      \
         if (MIPS_DEBUG_DISAS) {                                               \
-            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
-                          TARGET_FMT_lx ": %08x " fmt "\n",                   \
-                          ctx->pc, ctx->opcode , ## __VA_ARGS__);             \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__);                 \
         }                                                                     \
     } while (0)
 
-#define LOG_DISAS(...)                                                        \
+#define MIPS_INVAL(op)                                                        \
     do {                                                                      \
         if (MIPS_DEBUG_DISAS) {                                               \
-            qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__);                 \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
+                          TARGET_FMT_lx ": %08x Invalid %s %03x %03x %03x\n", \
+                          ctx->pc, ctx->opcode, op, ctx->opcode >> 26,        \
+                          ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F));  \
         }                                                                     \
     } while (0)
 
-#define MIPS_INVAL(op)                                                        \
-    MIPS_DEBUG("Invalid %s %03x %03x %03x", op, ctx->opcode >> 26,            \
-               ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F))
-
 /* General purpose registers moves. */
 static inline void gen_load_gpr (TCGv t, int reg)
 {
@@ -1615,21 +1608,26 @@ static inline void generate_exception_err(DisasContext *ctx, int excp, int err)
     gen_helper_raise_exception_err(cpu_env, texcp, terr);
     tcg_temp_free_i32(terr);
     tcg_temp_free_i32(texcp);
+    ctx->bstate = BS_EXCP;
 }
 
 static inline void generate_exception(DisasContext *ctx, int excp)
 {
-    save_cpu_state(ctx, 1);
     gen_helper_0e0i(raise_exception, excp);
 }
 
+static inline void generate_exception_end(DisasContext *ctx, int excp)
+{
+    generate_exception_err(ctx, excp, 0);
+}
+
 /* Floating point register moves. */
 static void gen_load_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_FRE) {
         generate_exception(ctx, EXCP_RI);
     }
-    tcg_gen_trunc_i64_i32(t, fpu_f64[reg]);
+    tcg_gen_extrl_i64_i32(t, fpu_f64[reg]);
 }
 
 static void gen_store_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
@@ -1647,10 +1645,7 @@ static void gen_store_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
 static void gen_load_fpr32h(DisasContext *ctx, TCGv_i32 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_F64) {
-        TCGv_i64 t64 = tcg_temp_new_i64();
-        tcg_gen_shri_i64(t64, fpu_f64[reg], 32);
-        tcg_gen_trunc_i64_i32(t, t64);
-        tcg_temp_free_i64(t64);
+        tcg_gen_extrh_i64_i32(t, fpu_f64[reg]);
     } else {
         gen_load_fpr32(ctx, t, reg | 1);
     }
@@ -1725,12 +1720,23 @@ static target_long addr_add(DisasContext *ctx, target_long base,
     return sum;
 }
 
+/* Sign-extract the low 32-bits to a target_long.  */
 static inline void gen_move_low32(TCGv ret, TCGv_i64 arg)
 {
 #if defined(TARGET_MIPS64)
-    tcg_gen_ext32s_tl(ret, arg);
+    tcg_gen_ext32s_i64(ret, arg);
 #else
-    tcg_gen_trunc_i64_tl(ret, arg);
+    tcg_gen_extrl_i64_i32(ret, arg);
+#endif
+}
+
+/* Sign-extract the high 32-bits to a target_long.  */
+static inline void gen_move_high32(TCGv ret, TCGv_i64 arg)
+{
+#if defined(TARGET_MIPS64)
+    tcg_gen_sari_i64(ret, arg, 32);
+#else
+    tcg_gen_extrh_i64_i32(ret, arg);
 #endif
 }
 
@@ -1753,7 +1759,7 @@ static inline void check_cp1_enabled(DisasContext *ctx)
 static inline void check_cop1x(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_COP1X)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /* Verify that the processor is running with 64-bit floating-point
@@ -1762,7 +1768,7 @@ static inline void check_cop1x(DisasContext *ctx)
 static inline void check_cp1_64bitmode(DisasContext *ctx)
 {
     if (unlikely(~ctx->hflags & (MIPS_HFLAG_F64 | MIPS_HFLAG_COP1X)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /*
@@ -1779,7 +1785,7 @@ static inline void check_cp1_64bitmode(DisasContext *ctx)
 static inline void check_cp1_registers(DisasContext *ctx, int regs)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_F64) && (regs & 1)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /* Verify that the processor is running with DSP instructions enabled.
@@ -1790,9 +1796,9 @@ static inline void check_dsp(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSP))) {
         if (ctx->insn_flags & ASE_DSP) {
-            generate_exception(ctx, EXCP_DSPDIS);
+            generate_exception_end(ctx, EXCP_DSPDIS);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
     }
 }
@@ -1801,9 +1807,9 @@ static inline void check_dspr2(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSPR2))) {
         if (ctx->insn_flags & ASE_DSP) {
-            generate_exception(ctx, EXCP_DSPDIS);
+            generate_exception_end(ctx, EXCP_DSPDIS);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
     }
 }
@@ -1813,7 +1819,7 @@ static inline void check_dspr2(DisasContext *ctx)
 static inline void check_insn(DisasContext *ctx, int flags)
 {
     if (unlikely(!(ctx->insn_flags & flags))) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
 }
 
@@ -1823,7 +1829,7 @@ static inline void check_insn(DisasContext *ctx, int flags)
 static inline void check_insn_opc_removed(DisasContext *ctx, int flags)
 {
     if (unlikely(ctx->insn_flags & flags)) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
 }
 
@@ -1843,7 +1849,7 @@ static inline void check_ps(DisasContext *ctx)
 static inline void check_mips_64(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_64)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 #endif
 
@@ -2052,7 +2058,7 @@ static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx)
     tcg_gen_movi_tl(t0, rt | ((almask << 3) & 0x20));                        \
     tcg_gen_st_tl(t0, cpu_env, offsetof(CPUMIPSState, llreg));                   \
     tcg_gen_st_tl(arg1, cpu_env, offsetof(CPUMIPSState, llnewval));              \
-    gen_helper_0e0i(raise_exception, EXCP_SC);                               \
+    generate_exception_end(ctx, EXCP_SC);                                    \
     gen_set_label(l2);                                                       \
     tcg_gen_movi_tl(t0, 0);                                                  \
     gen_store_gpr(t0, rt);                                                   \
@@ -2105,14 +2111,12 @@ static target_ulong pc_relative_pc (DisasContext *ctx)
 static void gen_ld(DisasContext *ctx, uint32_t opc,
                    int rt, int base, int16_t offset)
 {
-    const char *opn = "ld";
     TCGv t0, t1, t2;
 
     if (rt == 0 && ctx->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
         /* Loongson CPU uses a load to zero register for prefetch.
            We emulate it as a NOP. On other CPU we must perform the
            actual memory access. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2125,20 +2129,16 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUL |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lwu";
         break;
     case OPC_LD:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "ld";
         break;
     case OPC_LLD:
     case R6_OPC_LLD:
-        save_cpu_state(ctx, 1);
         op_ld_lld(t0, t0, ctx);
         gen_store_gpr(t0, rt);
-        opn = "lld";
         break;
     case OPC_LDL:
         t1 = tcg_temp_new();
@@ -2153,16 +2153,14 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_andi_tl(t0, t0, ~7);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         tcg_gen_shl_tl(t0, t0, t1);
-        tcg_gen_xori_tl(t1, t1, 63);
-        t2 = tcg_const_tl(0x7fffffffffffffffull);
-        tcg_gen_shr_tl(t2, t2, t1);
+        t2 = tcg_const_tl(-1);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        tcg_gen_and_tl(t1, t1, t2);
+        tcg_gen_andc_tl(t1, t1, t2);
         tcg_temp_free(t2);
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         gen_store_gpr(t0, rt);
-        opn = "ldl";
         break;
     case OPC_LDR:
         t1 = tcg_temp_new();
@@ -2186,7 +2184,6 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         gen_store_gpr(t0, rt);
-        opn = "ldr";
         break;
     case OPC_LDPC:
         t1 = tcg_const_tl(pc_relative_pc(ctx));
@@ -2194,7 +2191,6 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_temp_free(t1);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t0, rt);
-        opn = "ldpc";
         break;
 #endif
     case OPC_LWPC:
@@ -2203,35 +2199,29 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_temp_free(t1);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t0, rt);
-        opn = "lwpc";
         break;
     case OPC_LW:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lw";
         break;
     case OPC_LH:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESW |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lh";
         break;
     case OPC_LHU:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUW |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lhu";
         break;
     case OPC_LB:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_SB);
         gen_store_gpr(t0, rt);
-        opn = "lb";
         break;
     case OPC_LBU:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_UB);
         gen_store_gpr(t0, rt);
-        opn = "lbu";
         break;
     case OPC_LWL:
         t1 = tcg_temp_new();
@@ -2246,17 +2236,15 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_andi_tl(t0, t0, ~3);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUL);
         tcg_gen_shl_tl(t0, t0, t1);
-        tcg_gen_xori_tl(t1, t1, 31);
-        t2 = tcg_const_tl(0x7fffffffull);
-        tcg_gen_shr_tl(t2, t2, t1);
+        t2 = tcg_const_tl(-1);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        tcg_gen_and_tl(t1, t1, t2);
+        tcg_gen_andc_tl(t1, t1, t2);
         tcg_temp_free(t2);
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         tcg_gen_ext32s_tl(t0, t0);
         gen_store_gpr(t0, rt);
-        opn = "lwl";
         break;
     case OPC_LWR:
         t1 = tcg_temp_new();
@@ -2281,18 +2269,13 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_temp_free(t1);
         tcg_gen_ext32s_tl(t0, t0);
         gen_store_gpr(t0, rt);
-        opn = "lwr";
         break;
     case OPC_LL:
     case R6_OPC_LL:
-        save_cpu_state(ctx, 1);
         op_ld_ll(t0, t0, ctx);
         gen_store_gpr(t0, rt);
-        opn = "ll";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
 }
 
@@ -2300,7 +2283,6 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
 static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
                     int base, int16_t offset)
 {
-    const char *opn = "st";
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
 
@@ -2311,46 +2293,32 @@ static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
     case OPC_SD:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEQ |
                            ctx->default_tcg_memop_mask);
-        opn = "sd";
         break;
     case OPC_SDL:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(sdl, t1, t0, ctx->mem_idx);
-        opn = "sdl";
         break;
     case OPC_SDR:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(sdr, t1, t0, ctx->mem_idx);
-        opn = "sdr";
         break;
 #endif
     case OPC_SW:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUL |
                            ctx->default_tcg_memop_mask);
-        opn = "sw";
         break;
     case OPC_SH:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUW |
                            ctx->default_tcg_memop_mask);
-        opn = "sh";
         break;
     case OPC_SB:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_8);
-        opn = "sb";
         break;
     case OPC_SWL:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(swl, t1, t0, ctx->mem_idx);
-        opn = "swl";
         break;
     case OPC_SWR:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(swr, t1, t0, ctx->mem_idx);
-        opn = "swr";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -2360,7 +2328,6 @@ static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
 static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
                          int base, int16_t offset)
 {
-    const char *opn = "st_cond";
     TCGv t0, t1;
 
 #ifdef CONFIG_USER_ONLY
@@ -2376,20 +2343,14 @@ static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
 #if defined(TARGET_MIPS64)
     case OPC_SCD:
     case R6_OPC_SCD:
-        save_cpu_state(ctx, 1);
         op_st_scd(t1, t0, rt, ctx);
-        opn = "scd";
         break;
 #endif
     case OPC_SC:
     case R6_OPC_SC:
-        save_cpu_state(ctx, 1);
         op_st_sc(t1, t0, rt, ctx);
-        opn = "sc";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t1);
     tcg_temp_free(t0);
 }
@@ -2398,7 +2359,6 @@ static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
 static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
                           int base, int16_t offset)
 {
-    const char *opn = "flt_ldst";
     TCGv t0 = tcg_temp_new();
 
     gen_base_offset_addr(ctx, t0, base, offset);
@@ -2413,7 +2373,6 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
             gen_store_fpr32(ctx, fp0, ft);
             tcg_temp_free_i32(fp0);
         }
-        opn = "lwc1";
         break;
     case OPC_SWC1:
         {
@@ -2423,7 +2382,6 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
                                 ctx->default_tcg_memop_mask);
             tcg_temp_free_i32(fp0);
         }
-        opn = "swc1";
         break;
     case OPC_LDC1:
         {
@@ -2433,7 +2391,6 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
             gen_store_fpr64(ctx, fp0, ft);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ldc1";
         break;
     case OPC_SDC1:
         {
@@ -2443,15 +2400,12 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
                                 ctx->default_tcg_memop_mask);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sdc1";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("flt_ldst");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, fregnames[ft], offset, regnames[base]);
  out:
     tcg_temp_free(t0);
 }
@@ -2479,12 +2433,10 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
-    const char *opn = "imm arith";
 
     if (rt == 0 && opc != OPC_ADDI && opc != OPC_DADDI) {
         /* If no destination, treat it as a NOP.
            For addi, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
     switch (opc) {
@@ -2512,7 +2464,6 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rt);
             tcg_temp_free(t0);
         }
-        opn = "addi";
         break;
     case OPC_ADDIU:
         if (rs != 0) {
@@ -2521,7 +2472,6 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
         }
-        opn = "addiu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DADDI:
@@ -2546,7 +2496,6 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rt);
             tcg_temp_free(t0);
         }
-        opn = "daddi";
         break;
     case OPC_DADDIU:
         if (rs != 0) {
@@ -2554,12 +2503,9 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
         }
-        opn = "daddiu";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
 }
 
 /* Logic with immediate operand */
@@ -2570,7 +2516,6 @@ static void gen_logic_imm(DisasContext *ctx, uint32_t opc,
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     uimm = (uint16_t)imm;
@@ -2580,39 +2525,30 @@ static void gen_logic_imm(DisasContext *ctx, uint32_t opc,
             tcg_gen_andi_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], 0);
-        MIPS_DEBUG("andi %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_ORI:
         if (rs != 0)
             tcg_gen_ori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        MIPS_DEBUG("ori %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_XORI:
         if (likely(rs != 0))
             tcg_gen_xori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        MIPS_DEBUG("xori %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_LUI:
         if (rs != 0 && (ctx->insn_flags & ISA_MIPS32R6)) {
             /* OPC_AUI */
             tcg_gen_addi_tl(cpu_gpr[rt], cpu_gpr[rs], imm << 16);
             tcg_gen_ext32s_tl(cpu_gpr[rt], cpu_gpr[rt]);
-            MIPS_DEBUG("aui %s, %s, %04x", regnames[rt], regnames[rs], imm);
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], imm << 16);
-            MIPS_DEBUG("lui %s, " TARGET_FMT_lx, regnames[rt], uimm);
         }
         break;
 
     default:
-        MIPS_DEBUG("Unknown logical immediate opcode %08x", opc);
         break;
     }
 }
@@ -2622,12 +2558,10 @@ static void gen_slt_imm(DisasContext *ctx, uint32_t opc,
                         int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
-    const char *opn = "imm arith";
     TCGv t0;
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     t0 = tcg_temp_new();
@@ -2635,15 +2569,11 @@ static void gen_slt_imm(DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_SLTI:
         tcg_gen_setcondi_tl(TCG_COND_LT, cpu_gpr[rt], t0, uimm);
-        opn = "slti";
         break;
     case OPC_SLTIU:
         tcg_gen_setcondi_tl(TCG_COND_LTU, cpu_gpr[rt], t0, uimm);
-        opn = "sltiu";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
     tcg_temp_free(t0);
 }
 
@@ -2652,12 +2582,10 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm = ((uint16_t)imm) & 0x1f;
-    const char *opn = "imm shift";
     TCGv t0;
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2667,11 +2595,9 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
     case OPC_SLL:
         tcg_gen_shli_tl(t0, t0, uimm);
         tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
-        opn = "sll";
         break;
     case OPC_SRA:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm);
-        opn = "sra";
         break;
     case OPC_SRL:
         if (uimm != 0) {
@@ -2680,7 +2606,6 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
         }
-        opn = "srl";
         break;
     case OPC_ROTR:
         if (uimm != 0) {
@@ -2693,20 +2618,16 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
         }
-        opn = "rotr";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DSLL:
         tcg_gen_shli_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsll";
         break;
     case OPC_DSRA:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsra";
         break;
     case OPC_DSRL:
         tcg_gen_shri_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsrl";
         break;
     case OPC_DROTR:
         if (uimm != 0) {
@@ -2714,28 +2635,21 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_mov_tl(cpu_gpr[rt], t0);
         }
-        opn = "drotr";
         break;
     case OPC_DSLL32:
         tcg_gen_shli_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsll32";
         break;
     case OPC_DSRA32:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsra32";
         break;
     case OPC_DSRL32:
         tcg_gen_shri_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsrl32";
         break;
     case OPC_DROTR32:
         tcg_gen_rotri_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "drotr32";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
     tcg_temp_free(t0);
 }
 
@@ -2743,13 +2657,10 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
 static void gen_arith(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "arith";
-
     if (rd == 0 && opc != OPC_ADD && opc != OPC_SUB
        && opc != OPC_DADD && opc != OPC_DSUB) {
         /* If no destination, treat it as a NOP.
            For add & sub, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2777,7 +2688,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "add";
         break;
     case OPC_ADDU:
         if (rs != 0 && rt != 0) {
@@ -2790,7 +2700,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "addu";
         break;
     case OPC_SUB:
         {
@@ -2815,7 +2724,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "sub";
         break;
     case OPC_SUBU:
         if (rs != 0 && rt != 0) {
@@ -2829,7 +2737,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "subu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DADD:
@@ -2854,7 +2761,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "dadd";
         break;
     case OPC_DADDU:
         if (rs != 0 && rt != 0) {
@@ -2866,7 +2772,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "daddu";
         break;
     case OPC_DSUB:
         {
@@ -2890,7 +2795,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "dsub";
         break;
     case OPC_DSUBU:
         if (rs != 0 && rt != 0) {
@@ -2902,7 +2806,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "dsubu";
         break;
 #endif
     case OPC_MUL:
@@ -2912,23 +2815,18 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "mul";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Conditional move */
 static void gen_cond_move(DisasContext *ctx, uint32_t opc,
                           int rd, int rs, int rt)
 {
-    const char *opn = "cond move";
     TCGv t0, t1, t2;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2940,38 +2838,28 @@ static void gen_cond_move(DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_MOVN:
         tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
-        opn = "movn";
         break;
     case OPC_MOVZ:
         tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
-        opn = "movz";
         break;
     case OPC_SELNEZ:
         tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, t1);
-        opn = "selnez";
         break;
     case OPC_SELEQZ:
         tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, t1);
-        opn = "seleqz";
         break;
     }
     tcg_temp_free(t2);
     tcg_temp_free(t1);
     tcg_temp_free(t0);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Logic */
 static void gen_logic(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "logic";
-
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2982,7 +2870,6 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "and";
         break;
     case OPC_NOR:
         if (rs != 0 && rt != 0) {
@@ -2994,7 +2881,6 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], ~((target_ulong)0));
         }
-        opn = "nor";
         break;
     case OPC_OR:
         if (likely(rs != 0 && rt != 0)) {
@@ -3006,7 +2892,6 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "or";
         break;
     case OPC_XOR:
         if (likely(rs != 0 && rt != 0)) {
@@ -3018,23 +2903,18 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "xor";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Set on lower than */
 static void gen_slt(DisasContext *ctx, uint32_t opc,
                     int rd, int rs, int rt)
 {
-    const char *opn = "slt";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3045,15 +2925,11 @@ static void gen_slt(DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_SLT:
         tcg_gen_setcond_tl(TCG_COND_LT, cpu_gpr[rd], t0, t1);
-        opn = "slt";
         break;
     case OPC_SLTU:
         tcg_gen_setcond_tl(TCG_COND_LTU, cpu_gpr[rd], t0, t1);
-        opn = "sltu";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -3062,13 +2938,11 @@ static void gen_slt(DisasContext *ctx, uint32_t opc,
 static void gen_shift(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "shifts";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP.
            For add & sub, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3081,19 +2955,16 @@ static void gen_shift(DisasContext *ctx, uint32_t opc,
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_shl_tl(t0, t1, t0);
         tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
-        opn = "sllv";
         break;
     case OPC_SRAV:
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_sar_tl(cpu_gpr[rd], t1, t0);
-        opn = "srav";
         break;
     case OPC_SRLV:
         tcg_gen_ext32u_tl(t1, t1);
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_shr_tl(t0, t1, t0);
         tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
-        opn = "srlv";
         break;
     case OPC_ROTRV:
         {
@@ -3107,34 +2978,27 @@ static void gen_shift(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext_i32_tl(cpu_gpr[rd], t2);
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
-            opn = "rotrv";
         }
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DSLLV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_shl_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsllv";
         break;
     case OPC_DSRAV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_sar_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsrav";
         break;
     case OPC_DSRLV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_shr_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsrlv";
         break;
     case OPC_DROTRV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_rotr_tl(cpu_gpr[rd], t1, t0);
-        opn = "drotrv";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -3142,11 +3006,8 @@ static void gen_shift(DisasContext *ctx, uint32_t opc,
 /* Arithmetic on HI/LO registers */
 static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
 {
-    const char *opn = "hilo";
-
     if (reg == 0 && (opc == OPC_MFHI || opc == OPC_MFLO)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3164,7 +3025,6 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         {
             tcg_gen_mov_tl(cpu_gpr[reg], cpu_HI[acc]);
         }
-        opn = "mfhi";
         break;
     case OPC_MFLO:
 #if defined(TARGET_MIPS64)
@@ -3175,7 +3035,6 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         {
             tcg_gen_mov_tl(cpu_gpr[reg], cpu_LO[acc]);
         }
-        opn = "mflo";
         break;
     case OPC_MTHI:
         if (reg != 0) {
@@ -3190,7 +3049,6 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         } else {
             tcg_gen_movi_tl(cpu_HI[acc], 0);
         }
-        opn = "mthi";
         break;
     case OPC_MTLO:
         if (reg != 0) {
@@ -3205,11 +3063,8 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         } else {
             tcg_gen_movi_tl(cpu_LO[acc], 0);
         }
-        opn = "mtlo";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s", opn, regnames[reg]);
 }
 
 static inline void gen_r6_ld(target_long addr, int reg, int memidx,
@@ -3277,7 +3132,7 @@ static inline void gen_pcrel(DisasContext *ctx, int opc, target_ulong pc,
 #endif
         default:
             MIPS_INVAL("OPC_PCREL");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -3286,12 +3141,10 @@ static inline void gen_pcrel(DisasContext *ctx, int opc, target_ulong pc,
 
 static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
 {
-    const char *opn = "r6 mul/div";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3320,7 +3173,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "div";
         break;
     case R6_OPC_MOD:
         {
@@ -3340,7 +3192,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "mod";
         break;
     case R6_OPC_DIVU:
         {
@@ -3354,7 +3205,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "divu";
         break;
     case R6_OPC_MODU:
         {
@@ -3368,7 +3218,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "modu";
         break;
     case R6_OPC_MUL:
         {
@@ -3381,7 +3230,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mul";
         break;
     case R6_OPC_MUH:
         {
@@ -3394,7 +3242,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "muh";
         break;
     case R6_OPC_MULU:
         {
@@ -3407,7 +3254,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mulu";
         break;
     case R6_OPC_MUHU:
         {
@@ -3420,7 +3266,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "muhu";
         break;
 #if defined(TARGET_MIPS64)
     case R6_OPC_DDIV:
@@ -3438,7 +3283,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddiv";
         break;
     case R6_OPC_DMOD:
         {
@@ -3455,7 +3299,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "dmod";
         break;
     case R6_OPC_DDIVU:
         {
@@ -3466,7 +3309,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddivu";
         break;
     case R6_OPC_DMODU:
         {
@@ -3477,11 +3319,9 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "dmodu";
         break;
     case R6_OPC_DMUL:
         tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
-        opn = "dmul";
         break;
     case R6_OPC_DMUH:
         {
@@ -3489,11 +3329,9 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_gen_muls2_i64(t2, cpu_gpr[rd], t0, t1);
             tcg_temp_free(t2);
         }
-        opn = "dmuh";
         break;
     case R6_OPC_DMULU:
         tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
-        opn = "dmulu";
         break;
     case R6_OPC_DMUHU:
         {
@@ -3501,16 +3339,13 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_gen_mulu2_i64(t2, cpu_gpr[rd], t0, t1);
             tcg_temp_free(t2);
         }
-        opn = "dmuhu";
         break;
 #endif
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("r6 mul/div");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rs], regnames[rt]);
  out:
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -3519,7 +3354,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
 static void gen_muldiv(DisasContext *ctx, uint32_t opc,
                        int acc, int rs, int rt)
 {
-    const char *opn = "mul/div";
     TCGv t0, t1;
 
     t0 = tcg_temp_new();
@@ -3553,7 +3387,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "div";
         break;
     case OPC_DIVU:
         {
@@ -3569,7 +3402,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "divu";
         break;
     case OPC_MULT:
         {
@@ -3583,7 +3415,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mult";
         break;
     case OPC_MULTU:
         {
@@ -3597,7 +3428,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "multu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DDIV:
@@ -3616,7 +3446,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddiv";
         break;
     case OPC_DDIVU:
         {
@@ -3628,15 +3457,12 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddivu";
         break;
     case OPC_DMULT:
         tcg_gen_muls2_i64(cpu_LO[acc], cpu_HI[acc], t0, t1);
-        opn = "dmult";
         break;
     case OPC_DMULTU:
         tcg_gen_mulu2_i64(cpu_LO[acc], cpu_HI[acc], t0, t1);
-        opn = "dmultu";
         break;
 #endif
     case OPC_MADD:
@@ -3650,14 +3476,10 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "madd";
         break;
     case OPC_MADDU:
         {
@@ -3672,14 +3494,10 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "maddu";
         break;
     case OPC_MSUB:
         {
@@ -3692,14 +3510,10 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "msub";
         break;
     case OPC_MSUBU:
         {
@@ -3714,22 +3528,16 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "msubu";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("mul/div");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rs], regnames[rt]);
  out:
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -3738,7 +3546,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
 static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
                             int rd, int rs, int rt)
 {
-    const char *opn = "mul vr54xx";
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
 
@@ -3748,68 +3555,52 @@ static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_VR54XX_MULS:
         gen_helper_muls(t0, cpu_env, t0, t1);
-        opn = "muls";
         break;
     case OPC_VR54XX_MULSU:
         gen_helper_mulsu(t0, cpu_env, t0, t1);
-        opn = "mulsu";
         break;
     case OPC_VR54XX_MACC:
         gen_helper_macc(t0, cpu_env, t0, t1);
-        opn = "macc";
         break;
     case OPC_VR54XX_MACCU:
         gen_helper_maccu(t0, cpu_env, t0, t1);
-        opn = "maccu";
         break;
     case OPC_VR54XX_MSAC:
         gen_helper_msac(t0, cpu_env, t0, t1);
-        opn = "msac";
         break;
     case OPC_VR54XX_MSACU:
         gen_helper_msacu(t0, cpu_env, t0, t1);
-        opn = "msacu";
         break;
     case OPC_VR54XX_MULHI:
         gen_helper_mulhi(t0, cpu_env, t0, t1);
-        opn = "mulhi";
         break;
     case OPC_VR54XX_MULHIU:
         gen_helper_mulhiu(t0, cpu_env, t0, t1);
-        opn = "mulhiu";
         break;
     case OPC_VR54XX_MULSHI:
         gen_helper_mulshi(t0, cpu_env, t0, t1);
-        opn = "mulshi";
         break;
     case OPC_VR54XX_MULSHIU:
         gen_helper_mulshiu(t0, cpu_env, t0, t1);
-        opn = "mulshiu";
         break;
     case OPC_VR54XX_MACCHI:
         gen_helper_macchi(t0, cpu_env, t0, t1);
-        opn = "macchi";
         break;
     case OPC_VR54XX_MACCHIU:
         gen_helper_macchiu(t0, cpu_env, t0, t1);
-        opn = "macchiu";
         break;
     case OPC_VR54XX_MSACHI:
         gen_helper_msachi(t0, cpu_env, t0, t1);
-        opn = "msachi";
         break;
     case OPC_VR54XX_MSACHIU:
         gen_helper_msachiu(t0, cpu_env, t0, t1);
-        opn = "msachiu";
         break;
     default:
         MIPS_INVAL("mul vr54xx");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
     gen_store_gpr(t0, rd);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 
  out:
     tcg_temp_free(t0);
@@ -3819,12 +3610,10 @@ static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
 static void gen_cl (DisasContext *ctx, uint32_t opc,
                     int rd, int rs)
 {
-    const char *opn = "CLx";
     TCGv t0;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     t0 = tcg_temp_new();
@@ -3833,28 +3622,22 @@ static void gen_cl (DisasContext *ctx, uint32_t opc,
     case OPC_CLO:
     case R6_OPC_CLO:
         gen_helper_clo(cpu_gpr[rd], t0);
-        opn = "clo";
         break;
     case OPC_CLZ:
     case R6_OPC_CLZ:
         gen_helper_clz(cpu_gpr[rd], t0);
-        opn = "clz";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DCLO:
     case R6_OPC_DCLO:
         gen_helper_dclo(cpu_gpr[rd], t0);
-        opn = "dclo";
         break;
     case OPC_DCLZ:
     case R6_OPC_DCLZ:
         gen_helper_dclz(cpu_gpr[rd], t0);
-        opn = "dclz";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s", opn, regnames[rd], regnames[rs]);
     tcg_temp_free(t0);
 }
 
@@ -3862,12 +3645,10 @@ static void gen_cl (DisasContext *ctx, uint32_t opc,
 static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
                                  int rd, int rs, int rt)
 {
-    const char *opn = "loongson";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3899,7 +3680,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
     case OPC_MULT_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
         tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
-        opn = "mult.g";
         break;
     case OPC_MULTU_G_2E:
     case OPC_MULTU_G_2F:
@@ -3907,7 +3687,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
         tcg_gen_ext32u_tl(t1, t1);
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
         tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
-        opn = "multu.g";
         break;
     case OPC_DIV_G_2E:
     case OPC_DIV_G_2F:
@@ -3930,7 +3709,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l3);
         }
-        opn = "div.g";
         break;
     case OPC_DIVU_G_2E:
     case OPC_DIVU_G_2F:
@@ -3947,7 +3725,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l2);
         }
-        opn = "divu.g";
         break;
     case OPC_MOD_G_2E:
     case OPC_MOD_G_2F:
@@ -3968,7 +3745,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l3);
         }
-        opn = "mod.g";
         break;
     case OPC_MODU_G_2E:
     case OPC_MODU_G_2F:
@@ -3985,18 +3761,15 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l2);
         }
-        opn = "modu.g";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DMULT_G_2E:
     case OPC_DMULT_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
-        opn = "dmult.g";
         break;
     case OPC_DMULTU_G_2E:
     case OPC_DMULTU_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
-        opn = "dmultu.g";
         break;
     case OPC_DDIV_G_2E:
     case OPC_DDIV_G_2F:
@@ -4016,7 +3789,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_div_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l3);
         }
-        opn = "ddiv.g";
         break;
     case OPC_DDIVU_G_2E:
     case OPC_DDIVU_G_2F:
@@ -4030,7 +3802,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_divu_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l2);
         }
-        opn = "ddivu.g";
         break;
     case OPC_DMOD_G_2E:
     case OPC_DMOD_G_2F:
@@ -4048,7 +3819,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_rem_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l3);
         }
-        opn = "dmod.g";
         break;
     case OPC_DMODU_G_2E:
     case OPC_DMODU_G_2F:
@@ -4062,13 +3832,10 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_remu_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l2);
         }
-        opn = "dmodu.g";
         break;
 #endif
     }
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s", opn, regnames[rd], regnames[rs]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -4076,7 +3843,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
 /* Loongson multimedia instructions */
 static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 {
-    const char *opn = "loongson_cp2";
     uint32_t opc, shift_max;
     TCGv_i64 t0, t1;
 
@@ -4099,11 +3865,11 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
     gen_load_fpr64(ctx, t1, rt);
 
 #define LMI_HELPER(UP, LO) \
-    case OPC_##UP: gen_helper_##LO(t0, t0, t1); opn = #LO; break
+    case OPC_##UP: gen_helper_##LO(t0, t0, t1); break
 #define LMI_HELPER_1(UP, LO) \
-    case OPC_##UP: gen_helper_##LO(t0, t0); opn = #LO; break
+    case OPC_##UP: gen_helper_##LO(t0, t0); break
 #define LMI_DIRECT(UP, LO, OP) \
-    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); opn = #LO; break
+    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); break
 
     switch (opc) {
     LMI_HELPER(PADDSH, paddsh);
@@ -4174,19 +3940,15 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 
     case OPC_PINSRH_0:
         tcg_gen_deposit_i64(t0, t0, t1, 0, 16);
-        opn = "pinsrh_0";
         break;
     case OPC_PINSRH_1:
         tcg_gen_deposit_i64(t0, t0, t1, 16, 16);
-        opn = "pinsrh_1";
         break;
     case OPC_PINSRH_2:
         tcg_gen_deposit_i64(t0, t0, t1, 32, 16);
-        opn = "pinsrh_2";
         break;
     case OPC_PINSRH_3:
         tcg_gen_deposit_i64(t0, t0, t1, 48, 16);
-        opn = "pinsrh_3";
         break;
 
     case OPC_PEXTRH:
@@ -4194,42 +3956,33 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
         tcg_gen_shli_i64(t1, t1, 4);
         tcg_gen_shr_i64(t0, t0, t1);
         tcg_gen_ext16u_i64(t0, t0);
-        opn = "pextrh";
         break;
 
     case OPC_ADDU_CP2:
         tcg_gen_add_i64(t0, t0, t1);
         tcg_gen_ext32s_i64(t0, t0);
-        opn = "addu";
         break;
     case OPC_SUBU_CP2:
         tcg_gen_sub_i64(t0, t0, t1);
         tcg_gen_ext32s_i64(t0, t0);
-        opn = "addu";
         break;
 
     case OPC_SLL_CP2:
-        opn = "sll";
         shift_max = 32;
         goto do_shift;
     case OPC_SRL_CP2:
-        opn = "srl";
         shift_max = 32;
         goto do_shift;
     case OPC_SRA_CP2:
-        opn = "sra";
         shift_max = 32;
         goto do_shift;
     case OPC_DSLL_CP2:
-        opn = "dsll";
         shift_max = 64;
         goto do_shift;
     case OPC_DSRL_CP2:
-        opn = "dsrl";
         shift_max = 64;
         goto do_shift;
     case OPC_DSRA_CP2:
-        opn = "dsra";
         shift_max = 64;
         goto do_shift;
     do_shift:
@@ -4284,8 +4037,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
             tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
             generate_exception(ctx, EXCP_OVERFLOW);
             gen_set_label(lab);
-
-            opn = (opc == OPC_ADD_CP2 ? "add" : "dadd");
             break;
         }
 
@@ -4307,8 +4058,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
             tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
             generate_exception(ctx, EXCP_OVERFLOW);
             gen_set_label(lab);
-
-            opn = (opc == OPC_SUB_CP2 ? "sub" : "dsub");
             break;
         }
 
@@ -4316,7 +4065,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
         tcg_gen_ext32u_i64(t0, t0);
         tcg_gen_ext32u_i64(t1, t1);
         tcg_gen_mul_i64(t0, t0, t1);
-        opn = "pmuluw";
         break;
 
     case OPC_SEQU_CP2:
@@ -4328,8 +4076,8 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
         /* ??? Document is unclear: Set FCC[CC].  Does that mean the
            FD field is the CC field?  */
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("loongson_cp2");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -4338,9 +4086,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 
     gen_store_fpr64(ctx, t0, rd);
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn,
-               fregnames[rd], fregnames[rs], fregnames[rt]);
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
 }
@@ -4392,7 +4137,7 @@ static void gen_trap (DisasContext *ctx, uint32_t opc,
         case OPC_TGEU:  /* rs >= rs unsigned */
         case OPC_TGEIU: /* r0 >= 0  unsigned */
             /* Always trap */
-            generate_exception(ctx, EXCP_TRAP);
+            generate_exception_end(ctx, EXCP_TRAP);
             break;
         case OPC_TLT:   /* rs < rs           */
         case OPC_TLTI:  /* r0 < 0            */
@@ -4452,7 +4197,7 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
         gen_save_pc(dest);
         if (ctx->singlestep_enabled) {
             save_cpu_state(ctx, 0);
-            gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+            gen_helper_raise_exception_debug(cpu_env);
         }
         tcg_gen_exit_tb(0);
     }
@@ -4475,7 +4220,7 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x"
                   TARGET_FMT_lx "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -4535,14 +4280,14 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             /* Hint = 0 is JR/JALR, hint 16 is JR.HB/JALR.HB, the
                others are reserved. */
             MIPS_INVAL("jump hint");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
         gen_load_gpr(btarget, rs);
         break;
     default:
         MIPS_INVAL("branch/jump");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
     if (bcond_compute == 0) {
@@ -4556,20 +4301,17 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         case OPC_BLEZL:   /* 0 <= 0 likely   */
             /* Always take */
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("balways");
             break;
         case OPC_BGEZAL:  /* 0 >= 0          */
         case OPC_BGEZALL: /* 0 >= 0 likely   */
             /* Always take and link */
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("balways and link");
             break;
         case OPC_BNE:     /* rx != rx        */
         case OPC_BGTZ:    /* 0 > 0           */
         case OPC_BLTZ:    /* 0 < 0           */
             /* Treat as NOP. */
-            MIPS_DEBUG("bnever (NOP)");
             goto out;
         case OPC_BLTZAL:  /* 0 < 0           */
             /* Handle as an unconditional branch to get correct delay
@@ -4577,24 +4319,20 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             blink = 31;
             btgt = ctx->pc + insn_bytes + delayslot_size;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("bnever and link");
             break;
         case OPC_BLTZALL: /* 0 < 0 likely */
             tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 8);
             /* Skip the instruction in the delay slot */
-            MIPS_DEBUG("bnever, link and skip");
             ctx->pc += 4;
             goto out;
         case OPC_BNEL:    /* rx != rx likely */
         case OPC_BGTZL:   /* 0 > 0 likely */
         case OPC_BLTZL:   /* 0 < 0 likely */
             /* Skip the instruction in the delay slot */
-            MIPS_DEBUG("bnever and skip");
             ctx->pc += 4;
             goto out;
         case OPC_J:
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("j " TARGET_FMT_lx, btgt);
             break;
         case OPC_JALX:
             ctx->hflags |= MIPS_HFLAG_BX;
@@ -4602,118 +4340,91 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         case OPC_JAL:
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("jal " TARGET_FMT_lx, btgt);
             break;
         case OPC_JR:
             ctx->hflags |= MIPS_HFLAG_BR;
-            MIPS_DEBUG("jr %s", regnames[rs]);
             break;
         case OPC_JALR:
             blink = rt;
             ctx->hflags |= MIPS_HFLAG_BR;
-            MIPS_DEBUG("jalr %s, %s", regnames[rt], regnames[rs]);
             break;
         default:
             MIPS_INVAL("branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
     } else {
         switch (opc) {
         case OPC_BEQ:
             tcg_gen_setcond_tl(TCG_COND_EQ, bcond, t0, t1);
-            MIPS_DEBUG("beq %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto not_likely;
         case OPC_BEQL:
             tcg_gen_setcond_tl(TCG_COND_EQ, bcond, t0, t1);
-            MIPS_DEBUG("beql %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto likely;
         case OPC_BNE:
             tcg_gen_setcond_tl(TCG_COND_NE, bcond, t0, t1);
-            MIPS_DEBUG("bne %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto not_likely;
         case OPC_BNEL:
             tcg_gen_setcond_tl(TCG_COND_NE, bcond, t0, t1);
-            MIPS_DEBUG("bnel %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto likely;
         case OPC_BGEZ:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgez %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BGEZL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgezl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BGEZAL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgezal %s, " TARGET_FMT_lx, regnames[rs], btgt);
             blink = 31;
             goto not_likely;
         case OPC_BGEZALL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bgezall %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BGTZ:
             tcg_gen_setcondi_tl(TCG_COND_GT, bcond, t0, 0);
-            MIPS_DEBUG("bgtz %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BGTZL:
             tcg_gen_setcondi_tl(TCG_COND_GT, bcond, t0, 0);
-            MIPS_DEBUG("bgtzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BLEZ:
             tcg_gen_setcondi_tl(TCG_COND_LE, bcond, t0, 0);
-            MIPS_DEBUG("blez %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BLEZL:
             tcg_gen_setcondi_tl(TCG_COND_LE, bcond, t0, 0);
-            MIPS_DEBUG("blezl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BLTZ:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
-            MIPS_DEBUG("bltz %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BLTZL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
-            MIPS_DEBUG("bltzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BPOSGE32:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 32);
-            MIPS_DEBUG("bposge32 " TARGET_FMT_lx, btgt);
             goto not_likely;
 #if defined(TARGET_MIPS64)
         case OPC_BPOSGE64:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 64);
-            MIPS_DEBUG("bposge64 " TARGET_FMT_lx, btgt);
             goto not_likely;
 #endif
         case OPC_BLTZAL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bltzal %s, " TARGET_FMT_lx, regnames[rs], btgt);
         not_likely:
             ctx->hflags |= MIPS_HFLAG_BC;
             break;
         case OPC_BLTZALL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bltzall %s, " TARGET_FMT_lx, regnames[rs], btgt);
         likely:
             ctx->hflags |= MIPS_HFLAG_BL;
             break;
         default:
             MIPS_INVAL("conditional branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
     }
-    MIPS_DEBUG("enter ds: link %d cond %02x target " TARGET_FMT_lx,
-               blink, ctx->hflags, btgt);
 
     ctx->btarget = btgt;
 
@@ -4804,7 +4515,7 @@ static void gen_bitops (DisasContext *ctx, uint32_t opc, int rt,
     default:
 fail:
         MIPS_INVAL("bitops");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         tcg_temp_free(t0);
         tcg_temp_free(t1);
         return;
@@ -4820,7 +4531,6 @@ static void gen_bshfl (DisasContext *ctx, uint32_t op2, int rt, int rd)
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -4877,7 +4587,7 @@ static void gen_bshfl (DisasContext *ctx, uint32_t op2, int rt, int rd)
 #endif
     default:
         MIPS_INVAL("bsfhl");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         tcg_temp_free(t0);
         return;
     }
@@ -5050,12 +4760,6 @@ static inline void gen_mtc0_store32 (TCGv arg, target_ulong off)
     tcg_temp_free_i32(t0);
 }
 
-static inline void gen_mtc0_store64 (TCGv arg, target_ulong off)
-{
-    tcg_gen_ext32s_tl(arg, arg);
-    tcg_gen_st_tl(arg, cpu_env, off);
-}
-
 static void gen_mfhc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
@@ -5914,12 +5618,14 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         case 5:
             CP0_CHECK(ctx->insn_flags & ASE_MT);
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPESchedule));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
             CP0_CHECK(ctx->insn_flags & ASE_MT);
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPEScheFBack));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
@@ -6169,7 +5875,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
     case 14:
         switch (sel) {
         case 0:
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_EPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EPC));
             rn = "EPC";
             break;
         default:
@@ -6342,7 +6048,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         switch (sel) {
         case 0:
             /* EJTAG support */
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_DEPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_DEPC));
             rn = "DEPC";
             break;
         default:
@@ -6445,7 +6151,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
     case 30:
         switch (sel) {
         case 0:
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_ErrorEPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_ErrorEPC));
             rn = "ErrorEPC";
             break;
         default:
@@ -7944,7 +7650,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
 die:
     tcg_temp_free(t0);
     LOG_DISAS("mftr (reg %d u %d sel %d h %d)\n", rt, u, sel, h);
-    generate_exception(ctx, EXCP_RI);
+    generate_exception_end(ctx, EXCP_RI);
 }
 
 static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
@@ -8125,7 +7831,6 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
-        save_cpu_state(ctx, 1);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(rd);
 
@@ -8149,7 +7854,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
 die:
     tcg_temp_free(t0);
     LOG_DISAS("mttr (reg %d u %d sel %d h %d)\n", rd, u, sel, h);
-    generate_exception(ctx, EXCP_RI);
+    generate_exception_end(ctx, EXCP_RI);
 }
 
 static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt, int rd)
@@ -8278,7 +7983,6 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
     case OPC_ERET: /* OPC_ERETNC */
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         } else {
             int bit_shift = (ctx->hflags & MIPS_HFLAG_M16) ? 16 : 6;
@@ -8301,12 +8005,11 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         check_insn(ctx, ISA_MIPS32);
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         }
         if (!(ctx->hflags & MIPS_HFLAG_DM)) {
             MIPS_INVAL(opn);
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             gen_helper_deret(cpu_env);
             ctx->bstate = BS_EXCP;
@@ -8317,7 +8020,6 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         }
         /* If we get an exception, we want to restart at next instruction */
@@ -8330,11 +8032,10 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
     default:
  die:
         MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
     (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %d", opn, regnames[rt], rd);
 }
 #endif /* !CONFIG_USER_ONLY */
 
@@ -8343,12 +8044,10 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
                                 int32_t cc, int32_t offset)
 {
     target_ulong btarget;
-    const char *opn = "cp1 cond branch";
     TCGv_i32 t0 = tcg_temp_new_i32();
 
     if ((ctx->insn_flags & ISA_MIPS32R6) && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-        MIPS_DEBUG("CTI in delay / forbidden slot");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -8363,26 +8062,22 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
         tcg_gen_not_i32(t0, t0);
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1f";
         goto not_likely;
     case OPC_BC1FL:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_not_i32(t0, t0);
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1fl";
         goto likely;
     case OPC_BC1T:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1t";
         goto not_likely;
     case OPC_BC1TL:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1tl";
     likely:
         ctx->hflags |= MIPS_HFLAG_BL;
         break;
@@ -8396,7 +8091,6 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any2f";
         goto not_likely;
     case OPC_BC1TANY2:
         {
@@ -8408,7 +8102,6 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any2t";
         goto not_likely;
     case OPC_BC1FANY4:
         {
@@ -8424,7 +8117,6 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any4f";
         goto not_likely;
     case OPC_BC1TANY4:
         {
@@ -8440,18 +8132,14 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any4t";
     not_likely:
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("cp1 cond branch");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
-               ctx->hflags, btarget);
     ctx->btarget = btarget;
     ctx->hflags |= MIPS_HFLAG_BDS32;
  out:
@@ -8464,7 +8152,6 @@ static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
                                    int delayslot_size)
 {
     target_ulong btarget;
-    const char *opn = "cp1 cond branch";
     TCGv_i64 t0 = tcg_temp_new_i64();
 
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
@@ -8472,7 +8159,7 @@ static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
                   "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -8484,25 +8171,20 @@ static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
     switch (op) {
     case OPC_BC1EQZ:
         tcg_gen_xori_i64(t0, t0, 1);
-        opn = "bc1eqz";
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     case OPC_BC1NEZ:
         /* t0 already set */
-        opn = "bc1nez";
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("cp1 cond branch");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
     tcg_gen_trunc_i64_tl(bcond, t0);
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
-               ctx->hflags, btarget);
     ctx->btarget = btarget;
 
     switch (delayslot_size) {
@@ -8733,7 +8415,6 @@ enum r6_f_cmp_op {
 };
 static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
 {
-    const char *opn = "cp1 move";
     TCGv t0 = tcg_temp_new();
 
     switch (opc) {
@@ -8746,7 +8427,6 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             tcg_temp_free_i32(fp0);
         }
         gen_store_gpr(t0, rt);
-        opn = "mfc1";
         break;
     case OPC_MTC1:
         gen_load_gpr(t0, rt);
@@ -8757,16 +8437,14 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             gen_store_fpr32(ctx, fp0, fs);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mtc1";
         break;
     case OPC_CFC1:
         gen_helper_1e0i(cfc1, t0, fs);
         gen_store_gpr(t0, rt);
-        opn = "cfc1";
         break;
     case OPC_CTC1:
         gen_load_gpr(t0, rt);
-        save_cpu_state(ctx, 1);
+        save_cpu_state(ctx, 0);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(fs);
 
@@ -8775,18 +8453,15 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
         }
         /* Stop translation as we may have changed hflags */
         ctx->bstate = BS_STOP;
-        opn = "ctc1";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DMFC1:
         gen_load_fpr64(ctx, t0, fs);
         gen_store_gpr(t0, rt);
-        opn = "dmfc1";
         break;
     case OPC_DMTC1:
         gen_load_gpr(t0, rt);
         gen_store_fpr64(ctx, t0, fs);
-        opn = "dmtc1";
         break;
 #endif
     case OPC_MFHC1:
@@ -8798,7 +8473,6 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             tcg_temp_free_i32(fp0);
         }
         gen_store_gpr(t0, rt);
-        opn = "mfhc1";
         break;
     case OPC_MTHC1:
         gen_load_gpr(t0, rt);
@@ -8809,15 +8483,12 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             gen_store_fpr32h(ctx, fp0, fs);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mthc1";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("cp1 move");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rt], fregnames[fs]);
 
  out:
     tcg_temp_free(t0);
@@ -8947,7 +8618,7 @@ static void gen_sel_s(DisasContext *ctx, enum fopcode op1, int fd, int ft,
         break;
     default:
         MIPS_INVAL("gen_sel_s");
-        generate_exception (ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -8984,7 +8655,7 @@ static void gen_sel_d(DisasContext *ctx, enum fopcode op1, int fd, int ft,
         break;
     default:
         MIPS_INVAL("gen_sel_d");
-        generate_exception (ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -8998,44 +8669,6 @@ static void gen_sel_d(DisasContext *ctx, enum fopcode op1, int fd, int ft,
 static void gen_farith (DisasContext *ctx, enum fopcode op1,
                         int ft, int fs, int fd, int cc)
 {
-    const char *opn = "farith";
-    const char *condnames[] = {
-            "c.f",
-            "c.un",
-            "c.eq",
-            "c.ueq",
-            "c.olt",
-            "c.ult",
-            "c.ole",
-            "c.ule",
-            "c.sf",
-            "c.ngle",
-            "c.seq",
-            "c.ngl",
-            "c.lt",
-            "c.nge",
-            "c.le",
-            "c.ngt",
-    };
-    const char *condnames_abs[] = {
-            "cabs.f",
-            "cabs.un",
-            "cabs.eq",
-            "cabs.ueq",
-            "cabs.olt",
-            "cabs.ult",
-            "cabs.ole",
-            "cabs.ule",
-            "cabs.sf",
-            "cabs.ngle",
-            "cabs.seq",
-            "cabs.ngl",
-            "cabs.lt",
-            "cabs.nge",
-            "cabs.le",
-            "cabs.ngt",
-    };
-    enum { BINOP, CMPOP, OTHEROP } optype = OTHEROP;
     uint32_t func = ctx->opcode & 0x3f;
     switch (op1) {
     case OPC_ADD_S:
@@ -9050,8 +8683,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "add.s";
-        optype = BINOP;
         break;
     case OPC_SUB_S:
         {
@@ -9065,8 +8696,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "sub.s";
-        optype = BINOP;
         break;
     case OPC_MUL_S:
         {
@@ -9080,8 +8709,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mul.s";
-        optype = BINOP;
         break;
     case OPC_DIV_S:
         {
@@ -9095,8 +8722,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "div.s";
-        optype = BINOP;
         break;
     case OPC_SQRT_S:
         {
@@ -9107,7 +8732,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "sqrt.s";
         break;
     case OPC_ABS_S:
         {
@@ -9118,7 +8742,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "abs.s";
         break;
     case OPC_MOV_S:
         {
@@ -9128,7 +8751,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mov.s";
         break;
     case OPC_NEG_S:
         {
@@ -9139,7 +8761,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "neg.s";
         break;
     case OPC_ROUND_L_S:
         check_cp1_64bitmode(ctx);
@@ -9153,7 +8774,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "round.l.s";
         break;
     case OPC_TRUNC_L_S:
         check_cp1_64bitmode(ctx);
@@ -9167,7 +8787,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "trunc.l.s";
         break;
     case OPC_CEIL_L_S:
         check_cp1_64bitmode(ctx);
@@ -9181,7 +8800,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "ceil.l.s";
         break;
     case OPC_FLOOR_L_S:
         check_cp1_64bitmode(ctx);
@@ -9195,7 +8813,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "floor.l.s";
         break;
     case OPC_ROUND_W_S:
         {
@@ -9206,7 +8823,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "round.w.s";
         break;
     case OPC_TRUNC_W_S:
         {
@@ -9217,7 +8833,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "trunc.w.s";
         break;
     case OPC_CEIL_W_S:
         {
@@ -9228,7 +8843,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "ceil.w.s";
         break;
     case OPC_FLOOR_W_S:
         {
@@ -9239,27 +8853,22 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "floor.w.s";
         break;
     case OPC_SEL_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "sel.s";
         break;
     case OPC_SELEQZ_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "seleqz.s";
         break;
     case OPC_SELNEZ_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "selnez.s";
         break;
     case OPC_MOVCF_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_s(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.s";
         break;
     case OPC_MOVZ_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9276,7 +8885,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.s";
         break;
     case OPC_MOVN_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9293,10 +8901,8 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_set_label(l1);
             }
         }
-        opn = "movn.s";
         break;
     case OPC_RECIP_S:
-        check_cop1x(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
@@ -9305,10 +8911,8 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "recip.s";
         break;
     case OPC_RSQRT_S:
-        check_cop1x(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
@@ -9317,7 +8921,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "rsqrt.s";
         break;
     case OPC_MADDF_S:
         check_insn(ctx, ISA_MIPS32R6);
@@ -9333,7 +8936,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "maddf.s";
         }
         break;
     case OPC_MSUBF_S:
@@ -9350,7 +8952,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "msubf.s";
         }
         break;
     case OPC_RINT_S:
@@ -9361,7 +8962,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_rint_s(fp0, cpu_env, fp0);
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
-            opn = "rint.s";
         }
         break;
     case OPC_CLASS_S:
@@ -9372,7 +8972,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_class_s(fp0, fp0);
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
-            opn = "class.s";
         }
         break;
     case OPC_MIN_S: /* OPC_RECIP2_S */
@@ -9388,7 +8987,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "min.s";
         } else {
             /* OPC_RECIP2_S */
             check_cp1_64bitmode(ctx);
@@ -9403,7 +9001,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "recip2.s";
         }
         break;
     case OPC_MINA_S: /* OPC_RECIP1_S */
@@ -9419,7 +9016,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "mina.s";
         } else {
             /* OPC_RECIP1_S */
             check_cp1_64bitmode(ctx);
@@ -9431,7 +9027,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "recip1.s";
         }
         break;
     case OPC_MAX_S: /* OPC_RSQRT1_S */
@@ -9445,7 +9040,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp1, fd);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "max.s";
         } else {
             /* OPC_RSQRT1_S */
             check_cp1_64bitmode(ctx);
@@ -9457,7 +9051,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "rsqrt1.s";
         }
         break;
     case OPC_MAXA_S: /* OPC_RSQRT2_S */
@@ -9471,7 +9064,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp1, fd);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "maxa.s";
         } else {
             /* OPC_RSQRT2_S */
             check_cp1_64bitmode(ctx);
@@ -9486,7 +9078,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "rsqrt2.s";
         }
         break;
     case OPC_CVT_D_S:
@@ -9501,7 +9092,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.d.s";
         break;
     case OPC_CVT_W_S:
         {
@@ -9512,7 +9102,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.w.s";
         break;
     case OPC_CVT_L_S:
         check_cp1_64bitmode(ctx);
@@ -9526,7 +9115,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.l.s";
         break;
     case OPC_CVT_PS_S:
         check_ps(ctx);
@@ -9543,7 +9131,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.ps.s";
         break;
     case OPC_CMP_F_S:
     case OPC_CMP_UN_S:
@@ -9564,12 +9151,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_s(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_s(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
-        optype = CMPOP;
         break;
     case OPC_ADD_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9584,8 +9168,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "add.d";
-        optype = BINOP;
         break;
     case OPC_SUB_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9600,8 +9182,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sub.d";
-        optype = BINOP;
         break;
     case OPC_MUL_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9616,8 +9196,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mul.d";
-        optype = BINOP;
         break;
     case OPC_DIV_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9632,8 +9210,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "div.d";
-        optype = BINOP;
         break;
     case OPC_SQRT_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9645,7 +9221,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sqrt.d";
         break;
     case OPC_ABS_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9657,7 +9232,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "abs.d";
         break;
     case OPC_MOV_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9668,7 +9242,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mov.d";
         break;
     case OPC_NEG_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9680,7 +9253,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "neg.d";
         break;
     case OPC_ROUND_L_D:
         check_cp1_64bitmode(ctx);
@@ -9692,7 +9264,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "round.l.d";
         break;
     case OPC_TRUNC_L_D:
         check_cp1_64bitmode(ctx);
@@ -9704,7 +9275,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "trunc.l.d";
         break;
     case OPC_CEIL_L_D:
         check_cp1_64bitmode(ctx);
@@ -9716,7 +9286,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ceil.l.d";
         break;
     case OPC_FLOOR_L_D:
         check_cp1_64bitmode(ctx);
@@ -9728,7 +9297,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "floor.l.d";
         break;
     case OPC_ROUND_W_D:
         check_cp1_registers(ctx, fs);
@@ -9742,7 +9310,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "round.w.d";
         break;
     case OPC_TRUNC_W_D:
         check_cp1_registers(ctx, fs);
@@ -9756,7 +9323,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "trunc.w.d";
         break;
     case OPC_CEIL_W_D:
         check_cp1_registers(ctx, fs);
@@ -9770,7 +9336,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "ceil.w.d";
         break;
     case OPC_FLOOR_W_D:
         check_cp1_registers(ctx, fs);
@@ -9784,27 +9349,22 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "floor.w.d";
         break;
     case OPC_SEL_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "sel.d";
         break;
     case OPC_SELEQZ_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "seleqz.d";
         break;
     case OPC_SELNEZ_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "selnez.d";
         break;
     case OPC_MOVCF_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_d(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.d";
         break;
     case OPC_MOVZ_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9821,7 +9381,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.d";
         break;
     case OPC_MOVN_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9838,10 +9397,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_set_label(l1);
             }
         }
-        opn = "movn.d";
         break;
     case OPC_RECIP_D:
-        check_cp1_64bitmode(ctx);
+        check_cp1_registers(ctx, fs | fd);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -9850,10 +9408,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip.d";
         break;
     case OPC_RSQRT_D:
-        check_cp1_64bitmode(ctx);
+        check_cp1_registers(ctx, fs | fd);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -9862,7 +9419,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt.d";
         break;
     case OPC_MADDF_D:
         check_insn(ctx, ISA_MIPS32R6);
@@ -9878,7 +9434,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp2);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "maddf.d";
         }
         break;
     case OPC_MSUBF_D:
@@ -9895,7 +9450,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp2);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "msubf.d";
         }
         break;
     case OPC_RINT_D:
@@ -9906,7 +9460,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_rint_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
-            opn = "rint.d";
         }
         break;
     case OPC_CLASS_D:
@@ -9917,7 +9470,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_class_d(fp0, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
-            opn = "class.d";
         }
         break;
     case OPC_MIN_D: /* OPC_RECIP2_D */
@@ -9931,7 +9483,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "min.d";
         } else {
             /* OPC_RECIP2_D */
             check_cp1_64bitmode(ctx);
@@ -9946,7 +9497,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "recip2.d";
         }
         break;
     case OPC_MINA_D: /* OPC_RECIP1_D */
@@ -9960,7 +9510,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "mina.d";
         } else {
             /* OPC_RECIP1_D */
             check_cp1_64bitmode(ctx);
@@ -9972,7 +9521,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "recip1.d";
         }
         break;
     case OPC_MAX_D: /*  OPC_RSQRT1_D */
@@ -9986,7 +9534,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "max.d";
         } else {
             /* OPC_RSQRT1_D */
             check_cp1_64bitmode(ctx);
@@ -9998,7 +9545,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "rsqrt1.d";
         }
         break;
     case OPC_MAXA_D: /* OPC_RSQRT2_D */
@@ -10012,7 +9558,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "maxa.d";
         } else {
             /* OPC_RSQRT2_D */
             check_cp1_64bitmode(ctx);
@@ -10027,7 +9572,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "rsqrt2.d";
         }
         break;
     case OPC_CMP_F_D:
@@ -10049,12 +9593,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_d(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_d(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
-        optype = CMPOP;
         break;
     case OPC_CVT_S_D:
         check_cp1_registers(ctx, fs);
@@ -10068,7 +9609,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.s.d";
         break;
     case OPC_CVT_W_D:
         check_cp1_registers(ctx, fs);
@@ -10082,7 +9622,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.w.d";
         break;
     case OPC_CVT_L_D:
         check_cp1_64bitmode(ctx);
@@ -10094,7 +9633,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.l.d";
         break;
     case OPC_CVT_S_W:
         {
@@ -10105,7 +9643,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.w";
         break;
     case OPC_CVT_D_W:
         check_cp1_registers(ctx, fd);
@@ -10119,7 +9656,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.d.w";
         break;
     case OPC_CVT_S_L:
         check_cp1_64bitmode(ctx);
@@ -10133,7 +9669,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.s.l";
         break;
     case OPC_CVT_D_L:
         check_cp1_64bitmode(ctx);
@@ -10145,7 +9680,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.d.l";
         break;
     case OPC_CVT_PS_PW:
         check_ps(ctx);
@@ -10157,7 +9691,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.ps.pw";
         break;
     case OPC_ADD_PS:
         check_ps(ctx);
@@ -10172,7 +9705,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "add.ps";
         break;
     case OPC_SUB_PS:
         check_ps(ctx);
@@ -10187,7 +9719,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sub.ps";
         break;
     case OPC_MUL_PS:
         check_ps(ctx);
@@ -10202,7 +9733,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mul.ps";
         break;
     case OPC_ABS_PS:
         check_ps(ctx);
@@ -10214,7 +9744,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "abs.ps";
         break;
     case OPC_MOV_PS:
         check_ps(ctx);
@@ -10225,7 +9754,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mov.ps";
         break;
     case OPC_NEG_PS:
         check_ps(ctx);
@@ -10237,12 +9765,10 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "neg.ps";
         break;
     case OPC_MOVCF_PS:
         check_ps(ctx);
         gen_movcf_ps(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.ps";
         break;
     case OPC_MOVZ_PS:
         check_ps(ctx);
@@ -10258,7 +9784,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.ps";
         break;
     case OPC_MOVN_PS:
         check_ps(ctx);
@@ -10275,7 +9800,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_set_label(l1);
             }
         }
-        opn = "movn.ps";
         break;
     case OPC_ADDR_PS:
         check_ps(ctx);
@@ -10290,7 +9814,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "addr.ps";
         break;
     case OPC_MULR_PS:
         check_ps(ctx);
@@ -10305,7 +9828,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mulr.ps";
         break;
     case OPC_RECIP2_PS:
         check_ps(ctx);
@@ -10320,7 +9842,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip2.ps";
         break;
     case OPC_RECIP1_PS:
         check_ps(ctx);
@@ -10332,7 +9853,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip1.ps";
         break;
     case OPC_RSQRT1_PS:
         check_ps(ctx);
@@ -10344,7 +9864,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt1.ps";
         break;
     case OPC_RSQRT2_PS:
         check_ps(ctx);
@@ -10359,7 +9878,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt2.ps";
         break;
     case OPC_CVT_S_PU:
         check_cp1_64bitmode(ctx);
@@ -10371,7 +9889,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.pu";
         break;
     case OPC_CVT_PW_PS:
         check_ps(ctx);
@@ -10383,7 +9900,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.pw.ps";
         break;
     case OPC_CVT_S_PL:
         check_cp1_64bitmode(ctx);
@@ -10395,7 +9911,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.pl";
         break;
     case OPC_PLL_PS:
         check_ps(ctx);
@@ -10410,7 +9925,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "pll.ps";
         break;
     case OPC_PLU_PS:
         check_ps(ctx);
@@ -10425,7 +9939,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "plu.ps";
         break;
     case OPC_PUL_PS:
         check_ps(ctx);
@@ -10440,7 +9953,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "pul.ps";
         break;
     case OPC_PUU_PS:
         check_ps(ctx);
@@ -10455,7 +9967,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "puu.ps";
         break;
     case OPC_CMP_F_PS:
     case OPC_CMP_UN_PS:
@@ -10475,38 +9986,21 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
     case OPC_CMP_NGT_PS:
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_ps(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_ps(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
-        optype = CMPOP;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("farith");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
-    (void)opn; /* avoid a compiler warning */
-    switch (optype) {
-    case BINOP:
-        MIPS_DEBUG("%s %s, %s, %s", opn, fregnames[fd], fregnames[fs], fregnames[ft]);
-        break;
-    case CMPOP:
-        MIPS_DEBUG("%s %s,%s", opn, fregnames[fs], fregnames[ft]);
-        break;
-    default:
-        MIPS_DEBUG("%s %s,%s", opn, fregnames[fd], fregnames[fs]);
-        break;
-    }
 }
 
 /* Coprocessor 3 (FPU) */
 static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
                            int fd, int fs, int base, int index)
 {
-    const char *opn = "extended float load/store";
-    int store = 0;
     TCGv t0 = tcg_temp_new();
 
     if (base == 0) {
@@ -10529,7 +10023,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "lwxc1";
         break;
     case OPC_LDXC1:
         check_cop1x(ctx);
@@ -10540,7 +10033,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ldxc1";
         break;
     case OPC_LUXC1:
         check_cp1_64bitmode(ctx);
@@ -10552,7 +10044,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "luxc1";
         break;
     case OPC_SWXC1:
         check_cop1x(ctx);
@@ -10562,8 +10053,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             tcg_gen_qemu_st_i32(fp0, t0, ctx->mem_idx, MO_TEUL);
             tcg_temp_free_i32(fp0);
         }
-        opn = "swxc1";
-        store = 1;
         break;
     case OPC_SDXC1:
         check_cop1x(ctx);
@@ -10574,8 +10063,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             tcg_gen_qemu_st_i64(fp0, t0, ctx->mem_idx, MO_TEQ);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sdxc1";
-        store = 1;
         break;
     case OPC_SUXC1:
         check_cp1_64bitmode(ctx);
@@ -10586,21 +10073,14 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             tcg_gen_qemu_st_i64(fp0, t0, ctx->mem_idx, MO_TEQ);
             tcg_temp_free_i64(fp0);
         }
-        opn = "suxc1";
-        store = 1;
         break;
     }
     tcg_temp_free(t0);
-    (void)opn; (void)store; /* avoid compiler warnings */
-    MIPS_DEBUG("%s %s, %s(%s)", opn, fregnames[store ? fs : fd],
-               regnames[index], regnames[base]);
 }
 
 static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
                             int fd, int fr, int fs, int ft)
 {
-    const char *opn = "flt3_arith";
-
     switch (opc) {
     case OPC_ALNV_PS:
         check_ps(ctx);
@@ -10638,7 +10118,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             tcg_temp_free_i32(fp);
             tcg_temp_free_i32(fph);
         }
-        opn = "alnv.ps";
         break;
     case OPC_MADD_S:
         check_cop1x(ctx);
@@ -10656,7 +10135,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "madd.s";
         break;
     case OPC_MADD_D:
         check_cop1x(ctx);
@@ -10675,7 +10153,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "madd.d";
         break;
     case OPC_MADD_PS:
         check_ps(ctx);
@@ -10693,7 +10170,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "madd.ps";
         break;
     case OPC_MSUB_S:
         check_cop1x(ctx);
@@ -10711,7 +10187,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "msub.s";
         break;
     case OPC_MSUB_D:
         check_cop1x(ctx);
@@ -10730,7 +10205,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "msub.d";
         break;
     case OPC_MSUB_PS:
         check_ps(ctx);
@@ -10748,7 +10222,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "msub.ps";
         break;
     case OPC_NMADD_S:
         check_cop1x(ctx);
@@ -10766,7 +10239,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "nmadd.s";
         break;
     case OPC_NMADD_D:
         check_cop1x(ctx);
@@ -10785,7 +10257,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmadd.d";
         break;
     case OPC_NMADD_PS:
         check_ps(ctx);
@@ -10803,7 +10274,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmadd.ps";
         break;
     case OPC_NMSUB_S:
         check_cop1x(ctx);
@@ -10821,7 +10291,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "nmsub.s";
         break;
     case OPC_NMSUB_D:
         check_cop1x(ctx);
@@ -10840,7 +10309,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmsub.d";
         break;
     case OPC_NMSUB_PS:
         check_ps(ctx);
@@ -10858,19 +10326,15 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmsub.ps";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("flt3_arith");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s, %s", opn, fregnames[fd], fregnames[fr],
-               fregnames[fs], fregnames[ft]);
 }
 
-static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
+static void gen_rdhwr(DisasContext *ctx, int rt, int rd, int sel)
 {
     TCGv t0;
 
@@ -10883,25 +10347,37 @@ static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
 
     switch (rd) {
     case 0:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_cpunum(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 1:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_synci_step(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 2:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_cc(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 3:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_ccres(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
+    case 4:
+        check_insn(ctx, ISA_MIPS32R6);
+        if (sel != 0) {
+            /* Performance counter registers are not implemented other than
+             * control register 0.
+             */
+            generate_exception(ctx, EXCP_RI);
+        }
+        gen_helper_rdhwr_performance(t0, cpu_env);
+        gen_store_gpr(t0, rt);
+        break;
+    case 5:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_helper_rdhwr_xnp(t0, cpu_env);
+        gen_store_gpr(t0, rt);
+        break;
     case 29:
 #if defined(CONFIG_USER_ONLY)
         tcg_gen_ld_tl(t0, cpu_env,
@@ -10915,13 +10391,13 @@ static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
                           offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
             gen_store_gpr(t0, rt);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
 #endif
     default:            /* Invalid */
         MIPS_INVAL("rdhwr");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
     tcg_temp_free(t0);
@@ -10949,12 +10425,10 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
         /* FIXME: Need to clear can_do_io.  */
         switch (proc_hflags & MIPS_HFLAG_BMASK_BASE) {
         case MIPS_HFLAG_FBNSLOT:
-            MIPS_DEBUG("forbidden slot");
             gen_goto_tb(ctx, 0, ctx->pc + insn_bytes);
             break;
         case MIPS_HFLAG_B:
             /* unconditional branch */
-            MIPS_DEBUG("unconditional branch");
             if (proc_hflags & MIPS_HFLAG_BX) {
                 tcg_gen_xori_i32(hflags, hflags, MIPS_HFLAG_M16);
             }
@@ -10962,12 +10436,10 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
             break;
         case MIPS_HFLAG_BL:
             /* blikely taken case */
-            MIPS_DEBUG("blikely branch taken");
             gen_goto_tb(ctx, 0, ctx->btarget);
             break;
         case MIPS_HFLAG_BC:
             /* Conditional branch */
-            MIPS_DEBUG("conditional branch");
             {
                 TCGLabel *l1 = gen_new_label();
 
@@ -10979,7 +10451,6 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
             break;
         case MIPS_HFLAG_BR:
             /* unconditional branch to register */
-            MIPS_DEBUG("branch to register");
             if (ctx->insn_flags & (ASE_MIPS16 | ASE_MICROMIPS)) {
                 TCGv t0 = tcg_temp_new();
                 TCGv_i32 t1 = tcg_temp_new_i32();
@@ -10998,7 +10469,7 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
             }
             if (ctx->singlestep_enabled) {
                 save_cpu_state(ctx, 0);
-                gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+                gen_helper_raise_exception_debug(cpu_env);
             }
             tcg_gen_exit_tb(0);
             break;
@@ -11023,7 +10494,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
                   "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -11085,7 +10556,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
         break;
     default:
         MIPS_INVAL("Compact branch/jump");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -11106,7 +10577,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
             break;
         default:
             MIPS_INVAL("Compact branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
 
@@ -11229,7 +10700,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
             break;
         default:
             MIPS_INVAL("Compact conditional branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
 
@@ -11238,7 +10709,6 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
         gen_set_label(fs);
 
         ctx->hflags |= MIPS_HFLAG_FBNSLOT;
-        MIPS_DEBUG("Compact conditional branch");
     }
 
 out:
@@ -11404,7 +10874,7 @@ static void gen_mips16_save (DisasContext *ctx,
         args = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11500,7 +10970,7 @@ static void gen_mips16_save (DisasContext *ctx,
         astatic = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11606,7 +11076,7 @@ static void gen_mips16_restore (DisasContext *ctx,
         astatic = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11637,7 +11107,7 @@ static void gen_addiupc (DisasContext *ctx, int rx, int imm,
     TCGv t0;
 
     if (extended && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11686,7 +11156,7 @@ static void decode_i64_mips16 (DisasContext *ctx,
         check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         if (extended && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             offset = extended ? offset : offset << 3;
             gen_ld(ctx, OPC_LDPC, ry, 0, offset);
@@ -11761,7 +11231,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
@@ -11789,7 +11259,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
         } else {
             gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
@@ -11841,7 +11311,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             }
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -11904,7 +11374,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
         break;
 #endif
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -11985,7 +11455,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
@@ -12013,7 +11483,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 check_mips_64(ctx);
                 gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
 #endif
             } else {
                 gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
@@ -12097,7 +11567,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 gen_arith(ctx, OPC_ADDU, ry, reg32, 0);
                 break;
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         }
@@ -12187,7 +11657,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 break;
 #endif
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 goto done;
             }
 
@@ -12226,7 +11696,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                  *      when in debug mode...
                  */
                 check_insn(ctx, ISA_MIPS32);
-                generate_exception(ctx, EXCP_DBp);
+                generate_exception_end(ctx, EXCP_DBp);
             }
             break;
         case RR_SLT:
@@ -12236,7 +11706,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             gen_slt(ctx, OPC_SLTU, 24, rx, ry);
             break;
         case RR_BREAK:
-            generate_exception(ctx, EXCP_BREAK);
+            generate_exception_end(ctx, EXCP_BREAK);
             break;
         case RR_SLLV:
             gen_shift(ctx, OPC_SLLV, ry, rx, ry);
@@ -12303,7 +11773,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 break;
 #endif
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             break;
@@ -12367,7 +11837,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             break;
 #endif
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -12382,7 +11852,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
         break;
 #endif
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -12526,6 +11996,7 @@ enum {
     ROTR = 0x3,
     SELEQZ = 0x5,
     SELNEZ = 0x6,
+    R6_RDHWR = 0x7,
 
     SLLV = 0x0,
     SRLV = 0x1,
@@ -12556,11 +12027,13 @@ enum {
     MODU = 0x7,
 
     /* The following can be distinguished by their lower 6 bits. */
+    BREAK32 = 0x07,
     INS = 0x0c,
     LSA = 0x0f,
     ALIGN = 0x1f,
     EXT = 0x2c,
-    POOL32AXF = 0x3c
+    POOL32AXF = 0x3c,
+    SIGRIE = 0x3f
 };
 
 /* POOL32AXF encoding of minor opcode field extension */
@@ -13029,12 +12502,11 @@ static void gen_andi16(DisasContext *ctx)
 static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
                                int base, int16_t offset)
 {
-    const char *opn = "ldst_multiple";
     TCGv t0, t1;
     TCGv_i32 t2;
 
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -13049,25 +12521,19 @@ static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
     switch (opc) {
     case LWM32:
         gen_helper_lwm(cpu_env, t0, t1, t2);
-        opn = "lwm";
         break;
     case SWM32:
         gen_helper_swm(cpu_env, t0, t1, t2);
-        opn = "swm";
         break;
 #ifdef TARGET_MIPS64
     case LDM:
         gen_helper_ldm(cpu_env, t0, t1, t2);
-        opn = "ldm";
         break;
     case SDM:
         gen_helper_sdm(cpu_env, t0, t1, t2);
-        opn = "sdm";
         break;
 #endif
     }
-    (void)opn;
-    MIPS_DEBUG("%s, %x, %d(%s)", opn, reglist, offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
     tcg_temp_free_i32(t2);
@@ -13164,7 +12630,7 @@ static void gen_pool16c_insn(DisasContext *ctx)
         gen_HILO(ctx, OPC_MFLO, 0, uMIPS_RS5(ctx->opcode));
         break;
     case BREAK16:
-        generate_exception(ctx, EXCP_BREAK);
+        generate_exception_end(ctx, EXCP_BREAK);
         break;
     case SDBBP16:
         if (is_uhi(extract32(ctx->opcode, 0, 4))) {
@@ -13174,7 +12640,7 @@ static void gen_pool16c_insn(DisasContext *ctx)
              *      when in debug mode...
              */
             check_insn(ctx, ISA_MIPS32);
-            generate_exception(ctx, EXCP_DBp);
+            generate_exception_end(ctx, EXCP_DBp);
         }
         break;
     case JRADDIUSP + 0:
@@ -13188,7 +12654,7 @@ static void gen_pool16c_insn(DisasContext *ctx)
         }
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -13324,11 +12790,10 @@ static void gen_ldxs (DisasContext *ctx, int base, int index, int rd)
 static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
                            int base, int16_t offset)
 {
-    const char *opn = "ldst_pair";
     TCGv t0, t1;
 
     if (ctx->hflags & MIPS_HFLAG_BMASK || rd == 31) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -13340,7 +12805,7 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
     switch (opc) {
     case LWP:
         if (rd == base) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return;
         }
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TESL);
@@ -13349,7 +12814,6 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t1, rd+1);
-        opn = "lwp";
         break;
     case SWP:
         gen_load_gpr(t1, rd);
@@ -13358,12 +12822,11 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUL);
-        opn = "swp";
         break;
 #ifdef TARGET_MIPS64
     case LDP:
         if (rd == base) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return;
         }
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TEQ);
@@ -13372,7 +12835,6 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t1, rd+1);
-        opn = "ldp";
         break;
     case SDP:
         gen_load_gpr(t1, rd);
@@ -13381,12 +12843,9 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEQ);
-        opn = "sdp";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s, %s, %d(%s)", opn, regnames[rd], offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -13492,7 +12951,8 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
             gen_cl(ctx, mips32_op, rt, rs);
             break;
         case RDHWR:
-            gen_rdhwr(ctx, rt, rs);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            gen_rdhwr(ctx, rt, rs, 0);
             break;
         case WSBH:
             gen_bshfl(ctx, OPC_WSBH, rs, rt);
@@ -13668,8 +13128,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
             /* NOP */
             break;
         case SYSCALL:
-            generate_exception(ctx, EXCP_SYSCALL);
-            ctx->bstate = BS_STOP;
+            generate_exception_end(ctx, EXCP_SYSCALL);
             break;
         case SDBBP:
             if (is_uhi(extract32(ctx->opcode, 16, 10))) {
@@ -13677,9 +13136,9 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
             } else {
                 check_insn(ctx, ISA_MIPS32);
                 if (ctx->hflags & MIPS_HFLAG_SBRI) {
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                 } else {
-                    generate_exception(ctx, EXCP_DBp);
+                    generate_exception_end(ctx, EXCP_DBp);
                 }
             }
             break;
@@ -13727,7 +13186,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
     default:
     pool32axf_invalid:
         MIPS_INVAL("pool32axf");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -13995,7 +13454,7 @@ static void gen_pool32fxf(DisasContext *ctx, int rt, int rs)
         break;
     default:
         MIPS_INVAL("pool32fxf");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -14048,6 +13507,10 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
                 check_insn(ctx, ISA_MIPS32R6);
                 gen_cond_move(ctx, OPC_SELNEZ, rd, rs, rt);
                 break;
+            case R6_RDHWR:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_rdhwr(ctx, rt, rs, extract32(ctx->opcode, 11, 3));
+                break;
             default:
                 goto pool32a_invalid;
             }
@@ -14191,13 +13654,17 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
         case POOL32AXF:
             gen_pool32axf(env, ctx, rt, rs);
             break;
-        case 0x07:
-            generate_exception(ctx, EXCP_BREAK);
+        case BREAK32:
+            generate_exception_end(ctx, EXCP_BREAK);
+            break;
+        case SIGRIE:
+            check_insn(ctx, ISA_MIPS32R6);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         default:
         pool32a_invalid:
                 MIPS_INVAL("pool32a");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
         }
         break;
@@ -14237,7 +13704,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("pool32b");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -14717,7 +14184,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             default:
             pool32f_invalid:
                 MIPS_INVAL("pool32f");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else {
@@ -14873,7 +14340,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             /* Fall through */
         default:
             MIPS_INVAL("pool32i");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -14962,7 +14429,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("pool32c");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -15240,7 +14707,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
         gen_st(ctx, mips32_op, rt, rs, imm);
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -15252,8 +14719,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
     /* make sure instructions are on a halfword boundary */
     if (ctx->pc & 0x1) {
         env->CP0_BadVAddr = ctx->pc;
-        generate_exception(ctx, EXCP_AdEL);
-        ctx->bstate = BS_STOP;
+        generate_exception_end(ctx, EXCP_AdEL);
         return 2;
     }
 
@@ -15272,9 +14738,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
         case 7:
         /* LB32, LH32, LWC132, LDC132, LW32 */
             if (ctx->hflags & MIPS_HFLAG_BDS16) {
-                generate_exception(ctx, EXCP_RI);
-                /* Just stop translation; the user is confused.  */
-                ctx->bstate = BS_STOP;
+                generate_exception_end(ctx, EXCP_RI);
                 return 2;
             }
             break;
@@ -15285,9 +14749,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
         case 3:
         /* MOVE16, ANDI16, POOL16D, POOL16E, BEQZ16, BNEZ16, B16, LI16 */
             if (ctx->hflags & MIPS_HFLAG_BDS32) {
-                generate_exception(ctx, EXCP_RI);
-                /* Just stop translation; the user is confused.  */
-                ctx->bstate = BS_STOP;
+                generate_exception_end(ctx, EXCP_RI);
                 return 2;
             }
             break;
@@ -15359,7 +14821,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
     case POOL16F:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & 1) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             /* MOVEP */
             int enc_dest = uMIPS_RD(ctx->opcode);
@@ -15497,7 +14959,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
     case RES_29:
     case RES_31:
     case RES_39:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     default:
         decode_micromips32_opc(env, ctx);
@@ -15519,7 +14981,6 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
 static void gen_mipsdsp_ld(DisasContext *ctx, uint32_t opc,
                            int rd, int base, int offset)
 {
-    const char *opn = "ldx";
     TCGv t0;
 
     check_dsp(ctx);
@@ -15537,42 +14998,33 @@ static void gen_mipsdsp_ld(DisasContext *ctx, uint32_t opc,
     case OPC_LBUX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_UB);
         gen_store_gpr(t0, rd);
-        opn = "lbux";
         break;
     case OPC_LHX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESW);
         gen_store_gpr(t0, rd);
-        opn = "lhx";
         break;
     case OPC_LWX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t0, rd);
-        opn = "lwx";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_LDX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t0, rd);
-        opn = "ldx";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s(%s)", opn,
-               regnames[rd], regnames[offset], regnames[base]);
     tcg_temp_free(t0);
 }
 
 static void gen_mipsdsp_arith(DisasContext *ctx, uint32_t op1, uint32_t op2,
                               int ret, int v1, int v2)
 {
-    const char *opn = "mipsdsp arith";
     TCGv v1_t;
     TCGv v2_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16012,23 +15464,18 @@ static void gen_mipsdsp_arith(DisasContext *ctx, uint32_t op1, uint32_t op2,
 
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
                               int ret, int v1, int v2)
 {
     uint32_t op2;
-    const char *opn = "mipsdsp shift";
     TCGv t0;
     TCGv v1_t;
     TCGv v2_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16135,7 +15582,7 @@ static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
                 break;
             default:            /* Invalid */
                 MIPS_INVAL("MASK SHLL.QB");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             break;
@@ -16250,7 +15697,7 @@ static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK SHLL.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -16260,21 +15707,17 @@ static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
     tcg_temp_free(t0);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                  int ret, int v1, int v2, int check_ret)
 {
-    const char *opn = "mipsdsp multiply";
     TCGv_i32 t0;
     TCGv v1_t;
     TCGv v2_t;
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16572,23 +16015,17 @@ static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
     tcg_temp_free_i32(t0);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
-
 }
 
 static void gen_mipsdsp_bitinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                 int ret, int val)
 {
-    const char *opn = "mipsdsp Bit/ Manipulation";
     int16_t imm;
     TCGv t0;
     TCGv val_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16716,23 +16153,18 @@ static void gen_mipsdsp_bitinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
     }
     tcg_temp_free(t0);
     tcg_temp_free(val_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
                                      uint32_t op1, uint32_t op2,
                                      int ret, int v1, int v2, int check_ret)
 {
-    const char *opn = "mipsdsp add compare pick";
     TCGv t1;
     TCGv v1_t;
     TCGv v2_t;
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16907,22 +16339,17 @@ static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
     tcg_temp_free(t1);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
                                uint32_t op1, int rt, int rs, int sa)
 {
-    const char *opn = "mipsdsp append/dappend";
     TCGv t0;
 
     check_dspr2(ctx);
 
     if (rt == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16959,7 +16386,7 @@ static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK APPEND");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -16993,22 +16420,19 @@ static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DAPPEND");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
 #endif
     }
     tcg_temp_free(t0);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                 int ret, int v1, int v2, int check_ret)
 
 {
-    const char *opn = "mipsdsp accumulator";
     TCGv t0;
     TCGv t1;
     TCGv v1_t;
@@ -17017,7 +16441,6 @@ static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -17229,9 +16652,6 @@ static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
     tcg_temp_free(t1);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 /* End MIPSDSP functions. */
@@ -17266,7 +16686,7 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("special_r6 muldiv");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17281,7 +16701,7 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
                We need additionally to check other fields */
             gen_cl(ctx, op1, rd, rs);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case R6_OPC_SDBBP:
@@ -17289,9 +16709,9 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             gen_helper_do_semihosting(cpu_env);
         } else {
             if (ctx->hflags & MIPS_HFLAG_SBRI) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
             } else {
-                generate_exception(ctx, EXCP_DBp);
+                generate_exception_end(ctx, EXCP_DBp);
             }
         }
         break;
@@ -17308,7 +16728,7 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_cl(ctx, op1, rd, rs);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case OPC_DMULT ... OPC_DDIVU:
@@ -17327,14 +16747,14 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("special_r6 muldiv");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special_r6");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17402,16 +16822,16 @@ static void decode_opc_special_legacy(CPUMIPSState *env, DisasContext *ctx)
     case OPC_SPIM:
 #ifdef MIPS_STRICT_STANDARD
         MIPS_INVAL("SPIM");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 #else
         /* Implemented as RI exception for now. */
         MIPS_INVAL("spim (unofficial)");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 #endif
         break;
     default:            /* Invalid */
         MIPS_INVAL("special_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17433,8 +16853,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             rs == 0 && rt == 0) { /* PAUSE */
             if ((ctx->insn_flags & ISA_MIPS32R6) &&
                 (ctx->hflags & MIPS_HFLAG_BMASK)) {
-                MIPS_DEBUG("CTI in delay / forbidden slot");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         }
@@ -17454,7 +16873,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17477,7 +16896,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift(ctx, op1, rd, rs, rt);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17507,18 +16926,17 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             /* Pmon entry point, also R4010 selsl */
 #ifdef MIPS_STRICT_STANDARD
             MIPS_INVAL("PMON / selsl");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #else
             gen_helper_0e0i(pmon, sa);
 #endif
         }
         break;
     case OPC_SYSCALL:
-        generate_exception(ctx, EXCP_SYSCALL);
-        ctx->bstate = BS_STOP;
+        generate_exception_end(ctx, EXCP_SYSCALL);
         break;
     case OPC_BREAK:
-        generate_exception(ctx, EXCP_BREAK);
+        generate_exception_end(ctx, EXCP_BREAK);
         break;
     case OPC_SYNC:
         check_insn(ctx, ISA_MIPS2);
@@ -17549,7 +16967,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17567,7 +16985,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17596,7 +17014,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift(ctx, op1, rd, rs, rt);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17659,7 +17077,7 @@ static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
              *      when in debug mode...
              */
             check_insn(ctx, ISA_MIPS32);
-            generate_exception(ctx, EXCP_DBp);
+            generate_exception_end(ctx, EXCP_DBp);
         }
         break;
 #if defined(TARGET_MIPS64)
@@ -17681,7 +17099,7 @@ static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special2_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17703,7 +17121,7 @@ static void decode_opc_special3_r6(CPUMIPSState *env, DisasContext *ctx)
     case R6_OPC_PREF:
         if (rt >= 24) {
             /* hint codes 24-31 are reserved and signal RI */
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         /* Treat as NOP. */
         break;
@@ -17762,7 +17180,7 @@ static void decode_opc_special3_r6(CPUMIPSState *env, DisasContext *ctx)
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special3_r6");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17808,13 +17226,13 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
                 break;
             default:
                 MIPS_INVAL("MASK ADDUH.QB");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else if (ctx->insn_flags & INSN_LOONGSON2E) {
             gen_loongson_integer(ctx, op1, rd, rs, rt);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case OPC_LX_DSP:
@@ -17830,7 +17248,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK LX");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17861,7 +17279,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("MASK ABSQ_S.PH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17898,7 +17316,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ADDU.QB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
 
         }
@@ -17938,7 +17356,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK CMPU.EQ.QB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17974,7 +17392,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DPAW.PH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17987,7 +17405,6 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
                 TCGv t0, t1;
 
                 if (rt == 0) {
-                    MIPS_DEBUG("NOP");
                     break;
                 }
 
@@ -18005,7 +17422,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             }
         default:            /* Invalid */
             MIPS_INVAL("MASK INSV");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18040,7 +17457,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK EXTR.W");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18083,7 +17500,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ABSQ_S.QH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18122,7 +17539,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ADDU.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18167,7 +17584,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK CMPU_EQ.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18204,7 +17621,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK EXTR.W");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18243,7 +17660,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DPAQ.W.QH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18255,7 +17672,6 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             TCGv t0, t1;
 
             if (rt == 0) {
-                MIPS_DEBUG("NOP");
                 break;
             }
             check_dsp(ctx);
@@ -18274,7 +17690,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
         }
         default:            /* Invalid */
             MIPS_INVAL("MASK DINSV");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18284,7 +17700,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special3_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -18345,7 +17761,7 @@ static void decode_opc_special3(CPUMIPSState *env, DisasContext *ctx)
         break;
 #endif
     case OPC_RDHWR:
-        gen_rdhwr(ctx, rt, rd);
+        gen_rdhwr(ctx, rt, rd, extract32(ctx->opcode, 6, 3));
         break;
     case OPC_FORK:
         check_insn(ctx, ASE_MT);
@@ -18365,7 +17781,6 @@ static void decode_opc_special3(CPUMIPSState *env, DisasContext *ctx)
         {
             TCGv t0 = tcg_temp_new();
 
-            save_cpu_state(ctx, 1);
             gen_load_gpr(t0, rs);
             gen_helper_yield(t0, cpu_env, t0);
             gen_store_gpr(t0, rd);
@@ -18386,16 +17801,16 @@ static inline int check_msa_access(DisasContext *ctx)
 {
     if (unlikely((ctx->hflags & MIPS_HFLAG_FPU) &&
                  !(ctx->hflags & MIPS_HFLAG_F64))) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return 0;
     }
 
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_MSA))) {
         if (ctx->insn_flags & ASE_MSA) {
-            generate_exception(ctx, EXCP_MSADIS);
+            generate_exception_end(ctx, EXCP_MSADIS);
             return 0;
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return 0;
         }
     }
@@ -18452,8 +17867,7 @@ static void gen_msa_branch(CPUMIPSState *env, DisasContext *ctx, uint32_t op1)
     check_msa_access(ctx);
 
     if (ctx->insn_flags & ISA_MIPS32R6 && ctx->hflags & MIPS_HFLAG_BMASK) {
-        MIPS_DEBUG("CTI in delay / forbidden slot");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
     switch (op1) {
@@ -18528,7 +17942,7 @@ static void gen_msa_i8(CPUMIPSState *env, DisasContext *ctx)
         {
             uint8_t df = (ctx->opcode >> 24) & 0x3;
             if (df == DF_DOUBLE) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
             } else {
                 TCGv_i32 tdf = tcg_const_i32(df);
                 gen_helper_msa_shf_df(cpu_env, tdf, twd, tws, ti8);
@@ -18538,7 +17952,7 @@ static void gen_msa_i8(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18610,7 +18024,7 @@ static void gen_msa_i5(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18646,7 +18060,7 @@ static void gen_msa_bit(CPUMIPSState *env, DisasContext *ctx)
         m = dfm & 0x7;
         df = DF_BYTE;
     } else {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -18694,7 +18108,7 @@ static void gen_msa_bit(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18889,7 +18303,8 @@ static void gen_msa_3r(CPUMIPSState *env, DisasContext *ctx)
     case OPC_HSUB_S_df:
     case OPC_HSUB_U_df:
         if (df == DF_BYTE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
+            break;
         }
         switch (MASK_MSA_3R(ctx->opcode)) {
         case OPC_DOTP_S_df:
@@ -18926,7 +18341,7 @@ static void gen_msa_3r(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
     tcg_temp_free_i32(twd);
@@ -18958,7 +18373,7 @@ static void gen_msa_elm_3e(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18995,7 +18410,7 @@ static void gen_msa_elm_df(CPUMIPSState *env, DisasContext *ctx, uint32_t df,
 #if !defined(TARGET_MIPS64)
         /* Double format valid only for MIPS64 */
         if (df == DF_DOUBLE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
 #endif
@@ -19013,7 +18428,7 @@ static void gen_msa_elm_df(CPUMIPSState *env, DisasContext *ctx, uint32_t df,
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
     tcg_temp_free_i32(twd);
     tcg_temp_free_i32(tws);
@@ -19043,7 +18458,7 @@ static void gen_msa_elm(CPUMIPSState *env, DisasContext *ctx)
         gen_msa_elm_3e(env, ctx);
         return;
     } else {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -19198,7 +18613,7 @@ static void gen_msa_3rf(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19226,7 +18641,7 @@ static void gen_msa_2r(CPUMIPSState *env, DisasContext *ctx)
 #if !defined(TARGET_MIPS64)
         /* Double format valid only for MIPS64 */
         if (df == DF_DOUBLE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
 #endif
@@ -19243,7 +18658,7 @@ static void gen_msa_2r(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19358,7 +18773,7 @@ static void gen_msa_vec_v(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19387,7 +18802,7 @@ static void gen_msa_vec(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -19485,7 +18900,7 @@ static void gen_msa(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19502,7 +18917,6 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     if (ctx->pc & 0x3) {
         env->CP0_BadVAddr = ctx->pc;
         generate_exception_err(ctx, EXCP_AdEL, EXCP_INST_NOTAVAIL);
-        ctx->bstate = BS_STOP;
         return;
     }
 
@@ -19510,17 +18924,12 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     if ((ctx->hflags & MIPS_HFLAG_BMASK_BASE) == MIPS_HFLAG_BL) {
         TCGLabel *l1 = gen_new_label();
 
-        MIPS_DEBUG("blikely condition (" TARGET_FMT_lx ")", ctx->pc + 4);
         tcg_gen_brcondi_tl(TCG_COND_NE, bcond, 0, l1);
         tcg_gen_movi_i32(hflags, ctx->hflags & ~MIPS_HFLAG_BMASK);
         gen_goto_tb(ctx, 1, ctx->pc + 4);
         gen_set_label(l1);
     }
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     op = MASK_OP_MAJOR(ctx->opcode);
     rs = (ctx->opcode >> 21) & 0x1f;
     rt = (ctx->opcode >> 16) & 0x1f;
@@ -19558,7 +18967,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                     /* OPC_NAL, OPC_BAL */
                     gen_compute_branch(ctx, op1, 4, 0, -1, imm << 2, 4);
                 } else {
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                 }
             } else {
                 gen_compute_branch(ctx, op1, 4, rs, -1, imm << 2, 4);
@@ -19570,6 +18979,10 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_trap(ctx, op1, rs, -1, imm);
             break;
+        case OPC_SIGRIE:
+            check_insn(ctx, ISA_MIPS32R6);
+            generate_exception_end(ctx, EXCP_RI);
+            break;
         case OPC_SYNCI:
             check_insn(ctx, ISA_MIPS32R2);
             /* Break the TB to be able to sync copied instructions
@@ -19590,7 +19003,6 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             if (rs != 0) {
                 tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 32);
             }
-            MIPS_DEBUG("dahi %s, %04x", regnames[rs], imm);
             break;
         case OPC_DATI:
             check_insn(ctx, ISA_MIPS32R6);
@@ -19598,12 +19010,11 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             if (rs != 0) {
                 tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 48);
             }
-            MIPS_DEBUG("dati %s, %04x", regnames[rs], imm);
             break;
 #endif
         default:            /* Invalid */
             MIPS_INVAL("regimm");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -19678,7 +19089,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                     break;
                 default:            /* Invalid */
                     MIPS_INVAL("mfmc0");
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                     break;
                 }
                 tcg_temp_free(t0);
@@ -19695,7 +19106,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("cp0");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -19730,7 +19141,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC, OPC_BLEZL */
         if (ctx->insn_flags & ISA_MIPS32R6) {
             if (rt == 0) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             /* OPC_BLEZC, OPC_BGEZC, OPC_BGEC */
@@ -19743,7 +19154,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC, OPC_BGTZL */
         if (ctx->insn_flags & ISA_MIPS32R6) {
             if (rt == 0) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             /* OPC_BGTZC, OPC_BLTZC, OPC_BLTC */
@@ -19973,7 +19384,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("cp1");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -20055,7 +19466,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                 break;
             default:
                 MIPS_INVAL("cp3");
-                generate_exception (ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else {
@@ -20111,7 +19522,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
         } else {
             MIPS_INVAL("major opcode");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
 #endif
@@ -20120,15 +19531,16 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
 #if defined(TARGET_MIPS64)
             /* OPC_DAUI */
             check_mips_64(ctx);
-            if (rt != 0) {
+            if (rs == 0) {
+                generate_exception(ctx, EXCP_RI);
+            } else if (rt != 0) {
                 TCGv t0 = tcg_temp_new();
                 gen_load_gpr(t0, rs);
                 tcg_gen_addi_tl(cpu_gpr[rt], t0, imm << 16);
                 tcg_temp_free(t0);
             }
-            MIPS_DEBUG("daui %s, %s, %04x", regnames[rt], regnames[rs], imm);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             MIPS_INVAL("major opcode");
 #endif
         } else {
@@ -20148,30 +19560,23 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:            /* Invalid */
         MIPS_INVAL("major opcode");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
 
-static inline void
-gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMIPSState *env, struct TranslationBlock *tb)
 {
+    MIPSCPU *cpu = mips_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUMIPSState *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
     target_ulong next_page_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
     int insn_bytes;
     int is_slot;
 
-    if (search_pc)
-        qemu_log("search pc %d\n", search_pc);
-
     pc_start = tb->pc;
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
     ctx.pc = pc_start;
@@ -20181,6 +19586,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
     ctx.CP0_Config1 = env->CP0_Config1;
     ctx.tb = tb;
     ctx.bstate = BS_NONE;
+    ctx.btarget = 0;
     ctx.kscrexist = (env->CP0_Config4 >> CP0C4_KScrExist) & 0xff;
     ctx.rxi = (env->CP0_Config3 >> CP0C3_RXI) & 1;
     ctx.ie = (env->CP0_Config4 >> CP0C4_IE) & 3;
@@ -20204,40 +19610,34 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
                                  MO_UNALN : MO_ALIGN;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     LOG_DISAS("\ntb %p idx %d hflags %04x\n", tb, ctx.mem_idx, ctx.hflags);
     gen_tb_start(tb);
     while (ctx.bstate == BS_NONE) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.pc) {
-                    save_cpu_state(&ctx, 1);
-                    ctx.bstate = BS_BRANCH;
-                    gen_helper_0e0i(raise_exception, EXCP_DEBUG);
-                    /* Include the breakpoint location or the tb won't
-                     * be flushed when it must be.  */
-                    ctx.pc += 4;
-                    goto done_generating;
-                }
-            }
-        }
+        tcg_gen_insn_start(ctx.pc, ctx.hflags & MIPS_HFLAG_BMASK, ctx.btarget);
+        num_insns++;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            gen_opc_hflags[lj] = ctx.hflags & MIPS_HFLAG_BMASK;
-            gen_opc_btarget[lj] = ctx.btarget;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            save_cpu_state(&ctx, 1);
+            ctx.bstate = BS_BRANCH;
+            gen_helper_raise_exception_debug(cpu_env);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 4;
+            goto done_generating;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         is_slot = ctx.hflags & MIPS_HFLAG_BMASK;
         if (!(ctx.hflags & MIPS_HFLAG_M16)) {
@@ -20251,8 +19651,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
             ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_mips16_opc(env, &ctx);
         } else {
-            generate_exception(&ctx, EXCP_RI);
-            ctx.bstate = BS_STOP;
+            generate_exception_end(&ctx, EXCP_RI);
             break;
         }
 
@@ -20275,8 +19674,6 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
         }
         ctx.pc += insn_bytes;
 
-        num_insns++;
-
         /* Execute a branch and its delay slot as a single instruction.
            This is what GDB expects and is consistent with what the
            hardware does (e.g. if a delay slot instruction faults, the
@@ -20304,7 +19701,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
     }
     if (cs->singlestep_enabled && ctx.bstate != BS_BRANCH) {
         save_cpu_state(&ctx, ctx.bstate != BS_EXCP);
-        gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+        gen_helper_raise_exception_debug(cpu_env);
     } else {
         switch (ctx.bstate) {
         case BS_STOP:
@@ -20325,15 +19722,9 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
 done_generating:
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
+
 #ifdef DEBUG_DISAS
     LOG_DISAS("\n");
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -20344,16 +19735,6 @@ done_generating:
 #endif
 }
 
-void gen_intermediate_code (CPUMIPSState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mips_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUMIPSState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mips_env_get_cpu(env), tb, true);
-}
-
 static void fpu_dump_state(CPUMIPSState *env, FILE *f, fprintf_function fpu_fprintf,
                            int flags)
 {
@@ -20394,40 +19775,6 @@ static void fpu_dump_state(CPUMIPSState *env, FILE *f, fprintf_function fpu_fpri
 #undef printfpr
 }
 
-#if defined(TARGET_MIPS64) && defined(MIPS_DEBUG_SIGN_EXTENSIONS)
-/* Debug help: The architecture requires 32bit code to maintain proper
-   sign-extended values on 64bit machines.  */
-
-#define SIGN_EXT_P(val) ((((val) & ~0x7fffffff) == 0) || (((val) & ~0x7fffffff) == ~0x7fffffff))
-
-static void
-cpu_mips_check_sign_extensions (CPUMIPSState *env, FILE *f,
-                                fprintf_function cpu_fprintf,
-                                int flags)
-{
-    int i;
-
-    if (!SIGN_EXT_P(env->active_tc.PC))
-        cpu_fprintf(f, "BROKEN: pc=0x" TARGET_FMT_lx "\n", env->active_tc.PC);
-    if (!SIGN_EXT_P(env->active_tc.HI[0]))
-        cpu_fprintf(f, "BROKEN: HI=0x" TARGET_FMT_lx "\n", env->active_tc.HI[0]);
-    if (!SIGN_EXT_P(env->active_tc.LO[0]))
-        cpu_fprintf(f, "BROKEN: LO=0x" TARGET_FMT_lx "\n", env->active_tc.LO[0]);
-    if (!SIGN_EXT_P(env->btarget))
-        cpu_fprintf(f, "BROKEN: btarget=0x" TARGET_FMT_lx "\n", env->btarget);
-
-    for (i = 0; i < 32; i++) {
-        if (!SIGN_EXT_P(env->active_tc.gpr[i]))
-            cpu_fprintf(f, "BROKEN: %s=0x" TARGET_FMT_lx "\n", regnames[i], env->active_tc.gpr[i]);
-    }
-
-    if (!SIGN_EXT_P(env->CP0_EPC))
-        cpu_fprintf(f, "BROKEN: EPC=0x" TARGET_FMT_lx "\n", env->CP0_EPC);
-    if (!SIGN_EXT_P(env->lladdr))
-        cpu_fprintf(f, "BROKEN: LLAddr=0x" TARGET_FMT_lx "\n", env->lladdr);
-}
-#endif
-
 void mips_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -20459,9 +19806,6 @@ void mips_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                 env->CP0_Config4, env->CP0_Config5);
     if (env->hflags & MIPS_HFLAG_FPU)
         fpu_dump_state(env, f, cpu_fprintf, flags);
-#if defined(TARGET_MIPS64) && defined(MIPS_DEBUG_SIGN_EXTENSIONS)
-    cpu_mips_check_sign_extensions(env, f, cpu_fprintf, flags);
-#endif
 }
 
 void mips_tcg_init(void)
@@ -20714,18 +20058,19 @@ void cpu_state_reset(CPUMIPSState *env)
     }
 }
 
-void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->active_tc.PC = tcg_ctx.gen_opc_pc[pc_pos];
+    env->active_tc.PC = data[0];
     env->hflags &= ~MIPS_HFLAG_BMASK;
-    env->hflags |= gen_opc_hflags[pc_pos];
+    env->hflags |= data[1];
     switch (env->hflags & MIPS_HFLAG_BMASK_BASE) {
     case MIPS_HFLAG_BR:
         break;
     case MIPS_HFLAG_BC:
     case MIPS_HFLAG_BL:
     case MIPS_HFLAG_B:
-        env->btarget = gen_opc_btarget[pc_pos];
+        env->btarget = data[2];
         break;
     }
 }
diff --git a/target-mips/translate_init.c b/target-mips/translate_init.c
index 9304e746b..bb33c7cfe 100644
--- a/target-mips/translate_init.c
+++ b/target-mips/translate_init.c
@@ -389,39 +389,44 @@ static const mips_def_t mips_defs[] =
         .mmu_type = MMU_TYPE_R4000,
     },
     {
-        /* A generic CPU providing MIPS32 Release 5 features.
-           FIXME: Eventually this should be replaced by a real CPU model. */
-        .name = "mips32r5-generic",
-        .CP0_PRid = 0x00019700,
-        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+        /* FIXME:
+         * Config3: CMGCR, SC, PW, VZ, CTXTC, CDMM, TL
+         * Config4: MMUExtDef
+         * Config5: EVA, MRP
+         * FIR(FCR0): Has2008
+         * */
+        .name = "P5600",
+        .CP0_PRid = 0x0001A800,
+        .CP0_Config0 = MIPS_CONFIG0 | (1 << CP0C0_MM) | (1 << CP0C0_AR) |
                     (MMU_TYPE_R4000 << CP0C0_MT),
-        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (15 << CP0C1_MMU) |
-                       (0 << CP0C1_IS) | (3 << CP0C1_IL) | (1 << CP0C1_IA) |
-                       (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA) |
-                       (1 << CP0C1_CA),
+        .CP0_Config1 = MIPS_CONFIG1 | (0x3F << CP0C1_MMU) |
+                       (2 << CP0C1_IS) | (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_FP),
         .CP0_Config2 = MIPS_CONFIG2,
         .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_MSAP) |
-                       (1 << CP0C3_LPA),
-        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M),
+                       (1 << CP0C3_BP) | (1 << CP0C3_BI) | (1 << CP0C3_ULRI) |
+                       (1 << CP0C3_RXI) | (1 << CP0C3_LPA) | (1 << CP0C3_VInt),
+        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) | (2 << CP0C4_IE) |
+                       (0x1c << CP0C4_KScrExist),
         .CP0_Config4_rw_bitmask = 0,
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_UFR) | (1 << CP0C5_LLB) |
-                       (1 << CP0C5_MVH),
-        .CP0_Config5_rw_bitmask = (0 << CP0C5_M) | (1 << CP0C5_K) |
-                                  (1 << CP0C5_CV) | (0 << CP0C5_EVA) |
-                                  (1 << CP0C5_MSAEn) | (1 << CP0C5_UFR) |
-                                  (0 << CP0C5_NFExists),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_MVH) | (1 << CP0C5_LLB),
+        .CP0_Config5_rw_bitmask = (1 << CP0C5_K) | (1 << CP0C5_CV) |
+                                  (1 << CP0C5_MSAEn) | (1 << CP0C5_UFE) |
+                                  (1 << CP0C5_FRE) | (1 << CP0C5_UFR),
         .CP0_LLAddr_rw_bitmask = 0,
-        .CP0_LLAddr_shift = 4,
+        .CP0_LLAddr_shift = 0,
         .SYNCI_Step = 32,
         .CCRes = 2,
-        .CP0_Status_rw_bitmask = 0x3778FF1F,
-        .CP0_PageGrain_rw_bitmask = (1 << CP0PG_ELPA),
-        .CP1_fcr0 = (1 << FCR0_UFRP) | (1 << FCR0_F64) | (1 << FCR0_L) |
-                    (1 << FCR0_W) | (1 << FCR0_D) | (1 << FCR0_S) |
-                    (0x93 << FCR0_PRID),
+        .CP0_Status_rw_bitmask = 0x3C68FF1F,
+        .CP0_PageGrain_rw_bitmask = (1U << CP0PG_RIE) | (1 << CP0PG_XIE) |
+                    (1 << CP0PG_ELPA) | (1 << CP0PG_IEC),
+        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_UFRP) | (1 << FCR0_F64) |
+                    (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
+                    (1 << FCR0_S) | (0x03 << FCR0_PRID),
         .SEGBITS = 32,
         .PABITS = 40,
-        .insn_flags = CPU_MIPS32R5 | ASE_MIPS16 | ASE_MSA,
+        .insn_flags = CPU_MIPS32R5 | ASE_MSA,
         .mmu_type = MMU_TYPE_R4000,
     },
     {
@@ -442,7 +447,7 @@ static const mips_def_t mips_defs[] =
                        (1 << CP0C3_RXI) | (1U << CP0C3_M),
         .CP0_Config4 = MIPS_CONFIG4 | (0xfc << CP0C4_KScrExist) |
                        (3 << CP0C4_IE) | (1U << CP0C4_M),
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_LLB),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_XNP) | (1 << CP0C5_LLB),
         .CP0_Config5_rw_bitmask = (1 << CP0C5_SBRI) | (1 << CP0C5_FRE) |
                                   (1 << CP0C5_UFE),
         .CP0_LLAddr_rw_bitmask = 0,
@@ -660,7 +665,7 @@ static const mips_def_t mips_defs[] =
                        (1 << CP0C3_RXI) | (1 << CP0C3_LPA),
         .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) | (3 << CP0C4_IE) |
                        (0xfc << CP0C4_KScrExist),
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_LLB),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_XNP) | (1 << CP0C5_LLB),
         .CP0_Config5_rw_bitmask = (1 << CP0C5_MSAEn) | (1 << CP0C5_SBRI) |
                                   (1 << CP0C5_FRE) | (1 << CP0C5_UFE),
         .CP0_LLAddr_rw_bitmask = 0,
diff --git a/target-moxie/cpu.c b/target-moxie/cpu.c
index 3af37799b..0c60c65d3 100644
--- a/target-moxie/cpu.c
+++ b/target-moxie/cpu.c
@@ -48,6 +48,12 @@ static void moxie_cpu_reset(CPUState *s)
     tlb_flush(s, 1);
 }
 
+static void moxie_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_arch_moxie;
+    info->print_insn = print_insn_moxie;
+}
+
 static void moxie_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -114,6 +120,7 @@ static void moxie_cpu_class_init(ObjectClass *oc, void *data)
     cc->get_phys_page_debug = moxie_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_moxie_cpu;
 #endif
+    cc->disas_set_info = moxie_cpu_disas_set_info;
 
     /*
      * Reason: moxie_cpu_initfn() calls cpu_exec_init(), which saves
diff --git a/target-moxie/cpu.h b/target-moxie/cpu.h
index 29572aaba..a61274401 100644
--- a/target-moxie/cpu.h
+++ b/target-moxie/cpu.h
@@ -26,8 +26,6 @@
 
 #define CPUArchState struct CPUMoxieState
 
-#define ELF_MACHINE     0xFEED /* EM_MOXIE */
-
 #define MOXIE_EX_DIV0        0
 #define MOXIE_EX_BAD         1
 #define MOXIE_EX_IRQ         2
@@ -124,10 +122,9 @@ int cpu_moxie_signal_handler(int host_signum, void *pinfo,
 #define cpu_init(cpu_model) CPU(cpu_moxie_init(cpu_model))
 
 #define cpu_exec cpu_moxie_exec
-#define cpu_gen_code cpu_moxie_gen_code
 #define cpu_signal_handler cpu_moxie_signal_handler
 
-static inline int cpu_mmu_index(CPUMoxieState *env)
+static inline int cpu_mmu_index(CPUMoxieState *env, bool ifetch)
 {
     return 0;
 }
diff --git a/target-moxie/helper.c b/target-moxie/helper.c
index 6c98965b9..f91ac28d3 100644
--- a/target-moxie/helper.c
+++ b/target-moxie/helper.c
@@ -19,7 +19,6 @@
 
 #include <stdio.h>
 #include <string.h>
-#include <assert.h>
 
 #include "config.h"
 #include "cpu.h"
@@ -56,7 +55,7 @@ void helper_raise_exception(CPUMoxieState *env, int ex)
     /* Stash the address where the exception occurred.  */
     cpu_restore_state(cs, GETPC());
     env->sregs[5] = env->pc;
-    /* Jump the the exception handline routine.  */
+    /* Jump to the exception handline routine.  */
     env->pc = env->sregs[1];
     cpu_loop_exit(cs);
 }
diff --git a/target-moxie/translate.c b/target-moxie/translate.c
index e3e913906..6dedcb7a2 100644
--- a/target-moxie/translate.c
+++ b/target-moxie/translate.c
@@ -26,7 +26,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <inttypes.h>
-#include <assert.h>
 
 #include "cpu.h"
 #include "exec/exec-all.h"
@@ -154,10 +153,6 @@ static int decode_opc(MoxieCPU *cpu, DisasContext *ctx)
     /* Set the default instruction length.  */
     int length = 2;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     /* Examine the 16-bit opcode.  */
     opcode = ctx->opcode;
 
@@ -820,17 +815,13 @@ static int decode_opc(MoxieCPU *cpu, DisasContext *ctx)
 }
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(MoxieCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMoxieState *env, struct TranslationBlock *tb)
 {
+    MoxieCPU *cpu = moxie_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
     DisasContext ctx;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
-    CPUMoxieState *env = &cpu->env;
-    int num_insns;
+    int num_insns, max_insns;
 
     pc_start = tb->pc;
     ctx.pc = pc_start;
@@ -840,40 +831,40 @@ gen_intermediate_code_internal(MoxieCPU *cpu, TranslationBlock *tb,
     ctx.singlestep_enabled = 0;
     ctx.bstate = BS_NONE;
     num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (ctx.pc == bp->pc) {
-                    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-                    gen_helper_debug(cpu_env);
-                    ctx.bstate = BS_EXCP;
-                    goto done_generating;
-                }
-            }
-        }
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            tcg_gen_movi_i32(cpu_pc, ctx.pc);
+            gen_helper_debug(cpu_env);
+            ctx.bstate = BS_EXCP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 2;
+            goto done_generating;
         }
+
         ctx.opcode = cpu_lduw_code(env, ctx.pc);
         ctx.pc += decode_opc(cpu, &ctx);
-        num_insns++;
 
+        if (num_insns >= max_insns) {
+            break;
+        }
         if (cs->singlestep_enabled) {
             break;
         }
-
         if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0) {
             break;
         }
@@ -899,29 +890,12 @@ gen_intermediate_code_internal(MoxieCPU *cpu, TranslationBlock *tb,
  done_generating:
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUMoxieState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(moxie_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUMoxieState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(moxie_env_get_cpu(env), tb, true);
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 }
 
-void restore_state_to_opc(CPUMoxieState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMoxieState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h
index 36c4f20e2..eb7160739 100644
--- a/target-openrisc/cpu.h
+++ b/target-openrisc/cpu.h
@@ -21,7 +21,6 @@
 #define CPU_OPENRISC_H
 
 #define TARGET_LONG_BITS 32
-#define ELF_MACHINE    EM_OPENRISC
 
 #define CPUArchState struct CPUOpenRISCState
 
@@ -361,7 +360,6 @@ int cpu_openrisc_signal_handler(int host_signum, void *pinfo, void *puc);
 
 #define cpu_list cpu_openrisc_list
 #define cpu_exec cpu_openrisc_exec
-#define cpu_gen_code cpu_openrisc_gen_code
 #define cpu_signal_handler cpu_openrisc_signal_handler
 
 #ifndef CONFIG_USER_ONLY
@@ -403,7 +401,7 @@ static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env,
     *flags = (env->flags & D_FLAG);
 }
 
-static inline int cpu_mmu_index(CPUOpenRISCState *env)
+static inline int cpu_mmu_index(CPUOpenRISCState *env, bool ifetch)
 {
     if (!(env->sr & SR_IME)) {
         return MMU_NOMMU_IDX;
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index a62cbf401..606490a47 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -279,7 +279,7 @@ static void dec_calc(DisasContext *dc, uint32_t insn)
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_extu_i32_i64(tb, cpu_R[rb]);
                 tcg_gen_add_i64(td, ta, tb);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 31);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -324,7 +324,7 @@ static void dec_calc(DisasContext *dc, uint32_t insn)
                 tcg_gen_shri_i64(tcy, tcy, 10);
                 tcg_gen_add_i64(td, ta, tb);
                 tcg_gen_add_i64(td, td, tcy);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 32);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -366,7 +366,7 @@ static void dec_calc(DisasContext *dc, uint32_t insn)
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_extu_i32_i64(tb, cpu_R[rb]);
                 tcg_gen_sub_i64(td, ta, tb);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 31);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -779,9 +779,9 @@ static void dec_misc(DisasContext *dc, uint32_t insn)
             tcg_gen_ext_i32_i64(t1, dst);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_add_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(dst);
             tcg_temp_free(ttmp);
             tcg_temp_free_i64(t1);
@@ -898,7 +898,7 @@ static void dec_misc(DisasContext *dc, uint32_t insn)
                 TCGv_i32 sr_ove = tcg_temp_local_new_i32();
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_addi_i64(td, ta, sign_extend(I16, 16));
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 32);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -934,7 +934,7 @@ static void dec_misc(DisasContext *dc, uint32_t insn)
             tcg_gen_extu_i32_i64(tcy, sr_cy);
             tcg_gen_addi_i64(td, ta, sign_extend(I16, 16));
             tcg_gen_add_i64(td, td, tcy);
-            tcg_gen_trunc_i64_i32(res, td);
+            tcg_gen_extrl_i64_i32(res, td);
             tcg_gen_shri_i64(td, td, 32);
             tcg_gen_andi_i64(td, td, 0x3);
             /* Jump to lab when no overflow.  */
@@ -1073,9 +1073,9 @@ static void dec_mac(DisasContext *dc, uint32_t insn)
             tcg_gen_ext_i32_i64(t1, t0);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_add_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(t0);
             tcg_temp_free_i64(t1);
             tcg_temp_free_i64(t2);
@@ -1092,9 +1092,9 @@ static void dec_mac(DisasContext *dc, uint32_t insn)
             tcg_gen_ext_i32_i64(t1, t0);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_sub_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(t0);
             tcg_temp_free_i64(t1);
             tcg_temp_free_i64(t2);
@@ -1618,30 +1618,12 @@ static void disas_openrisc_insn(DisasContext *dc, OpenRISCCPU *cpu)
     }
 }
 
-static void check_breakpoint(OpenRISCCPU *cpu, DisasContext *dc)
-{
-    CPUState *cs = CPU(cpu);
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_tl(cpu_pc, dc->pc);
-                gen_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-            }
-        }
-    }
-}
-
-static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  int search_pc)
+void gen_intermediate_code(CPUOpenRISCState *env, struct TranslationBlock *tb)
 {
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
     struct DisasContext ctx, *dc = &ctx;
     uint32_t pc_start;
-    int j, k;
     uint32_t next_page_start;
     int num_insns;
     int max_insns;
@@ -1653,7 +1635,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     dc->ppc = pc_start;
     dc->pc = pc_start;
     dc->flags = cpu->env.cpucfgr;
-    dc->mem_idx = cpu_mmu_index(&cpu->env);
+    dc->mem_idx = cpu_mmu_index(&cpu->env, false);
     dc->synced_flags = dc->tb_flags = tb->flags;
     dc->delayed_branch = !!(dc->tb_flags & D_FLAG);
     dc->singlestep_enabled = cs->singlestep_enabled;
@@ -1663,36 +1645,35 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    k = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
 
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
     do {
-        check_breakpoint(cpu, dc);
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (k < j) {
-                k++;
-                while (k < j) {
-                    tcg_ctx.gen_opc_instr_start[k++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[k] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[k] = 1;
-            tcg_ctx.gen_opc_icount[k] = num_insns;
-        }
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            gen_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         dc->ppc = dc->pc - 4;
@@ -1701,7 +1682,6 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
         tcg_gen_movi_tl(cpu_npc, dc->npc);
         disas_openrisc_insn(dc, cpu);
         dc->pc = dc->npc;
-        num_insns++;
         /* delay slot */
         if (dc->delayed_branch) {
             dc->delayed_branch--;
@@ -1756,16 +1736,8 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        k++;
-        while (k <= j) {
-            tcg_ctx.gen_opc_instr_start[k++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -1777,17 +1749,6 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code(CPUOpenRISCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(openrisc_env_get_cpu(env), tb, 0);
-}
-
-void gen_intermediate_code_pc(CPUOpenRISCState *env,
-                              struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(openrisc_env_get_cpu(env), tb, 1);
-}
-
 void openrisc_cpu_dump_state(CPUState *cs, FILE *f,
                              fprintf_function cpu_fprintf,
                              int flags)
@@ -1804,7 +1765,7 @@ void openrisc_cpu_dump_state(CPUState *cs, FILE *f,
 }
 
 void restore_state_to_opc(CPUOpenRISCState *env, TranslationBlock *tb,
-                          int pc_pos)
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-ppc/Makefile.objs b/target-ppc/Makefile.objs
index a7ae392cc..e667e6970 100644
--- a/target-ppc/Makefile.objs
+++ b/target-ppc/Makefile.objs
@@ -1,10 +1,10 @@
 obj-y += cpu-models.o
 obj-y += translate.o
 ifeq ($(CONFIG_SOFTMMU),y)
-obj-y += machine.o mmu_helper.o mmu-hash32.o
+obj-y += machine.o mmu_helper.o mmu-hash32.o monitor.o
 obj-$(TARGET_PPC64) += mmu-hash64.o arch_dump.o
 endif
-obj-$(CONFIG_KVM) += kvm.o kvm_ppc.o
+obj-$(CONFIG_KVM) += kvm.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-y += dfp_helper.o
 obj-y += excp_helper.o
diff --git a/target-ppc/cpu-qom.h b/target-ppc/cpu-qom.h
index 6967a8028..bc20504b3 100644
--- a/target-ppc/cpu-qom.h
+++ b/target-ppc/cpu-qom.h
@@ -118,6 +118,8 @@ void ppc_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                         int flags);
 void ppc_cpu_dump_statistics(CPUState *cpu, FILE *f,
                              fprintf_function cpu_fprintf, int flags);
+int ppc_cpu_get_monitor_def(CPUState *cs, const char *name,
+                            uint64_t *pval);
 hwaddr ppc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int ppc_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int ppc_cpu_gdb_read_register_apple(CPUState *cpu, uint8_t *buf, int reg);
diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h
index 6f76674a4..9706000f8 100644
--- a/target-ppc/cpu.h
+++ b/target-ppc/cpu.h
@@ -81,9 +81,9 @@
 #include "fpu/softfloat.h"
 
 #if defined (TARGET_PPC64)
-#define ELF_MACHINE     EM_PPC64
+#define PPC_ELF_MACHINE     EM_PPC64
 #else
-#define ELF_MACHINE     EM_PPC
+#define PPC_ELF_MACHINE     EM_PPC
 #endif
 
 /*****************************************************************************/
@@ -117,14 +117,20 @@ enum powerpc_mmu_t {
 #define POWERPC_MMU_AMR      0x00040000
     /* 64 bits PowerPC MMU                                     */
     POWERPC_MMU_64B        = POWERPC_MMU_64 | 0x00000001,
+    /* Architecture 2.03 and later (has LPCR) */
+    POWERPC_MMU_2_03       = POWERPC_MMU_64 | 0x00000002,
     /* Architecture 2.06 variant                               */
     POWERPC_MMU_2_06       = POWERPC_MMU_64 | POWERPC_MMU_1TSEG
                              | POWERPC_MMU_AMR | 0x00000003,
     /* Architecture 2.06 "degraded" (no 1T segments)           */
     POWERPC_MMU_2_06a      = POWERPC_MMU_64 | POWERPC_MMU_AMR
                              | 0x00000003,
-    /* Architecture 2.06 "degraded" (no 1T segments or AMR)    */
-    POWERPC_MMU_2_06d      = POWERPC_MMU_64 | 0x00000003,
+    /* Architecture 2.07 variant                               */
+    POWERPC_MMU_2_07       = POWERPC_MMU_64 | POWERPC_MMU_1TSEG
+                             | POWERPC_MMU_AMR | 0x00000004,
+    /* Architecture 2.07 "degraded" (no 1T segments)           */
+    POWERPC_MMU_2_07a      = POWERPC_MMU_64 | POWERPC_MMU_AMR
+                             | 0x00000004,
 #endif /* defined(TARGET_PPC64) */
 };
 
@@ -678,6 +684,27 @@ enum {
 #define fpscr_eex (((env->fpscr) >> FPSCR_XX) & ((env->fpscr) >> FPSCR_XE) &  \
                    0x1F)
 
+#define FP_FX		(1ull << FPSCR_FX)
+#define FP_FEX		(1ull << FPSCR_FEX)
+#define FP_OX		(1ull << FPSCR_OX)
+#define FP_OE		(1ull << FPSCR_OE)
+#define FP_UX		(1ull << FPSCR_UX)
+#define FP_UE		(1ull << FPSCR_UE)
+#define FP_XX		(1ull << FPSCR_XX)
+#define FP_XE		(1ull << FPSCR_XE)
+#define FP_ZX		(1ull << FPSCR_ZX)
+#define FP_ZE		(1ull << FPSCR_ZE)
+#define FP_VX		(1ull << FPSCR_VX)
+#define FP_VXSNAN	(1ull << FPSCR_VXSNAN)
+#define FP_VXISI	(1ull << FPSCR_VXISI)
+#define FP_VXIMZ	(1ull << FPSCR_VXIMZ)
+#define FP_VXZDZ	(1ull << FPSCR_VXZDZ)
+#define FP_VXIDI	(1ull << FPSCR_VXIDI)
+#define FP_VXVC		(1ull << FPSCR_VXVC)
+#define FP_VXCVI	(1ull << FPSCR_VXCVI)
+#define FP_VE		(1ull << FPSCR_VE)
+#define FP_FI		(1ull << FPSCR_FI)
+
 /*****************************************************************************/
 /* Vector status and control register */
 #define VSCR_NJ		16 /* Vector non-java */
@@ -1073,6 +1100,7 @@ struct CPUPPCState {
     uint64_t insns_flags2;
 #if defined(TARGET_PPC64)
     struct ppc_segment_page_sizes sps;
+    bool ci_large_pages;
 #endif
 
 #if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
@@ -1241,7 +1269,6 @@ int ppc_dcr_write (ppc_dcr_t *dcr_env, int dcrn, uint32_t val);
 #define cpu_init(cpu_model) CPU(cpu_ppc_init(cpu_model))
 
 #define cpu_exec cpu_ppc_exec
-#define cpu_gen_code cpu_ppc_gen_code
 #define cpu_signal_handler cpu_ppc_signal_handler
 #define cpu_list ppc_cpu_list
 
@@ -1250,7 +1277,7 @@ int ppc_dcr_write (ppc_dcr_t *dcr_env, int dcrn, uint32_t val);
 #define MMU_MODE1_SUFFIX _kernel
 #define MMU_MODE2_SUFFIX _hypv
 #define MMU_USER_IDX 0
-static inline int cpu_mmu_index (CPUPPCState *env)
+static inline int cpu_mmu_index (CPUPPCState *env, bool ifetch)
 {
     return env->mmu_idx;
 }
diff --git a/target-ppc/dfp_helper.c b/target-ppc/dfp_helper.c
index 49820bf21..451e4340f 100644
--- a/target-ppc/dfp_helper.c
+++ b/target-ppc/dfp_helper.c
@@ -170,27 +170,6 @@ static void dfp_prepare_decimal128(struct PPC_DFP *dfp, uint64_t *a,
     }
 }
 
-#define FP_FX       (1ull << FPSCR_FX)
-#define FP_FEX      (1ull << FPSCR_FEX)
-#define FP_OX       (1ull << FPSCR_OX)
-#define FP_OE       (1ull << FPSCR_OE)
-#define FP_UX       (1ull << FPSCR_UX)
-#define FP_UE       (1ull << FPSCR_UE)
-#define FP_XX       (1ull << FPSCR_XX)
-#define FP_XE       (1ull << FPSCR_XE)
-#define FP_ZX       (1ull << FPSCR_ZX)
-#define FP_ZE       (1ull << FPSCR_ZE)
-#define FP_VX       (1ull << FPSCR_VX)
-#define FP_VXSNAN   (1ull << FPSCR_VXSNAN)
-#define FP_VXISI    (1ull << FPSCR_VXISI)
-#define FP_VXIMZ    (1ull << FPSCR_VXIMZ)
-#define FP_VXZDZ    (1ull << FPSCR_VXZDZ)
-#define FP_VXIDI    (1ull << FPSCR_VXIDI)
-#define FP_VXVC     (1ull << FPSCR_VXVC)
-#define FP_VXCVI    (1ull << FPSCR_VXCVI)
-#define FP_VE       (1ull << FPSCR_VE)
-#define FP_FI       (1ull << FPSCR_FI)
-
 static void dfp_set_FPSCR_flag(struct PPC_DFP *dfp, uint64_t flag,
                 uint64_t enabled)
 {
diff --git a/target-ppc/excp_helper.c b/target-ppc/excp_helper.c
index b80347506..4250106b3 100644
--- a/target-ppc/excp_helper.c
+++ b/target-ppc/excp_helper.c
@@ -200,7 +200,7 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
         /* Get rS/rD and rA from faulting opcode */
         env->spr[SPR_DSISR] |= (cpu_ldl_code(env, (env->nip - 4))
                                 & 0x03FF0000) >> 16;
-        goto store_current;
+        goto store_next;
     case POWERPC_EXCP_PROGRAM:   /* Program exception                        */
         switch (env->error_code & ~0xF) {
         case POWERPC_EXCP_FP:
diff --git a/target-ppc/fpu_helper.c b/target-ppc/fpu_helper.c
index 6cceffc55..9f2d53d74 100644
--- a/target-ppc/fpu_helper.c
+++ b/target-ppc/fpu_helper.c
@@ -194,7 +194,7 @@ static inline uint64_t fload_invalid_op_excp(CPUPPCState *env, int op,
     /* Update the floating-point invalid operation summary */
     env->fpscr |= 1 << FPSCR_VX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (ve != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -211,7 +211,7 @@ static inline void float_zero_divide_excp(CPUPPCState *env)
     env->fpscr |= 1 << FPSCR_ZX;
     env->fpscr &= ~((1 << FPSCR_FR) | (1 << FPSCR_FI));
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_ze != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -228,7 +228,7 @@ static inline void float_overflow_excp(CPUPPCState *env)
 
     env->fpscr |= 1 << FPSCR_OX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_oe != 0) {
         /* XXX: should adjust the result */
         /* Update the floating-point enabled exception summary */
@@ -248,7 +248,7 @@ static inline void float_underflow_excp(CPUPPCState *env)
 
     env->fpscr |= 1 << FPSCR_UX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_ue != 0) {
         /* XXX: should adjust the result */
         /* Update the floating-point enabled exception summary */
@@ -265,7 +265,7 @@ static inline void float_inexact_excp(CPUPPCState *env)
 
     env->fpscr |= 1 << FPSCR_XX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_xe != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -330,31 +330,31 @@ void helper_fpscr_setbit(CPUPPCState *env, uint32_t bit)
     if (prev == 0) {
         switch (bit) {
         case FPSCR_VX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ve) {
                 goto raise_ve;
             }
             break;
         case FPSCR_OX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_oe) {
                 goto raise_oe;
             }
             break;
         case FPSCR_UX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ue) {
                 goto raise_ue;
             }
             break;
         case FPSCR_ZX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ze) {
                 goto raise_ze;
             }
             break;
         case FPSCR_XX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_xe) {
                 goto raise_xe;
             }
@@ -369,7 +369,7 @@ void helper_fpscr_setbit(CPUPPCState *env, uint32_t bit)
         case FPSCR_VXSQRT:
         case FPSCR_VXCVI:
             env->fpscr |= 1 << FPSCR_VX;
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ve != 0) {
                 goto raise_ve;
             }
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index a307a13de..acd327538 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -259,7 +259,8 @@ static void kvm_get_fallback_smmu_info(PowerPCCPU *cpu,
             info->flags |= KVM_PPC_1T_SEGMENTS;
         }
 
-        if (env->mmu_model == POWERPC_MMU_2_06) {
+        if (env->mmu_model == POWERPC_MMU_2_06 ||
+            env->mmu_model == POWERPC_MMU_2_07) {
             info->slb_size = 32;
         } else {
             info->slb_size = 64;
@@ -272,8 +273,9 @@ static void kvm_get_fallback_smmu_info(PowerPCCPU *cpu,
         info->sps[i].enc[0].pte_enc = 0;
         i++;
 
-        /* 64K on MMU 2.06 */
-        if (env->mmu_model == POWERPC_MMU_2_06) {
+        /* 64K on MMU 2.06 and later */
+        if (env->mmu_model == POWERPC_MMU_2_06 ||
+            env->mmu_model == POWERPC_MMU_2_07) {
             info->sps[i].page_shift = 16;
             info->sps[i].slb_enc = 0x110;
             info->sps[i].enc[0].page_shift = 16;
@@ -412,6 +414,13 @@ static void kvm_fixup_page_sizes(PowerPCCPU *cpu)
     /* Convert to QEMU form */
     memset(&env->sps, 0, sizeof(env->sps));
 
+    /* If we have HV KVM, we need to forbid CI large pages if our
+     * host page size is smaller than 64K.
+     */
+    if (smmu_info.flags & KVM_PPC_PAGE_SIZES_REAL) {
+        env->ci_large_pages = getpagesize() >= 0x10000;
+    }
+
     /*
      * XXX This loop should be an entry wide AND of the capabilities that
      *     the selected CPU has with the capabilities that KVM supports.
@@ -641,8 +650,13 @@ static int kvm_put_fp(CPUState *cs)
         for (i = 0; i < 32; i++) {
             uint64_t vsr[2];
 
+#ifdef HOST_WORDS_BIGENDIAN
             vsr[0] = float64_val(env->fpr[i]);
             vsr[1] = env->vsr[i];
+#else
+            vsr[0] = env->vsr[i];
+            vsr[1] = float64_val(env->fpr[i]);
+#endif
             reg.addr = (uintptr_t) &vsr;
             reg.id = vsx ? KVM_REG_PPC_VSR(i) : KVM_REG_PPC_FPR(i);
 
@@ -712,10 +726,17 @@ static int kvm_get_fp(CPUState *cs)
                         vsx ? "VSR" : "FPR", i, strerror(errno));
                 return ret;
             } else {
+#ifdef HOST_WORDS_BIGENDIAN
                 env->fpr[i] = vsr[0];
                 if (vsx) {
                     env->vsr[i] = vsr[1];
                 }
+#else
+                env->fpr[i] = vsr[1];
+                if (vsx) {
+                    env->vsr[i] = vsr[0];
+                }
+#endif
             }
         }
     }
@@ -1782,8 +1803,7 @@ uint32_t kvmppc_get_tbfreq(void)
 
     ns++;
 
-    retval = atoi(ns);
-    return retval;
+    return atoi(ns);
 }
 
 bool kvmppc_get_host_serial(char **value)
@@ -1953,6 +1973,11 @@ void kvmppc_enable_logical_ci_hcalls(void)
     kvmppc_enable_hcall(kvm_state, H_LOGICAL_CI_STORE);
 }
 
+void kvmppc_enable_set_mode_hcall(void)
+{
+    kvmppc_enable_hcall(kvm_state, H_SET_MODE);
+}
+
 void kvmppc_set_papr(PowerPCCPU *cpu)
 {
     CPUState *cs = CPU(cpu);
@@ -2066,7 +2091,7 @@ bool kvmppc_spapr_use_multitce(void)
 }
 
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
-                              bool vfio_accel)
+                              bool need_vfio)
 {
     struct kvm_create_spapr_tce args = {
         .liobn = liobn,
@@ -2080,7 +2105,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
      * destroying the table, which the upper layers -will- do
      */
     *pfd = -1;
-    if (!cap_spapr_tce || (vfio_accel && !cap_spapr_vfio)) {
+    if (!cap_spapr_tce || (need_vfio && !cap_spapr_vfio)) {
         return NULL;
     }
 
@@ -2479,7 +2504,7 @@ error_out:
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
@@ -2488,3 +2513,12 @@ int kvm_arch_msi_data_to_gsi(uint32_t data)
 {
     return data & 0xffff;
 }
+
+int kvmppc_enable_hwrng(void)
+{
+    if (!kvm_enabled() || !kvm_check_extension(kvm_state, KVM_CAP_PPC_HWRNG)) {
+        return -1;
+    }
+
+    return kvmppc_enable_hcall(kvm_state, H_RANDOM);
+}
diff --git a/target-ppc/kvm_ppc.c b/target-ppc/kvm_ppc.c
deleted file mode 100644
index f769acd44..000000000
--- a/target-ppc/kvm_ppc.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * PowerPC KVM support
- *
- * Copyright IBM Corp. 2008
- *
- * Authors:
- *  Hollis Blanchard <hollisb@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#include "qemu-common.h"
-#include "qemu/timer.h"
-#include "kvm_ppc.h"
-#include "sysemu/device_tree.h"
-#include "qemu/main-loop.h"
-
-#define PROC_DEVTREE_PATH "/proc/device-tree"
-
-static QEMUTimer *kvmppc_timer;
-static unsigned int kvmppc_timer_rate;
-
-static void kvmppc_timer_hack(void *opaque)
-{
-    qemu_notify_event();
-    timer_mod(kvmppc_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + kvmppc_timer_rate);
-}
-
-void kvmppc_init(void)
-{
-    /* XXX The only reason KVM yields control back to qemu is device IO. Since
-     * an idle guest does no IO, qemu's device model will never get a chance to
-     * run. So, until QEMU gains IO threads, we create this timer to ensure
-     * that the device model gets a chance to run. */
-    kvmppc_timer_rate = get_ticks_per_sec() / 10;
-    kvmppc_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, &kvmppc_timer_hack, NULL);
-    timer_mod(kvmppc_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + kvmppc_timer_rate);
-}
-
diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
index 4d30e2795..5e1333d99 100644
--- a/target-ppc/kvm_ppc.h
+++ b/target-ppc/kvm_ppc.h
@@ -11,8 +11,6 @@
 
 #define TYPE_HOST_POWERPC_CPU "host-" TYPE_POWERPC_CPU
 
-void kvmppc_init(void);
-
 #ifdef CONFIG_KVM
 
 uint32_t kvmppc_get_tbfreq(void);
@@ -25,6 +23,7 @@ int kvmppc_get_hasidle(CPUPPCState *env);
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len);
 int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level);
 void kvmppc_enable_logical_ci_hcalls(void);
+void kvmppc_enable_set_mode_hcall(void);
 void kvmppc_set_papr(PowerPCCPU *cpu);
 int kvmppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version);
 void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
@@ -37,7 +36,7 @@ int kvmppc_booke_watchdog_enable(PowerPCCPU *cpu);
 off_t kvmppc_alloc_rma(void **rma);
 bool kvmppc_spapr_use_multitce(void);
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
-                              bool vfio_accel);
+                              bool need_vfio);
 int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
 int kvmppc_reset_htab(int shift_hint);
 uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
@@ -55,6 +54,7 @@ void kvmppc_hash64_free_pteg(uint64_t token);
 void kvmppc_hash64_write_pte(CPUPPCState *env, target_ulong pte_index,
                              target_ulong pte0, target_ulong pte1);
 bool kvmppc_has_cap_fixup_hcalls(void);
+int kvmppc_enable_hwrng(void);
 
 #else
 
@@ -112,6 +112,10 @@ static inline void kvmppc_enable_logical_ci_hcalls(void)
 {
 }
 
+static inline void kvmppc_enable_set_mode_hcall(void)
+{
+}
+
 static inline void kvmppc_set_papr(PowerPCCPU *cpu)
 {
 }
@@ -176,7 +180,7 @@ static inline int kvmppc_remove_spapr_tce(void *table, int pfd,
 
 static inline int kvmppc_reset_htab(int shift_hint)
 {
-    return -1;
+    return 0;
 }
 
 static inline uint64_t kvmppc_rma_size(uint64_t current_size,
@@ -248,6 +252,10 @@ static inline bool kvmppc_has_cap_fixup_hcalls(void)
     abort();
 }
 
+static inline int kvmppc_enable_hwrng(void)
+{
+    return -1;
+}
 #endif
 
 #ifndef CONFIG_KVM
diff --git a/target-ppc/mem_helper.c b/target-ppc/mem_helper.c
index 6d37dae7b..7e1f234aa 100644
--- a/target-ppc/mem_helper.c
+++ b/target-ppc/mem_helper.c
@@ -100,8 +100,9 @@ void helper_lswx(CPUPPCState *env, target_ulong addr, uint32_t reg,
                  uint32_t ra, uint32_t rb)
 {
     if (likely(xer_bc != 0)) {
-        if (unlikely((ra != 0 && reg < ra && (reg + xer_bc) > ra) ||
-                     (reg < rb && (reg + xer_bc) > rb))) {
+        int num_used_regs = (xer_bc + 3) / 4;
+        if (unlikely((ra != 0 && reg < ra && (reg + num_used_regs) > ra) ||
+                     (reg < rb && (reg + num_used_regs) > rb))) {
             helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                        POWERPC_EXCP_INVAL |
                                        POWERPC_EXCP_INVAL_LSWX);
diff --git a/target-ppc/mmu_helper.c b/target-ppc/mmu_helper.c
index 527c6adca..30298d8d4 100644
--- a/target-ppc/mmu_helper.c
+++ b/target-ppc/mmu_helper.c
@@ -1293,9 +1293,11 @@ void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUPPCState *env)
         break;
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
         dump_slb(f, cpu_fprintf, env);
         break;
 #endif
@@ -1433,9 +1435,11 @@ hwaddr ppc_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     switch (env->mmu_model) {
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
         return ppc_hash64_get_phys_page_debug(env, addr);
 #endif
 
@@ -1937,9 +1941,11 @@ void ppc_tlb_invalidate_all(CPUPPCState *env)
     case POWERPC_MMU_601:
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
 #endif /* defined(TARGET_PPC64) */
         tlb_flush(CPU(cpu), 1);
         break;
@@ -2011,9 +2017,11 @@ void ppc_tlb_invalidate_one(CPUPPCState *env, target_ulong addr)
         break;
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
         /* tlbie invalidate TLBs for all segments */
         /* XXX: given the fact that there are too many segments to invalidate,
          *      and we still don't have a tlb_flush_mask(env, n, mask) in QEMU,
diff --git a/target-ppc/monitor.c b/target-ppc/monitor.c
new file mode 100644
index 000000000..bc571b87a
--- /dev/null
+++ b/target-ppc/monitor.c
@@ -0,0 +1,146 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+static target_long monitor_get_ccr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    unsigned int u;
+    int i;
+
+    u = 0;
+    for (i = 0; i < 8; i++)
+        u |= env->crf[i] << (32 - (4 * (i + 1)));
+
+    return u;
+}
+
+static target_long monitor_get_decr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_decr(env);
+}
+
+static target_long monitor_get_tbu (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_tbu(env);
+}
+
+static target_long monitor_get_tbl (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_tbl(env);
+}
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
+
+const MonitorDef monitor_defs[] = {
+    { "fpscr", offsetof(CPUPPCState, fpscr) },
+    /* Next instruction pointer */
+    { "nip|pc", offsetof(CPUPPCState, nip) },
+    { "lr", offsetof(CPUPPCState, lr) },
+    { "ctr", offsetof(CPUPPCState, ctr) },
+    { "decr", 0, &monitor_get_decr, },
+    { "ccr|cr", 0, &monitor_get_ccr, },
+    /* Machine state register */
+    { "xer", offsetof(CPUPPCState, xer) },
+    { "msr", offsetof(CPUPPCState, msr) },
+    { "tbu", 0, &monitor_get_tbu, },
+    { "tbl", 0, &monitor_get_tbl, },
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
+
+static int ppc_cpu_get_reg_num(const char *numstr, int maxnum, int *pregnum)
+{
+    int regnum;
+    char *endptr = NULL;
+
+    if (!*numstr) {
+        return false;
+    }
+
+    regnum = strtoul(numstr, &endptr, 10);
+    if (*endptr || (regnum >= maxnum)) {
+        return false;
+    }
+    *pregnum = regnum;
+
+    return true;
+}
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval)
+{
+    int i, regnum;
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    /* General purpose registers */
+    if ((tolower(name[0]) == 'r') &&
+        ppc_cpu_get_reg_num(name + 1, ARRAY_SIZE(env->gpr), &regnum)) {
+        *pval = env->gpr[regnum];
+        return 0;
+    }
+
+    /* Floating point registers */
+    if ((tolower(name[0]) == 'f') &&
+        ppc_cpu_get_reg_num(name + 1, ARRAY_SIZE(env->fpr), &regnum)) {
+        *pval = env->fpr[regnum];
+        return 0;
+    }
+
+    /* Special purpose registers */
+    for (i = 0; i < ARRAY_SIZE(env->spr_cb); ++i) {
+        ppc_spr_t *spr = &env->spr_cb[i];
+
+        if (spr->name && (strcasecmp(name, spr->name) == 0)) {
+            *pval = env->spr[i];
+            return 0;
+        }
+    }
+
+    /* Segment registers */
+#if !defined(CONFIG_USER_ONLY)
+    if ((strncasecmp(name, "sr", 2) == 0) &&
+        ppc_cpu_get_reg_num(name + 2, ARRAY_SIZE(env->sr), &regnum)) {
+        *pval = env->sr[regnum];
+        return 0;
+    }
+#endif
+
+    return -EINVAL;
+}
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index c0eed13f9..41a725848 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -9884,7 +9884,7 @@ GEN_HANDLER(mtcrf, 0x1F, 0x10, 0x04, 0x00000801, PPC_MISC),
 GEN_HANDLER(mtmsrd, 0x1F, 0x12, 0x05, 0x001EF801, PPC_64B),
 #endif
 GEN_HANDLER(mtmsr, 0x1F, 0x12, 0x04, 0x001FF801, PPC_MISC),
-GEN_HANDLER(mtspr, 0x1F, 0x13, 0x0E, 0x00000001, PPC_MISC),
+GEN_HANDLER(mtspr, 0x1F, 0x13, 0x0E, 0x00000000, PPC_MISC),
 GEN_HANDLER(dcbf, 0x1F, 0x16, 0x02, 0x03C00001, PPC_CACHE),
 GEN_HANDLER(dcbi, 0x1F, 0x16, 0x0E, 0x03E00001, PPC_CACHE),
 GEN_HANDLER(dcbst, 0x1F, 0x16, 0x01, 0x03E00001, PPC_CACHE),
@@ -11327,9 +11327,9 @@ void ppc_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     case POWERPC_MMU_SOFT_74xx:
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
-    case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
 #endif
         cpu_fprintf(f, " SDR1 " TARGET_FMT_lx "   DAR " TARGET_FMT_lx
                        "  DSISR " TARGET_FMT_lx "\n", env->spr[SPR_SDR1],
@@ -11409,17 +11409,13 @@ void ppc_cpu_dump_statistics(CPUState *cs, FILE*f,
 }
 
 /*****************************************************************************/
-static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUPPCState *env, struct TranslationBlock *tb)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUPPCState *env = &cpu->env;
     DisasContext ctx, *ctxp = &ctx;
     opc_handler_t **table, *handler;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
 
@@ -11476,36 +11472,34 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
 #endif
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     tcg_clear_temp_count();
     /* Set env in case of segfault during code fetch */
     while (ctx.exception == POWERPC_EXCP_NONE && !tcg_op_buf_full()) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.nip) {
-                    gen_debug_exception(ctxp);
-                    break;
-                }
-            }
-        }
-        if (unlikely(search_pc)) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.nip;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(ctx.nip);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.nip, BP_ANY))) {
+            gen_debug_exception(ctxp);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.nip += 4;
+            break;
         }
+
         LOG_DISAS("----------------\n");
         LOG_DISAS("nip=" TARGET_FMT_lx " super=%d ir=%d\n",
                   ctx.nip, ctx.mem_idx, (int)msr_ir);
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
         if (unlikely(need_byteswap(&ctx))) {
             ctx.opcode = bswap32(cpu_ldl_code(env, ctx.nip));
@@ -11515,12 +11509,8 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
         LOG_DISAS("translate opcode %08x (%02x %02x %02x) (%s)\n",
                     ctx.opcode, opc1(ctx.opcode), opc2(ctx.opcode),
                     opc3(ctx.opcode), ctx.le_mode ? "little" : "big");
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(ctx.nip);
-        }
         ctx.nip += 4;
         table = env->opcodes;
-        num_insns++;
         handler = table[opc1(ctx.opcode)];
         if (is_indirect_opcode(handler)) {
             table = ind_table(handler);
@@ -11599,15 +11589,9 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
     }
     gen_tb_end(tb, num_insns);
 
-    if (unlikely(search_pc)) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = ctx.nip - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.nip - pc_start;
+    tb->icount = num_insns;
+
 #if defined(DEBUG_DISAS)
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         int flags;
@@ -11620,17 +11604,8 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code (CPUPPCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(ppc_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUPPCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(ppc_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->nip = tcg_ctx.gen_opc_pc[pc_pos];
+    env->nip = data[0];
 }
diff --git a/target-ppc/translate_init.c b/target-ppc/translate_init.c
index 16d7b16ac..e88dc7fc7 100644
--- a/target-ppc/translate_init.c
+++ b/target-ppc/translate_init.c
@@ -305,7 +305,7 @@ static void spr_read_ibat (DisasContext *ctx, int gprn, int sprn)
 
 static void spr_read_ibat_h (DisasContext *ctx, int gprn, int sprn)
 {
-    tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][(sprn - SPR_IBAT4U) / 2]));
+    tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][((sprn - SPR_IBAT4U) / 2) + 4]));
 }
 
 static void spr_write_ibatu (DisasContext *ctx, int sprn, int gprn)
@@ -7864,6 +7864,7 @@ static void init_proc_book3s_64(CPUPPCState *env, int version)
         gen_spr_book3s_ids(env);
         gen_spr_amr(env);
         gen_spr_book3s_purr(env);
+        env->ci_large_pages = true;
         break;
     default:
         g_assert_not_reached();
@@ -8019,7 +8020,7 @@ POWERPC_FAMILY(POWER5P)(ObjectClass *oc, void *data)
                     (1ull << MSR_DR) |
                     (1ull << MSR_PMM) |
                     (1ull << MSR_RI);
-    pcc->mmu_model = POWERPC_MMU_64B;
+    pcc->mmu_model = POWERPC_MMU_2_03;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
 #endif
@@ -8243,7 +8244,7 @@ POWERPC_FAMILY(POWER8)(ObjectClass *oc, void *data)
                     (1ull << MSR_PMM) |
                     (1ull << MSR_RI) |
                     (1ull << MSR_LE);
-    pcc->mmu_model = POWERPC_MMU_2_06;
+    pcc->mmu_model = POWERPC_MMU_2_07;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
 #endif
@@ -8577,11 +8578,7 @@ static int register_ind_insn (opc_handler_t **ppc_opcodes,
                               unsigned char idx1, unsigned char idx2,
                               opc_handler_t *handler)
 {
-    int ret;
-
-    ret = register_ind_in_table(ppc_opcodes, idx1, idx2, handler);
-
-    return ret;
+    return register_ind_in_table(ppc_opcodes, idx1, idx2, handler);
 }
 
 static int register_dblind_insn (opc_handler_t **ppc_opcodes,
@@ -9350,8 +9347,7 @@ static ObjectClass *ppc_cpu_class_by_name(const char *name)
                 break;
         }
         if (i == 8) {
-            ret = OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
-            return ret;
+            return OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
         }
     }
 
diff --git a/target-s390x/cpu.c b/target-s390x/cpu.c
index ccfaa8a91..189a2afc0 100644
--- a/target-s390x/cpu.c
+++ b/target-s390x/cpu.c
@@ -184,6 +184,12 @@ static void s390_cpu_machine_reset_cb(void *opaque)
 }
 #endif
 
+static void s390_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_s390_64;
+    info->print_insn = print_insn_s390;
+}
+
 static void s390_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -351,6 +357,8 @@ static void s390_cpu_class_init(ObjectClass *oc, void *data)
     cc->cpu_exec_interrupt = s390_cpu_exec_interrupt;
     cc->debug_excp_handler = s390x_cpu_debug_excp_handler;
 #endif
+    cc->disas_set_info = s390_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = S390_NUM_CORE_REGS;
     cc->gdb_core_xml_file = "s390x-core64.xml";
 
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 63aebf484..658cd9d55 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -27,7 +27,6 @@
 
 #define TARGET_LONG_BITS 64
 
-#define ELF_MACHINE	EM_S390
 #define ELF_MACHINE_UNAME "S390X"
 
 #define CPUArchState struct CPUS390XState
@@ -43,6 +42,7 @@
 #include "fpu/softfloat.h"
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 #define MMU_MODE0_SUFFIX _primary
 #define MMU_MODE1_SUFFIX _secondary
@@ -143,8 +143,6 @@ typedef struct CPUS390XState {
     uint32_t cpu_num;
     uint32_t machine_type;
 
-    uint8_t *storage_keys;
-
     uint64_t tod_offset;
     uint64_t tod_basetime;
     QEMUTimer *tod_timer;
@@ -310,7 +308,7 @@ static inline CPU_DoubleU *get_freg(CPUS390XState *cs, int nr)
 #define MMU_SECONDARY_IDX       1
 #define MMU_HOME_IDX            2
 
-static inline int cpu_mmu_index (CPUS390XState *env)
+static inline int cpu_mmu_index (CPUS390XState *env, bool ifetch)
 {
     switch (env->psw.mask & PSW_MASK_ASC) {
     case PSW_ASC_PRIMARY:
@@ -571,7 +569,7 @@ void cpu_unlock(void);
 typedef struct SubchDev SubchDev;
 
 #ifndef CONFIG_USER_ONLY
-extern void io_subsystem_reset(void);
+extern void subsystem_reset(void);
 SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
                          uint16_t schid);
 bool css_subch_visible(SubchDev *sch);
@@ -600,7 +598,6 @@ bool css_present(uint8_t cssid);
 
 #define cpu_init(model) CPU(cpu_s390x_init(model))
 #define cpu_exec cpu_s390x_exec
-#define cpu_gen_code cpu_s390x_gen_code
 #define cpu_signal_handler cpu_s390x_signal_handler
 
 void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf);
@@ -1163,12 +1160,13 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
 int kvm_s390_get_memslot_count(KVMState *s);
-void kvm_s390_clear_cmma_callback(void *opaque);
+void kvm_s390_cmma_reset(void);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_reset_vcpu(S390CPU *cpu);
 int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
 int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
+void kvm_s390_crypto_reset(void);
 #else
 static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
                                         uint16_t subchannel_nr,
@@ -1192,7 +1190,7 @@ static inline int kvm_s390_cpu_restart(S390CPU *cpu)
 {
     return -ENOSYS;
 }
-static inline void kvm_s390_clear_cmma_callback(void *opaque)
+static inline void kvm_s390_cmma_reset(void)
 {
 }
 static inline int kvm_s390_get_memslot_count(KVMState *s)
@@ -1218,6 +1216,9 @@ static inline int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
 {
     return 0;
 }
+static inline void kvm_s390_crypto_reset(void)
+{
+}
 #endif
 
 static inline int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit)
@@ -1228,11 +1229,10 @@ static inline int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit)
     return 0;
 }
 
-static inline void cmma_reset(S390CPU *cpu)
+static inline void s390_cmma_reset(void)
 {
     if (kvm_enabled()) {
-        CPUState *cs = CPU(cpu);
-        kvm_s390_clear_cmma_callback(cs->kvm_state);
+        kvm_s390_cmma_reset();
     }
 }
 
@@ -1264,6 +1264,13 @@ static inline int s390_assign_subch_ioeventfd(EventNotifier *notifier,
     return kvm_s390_assign_subch_ioeventfd(notifier, sch_id, vq, assign);
 }
 
+static inline void s390_crypto_reset(void)
+{
+    if (kvm_enabled()) {
+        kvm_s390_crypto_reset();
+    }
+}
+
 #ifdef CONFIG_KVM
 static inline bool vregs_needed(void *opaque)
 {
@@ -1278,4 +1285,49 @@ static inline bool vregs_needed(void *opaque)
     return 0;
 }
 #endif
+
+/* machine check interruption code */
+
+/* subclasses */
+#define MCIC_SC_SD 0x8000000000000000ULL
+#define MCIC_SC_PD 0x4000000000000000ULL
+#define MCIC_SC_SR 0x2000000000000000ULL
+#define MCIC_SC_CD 0x0800000000000000ULL
+#define MCIC_SC_ED 0x0400000000000000ULL
+#define MCIC_SC_DG 0x0100000000000000ULL
+#define MCIC_SC_W  0x0080000000000000ULL
+#define MCIC_SC_CP 0x0040000000000000ULL
+#define MCIC_SC_SP 0x0020000000000000ULL
+#define MCIC_SC_CK 0x0010000000000000ULL
+
+/* subclass modifiers */
+#define MCIC_SCM_B  0x0002000000000000ULL
+#define MCIC_SCM_DA 0x0000000020000000ULL
+#define MCIC_SCM_AP 0x0000000000080000ULL
+
+/* storage errors */
+#define MCIC_SE_SE 0x0000800000000000ULL
+#define MCIC_SE_SC 0x0000400000000000ULL
+#define MCIC_SE_KE 0x0000200000000000ULL
+#define MCIC_SE_DS 0x0000100000000000ULL
+#define MCIC_SE_IE 0x0000000080000000ULL
+
+/* validity bits */
+#define MCIC_VB_WP 0x0000080000000000ULL
+#define MCIC_VB_MS 0x0000040000000000ULL
+#define MCIC_VB_PM 0x0000020000000000ULL
+#define MCIC_VB_IA 0x0000010000000000ULL
+#define MCIC_VB_FA 0x0000008000000000ULL
+#define MCIC_VB_VR 0x0000004000000000ULL
+#define MCIC_VB_EC 0x0000002000000000ULL
+#define MCIC_VB_FP 0x0000001000000000ULL
+#define MCIC_VB_GR 0x0000000800000000ULL
+#define MCIC_VB_CR 0x0000000400000000ULL
+#define MCIC_VB_ST 0x0000000100000000ULL
+#define MCIC_VB_AR 0x0000000040000000ULL
+#define MCIC_VB_PR 0x0000000000200000ULL
+#define MCIC_VB_FC 0x0000000000100000ULL
+#define MCIC_VB_CT 0x0000000000020000ULL
+#define MCIC_VB_CC 0x0000000000010000ULL
+
 #endif
diff --git a/target-s390x/gdbstub.c b/target-s390x/gdbstub.c
index 31f204964..a05d1cd7a 100644
--- a/target-s390x/gdbstub.c
+++ b/target-s390x/gdbstub.c
@@ -174,6 +174,115 @@ static int cpu_write_vreg(CPUS390XState *env, uint8_t *mem_buf, int n)
     }
 }
 
+/* the values represent the positions in s390-cr.xml */
+#define S390_C0_REGNUM 0
+#define S390_C15_REGNUM 15
+/* total number of registers in s390-cr.xml */
+#define S390_NUM_C_REGS 16
+
+#ifndef CONFIG_USER_ONLY
+static int cpu_read_c_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_C0_REGNUM ... S390_C15_REGNUM:
+        return gdb_get_regl(mem_buf, env->cregs[n]);
+    default:
+        return 0;
+    }
+}
+
+static int cpu_write_c_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_C0_REGNUM ... S390_C15_REGNUM:
+        env->cregs[n] = ldtul_p(mem_buf);
+        if (tcg_enabled()) {
+            tlb_flush(ENV_GET_CPU(env), 1);
+        }
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    default:
+        return 0;
+    }
+}
+
+/* the values represent the positions in s390-virt.xml */
+#define S390_VIRT_CKC_REGNUM    0
+#define S390_VIRT_CPUTM_REGNUM  1
+#define S390_VIRT_BEA_REGNUM    2
+#define S390_VIRT_PREFIX_REGNUM 3
+#define S390_VIRT_PP_REGNUM     4
+#define S390_VIRT_PFT_REGNUM    5
+#define S390_VIRT_PFS_REGNUM    6
+#define S390_VIRT_PFC_REGNUM    7
+/* total number of registers in s390-virt.xml */
+#define S390_NUM_VIRT_REGS 8
+
+static int cpu_read_virt_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_VIRT_CKC_REGNUM:
+        return gdb_get_regl(mem_buf, env->ckc);
+    case S390_VIRT_CPUTM_REGNUM:
+        return gdb_get_regl(mem_buf, env->cputm);
+    case S390_VIRT_BEA_REGNUM:
+        return gdb_get_regl(mem_buf, env->gbea);
+    case S390_VIRT_PREFIX_REGNUM:
+        return gdb_get_regl(mem_buf, env->psa);
+    case S390_VIRT_PP_REGNUM:
+        return gdb_get_regl(mem_buf, env->pp);
+    case S390_VIRT_PFT_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_token);
+    case S390_VIRT_PFS_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_select);
+    case S390_VIRT_PFC_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_compare);
+    default:
+        return 0;
+    }
+}
+
+static int cpu_write_virt_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_VIRT_CKC_REGNUM:
+        env->ckc = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_CPUTM_REGNUM:
+        env->cputm = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_BEA_REGNUM:
+        env->gbea = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PREFIX_REGNUM:
+        env->psa = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PP_REGNUM:
+        env->pp = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFT_REGNUM:
+        env->pfault_token = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFS_REGNUM:
+        env->pfault_select = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFC_REGNUM:
+        env->pfault_compare = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    default:
+        return 0;
+    }
+}
+#endif
+
 void s390_cpu_gdb_init(CPUState *cs)
 {
     gdb_register_coprocessor(cs, cpu_read_ac_reg,
@@ -187,4 +296,16 @@ void s390_cpu_gdb_init(CPUState *cs)
     gdb_register_coprocessor(cs, cpu_read_vreg,
                              cpu_write_vreg,
                              S390_NUM_VREGS, "s390-vx.xml", 0);
+
+#ifndef CONFIG_USER_ONLY
+    gdb_register_coprocessor(cs, cpu_read_c_reg,
+                             cpu_write_c_reg,
+                             S390_NUM_C_REGS, "s390-cr.xml", 0);
+
+    if (kvm_enabled()) {
+        gdb_register_coprocessor(cs, cpu_read_virt_reg,
+                                 cpu_write_virt_reg,
+                                 S390_NUM_VIRT_REGS, "s390-virt.xml", 0);
+    }
+#endif
 }
diff --git a/target-s390x/ioinst.c b/target-s390x/ioinst.c
index 77f2a1fb9..57c2d8b22 100644
--- a/target-s390x/ioinst.c
+++ b/target-s390x/ioinst.c
@@ -616,7 +616,8 @@ static void ioinst_handle_chsc_sei(ChscReq *req, ChscResp *res)
             (*res_flags) &= ~0x80;
         }
     } else {
-        res->code = cpu_to_be16(0x0004);
+        res->code = cpu_to_be16(0x0005);
+        res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
     }
 }
 
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 6d1f284da..75a0e5d1c 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -173,16 +173,15 @@ int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit)
     return kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
 }
 
-void kvm_s390_clear_cmma_callback(void *opaque)
+void kvm_s390_cmma_reset(void)
 {
     int rc;
-    KVMState *s = opaque;
     struct kvm_device_attr attr = {
         .group = KVM_S390_VM_MEM_CTRL,
         .attr = KVM_S390_VM_MEM_CLR_CMMA,
     };
 
-    rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
+    rc = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
     trace_kvm_clear_cmma(rc);
 }
 
@@ -200,9 +199,6 @@ static void kvm_s390_enable_cmma(KVMState *s)
     }
 
     rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
-    if (!rc) {
-        qemu_register_reset(kvm_s390_clear_cmma_callback, s);
-    }
     trace_kvm_enable_cmma(rc);
 }
 
@@ -249,7 +245,7 @@ static void kvm_s390_init_dea_kw(void)
     }
 }
 
-static void kvm_s390_init_crypto(void)
+void kvm_s390_crypto_reset(void)
 {
     kvm_s390_init_aes_kw();
     kvm_s390_init_dea_kw();
@@ -262,7 +258,9 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
 
-    kvm_s390_enable_cmma(s);
+    if (!mem_path) {
+        kvm_s390_enable_cmma(s);
+    }
 
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
@@ -301,8 +299,6 @@ void kvm_s390_reset_vcpu(S390CPU *cpu)
     if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
         error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
     }
-
-    kvm_s390_init_crypto();
 }
 
 static int can_sync_regs(CPUState *cs, int regs)
@@ -588,9 +584,9 @@ int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
  * @addr:      the logical start address in guest memory
  * @ar:        the access register number
  * @hostbuf:   buffer in host memory. NULL = do only checks w/o copying
- * @len:       length that should be transfered
+ * @len:       length that should be transferred
  * @is_write:  true = write, false = read
- * Returns:    0 on success, non-zero if an exception or error occured
+ * Returns:    0 on success, non-zero if an exception or error occurred
  *
  * Use KVM ioctl to read/write from/to guest memory. An access exception
  * is injected into the vCPU in case of translation errors.
@@ -1796,13 +1792,6 @@ static bool is_special_wait_psw(CPUState *cs)
     return cs->kvm_run->psw_addr == 0xfffUL;
 }
 
-static void guest_panicked(void)
-{
-    qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE,
-                                   &error_abort);
-    vm_stop(RUN_STATE_GUEST_PANICKED);
-}
-
 static void unmanageable_intercept(S390CPU *cpu, const char *str, int pswoffset)
 {
     CPUState *cs = CPU(cpu);
@@ -1811,7 +1800,7 @@ static void unmanageable_intercept(S390CPU *cpu, const char *str, int pswoffset)
                  str, cs->cpu_index, ldq_phys(cs->as, cpu->env.psa + pswoffset),
                  ldq_phys(cs->as, cpu->env.psa + pswoffset + 8));
     s390_cpu_halt(cpu);
-    guest_panicked();
+    qemu_system_guest_panicked();
 }
 
 static int handle_intercept(S390CPU *cpu)
@@ -1844,7 +1833,7 @@ static int handle_intercept(S390CPU *cpu)
                 if (is_special_wait_psw(cs)) {
                     qemu_system_shutdown_request();
                 } else {
-                    guest_panicked();
+                    qemu_system_guest_panicked();
                 }
             }
             r = EXCP_HALTED;
@@ -2072,16 +2061,31 @@ void kvm_s390_io_interrupt(uint16_t subchannel_id,
     kvm_s390_floating_interrupt(&irq);
 }
 
+static uint64_t build_channel_report_mcic(void)
+{
+    uint64_t mcic;
+
+    /* subclass: indicate channel report pending */
+    mcic = MCIC_SC_CP |
+    /* subclass modifiers: none */
+    /* storage errors: none */
+    /* validity bits: no damage */
+        MCIC_VB_WP | MCIC_VB_MS | MCIC_VB_PM | MCIC_VB_IA | MCIC_VB_FP |
+        MCIC_VB_GR | MCIC_VB_CR | MCIC_VB_ST | MCIC_VB_AR | MCIC_VB_PR |
+        MCIC_VB_FC | MCIC_VB_CT | MCIC_VB_CC;
+    if (kvm_check_extension(kvm_state, KVM_CAP_S390_VECTOR_REGISTERS)) {
+        mcic |= MCIC_VB_VR;
+    }
+    return mcic;
+}
+
 void kvm_s390_crw_mchk(void)
 {
     struct kvm_s390_irq irq = {
         .type = KVM_S390_MCHK,
         .u.mchk.cr14 = 1 << 28,
-        .u.mchk.mcic = 0x00400f1d40330000ULL,
+        .u.mchk.mcic = build_channel_report_mcic(),
     };
-    if (kvm_check_extension(kvm_state, KVM_CAP_S390_VECTOR_REGISTERS)) {
-        irq.u.mchk.mcic |= 0x0000004000000000ULL;
-    }
     kvm_s390_floating_interrupt(&irq);
 }
 
@@ -2218,7 +2222,7 @@ int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                              uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     S390PCIBusDevice *pbdev;
     uint32_t fid = data >> ZPCI_MSI_VEC_BITS;
diff --git a/target-s390x/mem_helper.c b/target-s390x/mem_helper.c
index 6f8bd796a..90399f1b0 100644
--- a/target-s390x/mem_helper.c
+++ b/target-s390x/mem_helper.c
@@ -21,6 +21,7 @@
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
+#include "hw/s390x/storage-keys.h"
 
 /*****************************************************************************/
 /* Softmmu support */
@@ -68,7 +69,7 @@ static inline uint64_t adj_len_to_page(uint64_t len, uint64_t addr)
 static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
                         uint32_t l)
 {
-    int mmu_idx = cpu_mmu_index(env);
+    int mmu_idx = cpu_mmu_index(env, false);
 
     while (l > 0) {
         void *p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
@@ -91,7 +92,7 @@ static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
 static void fast_memmove(CPUS390XState *env, uint64_t dest, uint64_t src,
                          uint32_t l)
 {
-    int mmu_idx = cpu_mmu_index(env);
+    int mmu_idx = cpu_mmu_index(env, false);
 
     while (l > 0) {
         void *src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, mmu_idx);
@@ -937,40 +938,73 @@ uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
 /* insert storage key extended */
 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     uint64_t addr = get_address(env, 0, 0, r2);
+    uint8_t key;
 
     if (addr > ram_size) {
         return 0;
     }
 
-    return env->storage_keys[addr / TARGET_PAGE_SIZE];
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    if (skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
+    return key;
 }
 
 /* set storage key extended */
 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     uint64_t addr = get_address(env, 0, 0, r2);
+    uint8_t key;
 
     if (addr > ram_size) {
         return;
     }
 
-    env->storage_keys[addr / TARGET_PAGE_SIZE] = r1;
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    key = (uint8_t) r1;
+    skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
 }
 
 /* reset reference bit extended */
 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
 {
-    uint8_t re;
-    uint8_t key;
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
+    uint8_t re, key;
 
     if (r2 > ram_size) {
         return 0;
     }
 
-    key = env->storage_keys[r2 / TARGET_PAGE_SIZE];
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    if (skeyclass->get_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
+
     re = key & (SK_R | SK_C);
-    env->storage_keys[r2 / TARGET_PAGE_SIZE] = (key & ~SK_R);
+    key &= ~SK_R;
+
+    if (skeyclass->set_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
 
     /*
      * cc
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
index 8eac0e12b..b601a3360 100644
--- a/target-s390x/misc_helper.c
+++ b/target-s390x/misc_helper.c
@@ -127,8 +127,9 @@ static int modified_clear_reset(S390CPU *cpu)
     CPU_FOREACH(t) {
         run_on_cpu(t, s390_do_cpu_full_reset, t);
     }
-    cmma_reset(cpu);
-    io_subsystem_reset();
+    s390_cmma_reset();
+    subsystem_reset();
+    s390_crypto_reset();
     scc->load_normal(CPU(cpu));
     cpu_synchronize_all_post_reset();
     resume_all_vcpus();
@@ -145,8 +146,8 @@ static int load_normal_reset(S390CPU *cpu)
     CPU_FOREACH(t) {
         run_on_cpu(t, s390_do_cpu_reset, t);
     }
-    cmma_reset(cpu);
-    io_subsystem_reset();
+    s390_cmma_reset();
+    subsystem_reset();
     scc->initial_cpu_reset(CPU(cpu));
     scc->load_normal(CPU(cpu));
     cpu_synchronize_all_post_reset();
@@ -233,11 +234,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3)
         }
         iplb = g_malloc0(sizeof(struct IplParameterBlock));
         cpu_physical_memory_read(addr, iplb, sizeof(struct IplParameterBlock));
-        if (!s390_ipl_update_diag308(iplb)) {
-            env->regs[r1 + 1] = DIAG_308_RC_OK;
-        } else {
-            env->regs[r1 + 1] = DIAG_308_RC_INVALID;
-        }
+        s390_ipl_update_diag308(iplb);
+        env->regs[r1 + 1] = DIAG_308_RC_OK;
         g_free(iplb);
         return;
     case 6:
diff --git a/target-s390x/mmu_helper.c b/target-s390x/mmu_helper.c
index 1ea6d812c..058a37013 100644
--- a/target-s390x/mmu_helper.c
+++ b/target-s390x/mmu_helper.c
@@ -19,6 +19,8 @@
 #include "exec/address-spaces.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
+#include "trace.h"
+#include "hw/s390x/storage-keys.h"
 
 /* #define DEBUG_S390 */
 /* #define DEBUG_S390_PTE */
@@ -309,8 +311,15 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr,
 int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
                   target_ulong *raddr, int *flags, bool exc)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     int r = -1;
-    uint8_t *sk;
+    uint8_t key;
+
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
 
     *flags = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
     vaddr &= TARGET_PAGE_MASK;
@@ -358,14 +367,23 @@ int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
     /* Convert real address -> absolute address */
     *raddr = mmu_real2abs(env, *raddr);
 
-    if (*raddr < ram_size) {
-        sk = &env->storage_keys[*raddr / TARGET_PAGE_SIZE];
+    if (r == 0 && *raddr < ram_size) {
+        if (skeyclass->get_skeys(ss, *raddr / TARGET_PAGE_SIZE, 1, &key)) {
+            trace_get_skeys_nonzero(r);
+            return 0;
+        }
+
         if (*flags & PAGE_READ) {
-            *sk |= SK_R;
+            key |= SK_R;
         }
 
         if (*flags & PAGE_WRITE) {
-            *sk |= SK_C;
+            key |= SK_C;
+        }
+
+        if (skeyclass->set_skeys(ss, *raddr / TARGET_PAGE_SIZE, 1, &key)) {
+            trace_set_skeys_nonzero(r);
+            return 0;
         }
     }
 
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index c748290d5..c79a2cb57 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -124,7 +124,7 @@ void s390_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     for (i = 0; i < 32; i++) {
         cpu_fprintf(f, "V%02d=%016" PRIx64 "%016" PRIx64, i,
                     env->vregs[i][0].ll, env->vregs[i][1].ll);
-        cpu_fprintf(f, (i % 2) ? " " : "\n");
+        cpu_fprintf(f, (i % 2) ? "\n" : " ");
     }
 
 #ifndef CONFIG_USER_ONLY
@@ -161,8 +161,6 @@ static char cpu_reg_names[32][4];
 static TCGv_i64 regs[16];
 static TCGv_i64 fregs[16];
 
-static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
-
 void s390x_translate_init(void)
 {
     int i;
@@ -811,7 +809,7 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
     case CC_OP_LTGT0_32:
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_dst);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_dst);
         c->u.s32.b = tcg_const_i32(0);
         break;
     case CC_OP_LTGT_32:
@@ -819,9 +817,9 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
     case CC_OP_SUBU_32:
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_src);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_src);
         c->u.s32.b = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.b, cc_dst);
+        tcg_gen_extrl_i64_i32(c->u.s32.b, cc_dst);
         break;
 
     case CC_OP_LTGT0_64:
@@ -851,11 +849,11 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
         c->u.s32.b = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_vr);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_vr);
         if (cond == TCG_COND_EQ || cond == TCG_COND_NE) {
             tcg_gen_movi_i32(c->u.s32.b, 0);
         } else {
-            tcg_gen_trunc_i64_i32(c->u.s32.b, cc_src);
+            tcg_gen_extrl_i64_i32(c->u.s32.b, cc_src);
         }
         break;
 
@@ -1532,7 +1530,7 @@ static ExitStatus op_bct32(DisasContext *s, DisasOps *o)
     store_reg32_i64(r1, t);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_const_i32(0);
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
     tcg_temp_free_i64(t);
 
     return help_branch(s, &c, is_imm, imm, o->in2);
@@ -1556,7 +1554,7 @@ static ExitStatus op_bcth(DisasContext *s, DisasOps *o)
     store_reg32h_i64(r1, t);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_const_i32(0);
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
     tcg_temp_free_i64(t);
 
     return help_branch(s, &c, 1, imm, o->in2);
@@ -1599,8 +1597,8 @@ static ExitStatus op_bx32(DisasContext *s, DisasOps *o)
     tcg_gen_add_i64(t, regs[r1], regs[r3]);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
-    tcg_gen_trunc_i64_i32(c.u.s32.b, regs[r3 | 1]);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.b, regs[r3 | 1]);
     store_reg32_i64(r1, t);
     tcg_temp_free_i64(t);
 
@@ -1905,7 +1903,7 @@ static ExitStatus op_clm(DisasContext *s, DisasOps *o)
 {
     TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
     TCGv_i32 t1 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t1, o->in1);
+    tcg_gen_extrl_i64_i32(t1, o->in1);
     potential_page_fault(s);
     gen_helper_clm(cc_op, cpu_env, t1, m3, o->in2);
     set_cc_static(s);
@@ -1977,7 +1975,7 @@ static ExitStatus op_cs(DisasContext *s, DisasOps *o)
 
     /* Store CC back to cc_op.  Wait until after the store so that any
        exception gets the old cc_op value.  */
-    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_gen_extrl_i64_i32(cc_op, cc);
     tcg_temp_free_i64(cc);
     set_cc_static(s);
     return NO_EXIT;
@@ -2027,7 +2025,7 @@ static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
     /* Save back state now that we've passed all exceptions.  */
     tcg_gen_mov_i64(regs[r1], outh);
     tcg_gen_mov_i64(regs[r1 + 1], outl);
-    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_gen_extrl_i64_i32(cc_op, cc);
     tcg_temp_free_i64(outh);
     tcg_temp_free_i64(outl);
     tcg_temp_free_i64(cc);
@@ -2051,7 +2049,7 @@ static ExitStatus op_cvd(DisasContext *s, DisasOps *o)
 {
     TCGv_i64 t1 = tcg_temp_new_i64();
     TCGv_i32 t2 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t2, o->in1);
+    tcg_gen_extrl_i64_i32(t2, o->in1);
     gen_helper_cvd(t1, t2);
     tcg_temp_free_i32(t2);
     tcg_gen_qemu_st64(t1, o->in2, get_mem_index(s));
@@ -3235,8 +3233,8 @@ static ExitStatus op_rll32(DisasContext *s, DisasOps *o)
     TCGv_i32 t1 = tcg_temp_new_i32();
     TCGv_i32 t2 = tcg_temp_new_i32();
     TCGv_i32 to = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t1, o->in1);
-    tcg_gen_trunc_i64_i32(t2, o->in2);
+    tcg_gen_extrl_i64_i32(t1, o->in1);
+    tcg_gen_extrl_i64_i32(t2, o->in2);
     tcg_gen_rotl_i32(to, t1, t2);
     tcg_gen_extu_i32_i64(o->out, to);
     tcg_temp_free_i32(t1);
@@ -5319,18 +5317,14 @@ static ExitStatus translate_one(CPUS390XState *env, DisasContext *s)
     return ret;
 }
 
-static inline void gen_intermediate_code_internal(S390CPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUS390XState *env, struct TranslationBlock *tb)
 {
+    S390CPU *cpu = s390_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUS390XState *env = &cpu->env;
     DisasContext dc;
     target_ulong pc_start;
     uint64_t next_page_start;
-    int j, lj = -1;
     int num_insns, max_insns;
-    CPUBreakpoint *bp;
     ExitStatus status;
     bool do_debug;
 
@@ -5353,41 +5347,32 @@ static inline void gen_intermediate_code_internal(S390CPU *cpu,
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
     do {
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc.pc;
-            gen_opc_cc_op[lj] = dc.cc_op;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (++num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
-            gen_io_start();
+        tcg_gen_insn_start(dc.pc, dc.cc_op);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc.pc, BP_ANY))) {
+            status = EXIT_PC_STALE;
+            do_debug = true;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc.pc += 2;
+            break;
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc.pc);
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
+            gen_io_start();
         }
 
         status = NO_EXIT;
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc.pc) {
-                    status = EXIT_PC_STALE;
-                    do_debug = true;
-                    break;
-                }
-            }
-        }
         if (status == NO_EXIT) {
             status = translate_one(env, &dc);
         }
@@ -5432,16 +5417,8 @@ static inline void gen_intermediate_code_internal(S390CPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc.pc - pc_start;
+    tb->icount = num_insns;
 
 #if defined(S390X_DEBUG_DISAS)
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -5452,21 +5429,11 @@ static inline void gen_intermediate_code_internal(S390CPU *cpu,
 #endif
 }
 
-void gen_intermediate_code (CPUS390XState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(s390_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUS390XState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(s390_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    int cc_op;
-    env->psw.addr = tcg_ctx.gen_opc_pc[pc_pos];
-    cc_op = gen_opc_cc_op[pc_pos];
+    int cc_op = data[1];
+    env->psw.addr = data[0];
     if ((cc_op != CC_OP_DYNAMIC) && (cc_op != CC_OP_STATIC)) {
         env->cc_op = cc_op;
     }
diff --git a/target-sh4/Makefile.objs b/target-sh4/Makefile.objs
index a285358ad..2c25d96e6 100644
--- a/target-sh4/Makefile.objs
+++ b/target-sh4/Makefile.objs
@@ -1,2 +1,3 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
+obj-$(CONFIG_SOFTMMU) += monitor.o
 obj-y += gdbstub.o
diff --git a/target-sh4/cpu.c b/target-sh4/cpu.c
index 64e4467c0..d7e2fbd0e 100644
--- a/target-sh4/cpu.c
+++ b/target-sh4/cpu.c
@@ -70,6 +70,12 @@ static void superh_cpu_reset(CPUState *s)
     set_default_nan_mode(1, &env->fp_status);
 }
 
+static void superh_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_sh4;
+    info->print_insn = print_insn_sh;
+}
+
 typedef struct SuperHCPUListState {
     fprintf_function cpu_fprintf;
     FILE *file;
@@ -288,9 +294,12 @@ static void superh_cpu_class_init(ObjectClass *oc, void *data)
 #else
     cc->get_phys_page_debug = superh_cpu_get_phys_page_debug;
 #endif
-    dc->vmsd = &vmstate_sh_cpu;
+    cc->disas_set_info = superh_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = 59;
 
+    dc->vmsd = &vmstate_sh_cpu;
+
     /*
      * Reason: superh_cpu_initfn() calls cpu_exec_init(), which saves
      * the object in cpus -> dangling pointer after final
diff --git a/target-sh4/cpu.h b/target-sh4/cpu.h
index 34bb3d779..5b022c597 100644
--- a/target-sh4/cpu.h
+++ b/target-sh4/cpu.h
@@ -24,8 +24,6 @@
 
 #define TARGET_LONG_BITS 32
 
-#define ELF_MACHINE	EM_SH
-
 /* CPU Subtypes */
 #define SH_CPU_SH7750  (1 << 0)
 #define SH_CPU_SH7750S (1 << 1)
@@ -122,6 +120,7 @@ typedef struct tlb_t {
 #define ITLB_SIZE 4
 
 #define NB_MMU_MODES 2
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 enum sh_features {
     SH_FEATURE_SH4A = 1,
@@ -227,7 +226,6 @@ void cpu_load_tlb(CPUSH4State * env);
 #define cpu_init(cpu_model) CPU(cpu_sh4_init(cpu_model))
 
 #define cpu_exec cpu_sh4_exec
-#define cpu_gen_code cpu_sh4_gen_code
 #define cpu_signal_handler cpu_sh4_signal_handler
 #define cpu_list sh4_cpu_list
 
@@ -235,7 +233,7 @@ void cpu_load_tlb(CPUSH4State * env);
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUSH4State *env)
+static inline int cpu_mmu_index (CPUSH4State *env, bool ifetch)
 {
     return (env->sr & (1u << SR_MD)) == 0 ? 1 : 0;
 }
diff --git a/target-sh4/helper.c b/target-sh4/helper.c
index a533f08ea..eaececd89 100644
--- a/target-sh4/helper.c
+++ b/target-sh4/helper.c
@@ -21,7 +21,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <inttypes.h>
-#include <signal.h>
 
 #include "cpu.h"
 
@@ -60,7 +59,7 @@ int superh_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
 
 int cpu_sh4_is_cached(CPUSH4State * env, target_ulong addr)
 {
-    /* For user mode, only U0 area is cachable. */
+    /* For user mode, only U0 area is cacheable. */
     return !(addr & 0x80000000);
 }
 
@@ -826,11 +825,11 @@ int cpu_sh4_is_cached(CPUSH4State * env, target_ulong addr)
 
     /* check area */
     if (env->sr & (1u << SR_MD)) {
-        /* For previledged mode, P2 and P4 area is not cachable. */
+        /* For privileged mode, P2 and P4 area is not cacheable. */
         if ((0xA0000000 <= addr && addr < 0xC0000000) || 0xE0000000 <= addr)
             return 0;
     } else {
-        /* For user mode, only U0 area is cachable. */
+        /* For user mode, only U0 area is cacheable. */
         if (0x80000000 <= addr)
             return 0;
     }
diff --git a/target-sh4/helper.h b/target-sh4/helper.h
index c9bc40704..dce859cae 100644
--- a/target-sh4/helper.h
+++ b/target-sh4/helper.h
@@ -18,28 +18,28 @@ DEF_HELPER_2(ld_fpscr, void, env, i32)
 
 DEF_HELPER_FLAGS_1(fabs_FT, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_FLAGS_1(fabs_DT, TCG_CALL_NO_RWG_SE, f64, f64)
-DEF_HELPER_3(fadd_FT, f32, env, f32, f32)
-DEF_HELPER_3(fadd_DT, f64, env, f64, f64)
-DEF_HELPER_2(fcnvsd_FT_DT, f64, env, f32)
-DEF_HELPER_2(fcnvds_DT_FT, f32, env, f64)
+DEF_HELPER_FLAGS_3(fadd_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fadd_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(fcnvsd_FT_DT, TCG_CALL_NO_WG, f64, env, f32)
+DEF_HELPER_FLAGS_2(fcnvds_DT_FT, TCG_CALL_NO_WG, f32, env, f64)
 
 DEF_HELPER_3(fcmp_eq_FT, void, env, f32, f32)
 DEF_HELPER_3(fcmp_eq_DT, void, env, f64, f64)
 DEF_HELPER_3(fcmp_gt_FT, void, env, f32, f32)
 DEF_HELPER_3(fcmp_gt_DT, void, env, f64, f64)
-DEF_HELPER_3(fdiv_FT, f32, env, f32, f32)
-DEF_HELPER_3(fdiv_DT, f64, env, f64, f64)
-DEF_HELPER_2(float_FT, f32, env, i32)
-DEF_HELPER_2(float_DT, f64, env, i32)
-DEF_HELPER_4(fmac_FT, f32, env, f32, f32, f32)
-DEF_HELPER_3(fmul_FT, f32, env, f32, f32)
-DEF_HELPER_3(fmul_DT, f64, env, f64, f64)
+DEF_HELPER_FLAGS_3(fdiv_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fdiv_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(float_FT, TCG_CALL_NO_WG, f32, env, i32)
+DEF_HELPER_FLAGS_2(float_DT, TCG_CALL_NO_WG, f64, env, i32)
+DEF_HELPER_FLAGS_4(fmac_FT, TCG_CALL_NO_WG, f32, env, f32, f32, f32)
+DEF_HELPER_FLAGS_3(fmul_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fmul_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
 DEF_HELPER_FLAGS_1(fneg_T, TCG_CALL_NO_RWG_SE, f32, f32)
-DEF_HELPER_3(fsub_FT, f32, env, f32, f32)
-DEF_HELPER_3(fsub_DT, f64, env, f64, f64)
-DEF_HELPER_2(fsqrt_FT, f32, env, f32)
-DEF_HELPER_2(fsqrt_DT, f64, env, f64)
-DEF_HELPER_2(ftrc_FT, i32, env, f32)
-DEF_HELPER_2(ftrc_DT, i32, env, f64)
+DEF_HELPER_FLAGS_3(fsub_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fsub_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(fsqrt_FT, TCG_CALL_NO_WG, f32, env, f32)
+DEF_HELPER_FLAGS_2(fsqrt_DT, TCG_CALL_NO_WG, f64, env, f64)
+DEF_HELPER_FLAGS_2(ftrc_FT, TCG_CALL_NO_WG, i32, env, f32)
+DEF_HELPER_FLAGS_2(ftrc_DT, TCG_CALL_NO_WG, i32, env, f64)
 DEF_HELPER_3(fipr, void, env, i32, i32)
 DEF_HELPER_2(ftrv, void, env, i32)
diff --git a/target-sh4/monitor.c b/target-sh4/monitor.c
new file mode 100644
index 000000000..a06f0d46d
--- /dev/null
+++ b/target-sh4/monitor.c
@@ -0,0 +1,52 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
+{
+    monitor_printf(mon, " tlb%i:\t"
+                   "asid=%hhu vpn=%x\tppn=%x\tsz=%hhu size=%u\t"
+                   "v=%hhu shared=%hhu cached=%hhu prot=%hhu "
+                   "dirty=%hhu writethrough=%hhu\n",
+                   idx,
+                   tlb->asid, tlb->vpn, tlb->ppn, tlb->sz, tlb->size,
+                   tlb->v, tlb->sh, tlb->c, tlb->pr,
+                   tlb->d, tlb->wt);
+}
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    int i;
+
+    monitor_printf (mon, "ITLB:\n");
+    for (i = 0 ; i < ITLB_SIZE ; i++)
+        print_tlb (mon, i, &env->itlb[i]);
+    monitor_printf (mon, "UTLB:\n");
+    for (i = 0 ; i < UTLB_SIZE ; i++)
+        print_tlb (mon, i, &env->utlb[i]);
+}
diff --git a/target-sh4/op_helper.c b/target-sh4/op_helper.c
index cbc11aecc..a3121180d 100644
--- a/target-sh4/op_helper.c
+++ b/target-sh4/op_helper.c
@@ -16,7 +16,6 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <assert.h>
 #include <stdlib.h>
 #include "cpu.h"
 #include "exec/helper-proto.h"
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index 3b4a1b5ce..7bc621649 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -70,8 +70,6 @@ static TCGv cpu_fregs[32];
 /* internal register indexes */
 static TCGv cpu_flags, cpu_delayed_pc;
 
-static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 void sh4_translate_init(void)
@@ -288,10 +286,10 @@ static inline void gen_load_fpr64(TCGv_i64 t, int reg)
 static inline void gen_store_fpr64 (TCGv_i64 t, int reg)
 {
     TCGv_i32 tmp = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(tmp, t);
+    tcg_gen_extrl_i64_i32(tmp, t);
     tcg_gen_mov_i32(cpu_fregs[reg + 1], tmp);
     tcg_gen_shri_i64(t, t, 32);
-    tcg_gen_trunc_i64_i32(tmp, t);
+    tcg_gen_extrl_i64_i32(tmp, t);
     tcg_gen_mov_i32(cpu_fregs[reg], tmp);
     tcg_temp_free_i32(tmp);
 }
@@ -612,15 +610,11 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0x6008:		/* swap.b Rm,Rn */
 	{
-	    TCGv high, low;
-	    high = tcg_temp_new();
-	    tcg_gen_andi_i32(high, REG(B7_4), 0xffff0000);
-	    low = tcg_temp_new();
+            TCGv low = tcg_temp_new();;
 	    tcg_gen_ext16u_i32(low, REG(B7_4));
 	    tcg_gen_bswap16_i32(low, low);
-	    tcg_gen_or_i32(REG(B11_8), high, low);
+            tcg_gen_deposit_i32(REG(B11_8), REG(B7_4), low, 0, 16);
 	    tcg_temp_free(low);
-	    tcg_temp_free(high);
 	}
 	return;
     case 0x6009:		/* swap.w Rm,Rn */
@@ -692,18 +686,11 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv cmp1 = tcg_temp_new();
 	    TCGv cmp2 = tcg_temp_new();
-	    tcg_gen_xor_i32(cmp1, REG(B7_4), REG(B11_8));
-	    tcg_gen_andi_i32(cmp2, cmp1, 0xff000000);
-            tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, cmp2, 0);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x00ff0000);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-            tcg_gen_or_i32(cpu_sr_t, cpu_sr_t, cmp2);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x0000ff00);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-            tcg_gen_or_i32(cpu_sr_t, cpu_sr_t, cmp2);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x000000ff);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-            tcg_gen_or_i32(cpu_sr_t, cpu_sr_t, cmp2);
+            tcg_gen_xor_i32(cmp2, REG(B7_4), REG(B11_8));
+            tcg_gen_subi_i32(cmp1, cmp2, 0x01010101);
+            tcg_gen_andc_i32(cmp1, cmp1, cmp2);
+            tcg_gen_andi_i32(cmp1, cmp1, 0x80808080);
+            tcg_gen_setcondi_i32(TCG_COND_NE, cpu_sr_t, cmp1, 0);
 	    tcg_temp_free(cmp2);
 	    tcg_temp_free(cmp1);
 	}
@@ -843,67 +830,54 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0x400c:		/* shad Rm,Rn */
 	{
-            TCGLabel *label1 = gen_new_label();
-            TCGLabel *label2 = gen_new_label();
-            TCGLabel *label3 = gen_new_label();
-            TCGLabel *label4 = gen_new_label();
-	    TCGv shift;
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B7_4), 0, label1);
-	    /* Rm positive, shift to the left */
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_shl_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label4);
-	    /* Rm negative, shift to the right */
-	    gen_set_label(label1);
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_brcondi_i32(TCG_COND_EQ, shift, 0, label2);
-	    tcg_gen_not_i32(shift, REG(B7_4));
-	    tcg_gen_andi_i32(shift, shift, 0x1f);
-	    tcg_gen_addi_i32(shift, shift, 1);
-	    tcg_gen_sar_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label4);
-	    /* Rm = -32 */
-	    gen_set_label(label2);
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B11_8), 0, label3);
-	    tcg_gen_movi_i32(REG(B11_8), 0);
-	    tcg_gen_br(label4);
-	    gen_set_label(label3);
-	    tcg_gen_movi_i32(REG(B11_8), 0xffffffff);
-	    gen_set_label(label4);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            TCGv t2 = tcg_temp_new();
+
+            tcg_gen_andi_i32(t0, REG(B7_4), 0x1f);
+
+            /* positive case: shift to the left */
+            tcg_gen_shl_i32(t1, REG(B11_8), t0);
+
+            /* negative case: shift to the right in two steps to
+               correctly handle the -32 case */
+            tcg_gen_xori_i32(t0, t0, 0x1f);
+            tcg_gen_sar_i32(t2, REG(B11_8), t0);
+            tcg_gen_sari_i32(t2, t2, 1);
+
+            /* select between the two cases */
+            tcg_gen_movi_i32(t0, 0);
+            tcg_gen_movcond_i32(TCG_COND_GE, REG(B11_8), REG(B7_4), t0, t1, t2);
+
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
+            tcg_temp_free(t2);
 	}
 	return;
     case 0x400d:		/* shld Rm,Rn */
 	{
-            TCGLabel *label1 = gen_new_label();
-            TCGLabel *label2 = gen_new_label();
-            TCGLabel *label3 = gen_new_label();
-	    TCGv shift;
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B7_4), 0, label1);
-	    /* Rm positive, shift to the left */
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_shl_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label3);
-	    /* Rm negative, shift to the right */
-	    gen_set_label(label1);
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_brcondi_i32(TCG_COND_EQ, shift, 0, label2);
-	    tcg_gen_not_i32(shift, REG(B7_4));
-	    tcg_gen_andi_i32(shift, shift, 0x1f);
-	    tcg_gen_addi_i32(shift, shift, 1);
-	    tcg_gen_shr_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label3);
-	    /* Rm = -32 */
-	    gen_set_label(label2);
-	    tcg_gen_movi_i32(REG(B11_8), 0);
-	    gen_set_label(label3);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            TCGv t2 = tcg_temp_new();
+
+            tcg_gen_andi_i32(t0, REG(B7_4), 0x1f);
+
+            /* positive case: shift to the left */
+            tcg_gen_shl_i32(t1, REG(B11_8), t0);
+
+            /* negative case: shift to the right in two steps to
+               correctly handle the -32 case */
+            tcg_gen_xori_i32(t0, t0, 0x1f);
+            tcg_gen_shr_i32(t2, REG(B11_8), t0);
+            tcg_gen_shri_i32(t2, t2, 1);
+
+            /* select between the two cases */
+            tcg_gen_movi_i32(t0, 0);
+            tcg_gen_movcond_i32(TCG_COND_GE, REG(B11_8), REG(B7_4), t0, t1, t2);
+
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
+            tcg_temp_free(t2);
 	}
 	return;
     case 0x3008:		/* sub Rm,Rn */
@@ -1814,10 +1788,6 @@ static void decode_opc(DisasContext * ctx)
 {
     uint32_t old_flags = ctx->flags;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     _decode_opc(ctx);
 
     if (old_flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) {
@@ -1844,16 +1814,12 @@ static void decode_opc(DisasContext * ctx)
         gen_store_flags(ctx->flags);
 }
 
-static inline void
-gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUSH4State * env, struct TranslationBlock *tb)
 {
+    SuperHCPU *cpu = sh_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUSH4State *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int i, ii;
     int num_insns;
     int max_insns;
 
@@ -1870,45 +1836,39 @@ gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
     ctx.features = env->features;
     ctx.has_movcal = (ctx.flags & TB_FLAG_PENDING_MOVCA);
 
-    ii = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     gen_tb_start(tb);
     while (ctx.bstate == BS_NONE && !tcg_op_buf_full()) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (ctx.pc == bp->pc) {
-		    /* We have hit a breakpoint - make sure PC is up-to-date */
-		    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-                    gen_helper_debug(cpu_env);
-                    ctx.bstate = BS_BRANCH;
-		    break;
-		}
-	    }
-	}
-        if (search_pc) {
-            i = tcg_op_buf_count();
-            if (ii < i) {
-                ii++;
-                while (ii < i)
-                    tcg_ctx.gen_opc_instr_start[ii++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[ii] = ctx.pc;
-            gen_opc_hflags[ii] = ctx.flags;
-            tcg_ctx.gen_opc_instr_start[ii] = 1;
-            tcg_ctx.gen_opc_icount[ii] = num_insns;
+        tcg_gen_insn_start(ctx.pc, ctx.flags);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            /* We have hit a breakpoint - make sure PC is up-to-date */
+            tcg_gen_movi_i32(cpu_pc, ctx.pc);
+            gen_helper_debug(cpu_env);
+            ctx.bstate = BS_BRANCH;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 2;
+            break;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-#if 0
-	fprintf(stderr, "Loading opcode at address 0x%08x\n", ctx.pc);
-	fflush(stderr);
-#endif
+        }
+
         ctx.opcode = cpu_lduw_code(env, ctx.pc);
 	decode_opc(&ctx);
-        num_insns++;
 	ctx.pc += 2;
 	if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0)
 	    break;
@@ -1948,15 +1908,8 @@ gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        i = tcg_op_buf_count();
-        ii++;
-        while (ii <= i)
-            tcg_ctx.gen_opc_instr_start[ii++] = 0;
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -1967,18 +1920,9 @@ gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
 #endif
 }
 
-void gen_intermediate_code(CPUSH4State * env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(sh_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUSH4State * env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(sh_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
-    env->flags = gen_opc_hflags[pc_pos];
+    env->pc = data[0];
+    env->flags = data[1];
 }
diff --git a/target-sparc/Makefile.objs b/target-sparc/Makefile.objs
index 1cd81cccc..ec905698c 100644
--- a/target-sparc/Makefile.objs
+++ b/target-sparc/Makefile.objs
@@ -1,4 +1,4 @@
-obj-$(CONFIG_SOFTMMU) += machine.o
+obj-$(CONFIG_SOFTMMU) += machine.o monitor.o
 obj-y += translate.o helper.o cpu.o
 obj-y += fop_helper.o cc_helper.o win_helper.o mmu_helper.o ldst_helper.o
 obj-$(TARGET_SPARC) += int32_helper.o
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index 82bb72ab7..d98682b56 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -90,6 +90,14 @@ static bool sparc_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
     return false;
 }
 
+static void cpu_sparc_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->print_insn = print_insn_sparc;
+#ifdef TARGET_SPARC64
+    info->mach = bfd_mach_sparc_v9b;
+#endif
+}
+
 static int cpu_sparc_register(SPARCCPU *cpu, const char *cpu_model)
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
@@ -848,6 +856,7 @@ static void sparc_cpu_class_init(ObjectClass *oc, void *data)
     cc->do_unaligned_access = sparc_cpu_do_unaligned_access;
     cc->get_phys_page_debug = sparc_cpu_get_phys_page_debug;
 #endif
+    cc->disas_set_info = cpu_sparc_disas_set_info;
 
 #if defined(TARGET_SPARC64) && !defined(TARGET_ABI32)
     cc->gdb_num_core_regs = 86;
diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h
index 0522b65f1..9fa770b14 100644
--- a/target-sparc/cpu.h
+++ b/target-sparc/cpu.h
@@ -31,12 +31,6 @@
 
 #include "fpu/softfloat.h"
 
-#if !defined(TARGET_SPARC64)
-#define ELF_MACHINE     EM_SPARC
-#else
-#define ELF_MACHINE     EM_SPARCV9
-#endif
-
 /*#define EXCP_INTERRUPT 0x100*/
 
 /* trap definitions */
@@ -236,6 +230,7 @@ typedef struct trap_state {
     uint32_t tt;
 } trap_state;
 #endif
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 typedef struct sparc_def_t {
     const char *name;
@@ -598,7 +593,6 @@ int cpu_sparc_signal_handler(int host_signum, void *pinfo, void *puc);
 #endif
 
 #define cpu_exec cpu_sparc_exec
-#define cpu_gen_code cpu_sparc_gen_code
 #define cpu_signal_handler cpu_sparc_signal_handler
 #define cpu_list sparc_cpu_list
 
@@ -642,7 +636,7 @@ static inline int cpu_supervisor_mode(CPUSPARCState *env1)
 }
 #endif
 
-static inline int cpu_mmu_index(CPUSPARCState *env1)
+static inline int cpu_mmu_index(CPUSPARCState *env1, bool ifetch)
 {
 #if defined(CONFIG_USER_ONLY)
     return MMU_USER_IDX;
diff --git a/target-sparc/mmu_helper.c b/target-sparc/mmu_helper.c
index 2a0c6f0d3..7495406b3 100644
--- a/target-sparc/mmu_helper.c
+++ b/target-sparc/mmu_helper.c
@@ -849,7 +849,7 @@ hwaddr sparc_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     SPARCCPU *cpu = SPARC_CPU(cs);
     CPUSPARCState *env = &cpu->env;
     hwaddr phys_addr;
-    int mmu_idx = cpu_mmu_index(env);
+    int mmu_idx = cpu_mmu_index(env, false);
     MemoryRegionSection section;
 
     if (cpu_sparc_get_phys_page(env, &phys_addr, addr, 2, mmu_idx) != 0) {
diff --git a/target-sparc/monitor.c b/target-sparc/monitor.c
new file mode 100644
index 000000000..ca54d72d9
--- /dev/null
+++ b/target-sparc/monitor.c
@@ -0,0 +1,158 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
+
+#ifndef TARGET_SPARC64
+static target_long monitor_get_psr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+
+    return cpu_get_psr(env);
+}
+#endif
+
+static target_long monitor_get_reg(const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return env->regwptr[val];
+}
+
+const MonitorDef monitor_defs[] = {
+    { "g0", offsetof(CPUSPARCState, gregs[0]) },
+    { "g1", offsetof(CPUSPARCState, gregs[1]) },
+    { "g2", offsetof(CPUSPARCState, gregs[2]) },
+    { "g3", offsetof(CPUSPARCState, gregs[3]) },
+    { "g4", offsetof(CPUSPARCState, gregs[4]) },
+    { "g5", offsetof(CPUSPARCState, gregs[5]) },
+    { "g6", offsetof(CPUSPARCState, gregs[6]) },
+    { "g7", offsetof(CPUSPARCState, gregs[7]) },
+    { "o0", 0, monitor_get_reg },
+    { "o1", 1, monitor_get_reg },
+    { "o2", 2, monitor_get_reg },
+    { "o3", 3, monitor_get_reg },
+    { "o4", 4, monitor_get_reg },
+    { "o5", 5, monitor_get_reg },
+    { "o6", 6, monitor_get_reg },
+    { "o7", 7, monitor_get_reg },
+    { "l0", 8, monitor_get_reg },
+    { "l1", 9, monitor_get_reg },
+    { "l2", 10, monitor_get_reg },
+    { "l3", 11, monitor_get_reg },
+    { "l4", 12, monitor_get_reg },
+    { "l5", 13, monitor_get_reg },
+    { "l6", 14, monitor_get_reg },
+    { "l7", 15, monitor_get_reg },
+    { "i0", 16, monitor_get_reg },
+    { "i1", 17, monitor_get_reg },
+    { "i2", 18, monitor_get_reg },
+    { "i3", 19, monitor_get_reg },
+    { "i4", 20, monitor_get_reg },
+    { "i5", 21, monitor_get_reg },
+    { "i6", 22, monitor_get_reg },
+    { "i7", 23, monitor_get_reg },
+    { "pc", offsetof(CPUSPARCState, pc) },
+    { "npc", offsetof(CPUSPARCState, npc) },
+    { "y", offsetof(CPUSPARCState, y) },
+#ifndef TARGET_SPARC64
+    { "psr", 0, &monitor_get_psr, },
+    { "wim", offsetof(CPUSPARCState, wim) },
+#endif
+    { "tbr", offsetof(CPUSPARCState, tbr) },
+    { "fsr", offsetof(CPUSPARCState, fsr) },
+    { "f0", offsetof(CPUSPARCState, fpr[0].l.upper) },
+    { "f1", offsetof(CPUSPARCState, fpr[0].l.lower) },
+    { "f2", offsetof(CPUSPARCState, fpr[1].l.upper) },
+    { "f3", offsetof(CPUSPARCState, fpr[1].l.lower) },
+    { "f4", offsetof(CPUSPARCState, fpr[2].l.upper) },
+    { "f5", offsetof(CPUSPARCState, fpr[2].l.lower) },
+    { "f6", offsetof(CPUSPARCState, fpr[3].l.upper) },
+    { "f7", offsetof(CPUSPARCState, fpr[3].l.lower) },
+    { "f8", offsetof(CPUSPARCState, fpr[4].l.upper) },
+    { "f9", offsetof(CPUSPARCState, fpr[4].l.lower) },
+    { "f10", offsetof(CPUSPARCState, fpr[5].l.upper) },
+    { "f11", offsetof(CPUSPARCState, fpr[5].l.lower) },
+    { "f12", offsetof(CPUSPARCState, fpr[6].l.upper) },
+    { "f13", offsetof(CPUSPARCState, fpr[6].l.lower) },
+    { "f14", offsetof(CPUSPARCState, fpr[7].l.upper) },
+    { "f15", offsetof(CPUSPARCState, fpr[7].l.lower) },
+    { "f16", offsetof(CPUSPARCState, fpr[8].l.upper) },
+    { "f17", offsetof(CPUSPARCState, fpr[8].l.lower) },
+    { "f18", offsetof(CPUSPARCState, fpr[9].l.upper) },
+    { "f19", offsetof(CPUSPARCState, fpr[9].l.lower) },
+    { "f20", offsetof(CPUSPARCState, fpr[10].l.upper) },
+    { "f21", offsetof(CPUSPARCState, fpr[10].l.lower) },
+    { "f22", offsetof(CPUSPARCState, fpr[11].l.upper) },
+    { "f23", offsetof(CPUSPARCState, fpr[11].l.lower) },
+    { "f24", offsetof(CPUSPARCState, fpr[12].l.upper) },
+    { "f25", offsetof(CPUSPARCState, fpr[12].l.lower) },
+    { "f26", offsetof(CPUSPARCState, fpr[13].l.upper) },
+    { "f27", offsetof(CPUSPARCState, fpr[13].l.lower) },
+    { "f28", offsetof(CPUSPARCState, fpr[14].l.upper) },
+    { "f29", offsetof(CPUSPARCState, fpr[14].l.lower) },
+    { "f30", offsetof(CPUSPARCState, fpr[15].l.upper) },
+    { "f31", offsetof(CPUSPARCState, fpr[15].l.lower) },
+#ifdef TARGET_SPARC64
+    { "f32", offsetof(CPUSPARCState, fpr[16]) },
+    { "f34", offsetof(CPUSPARCState, fpr[17]) },
+    { "f36", offsetof(CPUSPARCState, fpr[18]) },
+    { "f38", offsetof(CPUSPARCState, fpr[19]) },
+    { "f40", offsetof(CPUSPARCState, fpr[20]) },
+    { "f42", offsetof(CPUSPARCState, fpr[21]) },
+    { "f44", offsetof(CPUSPARCState, fpr[22]) },
+    { "f46", offsetof(CPUSPARCState, fpr[23]) },
+    { "f48", offsetof(CPUSPARCState, fpr[24]) },
+    { "f50", offsetof(CPUSPARCState, fpr[25]) },
+    { "f52", offsetof(CPUSPARCState, fpr[26]) },
+    { "f54", offsetof(CPUSPARCState, fpr[27]) },
+    { "f56", offsetof(CPUSPARCState, fpr[28]) },
+    { "f58", offsetof(CPUSPARCState, fpr[29]) },
+    { "f60", offsetof(CPUSPARCState, fpr[30]) },
+    { "f62", offsetof(CPUSPARCState, fpr[31]) },
+    { "asi", offsetof(CPUSPARCState, asi) },
+    { "pstate", offsetof(CPUSPARCState, pstate) },
+    { "cansave", offsetof(CPUSPARCState, cansave) },
+    { "canrestore", offsetof(CPUSPARCState, canrestore) },
+    { "otherwin", offsetof(CPUSPARCState, otherwin) },
+    { "wstate", offsetof(CPUSPARCState, wstate) },
+    { "cleanwin", offsetof(CPUSPARCState, cleanwin) },
+    { "fprs", offsetof(CPUSPARCState, fprs) },
+#endif
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index c58dd4e95..63440dd79 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -64,9 +64,6 @@ static TCGv cpu_wim;
 /* Floating point registers */
 static TCGv_i64 cpu_fpr[TARGET_DPREGS];
 
-static target_ulong gen_opc_npc[OPC_BUF_SIZE];
-static target_ulong gen_opc_jump_pc[2];
-
 #include "exec/gen-icount.h"
 
 typedef struct DisasContext {
@@ -164,7 +161,7 @@ static TCGv_i32 gen_load_fpr_F(DisasContext *dc, unsigned int src)
         TCGv_i64 t = tcg_temp_new_i64();
 
         tcg_gen_shri_i64(t, cpu_fpr[src / 2], 32);
-        tcg_gen_trunc_i64_i32(ret, t);
+        tcg_gen_extrl_i64_i32(ret, t);
         tcg_temp_free_i64(t);
 
         return ret;
@@ -379,8 +376,8 @@ static TCGv_i32 gen_add32_carry32(void)
 #if TARGET_LONG_BITS == 64
     cc_src1_32 = tcg_temp_new_i32();
     cc_src2_32 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(cc_src1_32, cpu_cc_dst);
-    tcg_gen_trunc_i64_i32(cc_src2_32, cpu_cc_src);
+    tcg_gen_extrl_i64_i32(cc_src1_32, cpu_cc_dst);
+    tcg_gen_extrl_i64_i32(cc_src2_32, cpu_cc_src);
 #else
     cc_src1_32 = cpu_cc_dst;
     cc_src2_32 = cpu_cc_src;
@@ -405,8 +402,8 @@ static TCGv_i32 gen_sub32_carry32(void)
 #if TARGET_LONG_BITS == 64
     cc_src1_32 = tcg_temp_new_i32();
     cc_src2_32 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(cc_src1_32, cpu_cc_src);
-    tcg_gen_trunc_i64_i32(cc_src2_32, cpu_cc_src2);
+    tcg_gen_extrl_i64_i32(cc_src1_32, cpu_cc_src);
+    tcg_gen_extrl_i64_i32(cc_src2_32, cpu_cc_src2);
 #else
     cc_src1_32 = cpu_cc_src;
     cc_src2_32 = cpu_cc_src2;
@@ -955,17 +952,44 @@ static inline void gen_branch2(DisasContext *dc, target_ulong pc1,
     gen_goto_tb(dc, 1, pc2, pc2 + 4);
 }
 
-static inline void gen_branch_a(DisasContext *dc, target_ulong pc1,
-                                target_ulong pc2, TCGv r_cond)
+static void gen_branch_a(DisasContext *dc, target_ulong pc1)
 {
     TCGLabel *l1 = gen_new_label();
+    target_ulong npc = dc->npc;
 
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
+    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_cond, 0, l1);
 
-    gen_goto_tb(dc, 0, pc2, pc1);
+    gen_goto_tb(dc, 0, npc, pc1);
 
     gen_set_label(l1);
-    gen_goto_tb(dc, 1, pc2 + 4, pc2 + 8);
+    gen_goto_tb(dc, 1, npc + 4, npc + 8);
+
+    dc->is_br = 1;
+}
+
+static void gen_branch_n(DisasContext *dc, target_ulong pc1)
+{
+    target_ulong npc = dc->npc;
+
+    if (likely(npc != DYNAMIC_PC)) {
+        dc->pc = npc;
+        dc->jump_pc[0] = pc1;
+        dc->jump_pc[1] = npc + 4;
+        dc->npc = JUMP_PC;
+    } else {
+        TCGv t, z;
+
+        tcg_gen_mov_tl(cpu_pc, cpu_npc);
+
+        tcg_gen_addi_tl(cpu_npc, cpu_npc, 4);
+        t = tcg_const_tl(pc1);
+        z = tcg_const_tl(0);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_npc, cpu_cond, z, t, cpu_npc);
+        tcg_temp_free(t);
+        tcg_temp_free(z);
+
+        dc->pc = DYNAMIC_PC;
+    }
 }
 
 static inline void gen_generic_branch(DisasContext *dc)
@@ -1398,18 +1422,9 @@ static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
         flush_cond(dc);
         gen_cond(cpu_cond, cc, cond, dc);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, cpu_cond);
-            dc->is_br = 1;
+            gen_branch_a(dc, target);
         } else {
-            dc->pc = dc->npc;
-            dc->jump_pc[0] = target;
-            if (unlikely(dc->npc == DYNAMIC_PC)) {
-                dc->jump_pc[1] = DYNAMIC_PC;
-                tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-            } else {
-                dc->jump_pc[1] = dc->npc + 4;
-                dc->npc = JUMP_PC;
-            }
+            gen_branch_n(dc, target);
         }
     }
 }
@@ -1447,18 +1462,9 @@ static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
         flush_cond(dc);
         gen_fcond(cpu_cond, cc, cond);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, cpu_cond);
-            dc->is_br = 1;
+            gen_branch_a(dc, target);
         } else {
-            dc->pc = dc->npc;
-            dc->jump_pc[0] = target;
-            if (unlikely(dc->npc == DYNAMIC_PC)) {
-                dc->jump_pc[1] = DYNAMIC_PC;
-                tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-            } else {
-                dc->jump_pc[1] = dc->npc + 4;
-                dc->npc = JUMP_PC;
-            }
+            gen_branch_n(dc, target);
         }
     }
 }
@@ -1476,18 +1482,9 @@ static void do_branch_reg(DisasContext *dc, int32_t offset, uint32_t insn,
     flush_cond(dc);
     gen_cond_reg(cpu_cond, cond, r_reg);
     if (a) {
-        gen_branch_a(dc, target, dc->npc, cpu_cond);
-        dc->is_br = 1;
+        gen_branch_a(dc, target);
     } else {
-        dc->pc = dc->npc;
-        dc->jump_pc[0] = target;
-        if (unlikely(dc->npc == DYNAMIC_PC)) {
-            dc->jump_pc[1] = DYNAMIC_PC;
-            tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-        } else {
-            dc->jump_pc[1] = dc->npc + 4;
-            dc->npc = JUMP_PC;
-        }
+        gen_branch_n(dc, target);
     }
 }
 
@@ -2254,11 +2251,11 @@ static void gen_fmovs(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
        the later.  */
     c32 = tcg_temp_new_i32();
     if (cmp->is_bool) {
-        tcg_gen_trunc_i64_i32(c32, cmp->c1);
+        tcg_gen_extrl_i64_i32(c32, cmp->c1);
     } else {
         TCGv_i64 c64 = tcg_temp_new_i64();
         tcg_gen_setcond_i64(cmp->cond, c64, cmp->c1, cmp->c2);
-        tcg_gen_trunc_i64_i32(c32, c64);
+        tcg_gen_extrl_i64_i32(c32, c64);
         tcg_temp_free_i64(c64);
     }
 
@@ -2482,10 +2479,6 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     TCGv_i64 cpu_src1_64, cpu_src2_64, cpu_dst_64;
     target_long simm;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     opc = GET_FIELD(insn, 0, 1);
     rd = GET_FIELD(insn, 2, 6);
 
@@ -5104,7 +5097,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (IS_IMM) {
                         goto illegal_insn;
                     }
-                    if (!supervisor(dc)) {
+                    /* LEON3 allows CASA from user space with ASI 0xa */
+                    if ((GET_FIELD(insn, 19, 26) != 0xa) && !supervisor(dc)) {
                         goto priv_insn;
                     }
 #endif
@@ -5213,16 +5207,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     }
 }
 
-static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool spc)
+void gen_intermediate_code(CPUSPARCState * env, TranslationBlock * tb)
 {
+    SPARCCPU *cpu = sparc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUSPARCState *env = &cpu->env;
     target_ulong pc_start, last_pc;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
     unsigned int insn;
@@ -5234,7 +5224,7 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
     last_pc = dc->pc;
     dc->npc = (target_ulong) tb->cs_base;
     dc->cc_op = CC_OP_DYNAMIC;
-    dc->mem_idx = cpu_mmu_index(env);
+    dc->mem_idx = cpu_mmu_index(env, false);
     dc->def = env->def;
     dc->fpu_enabled = tb_fpu_enabled(tb->flags);
     dc->address_mask_32bit = tb_am_enabled(tb->flags);
@@ -5242,42 +5232,41 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
 
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    if (dc->pc != pc_start)
-                        save_state(dc);
-                    gen_helper_debug(cpu_env);
-                    tcg_gen_exit_tb(0);
-                    dc->is_br = 1;
-                    goto exit_gen_loop;
-                }
-            }
+        if (dc->npc & JUMP_PC) {
+            assert(dc->jump_pc[1] == dc->pc + 4);
+            tcg_gen_insn_start(dc->pc, dc->jump_pc[0] | JUMP_PC);
+        } else {
+            tcg_gen_insn_start(dc->pc, dc->npc);
         }
-        if (spc) {
-            qemu_log("Search PC...\n");
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                tcg_ctx.gen_opc_pc[lj] = dc->pc;
-                gen_opc_npc[lj] = dc->npc;
-                tcg_ctx.gen_opc_instr_start[lj] = 1;
-                tcg_ctx.gen_opc_icount[lj] = num_insns;
+        num_insns++;
+        last_pc = dc->pc;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            if (dc->pc != pc_start) {
+                save_state(dc);
             }
+            gen_helper_debug(cpu_env);
+            tcg_gen_exit_tb(0);
+            dc->is_br = 1;
+            goto exit_gen_loop;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-        last_pc = dc->pc;
+        }
+
         insn = cpu_ldl_code(env, dc->pc);
 
         disas_sparc_insn(dc, insn);
-        num_insns++;
 
         if (dc->is_br)
             break;
@@ -5316,20 +5305,9 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
     }
     gen_tb_end(tb, num_insns);
 
-    if (spc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-#if 0
-        log_page_dump();
-#endif
-        gen_opc_jump_pc[0] = dc->jump_pc[0];
-        gen_opc_jump_pc[1] = dc->jump_pc[1];
-    } else {
-        tb->size = last_pc + 4 - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = last_pc + 4 - pc_start;
+    tb->icount = num_insns;
+
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("--------------\n");
@@ -5340,16 +5318,6 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code(CPUSPARCState * env, TranslationBlock * tb)
-{
-    gen_intermediate_code_internal(sparc_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUSPARCState * env, TranslationBlock * tb)
-{
-    gen_intermediate_code_internal(sparc_env_get_cpu(env), tb, true);
-}
-
 void gen_intermediate_code_init(CPUSPARCState *env)
 {
     unsigned int i;
@@ -5451,19 +5419,21 @@ void gen_intermediate_code_init(CPUSPARCState *env)
     }
 }
 
-void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    target_ulong npc;
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
-    npc = gen_opc_npc[pc_pos];
-    if (npc == 1) {
+    target_ulong pc = data[0];
+    target_ulong npc = data[1];
+
+    env->pc = pc;
+    if (npc == DYNAMIC_PC) {
         /* dynamic NPC: already stored */
-    } else if (npc == 2) {
+    } else if (npc & JUMP_PC) {
         /* jump PC: use 'cond' and the jump targets of the translation */
         if (env->cond) {
-            env->npc = gen_opc_jump_pc[0];
+            env->npc = npc & ~3;
         } else {
-            env->npc = gen_opc_jump_pc[1];
+            env->npc = pc + 4;
         }
     } else {
         env->npc = npc;
diff --git a/target-sparc/vis_helper.c b/target-sparc/vis_helper.c
index 383cc8bdf..45fc7dbe8 100644
--- a/target-sparc/vis_helper.c
+++ b/target-sparc/vis_helper.c
@@ -447,7 +447,7 @@ uint32_t helper_fpackfix(uint64_t gsr, uint64_t rs2)
     for (word = 0; word < 2; word++) {
         uint32_t val;
         int32_t src = rs2 >> (word * 32);
-        int64_t scaled = src << scale;
+        int64_t scaled = (int64_t)src << scale;
         int64_t from_fixed = scaled >> 16;
 
         val = (from_fixed < -32768 ? -32768 :
diff --git a/target-tilegx/Makefile.objs b/target-tilegx/Makefile.objs
new file mode 100644
index 000000000..0db778f40
--- /dev/null
+++ b/target-tilegx/Makefile.objs
@@ -0,0 +1 @@
+obj-y += cpu.o translate.o helper.o simd_helper.o
diff --git a/target-tilegx/cpu.c b/target-tilegx/cpu.c
new file mode 100644
index 000000000..c24970436
--- /dev/null
+++ b/target-tilegx/cpu.c
@@ -0,0 +1,185 @@
+/*
+ * QEMU TILE-Gx CPU
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "cpu.h"
+#include "qemu-common.h"
+#include "hw/qdev-properties.h"
+#include "migration/vmstate.h"
+#include "linux-user/syscall_defs.h"
+
+static void tilegx_cpu_dump_state(CPUState *cs, FILE *f,
+                                  fprintf_function cpu_fprintf, int flags)
+{
+    static const char * const reg_names[TILEGX_R_COUNT] = {
+         "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+         "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+        "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+        "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+        "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
+        "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
+        "r48", "r49", "r50", "r51",  "bp",  "tp",  "sp",  "lr"
+    };
+
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+    CPUTLGState *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; i++) {
+        cpu_fprintf(f, "%-4s" TARGET_FMT_lx "%s",
+                    reg_names[i], env->regs[i],
+                    (i % 4) == 3 ? "\n" : " ");
+    }
+    cpu_fprintf(f, "PC  " TARGET_FMT_lx " CEX " TARGET_FMT_lx "\n\n",
+                env->pc, env->spregs[TILEGX_SPR_CMPEXCH]);
+}
+
+TileGXCPU *cpu_tilegx_init(const char *cpu_model)
+{
+    TileGXCPU *cpu;
+
+    cpu = TILEGX_CPU(object_new(TYPE_TILEGX_CPU));
+
+    object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
+
+    return cpu;
+}
+
+static void tilegx_cpu_set_pc(CPUState *cs, vaddr value)
+{
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+
+    cpu->env.pc = value;
+}
+
+static bool tilegx_cpu_has_work(CPUState *cs)
+{
+    return true;
+}
+
+static void tilegx_cpu_reset(CPUState *s)
+{
+    TileGXCPU *cpu = TILEGX_CPU(s);
+    TileGXCPUClass *tcc = TILEGX_CPU_GET_CLASS(cpu);
+    CPUTLGState *env = &cpu->env;
+
+    tcc->parent_reset(s);
+
+    memset(env, 0, sizeof(CPUTLGState));
+    tlb_flush(s, 1);
+}
+
+static void tilegx_cpu_realizefn(DeviceState *dev, Error **errp)
+{
+    CPUState *cs = CPU(dev);
+    TileGXCPUClass *tcc = TILEGX_CPU_GET_CLASS(dev);
+
+    cpu_reset(cs);
+    qemu_init_vcpu(cs);
+
+    tcc->parent_realize(dev, errp);
+}
+
+static void tilegx_cpu_initfn(Object *obj)
+{
+    CPUState *cs = CPU(obj);
+    TileGXCPU *cpu = TILEGX_CPU(obj);
+    CPUTLGState *env = &cpu->env;
+    static bool tcg_initialized;
+
+    cs->env_ptr = env;
+    cpu_exec_init(cs, &error_abort);
+
+    if (tcg_enabled() && !tcg_initialized) {
+        tcg_initialized = true;
+        tilegx_tcg_init();
+    }
+}
+
+static void tilegx_cpu_do_interrupt(CPUState *cs)
+{
+    cs->exception_index = -1;
+}
+
+static int tilegx_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
+                                       int mmu_idx)
+{
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+
+    /* The sigcode field will be filled in by do_signal in main.c.  */
+    cs->exception_index = TILEGX_EXCP_SIGNAL;
+    cpu->env.excaddr = address;
+    cpu->env.signo = TARGET_SIGSEGV;
+    cpu->env.sigcode = 0;
+
+    return 1;
+}
+
+static bool tilegx_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        tilegx_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
+
+static void tilegx_cpu_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    CPUClass *cc = CPU_CLASS(oc);
+    TileGXCPUClass *tcc = TILEGX_CPU_CLASS(oc);
+
+    tcc->parent_realize = dc->realize;
+    dc->realize = tilegx_cpu_realizefn;
+
+    tcc->parent_reset = cc->reset;
+    cc->reset = tilegx_cpu_reset;
+
+    cc->has_work = tilegx_cpu_has_work;
+    cc->do_interrupt = tilegx_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = tilegx_cpu_exec_interrupt;
+    cc->dump_state = tilegx_cpu_dump_state;
+    cc->set_pc = tilegx_cpu_set_pc;
+    cc->handle_mmu_fault = tilegx_cpu_handle_mmu_fault;
+    cc->gdb_num_core_regs = 0;
+
+    /*
+     * Reason: tilegx_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo tilegx_cpu_type_info = {
+    .name = TYPE_TILEGX_CPU,
+    .parent = TYPE_CPU,
+    .instance_size = sizeof(TileGXCPU),
+    .instance_init = tilegx_cpu_initfn,
+    .class_size = sizeof(TileGXCPUClass),
+    .class_init = tilegx_cpu_class_init,
+};
+
+static void tilegx_cpu_register_types(void)
+{
+    type_register_static(&tilegx_cpu_type_info);
+}
+
+type_init(tilegx_cpu_register_types)
diff --git a/target-tilegx/cpu.h b/target-tilegx/cpu.h
new file mode 100644
index 000000000..03df107a6
--- /dev/null
+++ b/target-tilegx/cpu.h
@@ -0,0 +1,182 @@
+/*
+ *  TILE-Gx virtual CPU header
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef CPU_TILEGX_H
+#define CPU_TILEGX_H
+
+#include "config.h"
+#include "qemu-common.h"
+
+#define TARGET_LONG_BITS 64
+
+#define CPUArchState struct CPUTLGState
+
+#include "exec/cpu-defs.h"
+
+
+/* TILE-Gx common register alias */
+#define TILEGX_R_RE    0   /*  0 register, for function/syscall return value */
+#define TILEGX_R_ERR   1   /*  1 register, for syscall errno flag */
+#define TILEGX_R_NR    10  /* 10 register, for syscall number */
+#define TILEGX_R_BP    52  /* 52 register, optional frame pointer */
+#define TILEGX_R_TP    53  /* TP register, thread local storage data */
+#define TILEGX_R_SP    54  /* SP register, stack pointer */
+#define TILEGX_R_LR    55  /* LR register, may save pc, but it is not pc */
+#define TILEGX_R_COUNT 56  /* Only 56 registers are really useful */
+#define TILEGX_R_SN    56  /* SN register, obsoleted, it likes zero register */
+#define TILEGX_R_IDN0  57  /* IDN0 register, cause IDN_ACCESS exception */
+#define TILEGX_R_IDN1  58  /* IDN1 register, cause IDN_ACCESS exception */
+#define TILEGX_R_UDN0  59  /* UDN0 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN1  60  /* UDN1 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN2  61  /* UDN2 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN3  62  /* UDN3 register, cause UDN_ACCESS exception */
+#define TILEGX_R_ZERO  63  /* Zero register, always zero */
+#define TILEGX_R_NOREG 255 /* Invalid register value */
+
+/* TILE-Gx special registers used by outside */
+enum {
+    TILEGX_SPR_CMPEXCH = 0,
+    TILEGX_SPR_CRITICAL_SEC = 1,
+    TILEGX_SPR_SIM_CONTROL = 2,
+    TILEGX_SPR_EX_CONTEXT_0_0 = 3,
+    TILEGX_SPR_EX_CONTEXT_0_1 = 4,
+    TILEGX_SPR_COUNT
+};
+
+/* Exception numbers */
+typedef enum {
+    TILEGX_EXCP_NONE = 0,
+    TILEGX_EXCP_SYSCALL = 1,
+    TILEGX_EXCP_SIGNAL = 2,
+    TILEGX_EXCP_OPCODE_UNKNOWN = 0x101,
+    TILEGX_EXCP_OPCODE_UNIMPLEMENTED = 0x102,
+    TILEGX_EXCP_OPCODE_CMPEXCH = 0x103,
+    TILEGX_EXCP_OPCODE_CMPEXCH4 = 0x104,
+    TILEGX_EXCP_OPCODE_EXCH = 0x105,
+    TILEGX_EXCP_OPCODE_EXCH4 = 0x106,
+    TILEGX_EXCP_OPCODE_FETCHADD = 0x107,
+    TILEGX_EXCP_OPCODE_FETCHADD4 = 0x108,
+    TILEGX_EXCP_OPCODE_FETCHADDGEZ = 0x109,
+    TILEGX_EXCP_OPCODE_FETCHADDGEZ4 = 0x10a,
+    TILEGX_EXCP_OPCODE_FETCHAND = 0x10b,
+    TILEGX_EXCP_OPCODE_FETCHAND4 = 0x10c,
+    TILEGX_EXCP_OPCODE_FETCHOR = 0x10d,
+    TILEGX_EXCP_OPCODE_FETCHOR4 = 0x10e,
+    TILEGX_EXCP_REG_IDN_ACCESS = 0x181,
+    TILEGX_EXCP_REG_UDN_ACCESS = 0x182,
+    TILEGX_EXCP_UNALIGNMENT = 0x201,
+    TILEGX_EXCP_DBUG_BREAK = 0x301
+} TileExcp;
+
+typedef struct CPUTLGState {
+    uint64_t regs[TILEGX_R_COUNT];     /* Common used registers by outside */
+    uint64_t spregs[TILEGX_SPR_COUNT]; /* Special used registers by outside */
+    uint64_t pc;                       /* Current pc */
+
+#if defined(CONFIG_USER_ONLY)
+    uint64_t excaddr;                  /* exception address */
+    uint64_t atomic_srca;              /* Arguments to atomic "exceptions" */
+    uint64_t atomic_srcb;
+    uint32_t atomic_dstr;
+    uint32_t signo;                    /* Signal number */
+    uint32_t sigcode;                  /* Signal code */
+#endif
+
+    CPU_COMMON
+} CPUTLGState;
+
+#include "qom/cpu.h"
+
+#define TYPE_TILEGX_CPU "tilegx-cpu"
+
+#define TILEGX_CPU_CLASS(klass) \
+    OBJECT_CLASS_CHECK(TileGXCPUClass, (klass), TYPE_TILEGX_CPU)
+#define TILEGX_CPU(obj) \
+    OBJECT_CHECK(TileGXCPU, (obj), TYPE_TILEGX_CPU)
+#define TILEGX_CPU_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(TileGXCPUClass, (obj), TYPE_TILEGX_CPU)
+
+/**
+ * TileGXCPUClass:
+ * @parent_realize: The parent class' realize handler.
+ * @parent_reset: The parent class' reset handler.
+ *
+ * A Tile-Gx CPU model.
+ */
+typedef struct TileGXCPUClass {
+    /*< private >*/
+    CPUClass parent_class;
+    /*< public >*/
+
+    DeviceRealize parent_realize;
+    void (*parent_reset)(CPUState *cpu);
+} TileGXCPUClass;
+
+/**
+ * TileGXCPU:
+ * @env: #CPUTLGState
+ *
+ * A Tile-GX CPU.
+ */
+typedef struct TileGXCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUTLGState env;
+} TileGXCPU;
+
+static inline TileGXCPU *tilegx_env_get_cpu(CPUTLGState *env)
+{
+    return container_of(env, TileGXCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(tilegx_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(TileGXCPU, env)
+
+/* TILE-Gx memory attributes */
+#define TARGET_PAGE_BITS 16  /* TILE-Gx uses 64KB page size */
+#define TARGET_PHYS_ADDR_SPACE_BITS 42
+#define TARGET_VIRT_ADDR_SPACE_BITS 64
+#define MMU_USER_IDX    0  /* Current memory operation is in user mode */
+
+#include "exec/cpu-all.h"
+
+void tilegx_tcg_init(void);
+int cpu_tilegx_exec(CPUState *s);
+int cpu_tilegx_signal_handler(int host_signum, void *pinfo, void *puc);
+
+TileGXCPU *cpu_tilegx_init(const char *cpu_model);
+
+#define cpu_init(cpu_model) CPU(cpu_tilegx_init(cpu_model))
+
+#define cpu_exec cpu_tilegx_exec
+#define cpu_signal_handler cpu_tilegx_signal_handler
+
+static inline void cpu_get_tb_cpu_state(CPUTLGState *env, target_ulong *pc,
+                                        target_ulong *cs_base, int *flags)
+{
+    *pc = env->pc;
+    *cs_base = 0;
+    *flags = 0;
+}
+
+#include "exec/exec-all.h"
+
+#endif
diff --git a/target-tilegx/helper.c b/target-tilegx/helper.c
new file mode 100644
index 000000000..dda821f5c
--- /dev/null
+++ b/target-tilegx/helper.c
@@ -0,0 +1,161 @@
+/*
+ * QEMU TILE-Gx helpers
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "cpu.h"
+#include "qemu-common.h"
+#include "exec/helper-proto.h"
+#include <zlib.h> /* For crc32 */
+#include "syscall_defs.h"
+
+void helper_exception(CPUTLGState *env, uint32_t excp)
+{
+    CPUState *cs = CPU(tilegx_env_get_cpu(env));
+
+    cs->exception_index = excp;
+    cpu_loop_exit(cs);
+}
+
+void helper_ext01_ics(CPUTLGState *env)
+{
+    uint64_t val = env->spregs[TILEGX_SPR_EX_CONTEXT_0_1];
+
+    switch (val) {
+    case 0:
+    case 1:
+        env->spregs[TILEGX_SPR_CRITICAL_SEC] = val;
+        break;
+    default:
+#if defined(CONFIG_USER_ONLY)
+        env->signo = TARGET_SIGILL;
+        env->sigcode = TARGET_ILL_ILLOPC;
+        helper_exception(env, TILEGX_EXCP_SIGNAL);
+#else
+        helper_exception(env, TILEGX_EXCP_OPCODE_UNIMPLEMENTED);
+#endif
+        break;
+    }
+}
+
+uint64_t helper_cntlz(uint64_t arg)
+{
+    return clz64(arg);
+}
+
+uint64_t helper_cnttz(uint64_t arg)
+{
+    return ctz64(arg);
+}
+
+uint64_t helper_pcnt(uint64_t arg)
+{
+    return ctpop64(arg);
+}
+
+uint64_t helper_revbits(uint64_t arg)
+{
+    return revbit64(arg);
+}
+
+/*
+ * Functional Description
+ *     uint64_t a = rf[SrcA];
+ *     uint64_t b = rf[SrcB];
+ *     uint64_t d = rf[Dest];
+ *     uint64_t output = 0;
+ *     unsigned int counter;
+ *     for (counter = 0; counter < (WORD_SIZE / BYTE_SIZE); counter++)
+ *     {
+ *         int sel = getByte (b, counter) & 0xf;
+ *         uint8_t byte = (sel < 8) ? getByte (d, sel) : getByte (a, (sel - 8));
+ *         output = setByte (output, counter, byte);
+ *     }
+ *     rf[Dest] = output;
+ */
+uint64_t helper_shufflebytes(uint64_t dest, uint64_t srca, uint64_t srcb)
+{
+    uint64_t vdst = 0;
+    int count;
+
+    for (count = 0; count < 64; count += 8) {
+        uint64_t sel = srcb >> count;
+        uint64_t src = (sel & 8) ? srca : dest;
+        vdst |= extract64(src, (sel & 7) * 8, 8) << count;
+    }
+
+    return vdst;
+}
+
+uint64_t helper_crc32_8(uint64_t accum, uint64_t input)
+{
+    uint8_t buf = input;
+
+    /* zlib crc32 converts the accumulator and output to one's complement.  */
+    return crc32(accum ^ 0xffffffff, &buf, 1) ^ 0xffffffff;
+}
+
+uint64_t helper_crc32_32(uint64_t accum, uint64_t input)
+{
+    uint8_t buf[4];
+
+    stl_le_p(buf, input);
+
+    /* zlib crc32 converts the accumulator and output to one's complement.  */
+    return crc32(accum ^ 0xffffffff, buf, 4) ^ 0xffffffff;
+}
+
+uint64_t helper_cmula(uint64_t srcd, uint64_t srca, uint64_t srcb)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    uint32_t reald = srcd;
+    uint32_t imagd = srcd >> 32;
+    uint32_t realr = reala * realb - imaga * imagb + reald;
+    uint32_t imagr = reala * imagb + imaga * realb + imagd;
+
+    return deposit64(realr, 32, 32, imagr);
+}
+
+uint64_t helper_cmulaf(uint64_t srcd, uint64_t srca, uint64_t srcb)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    uint32_t reald = (int16_t)srcd;
+    uint32_t imagd = (int16_t)(srcd >> 16);
+    int32_t realr = reala * realb - imaga * imagb;
+    int32_t imagr = reala * imagb + imaga * realb;
+
+    return deposit32((realr >> 15) + reald, 16, 16, (imagr >> 15) + imagd);
+}
+
+uint64_t helper_cmul2(uint64_t srca, uint64_t srcb, int shift, int round)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    int32_t realr = reala * realb - imaga * imagb + round;
+    int32_t imagr = reala * imagb + imaga * realb + round;
+
+    return deposit32(realr >> shift, 16, 16, imagr >> shift);
+}
diff --git a/target-tilegx/helper.h b/target-tilegx/helper.h
new file mode 100644
index 000000000..9281d0f42
--- /dev/null
+++ b/target-tilegx/helper.h
@@ -0,0 +1,26 @@
+DEF_HELPER_2(exception, noreturn, env, i32)
+DEF_HELPER_1(ext01_ics, void, env)
+DEF_HELPER_FLAGS_1(cntlz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(cnttz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(pcnt, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(revbits, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_3(shufflebytes, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(crc32_8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(crc32_32, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cmula, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cmulaf, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_4(cmul2, TCG_CALL_NO_RWG_SE, i64, i64, i64, int, int)
+
+DEF_HELPER_FLAGS_2(v1int_h, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1int_l, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2int_h, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2int_l, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(v1multu, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2mults, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shl, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shru, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shrs, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shl, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shru, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shrs, TCG_CALL_NO_RWG_SE, i64, i64, i64)
diff --git a/target-tilegx/opcode_tilegx.h b/target-tilegx/opcode_tilegx.h
new file mode 100644
index 000000000..989436d2f
--- /dev/null
+++ b/target-tilegx/opcode_tilegx.h
@@ -0,0 +1,1406 @@
+/* TILE-Gx opcode information.
+ *
+ * Copyright 2011 Tilera Corporation. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or
+ *   modify it under the terms of the GNU General Public License
+ *   as published by the Free Software Foundation, version 2.
+ *
+ *   This program is distributed in the hope that it will be useful, but
+ *   WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *   NON INFRINGEMENT.  See the GNU General Public License for
+ *   more details.
+ *
+ *
+ *
+ *
+ *
+ */
+
+#ifndef __ARCH_OPCODE_H__
+#define __ARCH_OPCODE_H__
+
+#ifndef __ASSEMBLER__
+
+typedef uint64_t tilegx_bundle_bits;
+
+/* These are the bits that determine if a bundle is in the X encoding. */
+#define TILEGX_BUNDLE_MODE_MASK ((tilegx_bundle_bits)3 << 62)
+
+enum
+{
+  /* Maximum number of instructions in a bundle (2 for X, 3 for Y). */
+  TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE = 3,
+
+  /* How many different pipeline encodings are there? X0, X1, Y0, Y1, Y2. */
+  TILEGX_NUM_PIPELINE_ENCODINGS = 5,
+
+  /* Log base 2 of TILEGX_BUNDLE_SIZE_IN_BYTES. */
+  TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES = 3,
+
+  /* Instructions take this many bytes. */
+  TILEGX_BUNDLE_SIZE_IN_BYTES = 1 << TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES,
+
+  /* Log base 2 of TILEGX_BUNDLE_ALIGNMENT_IN_BYTES. */
+  TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES = 3,
+
+  /* Bundles should be aligned modulo this number of bytes. */
+  TILEGX_BUNDLE_ALIGNMENT_IN_BYTES =
+    (1 << TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES),
+
+  /* Number of registers (some are magic, such as network I/O). */
+  TILEGX_NUM_REGISTERS = 64,
+};
+
+/* Make a few "tile_" variables to simplify common code between
+   architectures.  */
+
+typedef tilegx_bundle_bits tile_bundle_bits;
+#define TILE_BUNDLE_SIZE_IN_BYTES TILEGX_BUNDLE_SIZE_IN_BYTES
+#define TILE_BUNDLE_ALIGNMENT_IN_BYTES TILEGX_BUNDLE_ALIGNMENT_IN_BYTES
+#define TILE_LOG2_BUNDLE_ALIGNMENT_IN_BYTES \
+  TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES
+#define TILE_BPT_BUNDLE TILEGX_BPT_BUNDLE
+
+/* 64-bit pattern for a { bpt ; nop } bundle. */
+#define TILEGX_BPT_BUNDLE 0x286a44ae51485000ULL
+
+static inline unsigned int
+get_BFEnd_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_BFOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 24)) & 0xf);
+}
+
+static inline unsigned int
+get_BFStart_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3f);
+}
+
+static inline unsigned int
+get_BrOff_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 37)) & 0x0001ffc0);
+}
+
+static inline unsigned int
+get_BrType_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 54)) & 0x1f);
+}
+
+static inline unsigned int
+get_Dest_Imm8_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 43)) & 0x000000c0);
+}
+
+static inline unsigned int
+get_Dest_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 0)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 0)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x3f);
+}
+
+static inline unsigned int
+get_Imm16_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xffff);
+}
+
+static inline unsigned int
+get_Imm16_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xffff);
+}
+
+static inline unsigned int
+get_Imm8OpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 20)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8OpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 51)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xff);
+}
+
+static inline unsigned int
+get_JumpOff_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x7ffffff);
+}
+
+static inline unsigned int
+get_JumpOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 58)) & 0x1);
+}
+
+static inline unsigned int
+get_MF_Imm14_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3fff);
+}
+
+static inline unsigned int
+get_MT_Imm14_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 37)) & 0x00003fc0);
+}
+
+static inline unsigned int
+get_Mode(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 62)) & 0x3);
+}
+
+static inline unsigned int
+get_Opcode_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 28)) & 0x7);
+}
+
+static inline unsigned int
+get_Opcode_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 59)) & 0x7);
+}
+
+static inline unsigned int
+get_Opcode_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 27)) & 0xf);
+}
+
+static inline unsigned int
+get_Opcode_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 58)) & 0xf);
+}
+
+static inline unsigned int
+get_Opcode_Y2(tilegx_bundle_bits n)
+{
+  return (((n >> 26)) & 0x00000001) |
+         (((unsigned int)(n >> 56)) & 0x00000002);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3ff);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3ff);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3);
+}
+
+static inline unsigned int
+get_ShAmt_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3ff);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3ff);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3);
+}
+
+static inline unsigned int
+get_SrcA_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 6)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 6)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y2(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 20)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcBDest_Y2(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 51)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+
+static inline int
+sign_extend(int n, int num_bits)
+{
+  int shift = (int)(sizeof(int) * 8 - num_bits);
+  return (n << shift) >> shift;
+}
+
+
+
+static inline tilegx_bundle_bits
+create_BFEnd_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_BFOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xf) << 24);
+}
+
+static inline tilegx_bundle_bits
+create_BFStart_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_BrOff_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x0001ffc0)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_BrType_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x1f)) << 54);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Imm8_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x000000c0)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 0);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 0);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_Imm16_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xffff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm16_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xffff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8OpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 20);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8OpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 51);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_JumpOff_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x7ffffff)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_JumpOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x1)) << 58);
+}
+
+static inline tilegx_bundle_bits
+create_MF_Imm14_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3fff)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_MT_Imm14_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x00003fc0)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_Mode(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 62);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x7) << 28);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x7)) << 59);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xf) << 27);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xf)) << 58);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x00000001) << 26) |
+         (((tilegx_bundle_bits)(n & 0x00000002)) << 56);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3ff) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3ff)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3ff) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3ff)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 6);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 6);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 20);
+}
+
+static inline tilegx_bundle_bits
+create_SrcBDest_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 51);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+
+enum
+{
+  ADDI_IMM8_OPCODE_X0 = 1,
+  ADDI_IMM8_OPCODE_X1 = 1,
+  ADDI_OPCODE_Y0 = 0,
+  ADDI_OPCODE_Y1 = 1,
+  ADDLI_OPCODE_X0 = 1,
+  ADDLI_OPCODE_X1 = 0,
+  ADDXI_IMM8_OPCODE_X0 = 2,
+  ADDXI_IMM8_OPCODE_X1 = 2,
+  ADDXI_OPCODE_Y0 = 1,
+  ADDXI_OPCODE_Y1 = 2,
+  ADDXLI_OPCODE_X0 = 2,
+  ADDXLI_OPCODE_X1 = 1,
+  ADDXSC_RRR_0_OPCODE_X0 = 1,
+  ADDXSC_RRR_0_OPCODE_X1 = 1,
+  ADDX_RRR_0_OPCODE_X0 = 2,
+  ADDX_RRR_0_OPCODE_X1 = 2,
+  ADDX_RRR_0_OPCODE_Y0 = 0,
+  ADDX_RRR_0_OPCODE_Y1 = 0,
+  ADD_RRR_0_OPCODE_X0 = 3,
+  ADD_RRR_0_OPCODE_X1 = 3,
+  ADD_RRR_0_OPCODE_Y0 = 1,
+  ADD_RRR_0_OPCODE_Y1 = 1,
+  ANDI_IMM8_OPCODE_X0 = 3,
+  ANDI_IMM8_OPCODE_X1 = 3,
+  ANDI_OPCODE_Y0 = 2,
+  ANDI_OPCODE_Y1 = 3,
+  AND_RRR_0_OPCODE_X0 = 4,
+  AND_RRR_0_OPCODE_X1 = 4,
+  AND_RRR_5_OPCODE_Y0 = 0,
+  AND_RRR_5_OPCODE_Y1 = 0,
+  BEQZT_BRANCH_OPCODE_X1 = 16,
+  BEQZ_BRANCH_OPCODE_X1 = 17,
+  BFEXTS_BF_OPCODE_X0 = 4,
+  BFEXTU_BF_OPCODE_X0 = 5,
+  BFINS_BF_OPCODE_X0 = 6,
+  BF_OPCODE_X0 = 3,
+  BGEZT_BRANCH_OPCODE_X1 = 18,
+  BGEZ_BRANCH_OPCODE_X1 = 19,
+  BGTZT_BRANCH_OPCODE_X1 = 20,
+  BGTZ_BRANCH_OPCODE_X1 = 21,
+  BLBCT_BRANCH_OPCODE_X1 = 22,
+  BLBC_BRANCH_OPCODE_X1 = 23,
+  BLBST_BRANCH_OPCODE_X1 = 24,
+  BLBS_BRANCH_OPCODE_X1 = 25,
+  BLEZT_BRANCH_OPCODE_X1 = 26,
+  BLEZ_BRANCH_OPCODE_X1 = 27,
+  BLTZT_BRANCH_OPCODE_X1 = 28,
+  BLTZ_BRANCH_OPCODE_X1 = 29,
+  BNEZT_BRANCH_OPCODE_X1 = 30,
+  BNEZ_BRANCH_OPCODE_X1 = 31,
+  BRANCH_OPCODE_X1 = 2,
+  CMOVEQZ_RRR_0_OPCODE_X0 = 5,
+  CMOVEQZ_RRR_4_OPCODE_Y0 = 0,
+  CMOVNEZ_RRR_0_OPCODE_X0 = 6,
+  CMOVNEZ_RRR_4_OPCODE_Y0 = 1,
+  CMPEQI_IMM8_OPCODE_X0 = 4,
+  CMPEQI_IMM8_OPCODE_X1 = 4,
+  CMPEQI_OPCODE_Y0 = 3,
+  CMPEQI_OPCODE_Y1 = 4,
+  CMPEQ_RRR_0_OPCODE_X0 = 7,
+  CMPEQ_RRR_0_OPCODE_X1 = 5,
+  CMPEQ_RRR_3_OPCODE_Y0 = 0,
+  CMPEQ_RRR_3_OPCODE_Y1 = 2,
+  CMPEXCH4_RRR_0_OPCODE_X1 = 6,
+  CMPEXCH_RRR_0_OPCODE_X1 = 7,
+  CMPLES_RRR_0_OPCODE_X0 = 8,
+  CMPLES_RRR_0_OPCODE_X1 = 8,
+  CMPLES_RRR_2_OPCODE_Y0 = 0,
+  CMPLES_RRR_2_OPCODE_Y1 = 0,
+  CMPLEU_RRR_0_OPCODE_X0 = 9,
+  CMPLEU_RRR_0_OPCODE_X1 = 9,
+  CMPLEU_RRR_2_OPCODE_Y0 = 1,
+  CMPLEU_RRR_2_OPCODE_Y1 = 1,
+  CMPLTSI_IMM8_OPCODE_X0 = 5,
+  CMPLTSI_IMM8_OPCODE_X1 = 5,
+  CMPLTSI_OPCODE_Y0 = 4,
+  CMPLTSI_OPCODE_Y1 = 5,
+  CMPLTS_RRR_0_OPCODE_X0 = 10,
+  CMPLTS_RRR_0_OPCODE_X1 = 10,
+  CMPLTS_RRR_2_OPCODE_Y0 = 2,
+  CMPLTS_RRR_2_OPCODE_Y1 = 2,
+  CMPLTUI_IMM8_OPCODE_X0 = 6,
+  CMPLTUI_IMM8_OPCODE_X1 = 6,
+  CMPLTU_RRR_0_OPCODE_X0 = 11,
+  CMPLTU_RRR_0_OPCODE_X1 = 11,
+  CMPLTU_RRR_2_OPCODE_Y0 = 3,
+  CMPLTU_RRR_2_OPCODE_Y1 = 3,
+  CMPNE_RRR_0_OPCODE_X0 = 12,
+  CMPNE_RRR_0_OPCODE_X1 = 12,
+  CMPNE_RRR_3_OPCODE_Y0 = 1,
+  CMPNE_RRR_3_OPCODE_Y1 = 3,
+  CMULAF_RRR_0_OPCODE_X0 = 13,
+  CMULA_RRR_0_OPCODE_X0 = 14,
+  CMULFR_RRR_0_OPCODE_X0 = 15,
+  CMULF_RRR_0_OPCODE_X0 = 16,
+  CMULHR_RRR_0_OPCODE_X0 = 17,
+  CMULH_RRR_0_OPCODE_X0 = 18,
+  CMUL_RRR_0_OPCODE_X0 = 19,
+  CNTLZ_UNARY_OPCODE_X0 = 1,
+  CNTLZ_UNARY_OPCODE_Y0 = 1,
+  CNTTZ_UNARY_OPCODE_X0 = 2,
+  CNTTZ_UNARY_OPCODE_Y0 = 2,
+  CRC32_32_RRR_0_OPCODE_X0 = 20,
+  CRC32_8_RRR_0_OPCODE_X0 = 21,
+  DBLALIGN2_RRR_0_OPCODE_X0 = 22,
+  DBLALIGN2_RRR_0_OPCODE_X1 = 13,
+  DBLALIGN4_RRR_0_OPCODE_X0 = 23,
+  DBLALIGN4_RRR_0_OPCODE_X1 = 14,
+  DBLALIGN6_RRR_0_OPCODE_X0 = 24,
+  DBLALIGN6_RRR_0_OPCODE_X1 = 15,
+  DBLALIGN_RRR_0_OPCODE_X0 = 25,
+  DRAIN_UNARY_OPCODE_X1 = 1,
+  DTLBPR_UNARY_OPCODE_X1 = 2,
+  EXCH4_RRR_0_OPCODE_X1 = 16,
+  EXCH_RRR_0_OPCODE_X1 = 17,
+  FDOUBLE_ADDSUB_RRR_0_OPCODE_X0 = 26,
+  FDOUBLE_ADD_FLAGS_RRR_0_OPCODE_X0 = 27,
+  FDOUBLE_MUL_FLAGS_RRR_0_OPCODE_X0 = 28,
+  FDOUBLE_PACK1_RRR_0_OPCODE_X0 = 29,
+  FDOUBLE_PACK2_RRR_0_OPCODE_X0 = 30,
+  FDOUBLE_SUB_FLAGS_RRR_0_OPCODE_X0 = 31,
+  FDOUBLE_UNPACK_MAX_RRR_0_OPCODE_X0 = 32,
+  FDOUBLE_UNPACK_MIN_RRR_0_OPCODE_X0 = 33,
+  FETCHADD4_RRR_0_OPCODE_X1 = 18,
+  FETCHADDGEZ4_RRR_0_OPCODE_X1 = 19,
+  FETCHADDGEZ_RRR_0_OPCODE_X1 = 20,
+  FETCHADD_RRR_0_OPCODE_X1 = 21,
+  FETCHAND4_RRR_0_OPCODE_X1 = 22,
+  FETCHAND_RRR_0_OPCODE_X1 = 23,
+  FETCHOR4_RRR_0_OPCODE_X1 = 24,
+  FETCHOR_RRR_0_OPCODE_X1 = 25,
+  FINV_UNARY_OPCODE_X1 = 3,
+  FLUSHWB_UNARY_OPCODE_X1 = 4,
+  FLUSH_UNARY_OPCODE_X1 = 5,
+  FNOP_UNARY_OPCODE_X0 = 3,
+  FNOP_UNARY_OPCODE_X1 = 6,
+  FNOP_UNARY_OPCODE_Y0 = 3,
+  FNOP_UNARY_OPCODE_Y1 = 8,
+  FSINGLE_ADD1_RRR_0_OPCODE_X0 = 34,
+  FSINGLE_ADDSUB2_RRR_0_OPCODE_X0 = 35,
+  FSINGLE_MUL1_RRR_0_OPCODE_X0 = 36,
+  FSINGLE_MUL2_RRR_0_OPCODE_X0 = 37,
+  FSINGLE_PACK1_UNARY_OPCODE_X0 = 4,
+  FSINGLE_PACK1_UNARY_OPCODE_Y0 = 4,
+  FSINGLE_PACK2_RRR_0_OPCODE_X0 = 38,
+  FSINGLE_SUB1_RRR_0_OPCODE_X0 = 39,
+  ICOH_UNARY_OPCODE_X1 = 7,
+  ILL_UNARY_OPCODE_X1 = 8,
+  ILL_UNARY_OPCODE_Y1 = 9,
+  IMM8_OPCODE_X0 = 4,
+  IMM8_OPCODE_X1 = 3,
+  INV_UNARY_OPCODE_X1 = 9,
+  IRET_UNARY_OPCODE_X1 = 10,
+  JALRP_UNARY_OPCODE_X1 = 11,
+  JALRP_UNARY_OPCODE_Y1 = 10,
+  JALR_UNARY_OPCODE_X1 = 12,
+  JALR_UNARY_OPCODE_Y1 = 11,
+  JAL_JUMP_OPCODE_X1 = 0,
+  JRP_UNARY_OPCODE_X1 = 13,
+  JRP_UNARY_OPCODE_Y1 = 12,
+  JR_UNARY_OPCODE_X1 = 14,
+  JR_UNARY_OPCODE_Y1 = 13,
+  JUMP_OPCODE_X1 = 4,
+  J_JUMP_OPCODE_X1 = 1,
+  LD1S_ADD_IMM8_OPCODE_X1 = 7,
+  LD1S_OPCODE_Y2 = 0,
+  LD1S_UNARY_OPCODE_X1 = 15,
+  LD1U_ADD_IMM8_OPCODE_X1 = 8,
+  LD1U_OPCODE_Y2 = 1,
+  LD1U_UNARY_OPCODE_X1 = 16,
+  LD2S_ADD_IMM8_OPCODE_X1 = 9,
+  LD2S_OPCODE_Y2 = 2,
+  LD2S_UNARY_OPCODE_X1 = 17,
+  LD2U_ADD_IMM8_OPCODE_X1 = 10,
+  LD2U_OPCODE_Y2 = 3,
+  LD2U_UNARY_OPCODE_X1 = 18,
+  LD4S_ADD_IMM8_OPCODE_X1 = 11,
+  LD4S_OPCODE_Y2 = 1,
+  LD4S_UNARY_OPCODE_X1 = 19,
+  LD4U_ADD_IMM8_OPCODE_X1 = 12,
+  LD4U_OPCODE_Y2 = 2,
+  LD4U_UNARY_OPCODE_X1 = 20,
+  LDNA_UNARY_OPCODE_X1 = 21,
+  LDNT1S_ADD_IMM8_OPCODE_X1 = 13,
+  LDNT1S_UNARY_OPCODE_X1 = 22,
+  LDNT1U_ADD_IMM8_OPCODE_X1 = 14,
+  LDNT1U_UNARY_OPCODE_X1 = 23,
+  LDNT2S_ADD_IMM8_OPCODE_X1 = 15,
+  LDNT2S_UNARY_OPCODE_X1 = 24,
+  LDNT2U_ADD_IMM8_OPCODE_X1 = 16,
+  LDNT2U_UNARY_OPCODE_X1 = 25,
+  LDNT4S_ADD_IMM8_OPCODE_X1 = 17,
+  LDNT4S_UNARY_OPCODE_X1 = 26,
+  LDNT4U_ADD_IMM8_OPCODE_X1 = 18,
+  LDNT4U_UNARY_OPCODE_X1 = 27,
+  LDNT_ADD_IMM8_OPCODE_X1 = 19,
+  LDNT_UNARY_OPCODE_X1 = 28,
+  LD_ADD_IMM8_OPCODE_X1 = 20,
+  LD_OPCODE_Y2 = 3,
+  LD_UNARY_OPCODE_X1 = 29,
+  LNK_UNARY_OPCODE_X1 = 30,
+  LNK_UNARY_OPCODE_Y1 = 14,
+  LDNA_ADD_IMM8_OPCODE_X1 = 21,
+  MFSPR_IMM8_OPCODE_X1 = 22,
+  MF_UNARY_OPCODE_X1 = 31,
+  MM_BF_OPCODE_X0 = 7,
+  MNZ_RRR_0_OPCODE_X0 = 40,
+  MNZ_RRR_0_OPCODE_X1 = 26,
+  MNZ_RRR_4_OPCODE_Y0 = 2,
+  MNZ_RRR_4_OPCODE_Y1 = 2,
+  MODE_OPCODE_YA2 = 1,
+  MODE_OPCODE_YB2 = 2,
+  MODE_OPCODE_YC2 = 3,
+  MTSPR_IMM8_OPCODE_X1 = 23,
+  MULAX_RRR_0_OPCODE_X0 = 41,
+  MULAX_RRR_3_OPCODE_Y0 = 2,
+  MULA_HS_HS_RRR_0_OPCODE_X0 = 42,
+  MULA_HS_HS_RRR_9_OPCODE_Y0 = 0,
+  MULA_HS_HU_RRR_0_OPCODE_X0 = 43,
+  MULA_HS_LS_RRR_0_OPCODE_X0 = 44,
+  MULA_HS_LU_RRR_0_OPCODE_X0 = 45,
+  MULA_HU_HU_RRR_0_OPCODE_X0 = 46,
+  MULA_HU_HU_RRR_9_OPCODE_Y0 = 1,
+  MULA_HU_LS_RRR_0_OPCODE_X0 = 47,
+  MULA_HU_LU_RRR_0_OPCODE_X0 = 48,
+  MULA_LS_LS_RRR_0_OPCODE_X0 = 49,
+  MULA_LS_LS_RRR_9_OPCODE_Y0 = 2,
+  MULA_LS_LU_RRR_0_OPCODE_X0 = 50,
+  MULA_LU_LU_RRR_0_OPCODE_X0 = 51,
+  MULA_LU_LU_RRR_9_OPCODE_Y0 = 3,
+  MULX_RRR_0_OPCODE_X0 = 52,
+  MULX_RRR_3_OPCODE_Y0 = 3,
+  MUL_HS_HS_RRR_0_OPCODE_X0 = 53,
+  MUL_HS_HS_RRR_8_OPCODE_Y0 = 0,
+  MUL_HS_HU_RRR_0_OPCODE_X0 = 54,
+  MUL_HS_LS_RRR_0_OPCODE_X0 = 55,
+  MUL_HS_LU_RRR_0_OPCODE_X0 = 56,
+  MUL_HU_HU_RRR_0_OPCODE_X0 = 57,
+  MUL_HU_HU_RRR_8_OPCODE_Y0 = 1,
+  MUL_HU_LS_RRR_0_OPCODE_X0 = 58,
+  MUL_HU_LU_RRR_0_OPCODE_X0 = 59,
+  MUL_LS_LS_RRR_0_OPCODE_X0 = 60,
+  MUL_LS_LS_RRR_8_OPCODE_Y0 = 2,
+  MUL_LS_LU_RRR_0_OPCODE_X0 = 61,
+  MUL_LU_LU_RRR_0_OPCODE_X0 = 62,
+  MUL_LU_LU_RRR_8_OPCODE_Y0 = 3,
+  MZ_RRR_0_OPCODE_X0 = 63,
+  MZ_RRR_0_OPCODE_X1 = 27,
+  MZ_RRR_4_OPCODE_Y0 = 3,
+  MZ_RRR_4_OPCODE_Y1 = 3,
+  NAP_UNARY_OPCODE_X1 = 32,
+  NOP_UNARY_OPCODE_X0 = 5,
+  NOP_UNARY_OPCODE_X1 = 33,
+  NOP_UNARY_OPCODE_Y0 = 5,
+  NOP_UNARY_OPCODE_Y1 = 15,
+  NOR_RRR_0_OPCODE_X0 = 64,
+  NOR_RRR_0_OPCODE_X1 = 28,
+  NOR_RRR_5_OPCODE_Y0 = 1,
+  NOR_RRR_5_OPCODE_Y1 = 1,
+  ORI_IMM8_OPCODE_X0 = 7,
+  ORI_IMM8_OPCODE_X1 = 24,
+  OR_RRR_0_OPCODE_X0 = 65,
+  OR_RRR_0_OPCODE_X1 = 29,
+  OR_RRR_5_OPCODE_Y0 = 2,
+  OR_RRR_5_OPCODE_Y1 = 2,
+  PCNT_UNARY_OPCODE_X0 = 6,
+  PCNT_UNARY_OPCODE_Y0 = 6,
+  REVBITS_UNARY_OPCODE_X0 = 7,
+  REVBITS_UNARY_OPCODE_Y0 = 7,
+  REVBYTES_UNARY_OPCODE_X0 = 8,
+  REVBYTES_UNARY_OPCODE_Y0 = 8,
+  ROTLI_SHIFT_OPCODE_X0 = 1,
+  ROTLI_SHIFT_OPCODE_X1 = 1,
+  ROTLI_SHIFT_OPCODE_Y0 = 0,
+  ROTLI_SHIFT_OPCODE_Y1 = 0,
+  ROTL_RRR_0_OPCODE_X0 = 66,
+  ROTL_RRR_0_OPCODE_X1 = 30,
+  ROTL_RRR_6_OPCODE_Y0 = 0,
+  ROTL_RRR_6_OPCODE_Y1 = 0,
+  RRR_0_OPCODE_X0 = 5,
+  RRR_0_OPCODE_X1 = 5,
+  RRR_0_OPCODE_Y0 = 5,
+  RRR_0_OPCODE_Y1 = 6,
+  RRR_1_OPCODE_Y0 = 6,
+  RRR_1_OPCODE_Y1 = 7,
+  RRR_2_OPCODE_Y0 = 7,
+  RRR_2_OPCODE_Y1 = 8,
+  RRR_3_OPCODE_Y0 = 8,
+  RRR_3_OPCODE_Y1 = 9,
+  RRR_4_OPCODE_Y0 = 9,
+  RRR_4_OPCODE_Y1 = 10,
+  RRR_5_OPCODE_Y0 = 10,
+  RRR_5_OPCODE_Y1 = 11,
+  RRR_6_OPCODE_Y0 = 11,
+  RRR_6_OPCODE_Y1 = 12,
+  RRR_7_OPCODE_Y0 = 12,
+  RRR_7_OPCODE_Y1 = 13,
+  RRR_8_OPCODE_Y0 = 13,
+  RRR_9_OPCODE_Y0 = 14,
+  SHIFT_OPCODE_X0 = 6,
+  SHIFT_OPCODE_X1 = 6,
+  SHIFT_OPCODE_Y0 = 15,
+  SHIFT_OPCODE_Y1 = 14,
+  SHL16INSLI_OPCODE_X0 = 7,
+  SHL16INSLI_OPCODE_X1 = 7,
+  SHL1ADDX_RRR_0_OPCODE_X0 = 67,
+  SHL1ADDX_RRR_0_OPCODE_X1 = 31,
+  SHL1ADDX_RRR_7_OPCODE_Y0 = 1,
+  SHL1ADDX_RRR_7_OPCODE_Y1 = 1,
+  SHL1ADD_RRR_0_OPCODE_X0 = 68,
+  SHL1ADD_RRR_0_OPCODE_X1 = 32,
+  SHL1ADD_RRR_1_OPCODE_Y0 = 0,
+  SHL1ADD_RRR_1_OPCODE_Y1 = 0,
+  SHL2ADDX_RRR_0_OPCODE_X0 = 69,
+  SHL2ADDX_RRR_0_OPCODE_X1 = 33,
+  SHL2ADDX_RRR_7_OPCODE_Y0 = 2,
+  SHL2ADDX_RRR_7_OPCODE_Y1 = 2,
+  SHL2ADD_RRR_0_OPCODE_X0 = 70,
+  SHL2ADD_RRR_0_OPCODE_X1 = 34,
+  SHL2ADD_RRR_1_OPCODE_Y0 = 1,
+  SHL2ADD_RRR_1_OPCODE_Y1 = 1,
+  SHL3ADDX_RRR_0_OPCODE_X0 = 71,
+  SHL3ADDX_RRR_0_OPCODE_X1 = 35,
+  SHL3ADDX_RRR_7_OPCODE_Y0 = 3,
+  SHL3ADDX_RRR_7_OPCODE_Y1 = 3,
+  SHL3ADD_RRR_0_OPCODE_X0 = 72,
+  SHL3ADD_RRR_0_OPCODE_X1 = 36,
+  SHL3ADD_RRR_1_OPCODE_Y0 = 2,
+  SHL3ADD_RRR_1_OPCODE_Y1 = 2,
+  SHLI_SHIFT_OPCODE_X0 = 2,
+  SHLI_SHIFT_OPCODE_X1 = 2,
+  SHLI_SHIFT_OPCODE_Y0 = 1,
+  SHLI_SHIFT_OPCODE_Y1 = 1,
+  SHLXI_SHIFT_OPCODE_X0 = 3,
+  SHLXI_SHIFT_OPCODE_X1 = 3,
+  SHLX_RRR_0_OPCODE_X0 = 73,
+  SHLX_RRR_0_OPCODE_X1 = 37,
+  SHL_RRR_0_OPCODE_X0 = 74,
+  SHL_RRR_0_OPCODE_X1 = 38,
+  SHL_RRR_6_OPCODE_Y0 = 1,
+  SHL_RRR_6_OPCODE_Y1 = 1,
+  SHRSI_SHIFT_OPCODE_X0 = 4,
+  SHRSI_SHIFT_OPCODE_X1 = 4,
+  SHRSI_SHIFT_OPCODE_Y0 = 2,
+  SHRSI_SHIFT_OPCODE_Y1 = 2,
+  SHRS_RRR_0_OPCODE_X0 = 75,
+  SHRS_RRR_0_OPCODE_X1 = 39,
+  SHRS_RRR_6_OPCODE_Y0 = 2,
+  SHRS_RRR_6_OPCODE_Y1 = 2,
+  SHRUI_SHIFT_OPCODE_X0 = 5,
+  SHRUI_SHIFT_OPCODE_X1 = 5,
+  SHRUI_SHIFT_OPCODE_Y0 = 3,
+  SHRUI_SHIFT_OPCODE_Y1 = 3,
+  SHRUXI_SHIFT_OPCODE_X0 = 6,
+  SHRUXI_SHIFT_OPCODE_X1 = 6,
+  SHRUX_RRR_0_OPCODE_X0 = 76,
+  SHRUX_RRR_0_OPCODE_X1 = 40,
+  SHRU_RRR_0_OPCODE_X0 = 77,
+  SHRU_RRR_0_OPCODE_X1 = 41,
+  SHRU_RRR_6_OPCODE_Y0 = 3,
+  SHRU_RRR_6_OPCODE_Y1 = 3,
+  SHUFFLEBYTES_RRR_0_OPCODE_X0 = 78,
+  ST1_ADD_IMM8_OPCODE_X1 = 25,
+  ST1_OPCODE_Y2 = 0,
+  ST1_RRR_0_OPCODE_X1 = 42,
+  ST2_ADD_IMM8_OPCODE_X1 = 26,
+  ST2_OPCODE_Y2 = 1,
+  ST2_RRR_0_OPCODE_X1 = 43,
+  ST4_ADD_IMM8_OPCODE_X1 = 27,
+  ST4_OPCODE_Y2 = 2,
+  ST4_RRR_0_OPCODE_X1 = 44,
+  STNT1_ADD_IMM8_OPCODE_X1 = 28,
+  STNT1_RRR_0_OPCODE_X1 = 45,
+  STNT2_ADD_IMM8_OPCODE_X1 = 29,
+  STNT2_RRR_0_OPCODE_X1 = 46,
+  STNT4_ADD_IMM8_OPCODE_X1 = 30,
+  STNT4_RRR_0_OPCODE_X1 = 47,
+  STNT_ADD_IMM8_OPCODE_X1 = 31,
+  STNT_RRR_0_OPCODE_X1 = 48,
+  ST_ADD_IMM8_OPCODE_X1 = 32,
+  ST_OPCODE_Y2 = 3,
+  ST_RRR_0_OPCODE_X1 = 49,
+  SUBXSC_RRR_0_OPCODE_X0 = 79,
+  SUBXSC_RRR_0_OPCODE_X1 = 50,
+  SUBX_RRR_0_OPCODE_X0 = 80,
+  SUBX_RRR_0_OPCODE_X1 = 51,
+  SUBX_RRR_0_OPCODE_Y0 = 2,
+  SUBX_RRR_0_OPCODE_Y1 = 2,
+  SUB_RRR_0_OPCODE_X0 = 81,
+  SUB_RRR_0_OPCODE_X1 = 52,
+  SUB_RRR_0_OPCODE_Y0 = 3,
+  SUB_RRR_0_OPCODE_Y1 = 3,
+  SWINT0_UNARY_OPCODE_X1 = 34,
+  SWINT1_UNARY_OPCODE_X1 = 35,
+  SWINT2_UNARY_OPCODE_X1 = 36,
+  SWINT3_UNARY_OPCODE_X1 = 37,
+  TBLIDXB0_UNARY_OPCODE_X0 = 9,
+  TBLIDXB0_UNARY_OPCODE_Y0 = 9,
+  TBLIDXB1_UNARY_OPCODE_X0 = 10,
+  TBLIDXB1_UNARY_OPCODE_Y0 = 10,
+  TBLIDXB2_UNARY_OPCODE_X0 = 11,
+  TBLIDXB2_UNARY_OPCODE_Y0 = 11,
+  TBLIDXB3_UNARY_OPCODE_X0 = 12,
+  TBLIDXB3_UNARY_OPCODE_Y0 = 12,
+  UNARY_RRR_0_OPCODE_X0 = 82,
+  UNARY_RRR_0_OPCODE_X1 = 53,
+  UNARY_RRR_1_OPCODE_Y0 = 3,
+  UNARY_RRR_1_OPCODE_Y1 = 3,
+  V1ADDI_IMM8_OPCODE_X0 = 8,
+  V1ADDI_IMM8_OPCODE_X1 = 33,
+  V1ADDUC_RRR_0_OPCODE_X0 = 83,
+  V1ADDUC_RRR_0_OPCODE_X1 = 54,
+  V1ADD_RRR_0_OPCODE_X0 = 84,
+  V1ADD_RRR_0_OPCODE_X1 = 55,
+  V1ADIFFU_RRR_0_OPCODE_X0 = 85,
+  V1AVGU_RRR_0_OPCODE_X0 = 86,
+  V1CMPEQI_IMM8_OPCODE_X0 = 9,
+  V1CMPEQI_IMM8_OPCODE_X1 = 34,
+  V1CMPEQ_RRR_0_OPCODE_X0 = 87,
+  V1CMPEQ_RRR_0_OPCODE_X1 = 56,
+  V1CMPLES_RRR_0_OPCODE_X0 = 88,
+  V1CMPLES_RRR_0_OPCODE_X1 = 57,
+  V1CMPLEU_RRR_0_OPCODE_X0 = 89,
+  V1CMPLEU_RRR_0_OPCODE_X1 = 58,
+  V1CMPLTSI_IMM8_OPCODE_X0 = 10,
+  V1CMPLTSI_IMM8_OPCODE_X1 = 35,
+  V1CMPLTS_RRR_0_OPCODE_X0 = 90,
+  V1CMPLTS_RRR_0_OPCODE_X1 = 59,
+  V1CMPLTUI_IMM8_OPCODE_X0 = 11,
+  V1CMPLTUI_IMM8_OPCODE_X1 = 36,
+  V1CMPLTU_RRR_0_OPCODE_X0 = 91,
+  V1CMPLTU_RRR_0_OPCODE_X1 = 60,
+  V1CMPNE_RRR_0_OPCODE_X0 = 92,
+  V1CMPNE_RRR_0_OPCODE_X1 = 61,
+  V1DDOTPUA_RRR_0_OPCODE_X0 = 161,
+  V1DDOTPUSA_RRR_0_OPCODE_X0 = 93,
+  V1DDOTPUS_RRR_0_OPCODE_X0 = 94,
+  V1DDOTPU_RRR_0_OPCODE_X0 = 162,
+  V1DOTPA_RRR_0_OPCODE_X0 = 95,
+  V1DOTPUA_RRR_0_OPCODE_X0 = 163,
+  V1DOTPUSA_RRR_0_OPCODE_X0 = 96,
+  V1DOTPUS_RRR_0_OPCODE_X0 = 97,
+  V1DOTPU_RRR_0_OPCODE_X0 = 164,
+  V1DOTP_RRR_0_OPCODE_X0 = 98,
+  V1INT_H_RRR_0_OPCODE_X0 = 99,
+  V1INT_H_RRR_0_OPCODE_X1 = 62,
+  V1INT_L_RRR_0_OPCODE_X0 = 100,
+  V1INT_L_RRR_0_OPCODE_X1 = 63,
+  V1MAXUI_IMM8_OPCODE_X0 = 12,
+  V1MAXUI_IMM8_OPCODE_X1 = 37,
+  V1MAXU_RRR_0_OPCODE_X0 = 101,
+  V1MAXU_RRR_0_OPCODE_X1 = 64,
+  V1MINUI_IMM8_OPCODE_X0 = 13,
+  V1MINUI_IMM8_OPCODE_X1 = 38,
+  V1MINU_RRR_0_OPCODE_X0 = 102,
+  V1MINU_RRR_0_OPCODE_X1 = 65,
+  V1MNZ_RRR_0_OPCODE_X0 = 103,
+  V1MNZ_RRR_0_OPCODE_X1 = 66,
+  V1MULTU_RRR_0_OPCODE_X0 = 104,
+  V1MULUS_RRR_0_OPCODE_X0 = 105,
+  V1MULU_RRR_0_OPCODE_X0 = 106,
+  V1MZ_RRR_0_OPCODE_X0 = 107,
+  V1MZ_RRR_0_OPCODE_X1 = 67,
+  V1SADAU_RRR_0_OPCODE_X0 = 108,
+  V1SADU_RRR_0_OPCODE_X0 = 109,
+  V1SHLI_SHIFT_OPCODE_X0 = 7,
+  V1SHLI_SHIFT_OPCODE_X1 = 7,
+  V1SHL_RRR_0_OPCODE_X0 = 110,
+  V1SHL_RRR_0_OPCODE_X1 = 68,
+  V1SHRSI_SHIFT_OPCODE_X0 = 8,
+  V1SHRSI_SHIFT_OPCODE_X1 = 8,
+  V1SHRS_RRR_0_OPCODE_X0 = 111,
+  V1SHRS_RRR_0_OPCODE_X1 = 69,
+  V1SHRUI_SHIFT_OPCODE_X0 = 9,
+  V1SHRUI_SHIFT_OPCODE_X1 = 9,
+  V1SHRU_RRR_0_OPCODE_X0 = 112,
+  V1SHRU_RRR_0_OPCODE_X1 = 70,
+  V1SUBUC_RRR_0_OPCODE_X0 = 113,
+  V1SUBUC_RRR_0_OPCODE_X1 = 71,
+  V1SUB_RRR_0_OPCODE_X0 = 114,
+  V1SUB_RRR_0_OPCODE_X1 = 72,
+  V2ADDI_IMM8_OPCODE_X0 = 14,
+  V2ADDI_IMM8_OPCODE_X1 = 39,
+  V2ADDSC_RRR_0_OPCODE_X0 = 115,
+  V2ADDSC_RRR_0_OPCODE_X1 = 73,
+  V2ADD_RRR_0_OPCODE_X0 = 116,
+  V2ADD_RRR_0_OPCODE_X1 = 74,
+  V2ADIFFS_RRR_0_OPCODE_X0 = 117,
+  V2AVGS_RRR_0_OPCODE_X0 = 118,
+  V2CMPEQI_IMM8_OPCODE_X0 = 15,
+  V2CMPEQI_IMM8_OPCODE_X1 = 40,
+  V2CMPEQ_RRR_0_OPCODE_X0 = 119,
+  V2CMPEQ_RRR_0_OPCODE_X1 = 75,
+  V2CMPLES_RRR_0_OPCODE_X0 = 120,
+  V2CMPLES_RRR_0_OPCODE_X1 = 76,
+  V2CMPLEU_RRR_0_OPCODE_X0 = 121,
+  V2CMPLEU_RRR_0_OPCODE_X1 = 77,
+  V2CMPLTSI_IMM8_OPCODE_X0 = 16,
+  V2CMPLTSI_IMM8_OPCODE_X1 = 41,
+  V2CMPLTS_RRR_0_OPCODE_X0 = 122,
+  V2CMPLTS_RRR_0_OPCODE_X1 = 78,
+  V2CMPLTUI_IMM8_OPCODE_X0 = 17,
+  V2CMPLTUI_IMM8_OPCODE_X1 = 42,
+  V2CMPLTU_RRR_0_OPCODE_X0 = 123,
+  V2CMPLTU_RRR_0_OPCODE_X1 = 79,
+  V2CMPNE_RRR_0_OPCODE_X0 = 124,
+  V2CMPNE_RRR_0_OPCODE_X1 = 80,
+  V2DOTPA_RRR_0_OPCODE_X0 = 125,
+  V2DOTP_RRR_0_OPCODE_X0 = 126,
+  V2INT_H_RRR_0_OPCODE_X0 = 127,
+  V2INT_H_RRR_0_OPCODE_X1 = 81,
+  V2INT_L_RRR_0_OPCODE_X0 = 128,
+  V2INT_L_RRR_0_OPCODE_X1 = 82,
+  V2MAXSI_IMM8_OPCODE_X0 = 18,
+  V2MAXSI_IMM8_OPCODE_X1 = 43,
+  V2MAXS_RRR_0_OPCODE_X0 = 129,
+  V2MAXS_RRR_0_OPCODE_X1 = 83,
+  V2MINSI_IMM8_OPCODE_X0 = 19,
+  V2MINSI_IMM8_OPCODE_X1 = 44,
+  V2MINS_RRR_0_OPCODE_X0 = 130,
+  V2MINS_RRR_0_OPCODE_X1 = 84,
+  V2MNZ_RRR_0_OPCODE_X0 = 131,
+  V2MNZ_RRR_0_OPCODE_X1 = 85,
+  V2MULFSC_RRR_0_OPCODE_X0 = 132,
+  V2MULS_RRR_0_OPCODE_X0 = 133,
+  V2MULTS_RRR_0_OPCODE_X0 = 134,
+  V2MZ_RRR_0_OPCODE_X0 = 135,
+  V2MZ_RRR_0_OPCODE_X1 = 86,
+  V2PACKH_RRR_0_OPCODE_X0 = 136,
+  V2PACKH_RRR_0_OPCODE_X1 = 87,
+  V2PACKL_RRR_0_OPCODE_X0 = 137,
+  V2PACKL_RRR_0_OPCODE_X1 = 88,
+  V2PACKUC_RRR_0_OPCODE_X0 = 138,
+  V2PACKUC_RRR_0_OPCODE_X1 = 89,
+  V2SADAS_RRR_0_OPCODE_X0 = 139,
+  V2SADAU_RRR_0_OPCODE_X0 = 140,
+  V2SADS_RRR_0_OPCODE_X0 = 141,
+  V2SADU_RRR_0_OPCODE_X0 = 142,
+  V2SHLI_SHIFT_OPCODE_X0 = 10,
+  V2SHLI_SHIFT_OPCODE_X1 = 10,
+  V2SHLSC_RRR_0_OPCODE_X0 = 143,
+  V2SHLSC_RRR_0_OPCODE_X1 = 90,
+  V2SHL_RRR_0_OPCODE_X0 = 144,
+  V2SHL_RRR_0_OPCODE_X1 = 91,
+  V2SHRSI_SHIFT_OPCODE_X0 = 11,
+  V2SHRSI_SHIFT_OPCODE_X1 = 11,
+  V2SHRS_RRR_0_OPCODE_X0 = 145,
+  V2SHRS_RRR_0_OPCODE_X1 = 92,
+  V2SHRUI_SHIFT_OPCODE_X0 = 12,
+  V2SHRUI_SHIFT_OPCODE_X1 = 12,
+  V2SHRU_RRR_0_OPCODE_X0 = 146,
+  V2SHRU_RRR_0_OPCODE_X1 = 93,
+  V2SUBSC_RRR_0_OPCODE_X0 = 147,
+  V2SUBSC_RRR_0_OPCODE_X1 = 94,
+  V2SUB_RRR_0_OPCODE_X0 = 148,
+  V2SUB_RRR_0_OPCODE_X1 = 95,
+  V4ADDSC_RRR_0_OPCODE_X0 = 149,
+  V4ADDSC_RRR_0_OPCODE_X1 = 96,
+  V4ADD_RRR_0_OPCODE_X0 = 150,
+  V4ADD_RRR_0_OPCODE_X1 = 97,
+  V4INT_H_RRR_0_OPCODE_X0 = 151,
+  V4INT_H_RRR_0_OPCODE_X1 = 98,
+  V4INT_L_RRR_0_OPCODE_X0 = 152,
+  V4INT_L_RRR_0_OPCODE_X1 = 99,
+  V4PACKSC_RRR_0_OPCODE_X0 = 153,
+  V4PACKSC_RRR_0_OPCODE_X1 = 100,
+  V4SHLSC_RRR_0_OPCODE_X0 = 154,
+  V4SHLSC_RRR_0_OPCODE_X1 = 101,
+  V4SHL_RRR_0_OPCODE_X0 = 155,
+  V4SHL_RRR_0_OPCODE_X1 = 102,
+  V4SHRS_RRR_0_OPCODE_X0 = 156,
+  V4SHRS_RRR_0_OPCODE_X1 = 103,
+  V4SHRU_RRR_0_OPCODE_X0 = 157,
+  V4SHRU_RRR_0_OPCODE_X1 = 104,
+  V4SUBSC_RRR_0_OPCODE_X0 = 158,
+  V4SUBSC_RRR_0_OPCODE_X1 = 105,
+  V4SUB_RRR_0_OPCODE_X0 = 159,
+  V4SUB_RRR_0_OPCODE_X1 = 106,
+  WH64_UNARY_OPCODE_X1 = 38,
+  XORI_IMM8_OPCODE_X0 = 20,
+  XORI_IMM8_OPCODE_X1 = 45,
+  XOR_RRR_0_OPCODE_X0 = 160,
+  XOR_RRR_0_OPCODE_X1 = 107,
+  XOR_RRR_5_OPCODE_Y0 = 3,
+  XOR_RRR_5_OPCODE_Y1 = 3
+};
+
+
+#endif /* __ASSEMBLER__ */
+
+#endif /* __ARCH_OPCODE_H__ */
diff --git a/target-tilegx/simd_helper.c b/target-tilegx/simd_helper.c
new file mode 100644
index 000000000..6d7bb5ce2
--- /dev/null
+++ b/target-tilegx/simd_helper.c
@@ -0,0 +1,165 @@
+/*
+ * QEMU TILE-Gx helpers
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "cpu.h"
+#include "qemu-common.h"
+#include "exec/helper-proto.h"
+
+
+/* Broadcast a value to all elements of a vector.  */
+#define V1(X)      (((X) & 0xff) * 0x0101010101010101ull)
+#define V2(X)      (((X) & 0xffff) * 0x0001000100010001ull)
+
+
+uint64_t helper_v1multu(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 64; i += 8) {
+        unsigned ae = extract64(a, i, 8);
+        unsigned be = extract64(b, i, 8);
+        r = deposit64(r, i, 8, ae * be);
+    }
+    return r;
+}
+
+uint64_t helper_v2mults(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    /* While the instruction talks about signed inputs, with a
+       truncated result the sign of the inputs doesn't matter.  */
+    for (i = 0; i < 64; i += 16) {
+        unsigned ae = extract64(a, i, 16);
+        unsigned be = extract64(b, i, 16);
+        r = deposit64(r, i, 16, ae * be);
+    }
+    return r;
+}
+
+uint64_t helper_v1shl(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 7;
+    m = V1(0xff >> b);
+    return (a & m) << b;
+}
+
+uint64_t helper_v2shl(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 15;
+    m = V2(0xffff >> b);
+    return (a & m) << b;
+}
+
+uint64_t helper_v1shru(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 7;
+    m = V1(0xff << b);
+    return (a & m) >> b;
+}
+
+uint64_t helper_v2shru(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 15;
+    m = V2(0xffff << b);
+    return (a & m) >> b;
+}
+
+uint64_t helper_v1shrs(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    b &= 7;
+    for (i = 0; i < 64; i += 8) {
+        r = deposit64(r, i, 8, sextract64(a, i + b, 8 - b));
+    }
+    return r;
+}
+
+uint64_t helper_v2shrs(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    b &= 15;
+    for (i = 0; i < 64; i += 16) {
+        r = deposit64(r, i, 16, sextract64(a, i + b, 16 - b));
+    }
+    return r;
+}
+
+uint64_t helper_v1int_h(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 8) {
+        r = deposit64(r, 2 * i + 8, 8, extract64(a, i + 32, 8));
+        r = deposit64(r, 2 * i, 8, extract64(b, i + 32, 8));
+    }
+    return r;
+}
+
+uint64_t helper_v1int_l(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 8) {
+        r = deposit64(r, 2 * i + 8, 8, extract64(a, i, 8));
+        r = deposit64(r, 2 * i, 8, extract64(b, i, 8));
+    }
+    return r;
+}
+
+uint64_t helper_v2int_h(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 16) {
+        r = deposit64(r, 2 * i + 16, 16, extract64(a, i + 32, 16));
+        r = deposit64(r, 2 * i, 16, extract64(b, i + 32, 16));
+    }
+    return r;
+}
+
+uint64_t helper_v2int_l(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 16) {
+        r = deposit64(r, 2 * i + 16, 16, extract64(a, i, 16));
+        r = deposit64(r, 2 * i, 16, extract64(b, i, 16));
+    }
+    return r;
+}
diff --git a/target-tilegx/spr_def_64.h b/target-tilegx/spr_def_64.h
new file mode 100644
index 000000000..67a6c1751
--- /dev/null
+++ b/target-tilegx/spr_def_64.h
@@ -0,0 +1,216 @@
+/*
+ * Copyright 2011 Tilera Corporation. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or
+ *   modify it under the terms of the GNU General Public License
+ *   as published by the Free Software Foundation, version 2.
+ *
+ *   This program is distributed in the hope that it will be useful, but
+ *   WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *   NON INFRINGEMENT.  See the GNU General Public License for
+ *   more details.
+ */
+
+#ifndef __DOXYGEN__
+
+#ifndef __ARCH_SPR_DEF_64_H__
+#define __ARCH_SPR_DEF_64_H__
+
+#define SPR_AUX_PERF_COUNT_0 0x2105
+#define SPR_AUX_PERF_COUNT_1 0x2106
+#define SPR_AUX_PERF_COUNT_CTL 0x2107
+#define SPR_AUX_PERF_COUNT_STS 0x2108
+#define SPR_CMPEXCH_VALUE 0x2780
+#define SPR_CYCLE 0x2781
+#define SPR_DONE 0x2705
+#define SPR_DSTREAM_PF 0x2706
+#define SPR_EVENT_BEGIN 0x2782
+#define SPR_EVENT_END 0x2783
+#define SPR_EX_CONTEXT_0_0 0x2580
+#define SPR_EX_CONTEXT_0_1 0x2581
+#define SPR_EX_CONTEXT_0_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_0_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_0_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_0_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_0_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_0_1__ICS_MASK  0x4
+#define SPR_EX_CONTEXT_1_0 0x2480
+#define SPR_EX_CONTEXT_1_1 0x2481
+#define SPR_EX_CONTEXT_1_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_1_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_1_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_1_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_1_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_1_1__ICS_MASK  0x4
+#define SPR_EX_CONTEXT_2_0 0x2380
+#define SPR_EX_CONTEXT_2_1 0x2381
+#define SPR_EX_CONTEXT_2_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_2_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_2_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_2_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_2_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_2_1__ICS_MASK  0x4
+#define SPR_FAIL 0x2707
+#define SPR_IDN_AVAIL_EN 0x1a05
+#define SPR_IDN_DATA_AVAIL 0x0a80
+#define SPR_IDN_DEADLOCK_TIMEOUT 0x1806
+#define SPR_IDN_DEMUX_COUNT_0 0x0a05
+#define SPR_IDN_DEMUX_COUNT_1 0x0a06
+#define SPR_IDN_DIRECTION_PROTECT 0x1405
+#define SPR_IDN_PENDING 0x0a08
+#define SPR_ILL_TRANS_REASON__I_STREAM_VA_RMASK 0x1
+#define SPR_INTCTRL_0_STATUS 0x2505
+#define SPR_INTCTRL_1_STATUS 0x2405
+#define SPR_INTCTRL_2_STATUS 0x2305
+#define SPR_INTERRUPT_CRITICAL_SECTION 0x2708
+#define SPR_INTERRUPT_MASK_0 0x2506
+#define SPR_INTERRUPT_MASK_1 0x2406
+#define SPR_INTERRUPT_MASK_2 0x2306
+#define SPR_INTERRUPT_MASK_RESET_0 0x2507
+#define SPR_INTERRUPT_MASK_RESET_1 0x2407
+#define SPR_INTERRUPT_MASK_RESET_2 0x2307
+#define SPR_INTERRUPT_MASK_SET_0 0x2508
+#define SPR_INTERRUPT_MASK_SET_1 0x2408
+#define SPR_INTERRUPT_MASK_SET_2 0x2308
+#define SPR_INTERRUPT_VECTOR_BASE_0 0x2509
+#define SPR_INTERRUPT_VECTOR_BASE_1 0x2409
+#define SPR_INTERRUPT_VECTOR_BASE_2 0x2309
+#define SPR_INTERRUPT_VECTOR_BASE_3 0x2209
+#define SPR_IPI_EVENT_0 0x1f05
+#define SPR_IPI_EVENT_1 0x1e05
+#define SPR_IPI_EVENT_2 0x1d05
+#define SPR_IPI_EVENT_RESET_0 0x1f06
+#define SPR_IPI_EVENT_RESET_1 0x1e06
+#define SPR_IPI_EVENT_RESET_2 0x1d06
+#define SPR_IPI_EVENT_SET_0 0x1f07
+#define SPR_IPI_EVENT_SET_1 0x1e07
+#define SPR_IPI_EVENT_SET_2 0x1d07
+#define SPR_IPI_MASK_0 0x1f08
+#define SPR_IPI_MASK_1 0x1e08
+#define SPR_IPI_MASK_2 0x1d08
+#define SPR_IPI_MASK_RESET_0 0x1f09
+#define SPR_IPI_MASK_RESET_1 0x1e09
+#define SPR_IPI_MASK_RESET_2 0x1d09
+#define SPR_IPI_MASK_SET_0 0x1f0a
+#define SPR_IPI_MASK_SET_1 0x1e0a
+#define SPR_IPI_MASK_SET_2 0x1d0a
+#define SPR_MPL_AUX_PERF_COUNT_SET_0 0x2100
+#define SPR_MPL_AUX_PERF_COUNT_SET_1 0x2101
+#define SPR_MPL_AUX_PERF_COUNT_SET_2 0x2102
+#define SPR_MPL_AUX_TILE_TIMER_SET_0 0x1700
+#define SPR_MPL_AUX_TILE_TIMER_SET_1 0x1701
+#define SPR_MPL_AUX_TILE_TIMER_SET_2 0x1702
+#define SPR_MPL_IDN_ACCESS_SET_0 0x0a00
+#define SPR_MPL_IDN_ACCESS_SET_1 0x0a01
+#define SPR_MPL_IDN_ACCESS_SET_2 0x0a02
+#define SPR_MPL_IDN_AVAIL_SET_0 0x1a00
+#define SPR_MPL_IDN_AVAIL_SET_1 0x1a01
+#define SPR_MPL_IDN_AVAIL_SET_2 0x1a02
+#define SPR_MPL_IDN_COMPLETE_SET_0 0x0500
+#define SPR_MPL_IDN_COMPLETE_SET_1 0x0501
+#define SPR_MPL_IDN_COMPLETE_SET_2 0x0502
+#define SPR_MPL_IDN_FIREWALL_SET_0 0x1400
+#define SPR_MPL_IDN_FIREWALL_SET_1 0x1401
+#define SPR_MPL_IDN_FIREWALL_SET_2 0x1402
+#define SPR_MPL_IDN_TIMER_SET_0 0x1800
+#define SPR_MPL_IDN_TIMER_SET_1 0x1801
+#define SPR_MPL_IDN_TIMER_SET_2 0x1802
+#define SPR_MPL_INTCTRL_0_SET_0 0x2500
+#define SPR_MPL_INTCTRL_0_SET_1 0x2501
+#define SPR_MPL_INTCTRL_0_SET_2 0x2502
+#define SPR_MPL_INTCTRL_1_SET_0 0x2400
+#define SPR_MPL_INTCTRL_1_SET_1 0x2401
+#define SPR_MPL_INTCTRL_1_SET_2 0x2402
+#define SPR_MPL_INTCTRL_2_SET_0 0x2300
+#define SPR_MPL_INTCTRL_2_SET_1 0x2301
+#define SPR_MPL_INTCTRL_2_SET_2 0x2302
+#define SPR_MPL_IPI_0 0x1f04
+#define SPR_MPL_IPI_0_SET_0 0x1f00
+#define SPR_MPL_IPI_0_SET_1 0x1f01
+#define SPR_MPL_IPI_0_SET_2 0x1f02
+#define SPR_MPL_IPI_1 0x1e04
+#define SPR_MPL_IPI_1_SET_0 0x1e00
+#define SPR_MPL_IPI_1_SET_1 0x1e01
+#define SPR_MPL_IPI_1_SET_2 0x1e02
+#define SPR_MPL_IPI_2 0x1d04
+#define SPR_MPL_IPI_2_SET_0 0x1d00
+#define SPR_MPL_IPI_2_SET_1 0x1d01
+#define SPR_MPL_IPI_2_SET_2 0x1d02
+#define SPR_MPL_PERF_COUNT_SET_0 0x2000
+#define SPR_MPL_PERF_COUNT_SET_1 0x2001
+#define SPR_MPL_PERF_COUNT_SET_2 0x2002
+#define SPR_MPL_UDN_ACCESS_SET_0 0x0b00
+#define SPR_MPL_UDN_ACCESS_SET_1 0x0b01
+#define SPR_MPL_UDN_ACCESS_SET_2 0x0b02
+#define SPR_MPL_UDN_AVAIL_SET_0 0x1b00
+#define SPR_MPL_UDN_AVAIL_SET_1 0x1b01
+#define SPR_MPL_UDN_AVAIL_SET_2 0x1b02
+#define SPR_MPL_UDN_COMPLETE_SET_0 0x0600
+#define SPR_MPL_UDN_COMPLETE_SET_1 0x0601
+#define SPR_MPL_UDN_COMPLETE_SET_2 0x0602
+#define SPR_MPL_UDN_FIREWALL_SET_0 0x1500
+#define SPR_MPL_UDN_FIREWALL_SET_1 0x1501
+#define SPR_MPL_UDN_FIREWALL_SET_2 0x1502
+#define SPR_MPL_UDN_TIMER_SET_0 0x1900
+#define SPR_MPL_UDN_TIMER_SET_1 0x1901
+#define SPR_MPL_UDN_TIMER_SET_2 0x1902
+#define SPR_MPL_WORLD_ACCESS_SET_0 0x2700
+#define SPR_MPL_WORLD_ACCESS_SET_1 0x2701
+#define SPR_MPL_WORLD_ACCESS_SET_2 0x2702
+#define SPR_PASS 0x2709
+#define SPR_PERF_COUNT_0 0x2005
+#define SPR_PERF_COUNT_1 0x2006
+#define SPR_PERF_COUNT_CTL 0x2007
+#define SPR_PERF_COUNT_DN_CTL 0x2008
+#define SPR_PERF_COUNT_STS 0x2009
+#define SPR_PROC_STATUS 0x2784
+#define SPR_SIM_CONTROL 0x2785
+#define SPR_SINGLE_STEP_CONTROL_0 0x0405
+#define SPR_SINGLE_STEP_CONTROL_0__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_0__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_CONTROL_1 0x0305
+#define SPR_SINGLE_STEP_CONTROL_1__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_1__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_CONTROL_2 0x0205
+#define SPR_SINGLE_STEP_CONTROL_2__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_2__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_EN_0_0 0x250a
+#define SPR_SINGLE_STEP_EN_0_1 0x240a
+#define SPR_SINGLE_STEP_EN_0_2 0x230a
+#define SPR_SINGLE_STEP_EN_1_0 0x250b
+#define SPR_SINGLE_STEP_EN_1_1 0x240b
+#define SPR_SINGLE_STEP_EN_1_2 0x230b
+#define SPR_SINGLE_STEP_EN_2_0 0x250c
+#define SPR_SINGLE_STEP_EN_2_1 0x240c
+#define SPR_SINGLE_STEP_EN_2_2 0x230c
+#define SPR_SYSTEM_SAVE_0_0 0x2582
+#define SPR_SYSTEM_SAVE_0_1 0x2583
+#define SPR_SYSTEM_SAVE_0_2 0x2584
+#define SPR_SYSTEM_SAVE_0_3 0x2585
+#define SPR_SYSTEM_SAVE_1_0 0x2482
+#define SPR_SYSTEM_SAVE_1_1 0x2483
+#define SPR_SYSTEM_SAVE_1_2 0x2484
+#define SPR_SYSTEM_SAVE_1_3 0x2485
+#define SPR_SYSTEM_SAVE_2_0 0x2382
+#define SPR_SYSTEM_SAVE_2_1 0x2383
+#define SPR_SYSTEM_SAVE_2_2 0x2384
+#define SPR_SYSTEM_SAVE_2_3 0x2385
+#define SPR_TILE_COORD 0x270b
+#define SPR_TILE_RTF_HWM 0x270c
+#define SPR_TILE_TIMER_CONTROL 0x1605
+#define SPR_UDN_AVAIL_EN 0x1b05
+#define SPR_UDN_DATA_AVAIL 0x0b80
+#define SPR_UDN_DEADLOCK_TIMEOUT 0x1906
+#define SPR_UDN_DEMUX_COUNT_0 0x0b05
+#define SPR_UDN_DEMUX_COUNT_1 0x0b06
+#define SPR_UDN_DEMUX_COUNT_2 0x0b07
+#define SPR_UDN_DEMUX_COUNT_3 0x0b08
+#define SPR_UDN_DIRECTION_PROTECT 0x1505
+#define SPR_UDN_PENDING 0x0b0a
+#define SPR_WATCH_MASK 0x200a
+#define SPR_WATCH_VAL 0x200b
+
+#endif /* !defined(__ARCH_SPR_DEF_64_H__) */
+
+#endif /* !defined(__DOXYGEN__) */
diff --git a/target-tilegx/translate.c b/target-tilegx/translate.c
new file mode 100644
index 000000000..354f25a52
--- /dev/null
+++ b/target-tilegx/translate.c
@@ -0,0 +1,2449 @@
+/*
+ * QEMU TILE-Gx CPU
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "cpu.h"
+#include "qemu/log.h"
+#include "disas/disas.h"
+#include "tcg-op.h"
+#include "exec/cpu_ldst.h"
+#include "linux-user/syscall_defs.h"
+
+#include "opcode_tilegx.h"
+#include "spr_def_64.h"
+
+#define FMT64X                          "%016" PRIx64
+
+static TCGv_ptr cpu_env;
+static TCGv cpu_pc;
+static TCGv cpu_regs[TILEGX_R_COUNT];
+
+static const char * const reg_names[64] = {
+     "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+     "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+    "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+    "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+    "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
+    "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
+    "r48", "r49", "r50", "r51",  "bp",  "tp",  "sp",  "lr",
+    "sn", "idn0", "idn1", "udn0", "udn1", "udn2", "udn2", "zero"
+};
+
+/* Modified registers are cached in temporaries until the end of the bundle. */
+typedef struct {
+    unsigned reg;
+    TCGv val;
+} DisasContextTemp;
+
+#define MAX_WRITEBACK 4
+
+/* This is the state at translation time.  */
+typedef struct {
+    uint64_t pc;		/* Current pc */
+
+    TCGv zero;                  /* For zero register */
+
+    DisasContextTemp wb[MAX_WRITEBACK];
+    int num_wb;
+    int mmuidx;
+    bool exit_tb;
+    TileExcp atomic_excp;
+
+    struct {
+        TCGCond cond;    /* branch condition */
+        TCGv dest;       /* branch destination */
+        TCGv val1;       /* value to be compared against zero, for cond */
+    } jmp;               /* Jump object, only once in each TB block */
+} DisasContext;
+
+#include "exec/gen-icount.h"
+
+/* Differentiate the various pipe encodings.  */
+#define TY_X0  0
+#define TY_X1  1
+#define TY_Y0  2
+#define TY_Y1  3
+
+/* Remerge the base opcode and extension fields for switching.
+   The X opcode fields are 3 bits; Y0/Y1 opcode fields are 4 bits;
+   Y2 opcode field is 2 bits.  */
+#define OE(OP, EXT, XY) (TY_##XY + OP * 4 + EXT * 64)
+
+/* Similar, but for Y2 only.  */
+#define OEY2(OP, MODE) (OP + MODE * 4)
+
+/* Similar, but make sure opcode names match up.  */
+#define OE_RR_X0(E)    OE(RRR_0_OPCODE_X0, E##_UNARY_OPCODE_X0, X0)
+#define OE_RR_X1(E)    OE(RRR_0_OPCODE_X1, E##_UNARY_OPCODE_X1, X1)
+#define OE_RR_Y0(E)    OE(RRR_1_OPCODE_Y0, E##_UNARY_OPCODE_Y0, Y0)
+#define OE_RR_Y1(E)    OE(RRR_1_OPCODE_Y1, E##_UNARY_OPCODE_Y1, Y1)
+#define OE_RRR(E,N,XY) OE(RRR_##N##_OPCODE_##XY, E##_RRR_##N##_OPCODE_##XY, XY)
+#define OE_IM(E,XY)    OE(IMM8_OPCODE_##XY, E##_IMM8_OPCODE_##XY, XY)
+#define OE_SH(E,XY)    OE(SHIFT_OPCODE_##XY, E##_SHIFT_OPCODE_##XY, XY)
+
+#define V1_IMM(X)      (((X) & 0xff) * 0x0101010101010101ull)
+#define V2_IMM(X)      (((X) & 0xffff) * 0x0001000100010001ull)
+
+
+static void gen_exception(DisasContext *dc, TileExcp num)
+{
+    TCGv_i32 tmp;
+
+    tcg_gen_movi_tl(cpu_pc, dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+
+    tmp = tcg_const_i32(num);
+    gen_helper_exception(cpu_env, tmp);
+    tcg_temp_free_i32(tmp);
+    dc->exit_tb = true;
+}
+
+static bool check_gr(DisasContext *dc, uint8_t reg)
+{
+    if (likely(reg < TILEGX_R_COUNT)) {
+        return true;
+    }
+
+    switch (reg) {
+    case TILEGX_R_SN:
+    case TILEGX_R_ZERO:
+        break;
+    case TILEGX_R_IDN0:
+    case TILEGX_R_IDN1:
+        gen_exception(dc, TILEGX_EXCP_REG_IDN_ACCESS);
+        break;
+    case TILEGX_R_UDN0:
+    case TILEGX_R_UDN1:
+    case TILEGX_R_UDN2:
+    case TILEGX_R_UDN3:
+        gen_exception(dc, TILEGX_EXCP_REG_UDN_ACCESS);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    return false;
+}
+
+static TCGv load_zero(DisasContext *dc)
+{
+    if (TCGV_IS_UNUSED_I64(dc->zero)) {
+        dc->zero = tcg_const_i64(0);
+    }
+    return dc->zero;
+}
+
+static TCGv load_gr(DisasContext *dc, unsigned reg)
+{
+    if (check_gr(dc, reg)) {
+        return cpu_regs[reg];
+    }
+    return load_zero(dc);
+}
+
+static TCGv dest_gr(DisasContext *dc, unsigned reg)
+{
+    int n;
+
+    /* Skip the result, mark the exception if necessary, and continue */
+    check_gr(dc, reg);
+
+    n = dc->num_wb++;
+    dc->wb[n].reg = reg;
+    return dc->wb[n].val = tcg_temp_new_i64();
+}
+
+static void gen_saturate_op(TCGv tdest, TCGv tsrca, TCGv tsrcb,
+                            void (*operate)(TCGv, TCGv, TCGv))
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_ext32s_tl(tdest, tsrca);
+    tcg_gen_ext32s_tl(t0, tsrcb);
+    operate(tdest, tdest, t0);
+
+    tcg_gen_movi_tl(t0, 0x7fffffff);
+    tcg_gen_movcond_tl(TCG_COND_GT, tdest, tdest, t0, t0, tdest);
+    tcg_gen_movi_tl(t0, -0x80000000LL);
+    tcg_gen_movcond_tl(TCG_COND_LT, tdest, tdest, t0, t0, tdest);
+
+    tcg_temp_free(t0);
+}
+
+static void gen_atomic_excp(DisasContext *dc, unsigned dest, TCGv tdest,
+                            TCGv tsrca, TCGv tsrcb, TileExcp excp)
+{
+#ifdef CONFIG_USER_ONLY
+    TCGv_i32 t;
+
+    tcg_gen_st_tl(tsrca, cpu_env, offsetof(CPUTLGState, atomic_srca));
+    tcg_gen_st_tl(tsrcb, cpu_env, offsetof(CPUTLGState, atomic_srcb));
+    t = tcg_const_i32(dest);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUTLGState, atomic_dstr));
+    tcg_temp_free_i32(t);
+
+    /* We're going to write the real result in the exception.  But in
+       the meantime we've already created a writeback register, and
+       we don't want that to remain uninitialized.  */
+    tcg_gen_movi_tl(tdest, 0);
+
+    /* Note that we need to delay issuing the exception that implements
+       the atomic operation until after writing back the results of the
+       instruction occupying the X0 pipe.  */
+    dc->atomic_excp = excp;
+#else
+    gen_exception(dc, TILEGX_EXCP_OPCODE_UNIMPLEMENTED);
+#endif
+}
+
+/* Shift the 128-bit value TSRCA:TSRCD right by the number of bytes
+   specified by the bottom 3 bits of TSRCB, and set TDEST to the
+   low 64 bits of the resulting value.  */
+static void gen_dblalign(TCGv tdest, TCGv tsrcd, TCGv tsrca, TCGv tsrcb)
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_andi_tl(t0, tsrcb, 7);
+    tcg_gen_shli_tl(t0, t0, 3);
+    tcg_gen_shr_tl(tdest, tsrcd, t0);
+
+    /* We want to do "t0 = tsrca << (64 - t0)".  Two's complement
+       arithmetic on a 6-bit field tells us that 64 - t0 is equal
+       to (t0 ^ 63) + 1.  So we can do the shift in two parts,
+       neither of which will be an invalid shift by 64.  */
+    tcg_gen_xori_tl(t0, t0, 63);
+    tcg_gen_shl_tl(t0, tsrca, t0);
+    tcg_gen_shli_tl(t0, t0, 1);
+    tcg_gen_or_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t0);
+}
+
+/* Similarly, except that the 128-bit value is TSRCA:TSRCB, and the
+   right shift is an immediate.  */
+static void gen_dblaligni(TCGv tdest, TCGv tsrca, TCGv tsrcb, int shr)
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_shri_tl(t0, tsrcb, shr);
+    tcg_gen_shli_tl(tdest, tsrca, 64 - shr);
+    tcg_gen_or_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t0);
+}
+
+typedef enum {
+    LU, LS, HU, HS
+} MulHalf;
+
+static void gen_ext_half(TCGv d, TCGv s, MulHalf h)
+{
+    switch (h) {
+    case LU:
+        tcg_gen_ext32u_tl(d, s);
+        break;
+    case LS:
+        tcg_gen_ext32s_tl(d, s);
+        break;
+    case HU:
+        tcg_gen_shri_tl(d, s, 32);
+        break;
+    case HS:
+        tcg_gen_sari_tl(d, s, 32);
+        break;
+    }
+}
+
+static void gen_mul_half(TCGv tdest, TCGv tsrca, TCGv tsrcb,
+                         MulHalf ha, MulHalf hb)
+{
+    TCGv t = tcg_temp_new();
+    gen_ext_half(t, tsrca, ha);
+    gen_ext_half(tdest, tsrcb, hb);
+    tcg_gen_mul_tl(tdest, tdest, t);
+    tcg_temp_free(t);
+}
+
+static void gen_cmul2(TCGv tdest, TCGv tsrca, TCGv tsrcb, int sh, int rd)
+{
+    TCGv_i32 tsh = tcg_const_i32(sh);
+    TCGv_i32 trd = tcg_const_i32(rd);
+    gen_helper_cmul2(tdest, tsrca, tsrcb, tsh, trd);
+    tcg_temp_free_i32(tsh);
+    tcg_temp_free_i32(trd);
+}
+
+static TileExcp gen_st_opcode(DisasContext *dc, unsigned dest, unsigned srca,
+                              unsigned srcb, TCGMemOp memop, const char *name)
+{
+    if (dest) {
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    tcg_gen_qemu_st_tl(load_gr(dc, srcb), load_gr(dc, srca),
+		       dc->mmuidx, memop);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", name,
+                  reg_names[srca], reg_names[srcb]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_st_add_opcode(DisasContext *dc, unsigned srca, unsigned srcb,
+                                  int imm, TCGMemOp memop, const char *name)
+{
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcb = load_gr(dc, srcb);
+
+    tcg_gen_qemu_st_tl(tsrcb, tsrca, dc->mmuidx, memop);
+    tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %d", name,
+                  reg_names[srca], reg_names[srcb], imm);
+    return TILEGX_EXCP_NONE;
+}
+
+/* Equality comparison with zero can be done quickly and efficiently.  */
+static void gen_v1cmpeq0(TCGv v)
+{
+    TCGv m = tcg_const_tl(V1_IMM(0x7f));
+    TCGv c = tcg_temp_new();
+
+    /* ~(((v & m) + m) | m | v).  Sets the msb for each byte == 0.  */
+    tcg_gen_and_tl(c, v, m);
+    tcg_gen_add_tl(c, c, m);
+    tcg_gen_or_tl(c, c, m);
+    tcg_gen_nor_tl(c, c, v);
+    tcg_temp_free(m);
+
+    /* Shift the msb down to form the lsb boolean result.  */
+    tcg_gen_shri_tl(v, c, 7);
+    tcg_temp_free(c);
+}
+
+static void gen_v1cmpne0(TCGv v)
+{
+    TCGv m = tcg_const_tl(V1_IMM(0x7f));
+    TCGv c = tcg_temp_new();
+
+    /* (((v & m) + m) | v) & ~m.  Sets the msb for each byte != 0.  */
+    tcg_gen_and_tl(c, v, m);
+    tcg_gen_add_tl(c, c, m);
+    tcg_gen_or_tl(c, c, v);
+    tcg_gen_andc_tl(c, c, m);
+    tcg_temp_free(m);
+
+    /* Shift the msb down to form the lsb boolean result.  */
+    tcg_gen_shri_tl(v, c, 7);
+    tcg_temp_free(c);
+}
+
+/* Vector addition can be performed via arithmetic plus masking.  It is
+   efficient this way only for 4 or more elements.  */
+static void gen_v12add(TCGv tdest, TCGv tsrca, TCGv tsrcb, uint64_t sign)
+{
+    TCGv tmask = tcg_const_tl(~sign);
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+
+    /* ((a & ~sign) + (b & ~sign)) ^ ((a ^ b) & sign).  */
+    tcg_gen_and_tl(t0, tsrca, tmask);
+    tcg_gen_and_tl(t1, tsrcb, tmask);
+    tcg_gen_add_tl(tdest, t0, t1);
+    tcg_gen_xor_tl(t0, tsrca, tsrcb);
+    tcg_gen_andc_tl(t0, t0, tmask);
+    tcg_gen_xor_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
+    tcg_temp_free(tmask);
+}
+
+/* Similarly for vector subtraction.  */
+static void gen_v12sub(TCGv tdest, TCGv tsrca, TCGv tsrcb, uint64_t sign)
+{
+    TCGv tsign = tcg_const_tl(sign);
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+
+    /* ((a | sign) - (b & ~sign)) ^ ((a ^ ~b) & sign).  */
+    tcg_gen_or_tl(t0, tsrca, tsign);
+    tcg_gen_andc_tl(t1, tsrcb, tsign);
+    tcg_gen_sub_tl(tdest, t0, t1);
+    tcg_gen_eqv_tl(t0, tsrca, tsrcb);
+    tcg_gen_and_tl(t0, t0, tsign);
+    tcg_gen_xor_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
+    tcg_temp_free(tsign);
+}
+
+static void gen_v4sh(TCGv d64, TCGv a64, TCGv b64,
+                     void (*generate)(TCGv_i32, TCGv_i32, TCGv_i32))
+{
+    TCGv_i32 al = tcg_temp_new_i32();
+    TCGv_i32 ah = tcg_temp_new_i32();
+    TCGv_i32 bl = tcg_temp_new_i32();
+
+    tcg_gen_extr_i64_i32(al, ah, a64);
+    tcg_gen_extrl_i64_i32(bl, b64);
+    tcg_gen_andi_i32(bl, bl, 31);
+    generate(al, al, bl);
+    generate(ah, ah, bl);
+    tcg_gen_concat_i32_i64(d64, al, ah);
+
+    tcg_temp_free_i32(al);
+    tcg_temp_free_i32(ah);
+    tcg_temp_free_i32(bl);
+}
+
+static void gen_v4op(TCGv d64, TCGv a64, TCGv b64,
+                     void (*generate)(TCGv_i32, TCGv_i32, TCGv_i32))
+{
+    TCGv_i32 al = tcg_temp_new_i32();
+    TCGv_i32 ah = tcg_temp_new_i32();
+    TCGv_i32 bl = tcg_temp_new_i32();
+    TCGv_i32 bh = tcg_temp_new_i32();
+
+    tcg_gen_extr_i64_i32(al, ah, a64);
+    tcg_gen_extr_i64_i32(bl, bh, b64);
+    generate(al, al, bl);
+    generate(ah, ah, bh);
+    tcg_gen_concat_i32_i64(d64, al, ah);
+
+    tcg_temp_free_i32(al);
+    tcg_temp_free_i32(ah);
+    tcg_temp_free_i32(bl);
+    tcg_temp_free_i32(bh);
+}
+
+static TileExcp gen_signal(DisasContext *dc, int signo, int sigcode,
+                           const char *mnemonic)
+{
+    TCGv_i32 t0 = tcg_const_i32(signo);
+    TCGv_i32 t1 = tcg_const_i32(sigcode);
+
+    tcg_gen_st_i32(t0, cpu_env, offsetof(CPUTLGState, signo));
+    tcg_gen_st_i32(t1, cpu_env, offsetof(CPUTLGState, sigcode));
+
+    tcg_temp_free_i32(t1);
+    tcg_temp_free_i32(t0);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s", mnemonic);
+    return TILEGX_EXCP_SIGNAL;
+}
+
+static bool parse_from_addli(uint64_t bundle, int *signo, int *sigcode)
+{
+    int imm;
+
+    if ((get_Opcode_X0(bundle) != ADDLI_OPCODE_X0)
+        || (get_Dest_X0(bundle) != TILEGX_R_ZERO)
+        || (get_SrcA_X0(bundle) != TILEGX_R_ZERO)) {
+        return false;
+    }
+
+    imm = get_Imm16_X0(bundle);
+    *signo = imm & 0x3f;
+    *sigcode = (imm >> 6) & 0xf;
+
+    /* ??? The linux kernel validates both signo and the sigcode vs the
+       known max for each signal.  Don't bother here.  */
+    return true;
+}
+
+static TileExcp gen_specill(DisasContext *dc, unsigned dest, unsigned srca,
+                            uint64_t bundle)
+{
+    const char *mnemonic;
+    int signo;
+    int sigcode;
+
+    if (dest == 0x1c && srca == 0x25) {
+        signo = TARGET_SIGTRAP;
+        sigcode = TARGET_TRAP_BRKPT;
+        mnemonic = "bpt";
+    } else if (dest == 0x1d && srca == 0x25
+               && parse_from_addli(bundle, &signo, &sigcode)) {
+        mnemonic = "raise";
+    } else {
+        signo = TARGET_SIGILL;
+        sigcode = TARGET_ILL_ILLOPC;
+        mnemonic = "ill";
+    }
+
+    return gen_signal(dc, signo, sigcode, mnemonic);
+}
+
+static TileExcp gen_rr_opcode(DisasContext *dc, unsigned opext,
+                              unsigned dest, unsigned srca, uint64_t bundle)
+{
+    TCGv tdest, tsrca;
+    const char *mnemonic;
+    TCGMemOp memop;
+    TileExcp ret = TILEGX_EXCP_NONE;
+    bool prefetch_nofault = false;
+
+    /* Eliminate instructions with no output before doing anything else.  */
+    switch (opext) {
+    case OE_RR_Y0(NOP):
+    case OE_RR_Y1(NOP):
+    case OE_RR_X0(NOP):
+    case OE_RR_X1(NOP):
+        mnemonic = "nop";
+        goto done0;
+    case OE_RR_Y0(FNOP):
+    case OE_RR_Y1(FNOP):
+    case OE_RR_X0(FNOP):
+    case OE_RR_X1(FNOP):
+        mnemonic = "fnop";
+        goto done0;
+    case OE_RR_X1(DRAIN):
+        mnemonic = "drain";
+        goto done0;
+    case OE_RR_X1(FLUSHWB):
+        mnemonic = "flushwb";
+        goto done0;
+    case OE_RR_X1(ILL):
+        return gen_specill(dc, dest, srca, bundle);
+    case OE_RR_Y1(ILL):
+        return gen_signal(dc, TARGET_SIGILL, TARGET_ILL_ILLOPC, "ill");
+    case OE_RR_X1(MF):
+        mnemonic = "mf";
+        goto done0;
+    case OE_RR_X1(NAP):
+        /* ??? This should yield, especially in system mode.  */
+        mnemonic = "nap";
+        goto done0;
+    case OE_RR_X1(IRET):
+        gen_helper_ext01_ics(cpu_env);
+        dc->jmp.cond = TCG_COND_ALWAYS;
+        dc->jmp.dest = tcg_temp_new();
+        tcg_gen_ld_tl(dc->jmp.dest, cpu_env,
+                      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_0]));
+        tcg_gen_andi_tl(dc->jmp.dest, dc->jmp.dest, ~7);
+        mnemonic = "iret";
+        goto done0;
+    case OE_RR_X1(SWINT0):
+    case OE_RR_X1(SWINT2):
+    case OE_RR_X1(SWINT3):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(SWINT1):
+        ret = TILEGX_EXCP_SYSCALL;
+        mnemonic = "swint1";
+    done0:
+        if (srca || dest) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s", mnemonic);
+        return ret;
+
+    case OE_RR_X1(DTLBPR):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(FINV):
+        mnemonic = "finv";
+        goto done1;
+    case OE_RR_X1(FLUSH):
+        mnemonic = "flush";
+        goto done1;
+    case OE_RR_X1(ICOH):
+        mnemonic = "icoh";
+        goto done1;
+    case OE_RR_X1(INV):
+        mnemonic = "inv";
+        goto done1;
+    case OE_RR_X1(WH64):
+        mnemonic = "wh64";
+        goto done1;
+    case OE_RR_X1(JRP):
+    case OE_RR_Y1(JRP):
+        mnemonic = "jrp";
+        goto do_jr;
+    case OE_RR_X1(JR):
+    case OE_RR_Y1(JR):
+        mnemonic = "jr";
+        goto do_jr;
+    case OE_RR_X1(JALRP):
+    case OE_RR_Y1(JALRP):
+        mnemonic = "jalrp";
+        goto do_jalr;
+    case OE_RR_X1(JALR):
+    case OE_RR_Y1(JALR):
+        mnemonic = "jalr";
+    do_jalr:
+        tcg_gen_movi_tl(dest_gr(dc, TILEGX_R_LR),
+                        dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+    do_jr:
+        dc->jmp.cond = TCG_COND_ALWAYS;
+        dc->jmp.dest = tcg_temp_new();
+        tcg_gen_andi_tl(dc->jmp.dest, load_gr(dc, srca), ~7);
+    done1:
+        if (dest) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s", mnemonic, reg_names[srca]);
+        return ret;
+    }
+
+    tdest = dest_gr(dc, dest);
+    tsrca = load_gr(dc, srca);
+
+    switch (opext) {
+    case OE_RR_X0(CNTLZ):
+    case OE_RR_Y0(CNTLZ):
+        gen_helper_cntlz(tdest, tsrca);
+        mnemonic = "cntlz";
+        break;
+    case OE_RR_X0(CNTTZ):
+    case OE_RR_Y0(CNTTZ):
+        gen_helper_cnttz(tdest, tsrca);
+        mnemonic = "cnttz";
+        break;
+    case OE_RR_X0(FSINGLE_PACK1):
+    case OE_RR_Y0(FSINGLE_PACK1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(LD1S):
+        memop = MO_SB;
+        mnemonic = "ld1s"; /* prefetch_l1_fault */
+        goto do_load;
+    case OE_RR_X1(LD1U):
+        memop = MO_UB;
+        mnemonic = "ld1u"; /* prefetch, prefetch_l1 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LD2S):
+        memop = MO_TESW;
+        mnemonic = "ld2s"; /* prefetch_l2_fault */
+        goto do_load;
+    case OE_RR_X1(LD2U):
+        memop = MO_TEUW;
+        mnemonic = "ld2u"; /* prefetch_l2 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LD4S):
+        memop = MO_TESL;
+        mnemonic = "ld4s"; /* prefetch_l3_fault */
+        goto do_load;
+    case OE_RR_X1(LD4U):
+        memop = MO_TEUL;
+        mnemonic = "ld4u"; /* prefetch_l3 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LDNT1S):
+        memop = MO_SB;
+        mnemonic = "ldnt1s";
+        goto do_load;
+    case OE_RR_X1(LDNT1U):
+        memop = MO_UB;
+        mnemonic = "ldnt1u";
+        goto do_load;
+    case OE_RR_X1(LDNT2S):
+        memop = MO_TESW;
+        mnemonic = "ldnt2s";
+        goto do_load;
+    case OE_RR_X1(LDNT2U):
+        memop = MO_TEUW;
+        mnemonic = "ldnt2u";
+        goto do_load;
+    case OE_RR_X1(LDNT4S):
+        memop = MO_TESL;
+        mnemonic = "ldnt4s";
+        goto do_load;
+    case OE_RR_X1(LDNT4U):
+        memop = MO_TEUL;
+        mnemonic = "ldnt4u";
+        goto do_load;
+    case OE_RR_X1(LDNT):
+        memop = MO_TEQ;
+        mnemonic = "ldnt";
+        goto do_load;
+    case OE_RR_X1(LD):
+        memop = MO_TEQ;
+        mnemonic = "ld";
+    do_load:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(tdest, tsrca, dc->mmuidx, memop);
+        }
+        break;
+    case OE_RR_X1(LDNA):
+        tcg_gen_andi_tl(tdest, tsrca, ~7);
+        tcg_gen_qemu_ld_tl(tdest, tdest, dc->mmuidx, MO_TEQ);
+        mnemonic = "ldna";
+        break;
+    case OE_RR_X1(LNK):
+    case OE_RR_Y1(LNK):
+        if (srca) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        tcg_gen_movi_tl(tdest, dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+        mnemonic = "lnk";
+        break;
+    case OE_RR_X0(PCNT):
+    case OE_RR_Y0(PCNT):
+        gen_helper_pcnt(tdest, tsrca);
+        mnemonic = "pcnt";
+        break;
+    case OE_RR_X0(REVBITS):
+    case OE_RR_Y0(REVBITS):
+        gen_helper_revbits(tdest, tsrca);
+        mnemonic = "revbits";
+        break;
+    case OE_RR_X0(REVBYTES):
+    case OE_RR_Y0(REVBYTES):
+        tcg_gen_bswap64_tl(tdest, tsrca);
+        mnemonic = "revbytes";
+        break;
+    case OE_RR_X0(TBLIDXB0):
+    case OE_RR_Y0(TBLIDXB0):
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tsrca, 2, 8);
+        mnemonic = "tblidxb0";
+        break;
+    case OE_RR_X0(TBLIDXB1):
+    case OE_RR_Y0(TBLIDXB1):
+        tcg_gen_shri_tl(tdest, tsrca, 8);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb1";
+        break;
+    case OE_RR_X0(TBLIDXB2):
+    case OE_RR_Y0(TBLIDXB2):
+        tcg_gen_shri_tl(tdest, tsrca, 16);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb2";
+        break;
+    case OE_RR_X0(TBLIDXB3):
+    case OE_RR_Y0(TBLIDXB3):
+        tcg_gen_shri_tl(tdest, tsrca, 24);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb3";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", mnemonic,
+                  reg_names[dest], reg_names[srca]);
+    return ret;
+}
+
+static TileExcp gen_rrr_opcode(DisasContext *dc, unsigned opext,
+                               unsigned dest, unsigned srca, unsigned srcb)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcb = load_gr(dc, srcb);
+    TCGv t0;
+    const char *mnemonic;
+
+    switch (opext) {
+    case OE_RRR(ADDXSC, 0, X0):
+    case OE_RRR(ADDXSC, 0, X1):
+        gen_saturate_op(tdest, tsrca, tsrcb, tcg_gen_add_tl);
+        mnemonic = "addxsc";
+        break;
+    case OE_RRR(ADDX, 0, X0):
+    case OE_RRR(ADDX, 0, X1):
+    case OE_RRR(ADDX, 0, Y0):
+    case OE_RRR(ADDX, 0, Y1):
+        tcg_gen_add_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addx";
+        break;
+    case OE_RRR(ADD, 0, X0):
+    case OE_RRR(ADD, 0, X1):
+    case OE_RRR(ADD, 0, Y0):
+    case OE_RRR(ADD, 0, Y1):
+        tcg_gen_add_tl(tdest, tsrca, tsrcb);
+        mnemonic = "add";
+        break;
+    case OE_RRR(AND, 0, X0):
+    case OE_RRR(AND, 0, X1):
+    case OE_RRR(AND, 5, Y0):
+    case OE_RRR(AND, 5, Y1):
+        tcg_gen_and_tl(tdest, tsrca, tsrcb);
+        mnemonic = "and";
+        break;
+    case OE_RRR(CMOVEQZ, 0, X0):
+    case OE_RRR(CMOVEQZ, 4, Y0):
+        tcg_gen_movcond_tl(TCG_COND_EQ, tdest, tsrca, load_zero(dc),
+                           tsrcb, load_gr(dc, dest));
+        mnemonic = "cmoveqz";
+        break;
+    case OE_RRR(CMOVNEZ, 0, X0):
+    case OE_RRR(CMOVNEZ, 4, Y0):
+        tcg_gen_movcond_tl(TCG_COND_NE, tdest, tsrca, load_zero(dc),
+                           tsrcb, load_gr(dc, dest));
+        mnemonic = "cmovnez";
+        break;
+    case OE_RRR(CMPEQ, 0, X0):
+    case OE_RRR(CMPEQ, 0, X1):
+    case OE_RRR(CMPEQ, 3, Y0):
+    case OE_RRR(CMPEQ, 3, Y1):
+        tcg_gen_setcond_tl(TCG_COND_EQ, tdest, tsrca, tsrcb);
+        mnemonic = "cmpeq";
+        break;
+    case OE_RRR(CMPEXCH4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_CMPEXCH4);
+        mnemonic = "cmpexch4";
+        break;
+    case OE_RRR(CMPEXCH, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_CMPEXCH);
+        mnemonic = "cmpexch";
+        break;
+    case OE_RRR(CMPLES, 0, X0):
+    case OE_RRR(CMPLES, 0, X1):
+    case OE_RRR(CMPLES, 2, Y0):
+    case OE_RRR(CMPLES, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LE, tdest, tsrca, tsrcb);
+        mnemonic = "cmples";
+        break;
+    case OE_RRR(CMPLEU, 0, X0):
+    case OE_RRR(CMPLEU, 0, X1):
+    case OE_RRR(CMPLEU, 2, Y0):
+    case OE_RRR(CMPLEU, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LEU, tdest, tsrca, tsrcb);
+        mnemonic = "cmpleu";
+        break;
+    case OE_RRR(CMPLTS, 0, X0):
+    case OE_RRR(CMPLTS, 0, X1):
+    case OE_RRR(CMPLTS, 2, Y0):
+    case OE_RRR(CMPLTS, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LT, tdest, tsrca, tsrcb);
+        mnemonic = "cmplts";
+        break;
+    case OE_RRR(CMPLTU, 0, X0):
+    case OE_RRR(CMPLTU, 0, X1):
+    case OE_RRR(CMPLTU, 2, Y0):
+    case OE_RRR(CMPLTU, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LTU, tdest, tsrca, tsrcb);
+        mnemonic = "cmpltu";
+        break;
+    case OE_RRR(CMPNE, 0, X0):
+    case OE_RRR(CMPNE, 0, X1):
+    case OE_RRR(CMPNE, 3, Y0):
+    case OE_RRR(CMPNE, 3, Y1):
+        tcg_gen_setcond_tl(TCG_COND_NE, tdest, tsrca, tsrcb);
+        mnemonic = "cmpne";
+        break;
+    case OE_RRR(CMULAF, 0, X0):
+        gen_helper_cmulaf(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "cmulaf";
+        break;
+    case OE_RRR(CMULA, 0, X0):
+        gen_helper_cmula(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "cmula";
+        break;
+    case OE_RRR(CMULFR, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 15, 1 << 14);
+        mnemonic = "cmulfr";
+        break;
+    case OE_RRR(CMULF, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 15, 0);
+        mnemonic = "cmulf";
+        break;
+    case OE_RRR(CMULHR, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 16, 1 << 15);
+        mnemonic = "cmulhr";
+        break;
+    case OE_RRR(CMULH, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 16, 0);
+        mnemonic = "cmulh";
+        break;
+    case OE_RRR(CMUL, 0, X0):
+        gen_helper_cmula(tdest, load_zero(dc), tsrca, tsrcb);
+        mnemonic = "cmul";
+        break;
+    case OE_RRR(CRC32_32, 0, X0):
+        gen_helper_crc32_32(tdest, tsrca, tsrcb);
+        mnemonic = "crc32_32";
+        break;
+    case OE_RRR(CRC32_8, 0, X0):
+        gen_helper_crc32_8(tdest, tsrca, tsrcb);
+        mnemonic = "crc32_8";
+        break;
+    case OE_RRR(DBLALIGN2, 0, X0):
+    case OE_RRR(DBLALIGN2, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 16);
+        mnemonic = "dblalign2";
+        break;
+    case OE_RRR(DBLALIGN4, 0, X0):
+    case OE_RRR(DBLALIGN4, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 32);
+        mnemonic = "dblalign4";
+        break;
+    case OE_RRR(DBLALIGN6, 0, X0):
+    case OE_RRR(DBLALIGN6, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 48);
+        mnemonic = "dblalign6";
+        break;
+    case OE_RRR(DBLALIGN, 0, X0):
+        gen_dblalign(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "dblalign";
+        break;
+    case OE_RRR(EXCH4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_EXCH4);
+        mnemonic = "exch4";
+        break;
+    case OE_RRR(EXCH, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_EXCH);
+        mnemonic = "exch";
+        break;
+    case OE_RRR(FDOUBLE_ADDSUB, 0, X0):
+    case OE_RRR(FDOUBLE_ADD_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_MUL_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_PACK1, 0, X0):
+    case OE_RRR(FDOUBLE_PACK2, 0, X0):
+    case OE_RRR(FDOUBLE_SUB_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_UNPACK_MAX, 0, X0):
+    case OE_RRR(FDOUBLE_UNPACK_MIN, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(FETCHADD4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADD4);
+        mnemonic = "fetchadd4";
+        break;
+    case OE_RRR(FETCHADDGEZ4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADDGEZ4);
+        mnemonic = "fetchaddgez4";
+        break;
+    case OE_RRR(FETCHADDGEZ, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADDGEZ);
+        mnemonic = "fetchaddgez";
+        break;
+    case OE_RRR(FETCHADD, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADD);
+        mnemonic = "fetchadd";
+        break;
+    case OE_RRR(FETCHAND4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHAND4);
+        mnemonic = "fetchand4";
+        break;
+    case OE_RRR(FETCHAND, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHAND);
+        mnemonic = "fetchand";
+        break;
+    case OE_RRR(FETCHOR4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHOR4);
+        mnemonic = "fetchor4";
+        break;
+    case OE_RRR(FETCHOR, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHOR);
+        mnemonic = "fetchor";
+        break;
+    case OE_RRR(FSINGLE_ADD1, 0, X0):
+    case OE_RRR(FSINGLE_ADDSUB2, 0, X0):
+    case OE_RRR(FSINGLE_MUL1, 0, X0):
+    case OE_RRR(FSINGLE_MUL2, 0, X0):
+    case OE_RRR(FSINGLE_PACK2, 0, X0):
+    case OE_RRR(FSINGLE_SUB1, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(MNZ, 0, X0):
+    case OE_RRR(MNZ, 0, X1):
+    case OE_RRR(MNZ, 4, Y0):
+    case OE_RRR(MNZ, 4, Y1):
+        t0 = load_zero(dc);
+        tcg_gen_movcond_tl(TCG_COND_NE, tdest, tsrca, t0, tsrcb, t0);
+        mnemonic = "mnz";
+        break;
+    case OE_RRR(MULAX, 0, X0):
+    case OE_RRR(MULAX, 3, Y0):
+        tcg_gen_mul_tl(tdest, tsrca, tsrcb);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "mulax";
+        break;
+    case OE_RRR(MULA_HS_HS, 0, X0):
+    case OE_RRR(MULA_HS_HS, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_hs";
+        break;
+    case OE_RRR(MULA_HS_HU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_hu";
+        break;
+    case OE_RRR(MULA_HS_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_ls";
+        break;
+    case OE_RRR(MULA_HS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_lu";
+        break;
+    case OE_RRR(MULA_HU_HU, 0, X0):
+    case OE_RRR(MULA_HU_HU, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, HU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_hu";
+        break;
+    case OE_RRR(MULA_HU_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_ls";
+        break;
+    case OE_RRR(MULA_HU_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_lu";
+        break;
+    case OE_RRR(MULA_LS_LS, 0, X0):
+    case OE_RRR(MULA_LS_LS, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_ls_ls";
+        break;
+    case OE_RRR(MULA_LS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_ls_lu";
+        break;
+    case OE_RRR(MULA_LU_LU, 0, X0):
+    case OE_RRR(MULA_LU_LU, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LU, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_lu_lu";
+        break;
+    case OE_RRR(MULX, 0, X0):
+    case OE_RRR(MULX, 3, Y0):
+        tcg_gen_mul_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "mulx";
+        break;
+    case OE_RRR(MUL_HS_HS, 0, X0):
+    case OE_RRR(MUL_HS_HS, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HS);
+        mnemonic = "mul_hs_hs";
+        break;
+    case OE_RRR(MUL_HS_HU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HU);
+        mnemonic = "mul_hs_hu";
+        break;
+    case OE_RRR(MUL_HS_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LS);
+        mnemonic = "mul_hs_ls";
+        break;
+    case OE_RRR(MUL_HS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LU);
+        mnemonic = "mul_hs_lu";
+        break;
+    case OE_RRR(MUL_HU_HU, 0, X0):
+    case OE_RRR(MUL_HU_HU, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, HU);
+        mnemonic = "mul_hu_hu";
+        break;
+    case OE_RRR(MUL_HU_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LS);
+        mnemonic = "mul_hu_ls";
+        break;
+    case OE_RRR(MUL_HU_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LU);
+        mnemonic = "mul_hu_lu";
+        break;
+    case OE_RRR(MUL_LS_LS, 0, X0):
+    case OE_RRR(MUL_LS_LS, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LS);
+        mnemonic = "mul_ls_ls";
+        break;
+    case OE_RRR(MUL_LS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LU);
+        mnemonic = "mul_ls_lu";
+        break;
+    case OE_RRR(MUL_LU_LU, 0, X0):
+    case OE_RRR(MUL_LU_LU, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LU, LU);
+        mnemonic = "mul_lu_lu";
+        break;
+    case OE_RRR(MZ, 0, X0):
+    case OE_RRR(MZ, 0, X1):
+    case OE_RRR(MZ, 4, Y0):
+    case OE_RRR(MZ, 4, Y1):
+        t0 = load_zero(dc);
+        tcg_gen_movcond_tl(TCG_COND_EQ, tdest, tsrca, t0, tsrcb, t0);
+        mnemonic = "mz";
+        break;
+    case OE_RRR(NOR, 0, X0):
+    case OE_RRR(NOR, 0, X1):
+    case OE_RRR(NOR, 5, Y0):
+    case OE_RRR(NOR, 5, Y1):
+        tcg_gen_nor_tl(tdest, tsrca, tsrcb);
+        mnemonic = "nor";
+        break;
+    case OE_RRR(OR, 0, X0):
+    case OE_RRR(OR, 0, X1):
+    case OE_RRR(OR, 5, Y0):
+    case OE_RRR(OR, 5, Y1):
+        tcg_gen_or_tl(tdest, tsrca, tsrcb);
+        mnemonic = "or";
+        break;
+    case OE_RRR(ROTL, 0, X0):
+    case OE_RRR(ROTL, 0, X1):
+    case OE_RRR(ROTL, 6, Y0):
+    case OE_RRR(ROTL, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_rotl_tl(tdest, tsrca, tdest);
+        mnemonic = "rotl";
+        break;
+    case OE_RRR(SHL1ADDX, 0, X0):
+    case OE_RRR(SHL1ADDX, 0, X1):
+    case OE_RRR(SHL1ADDX, 7, Y0):
+    case OE_RRR(SHL1ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 1);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl1addx";
+        break;
+    case OE_RRR(SHL1ADD, 0, X0):
+    case OE_RRR(SHL1ADD, 0, X1):
+    case OE_RRR(SHL1ADD, 1, Y0):
+    case OE_RRR(SHL1ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 1);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl1add";
+        break;
+    case OE_RRR(SHL2ADDX, 0, X0):
+    case OE_RRR(SHL2ADDX, 0, X1):
+    case OE_RRR(SHL2ADDX, 7, Y0):
+    case OE_RRR(SHL2ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 2);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl2addx";
+        break;
+    case OE_RRR(SHL2ADD, 0, X0):
+    case OE_RRR(SHL2ADD, 0, X1):
+    case OE_RRR(SHL2ADD, 1, Y0):
+    case OE_RRR(SHL2ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 2);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl2add";
+        break;
+    case OE_RRR(SHL3ADDX, 0, X0):
+    case OE_RRR(SHL3ADDX, 0, X1):
+    case OE_RRR(SHL3ADDX, 7, Y0):
+    case OE_RRR(SHL3ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 3);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl3addx";
+        break;
+    case OE_RRR(SHL3ADD, 0, X0):
+    case OE_RRR(SHL3ADD, 0, X1):
+    case OE_RRR(SHL3ADD, 1, Y0):
+    case OE_RRR(SHL3ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 3);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl3add";
+        break;
+    case OE_RRR(SHLX, 0, X0):
+    case OE_RRR(SHLX, 0, X1):
+        tcg_gen_andi_tl(tdest, tsrcb, 31);
+        tcg_gen_shl_tl(tdest, tsrca, tdest);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shlx";
+        break;
+    case OE_RRR(SHL, 0, X0):
+    case OE_RRR(SHL, 0, X1):
+    case OE_RRR(SHL, 6, Y0):
+    case OE_RRR(SHL, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_shl_tl(tdest, tsrca, tdest);
+        mnemonic = "shl";
+        break;
+    case OE_RRR(SHRS, 0, X0):
+    case OE_RRR(SHRS, 0, X1):
+    case OE_RRR(SHRS, 6, Y0):
+    case OE_RRR(SHRS, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_sar_tl(tdest, tsrca, tdest);
+        mnemonic = "shrs";
+        break;
+    case OE_RRR(SHRUX, 0, X0):
+    case OE_RRR(SHRUX, 0, X1):
+        t0 = tcg_temp_new();
+        tcg_gen_andi_tl(t0, tsrcb, 31);
+        tcg_gen_ext32u_tl(tdest, tsrca);
+        tcg_gen_shr_tl(tdest, tdest, t0);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        tcg_temp_free(t0);
+        mnemonic = "shrux";
+        break;
+    case OE_RRR(SHRU, 0, X0):
+    case OE_RRR(SHRU, 0, X1):
+    case OE_RRR(SHRU, 6, Y0):
+    case OE_RRR(SHRU, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_shr_tl(tdest, tsrca, tdest);
+        mnemonic = "shru";
+        break;
+    case OE_RRR(SHUFFLEBYTES, 0, X0):
+        gen_helper_shufflebytes(tdest, load_gr(dc, dest), tsrca, tsrca);
+        mnemonic = "shufflebytes";
+        break;
+    case OE_RRR(SUBXSC, 0, X0):
+    case OE_RRR(SUBXSC, 0, X1):
+        gen_saturate_op(tdest, tsrca, tsrcb, tcg_gen_sub_tl);
+        mnemonic = "subxsc";
+        break;
+    case OE_RRR(SUBX, 0, X0):
+    case OE_RRR(SUBX, 0, X1):
+    case OE_RRR(SUBX, 0, Y0):
+    case OE_RRR(SUBX, 0, Y1):
+        tcg_gen_sub_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "subx";
+        break;
+    case OE_RRR(SUB, 0, X0):
+    case OE_RRR(SUB, 0, X1):
+    case OE_RRR(SUB, 0, Y0):
+    case OE_RRR(SUB, 0, Y1):
+        tcg_gen_sub_tl(tdest, tsrca, tsrcb);
+        mnemonic = "sub";
+        break;
+    case OE_RRR(V1ADDUC, 0, X0):
+    case OE_RRR(V1ADDUC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1ADD, 0, X0):
+    case OE_RRR(V1ADD, 0, X1):
+        gen_v12add(tdest, tsrca, tsrcb, V1_IMM(0x80));
+        mnemonic = "v1add";
+        break;
+    case OE_RRR(V1ADIFFU, 0, X0):
+    case OE_RRR(V1AVGU, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1CMPEQ, 0, X0):
+    case OE_RRR(V1CMPEQ, 0, X1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        gen_v1cmpeq0(tdest);
+        mnemonic = "v1cmpeq";
+        break;
+    case OE_RRR(V1CMPLES, 0, X0):
+    case OE_RRR(V1CMPLES, 0, X1):
+    case OE_RRR(V1CMPLEU, 0, X0):
+    case OE_RRR(V1CMPLEU, 0, X1):
+    case OE_RRR(V1CMPLTS, 0, X0):
+    case OE_RRR(V1CMPLTS, 0, X1):
+    case OE_RRR(V1CMPLTU, 0, X0):
+    case OE_RRR(V1CMPLTU, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1CMPNE, 0, X0):
+    case OE_RRR(V1CMPNE, 0, X1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        gen_v1cmpne0(tdest);
+        mnemonic = "v1cmpne";
+        break;
+    case OE_RRR(V1DDOTPUA, 0, X0):
+    case OE_RRR(V1DDOTPUSA, 0, X0):
+    case OE_RRR(V1DDOTPUS, 0, X0):
+    case OE_RRR(V1DDOTPU, 0, X0):
+    case OE_RRR(V1DOTPA, 0, X0):
+    case OE_RRR(V1DOTPUA, 0, X0):
+    case OE_RRR(V1DOTPUSA, 0, X0):
+    case OE_RRR(V1DOTPUS, 0, X0):
+    case OE_RRR(V1DOTPU, 0, X0):
+    case OE_RRR(V1DOTP, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1INT_H, 0, X0):
+    case OE_RRR(V1INT_H, 0, X1):
+        gen_helper_v1int_h(tdest, tsrca, tsrcb);
+        mnemonic = "v1int_h";
+        break;
+    case OE_RRR(V1INT_L, 0, X0):
+    case OE_RRR(V1INT_L, 0, X1):
+        gen_helper_v1int_l(tdest, tsrca, tsrcb);
+        mnemonic = "v1int_l";
+        break;
+    case OE_RRR(V1MAXU, 0, X0):
+    case OE_RRR(V1MAXU, 0, X1):
+    case OE_RRR(V1MINU, 0, X0):
+    case OE_RRR(V1MINU, 0, X1):
+    case OE_RRR(V1MNZ, 0, X0):
+    case OE_RRR(V1MNZ, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1MULTU, 0, X0):
+        gen_helper_v1multu(tdest, tsrca, tsrcb);
+        mnemonic = "v1multu";
+        break;
+    case OE_RRR(V1MULUS, 0, X0):
+    case OE_RRR(V1MULU, 0, X0):
+    case OE_RRR(V1MZ, 0, X0):
+    case OE_RRR(V1MZ, 0, X1):
+    case OE_RRR(V1SADAU, 0, X0):
+    case OE_RRR(V1SADU, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1SHL, 0, X0):
+    case OE_RRR(V1SHL, 0, X1):
+        gen_helper_v1shl(tdest, tsrca, tsrcb);
+        mnemonic = "v1shl";
+        break;
+    case OE_RRR(V1SHRS, 0, X0):
+    case OE_RRR(V1SHRS, 0, X1):
+        gen_helper_v1shrs(tdest, tsrca, tsrcb);
+        mnemonic = "v1shrs";
+        break;
+    case OE_RRR(V1SHRU, 0, X0):
+    case OE_RRR(V1SHRU, 0, X1):
+        gen_helper_v1shru(tdest, tsrca, tsrcb);
+        mnemonic = "v1shru";
+        break;
+    case OE_RRR(V1SUBUC, 0, X0):
+    case OE_RRR(V1SUBUC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1SUB, 0, X0):
+    case OE_RRR(V1SUB, 0, X1):
+        gen_v12sub(tdest, tsrca, tsrcb, V1_IMM(0x80));
+        mnemonic = "v1sub";
+        break;
+    case OE_RRR(V2ADDSC, 0, X0):
+    case OE_RRR(V2ADDSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2ADD, 0, X0):
+    case OE_RRR(V2ADD, 0, X1):
+        gen_v12add(tdest, tsrca, tsrcb, V2_IMM(0x8000));
+        mnemonic = "v2add";
+        break;
+    case OE_RRR(V2ADIFFS, 0, X0):
+    case OE_RRR(V2AVGS, 0, X0):
+    case OE_RRR(V2CMPEQ, 0, X0):
+    case OE_RRR(V2CMPEQ, 0, X1):
+    case OE_RRR(V2CMPLES, 0, X0):
+    case OE_RRR(V2CMPLES, 0, X1):
+    case OE_RRR(V2CMPLEU, 0, X0):
+    case OE_RRR(V2CMPLEU, 0, X1):
+    case OE_RRR(V2CMPLTS, 0, X0):
+    case OE_RRR(V2CMPLTS, 0, X1):
+    case OE_RRR(V2CMPLTU, 0, X0):
+    case OE_RRR(V2CMPLTU, 0, X1):
+    case OE_RRR(V2CMPNE, 0, X0):
+    case OE_RRR(V2CMPNE, 0, X1):
+    case OE_RRR(V2DOTPA, 0, X0):
+    case OE_RRR(V2DOTP, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2INT_H, 0, X0):
+    case OE_RRR(V2INT_H, 0, X1):
+        gen_helper_v2int_h(tdest, tsrca, tsrcb);
+        mnemonic = "v2int_h";
+        break;
+    case OE_RRR(V2INT_L, 0, X0):
+    case OE_RRR(V2INT_L, 0, X1):
+        gen_helper_v2int_l(tdest, tsrca, tsrcb);
+        mnemonic = "v2int_l";
+        break;
+    case OE_RRR(V2MAXS, 0, X0):
+    case OE_RRR(V2MAXS, 0, X1):
+    case OE_RRR(V2MINS, 0, X0):
+    case OE_RRR(V2MINS, 0, X1):
+    case OE_RRR(V2MNZ, 0, X0):
+    case OE_RRR(V2MNZ, 0, X1):
+    case OE_RRR(V2MULFSC, 0, X0):
+    case OE_RRR(V2MULS, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2MULTS, 0, X0):
+        gen_helper_v2mults(tdest, tsrca, tsrcb);
+        mnemonic = "v2mults";
+        break;
+    case OE_RRR(V2MZ, 0, X0):
+    case OE_RRR(V2MZ, 0, X1):
+    case OE_RRR(V2PACKH, 0, X0):
+    case OE_RRR(V2PACKH, 0, X1):
+    case OE_RRR(V2PACKL, 0, X0):
+    case OE_RRR(V2PACKL, 0, X1):
+    case OE_RRR(V2PACKUC, 0, X0):
+    case OE_RRR(V2PACKUC, 0, X1):
+    case OE_RRR(V2SADAS, 0, X0):
+    case OE_RRR(V2SADAU, 0, X0):
+    case OE_RRR(V2SADS, 0, X0):
+    case OE_RRR(V2SADU, 0, X0):
+    case OE_RRR(V2SHLSC, 0, X0):
+    case OE_RRR(V2SHLSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2SHL, 0, X0):
+    case OE_RRR(V2SHL, 0, X1):
+        gen_helper_v2shl(tdest, tsrca, tsrcb);
+        mnemonic = "v2shl";
+        break;
+    case OE_RRR(V2SHRS, 0, X0):
+    case OE_RRR(V2SHRS, 0, X1):
+        gen_helper_v2shrs(tdest, tsrca, tsrcb);
+        mnemonic = "v2shrs";
+        break;
+    case OE_RRR(V2SHRU, 0, X0):
+    case OE_RRR(V2SHRU, 0, X1):
+        gen_helper_v2shru(tdest, tsrca, tsrcb);
+        mnemonic = "v2shru";
+        break;
+    case OE_RRR(V2SUBSC, 0, X0):
+    case OE_RRR(V2SUBSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2SUB, 0, X0):
+    case OE_RRR(V2SUB, 0, X1):
+        gen_v12sub(tdest, tsrca, tsrcb, V2_IMM(0x8000));
+        mnemonic = "v2sub";
+        break;
+    case OE_RRR(V4ADDSC, 0, X0):
+    case OE_RRR(V4ADDSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4ADD, 0, X0):
+    case OE_RRR(V4ADD, 0, X1):
+        gen_v4op(tdest, tsrca, tsrcb, tcg_gen_add_i32);
+        mnemonic = "v4add";
+        break;
+    case OE_RRR(V4INT_H, 0, X0):
+    case OE_RRR(V4INT_H, 0, X1):
+        tcg_gen_shri_tl(tdest, tsrcb, 32);
+        tcg_gen_deposit_tl(tdest, tsrca, tdest, 0, 32);
+        mnemonic = "v4int_h";
+        break;
+    case OE_RRR(V4INT_L, 0, X0):
+    case OE_RRR(V4INT_L, 0, X1):
+        tcg_gen_deposit_tl(tdest, tsrcb, tsrca, 32, 32);
+        mnemonic = "v4int_l";
+        break;
+    case OE_RRR(V4PACKSC, 0, X0):
+    case OE_RRR(V4PACKSC, 0, X1):
+    case OE_RRR(V4SHLSC, 0, X0):
+    case OE_RRR(V4SHLSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4SHL, 0, X0):
+    case OE_RRR(V4SHL, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_shl_i32);
+        mnemonic = "v4shl";
+        break;
+    case OE_RRR(V4SHRS, 0, X0):
+    case OE_RRR(V4SHRS, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_sar_i32);
+        mnemonic = "v4shrs";
+        break;
+    case OE_RRR(V4SHRU, 0, X0):
+    case OE_RRR(V4SHRU, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_shr_i32);
+        mnemonic = "v4shru";
+        break;
+    case OE_RRR(V4SUBSC, 0, X0):
+    case OE_RRR(V4SUBSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4SUB, 0, X0):
+    case OE_RRR(V4SUB, 0, X1):
+        gen_v4op(tdest, tsrca, tsrcb, tcg_gen_sub_i32);
+        mnemonic = "v2sub";
+        break;
+    case OE_RRR(XOR, 0, X0):
+    case OE_RRR(XOR, 0, X1):
+    case OE_RRR(XOR, 5, Y0):
+    case OE_RRR(XOR, 5, Y1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        mnemonic = "xor";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %s", mnemonic,
+                  reg_names[dest], reg_names[srca], reg_names[srcb]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_rri_opcode(DisasContext *dc, unsigned opext,
+                               unsigned dest, unsigned srca, int imm)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    bool prefetch_nofault = false;
+    const char *mnemonic;
+    TCGMemOp memop;
+    int i2, i3;
+    TCGv t0;
+
+    switch (opext) {
+    case OE(ADDI_OPCODE_Y0, 0, Y0):
+    case OE(ADDI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ADDI, X0):
+    case OE_IM(ADDI, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        mnemonic = "addi";
+        break;
+    case OE(ADDXI_OPCODE_Y0, 0, Y0):
+    case OE(ADDXI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ADDXI, X0):
+    case OE_IM(ADDXI, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addxi";
+        break;
+    case OE(ANDI_OPCODE_Y0, 0, Y0):
+    case OE(ANDI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ANDI, X0):
+    case OE_IM(ANDI, X1):
+        tcg_gen_andi_tl(tdest, tsrca, imm);
+        mnemonic = "andi";
+        break;
+    case OE(CMPEQI_OPCODE_Y0, 0, Y0):
+    case OE(CMPEQI_OPCODE_Y1, 0, Y1):
+    case OE_IM(CMPEQI, X0):
+    case OE_IM(CMPEQI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_EQ, tdest, tsrca, imm);
+        mnemonic = "cmpeqi";
+        break;
+    case OE(CMPLTSI_OPCODE_Y0, 0, Y0):
+    case OE(CMPLTSI_OPCODE_Y1, 0, Y1):
+    case OE_IM(CMPLTSI, X0):
+    case OE_IM(CMPLTSI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_LT, tdest, tsrca, imm);
+        mnemonic = "cmpltsi";
+        break;
+    case OE_IM(CMPLTUI, X0):
+    case OE_IM(CMPLTUI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_LTU, tdest, tsrca, imm);
+        mnemonic = "cmpltui";
+        break;
+    case OE_IM(LD1S_ADD, X1):
+        memop = MO_SB;
+        mnemonic = "ld1s_add"; /* prefetch_add_l1_fault */
+        goto do_load_add;
+    case OE_IM(LD1U_ADD, X1):
+        memop = MO_UB;
+        mnemonic = "ld1u_add"; /* prefetch_add_l1 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LD2S_ADD, X1):
+        memop = MO_TESW;
+        mnemonic = "ld2s_add"; /* prefetch_add_l2_fault */
+        goto do_load_add;
+    case OE_IM(LD2U_ADD, X1):
+        memop = MO_TEUW;
+        mnemonic = "ld2u_add"; /* prefetch_add_l2 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LD4S_ADD, X1):
+        memop = MO_TESL;
+        mnemonic = "ld4s_add"; /* prefetch_add_l3_fault */
+        goto do_load_add;
+    case OE_IM(LD4U_ADD, X1):
+        memop = MO_TEUL;
+        mnemonic = "ld4u_add"; /* prefetch_add_l3 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LDNT1S_ADD, X1):
+        memop = MO_SB;
+        mnemonic = "ldnt1s_add";
+        goto do_load_add;
+    case OE_IM(LDNT1U_ADD, X1):
+        memop = MO_UB;
+        mnemonic = "ldnt1u_add";
+        goto do_load_add;
+    case OE_IM(LDNT2S_ADD, X1):
+        memop = MO_TESW;
+        mnemonic = "ldnt2s_add";
+        goto do_load_add;
+    case OE_IM(LDNT2U_ADD, X1):
+        memop = MO_TEUW;
+        mnemonic = "ldnt2u_add";
+        goto do_load_add;
+    case OE_IM(LDNT4S_ADD, X1):
+        memop = MO_TESL;
+        mnemonic = "ldnt4s_add";
+        goto do_load_add;
+    case OE_IM(LDNT4U_ADD, X1):
+        memop = MO_TEUL;
+        mnemonic = "ldnt4u_add";
+        goto do_load_add;
+    case OE_IM(LDNT_ADD, X1):
+        memop = MO_TEQ;
+        mnemonic = "ldnt_add";
+        goto do_load_add;
+    case OE_IM(LD_ADD, X1):
+        memop = MO_TEQ;
+        mnemonic = "ld_add";
+    do_load_add:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(tdest, tsrca, dc->mmuidx, memop);
+        }
+        tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+        break;
+    case OE_IM(LDNA_ADD, X1):
+        tcg_gen_andi_tl(tdest, tsrca, ~7);
+        tcg_gen_qemu_ld_tl(tdest, tdest, dc->mmuidx, MO_TEQ);
+        tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+        mnemonic = "ldna_add";
+        break;
+    case OE_IM(ORI, X0):
+    case OE_IM(ORI, X1):
+        tcg_gen_ori_tl(tdest, tsrca, imm);
+        mnemonic = "ori";
+        break;
+    case OE_IM(V1ADDI, X0):
+    case OE_IM(V1ADDI, X1):
+        t0 = tcg_const_tl(V1_IMM(imm));
+        gen_v12add(tdest, tsrca, t0, V1_IMM(0x80));
+        tcg_temp_free(t0);
+        mnemonic = "v1addi";
+        break;
+    case OE_IM(V1CMPEQI, X0):
+    case OE_IM(V1CMPEQI, X1):
+        tcg_gen_xori_tl(tdest, tsrca, V1_IMM(imm));
+        gen_v1cmpeq0(tdest);
+        mnemonic = "v1cmpeqi";
+        break;
+    case OE_IM(V1CMPLTSI, X0):
+    case OE_IM(V1CMPLTSI, X1):
+    case OE_IM(V1CMPLTUI, X0):
+    case OE_IM(V1CMPLTUI, X1):
+    case OE_IM(V1MAXUI, X0):
+    case OE_IM(V1MAXUI, X1):
+    case OE_IM(V1MINUI, X0):
+    case OE_IM(V1MINUI, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_IM(V2ADDI, X0):
+    case OE_IM(V2ADDI, X1):
+        t0 = tcg_const_tl(V2_IMM(imm));
+        gen_v12add(tdest, tsrca, t0, V2_IMM(0x8000));
+        tcg_temp_free(t0);
+        mnemonic = "v2addi";
+        break;
+    case OE_IM(V2CMPEQI, X0):
+    case OE_IM(V2CMPEQI, X1):
+    case OE_IM(V2CMPLTSI, X0):
+    case OE_IM(V2CMPLTSI, X1):
+    case OE_IM(V2CMPLTUI, X0):
+    case OE_IM(V2CMPLTUI, X1):
+    case OE_IM(V2MAXSI, X0):
+    case OE_IM(V2MAXSI, X1):
+    case OE_IM(V2MINSI, X0):
+    case OE_IM(V2MINSI, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_IM(XORI, X0):
+    case OE_IM(XORI, X1):
+        tcg_gen_xori_tl(tdest, tsrca, imm);
+        mnemonic = "xori";
+        break;
+
+    case OE_SH(ROTLI, X0):
+    case OE_SH(ROTLI, X1):
+    case OE_SH(ROTLI, Y0):
+    case OE_SH(ROTLI, Y1):
+        tcg_gen_rotli_tl(tdest, tsrca, imm);
+        mnemonic = "rotli";
+        break;
+    case OE_SH(SHLI, X0):
+    case OE_SH(SHLI, X1):
+    case OE_SH(SHLI, Y0):
+    case OE_SH(SHLI, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, imm);
+        mnemonic = "shli";
+        break;
+    case OE_SH(SHLXI, X0):
+    case OE_SH(SHLXI, X1):
+        tcg_gen_shli_tl(tdest, tsrca, imm & 31);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shlxi";
+        break;
+    case OE_SH(SHRSI, X0):
+    case OE_SH(SHRSI, X1):
+    case OE_SH(SHRSI, Y0):
+    case OE_SH(SHRSI, Y1):
+        tcg_gen_sari_tl(tdest, tsrca, imm);
+        mnemonic = "shrsi";
+        break;
+    case OE_SH(SHRUI, X0):
+    case OE_SH(SHRUI, X1):
+    case OE_SH(SHRUI, Y0):
+    case OE_SH(SHRUI, Y1):
+        tcg_gen_shri_tl(tdest, tsrca, imm);
+        mnemonic = "shrui";
+        break;
+    case OE_SH(SHRUXI, X0):
+    case OE_SH(SHRUXI, X1):
+        if ((imm & 31) == 0) {
+            tcg_gen_ext32s_tl(tdest, tsrca);
+        } else {
+            tcg_gen_ext32u_tl(tdest, tsrca);
+            tcg_gen_shri_tl(tdest, tdest, imm & 31);
+        }
+        mnemonic = "shlxi";
+        break;
+    case OE_SH(V1SHLI, X0):
+    case OE_SH(V1SHLI, X1):
+        i2 = imm & 7;
+        i3 = 0xff >> i2;
+        tcg_gen_andi_tl(tdest, tsrca, V1_IMM(i3));
+        tcg_gen_shli_tl(tdest, tdest, i2);
+        mnemonic = "v1shli";
+        break;
+    case OE_SH(V1SHRSI, X0):
+    case OE_SH(V1SHRSI, X1):
+        t0 = tcg_const_tl(imm & 7);
+        gen_helper_v1shrs(tdest, tsrca, t0);
+        tcg_temp_free(t0);
+        mnemonic = "v1shrsi";
+        break;
+    case OE_SH(V1SHRUI, X0):
+    case OE_SH(V1SHRUI, X1):
+        i2 = imm & 7;
+        i3 = (0xff << i2) & 0xff;
+        tcg_gen_andi_tl(tdest, tsrca, V1_IMM(i3));
+        tcg_gen_shri_tl(tdest, tdest, i2);
+        mnemonic = "v1shrui";
+        break;
+    case OE_SH(V2SHLI, X0):
+    case OE_SH(V2SHLI, X1):
+        i2 = imm & 15;
+        i3 = 0xffff >> i2;
+        tcg_gen_andi_tl(tdest, tsrca, V2_IMM(i3));
+        tcg_gen_shli_tl(tdest, tdest, i2);
+        mnemonic = "v2shli";
+        break;
+    case OE_SH(V2SHRSI, X0):
+    case OE_SH(V2SHRSI, X1):
+        t0 = tcg_const_tl(imm & 15);
+        gen_helper_v2shrs(tdest, tsrca, t0);
+        tcg_temp_free(t0);
+        mnemonic = "v2shrsi";
+        break;
+    case OE_SH(V2SHRUI, X0):
+    case OE_SH(V2SHRUI, X1):
+        i2 = imm & 15;
+        i3 = (0xffff << i2) & 0xffff;
+        tcg_gen_andi_tl(tdest, tsrca, V2_IMM(i3));
+        tcg_gen_shri_tl(tdest, tdest, i2);
+        mnemonic = "v2shrui";
+        break;
+
+    case OE(ADDLI_OPCODE_X0, 0, X0):
+    case OE(ADDLI_OPCODE_X1, 0, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        mnemonic = "addli";
+        break;
+    case OE(ADDXLI_OPCODE_X0, 0, X0):
+    case OE(ADDXLI_OPCODE_X1, 0, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addxli";
+        break;
+    case OE(SHL16INSLI_OPCODE_X0, 0, X0):
+    case OE(SHL16INSLI_OPCODE_X1, 0, X1):
+        tcg_gen_shli_tl(tdest, tsrca, 16);
+        tcg_gen_ori_tl(tdest, tdest, imm & 0xffff);
+        mnemonic = "shl16insli";
+        break;
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %d", mnemonic,
+                  reg_names[dest], reg_names[srca], imm);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_bf_opcode_x0(DisasContext *dc, unsigned ext,
+                                 unsigned dest, unsigned srca,
+                                 unsigned bfs, unsigned bfe)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcd;
+    int len;
+    const char *mnemonic;
+
+    /* The bitfield is either between E and S inclusive,
+       or up from S and down from E inclusive.  */
+    if (bfs <= bfe) {
+        len = bfe - bfs + 1;
+    } else {
+        len = (64 - bfs) + (bfe + 1);
+    }
+
+    switch (ext) {
+    case BFEXTU_BF_OPCODE_X0:
+        if (bfs == 0 && bfe == 7) {
+            tcg_gen_ext8u_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 15) {
+            tcg_gen_ext16u_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 31) {
+            tcg_gen_ext32u_tl(tdest, tsrca);
+        } else {
+            int rol = 63 - bfe;
+            if (bfs <= bfe) {
+                tcg_gen_shli_tl(tdest, tsrca, rol);
+            } else {
+                tcg_gen_rotli_tl(tdest, tsrca, rol);
+            }
+            tcg_gen_shri_tl(tdest, tdest, (bfs + rol) & 63);
+        }
+        mnemonic = "bfextu";
+        break;
+
+    case BFEXTS_BF_OPCODE_X0:
+        if (bfs == 0 && bfe == 7) {
+            tcg_gen_ext8s_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 15) {
+            tcg_gen_ext16s_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 31) {
+            tcg_gen_ext32s_tl(tdest, tsrca);
+        } else {
+            int rol = 63 - bfe;
+            if (bfs <= bfe) {
+                tcg_gen_shli_tl(tdest, tsrca, rol);
+            } else {
+                tcg_gen_rotli_tl(tdest, tsrca, rol);
+            }
+            tcg_gen_sari_tl(tdest, tdest, (bfs + rol) & 63);
+        }
+        mnemonic = "bfexts";
+        break;
+
+    case BFINS_BF_OPCODE_X0:
+        tsrcd = load_gr(dc, dest);
+        if (bfs <= bfe) {
+            tcg_gen_deposit_tl(tdest, tsrcd, tsrca, bfs, len);
+        } else {
+            tcg_gen_rotri_tl(tdest, tsrcd, bfs);
+            tcg_gen_deposit_tl(tdest, tdest, tsrca, 0, len);
+            tcg_gen_rotli_tl(tdest, tdest, bfs);
+        }
+        mnemonic = "bfins";
+        break;
+
+    case MM_BF_OPCODE_X0:
+        tsrcd = load_gr(dc, dest);
+        if (bfs == 0) {
+            tcg_gen_deposit_tl(tdest, tsrca, tsrcd, 0, len);
+        } else {
+            uint64_t mask = len == 64 ? -1 : rol64((1ULL << len) - 1, bfs);
+            TCGv tmp = tcg_const_tl(mask);
+
+            tcg_gen_and_tl(tdest, tsrcd, tmp);
+            tcg_gen_andc_tl(tmp, tsrca, tmp);
+            tcg_gen_or_tl(tdest, tdest, tmp);
+            tcg_temp_free(tmp);
+        }
+        mnemonic = "mm";
+        break;
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %u, %u", mnemonic,
+                  reg_names[dest], reg_names[srca], bfs, bfe);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_branch_opcode_x1(DisasContext *dc, unsigned ext,
+                                     unsigned srca, int off)
+{
+    target_ulong tgt = dc->pc + off * TILEGX_BUNDLE_SIZE_IN_BYTES;
+    const char *mnemonic;
+
+    dc->jmp.dest = tcg_const_tl(tgt);
+    dc->jmp.val1 = tcg_temp_new();
+    tcg_gen_mov_tl(dc->jmp.val1, load_gr(dc, srca));
+
+    /* Note that the "predict taken" opcodes have bit 0 clear.
+       Therefore, fold the two cases together by setting bit 0.  */
+    switch (ext | 1) {
+    case BEQZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_EQ;
+        mnemonic = "beqz";
+        break;
+    case BNEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_NE;
+        mnemonic = "bnez";
+        break;
+    case BGEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_GE;
+        mnemonic = "bgez";
+        break;
+    case BGTZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_GT;
+        mnemonic = "bgtz";
+        break;
+    case BLEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_LE;
+        mnemonic = "blez";
+        break;
+    case BLTZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_LT;
+        mnemonic = "bltz";
+        break;
+    case BLBC_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_EQ;
+        tcg_gen_andi_tl(dc->jmp.val1, dc->jmp.val1, 1);
+        mnemonic = "blbc";
+        break;
+    case BLBS_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_NE;
+        tcg_gen_andi_tl(dc->jmp.val1, dc->jmp.val1, 1);
+        mnemonic = "blbs";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("%s%s %s, " TARGET_FMT_lx " <%s>",
+                 mnemonic, ext & 1 ? "" : "t",
+                 reg_names[srca], tgt, lookup_symbol(tgt));
+    }
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_jump_opcode_x1(DisasContext *dc, unsigned ext, int off)
+{
+    target_ulong tgt = dc->pc + off * TILEGX_BUNDLE_SIZE_IN_BYTES;
+    const char *mnemonic = "j";
+
+    /* The extension field is 1 bit, therefore we only have JAL and J.  */
+    if (ext == JAL_JUMP_OPCODE_X1) {
+        tcg_gen_movi_tl(dest_gr(dc, TILEGX_R_LR),
+                        dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+        mnemonic = "jal";
+    }
+    dc->jmp.cond = TCG_COND_ALWAYS;
+    dc->jmp.dest = tcg_const_tl(tgt);
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("%s " TARGET_FMT_lx " <%s>",
+                 mnemonic, tgt, lookup_symbol(tgt));
+    }
+    return TILEGX_EXCP_NONE;
+}
+
+typedef struct {
+    const char *name;
+    intptr_t offset;
+    void (*get)(TCGv, TCGv_ptr);
+    void (*put)(TCGv_ptr, TCGv);
+} TileSPR;
+
+static const TileSPR *find_spr(unsigned spr)
+{
+    /* Allow the compiler to construct the binary search tree.  */
+#define D(N, O, G, P) \
+    case SPR_##N: { static const TileSPR x = { #N, O, G, P }; return &x; }
+
+    switch (spr) {
+    D(CMPEXCH_VALUE,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_CMPEXCH]), 0, 0)
+    D(INTERRUPT_CRITICAL_SECTION,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_CRITICAL_SEC]), 0, 0)
+    D(SIM_CONTROL,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_SIM_CONTROL]), 0, 0)
+    D(EX_CONTEXT_0_0,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_0]), 0, 0)
+    D(EX_CONTEXT_0_1,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_1]), 0, 0)
+    }
+
+#undef D
+
+    qemu_log_mask(LOG_UNIMP, "UNIMP SPR %u\n", spr);
+    return NULL;
+}
+
+static TileExcp gen_mtspr_x1(DisasContext *dc, unsigned spr, unsigned srca)
+{
+    const TileSPR *def = find_spr(spr);
+    TCGv tsrca;
+
+    if (def == NULL) {
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr spr[%u], %s", spr, reg_names[srca]);
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    }
+
+    tsrca = load_gr(dc, srca);
+    if (def->put) {
+        def->put(cpu_env, tsrca);
+    } else {
+        tcg_gen_st_tl(tsrca, cpu_env, def->offset);
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr %s, %s", def->name, reg_names[srca]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_mfspr_x1(DisasContext *dc, unsigned dest, unsigned spr)
+{
+    const TileSPR *def = find_spr(spr);
+    TCGv tdest;
+
+    if (def == NULL) {
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr %s, spr[%u]", reg_names[dest], spr);
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    }
+
+    tdest = dest_gr(dc, dest);
+    if (def->get) {
+        def->get(tdest, cpu_env);
+    } else {
+        tcg_gen_ld_tl(tdest, cpu_env, def->offset);
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "mfspr %s, %s", reg_names[dest], def->name);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp decode_y0(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_Y0(bundle);
+    unsigned ext = get_RRROpcodeExtension_Y0(bundle);
+    unsigned dest = get_Dest_Y0(bundle);
+    unsigned srca = get_SrcA_Y0(bundle);
+    unsigned srcb;
+    int imm;
+
+    switch (opc) {
+    case RRR_1_OPCODE_Y0:
+        if (ext == UNARY_RRR_1_OPCODE_Y0) {
+            ext = get_UnaryOpcodeExtension_Y0(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, Y0), dest, srca, bundle);
+        }
+        /* fallthru */
+    case RRR_0_OPCODE_Y0:
+    case RRR_2_OPCODE_Y0:
+    case RRR_3_OPCODE_Y0:
+    case RRR_4_OPCODE_Y0:
+    case RRR_5_OPCODE_Y0:
+    case RRR_6_OPCODE_Y0:
+    case RRR_7_OPCODE_Y0:
+    case RRR_8_OPCODE_Y0:
+    case RRR_9_OPCODE_Y0:
+        srcb = get_SrcB_Y0(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, Y0), dest, srca, srcb);
+
+    case SHIFT_OPCODE_Y0:
+        ext = get_ShiftOpcodeExtension_Y0(bundle);
+        imm = get_ShAmt_Y0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, Y0), dest, srca, imm);
+
+    case ADDI_OPCODE_Y0:
+    case ADDXI_OPCODE_Y0:
+    case ANDI_OPCODE_Y0:
+    case CMPEQI_OPCODE_Y0:
+    case CMPLTSI_OPCODE_Y0:
+        imm = (int8_t)get_Imm8_Y0(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, Y0), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_y1(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_Y1(bundle);
+    unsigned ext = get_RRROpcodeExtension_Y1(bundle);
+    unsigned dest = get_Dest_Y1(bundle);
+    unsigned srca = get_SrcA_Y1(bundle);
+    unsigned srcb;
+    int imm;
+
+    switch (get_Opcode_Y1(bundle)) {
+    case RRR_1_OPCODE_Y1:
+        if (ext == UNARY_RRR_1_OPCODE_Y0) {
+            ext = get_UnaryOpcodeExtension_Y1(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, Y1), dest, srca, bundle);
+        }
+        /* fallthru */
+    case RRR_0_OPCODE_Y1:
+    case RRR_2_OPCODE_Y1:
+    case RRR_3_OPCODE_Y1:
+    case RRR_4_OPCODE_Y1:
+    case RRR_5_OPCODE_Y1:
+    case RRR_6_OPCODE_Y1:
+    case RRR_7_OPCODE_Y1:
+        srcb = get_SrcB_Y1(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, Y1), dest, srca, srcb);
+
+    case SHIFT_OPCODE_Y1:
+        ext = get_ShiftOpcodeExtension_Y1(bundle);
+        imm = get_ShAmt_Y1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, Y1), dest, srca, imm);
+
+    case ADDI_OPCODE_Y1:
+    case ADDXI_OPCODE_Y1:
+    case ANDI_OPCODE_Y1:
+    case CMPEQI_OPCODE_Y1:
+    case CMPLTSI_OPCODE_Y1:
+        imm = (int8_t)get_Imm8_Y1(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, Y1), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_y2(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned mode = get_Mode(bundle);
+    unsigned opc = get_Opcode_Y2(bundle);
+    unsigned srca = get_SrcA_Y2(bundle);
+    unsigned srcbdest = get_SrcBDest_Y2(bundle);
+    const char *mnemonic;
+    TCGMemOp memop;
+    bool prefetch_nofault = false;
+
+    switch (OEY2(opc, mode)) {
+    case OEY2(LD1S_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_SB;
+        mnemonic = "ld1s"; /* prefetch_l1_fault */
+        goto do_load;
+    case OEY2(LD1U_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_UB;
+        mnemonic = "ld1u"; /* prefetch, prefetch_l1 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD2S_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_TESW;
+        mnemonic = "ld2s"; /* prefetch_l2_fault */
+        goto do_load;
+    case OEY2(LD2U_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_TEUW;
+        mnemonic = "ld2u"; /* prefetch_l2 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD4S_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TESL;
+        mnemonic = "ld4s"; /* prefetch_l3_fault */
+        goto do_load;
+    case OEY2(LD4U_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TEUL;
+        mnemonic = "ld4u"; /* prefetch_l3 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TEQ;
+        mnemonic = "ld";
+    do_load:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(dest_gr(dc, srcbdest), load_gr(dc, srca),
+                               dc->mmuidx, memop);
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", mnemonic,
+                      reg_names[srcbdest], reg_names[srca]);
+        return TILEGX_EXCP_NONE;
+
+    case OEY2(ST1_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_UB, "st1");
+    case OEY2(ST2_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEUW, "st2");
+    case OEY2(ST4_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEUL, "st4");
+    case OEY2(ST_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEQ, "st");
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_x0(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_X0(bundle);
+    unsigned dest = get_Dest_X0(bundle);
+    unsigned srca = get_SrcA_X0(bundle);
+    unsigned ext, srcb, bfs, bfe;
+    int imm;
+
+    switch (opc) {
+    case RRR_0_OPCODE_X0:
+        ext = get_RRROpcodeExtension_X0(bundle);
+        if (ext == UNARY_RRR_0_OPCODE_X0) {
+            ext = get_UnaryOpcodeExtension_X0(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, X0), dest, srca, bundle);
+        }
+        srcb = get_SrcB_X0(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, X0), dest, srca, srcb);
+
+    case SHIFT_OPCODE_X0:
+        ext = get_ShiftOpcodeExtension_X0(bundle);
+        imm = get_ShAmt_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X0), dest, srca, imm);
+
+    case IMM8_OPCODE_X0:
+        ext = get_Imm8OpcodeExtension_X0(bundle);
+        imm = (int8_t)get_Imm8_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X0), dest, srca, imm);
+
+    case BF_OPCODE_X0:
+        ext = get_BFOpcodeExtension_X0(bundle);
+        bfs = get_BFStart_X0(bundle);
+        bfe = get_BFEnd_X0(bundle);
+        return gen_bf_opcode_x0(dc, ext, dest, srca, bfs, bfe);
+
+    case ADDLI_OPCODE_X0:
+    case SHL16INSLI_OPCODE_X0:
+    case ADDXLI_OPCODE_X0:
+        imm = (int16_t)get_Imm16_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, X0), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_x1(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_X1(bundle);
+    unsigned dest = get_Dest_X1(bundle);
+    unsigned srca = get_SrcA_X1(bundle);
+    unsigned ext, srcb;
+    int imm;
+
+    switch (opc) {
+    case RRR_0_OPCODE_X1:
+        ext = get_RRROpcodeExtension_X1(bundle);
+        srcb = get_SrcB_X1(bundle);
+        switch (ext) {
+        case UNARY_RRR_0_OPCODE_X1:
+            ext = get_UnaryOpcodeExtension_X1(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, X1), dest, srca, bundle);
+        case ST1_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_UB, "st1");
+        case ST2_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUW, "st2");
+        case ST4_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUL, "st4");
+        case STNT1_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_UB, "stnt1");
+        case STNT2_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUW, "stnt2");
+        case STNT4_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUL, "stnt4");
+        case STNT_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEQ, "stnt");
+        case ST_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEQ, "st");
+        }
+        return gen_rrr_opcode(dc, OE(opc, ext, X1), dest, srca, srcb);
+
+    case SHIFT_OPCODE_X1:
+        ext = get_ShiftOpcodeExtension_X1(bundle);
+        imm = get_ShAmt_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X1), dest, srca, imm);
+
+    case IMM8_OPCODE_X1:
+        ext = get_Imm8OpcodeExtension_X1(bundle);
+        imm = (int8_t)get_Dest_Imm8_X1(bundle);
+        srcb = get_SrcB_X1(bundle);
+        switch (ext) {
+        case ST1_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_UB, "st1_add");
+        case ST2_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUW, "st2_add");
+        case ST4_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUL, "st4_add");
+        case STNT1_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_UB, "stnt1_add");
+        case STNT2_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUW, "stnt2_add");
+        case STNT4_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUL, "stnt4_add");
+        case STNT_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEQ, "stnt_add");
+        case ST_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEQ, "st_add");
+        case MFSPR_IMM8_OPCODE_X1:
+            return gen_mfspr_x1(dc, dest, get_MF_Imm14_X1(bundle));
+        case MTSPR_IMM8_OPCODE_X1:
+            return gen_mtspr_x1(dc, get_MT_Imm14_X1(bundle), srca);
+        }
+        imm = (int8_t)get_Imm8_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X1), dest, srca, imm);
+
+    case BRANCH_OPCODE_X1:
+        ext = get_BrType_X1(bundle);
+        imm = sextract32(get_BrOff_X1(bundle), 0, 17);
+        return gen_branch_opcode_x1(dc, ext, srca, imm);
+
+    case JUMP_OPCODE_X1:
+        ext = get_JumpOpcodeExtension_X1(bundle);
+        imm = sextract32(get_JumpOff_X1(bundle), 0, 27);
+        return gen_jump_opcode_x1(dc, ext, imm);
+
+    case ADDLI_OPCODE_X1:
+    case SHL16INSLI_OPCODE_X1:
+    case ADDXLI_OPCODE_X1:
+        imm = (int16_t)get_Imm16_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, X1), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static void notice_excp(DisasContext *dc, uint64_t bundle,
+                        const char *type, TileExcp excp)
+{
+    if (likely(excp == TILEGX_EXCP_NONE)) {
+        return;
+    }
+    gen_exception(dc, excp);
+    switch (excp) {
+    case TILEGX_EXCP_OPCODE_UNIMPLEMENTED:
+        qemu_log_mask(LOG_UNIMP, "UNIMP %s, [" FMT64X "]\n", type, bundle);
+        break;
+    case TILEGX_EXCP_OPCODE_UNKNOWN:
+        qemu_log_mask(LOG_UNIMP, "UNKNOWN %s, [" FMT64X "]\n", type, bundle);
+        break;
+    default:
+        break;
+    }
+}
+
+static void translate_one_bundle(DisasContext *dc, uint64_t bundle)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(dc->wb); i++) {
+        DisasContextTemp *wb = &dc->wb[i];
+        wb->reg = TILEGX_R_NOREG;
+        TCGV_UNUSED_I64(wb->val);
+    }
+    dc->num_wb = 0;
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "  %" PRIx64 ":  { ", dc->pc);
+    if (get_Mode(bundle)) {
+        notice_excp(dc, bundle, "y0", decode_y0(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "y1", decode_y1(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "y2", decode_y2(dc, bundle));
+    } else {
+        notice_excp(dc, bundle, "x0", decode_x0(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "x1", decode_x1(dc, bundle));
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, " }\n");
+
+    for (i = dc->num_wb - 1; i >= 0; --i) {
+        DisasContextTemp *wb = &dc->wb[i];
+        if (wb->reg < TILEGX_R_COUNT) {
+            tcg_gen_mov_i64(cpu_regs[wb->reg], wb->val);
+        }
+        tcg_temp_free_i64(wb->val);
+    }
+
+    if (dc->jmp.cond != TCG_COND_NEVER) {
+        if (dc->jmp.cond == TCG_COND_ALWAYS) {
+            tcg_gen_mov_i64(cpu_pc, dc->jmp.dest);
+        } else {
+            TCGv next = tcg_const_i64(dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+            tcg_gen_movcond_i64(dc->jmp.cond, cpu_pc,
+                                dc->jmp.val1, load_zero(dc),
+                                dc->jmp.dest, next);
+            tcg_temp_free_i64(dc->jmp.val1);
+            tcg_temp_free_i64(next);
+        }
+        tcg_temp_free_i64(dc->jmp.dest);
+        tcg_gen_exit_tb(0);
+        dc->exit_tb = true;
+    } else if (dc->atomic_excp != TILEGX_EXCP_NONE) {
+        gen_exception(dc, dc->atomic_excp);
+    }
+}
+
+void gen_intermediate_code(CPUTLGState *env, struct TranslationBlock *tb)
+{
+    TileGXCPU *cpu = tilegx_env_get_cpu(env);
+    DisasContext ctx;
+    DisasContext *dc = &ctx;
+    CPUState *cs = CPU(cpu);
+    uint64_t pc_start = tb->pc;
+    uint64_t next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
+    int num_insns = 0;
+    int max_insns = tb->cflags & CF_COUNT_MASK;
+
+    dc->pc = pc_start;
+    dc->mmuidx = 0;
+    dc->exit_tb = false;
+    dc->atomic_excp = TILEGX_EXCP_NONE;
+    dc->jmp.cond = TCG_COND_NEVER;
+    TCGV_UNUSED_I64(dc->jmp.dest);
+    TCGV_UNUSED_I64(dc->jmp.val1);
+    TCGV_UNUSED_I64(dc->zero);
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
+    }
+    if (!max_insns) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (cs->singlestep_enabled || singlestep) {
+        max_insns = 1;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+    gen_tb_start(tb);
+
+    while (1) {
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        translate_one_bundle(dc, cpu_ldq_data(env, dc->pc));
+
+        if (dc->exit_tb) {
+            /* PC updated and EXIT_TB/GOTO_TB/exception emitted.  */
+            break;
+        }
+        dc->pc += TILEGX_BUNDLE_SIZE_IN_BYTES;
+        if (num_insns >= max_insns
+            || dc->pc >= next_page_start
+            || tcg_op_buf_full()) {
+            /* Ending the TB due to TB size or page boundary.  Set PC.  */
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            tcg_gen_exit_tb(0);
+            break;
+        }
+    }
+
+    gen_tb_end(tb, num_insns);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "\n");
+}
+
+void restore_state_to_opc(CPUTLGState *env, TranslationBlock *tb,
+                          target_ulong *data)
+{
+    env->pc = data[0];
+}
+
+void tilegx_tcg_init(void)
+{
+    int i;
+
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    cpu_pc = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUTLGState, pc), "pc");
+    for (i = 0; i < TILEGX_R_COUNT; i++) {
+        cpu_regs[i] = tcg_global_mem_new_i64(TCG_AREG0,
+                                             offsetof(CPUTLGState, regs[i]),
+                                             reg_names[i]);
+    }
+}
diff --git a/target-tricore/cpu.h b/target-tricore/cpu.h
index 916ee27ad..20a12f3a4 100644
--- a/target-tricore/cpu.h
+++ b/target-tricore/cpu.h
@@ -25,8 +25,6 @@
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
 
-#define ELF_MACHINE     EM_TRICORE
-
 #define CPUArchState struct CPUTriCoreState
 
 struct CPUTriCoreState;
@@ -350,7 +348,7 @@ void tricore_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 #define cpu_signal_handler cpu_tricore_signal_handler
 #define cpu_list tricore_cpu_list
 
-static inline int cpu_mmu_index(CPUTriCoreState *env)
+static inline int cpu_mmu_index(CPUTriCoreState *env, bool ifetch)
 {
     return 0;
 }
diff --git a/target-tricore/helper.c b/target-tricore/helper.c
index f52504c9f..1808b281f 100644
--- a/target-tricore/helper.c
+++ b/target-tricore/helper.c
@@ -20,7 +20,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <inttypes.h>
-#include <signal.h>
 
 #include "cpu.h"
 
diff --git a/target-tricore/translate.c b/target-tricore/translate.c
index 7dc7a325b..135c58347 100644
--- a/target-tricore/translate.c
+++ b/target-tricore/translate.c
@@ -457,11 +457,11 @@ gen_add64_d(TCGv_i64 ret, TCGv_i64 r1, TCGv_i64 r2)
     tcg_gen_xor_i64(t1, result, r1);
     tcg_gen_xor_i64(t0, r1, r2);
     tcg_gen_andc_i64(t1, t1, t0);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t1, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t1);
     /* calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
     /* calc AV/SAV bits */
-    tcg_gen_trunc_shr_i64_i32(temp, result, 32);
+    tcg_gen_extrh_i64_i32(temp, result);
     tcg_gen_add_tl(cpu_PSW_AV, temp, temp);
     tcg_gen_xor_tl(cpu_PSW_AV, temp, cpu_PSW_AV);
     /* calc SAV */
@@ -540,14 +540,14 @@ static inline void gen_madd32_d(TCGv ret, TCGv r1, TCGv r2, TCGv r3)
     tcg_gen_mul_i64(t1, t1, t3);
     tcg_gen_add_i64(t1, t2, t1);
 
-    tcg_gen_trunc_i64_i32(ret, t1);
+    tcg_gen_extrl_i64_i32(ret, t1);
     /* calc V
        t1 > 0x7fffffff */
     tcg_gen_setcondi_i64(TCG_COND_GT, t3, t1, 0x7fffffffLL);
     /* t1 < -0x80000000 */
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t1, -0x80000000LL);
     tcg_gen_or_i64(t2, t2, t3);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -621,7 +621,7 @@ gen_maddu64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
     /* only the add overflows, if t2 < t1
        calc V bit */
     tcg_gen_setcond_i64(TCG_COND_LTU, t2, t2, t1);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -1110,12 +1110,12 @@ gen_madd32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
     tcg_gen_sari_i64(t2, t2, up_shift);
 
     tcg_gen_add_i64(t3, t1, t2);
-    tcg_gen_trunc_i64_i32(temp3, t3);
+    tcg_gen_extrl_i64_i32(temp3, t3);
     /* calc v bit */
     tcg_gen_setcondi_i64(TCG_COND_GT, t1, t3, 0x7fffffffLL);
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t3, -0x80000000LL);
     tcg_gen_or_i64(t1, t1, t2);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* We produce an overflow on the host if the mul before was
        (0x80000000 * 0x80000000) << 1). If this is the
@@ -1273,7 +1273,7 @@ gen_madd64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
     tcg_gen_xor_i64(t3, t4, t1);
     tcg_gen_xor_i64(t2, t1, t2);
     tcg_gen_andc_i64(t3, t3, t2);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t3, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t3);
     /* We produce an overflow on the host if the mul before was
        (0x80000000 * 0x80000000) << 1). If this is the
        case, we negate the ovf. */
@@ -1356,14 +1356,14 @@ static inline void gen_msub32_d(TCGv ret, TCGv r1, TCGv r2, TCGv r3)
     tcg_gen_mul_i64(t1, t1, t3);
     tcg_gen_sub_i64(t1, t2, t1);
 
-    tcg_gen_trunc_i64_i32(ret, t1);
+    tcg_gen_extrl_i64_i32(ret, t1);
     /* calc V
        t2 > 0x7fffffff */
     tcg_gen_setcondi_i64(TCG_COND_GT, t3, t1, 0x7fffffffLL);
     /* result < -0x80000000 */
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t1, -0x80000000LL);
     tcg_gen_or_i64(t2, t2, t3);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
 
     /* Calc SV bit */
@@ -1445,7 +1445,7 @@ gen_msubu64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
     tcg_gen_extr_i64_i32(ret_low, ret_high, t3);
     /* calc V bit, only the sub can overflow, if t1 > t2 */
     tcg_gen_setcond_i64(TCG_COND_GTU, t1, t1, t2);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -1630,11 +1630,11 @@ gen_sub64_d(TCGv_i64 ret, TCGv_i64 r1, TCGv_i64 r2)
     tcg_gen_xor_i64(t1, result, r1);
     tcg_gen_xor_i64(t0, r1, r2);
     tcg_gen_and_i64(t1, t1, t0);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t1, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t1);
     /* calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
     /* calc AV/SAV bits */
-    tcg_gen_trunc_shr_i64_i32(temp, result, 32);
+    tcg_gen_extrh_i64_i32(temp, result);
     tcg_gen_add_tl(cpu_PSW_AV, temp, temp);
     tcg_gen_xor_tl(cpu_PSW_AV, temp, cpu_PSW_AV);
     /* calc SAV */
@@ -1973,12 +1973,12 @@ gen_msub32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
     tcg_gen_add_i64(t2, t2, t4);
 
     tcg_gen_sub_i64(t3, t1, t2);
-    tcg_gen_trunc_i64_i32(temp3, t3);
+    tcg_gen_extrl_i64_i32(temp3, t3);
     /* calc v bit */
     tcg_gen_setcondi_i64(TCG_COND_GT, t1, t3, 0x7fffffffLL);
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t3, -0x80000000LL);
     tcg_gen_or_i64(t1, t1, t2);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -2126,7 +2126,7 @@ gen_msub64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
     tcg_gen_xor_i64(t3, t4, t1);
     tcg_gen_xor_i64(t2, t1, t2);
     tcg_gen_and_i64(t3, t3, t2);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t3, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t3);
     /* We produce an overflow on the host if the mul before was
        (0x80000000 * 0x80000000) << 1). If this is the
        case, we negate the ovf. */
@@ -8266,43 +8266,44 @@ static void decode_opc(CPUTriCoreState *env, DisasContext *ctx, int *is_branch)
     }
 }
 
-static inline void
-gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
-                              int search_pc)
+void gen_intermediate_code(CPUTriCoreState *env, struct TranslationBlock *tb)
 {
+    TriCoreCPU *cpu = tricore_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUTriCoreState *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
-    int num_insns;
+    int num_insns, max_insns;
 
-    if (search_pc) {
-        qemu_log("search pc %d\n", search_pc);
+    num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (singlestep) {
+        max_insns = 1;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
     }
 
-    num_insns = 0;
     pc_start = tb->pc;
     ctx.pc = pc_start;
     ctx.saved_pc = -1;
     ctx.tb = tb;
     ctx.singlestep_enabled = cs->singlestep_enabled;
     ctx.bstate = BS_NONE;
-    ctx.mem_idx = cpu_mmu_index(env);
+    ctx.mem_idx = cpu_mmu_index(env, false);
 
     tcg_clear_temp_count();
     gen_tb_start(tb);
     while (ctx.bstate == BS_NONE) {
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
+
         ctx.opcode = cpu_ldl_code(env, ctx.pc);
         decode_opc(env, &ctx, 0);
 
-        num_insns++;
-
-        if (tcg_op_buf_full()) {
-            gen_save_pc(ctx.next_pc);
-            tcg_gen_exit_tb(0);
-            break;
-        }
-        if (singlestep) {
+        if (num_insns >= max_insns || tcg_op_buf_full()) {
             gen_save_pc(ctx.next_pc);
             tcg_gen_exit_tb(0);
             break;
@@ -8311,12 +8312,9 @@ gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
     }
 
     gen_tb_end(tb, num_insns);
-    if (search_pc) {
-        printf("done_generating search pc\n");
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
+
     if (tcg_check_temp_count()) {
         printf("LEAK at %08x\n", env->PC);
     }
@@ -8331,21 +8329,10 @@ gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
 }
 
 void
-gen_intermediate_code(CPUTriCoreState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, false);
-}
-
-void
-gen_intermediate_code_pc(CPUTriCoreState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, true);
-}
-
-void
-restore_state_to_opc(CPUTriCoreState *env, TranslationBlock *tb, int pc_pos)
+restore_state_to_opc(CPUTriCoreState *env, TranslationBlock *tb,
+                     target_ulong *data)
 {
-    env->PC = tcg_ctx.gen_opc_pc[pc_pos];
+    env->PC = data[0];
 }
 /*
  *
diff --git a/target-unicore32/cpu.h b/target-unicore32/cpu.h
index 45e31e54b..01c370c30 100644
--- a/target-unicore32/cpu.h
+++ b/target-unicore32/cpu.h
@@ -17,8 +17,6 @@
 #define TARGET_PHYS_ADDR_SPACE_BITS     32
 #define TARGET_VIRT_ADDR_SPACE_BITS     32
 
-#define ELF_MACHINE             EM_UNICORE32
-
 #define CPUArchState                struct CPUUniCore32State
 
 #include "config.h"
@@ -131,7 +129,7 @@ int uc32_cpu_signal_handler(int host_signum, void *pinfo, void *puc);
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index(CPUUniCore32State *env)
+static inline int cpu_mmu_index(CPUUniCore32State *env, bool ifetch)
 {
     return (env->uncached_asr & ASR_M) == ASR_MODE_USER ? 1 : 0;
 }
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index 2fc78e6f3..d2f92f02f 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -1794,10 +1794,6 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
     UniCore32CPU *cpu = uc32_env_get_cpu(env);
     unsigned int insn;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(s->pc);
-    }
-
     insn = cpu_ldl_code(env, s->pc);
     s->pc += 4;
 
@@ -1868,16 +1864,12 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
 }
 
 /* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
-        TranslationBlock *tb, bool search_pc)
+   basic block 'tb'.  */
+void gen_intermediate_code(CPUUniCore32State *env, TranslationBlock *tb)
 {
+    UniCore32CPU *cpu = uc32_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUUniCore32State *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     uint32_t next_page_start;
     int num_insns;
@@ -1899,12 +1891,14 @@ static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
     cpu_F0d = tcg_temp_new_i64();
     cpu_F1d = tcg_temp_new_i64();
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
 #ifndef CONFIG_USER_ONLY
     if ((env->uncached_asr & ASR_M) == ASR_MODE_USER) {
@@ -1916,33 +1910,22 @@ static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
 
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    gen_set_pc_im(dc->pc);
-                    gen_exception(EXCP_DEBUG);
-                    dc->is_jmp = DISAS_JUMP;
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2; /* FIXME */
-                    goto done_generating;
-                }
-            }
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            gen_set_pc_im(dc->pc);
+            gen_exception(EXCP_DEBUG);
+            dc->is_jmp = DISAS_JUMP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            goto done_generating;
         }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
@@ -1961,7 +1944,6 @@ static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
          * Otherwise the subsequent code could get translated several times.
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -2043,26 +2025,8 @@ done_generating:
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUUniCore32State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(uc32_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUUniCore32State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(uc32_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 static const char *cpu_mode_names[16] = {
@@ -2133,7 +2097,8 @@ void uc32_cpu_dump_state(CPUState *cs, FILE *f,
     cpu_dump_state_ucf64(env, f, cpu_fprintf, flags);
 }
 
-void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->regs[31] = tcg_ctx.gen_opc_pc[pc_pos];
+    env->regs[31] = data[0];
 }
diff --git a/target-xtensa/Makefile.objs b/target-xtensa/Makefile.objs
index 5c150a870..481de9197 100644
--- a/target-xtensa/Makefile.objs
+++ b/target-xtensa/Makefile.objs
@@ -2,5 +2,6 @@ obj-y += xtensa-semi.o
 obj-y += core-dc232b.o
 obj-y += core-dc233c.o
 obj-y += core-fsf.o
+obj-$(CONFIG_SOFTMMU) += monitor.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += gdbstub.o
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index 96bfc82e9..15ce0107d 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -30,7 +30,6 @@
 
 #define ALIGNED_ONLY
 #define TARGET_LONG_BITS 32
-#define ELF_MACHINE EM_XTENSA
 
 #define CPUArchState struct CPUXtensaState
 
@@ -65,6 +64,7 @@ enum {
     XTENSA_OPTION_MP_SYNCHRO,
     XTENSA_OPTION_CONDITIONAL_STORE,
     XTENSA_OPTION_ATOMCTL,
+    XTENSA_OPTION_DEPBITS,
 
     /* Interrupts and exceptions */
     XTENSA_OPTION_EXCEPTION,
@@ -383,7 +383,6 @@ typedef struct CPUXtensaState {
 #include "cpu-qom.h"
 
 #define cpu_exec cpu_xtensa_exec
-#define cpu_gen_code cpu_xtensa_gen_code
 #define cpu_signal_handler cpu_xtensa_signal_handler
 #define cpu_list xtensa_cpu_list
 
@@ -492,7 +491,7 @@ static inline uint32_t xtensa_replicate_windowstart(CPUXtensaState *env)
 #define MMU_MODE2_SUFFIX _ring2
 #define MMU_MODE3_SUFFIX _ring3
 
-static inline int cpu_mmu_index(CPUXtensaState *env)
+static inline int cpu_mmu_index(CPUXtensaState *env, bool ifetch)
 {
     return xtensa_get_cring(env);
 }
diff --git a/target-xtensa/helper.c b/target-xtensa/helper.c
index 76be50d09..2c3447ba6 100644
--- a/target-xtensa/helper.c
+++ b/target-xtensa/helper.c
@@ -541,8 +541,8 @@ static int get_physical_addr_mmu(CPUXtensaState *env, bool update_tlb,
             wi = ++env->autorefill_idx & 0x3;
             xtensa_tlb_set_entry(env, dtlb, wi, ei, vpn, pte);
             env->sregs[EXCVADDR] = vaddr;
-            qemu_log("%s: autorefill(%08x): %08x -> %08x\n",
-                    __func__, vaddr, vpn, pte);
+            qemu_log_mask(CPU_LOG_MMU, "%s: autorefill(%08x): %08x -> %08x\n",
+                          __func__, vaddr, vpn, pte);
         } else {
             xtensa_tlb_set_entry_mmu(env, &tmp_entry, dtlb, wi, ei, vpn, pte);
             entry = &tmp_entry;
@@ -590,8 +590,8 @@ static int get_pte(CPUXtensaState *env, uint32_t vaddr, uint32_t *pte)
     int ret = get_physical_addr_mmu(env, false, pt_vaddr, 0, 0,
             &paddr, &page_size, &access, false);
 
-    qemu_log("%s: trying autorefill(%08x) -> %08x\n", __func__,
-            vaddr, ret ? ~0 : paddr);
+    qemu_log_mask(CPU_LOG_MMU, "%s: trying autorefill(%08x) -> %08x\n",
+                  __func__, vaddr, ret ? ~0 : paddr);
 
     if (ret == 0) {
         *pte = ldl_phys(cs->as, paddr);
diff --git a/target-xtensa/monitor.c b/target-xtensa/monitor.c
new file mode 100644
index 000000000..554b2123d
--- /dev/null
+++ b/target-xtensa/monitor.c
@@ -0,0 +1,34 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c
index be657e615..718e54e7b 100644
--- a/target-xtensa/op_helper.c
+++ b/target-xtensa/op_helper.c
@@ -57,8 +57,8 @@ void tlb_fill(CPUState *cs,
     int ret = xtensa_get_physical_addr(env, true, vaddr, is_write, mmu_idx,
             &paddr, &page_size, &access);
 
-    qemu_log("%s(%08x, %d, %d) -> %08x, ret = %d\n", __func__,
-            vaddr, is_write, mmu_idx, paddr, ret);
+    qemu_log_mask(CPU_LOG_MMU, "%s(%08x, %d, %d) -> %08x, ret = %d\n",
+                  __func__, vaddr, is_write, mmu_idx, paddr, ret);
 
     if (ret == 0) {
         tlb_set_page(cs,
diff --git a/target-xtensa/overlay_tool.h b/target-xtensa/overlay_tool.h
index eda03aaca..e8a7fda3d 100644
--- a/target-xtensa/overlay_tool.h
+++ b/target-xtensa/overlay_tool.h
@@ -30,6 +30,10 @@
     { .targno = (no), .type = (typ), .group = (grp), .size = (sz) },
 #define XTREG_END { .targno = -1 },
 
+#ifndef XCHAL_HAVE_DEPBITS
+#define XCHAL_HAVE_DEPBITS 0
+#endif
+
 #ifndef XCHAL_HAVE_DIV32
 #define XCHAL_HAVE_DIV32 0
 #endif
@@ -69,6 +73,7 @@
     XCHAL_OPTION(XCHAL_HAVE_S32C1I, XTENSA_OPTION_CONDITIONAL_STORE) | \
     XCHAL_OPTION(XCHAL_HAVE_S32C1I && XCHAL_HW_MIN_VERSION >= 230000, \
         XTENSA_OPTION_ATOMCTL) | \
+    XCHAL_OPTION(XCHAL_HAVE_DEPBITS, XTENSA_OPTION_DEPBITS) | \
     /* Interrupts and exceptions */ \
     XCHAL_OPTION(XCHAL_HAVE_EXCEPTIONS, XTENSA_OPTION_EXCEPTION) | \
     XCHAL_OPTION(XCHAL_HAVE_VECBASE, XTENSA_OPTION_RELOCATABLE_VECTOR) | \
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 7a074e0e9..06b016341 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -1544,7 +1544,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                     TCGv_i64 tmp = tcg_temp_new_i64(); \
                     tcg_gen_extu_i32_i64(tmp, reg); \
                     tcg_gen_##cmd##_i64(v, v, tmp); \
-                    tcg_gen_trunc_i64_i32(cpu_R[RRR_R], v); \
+                    tcg_gen_extrl_i64_i32(cpu_R[RRR_R], v); \
                     tcg_temp_free_i64(v); \
                     tcg_temp_free_i64(tmp); \
                 } while (0)
@@ -1965,6 +1965,17 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 }
                 break;
 
+            case 5: /*S32N*/
+                if (gen_window_check2(dc, RRI4_S, RRI4_T)) {
+                    TCGv_i32 addr = tcg_temp_new_i32();
+
+                    tcg_gen_addi_i32(addr, cpu_R[RRI4_S], RRI4_IMM4 << 2);
+                    gen_load_store_alignment(dc, 2, addr, false);
+                    tcg_gen_qemu_st32(cpu_R[RRI4_T], addr, dc->cring);
+                    tcg_temp_free(addr);
+                }
+                break;
+
             default:
                 RESERVED();
                 break;
@@ -1972,6 +1983,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
             break;
 
         case 10: /*FP0*/
+            /*DEPBITS*/
+            if (option_enabled(dc, XTENSA_OPTION_DEPBITS)) {
+                if (!gen_window_check2(dc, RRR_S, RRR_T)) {
+                    break;
+                }
+                tcg_gen_deposit_i32(cpu_R[RRR_T], cpu_R[RRR_T], cpu_R[RRR_S],
+                                    OP2, RRR_R + 1);
+                break;
+            }
+
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
             switch (OP2) {
             case 0: /*ADD.Sf*/
@@ -2106,6 +2127,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
             break;
 
         case 11: /*FP1*/
+            /*DEPBITS*/
+            if (option_enabled(dc, XTENSA_OPTION_DEPBITS)) {
+                if (!gen_window_check2(dc, RRR_S, RRR_T)) {
+                    break;
+                }
+                tcg_gen_deposit_i32(cpu_R[RRR_T], cpu_R[RRR_T], cpu_R[RRR_S],
+                                    OP2 + 16, RRR_R + 1);
+                break;
+            }
+
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
 
 #define gen_compare(rel, br, a, b) \
@@ -2986,22 +3017,6 @@ static inline unsigned xtensa_insn_len(CPUXtensaState *env, DisasContext *dc)
     return xtensa_op0_insn_len(OP0);
 }
 
-static void check_breakpoint(CPUXtensaState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(xtensa_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_i32(cpu_pc, dc->pc);
-                gen_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 static void gen_ibreak_check(CPUXtensaState *env, DisasContext *dc)
 {
     unsigned i;
@@ -3015,15 +3030,12 @@ static void gen_ibreak_check(CPUXtensaState *env, DisasContext *dc)
     }
 }
 
-static inline
-void gen_intermediate_code_internal(XtensaCPU *cpu,
-                                    TranslationBlock *tb, bool search_pc)
+void gen_intermediate_code(CPUXtensaState *env, TranslationBlock *tb)
 {
+    XtensaCPU *cpu = xtensa_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUXtensaState *env = &cpu->env;
     DisasContext dc;
     int insn_count = 0;
-    int j, lj = -1;
     int max_insns = tb->cflags & CF_COUNT_MASK;
     uint32_t pc_start = tb->pc;
     uint32_t next_page_start =
@@ -3032,6 +3044,9 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     dc.config = env->config;
     dc.singlestep_enabled = cs->singlestep_enabled;
@@ -3064,28 +3079,24 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
     }
 
     do {
-        check_breakpoint(env, &dc);
+        tcg_gen_insn_start(dc.pc);
+        ++insn_count;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = insn_count;
-        }
+        ++dc.ccount_delta;
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc.pc);
+        if (unlikely(cpu_breakpoint_test(cs, dc.pc, BP_ANY))) {
+            tcg_gen_movi_i32(cpu_pc, dc.pc);
+            gen_exception(&dc, EXCP_DEBUG);
+            dc.is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc.pc += 2;
+            break;
         }
 
-        ++dc.ccount_delta;
-
-        if (insn_count + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (insn_count == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
@@ -3106,7 +3117,6 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
         }
 
         disas_xtensa_insn(env, &dc);
-        ++insn_count;
         if (dc.icount) {
             tcg_gen_mov_i32(cpu_SR[ICOUNT], dc.next_icount);
         }
@@ -3144,24 +3154,8 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        memset(tcg_ctx.gen_opc_instr_start + lj + 1, 0,
-                (j - lj) * sizeof(tcg_ctx.gen_opc_instr_start[0]));
-    } else {
-        tb->size = dc.pc - pc_start;
-        tb->icount = insn_count;
-    }
-}
-
-void gen_intermediate_code(CPUXtensaState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(xtensa_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUXtensaState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(xtensa_env_get_cpu(env), tb, true);
+    tb->size = dc.pc - pc_start;
+    tb->icount = insn_count;
 }
 
 void xtensa_cpu_dump_state(CPUState *cs, FILE *f,
@@ -3215,7 +3209,8 @@ void xtensa_cpu_dump_state(CPUState *cs, FILE *f,
     }
 }
 
-void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/tcg/README b/tcg/README
index a550ff176..34c0775cf 100644
--- a/tcg/README
+++ b/tcg/README
@@ -314,11 +314,17 @@ This operation would be equivalent to
 
   dest = (t1 & ~0x0f00) | ((t2 << 8) & 0x0f00)
 
-* trunc_shr_i32 t0, t1, pos
+* extrl_i64_i32 t0, t1
 
-For 64-bit hosts only, right shift the 64-bit input T1 by POS and
-truncate to 32-bit output T0.  Depending on the host, this may be
-a simple mov/shift, or may require additional canonicalization.
+For 64-bit hosts only, extract the low 32-bits of input T1 and place it
+into 32-bit output T0.  Depending on the host, this may be a simple move,
+or may require additional canonicalization.
+
+* extrh_i64_i32 t0, t1
+
+For 64-bit hosts only, extract the high 32-bits of input T1 and place it
+into 32-bit output T0.  Depending on the host, this may be a simple shift,
+or may require additional canonicalization.
 
 ********* Conditional moves
 
@@ -466,13 +472,25 @@ On a 32 bit target, all 64 bit operations are converted to 32 bits. A
 few specific operations must be implemented to allow it (see add2_i32,
 sub2_i32, brcond2_i32).
 
+On a 64 bit target, the values are transfered between 32 and 64-bit
+registers using the following ops:
+- trunc_shr_i64_i32
+- ext_i32_i64
+- extu_i32_i64
+
+They ensure that the values are correctly truncated or extended when
+moved from a 32-bit to a 64-bit register or vice-versa. Note that the
+trunc_shr_i64_i32 is an optional op. It is not necessary to implement
+it if all the following conditions are met:
+- 64-bit registers can hold 32-bit values
+- 32-bit values in a 64-bit register do not need to stay zero or
+  sign extended
+- all 32-bit TCG ops ignore the high part of 64-bit registers
+
 Floating point operations are not supported in this version. A
 previous incarnation of the code generator had full support of them,
 but it is better to concentrate on integer operations first.
 
-On a 64 bit target, no assumption is made in TCG about the storage of
-the 32 bit values in 64 bit registers.
-
 4.2) Constraints
 
 GCC like constraints are used to define the constraints of every
diff --git a/tcg/aarch64/tcg-target.c b/tcg/aarch64/tcg-target.c
index bad8b1de9..0ed10a974 100644
--- a/tcg/aarch64/tcg-target.c
+++ b/tcg/aarch64/tcg-target.c
@@ -30,7 +30,7 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
     TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
-    TCG_REG_X28, /* we will reserve this for GUEST_BASE if configured */
+    TCG_REG_X28, /* we will reserve this for guest_base if configured */
 
     TCG_REG_X8, TCG_REG_X9, TCG_REG_X10, TCG_REG_X11,
     TCG_REG_X12, TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
@@ -60,13 +60,8 @@ static const int tcg_target_call_oarg_regs[1] = {
    as that actaully encodes SP.  So if we need to zero-extend the guest
    address, via the address index register slot, we need to load even
    a zero guest base into a register.  */
-#define USE_GUEST_BASE     (GUEST_BASE != 0 || TARGET_LONG_BITS == 32)
-
-# ifdef CONFIG_USE_GUEST_BASE
-#  define TCG_REG_GUEST_BASE TCG_REG_X28
-# else
-#  define TCG_REG_GUEST_BASE TCG_REG_XZR
-# endif
+#define USE_GUEST_BASE     (guest_base != 0 || TARGET_LONG_BITS == 32)
+#define TCG_REG_GUEST_BASE TCG_REG_X28
 #endif
 
 static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
@@ -1057,14 +1052,29 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
    slow path for the failure case, which will be patched later when finalizing
    the slow path. Generated code returns the host addend in X1,
    clobbers X0,X2,X3,TMP. */
-static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp s_bits,
+static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc,
                              tcg_insn_unit **label_ptr, int mem_index,
                              bool is_read)
 {
-    TCGReg base = TCG_AREG0;
     int tlb_offset = is_read ?
         offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
         : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+    int s_mask = (1 << (opc & MO_SIZE)) - 1;
+    TCGReg base = TCG_AREG0, x3;
+    uint64_t tlb_mask;
+
+    /* For aligned accesses, we check the first byte and include the alignment
+       bits within the address.  For unaligned access, we check that we don't
+       cross pages using the address of the last byte of the access.  */
+    if ((opc & MO_AMASK) == MO_ALIGN || s_mask == 0) {
+        tlb_mask = TARGET_PAGE_MASK | s_mask;
+        x3 = addr_reg;
+    } else {
+        tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
+                     TCG_REG_X3, addr_reg, s_mask);
+        tlb_mask = TARGET_PAGE_MASK;
+        x3 = TCG_REG_X3;
+    }
 
     /* Extract the TLB index from the address into X0.
        X0<CPU_TLB_BITS:0> =
@@ -1072,11 +1082,9 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp s_bits,
     tcg_out_ubfm(s, TARGET_LONG_BITS == 64, TCG_REG_X0, addr_reg,
                  TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
 
-    /* Store the page mask part of the address and the low s_bits into X3.
-       Later this allows checking for equality and alignment at the same time.
-       X3 = addr_reg & (PAGE_MASK | ((1 << s_bits) - 1)) */
-    tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64, TCG_REG_X3,
-                     addr_reg, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
+    /* Store the page mask part of the address into X3.  */
+    tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
+                     TCG_REG_X3, x3, tlb_mask);
 
     /* Add any "high bits" from the tlb offset to the env address into X2,
        to take advantage of the LSL12 form of the ADDI instruction.
@@ -1213,10 +1221,9 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
     const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
 #ifdef CONFIG_SOFTMMU
     unsigned mem_index = get_mmuidx(oi);
-    TCGMemOp s_bits = memop & MO_SIZE;
     tcg_insn_unit *label_ptr;
 
-    tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 1);
+    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1);
     tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
                            TCG_REG_X1, otype, addr_reg);
     add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg,
@@ -1239,14 +1246,13 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
     const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
 #ifdef CONFIG_SOFTMMU
     unsigned mem_index = get_mmuidx(oi);
-    TCGMemOp s_bits = memop & MO_SIZE;
     tcg_insn_unit *label_ptr;
 
-    tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 0);
+    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0);
     tcg_out_qemu_st_direct(s, memop, data_reg,
                            TCG_REG_X1, otype, addr_reg);
-    add_qemu_ldst_label(s, false, oi, s_bits == MO_64, data_reg, addr_reg,
-                        s->code_ptr, label_ptr);
+    add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64,
+                        data_reg, addr_reg, s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
     if (USE_GUEST_BASE) {
         tcg_out_qemu_st_direct(s, memop, data_reg,
@@ -1570,6 +1576,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i32:
         tcg_out_sxt(s, ext, MO_16, a0, a1);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
         break;
@@ -1581,6 +1588,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
         tcg_out_uxt(s, MO_16, a0, a1);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
         break;
@@ -1726,6 +1734,8 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_ext8u_i64, { "r", "r" } },
     { INDEX_op_ext16u_i64, { "r", "r" } },
     { INDEX_op_ext32u_i64, { "r", "r" } },
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
 
     { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
     { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
@@ -1808,9 +1818,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
                   CPU_TEMP_BUF_NLONGS * sizeof(long));
 
-#if defined(CONFIG_USE_GUEST_BASE)
+#if !defined(CONFIG_SOFTMMU)
     if (USE_GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, GUEST_BASE);
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
     }
 #endif
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index 8aec04d2b..19a04a6e7 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -70,7 +70,8 @@ typedef enum {
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          1
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index ae2ec7a92..3edf6a6f9 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -1493,8 +1493,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
     add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
                         s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base);
         tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, TCG_REG_TMP);
     } else {
         tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo);
@@ -1623,8 +1623,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
     add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
                         s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base);
         tcg_out_qemu_st_index(s, COND_AL, opc, datalo,
                               datahi, addrlo, TCG_REG_TMP);
     } else {
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index 887f22f67..9187d34ca 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -1172,14 +1172,16 @@ static void * const qemu_st_helpers[16] = {
    First argument register is clobbered.  */
 
 static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
-                                    int mem_index, TCGMemOp s_bits,
+                                    int mem_index, TCGMemOp opc,
                                     tcg_insn_unit **label_ptr, int which)
 {
     const TCGReg r0 = TCG_REG_L0;
     const TCGReg r1 = TCG_REG_L1;
     TCGType ttype = TCG_TYPE_I32;
-    TCGType htype = TCG_TYPE_I32;
-    int trexw = 0, hrexw = 0;
+    TCGType tlbtype = TCG_TYPE_I32;
+    int trexw = 0, hrexw = 0, tlbrexw = 0;
+    int s_mask = (1 << (opc & MO_SIZE)) - 1;
+    bool aligned = (opc & MO_AMASK) == MO_ALIGN || s_mask == 0;
 
     if (TCG_TARGET_REG_BITS == 64) {
         if (TARGET_LONG_BITS == 64) {
@@ -1187,20 +1189,29 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
             trexw = P_REXW;
         }
         if (TCG_TYPE_PTR == TCG_TYPE_I64) {
-            htype = TCG_TYPE_I64;
             hrexw = P_REXW;
+            if (TARGET_PAGE_BITS + CPU_TLB_BITS > 32) {
+                tlbtype = TCG_TYPE_I64;
+                tlbrexw = P_REXW;
+            }
         }
     }
 
-    tcg_out_mov(s, htype, r0, addrlo);
-    tcg_out_mov(s, ttype, r1, addrlo);
+    tcg_out_mov(s, tlbtype, r0, addrlo);
+    if (aligned) {
+        tcg_out_mov(s, ttype, r1, addrlo);
+    } else {
+        /* For unaligned access check that we don't cross pages using
+           the page address of the last byte.  */
+        tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask);
+    }
 
-    tcg_out_shifti(s, SHIFT_SHR + hrexw, r0,
+    tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0,
                    TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
 
     tgen_arithi(s, ARITH_AND + trexw, r1,
-                TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
-    tgen_arithi(s, ARITH_AND + hrexw, r0,
+                TARGET_PAGE_MASK | (aligned ? s_mask : 0), 0);
+    tgen_arithi(s, ARITH_AND + tlbrexw, r0,
                 (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
 
     tcg_out_modrm_sib_offset(s, OPC_LEA + hrexw, r0, TCG_AREG0, r0, 0,
@@ -1424,7 +1435,7 @@ int arch_prctl(int code, unsigned long addr);
 static int guest_base_flags;
 static inline void setup_guest_base_seg(void)
 {
-    if (arch_prctl(ARCH_SET_GS, GUEST_BASE) == 0) {
+    if (arch_prctl(ARCH_SET_GS, guest_base) == 0) {
         guest_base_flags = P_GS;
     }
 }
@@ -1545,7 +1556,6 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     int mem_index;
-    TCGMemOp s_bits;
     tcg_insn_unit *label_ptr[2];
 #endif
 
@@ -1558,9 +1568,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
 
 #if defined(CONFIG_SOFTMMU)
     mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
 
-    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, s_bits,
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
                      label_ptr, offsetof(CPUTLBEntry, addr_read));
 
     /* TLB Hit.  */
@@ -1571,7 +1580,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
                         s->code_ptr, label_ptr);
 #else
     {
-        int32_t offset = GUEST_BASE;
+        int32_t offset = guest_base;
         TCGReg base = addrlo;
         int index = -1;
         int seg = 0;
@@ -1580,7 +1589,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
            We can do this with the ADDR32 prefix if we're not using
            a guest base, or when using segmentation.  Otherwise we
            need to zero-extend manually.  */
-        if (GUEST_BASE == 0 || guest_base_flags) {
+        if (guest_base == 0 || guest_base_flags) {
             seg = guest_base_flags;
             offset = 0;
             if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
@@ -1591,8 +1600,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
                 tcg_out_ext32u(s, TCG_REG_L0, base);
                 base = TCG_REG_L0;
             }
-            if (offset != GUEST_BASE) {
-                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+            if (offset != guest_base) {
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, guest_base);
                 index = TCG_REG_L1;
                 offset = 0;
             }
@@ -1687,7 +1696,6 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     int mem_index;
-    TCGMemOp s_bits;
     tcg_insn_unit *label_ptr[2];
 #endif
 
@@ -1700,9 +1708,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
 
 #if defined(CONFIG_SOFTMMU)
     mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
 
-    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, s_bits,
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
                      label_ptr, offsetof(CPUTLBEntry, addr_write));
 
     /* TLB Hit.  */
@@ -1713,12 +1720,12 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
                         s->code_ptr, label_ptr);
 #else
     {
-        int32_t offset = GUEST_BASE;
+        int32_t offset = guest_base;
         TCGReg base = addrlo;
         int seg = 0;
 
         /* See comment in tcg_out_qemu_ld re zero-extension of addrlo.  */
-        if (GUEST_BASE == 0 || guest_base_flags) {
+        if (guest_base == 0 || guest_base_flags) {
             seg = guest_base_flags;
             offset = 0;
             if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
@@ -1727,12 +1734,12 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
         } else if (TCG_TARGET_REG_BITS == 64) {
             /* ??? Note that we can't use the same SIB addressing scheme
                as for loads, since we require L0 free for bswap.  */
-            if (offset != GUEST_BASE) {
+            if (offset != guest_base) {
                 if (TARGET_LONG_BITS == 32) {
                     tcg_out_ext32u(s, TCG_REG_L0, base);
                     base = TCG_REG_L0;
                 }
-                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, guest_base);
                 tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
                 base = TCG_REG_L1;
                 offset = 0;
@@ -2064,9 +2071,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_bswap64_i64:
         tcg_out_bswap64(s, args[0]);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_ext32u(s, args[0], args[1]);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_ext32s(s, args[0], args[1]);
         break;
@@ -2201,6 +2210,9 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ext16u_i64, { "r", "r" } },
     { INDEX_op_ext32u_i64, { "r", "r" } },
 
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+
     { INDEX_op_deposit_i64, { "Q", "0", "Q" } },
     { INDEX_op_movcond_i64, { "r", "r", "re", "r", "0" } },
 
@@ -2306,8 +2318,8 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tcg_out_opc(s, OPC_RET, 0, 0, 0);
 
 #if !defined(CONFIG_SOFTMMU)
-    /* Try to set up a segment register to point to GUEST_BASE.  */
-    if (GUEST_BASE) {
+    /* Try to set up a segment register to point to guest_base.  */
+    if (guest_base) {
         setup_guest_base_seg();
     }
 #endif
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index 25b513354..92be34171 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -102,7 +102,8 @@ extern bool have_bmi1;
 #define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div2_i64         1
 #define TCG_TARGET_HAS_rot_i64          1
 #define TCG_TARGET_HAS_ext8s_i64        1
diff --git a/tcg/ia64/tcg-target.c b/tcg/ia64/tcg-target.c
index 81cb9f79f..647e9a6f2 100644
--- a/tcg/ia64/tcg-target.c
+++ b/tcg/ia64/tcg-target.c
@@ -40,13 +40,8 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 };
 #endif
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 #define TCG_GUEST_BASE_REG TCG_REG_R55
-#else
-#define TCG_GUEST_BASE_REG TCG_REG_R0
-#endif
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
 #endif
 
 /* Branch registers */
@@ -1599,7 +1594,7 @@ static void tcg_out_tb_finalize(TCGContext *s)
         /* The out-of-line thunks are all the same; load the return address
            from B0, load the GP, and branch to the code.  Note that we are
            always post-call, so the register window has rolled, so we're
-           using incomming parameter register numbers, not outgoing.  */
+           using incoming parameter register numbers, not outgoing.  */
         if (dest == NULL) {
             uintptr_t *desc = (uintptr_t *)helpers[x];
             uintptr_t func = desc[0], gp = desc[1], disp;
@@ -1765,7 +1760,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
     bswap = opc & MO_BSWAP;
 
 #if TARGET_LONG_BITS == 32
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, mII,
                        INSN_NOP_M,
                        tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
@@ -1829,7 +1824,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
         }
     }
 #else
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, MmI,
                        tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
                                    TCG_GUEST_BASE_REG, addr_reg),
@@ -1889,7 +1884,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
     bswap = opc & MO_BSWAP;
 
 #if TARGET_LONG_BITS == 32
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, mII,
                        INSN_NOP_M,
                        tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
@@ -1935,7 +1930,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
                    INSN_NOP_M,
                    INSN_NOP_I);
 #else
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         add_guest_base = tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
                                      TCG_GUEST_BASE_REG, addr_reg);
         addr_reg = TCG_REG_R2;
@@ -1944,7 +1939,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
     }
 
     if (!bswap) {
-        tcg_out_bundle(s, (GUEST_BASE ? MmI : mmI),
+        tcg_out_bundle(s, (guest_base ? MmI : mmI),
                        add_guest_base,
                        tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
                                    data_reg, addr_reg),
@@ -2148,9 +2143,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
         tcg_out_ext(s, OPC_ZXT2_I29, args[0], args[1]);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_ext(s, OPC_SXT4_I29, args[0], args[1]);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_ext(s, OPC_ZXT4_I29, args[0], args[1]);
         break;
@@ -2301,6 +2298,8 @@ static const TCGTargetOpDef ia64_op_defs[] = {
     { INDEX_op_ext16u_i64, { "r", "rZ"} },
     { INDEX_op_ext32s_i64, { "r", "rZ"} },
     { INDEX_op_ext32u_i64, { "r", "rZ"} },
+    { INDEX_op_ext_i32_i64, { "r", "rZ" } },
+    { INDEX_op_extu_i32_i64, { "r", "rZ" } },
 
     { INDEX_op_bswap16_i64, { "r", "rZ" } },
     { INDEX_op_bswap32_i64, { "r", "rZ" } },
@@ -2349,14 +2348,14 @@ static void tcg_target_qemu_prologue(TCGContext *s)
                    tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21,
                                TCG_REG_B6, TCG_REG_R33, 0));
 
-    /* ??? If GUEST_BASE < 0x200000, we could load the register via
+    /* ??? If guest_base < 0x200000, we could load the register via
        an ADDL in the M slot of the next bundle.  */
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, mlx,
                        INSN_NOP_M,
-                       tcg_opc_l2 (GUEST_BASE),
+                       tcg_opc_l2(guest_base),
                        tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2,
-                                   TCG_GUEST_BASE_REG, GUEST_BASE));
+                                   TCG_GUEST_BASE_REG, guest_base));
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
 
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index a04ed8126..ae9b79f02 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -160,7 +160,8 @@ typedef enum {
 #define TCG_TARGET_HAS_muluh_i64        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 #define TCG_TARGET_HAS_mulsh_i64        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
 #define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index 2ccd0e826..79e052ff7 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -288,16 +288,24 @@ typedef enum {
     OPC_SRLV     = OPC_SPECIAL | 0x06,
     OPC_ROTRV    = OPC_SPECIAL | (0x01 <<  6) | 0x06,
     OPC_SRAV     = OPC_SPECIAL | 0x07,
-    OPC_JR       = OPC_SPECIAL | 0x08,
+    OPC_JR_R5    = OPC_SPECIAL | 0x08,
     OPC_JALR     = OPC_SPECIAL | 0x09,
     OPC_MOVZ     = OPC_SPECIAL | 0x0A,
     OPC_MOVN     = OPC_SPECIAL | 0x0B,
     OPC_MFHI     = OPC_SPECIAL | 0x10,
     OPC_MFLO     = OPC_SPECIAL | 0x12,
     OPC_MULT     = OPC_SPECIAL | 0x18,
+    OPC_MUL_R6   = OPC_SPECIAL | (0x02 <<  6) | 0x18,
+    OPC_MUH      = OPC_SPECIAL | (0x03 <<  6) | 0x18,
     OPC_MULTU    = OPC_SPECIAL | 0x19,
+    OPC_MULU     = OPC_SPECIAL | (0x02 <<  6) | 0x19,
+    OPC_MUHU     = OPC_SPECIAL | (0x03 <<  6) | 0x19,
     OPC_DIV      = OPC_SPECIAL | 0x1A,
+    OPC_DIV_R6   = OPC_SPECIAL | (0x02 <<  6) | 0x1A,
+    OPC_MOD      = OPC_SPECIAL | (0x03 <<  6) | 0x1A,
     OPC_DIVU     = OPC_SPECIAL | 0x1B,
+    OPC_DIVU_R6  = OPC_SPECIAL | (0x02 <<  6) | 0x1B,
+    OPC_MODU     = OPC_SPECIAL | (0x03 <<  6) | 0x1B,
     OPC_ADDU     = OPC_SPECIAL | 0x21,
     OPC_SUBU     = OPC_SPECIAL | 0x23,
     OPC_AND      = OPC_SPECIAL | 0x24,
@@ -306,13 +314,15 @@ typedef enum {
     OPC_NOR      = OPC_SPECIAL | 0x27,
     OPC_SLT      = OPC_SPECIAL | 0x2A,
     OPC_SLTU     = OPC_SPECIAL | 0x2B,
+    OPC_SELEQZ   = OPC_SPECIAL | 0x35,
+    OPC_SELNEZ   = OPC_SPECIAL | 0x37,
 
     OPC_REGIMM   = 0x01 << 26,
     OPC_BLTZ     = OPC_REGIMM | (0x00 << 16),
     OPC_BGEZ     = OPC_REGIMM | (0x01 << 16),
 
     OPC_SPECIAL2 = 0x1c << 26,
-    OPC_MUL      = OPC_SPECIAL2 | 0x002,
+    OPC_MUL_R5   = OPC_SPECIAL2 | 0x002,
 
     OPC_SPECIAL3 = 0x1f << 26,
     OPC_EXT      = OPC_SPECIAL3 | 0x000,
@@ -320,6 +330,15 @@ typedef enum {
     OPC_WSBH     = OPC_SPECIAL3 | 0x0a0,
     OPC_SEB      = OPC_SPECIAL3 | 0x420,
     OPC_SEH      = OPC_SPECIAL3 | 0x620,
+
+    /* MIPS r6 doesn't have JR, JALR should be used instead */
+    OPC_JR       = use_mips32r6_instructions ? OPC_JALR : OPC_JR_R5,
+
+    /*
+     * MIPS r6 replaces MUL with an alternative encoding which is
+     * backwards-compatible at the assembly level.
+     */
+    OPC_MUL      = use_mips32r6_instructions ? OPC_MUL_R6 : OPC_MUL_R5,
 } MIPSInsn;
 
 /*
@@ -567,6 +586,55 @@ static inline void tcg_out_addi(TCGContext *s, TCGReg reg, TCGArg val)
     }
 }
 
+static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al,
+                            TCGReg ah, TCGArg bl, TCGArg bh, bool cbl,
+                            bool cbh, bool is_sub)
+{
+    TCGReg th = TCG_TMP1;
+
+    /* If we have a negative constant such that negating it would
+       make the high part zero, we can (usually) eliminate one insn.  */
+    if (cbl && cbh && bh == -1 && bl != 0) {
+        bl = -bl;
+        bh = 0;
+        is_sub = !is_sub;
+    }
+
+    /* By operating on the high part first, we get to use the final
+       carry operation to move back from the temporary.  */
+    if (!cbh) {
+        tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh);
+    } else if (bh != 0 || ah == rl) {
+        tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh));
+    } else {
+        th = ah;
+    }
+
+    /* Note that tcg optimization should eliminate the bl == 0 case.  */
+    if (is_sub) {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl);
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl);
+            tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl);
+        }
+        tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0);
+    } else {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl);
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl);
+        } else if (rl == al && rl == bl) {
+            tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, al, 31);
+            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl));
+        }
+        tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0);
+    }
+}
+
 /* Bit 0 set if inversion required; bit 1 set if swapping required.  */
 #define MIPS_CMP_INV  1
 #define MIPS_CMP_SWAP 2
@@ -792,13 +860,20 @@ static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah,
 }
 
 static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
-                            TCGReg c1, TCGReg c2, TCGReg v)
+                            TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2)
 {
-    MIPSInsn m_opc = OPC_MOVN;
+    bool eqz = false;
+
+    /* If one of the values is zero, put it last to match SEL*Z instructions */
+    if (use_mips32r6_instructions && v1 == 0) {
+        v1 = v2;
+        v2 = 0;
+        cond = tcg_invert_cond(cond);
+    }
 
     switch (cond) {
     case TCG_COND_EQ:
-        m_opc = OPC_MOVZ;
+        eqz = true;
         /* FALLTHRU */
     case TCG_COND_NE:
         if (c2 != 0) {
@@ -811,14 +886,32 @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
         /* Minimize code size by preferring a compare not requiring INV.  */
         if (mips_cmp_map[cond] & MIPS_CMP_INV) {
             cond = tcg_invert_cond(cond);
-            m_opc = OPC_MOVZ;
+            eqz = true;
         }
         tcg_out_setcond(s, cond, TCG_TMP0, c1, c2);
         c1 = TCG_TMP0;
         break;
     }
 
-    tcg_out_opc_reg(s, m_opc, ret, v, c1);
+    if (use_mips32r6_instructions) {
+        MIPSInsn m_opc_t = eqz ? OPC_SELEQZ : OPC_SELNEZ;
+        MIPSInsn m_opc_f = eqz ? OPC_SELNEZ : OPC_SELEQZ;
+
+        if (v2 != 0) {
+            tcg_out_opc_reg(s, m_opc_f, TCG_TMP1, v2, c1);
+        }
+        tcg_out_opc_reg(s, m_opc_t, ret, v1, c1);
+        if (v2 != 0) {
+            tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP1);
+        }
+    } else {
+        MIPSInsn m_opc = eqz ? OPC_MOVZ : OPC_MOVN;
+
+        tcg_out_opc_reg(s, m_opc, ret, v1, c1);
+
+        /* This should be guaranteed via constraints */
+        tcg_debug_assert(v2 == ret);
+    }
 }
 
 static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
@@ -934,9 +1027,11 @@ static int tcg_out_call_iarg_reg2(TCGContext *s, int i, TCGReg al, TCGReg ah)
 /* Perform the tlb comparison operation.  The complete host address is
    placed in BASE.  Clobbers AT, T0, A0.  */
 static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl,
-                             TCGReg addrh, int mem_index, TCGMemOp s_bits,
+                             TCGReg addrh, TCGMemOpIdx oi,
                              tcg_insn_unit *label_ptr[2], bool is_load)
 {
+    TCGMemOp s_bits = get_memop(oi) & MO_SIZE;
+    int mem_index = get_mmuidx(oi);
     int cmp_off
         = (is_load
            ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
@@ -1160,8 +1255,6 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     tcg_insn_unit *label_ptr[2];
-    int mem_index;
-    TCGMemOp s_bits;
 #endif
     /* Note that we've eliminated V0 from the output registers,
        so we won't overwrite the base register during loading.  */
@@ -1175,21 +1268,17 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
     opc = get_memop(oi);
 
 #if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
-
-    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
-                     s_bits, label_ptr, 1);
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 1);
     tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
     add_qemu_ldst_label(s, 1, oi, data_regl, data_regh, addr_regl, addr_regh,
                         s->code_ptr, label_ptr);
 #else
-    if (GUEST_BASE == 0 && data_regl != addr_regl) {
+    if (guest_base == 0 && data_regl != addr_regl) {
         base = addr_regl;
-    } else if (GUEST_BASE == (int16_t)GUEST_BASE) {
-        tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
+    } else if (guest_base == (int16_t)guest_base) {
+        tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, guest_base);
     } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
+        tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base);
         tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
     }
     tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
@@ -1237,55 +1326,6 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
     }
 }
 
-static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al,
-                            TCGReg ah, TCGArg bl, TCGArg bh, bool cbl,
-                            bool cbh, bool is_sub)
-{
-    TCGReg th = TCG_TMP1;
-
-    /* If we have a negative constant such that negating it would
-       make the high part zero, we can (usually) eliminate one insn.  */
-    if (cbl && cbh && bh == -1 && bl != 0) {
-        bl = -bl;
-        bh = 0;
-        is_sub = !is_sub;
-    }
-
-    /* By operating on the high part first, we get to use the final
-       carry operation to move back from the temporary.  */
-    if (!cbh) {
-        tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh);
-    } else if (bh != 0 || ah == rl) {
-        tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh));
-    } else {
-        th = ah;
-    }
-
-    /* Note that tcg optimization should eliminate the bl == 0 case.  */
-    if (is_sub) {
-        if (cbl) {
-            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl);
-            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl);
-            tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl);
-        }
-        tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0);
-    } else {
-        if (cbl) {
-            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl);
-            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl);
-        } else if (rl == al && rl == bl) {
-            tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, al, 31);
-            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
-        } else {
-            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
-            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl));
-        }
-        tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0);
-    }
-}
-
 static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
 {
     TCGReg addr_regl, addr_regh __attribute__((unused));
@@ -1294,8 +1334,6 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     tcg_insn_unit *label_ptr[2];
-    int mem_index;
-    TCGMemOp s_bits;
 #endif
 
     data_regl = *args++;
@@ -1306,26 +1344,22 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
     opc = get_memop(oi);
 
 #if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & 3;
-
     /* Note that we eliminated the helper's address argument,
        so we can reuse that for the base.  */
     base = (TARGET_LONG_BITS == 32 ? TCG_REG_A1 : TCG_REG_A2);
-    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
-                     s_bits, label_ptr, 0);
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 0);
     tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
     add_qemu_ldst_label(s, 0, oi, data_regl, data_regh, addr_regl, addr_regh,
                         s->code_ptr, label_ptr);
 #else
-    if (GUEST_BASE == 0) {
+    if (guest_base == 0) {
         base = addr_regl;
     } else {
         base = TCG_REG_A0;
-        if (GUEST_BASE == (int16_t)GUEST_BASE) {
-            tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
+        if (guest_base == (int16_t)guest_base) {
+            tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, guest_base);
         } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
+            tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base);
             tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
         }
     }
@@ -1455,21 +1489,45 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         i1 = OPC_MULT, i2 = OPC_MFLO;
         goto do_hilo1;
     case INDEX_op_mulsh_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MUH, a0, a1, a2);
+            break;
+        }
         i1 = OPC_MULT, i2 = OPC_MFHI;
         goto do_hilo1;
     case INDEX_op_muluh_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MUHU, a0, a1, a2);
+            break;
+        }
         i1 = OPC_MULTU, i2 = OPC_MFHI;
         goto do_hilo1;
     case INDEX_op_div_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_DIV_R6, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIV, i2 = OPC_MFLO;
         goto do_hilo1;
     case INDEX_op_divu_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_DIVU_R6, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIVU, i2 = OPC_MFLO;
         goto do_hilo1;
     case INDEX_op_rem_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MOD, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIV, i2 = OPC_MFHI;
         goto do_hilo1;
     case INDEX_op_remu_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MODU, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIVU, i2 = OPC_MFHI;
     do_hilo1:
         tcg_out_opc_reg(s, i1, 0, a1, a2);
@@ -1546,7 +1604,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_movcond_i32:
-        tcg_out_movcond(s, args[5], a0, a1, a2, args[3]);
+        tcg_out_movcond(s, args[5], a0, a1, a2, args[3], args[4]);
         break;
 
     case INDEX_op_setcond_i32:
@@ -1602,8 +1660,10 @@ static const TCGTargetOpDef mips_op_defs[] = {
 
     { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
     { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
+#if !use_mips32r6_instructions
     { INDEX_op_muls2_i32, { "r", "r", "rZ", "rZ" } },
     { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rZ" } },
+#endif
     { INDEX_op_mulsh_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_muluh_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_div_i32, { "r", "rZ", "rZ" } },
@@ -1633,7 +1693,11 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
 
     { INDEX_op_brcond_i32, { "rZ", "rZ" } },
+#if use_mips32r6_instructions
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
+#else
     { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "0" } },
+#endif
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
 
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index f5ba52cac..b1cda37b6 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -96,6 +96,13 @@ extern bool use_mips32_instructions;
 extern bool use_mips32r2_instructions;
 #endif
 
+/* MIPS32R6 instruction set detection */
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 6)
+#define use_mips32r6_instructions  1
+#else
+#define use_mips32r6_instructions  0
+#endif
+
 /* optional instructions */
 #define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_rem_i32          1
@@ -105,8 +112,8 @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
-#define TCG_TARGET_HAS_mulu2_i32        1
-#define TCG_TARGET_HAS_muls2_i32        1
+#define TCG_TARGET_HAS_mulu2_i32        (!use_mips32r6_instructions)
+#define TCG_TARGET_HAS_muls2_i32        (!use_mips32r6_instructions)
 #define TCG_TARGET_HAS_muluh_i32        1
 #define TCG_TARGET_HAS_mulsh_i32        1
 
diff --git a/tcg/optimize.c b/tcg/optimize.c
index 18283cfd7..10795ec9d 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -35,14 +35,8 @@
         glue(glue(case INDEX_op_, x), _i32):    \
         glue(glue(case INDEX_op_, x), _i64)
 
-typedef enum {
-    TCG_TEMP_UNDEF = 0,
-    TCG_TEMP_CONST,
-    TCG_TEMP_COPY,
-} tcg_temp_state;
-
 struct tcg_temp_info {
-    tcg_temp_state state;
+    bool is_const;
     uint16_t prev_copy;
     uint16_t next_copy;
     tcg_target_ulong val;
@@ -50,23 +44,47 @@ struct tcg_temp_info {
 };
 
 static struct tcg_temp_info temps[TCG_MAX_TEMPS];
+static TCGTempSet temps_used;
+
+static inline bool temp_is_const(TCGArg arg)
+{
+    return temps[arg].is_const;
+}
+
+static inline bool temp_is_copy(TCGArg arg)
+{
+    return temps[arg].next_copy != arg;
+}
 
-/* Reset TEMP's state to TCG_TEMP_UNDEF.  If TEMP only had one copy, remove
-   the copy flag from the left temp.  */
+/* Reset TEMP's state, possibly removing the temp for the list of copies.  */
 static void reset_temp(TCGArg temp)
 {
-    if (temps[temp].state == TCG_TEMP_COPY) {
-        if (temps[temp].prev_copy == temps[temp].next_copy) {
-            temps[temps[temp].next_copy].state = TCG_TEMP_UNDEF;
-        } else {
-            temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-            temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-        }
-    }
-    temps[temp].state = TCG_TEMP_UNDEF;
+    temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
+    temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
+    temps[temp].next_copy = temp;
+    temps[temp].prev_copy = temp;
+    temps[temp].is_const = false;
     temps[temp].mask = -1;
 }
 
+/* Reset all temporaries, given that there are NB_TEMPS of them.  */
+static void reset_all_temps(int nb_temps)
+{
+    bitmap_zero(temps_used.l, nb_temps);
+}
+
+/* Initialize and activate a temporary.  */
+static void init_temp_info(TCGArg temp)
+{
+    if (!test_bit(temp, temps_used.l)) {
+        temps[temp].next_copy = temp;
+        temps[temp].prev_copy = temp;
+        temps[temp].is_const = false;
+        temps[temp].mask = -1;
+        set_bit(temp, temps_used.l);
+    }
+}
+
 static TCGOp *insert_op_before(TCGContext *s, TCGOp *old_op,
                                 TCGOpcode opc, int nargs)
 {
@@ -98,16 +116,6 @@ static TCGOp *insert_op_before(TCGContext *s, TCGOp *old_op,
     return new_op;
 }
 
-/* Reset all temporaries, given that there are NB_TEMPS of them.  */
-static void reset_all_temps(int nb_temps)
-{
-    int i;
-    for (i = 0; i < nb_temps; i++) {
-        temps[i].state = TCG_TEMP_UNDEF;
-        temps[i].mask = -1;
-    }
-}
-
 static int op_bits(TCGOpcode op)
 {
     const TCGOpDef *def = &tcg_op_defs[op];
@@ -179,8 +187,7 @@ static bool temps_are_copies(TCGArg arg1, TCGArg arg2)
         return true;
     }
 
-    if (temps[arg1].state != TCG_TEMP_COPY
-        || temps[arg2].state != TCG_TEMP_COPY) {
+    if (!temp_is_copy(arg1) || !temp_is_copy(arg2)) {
         return false;
     }
 
@@ -202,7 +209,7 @@ static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg *args,
     op->opc = new_op;
 
     reset_temp(dst);
-    temps[dst].state = TCG_TEMP_CONST;
+    temps[dst].is_const = true;
     temps[dst].val = val;
     mask = val;
     if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_movi_i32) {
@@ -223,11 +230,6 @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg *args,
         return;
     }
 
-    if (temps[src].state == TCG_TEMP_CONST) {
-        tcg_opt_gen_movi(s, op, args, dst, temps[src].val);
-        return;
-    }
-
     TCGOpcode new_op = op_to_mov(op->opc);
     tcg_target_ulong mask;
 
@@ -241,19 +243,13 @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg *args,
     }
     temps[dst].mask = mask;
 
-    assert(temps[src].state != TCG_TEMP_CONST);
-
     if (s->temps[src].type == s->temps[dst].type) {
-        if (temps[src].state != TCG_TEMP_COPY) {
-            temps[src].state = TCG_TEMP_COPY;
-            temps[src].next_copy = src;
-            temps[src].prev_copy = src;
-        }
-        temps[dst].state = TCG_TEMP_COPY;
         temps[dst].next_copy = temps[src].next_copy;
         temps[dst].prev_copy = src;
         temps[temps[dst].next_copy].prev_copy = dst;
         temps[src].next_copy = dst;
+        temps[dst].is_const = temps[src].is_const;
+        temps[dst].val = temps[src].val;
     }
 
     args[0] = dst;
@@ -292,7 +288,6 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     case INDEX_op_shr_i32:
         return (uint32_t)x >> (y & 31);
 
-    case INDEX_op_trunc_shr_i32:
     case INDEX_op_shr_i64:
         return (uint64_t)x >> (y & 63);
 
@@ -347,12 +342,18 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     CASE_OP_32_64(ext16u):
         return (uint16_t)x;
 
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         return (int32_t)x;
 
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     case INDEX_op_ext32u_i64:
         return (uint32_t)x;
 
+    case INDEX_op_extrh_i64_i32:
+        return (uint64_t)x >> 32;
+
     case INDEX_op_muluh_i32:
         return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
     case INDEX_op_mulsh_i32:
@@ -395,7 +396,7 @@ static TCGArg do_constant_folding(TCGOpcode op, TCGArg x, TCGArg y)
 {
     TCGArg res = do_constant_folding_2(op, x, y);
     if (op_bits(op) == 32) {
-        res &= 0xffffffff;
+        res = (int32_t)res;
     }
     return res;
 }
@@ -481,7 +482,7 @@ static bool do_constant_folding_cond_eq(TCGCond c)
 static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
                                        TCGArg y, TCGCond c)
 {
-    if (temps[x].state == TCG_TEMP_CONST && temps[y].state == TCG_TEMP_CONST) {
+    if (temp_is_const(x) && temp_is_const(y)) {
         switch (op_bits(op)) {
         case 32:
             return do_constant_folding_cond_32(temps[x].val, temps[y].val, c);
@@ -492,7 +493,7 @@ static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
         }
     } else if (temps_are_copies(x, y)) {
         return do_constant_folding_cond_eq(c);
-    } else if (temps[y].state == TCG_TEMP_CONST && temps[y].val == 0) {
+    } else if (temp_is_const(y) && temps[y].val == 0) {
         switch (c) {
         case TCG_COND_LTU:
             return 0;
@@ -513,12 +514,10 @@ static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c)
     TCGArg al = p1[0], ah = p1[1];
     TCGArg bl = p2[0], bh = p2[1];
 
-    if (temps[bl].state == TCG_TEMP_CONST
-        && temps[bh].state == TCG_TEMP_CONST) {
+    if (temp_is_const(bl) && temp_is_const(bh)) {
         uint64_t b = ((uint64_t)temps[bh].val << 32) | (uint32_t)temps[bl].val;
 
-        if (temps[al].state == TCG_TEMP_CONST
-            && temps[ah].state == TCG_TEMP_CONST) {
+        if (temp_is_const(al) && temp_is_const(ah)) {
             uint64_t a;
             a = ((uint64_t)temps[ah].val << 32) | (uint32_t)temps[al].val;
             return do_constant_folding_cond_64(a, b, c);
@@ -544,8 +543,8 @@ static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
 {
     TCGArg a1 = *p1, a2 = *p2;
     int sum = 0;
-    sum += temps[a1].state == TCG_TEMP_CONST;
-    sum -= temps[a2].state == TCG_TEMP_CONST;
+    sum += temp_is_const(a1);
+    sum -= temp_is_const(a2);
 
     /* Prefer the constant in second argument, and then the form
        op a, a, b, which is better handled on non-RISC hosts. */
@@ -560,10 +559,10 @@ static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
 static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
 {
     int sum = 0;
-    sum += temps[p1[0]].state == TCG_TEMP_CONST;
-    sum += temps[p1[1]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[0]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[1]].state == TCG_TEMP_CONST;
+    sum += temp_is_const(p1[0]);
+    sum += temp_is_const(p1[1]);
+    sum -= temp_is_const(p2[0]);
+    sum -= temp_is_const(p2[1]);
     if (sum > 0) {
         TCGArg t;
         t = p1[0], p1[0] = p2[0], p2[0] = t;
@@ -598,17 +597,29 @@ void tcg_optimize(TCGContext *s)
         const TCGOpDef *def = &tcg_op_defs[opc];
 
         oi_next = op->next;
+
+        /* Count the arguments, and initialize the temps that are
+           going to be used */
         if (opc == INDEX_op_call) {
             nb_oargs = op->callo;
             nb_iargs = op->calli;
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                tmp = args[i];
+                if (tmp != TCG_CALL_DUMMY_ARG) {
+                    init_temp_info(tmp);
+                }
+            }
         } else {
             nb_oargs = def->nb_oargs;
             nb_iargs = def->nb_iargs;
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                init_temp_info(args[i]);
+            }
         }
 
         /* Do copy propagation */
         for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
-            if (temps[args[i]].state == TCG_TEMP_COPY) {
+            if (temp_is_copy(args[i])) {
                 args[i] = find_better_copy(s, args[i]);
             }
         }
@@ -678,8 +689,7 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(sar):
         CASE_OP_32_64(rotl):
         CASE_OP_32_64(rotr):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
+            if (temp_is_const(args[1]) && temps[args[1]].val == 0) {
                 tcg_opt_gen_movi(s, op, args, args[0], 0);
                 continue;
             }
@@ -689,7 +699,7 @@ void tcg_optimize(TCGContext *s)
                 TCGOpcode neg_op;
                 bool have_neg;
 
-                if (temps[args[2]].state == TCG_TEMP_CONST) {
+                if (temp_is_const(args[2])) {
                     /* Proceed with possible constant folding. */
                     break;
                 }
@@ -703,8 +713,7 @@ void tcg_optimize(TCGContext *s)
                 if (!have_neg) {
                     break;
                 }
-                if (temps[args[1]].state == TCG_TEMP_CONST
-                    && temps[args[1]].val == 0) {
+                if (temp_is_const(args[1]) && temps[args[1]].val == 0) {
                     op->opc = neg_op;
                     reset_temp(args[0]);
                     args[1] = args[2];
@@ -714,34 +723,30 @@ void tcg_optimize(TCGContext *s)
             break;
         CASE_OP_32_64(xor):
         CASE_OP_32_64(nand):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == -1) {
                 i = 1;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(nor):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == 0) {
                 i = 1;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(andc):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == -1) {
+            if (!temp_is_const(args[2])
+                && temp_is_const(args[1]) && temps[args[1]].val == -1) {
                 i = 2;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
+            if (!temp_is_const(args[2])
+                && temp_is_const(args[1]) && temps[args[1]].val == 0) {
                 i = 2;
                 goto try_not;
             }
@@ -782,9 +787,8 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(or):
         CASE_OP_32_64(xor):
         CASE_OP_32_64(andc):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == 0) {
                 tcg_opt_gen_mov(s, op, args, args[0], args[1]);
                 continue;
             }
@@ -792,9 +796,8 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(and):
         CASE_OP_32_64(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == -1) {
                 tcg_opt_gen_mov(s, op, args, args[0], args[1]);
                 continue;
             }
@@ -832,17 +835,26 @@ void tcg_optimize(TCGContext *s)
 
         CASE_OP_32_64(and):
             mask = temps[args[2]].mask;
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
         and_const:
                 affected = temps[args[1]].mask & ~mask;
             }
             mask = temps[args[1]].mask & mask;
             break;
 
+        case INDEX_op_ext_i32_i64:
+            if ((temps[args[1]].mask & 0x80000000) != 0) {
+                break;
+            }
+        case INDEX_op_extu_i32_i64:
+            /* We do not compute affected as it is a size changing op.  */
+            mask = (uint32_t)temps[args[1]].mask;
+            break;
+
         CASE_OP_32_64(andc):
             /* Known-zeros does not imply known-ones.  Therefore unless
                args[2] is constant, we can't infer anything from it.  */
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 mask = ~temps[args[2]].mask;
                 goto and_const;
             }
@@ -851,37 +863,40 @@ void tcg_optimize(TCGContext *s)
             break;
 
         case INDEX_op_sar_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 31;
                 mask = (int32_t)temps[args[1]].mask >> tmp;
             }
             break;
         case INDEX_op_sar_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 63;
                 mask = (int64_t)temps[args[1]].mask >> tmp;
             }
             break;
 
         case INDEX_op_shr_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 31;
                 mask = (uint32_t)temps[args[1]].mask >> tmp;
             }
             break;
         case INDEX_op_shr_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 63;
                 mask = (uint64_t)temps[args[1]].mask >> tmp;
             }
             break;
 
-        case INDEX_op_trunc_shr_i32:
-            mask = (uint64_t)temps[args[1]].mask >> args[2];
+        case INDEX_op_extrl_i64_i32:
+            mask = (uint32_t)temps[args[1]].mask;
+            break;
+        case INDEX_op_extrh_i64_i32:
+            mask = (uint64_t)temps[args[1]].mask >> 32;
             break;
 
         CASE_OP_32_64(shl):
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & (TCG_TARGET_REG_BITS - 1);
                 mask = temps[args[1]].mask << tmp;
             }
@@ -962,8 +977,7 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(mul):
         CASE_OP_32_64(muluh):
         CASE_OP_32_64(mulsh):
-            if ((temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0)) {
+            if ((temp_is_const(args[2]) && temps[args[2]].val == 0)) {
                 tcg_opt_gen_movi(s, op, args, args[0], 0);
                 continue;
             }
@@ -1018,21 +1032,17 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(ext16u):
         case INDEX_op_ext32s_i64:
         case INDEX_op_ext32u_i64:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
+        case INDEX_op_ext_i32_i64:
+        case INDEX_op_extu_i32_i64:
+        case INDEX_op_extrl_i64_i32:
+        case INDEX_op_extrh_i64_i32:
+            if (temp_is_const(args[1])) {
                 tmp = do_constant_folding(opc, temps[args[1]].val, 0);
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
                 break;
             }
             goto do_default;
 
-        case INDEX_op_trunc_shr_i32:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
-                tmp = do_constant_folding(opc, temps[args[1]].val, args[2]);
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
-                break;
-            }
-            goto do_default;
-
         CASE_OP_32_64(add):
         CASE_OP_32_64(sub):
         CASE_OP_32_64(mul):
@@ -1055,8 +1065,7 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(divu):
         CASE_OP_32_64(rem):
         CASE_OP_32_64(remu):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[1]) && temp_is_const(args[2])) {
                 tmp = do_constant_folding(opc, temps[args[1]].val,
                                           temps[args[2]].val);
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
@@ -1065,8 +1074,7 @@ void tcg_optimize(TCGContext *s)
             goto do_default;
 
         CASE_OP_32_64(deposit):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[1]) && temp_is_const(args[2])) {
                 tmp = deposit64(temps[args[1]].val, args[3], args[4],
                                 temps[args[2]].val);
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
@@ -1106,10 +1114,8 @@ void tcg_optimize(TCGContext *s)
 
         case INDEX_op_add2_i32:
         case INDEX_op_sub2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST
-                && temps[args[4]].state == TCG_TEMP_CONST
-                && temps[args[5]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2]) && temp_is_const(args[3])
+                && temp_is_const(args[4]) && temp_is_const(args[5])) {
                 uint32_t al = temps[args[2]].val;
                 uint32_t ah = temps[args[3]].val;
                 uint32_t bl = temps[args[4]].val;
@@ -1128,8 +1134,8 @@ void tcg_optimize(TCGContext *s)
 
                 rl = args[0];
                 rh = args[1];
-                tcg_opt_gen_movi(s, op, args, rl, (uint32_t)a);
-                tcg_opt_gen_movi(s, op2, args2, rh, (uint32_t)(a >> 32));
+                tcg_opt_gen_movi(s, op, args, rl, (int32_t)a);
+                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(a >> 32));
 
                 /* We've done all we need to do with the movi.  Skip it.  */
                 oi_next = op2->next;
@@ -1138,8 +1144,7 @@ void tcg_optimize(TCGContext *s)
             goto do_default;
 
         case INDEX_op_mulu2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2]) && temp_is_const(args[3])) {
                 uint32_t a = temps[args[2]].val;
                 uint32_t b = temps[args[3]].val;
                 uint64_t r = (uint64_t)a * b;
@@ -1149,8 +1154,8 @@ void tcg_optimize(TCGContext *s)
 
                 rl = args[0];
                 rh = args[1];
-                tcg_opt_gen_movi(s, op, args, rl, (uint32_t)r);
-                tcg_opt_gen_movi(s, op2, args2, rh, (uint32_t)(r >> 32));
+                tcg_opt_gen_movi(s, op, args, rl, (int32_t)r);
+                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(r >> 32));
 
                 /* We've done all we need to do with the movi.  Skip it.  */
                 oi_next = op2->next;
@@ -1171,10 +1176,8 @@ void tcg_optimize(TCGContext *s)
                     tcg_op_remove(s, op);
                 }
             } else if ((args[4] == TCG_COND_LT || args[4] == TCG_COND_GE)
-                       && temps[args[2]].state == TCG_TEMP_CONST
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[2]].val == 0
-                       && temps[args[3]].val == 0) {
+                       && temp_is_const(args[2]) && temps[args[2]].val == 0
+                       && temp_is_const(args[3]) && temps[args[3]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_brcond_high:
@@ -1236,10 +1239,8 @@ void tcg_optimize(TCGContext *s)
             do_setcond_const:
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
             } else if ((args[5] == TCG_COND_LT || args[5] == TCG_COND_GE)
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[4]].state == TCG_TEMP_CONST
-                       && temps[args[3]].val == 0
-                       && temps[args[4]].val == 0) {
+                       && temp_is_const(args[3]) && temps[args[3]].val == 0
+                       && temp_is_const(args[4]) && temps[args[4]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_setcond_high:
@@ -1299,7 +1300,9 @@ void tcg_optimize(TCGContext *s)
             if (!(args[nb_oargs + nb_iargs + 1]
                   & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
                 for (i = 0; i < nb_globals; i++) {
-                    reset_temp(i);
+                    if (test_bit(i, temps_used.l)) {
+                        reset_temp(i);
+                    }
                 }
             }
             goto do_reset_output;
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
index 2b6eafa03..2c72565fb 100644
--- a/tcg/ppc/tcg-target.c
+++ b/tcg/ppc/tcg-target.c
@@ -80,19 +80,13 @@
 
 static tcg_insn_unit *tb_ret_addr;
 
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
-#endif
-
 #include "elf.h"
 static bool have_isa_2_06;
 #define HAVE_ISA_2_06  have_isa_2_06
 #define HAVE_ISEL      have_isa_2_06
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 #define TCG_GUEST_BASE_REG 30
-#else
-#define TCG_GUEST_BASE_REG 0
 #endif
 
 #ifndef NDEBUG
@@ -706,14 +700,14 @@ static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
 {
     int mb, me;
 
-    if ((c & 0xffff) == c) {
+    if (mask_operand(c, &mb, &me)) {
+        tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
+    } else if ((c & 0xffff) == c) {
         tcg_out32(s, ANDI | SAI(src, dst, c));
         return;
     } else if ((c & 0xffff0000) == c) {
         tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
         return;
-    } else if (mask_operand(c, &mb, &me)) {
-        tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
     } else {
         tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
         tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
@@ -725,18 +719,18 @@ static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
     int mb, me;
 
     assert(TCG_TARGET_REG_BITS == 64);
-    if ((c & 0xffff) == c) {
-        tcg_out32(s, ANDI | SAI(src, dst, c));
-        return;
-    } else if ((c & 0xffff0000) == c) {
-        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
-        return;
-    } else if (mask64_operand(c, &mb, &me)) {
+    if (mask64_operand(c, &mb, &me)) {
         if (mb == 0) {
             tcg_out_rld(s, RLDICR, dst, src, 0, me);
         } else {
             tcg_out_rld(s, RLDICL, dst, src, 0, mb);
         }
+    } else if ((c & 0xffff) == c) {
+        tcg_out32(s, ANDI | SAI(src, dst, c));
+        return;
+    } else if ((c & 0xffff0000) == c) {
+        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
+        return;
     } else {
         tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
         tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
@@ -1245,11 +1239,36 @@ static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
 
 void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
 {
-    TCGContext s;
+    tcg_insn_unit i1, i2;
+    uint64_t pair;
+    intptr_t diff = addr - jmp_addr;
+
+    if (in_range_b(diff)) {
+        i1 = B | (diff & 0x3fffffc);
+        i2 = NOP;
+    } else if (USE_REG_RA) {
+        intptr_t lo, hi;
+        diff = addr - (uintptr_t)tb_ret_addr;
+        lo = (int16_t)diff;
+        hi = (int32_t)(diff - lo);
+        assert(diff == hi + lo);
+        i1 = ADDIS | TAI(TCG_REG_TMP1, TCG_REG_RA, hi >> 16);
+        i2 = ADDI | TAI(TCG_REG_TMP1, TCG_REG_TMP1, lo);
+    } else {
+        assert(TCG_TARGET_REG_BITS == 32 || addr == (int32_t)addr);
+        i1 = ADDIS | TAI(TCG_REG_TMP1, 0, addr >> 16);
+        i2 = ORI | SAI(TCG_REG_TMP1, TCG_REG_TMP1, addr);
+    }
+#ifdef HOST_WORDS_BIGENDIAN
+    pair = (uint64_t)i1 << 32 | i2;
+#else
+    pair = (uint64_t)i2 << 32 | i1;
+#endif
 
-    s.code_buf = s.code_ptr = (tcg_insn_unit *)jmp_addr;
-    tcg_out_b(&s, 0, (tcg_insn_unit *)addr);
-    flush_icache_range(jmp_addr, jmp_addr + tcg_current_code_size(&s));
+    /* ??? __atomic_store_8, presuming there's some way to do that
+       for 32-bit, otherwise this is good enough for 64-bit.  */
+    *(uint64_t *)jmp_addr = pair;
+    flush_icache_range(jmp_addr, jmp_addr + 8);
 }
 
 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
@@ -1361,7 +1380,7 @@ static void * const qemu_st_helpers[16] = {
    in CR7, loads the addend of the TLB into R3, and returns the register
    containing the guest address (zero-extended into R4).  Clobbers R0 and R2. */
 
-static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp s_bits,
+static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc,
                                TCGReg addrlo, TCGReg addrhi,
                                int mem_index, bool is_read)
 {
@@ -1371,6 +1390,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp s_bits,
            : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
     int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
     TCGReg base = TCG_AREG0;
+    TCGMemOp s_bits = opc & MO_SIZE;
 
     /* Extract the page index, shifted into place for tlb index.  */
     if (TCG_TARGET_REG_BITS == 64) {
@@ -1422,17 +1442,37 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp s_bits,
        to minimize any load use delay.  */
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3, add_off);
 
-    /* Clear the non-page, non-alignment bits from the address.  */
+    /* Clear the non-page, non-alignment bits from the address */
     if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
+        /* We don't support unaligned accesses on 32-bits, preserve
+         * the bottom bits and thus trigger a comparison failure on
+         * unaligned accesses
+         */
         tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,
                     (32 - s_bits) & 31, 31 - TARGET_PAGE_BITS);
-    } else if (!s_bits) {
-        tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo,
-                    0, 63 - TARGET_PAGE_BITS);
+    } else if (s_bits) {
+        /* > byte access, we need to handle alignment */
+        if ((opc & MO_AMASK) == MO_ALIGN) {
+            /* Alignment required by the front-end, same as 32-bits */
+            tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,
+                        64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - s_bits);
+            tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
+       } else {
+           /* We support unaligned accesses, we need to make sure we fail
+            * if we cross a page boundary. The trick is to add the
+            * access_size-1 to the address before masking the low bits.
+            * That will make the address overflow to the next page if we
+            * cross a page boundary which will then force a mismatch of
+            * the TLB compare since the next page cannot possibly be in
+            * the same TLB index.
+            */
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, (1 << s_bits) - 1));
+            tcg_out_rld(s, RLDICR, TCG_REG_R0, TCG_REG_R0,
+                        0, 63 - TARGET_PAGE_BITS);
+        }
     } else {
-        tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,
-                    64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - s_bits);
-        tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
+        /* Byte access, just chop off the bits below the page index */
+        tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo, 0, 63 - TARGET_PAGE_BITS);
     }
 
     if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
@@ -1592,7 +1632,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
 
 #ifdef CONFIG_SOFTMMU
     mem_index = get_mmuidx(oi);
-    addrlo = tcg_out_tlb_read(s, s_bits, addrlo, addrhi, mem_index, true);
+    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true);
 
     /* Load a pointer into the current opcode w/conditional branch-link. */
     label_ptr = s->code_ptr;
@@ -1600,7 +1640,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
 
     rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+    rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
     if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
         tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
         addrlo = TCG_REG_TMP1;
@@ -1667,7 +1707,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
 
 #ifdef CONFIG_SOFTMMU
     mem_index = get_mmuidx(oi);
-    addrlo = tcg_out_tlb_read(s, s_bits, addrlo, addrhi, mem_index, false);
+    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false);
 
     /* Load a pointer into the current opcode w/conditional branch-link. */
     label_ptr = s->code_ptr;
@@ -1675,7 +1715,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
 
     rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+    rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
     if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
         tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
         addrlo = TCG_REG_TMP1;
@@ -1779,9 +1819,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     }
     tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
 
-#ifdef CONFIG_USE_GUEST_BASE
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+#ifndef CONFIG_SOFTMMU
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
 #endif
@@ -1840,12 +1880,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         if (USE_REG_RA) {
             ptrdiff_t disp = tcg_pcrel_diff(s, tb_ret_addr);
 
-            /* If we can use a direct branch, otherwise use the value in RA.
-               Note that the direct branch is always forward.  If it's in
-               range now, it'll still be in range after the movi.  Don't
-               bother about the 20 bytes where the test here fails but it
-               would succeed below.  */
-            if (!in_range_b(disp)) {
+            /* Use a direct branch if we can, otherwise use the value in RA.
+               Note that the direct branch is always backward, thus we need
+               to account for the possibility of 5 insns from the movi.  */
+            if (!in_range_b(disp - 20)) {
                 tcg_out32(s, MTSPR | RS(TCG_REG_RA) | CTR);
                 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
                 tcg_out32(s, BCCTR | BO_ALWAYS);
@@ -1856,14 +1894,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_b(s, 0, tb_ret_addr);
         break;
     case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* Direct jump method.  */
-            s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
-            s->code_ptr += 7;
-        } else {
-            /* Indirect jump method.  */
-            tcg_abort();
+        tcg_debug_assert(s->tb_jmp_offset);
+        /* Direct jump.  Ensure the next insns are 8-byte aligned. */
+        if ((uintptr_t)s->code_ptr & 7) {
+            tcg_out32(s, NOP);
         }
+        s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
+        /* To be replaced by either a branch+nop or a load into TMP1.  */
+        s->code_ptr += 2;
+        tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
+        tcg_out32(s, BCCTR | BO_ALWAYS);
         s->tb_next_offset[args[0]] = tcg_current_code_size(s);
         break;
     case INDEX_op_br:
@@ -2200,12 +2240,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_ext16s_i64:
         c = EXTSH;
         goto gen_ext;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         c = EXTSW;
         goto gen_ext;
     gen_ext:
         tcg_out32(s, c | RS(args[1]) | RA(args[0]));
         break;
+    case INDEX_op_extu_i32_i64:
+        tcg_out_ext32u(s, args[0], args[1]);
+        break;
 
     case INDEX_op_setcond_i32:
         tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
@@ -2482,6 +2526,8 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_ext8s_i64, { "r", "r" } },
     { INDEX_op_ext16s_i64, { "r", "r" } },
     { INDEX_op_ext32s_i64, { "r", "r" } },
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
     { INDEX_op_bswap16_i64, { "r", "r" } },
     { INDEX_op_bswap32_i64, { "r", "r" } },
     { INDEX_op_bswap64_i64, { "r", "r" } },
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index 7ce704882..b4f081876 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -77,7 +77,8 @@ typedef enum {
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_add2_i32         0
 #define TCG_TARGET_HAS_sub2_i32         0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_rot_i64          1
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
index aa718eca0..fbf97bb2e 100644
--- a/tcg/s390/tcg-target.c
+++ b/tcg/s390/tcg-target.c
@@ -51,17 +51,10 @@
 /* A scratch register that may be be used throughout the backend.  */
 #define TCG_TMP0        TCG_REG_R14
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 #define TCG_GUEST_BASE_REG TCG_REG_R13
-#else
-#define TCG_GUEST_BASE_REG TCG_REG_R0
-#endif
-
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
 #endif
 
-
 /* All of the following instructions are prefixed with their instruction
    format, and are defined as 8- or 16-bit quantities, even when the two
    halves of the 16-bit quantity may appear 32 bits apart in the insn.
@@ -1504,20 +1497,36 @@ QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
 static TCGReg tcg_out_tlb_read(TCGContext* s, TCGReg addr_reg, TCGMemOp opc,
                                int mem_index, bool is_ld)
 {
-    TCGMemOp s_bits = opc & MO_SIZE;
-    uint64_t tlb_mask = TARGET_PAGE_MASK | ((1 << s_bits) - 1);
-    int ofs;
+    int s_mask = (1 << (opc & MO_SIZE)) - 1;
+    int ofs, a_off;
+    uint64_t tlb_mask;
+
+    /* For aligned accesses, we check the first byte and include the alignment
+       bits within the address.  For unaligned access, we check that we don't
+       cross pages using the address of the last byte of the access.  */
+    if ((opc & MO_AMASK) == MO_ALIGN || s_mask == 0) {
+        a_off = 0;
+        tlb_mask = TARGET_PAGE_MASK | s_mask;
+    } else {
+        a_off = s_mask;
+        tlb_mask = TARGET_PAGE_MASK;
+    }
 
     if (facilities & FACILITY_GEN_INST_EXT) {
         tcg_out_risbg(s, TCG_REG_R2, addr_reg,
                       64 - CPU_TLB_BITS - CPU_TLB_ENTRY_BITS,
                       63 - CPU_TLB_ENTRY_BITS,
                       64 + CPU_TLB_ENTRY_BITS - TARGET_PAGE_BITS, 1);
-        tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
+        if (a_off) {
+            tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
+            tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
+        } else {
+            tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
+        }
     } else {
         tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
                      TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-        tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_R3, addr_reg);
+        tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
         tgen_andi(s, TCG_TYPE_I64, TCG_REG_R2,
                   (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
         tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
@@ -1622,9 +1631,9 @@ static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
         tgen_ext32u(s, TCG_TMP0, *addr_reg);
         *addr_reg = TCG_TMP0;
     }
-    if (GUEST_BASE < 0x80000) {
+    if (guest_base < 0x80000) {
         *index_reg = TCG_REG_NONE;
-        *disp = GUEST_BASE;
+        *disp = guest_base;
     } else {
         *index_reg = TCG_GUEST_BASE_REG;
         *disp = 0;
@@ -2090,6 +2099,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
         tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tgen_ext32s(s, args[0], args[1]);
         break;
@@ -2099,6 +2109,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
         tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tgen_ext32u(s, args[0], args[1]);
         break;
@@ -2251,6 +2262,9 @@ static const TCGTargetOpDef s390_op_defs[] = {
     { INDEX_op_ext32s_i64, { "r", "r" } },
     { INDEX_op_ext32u_i64, { "r", "r" } },
 
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+
     { INDEX_op_bswap16_i64, { "r", "r" } },
     { INDEX_op_bswap32_i64, { "r", "r" } },
     { INDEX_op_bswap64_i64, { "r", "r" } },
@@ -2328,10 +2342,12 @@ static void tcg_target_qemu_prologue(TCGContext *s)
                   TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
                   CPU_TEMP_BUF_NLONGS * sizeof(long));
 
-    if (GUEST_BASE >= 0x80000) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+#ifndef CONFIG_SOFTMMU
+    if (guest_base >= 0x80000) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
+#endif
 
     tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
     /* br %r3 (go to TB) */
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index 91576d594..d9dc03873 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -72,7 +72,8 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_HAS_div2_i64         1
 #define TCG_TARGET_HAS_rot_i64          1
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index 1a870a81d..54df1bc42 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -83,10 +83,8 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #define TCG_REG_T1  TCG_REG_G1
 #define TCG_REG_T2  TCG_REG_O7
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 # define TCG_GUEST_BASE_REG TCG_REG_I5
-#else
-# define TCG_GUEST_BASE_REG TCG_REG_G0
 #endif
 
 static const int tcg_target_reg_alloc_order[] = {
@@ -955,9 +953,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tcg_out32(s, SAVE | INSN_RD(TCG_REG_O6) | INSN_RS1(TCG_REG_O6) |
               INSN_IMM13(-frame_size));
 
-#ifdef CONFIG_USE_GUEST_BASE
-    if (GUEST_BASE != 0) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+#ifndef CONFIG_SOFTMMU
+    if (guest_base != 0) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
 #endif
@@ -1146,7 +1144,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
         addr = TCG_REG_T1;
     }
     tcg_out_ldst_rr(s, data, addr,
-                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0),
                     qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]);
 #endif /* CONFIG_SOFTMMU */
 }
@@ -1201,7 +1199,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
         addr = TCG_REG_T1;
     }
     tcg_out_ldst_rr(s, data, addr,
-                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0),
                     qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]);
 #endif /* CONFIG_SOFTMMU */
 }
@@ -1407,18 +1405,19 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_divu_i64:
         c = ARITH_UDIVX;
         goto gen_arith;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_arithi(s, a0, a1, 0, SHIFT_SRA);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_arithi(s, a0, a1, 0, SHIFT_SRL);
         break;
-    case INDEX_op_trunc_shr_i32:
-        if (a2 == 0) {
-            tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
-        } else {
-            tcg_out_arithi(s, a0, a1, a2, SHIFT_SRLX);
-        }
+    case INDEX_op_extrl_i64_i32:
+        tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
+        break;
+    case INDEX_op_extrh_i64_i32:
+        tcg_out_arithi(s, a0, a1, 32, SHIFT_SRLX);
         break;
 
     case INDEX_op_brcond_i64:
@@ -1531,9 +1530,12 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_neg_i64, { "R", "RJ" } },
     { INDEX_op_not_i64, { "R", "RJ" } },
 
-    { INDEX_op_ext32s_i64, { "R", "r" } },
-    { INDEX_op_ext32u_i64, { "R", "r" } },
-    { INDEX_op_trunc_shr_i32,  { "r", "R" } },
+    { INDEX_op_ext32s_i64, { "R", "R" } },
+    { INDEX_op_ext32u_i64, { "R", "R" } },
+    { INDEX_op_ext_i32_i64, { "R", "r" } },
+    { INDEX_op_extu_i32_i64, { "R", "r" } },
+    { INDEX_op_extrl_i64_i32,  { "r", "R" } },
+    { INDEX_op_extrh_i64_i32,  { "r", "R" } },
 
     { INDEX_op_brcond_i64, { "RZ", "RJ" } },
     { INDEX_op_setcond_i64, { "R", "RZ", "RJ" } },
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index f584de476..2cd72d2d4 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -118,7 +118,8 @@ extern bool use_vis3_instructions;
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 
-#define TCG_TARGET_HAS_trunc_shr_i32    1
+#define TCG_TARGET_HAS_extrl_i64_i32    1
+#define TCG_TARGET_HAS_extrh_i64_i32    1
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_rot_i64          0
diff --git a/tcg/tcg-common.c b/tcg/tcg-common.c
new file mode 100644
index 000000000..8fa4e1330
--- /dev/null
+++ b/tcg/tcg-common.c
@@ -0,0 +1,37 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "tcg/tcg.h"
+
+#if defined(CONFIG_TCG_INTERPRETER)
+uintptr_t tci_tb_ptr;
+#endif
+
+TCGOpDef tcg_op_defs[] = {
+#define DEF(s, oargs, iargs, cargs, flags) \
+         { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
+#include "tcg-opc.h"
+#undef DEF
+};
+const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index 45098c310..0b9dd8ff9 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -1737,28 +1737,28 @@ void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
 
 /* Size changing operations.  */
 
-void tcg_gen_trunc_shr_i64_i32(TCGv_i32 ret, TCGv_i64 arg, unsigned count)
+void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
 {
-    tcg_debug_assert(count < 64);
     if (TCG_TARGET_REG_BITS == 32) {
-        if (count >= 32) {
-            tcg_gen_shri_i32(ret, TCGV_HIGH(arg), count - 32);
-        } else if (count == 0) {
-            tcg_gen_mov_i32(ret, TCGV_LOW(arg));
-        } else {
-            TCGv_i64 t = tcg_temp_new_i64();
-            tcg_gen_shri_i64(t, arg, count);
-            tcg_gen_mov_i32(ret, TCGV_LOW(t));
-            tcg_temp_free_i64(t);
-        }
-    } else if (TCG_TARGET_HAS_trunc_shr_i32) {
-        tcg_gen_op3i_i32(INDEX_op_trunc_shr_i32, ret,
-                         MAKE_TCGV_I32(GET_TCGV_I64(arg)), count);
-    } else if (count == 0) {
+        tcg_gen_mov_i32(ret, TCGV_LOW(arg));
+    } else if (TCG_TARGET_HAS_extrl_i64_i32) {
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extrl_i64_i32,
+                    GET_TCGV_I32(ret), GET_TCGV_I64(arg));
+    } else {
         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg)));
+    }
+}
+
+void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
+{
+    if (TCG_TARGET_REG_BITS == 32) {
+        tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
+    } else if (TCG_TARGET_HAS_extrh_i64_i32) {
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extrh_i64_i32,
+                    GET_TCGV_I32(ret), GET_TCGV_I64(arg));
     } else {
         TCGv_i64 t = tcg_temp_new_i64();
-        tcg_gen_shri_i64(t, arg, count);
+        tcg_gen_shri_i64(t, arg, 32);
         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(t)));
         tcg_temp_free_i64(t);
     }
@@ -1770,9 +1770,8 @@ void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
     } else {
-        /* Note: we assume the target supports move between
-           32 and 64 bit registers.  */
-        tcg_gen_ext32u_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)));
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extu_i32_i64,
+                    GET_TCGV_I64(ret), GET_TCGV_I32(arg));
     }
 }
 
@@ -1782,9 +1781,8 @@ void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
     } else {
-        /* Note: we assume the target supports move between
-           32 and 64 bit registers.  */
-        tcg_gen_ext32s_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)));
+        tcg_gen_op2(&tcg_ctx, INDEX_op_ext_i32_i64,
+                    GET_TCGV_I64(ret), GET_TCGV_I32(arg));
     }
 }
 
@@ -1820,8 +1818,8 @@ void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
     } else {
-        tcg_gen_trunc_shr_i64_i32(lo, arg, 0);
-        tcg_gen_trunc_shr_i64_i32(hi, arg, 32);
+        tcg_gen_extrl_i64_i32(lo, arg);
+        tcg_gen_extrh_i64_i32(hi, arg);
     }
 }
 
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index d1d763f6f..4e20dc1a6 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -684,7 +684,8 @@ static inline void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg)
 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg);
 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg);
 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high);
-void tcg_gen_trunc_shr_i64_i32(TCGv_i32 ret, TCGv_i64 arg, unsigned int c);
+void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg);
+void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg);
 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg);
 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg);
 
@@ -693,28 +694,59 @@ static inline void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi)
     tcg_gen_deposit_i64(ret, lo, hi, 32, 32);
 }
 
-static inline void tcg_gen_trunc_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
-{
-    tcg_gen_trunc_shr_i64_i32(ret, arg, 0);
-}
-
 /* QEMU specific operations.  */
 
 #ifndef TARGET_LONG_BITS
 #error must include QEMU headers
 #endif
 
-/* debug info: write the PC of the corresponding QEMU CPU instruction */
-static inline void tcg_gen_debug_insn_start(uint64_t pc)
+#if TARGET_INSN_START_WORDS == 1
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc)
+{
+    tcg_gen_op1(&tcg_ctx, INDEX_op_insn_start, pc);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc)
+{
+    tcg_gen_op2(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32));
+}
+# endif
+#elif TARGET_INSN_START_WORDS == 2
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1)
 {
-    /* XXX: must really use a 32 bit size for TCGArg in all cases */
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-    tcg_gen_op2ii(INDEX_op_debug_insn_start,
-                  (uint32_t)(pc), (uint32_t)(pc >> 32));
+    tcg_gen_op2(&tcg_ctx, INDEX_op_insn_start, pc, a1);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1)
+{
+    tcg_gen_op4(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32),
+                (uint32_t)a1, (uint32_t)(a1 >> 32));
+}
+# endif
+#elif TARGET_INSN_START_WORDS == 3
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1,
+                                      target_ulong a2)
+{
+    tcg_gen_op3(&tcg_ctx, INDEX_op_insn_start, pc, a1, a2);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1,
+                                      target_ulong a2)
+{
+    tcg_gen_op6(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32),
+                (uint32_t)a1, (uint32_t)(a1 >> 32),
+                (uint32_t)a2, (uint32_t)(a2 >> 32));
+}
+# endif
 #else
-    tcg_gen_op1i(INDEX_op_debug_insn_start, pc);
+# error "Unhandled number of operands to insn_start"
 #endif
-}
 
 static inline void tcg_gen_exit_tb(uintptr_t val)
 {
@@ -853,7 +885,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_divu_tl tcg_gen_divu_i64
 #define tcg_gen_remu_tl tcg_gen_remu_i64
 #define tcg_gen_discard_tl tcg_gen_discard_i64
-#define tcg_gen_trunc_tl_i32 tcg_gen_trunc_i64_i32
+#define tcg_gen_trunc_tl_i32 tcg_gen_extrl_i64_i32
 #define tcg_gen_trunc_i64_tl tcg_gen_mov_i64
 #define tcg_gen_extu_i32_tl tcg_gen_extu_i32_i64
 #define tcg_gen_ext_i32_tl tcg_gen_ext_i32_i64
@@ -932,7 +964,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_remu_tl tcg_gen_remu_i32
 #define tcg_gen_discard_tl tcg_gen_discard_i32
 #define tcg_gen_trunc_tl_i32 tcg_gen_mov_i32
-#define tcg_gen_trunc_i64_tl tcg_gen_trunc_i64_i32
+#define tcg_gen_trunc_i64_tl tcg_gen_extrl_i64_i32
 #define tcg_gen_extu_i32_tl tcg_gen_mov_i32
 #define tcg_gen_ext_i32_tl tcg_gen_mov_i32
 #define tcg_gen_extu_tl_i64 tcg_gen_extu_i32_i64
diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h
index 13ccb60a5..6d0410c4b 100644
--- a/tcg/tcg-opc.h
+++ b/tcg/tcg-opc.h
@@ -138,8 +138,14 @@ DEF(rotl_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(rotr_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(deposit_i64, 1, 2, 2, IMPL64 | IMPL(TCG_TARGET_HAS_deposit_i64))
 
-DEF(trunc_shr_i32, 1, 1, 1,
-    IMPL(TCG_TARGET_HAS_trunc_shr_i32)
+/* size changing ops */
+DEF(ext_i32_i64, 1, 1, 0, IMPL64)
+DEF(extu_i32_i64, 1, 1, 0, IMPL64)
+DEF(extrl_i64_i32, 1, 1, 0,
+    IMPL(TCG_TARGET_HAS_extrl_i64_i32)
+    | (TCG_TARGET_REG_BITS == 32 ? TCG_OPF_NOT_PRESENT : 0))
+DEF(extrh_i64_i32, 1, 1, 0,
+    IMPL(TCG_TARGET_HAS_extrh_i64_i32)
     | (TCG_TARGET_REG_BITS == 32 ? TCG_OPF_NOT_PRESENT : 0))
 
 DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | IMPL64)
@@ -167,18 +173,15 @@ DEF(muls2_i64, 2, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_muls2_i64))
 DEF(muluh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_muluh_i64))
 DEF(mulsh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_mulsh_i64))
 
+#define TLADDR_ARGS  (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2)
+#define DATA64_ARGS  (TCG_TARGET_REG_BITS == 64 ? 1 : 2)
+
 /* QEMU specific */
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-DEF(debug_insn_start, 0, 0, 2, TCG_OPF_NOT_PRESENT)
-#else
-DEF(debug_insn_start, 0, 0, 1, TCG_OPF_NOT_PRESENT)
-#endif
+DEF(insn_start, 0, 0, TLADDR_ARGS * TARGET_INSN_START_WORDS,
+    TCG_OPF_NOT_PRESENT)
 DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END)
 DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END)
 
-#define TLADDR_ARGS    (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2)
-#define DATA64_ARGS  (TCG_TARGET_REG_BITS == 64 ? 1 : 2)
-
 DEF(qemu_ld_i32, 1, TLADDR_ARGS, 1,
     TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
 DEF(qemu_st_i32, 0, TLADDR_ARGS + 1, 1,
diff --git a/tcg/tcg.c b/tcg/tcg.c
index 0892a9bbf..a163541d3 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -113,12 +113,6 @@ static void tcg_out_tb_init(TCGContext *s);
 static void tcg_out_tb_finalize(TCGContext *s);
 
 
-TCGOpDef tcg_op_defs[] = {
-#define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
-#include "tcg-opc.h"
-#undef DEF
-};
-const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
 
 static TCGRegSet tcg_target_available_regs[2];
 static TCGRegSet tcg_target_call_clobber_regs;
@@ -369,17 +363,43 @@ void tcg_context_init(TCGContext *s)
 
 void tcg_prologue_init(TCGContext *s)
 {
-    /* init global prologue and epilogue */
-    s->code_buf = s->code_gen_prologue;
-    s->code_ptr = s->code_buf;
+    size_t prologue_size, total_size;
+    void *buf0, *buf1;
+
+    /* Put the prologue at the beginning of code_gen_buffer.  */
+    buf0 = s->code_gen_buffer;
+    s->code_ptr = buf0;
+    s->code_buf = buf0;
+    s->code_gen_prologue = buf0;
+
+    /* Generate the prologue.  */
     tcg_target_qemu_prologue(s);
-    flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
+    buf1 = s->code_ptr;
+    flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
+
+    /* Deduct the prologue from the buffer.  */
+    prologue_size = tcg_current_code_size(s);
+    s->code_gen_ptr = buf1;
+    s->code_gen_buffer = buf1;
+    s->code_buf = buf1;
+    total_size = s->code_gen_buffer_size - prologue_size;
+    s->code_gen_buffer_size = total_size;
+
+    /* Compute a high-water mark, at which we voluntarily flush the buffer
+       and start over.  The size here is arbitrary, significantly larger
+       than we expect the code generation for any one opcode to require.  */
+    /* ??? We currently have no good estimate for, or checks in,
+       tcg_out_tb_finalize.  If there are quite a lot of guest memory ops,
+       the number of out-of-line fragments could be quite high.  In the
+       short-term, increase the highwater buffer.  */
+    s->code_gen_highwater = s->code_gen_buffer + (total_size - 64*1024);
+
+    tcg_register_jit(s->code_gen_buffer, total_size);
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
-        size_t size = tcg_current_code_size(s);
-        qemu_log("PROLOGUE: [size=%zu]\n", size);
-        log_disas(s->code_buf, size);
+        qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
+        log_disas(buf0, prologue_size);
         qemu_log("\n");
         qemu_log_flush();
     }
@@ -996,17 +1016,18 @@ void tcg_dump_ops(TCGContext *s)
         def = &tcg_op_defs[c];
         args = &s->gen_opparam_buf[op->args];
 
-        if (c == INDEX_op_debug_insn_start) {
-            uint64_t pc;
+        if (c == INDEX_op_insn_start) {
+            qemu_log("%s ----", oi != s->gen_first_op_idx ? "\n" : "");
+
+            for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+                target_ulong a;
 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-            pc = ((uint64_t)args[1] << 32) | args[0];
+                a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
 #else
-            pc = args[0];
+                a = args[i];
 #endif
-            if (oi != s->gen_first_op_idx) {
-                qemu_log("\n");
+                qemu_log(" " TARGET_FMT_lx, a);
             }
-            qemu_log(" ---- 0x%" PRIx64, pc);
         } else if (c == INDEX_op_call) {
             /* variable number of arguments */
             nb_oargs = op->callo;
@@ -1240,7 +1261,7 @@ void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
 
 #if defined(CONFIG_DEBUG_TCG)
     i = 0;
-    for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) {
+    for (op = 0; op < tcg_op_defs_max; op++) {
         const TCGOpDef *def = &tcg_op_defs[op];
         if (def->flags & TCG_OPF_NOT_PRESENT) {
             /* Wrong entry in op definitions? */
@@ -1396,7 +1417,7 @@ static void tcg_liveness_analysis(TCGContext *s)
                             }
                         }
                     }
-                    /* input arguments are live for preceeding opcodes */
+                    /* input arguments are live for preceding opcodes */
                     for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
                         arg = args[i];
                         dead_temps[arg] = 0;
@@ -1406,7 +1427,7 @@ static void tcg_liveness_analysis(TCGContext *s)
                 }
             }
             break;
-        case INDEX_op_debug_insn_start:
+        case INDEX_op_insn_start:
             break;
         case INDEX_op_discard:
             /* mark the temporary as dead */
@@ -1542,7 +1563,7 @@ static void tcg_liveness_analysis(TCGContext *s)
                         dead_args |= (1 << i);
                     }
                 }
-                /* input arguments are live for preceeding opcodes */
+                /* input arguments are live for preceding opcodes */
                 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
                     arg = args[i];
                     dead_temps[arg] = 0;
@@ -2295,11 +2316,27 @@ void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
 #endif
 
 
-static inline int tcg_gen_code_common(TCGContext *s,
-                                      tcg_insn_unit *gen_code_buf,
-                                      long search_pc)
+int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf)
 {
-    int oi, oi_next;
+    int i, oi, oi_next, num_insns;
+
+#ifdef CONFIG_PROFILER
+    {
+        int n;
+
+        n = s->gen_last_op_idx + 1;
+        s->op_count += n;
+        if (n > s->op_count_max) {
+            s->op_count_max = n;
+        }
+
+        n = s->nb_temps;
+        s->temp_count += n;
+        if (n > s->temp_count_max) {
+            s->temp_count_max = n;
+        }
+    }
+#endif
 
 #ifdef DEBUG_DISAS
     if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
@@ -2343,6 +2380,7 @@ static inline int tcg_gen_code_common(TCGContext *s,
 
     tcg_out_tb_init(s);
 
+    num_insns = -1;
     for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) {
         TCGOp * const op = &s->gen_op_buf[oi];
         TCGArg * const args = &s->gen_opparam_buf[op->args];
@@ -2365,7 +2403,20 @@ static inline int tcg_gen_code_common(TCGContext *s,
         case INDEX_op_movi_i64:
             tcg_reg_alloc_movi(s, args, dead_args, sync_args);
             break;
-        case INDEX_op_debug_insn_start:
+        case INDEX_op_insn_start:
+            if (num_insns >= 0) {
+                s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
+            }
+            num_insns++;
+            for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+                target_ulong a;
+#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
+                a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
+#else
+                a = args[i];
+#endif
+                s->gen_insn_data[num_insns][i] = a;
+            }
             break;
         case INDEX_op_discard:
             temp_dead(s, args[0]);
@@ -2389,40 +2440,22 @@ static inline int tcg_gen_code_common(TCGContext *s,
             tcg_reg_alloc_op(s, def, opc, args, dead_args, sync_args);
             break;
         }
-        if (search_pc >= 0 && search_pc < tcg_current_code_size(s)) {
-            return oi;
-        }
 #ifndef NDEBUG
         check_regs(s);
 #endif
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one operation beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           generating code without having to check during generation.  */
+        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+            return -1;
+        }
     }
+    tcg_debug_assert(num_insns >= 0);
+    s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
 
     /* Generate TB finalization at the end of block */
     tcg_out_tb_finalize(s);
-    return -1;
-}
-
-int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf)
-{
-#ifdef CONFIG_PROFILER
-    {
-        int n;
-
-        n = s->gen_last_op_idx + 1;
-        s->op_count += n;
-        if (n > s->op_count_max) {
-            s->op_count_max = n;
-        }
-
-        n = s->nb_temps;
-        s->temp_count += n;
-        if (n > s->temp_count_max) {
-            s->temp_count_max = n;
-        }
-    }
-#endif
-
-    tcg_gen_code_common(s, gen_code_buf, -1);
 
     /* flush instruction cache */
     flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
@@ -2430,38 +2463,30 @@ int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf)
     return tcg_current_code_size(s);
 }
 
-/* Return the index of the micro operation such as the pc after is <
-   offset bytes from the start of the TB.  The contents of gen_code_buf must
-   not be changed, though writing the same values is ok.
-   Return -1 if not found. */
-int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf,
-                           long offset)
-{
-    return tcg_gen_code_common(s, gen_code_buf, offset);
-}
-
 #ifdef CONFIG_PROFILER
 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
 {
     TCGContext *s = &tcg_ctx;
-    int64_t tot;
+    int64_t tb_count = s->tb_count;
+    int64_t tb_div_count = tb_count ? tb_count : 1;
+    int64_t tot = s->interm_time + s->code_time;
 
-    tot = s->interm_time + s->code_time;
     cpu_fprintf(f, "JIT cycles          %" PRId64 " (%0.3f s at 2.4 GHz)\n",
                 tot, tot / 2.4e9);
     cpu_fprintf(f, "translated TBs      %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 
-                s->tb_count, 
-                s->tb_count1 - s->tb_count,
-                s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0);
+                tb_count, s->tb_count1 - tb_count,
+                (double)(s->tb_count1 - s->tb_count)
+                / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
     cpu_fprintf(f, "avg ops/TB          %0.1f max=%d\n", 
-                s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max);
+                (double)s->op_count / tb_div_count, s->op_count_max);
     cpu_fprintf(f, "deleted ops/TB      %0.2f\n",
-                s->tb_count ? 
-                (double)s->del_op_count / s->tb_count : 0);
+                (double)s->del_op_count / tb_div_count);
     cpu_fprintf(f, "avg temps/TB        %0.2f max=%d\n",
-                s->tb_count ? 
-                (double)s->temp_count / s->tb_count : 0,
-                s->temp_count_max);
+                (double)s->temp_count / tb_div_count, s->temp_count_max);
+    cpu_fprintf(f, "avg host code/TB    %0.1f\n",
+                (double)s->code_out_len / tb_div_count);
+    cpu_fprintf(f, "avg search data/TB  %0.1f\n",
+                (double)s->search_out_len / tb_div_count);
     
     cpu_fprintf(f, "cycles/op           %0.1f\n", 
                 s->op_count ? (double)tot / s->op_count : 0);
@@ -2469,8 +2494,11 @@ void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
                 s->code_in_len ? (double)tot / s->code_in_len : 0);
     cpu_fprintf(f, "cycles/out byte     %0.1f\n", 
                 s->code_out_len ? (double)tot / s->code_out_len : 0);
-    if (tot == 0)
+    cpu_fprintf(f, "cycles/search byte     %0.1f\n",
+                s->search_out_len ? (double)tot / s->search_out_len : 0);
+    if (tot == 0) {
         tot = 1;
+    }
     cpu_fprintf(f, "  gen_interm time   %0.1f%%\n", 
                 (double)s->interm_time / tot * 100.0);
     cpu_fprintf(f, "  gen_code time     %0.1f%%\n", 
diff --git a/tcg/tcg.h b/tcg/tcg.h
index 231a78152..a69692242 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -66,7 +66,8 @@ typedef uint64_t TCGRegSet;
 
 #if TCG_TARGET_REG_BITS == 32
 /* Turn some undef macros into false macros.  */
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div_i64          0
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_div2_i64         0
@@ -128,6 +129,12 @@ typedef uint64_t TCGRegSet;
 # error "Missing unsigned widening multiply"
 #endif
 
+#ifndef TARGET_INSN_START_EXTRA_WORDS
+# define TARGET_INSN_START_WORDS 1
+#else
+# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
+#endif
+
 typedef enum TCGOpcode {
 #define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
 #include "tcg-opc.h"
@@ -187,6 +194,7 @@ typedef struct TCGPool {
 #define TCG_POOL_CHUNK_SIZE 32768
 
 #define TCG_MAX_TEMPS 512
+#define TCG_MAX_INSNS 512
 
 /* when the size of the arguments of a called function is smaller than
    this value, they are statically allocated in the TB stack frame */
@@ -524,6 +532,7 @@ struct TCGContext {
     int64_t del_op_count;
     int64_t code_in_len;
     int64_t code_out_len;
+    int64_t search_out_len;
     int64_t interm_time;
     int64_t code_time;
     int64_t la_time;
@@ -550,10 +559,11 @@ struct TCGContext {
     void *code_gen_prologue;
     void *code_gen_buffer;
     size_t code_gen_buffer_size;
-    /* threshold to flush the translated code buffer */
-    size_t code_gen_buffer_max_size;
     void *code_gen_ptr;
 
+    /* Threshold to flush the translated code buffer.  */
+    void *code_gen_highwater;
+
     TBContext tb_ctx;
 
     /* The TCGBackendData structure is private to tcg-target.c.  */
@@ -569,9 +579,8 @@ struct TCGContext {
     TCGOp gen_op_buf[OPC_BUF_SIZE];
     TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
 
-    target_ulong gen_opc_pc[OPC_BUF_SIZE];
-    uint16_t gen_opc_icount[OPC_BUF_SIZE];
-    uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
+    uint16_t gen_insn_end_off[TCG_MAX_INSNS];
+    target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
 };
 
 extern TCGContext tcg_ctx;
@@ -594,6 +603,10 @@ void *tcg_malloc_internal(TCGContext *s, int size);
 void tcg_pool_reset(TCGContext *s);
 void tcg_pool_delete(TCGContext *s);
 
+void tb_lock(void);
+void tb_unlock(void);
+void tb_lock_reset(void);
+
 static inline void *tcg_malloc(int size)
 {
     TCGContext *s = &tcg_ctx;
@@ -614,8 +627,6 @@ void tcg_prologue_init(TCGContext *s);
 void tcg_func_start(TCGContext *s);
 
 int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf);
-int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf,
-                           long offset);
 
 void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size);
 
@@ -985,25 +996,48 @@ void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
 void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
                        TCGMemOpIdx oi, uintptr_t retaddr);
 
+uint8_t helper_ret_ldb_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint16_t helper_le_ldw_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint32_t helper_le_ldl_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_le_ldq_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint16_t helper_be_ldw_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint32_t helper_be_ldl_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_be_ldq_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+
 /* Temporary aliases until backends are converted.  */
 #ifdef TARGET_WORDS_BIGENDIAN
 # define helper_ret_ldsw_mmu  helper_be_ldsw_mmu
 # define helper_ret_lduw_mmu  helper_be_lduw_mmu
 # define helper_ret_ldsl_mmu  helper_be_ldsl_mmu
 # define helper_ret_ldul_mmu  helper_be_ldul_mmu
+# define helper_ret_ldl_mmu   helper_be_ldul_mmu
 # define helper_ret_ldq_mmu   helper_be_ldq_mmu
 # define helper_ret_stw_mmu   helper_be_stw_mmu
 # define helper_ret_stl_mmu   helper_be_stl_mmu
 # define helper_ret_stq_mmu   helper_be_stq_mmu
+# define helper_ret_ldw_cmmu  helper_be_ldw_cmmu
+# define helper_ret_ldl_cmmu  helper_be_ldl_cmmu
+# define helper_ret_ldq_cmmu  helper_be_ldq_cmmu
 #else
 # define helper_ret_ldsw_mmu  helper_le_ldsw_mmu
 # define helper_ret_lduw_mmu  helper_le_lduw_mmu
 # define helper_ret_ldsl_mmu  helper_le_ldsl_mmu
 # define helper_ret_ldul_mmu  helper_le_ldul_mmu
+# define helper_ret_ldl_mmu   helper_le_ldul_mmu
 # define helper_ret_ldq_mmu   helper_le_ldq_mmu
 # define helper_ret_stw_mmu   helper_le_stw_mmu
 # define helper_ret_stl_mmu   helper_le_stl_mmu
 # define helper_ret_stq_mmu   helper_le_stq_mmu
+# define helper_ret_ldw_cmmu  helper_le_ldw_cmmu
+# define helper_ret_ldl_cmmu  helper_le_ldl_cmmu
+# define helper_ret_ldq_cmmu  helper_le_ldq_cmmu
 #endif
 
 #endif /* CONFIG_SOFTMMU */
diff --git a/tcg/tci/tcg-target.c b/tcg/tci/tcg-target.c
index 83472dbcd..4afe4d7a8 100644
--- a/tcg/tci/tcg-target.c
+++ b/tcg/tci/tcg-target.c
@@ -210,6 +210,8 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
 #if TCG_TARGET_HAS_ext32u_i64
     { INDEX_op_ext32u_i64, { R, R } },
 #endif
+    { INDEX_op_ext_i32_i64, { R, R } },
+    { INDEX_op_extu_i32_i64, { R, R } },
 #if TCG_TARGET_HAS_bswap16_i64
     { INDEX_op_bswap16_i64, { R, R } },
 #endif
@@ -701,6 +703,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_ext16u_i64:   /* Optional (TCG_TARGET_HAS_ext16u_i64). */
     case INDEX_op_ext32s_i64:   /* Optional (TCG_TARGET_HAS_ext32s_i64). */
     case INDEX_op_ext32u_i64:   /* Optional (TCG_TARGET_HAS_ext32u_i64). */
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
 #endif /* TCG_TARGET_REG_BITS == 64 */
     case INDEX_op_neg_i32:      /* Optional (TCG_TARGET_HAS_neg_i32). */
     case INDEX_op_not_i32:      /* Optional (TCG_TARGET_HAS_not_i32). */
@@ -846,7 +850,7 @@ static void tcg_target_init(TCGContext *s)
 #endif
 
     /* The current code uses uint8_t for tcg operations. */
-    assert(ARRAY_SIZE(tcg_op_defs) <= UINT8_MAX);
+    assert(tcg_op_defs_max <= UINT8_MAX);
 
     /* Registers available for 32 bit operations. */
     tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0,
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index cbf3f9b5a..77e595278 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -84,7 +84,8 @@
 #define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_bswap16_i64      1
 #define TCG_TARGET_HAS_bswap32_i64      1
 #define TCG_TARGET_HAS_bswap64_i64      1
diff --git a/tci.c b/tci.c
index 84449489d..b5ed7b1f7 100644
--- a/tci.c
+++ b/tci.c
@@ -52,12 +52,6 @@ typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong);
 #endif
 
-/* Targets which don't use GETPC also don't need tci_tb_ptr
-   which makes them a little faster. */
-#if defined(GETPC)
-uintptr_t tci_tb_ptr;
-#endif
-
 static tcg_target_ulong tci_reg[TCG_TARGET_NB_REGS];
 
 static tcg_target_ulong tci_read_reg(TCGReg index)
@@ -1033,18 +1027,20 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
 #endif
 #if TCG_TARGET_HAS_ext32s_i64
         case INDEX_op_ext32s_i64:
+#endif
+        case INDEX_op_ext_i32_i64:
             t0 = *tb_ptr++;
             t1 = tci_read_r32s(&tb_ptr);
             tci_write_reg64(t0, t1);
             break;
-#endif
 #if TCG_TARGET_HAS_ext32u_i64
         case INDEX_op_ext32u_i64:
+#endif
+        case INDEX_op_extu_i32_i64:
             t0 = *tb_ptr++;
             t1 = tci_read_r32(&tb_ptr);
             tci_write_reg64(t0, t1);
             break;
-#endif
 #if TCG_TARGET_HAS_bswap16_i64
         case INDEX_op_bswap16_i64:
             TODO();
@@ -1085,15 +1081,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
 
             /* QEMU specific operations. */
 
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-        case INDEX_op_debug_insn_start:
-            TODO();
-            break;
-#else
-        case INDEX_op_debug_insn_start:
-            TODO();
-            break;
-#endif
         case INDEX_op_exit_tb:
             next_tb = *(uint64_t *)tb_ptr;
             goto exit;
diff --git a/tests/.gitignore b/tests/.gitignore
index ccc92e476..1e55722b6 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -9,9 +9,17 @@ check-qom-proplist
 rcutorture
 test-aio
 test-bitops
+test-blockjob-txn
 test-coroutine
 test-crypto-cipher
 test-crypto-hash
+test-crypto-tlscredsx509
+test-crypto-tlscredsx509-work/
+test-crypto-tlscredsx509-certs/
+test-crypto-tlssession
+test-crypto-tlssession-work/
+test-crypto-tlssession-client/
+test-crypto-tlssession-server/
 test-cutils
 test-hbitmap
 test-int128
@@ -23,11 +31,13 @@ test-qapi-types.[ch]
 test-qapi-visit.[ch]
 test-qdev-global-props
 test-qemu-opts
+test-qga
 test-qmp-commands
 test-qmp-commands.h
 test-qmp-event
 test-qmp-input-strict
 test-qmp-input-visitor
+test-qmp-introspect.[ch]
 test-qmp-marshal.c
 test-qmp-output-visitor
 test-rcu-list
@@ -36,10 +46,12 @@ test-string-input-visitor
 test-string-output-visitor
 test-thread-pool
 test-throttle
+test-timed-average
 test-visitor-serialization
 test-vmstate
 test-write-threshold
 test-x86-cpuid
 test-xbzrle
+test-netfilter
 *-test
 qapi-schema/*.test.*
diff --git a/tests/Makefile b/tests/Makefile
index b453eb44a..a1d03b457 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,5 +1,7 @@
 export SRC_PATH
 
+qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py
+
 # Get the list of all supported sysemu targets
 SYSEMU_TARGET_LIST := $(subst -softmmu.mak,,$(notdir \
    $(wildcard $(SRC_PATH)/default-configs/*-softmmu.mak)))
@@ -45,6 +47,8 @@ check-unit-y += tests/test-thread-pool$(EXESUF)
 gcov-files-test-thread-pool-y = thread-pool.c
 gcov-files-test-hbitmap-y = util/hbitmap.c
 check-unit-y += tests/test-hbitmap$(EXESUF)
+gcov-files-test-hbitmap-y = blockjob.c
+check-unit-y += tests/test-blockjob-txn$(EXESUF)
 check-unit-y += tests/test-x86-cpuid$(EXESUF)
 # all code tested by test-x86-cpuid is inside topology.h
 gcov-files-test-x86-cpuid-y =
@@ -76,6 +80,10 @@ check-unit-y += tests/test-write-threshold$(EXESUF)
 gcov-files-test-write-threshold-y = block/write-threshold.c
 check-unit-$(CONFIG_GNUTLS_HASH) += tests/test-crypto-hash$(EXESUF)
 check-unit-y += tests/test-crypto-cipher$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-crypto-tlscredsx509$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-crypto-tlssession$(EXESUF)
+check-unit-$(CONFIG_LINUX) += tests/test-qga$(EXESUF)
+check-unit-y += tests/test-timed-average$(EXESUF)
 
 check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
 
@@ -145,6 +153,8 @@ gcov-files-pci-y += hw/display/virtio-gpu-pci.c
 gcov-files-pci-$(CONFIG_VIRTIO_VGA) += hw/display/virtio-vga.c
 check-qtest-pci-y += tests/intel-hda-test$(EXESUF)
 gcov-files-pci-y += hw/audio/intel-hda.c hw/audio/hda-codec.c
+check-qtest-pci-$(CONFIG_POSIX) += tests/ivshmem-test$(EXESUF)
+gcov-files-pci-y += hw/misc/ivshmem.c
 
 check-qtest-i386-y = tests/endianness-test$(EXESUF)
 check-qtest-i386-y += tests/fdc-test$(EXESUF)
@@ -187,7 +197,11 @@ gcov-files-i386-y += hw/usb/hcd-xhci.c
 check-qtest-i386-y += tests/pc-cpu-test$(EXESUF)
 check-qtest-i386-y += tests/q35-test$(EXESUF)
 gcov-files-i386-y += hw/pci-host/q35.c
-check-qtest-i386-$(CONFIG_LINUX) += tests/vhost-user-test$(EXESUF)
+check-qtest-i386-$(CONFIG_VHOST_NET_TEST_i386) += tests/vhost-user-test$(EXESUF)
+ifeq ($(CONFIG_VHOST_NET_TEST_i386),)
+check-qtest-x86_64-$(CONFIG_VHOST_NET_TEST_x86_64) += tests/vhost-user-test$(EXESUF)
+endif
+check-qtest-i386-y += tests/test-netfilter$(EXESUF)
 check-qtest-x86_64-y = $(check-qtest-i386-y)
 gcov-files-i386-y += i386-softmmu/hw/timer/mc146818rtc.c
 gcov-files-x86_64-y = $(subst i386-softmmu/,x86_64-softmmu/,$(gcov-files-i386-y))
@@ -204,6 +218,7 @@ check-qtest-sparc64-y = tests/endianness-test$(EXESUF)
 gcov-files-sparc-y += hw/timer/m48t59.c
 gcov-files-sparc64-y += hw/timer/m48t59.c
 check-qtest-arm-y = tests/tmp105-test$(EXESUF)
+check-qtest-arm-y = tests/ds1338-test$(EXESUF)
 gcov-files-arm-y += hw/misc/tmp105.c
 check-qtest-arm-y += tests/virtio-blk-test$(EXESUF)
 gcov-files-arm-y += arm-softmmu/hw/block/virtio-blk.c
@@ -216,49 +231,133 @@ check-qtest-xtensaeb-y = $(check-qtest-xtensa-y)
 
 check-qtest-generic-y += tests/qom-test$(EXESUF)
 
-check-qapi-schema-y := $(addprefix tests/qapi-schema/, \
-	comments.json empty.json enum-empty.json enum-missing-data.json \
-	enum-wrong-data.json enum-int-member.json enum-dict-member.json \
-	enum-clash-member.json enum-max-member.json enum-union-clash.json \
-	enum-bad-name.json funny-char.json indented-expr.json \
-	missing-type.json bad-ident.json ident-with-escape.json \
-	escape-outside-string.json unknown-escape.json \
-	escape-too-short.json escape-too-big.json unicode-str.json \
-	double-type.json bad-base.json bad-type-bool.json bad-type-int.json \
-	bad-type-dict.json double-data.json unknown-expr-key.json \
-	redefined-type.json redefined-command.json redefined-builtin.json \
-	redefined-event.json command-int.json bad-data.json event-max.json \
-	type-bypass.json type-bypass-no-gen.json type-bypass-bad-gen.json \
-	data-array-empty.json data-array-unknown.json data-int.json \
-	data-unknown.json data-member-unknown.json data-member-array.json \
-	data-member-array-bad.json returns-array-bad.json returns-int.json \
-	returns-unknown.json returns-alternate.json returns-whitelist.json \
-	missing-colon.json missing-comma-list.json missing-comma-object.json \
-	nested-struct-data.json nested-struct-returns.json non-objects.json \
-	qapi-schema-test.json quoted-structural-chars.json \
-	trailing-comma-list.json trailing-comma-object.json \
-	unclosed-list.json unclosed-object.json unclosed-string.json \
-	duplicate-key.json union-invalid-base.json union-bad-branch.json \
-	union-optional-branch.json union-unknown.json union-max.json \
-	flat-union-optional-discriminator.json flat-union-no-base.json \
-	flat-union-invalid-discriminator.json flat-union-inline.json \
-	flat-union-invalid-branch-key.json flat-union-reverse-define.json \
-	flat-union-string-discriminator.json union-base-no-discriminator.json \
-	flat-union-bad-discriminator.json flat-union-bad-base.json \
-	flat-union-base-star.json \
-	flat-union-array-branch.json flat-union-int-branch.json \
-	flat-union-base-union.json flat-union-branch-clash.json \
-	alternate-nested.json alternate-unknown.json alternate-clash.json \
-	alternate-good.json alternate-base.json alternate-array.json \
-	alternate-conflict-string.json alternate-conflict-dict.json \
-	include-simple.json include-relpath.json include-format-err.json \
-	include-non-file.json include-no-file.json include-before-err.json \
-	include-nested-err.json include-self-cycle.json include-cycle.json \
-	include-repetition.json event-nest-struct.json event-case.json \
-	struct-base-clash.json struct-base-clash-deep.json )
+qapi-schema += alternate-array.json
+qapi-schema += alternate-base.json
+qapi-schema += alternate-clash.json
+qapi-schema += alternate-conflict-dict.json
+qapi-schema += alternate-conflict-string.json
+qapi-schema += alternate-empty.json
+qapi-schema += alternate-nested.json
+qapi-schema += alternate-unknown.json
+qapi-schema += args-alternate.json
+qapi-schema += args-any.json
+qapi-schema += args-array-empty.json
+qapi-schema += args-array-unknown.json
+qapi-schema += args-int.json
+qapi-schema += args-invalid.json
+qapi-schema += args-member-array-bad.json
+qapi-schema += args-member-unknown.json
+qapi-schema += args-name-clash.json
+qapi-schema += args-union.json
+qapi-schema += args-unknown.json
+qapi-schema += bad-base.json
+qapi-schema += bad-data.json
+qapi-schema += bad-ident.json
+qapi-schema += bad-type-bool.json
+qapi-schema += bad-type-dict.json
+qapi-schema += bad-type-int.json
+qapi-schema += command-int.json
+qapi-schema += comments.json
+qapi-schema += double-data.json
+qapi-schema += double-type.json
+qapi-schema += duplicate-key.json
+qapi-schema += empty.json
+qapi-schema += enum-bad-name.json
+qapi-schema += enum-bad-prefix.json
+qapi-schema += enum-clash-member.json
+qapi-schema += enum-dict-member.json
+qapi-schema += enum-int-member.json
+qapi-schema += enum-max-member.json
+qapi-schema += enum-missing-data.json
+qapi-schema += enum-wrong-data.json
+qapi-schema += escape-outside-string.json
+qapi-schema += escape-too-big.json
+qapi-schema += escape-too-short.json
+qapi-schema += event-case.json
+qapi-schema += event-max.json
+qapi-schema += event-nest-struct.json
+qapi-schema += flat-union-array-branch.json
+qapi-schema += flat-union-bad-base.json
+qapi-schema += flat-union-bad-discriminator.json
+qapi-schema += flat-union-base-any.json
+qapi-schema += flat-union-base-union.json
+qapi-schema += flat-union-clash-branch.json
+qapi-schema += flat-union-clash-member.json
+qapi-schema += flat-union-clash-type.json
+qapi-schema += flat-union-empty.json
+qapi-schema += flat-union-inline.json
+qapi-schema += flat-union-int-branch.json
+qapi-schema += flat-union-invalid-branch-key.json
+qapi-schema += flat-union-invalid-discriminator.json
+qapi-schema += flat-union-no-base.json
+qapi-schema += flat-union-optional-discriminator.json
+qapi-schema += flat-union-string-discriminator.json
+qapi-schema += funny-char.json
+qapi-schema += ident-with-escape.json
+qapi-schema += include-before-err.json
+qapi-schema += include-cycle.json
+qapi-schema += include-format-err.json
+qapi-schema += include-nested-err.json
+qapi-schema += include-no-file.json
+qapi-schema += include-non-file.json
+qapi-schema += include-relpath.json
+qapi-schema += include-repetition.json
+qapi-schema += include-self-cycle.json
+qapi-schema += include-simple.json
+qapi-schema += indented-expr.json
+qapi-schema += leading-comma-list.json
+qapi-schema += leading-comma-object.json
+qapi-schema += missing-colon.json
+qapi-schema += missing-comma-list.json
+qapi-schema += missing-comma-object.json
+qapi-schema += missing-type.json
+qapi-schema += nested-struct-data.json
+qapi-schema += non-objects.json
+qapi-schema += qapi-schema-test.json
+qapi-schema += quoted-structural-chars.json
+qapi-schema += redefined-builtin.json
+qapi-schema += redefined-command.json
+qapi-schema += redefined-event.json
+qapi-schema += redefined-type.json
+qapi-schema += reserved-command-q.json
+qapi-schema += reserved-member-has.json
+qapi-schema += reserved-member-q.json
+qapi-schema += reserved-member-u.json
+qapi-schema += reserved-type-kind.json
+qapi-schema += reserved-type-list.json
+qapi-schema += returns-alternate.json
+qapi-schema += returns-array-bad.json
+qapi-schema += returns-dict.json
+qapi-schema += returns-unknown.json
+qapi-schema += returns-whitelist.json
+qapi-schema += struct-base-clash-deep.json
+qapi-schema += struct-base-clash.json
+qapi-schema += struct-data-invalid.json
+qapi-schema += struct-member-invalid.json
+qapi-schema += trailing-comma-list.json
+qapi-schema += trailing-comma-object.json
+qapi-schema += type-bypass-bad-gen.json
+qapi-schema += unclosed-list.json
+qapi-schema += unclosed-object.json
+qapi-schema += unclosed-string.json
+qapi-schema += unicode-str.json
+qapi-schema += union-bad-branch.json
+qapi-schema += union-base-no-discriminator.json
+qapi-schema += union-clash-branches.json
+qapi-schema += union-clash-data.json
+qapi-schema += union-clash-type.json
+qapi-schema += union-empty.json
+qapi-schema += union-invalid-base.json
+qapi-schema += union-max.json
+qapi-schema += union-optional-branch.json
+qapi-schema += union-unknown.json
+qapi-schema += unknown-escape.json
+qapi-schema += unknown-expr-key.json
+check-qapi-schema-y := $(addprefix tests/qapi-schema/, $(qapi-schema))
 
 GENERATED_HEADERS += tests/test-qapi-types.h tests/test-qapi-visit.h \
-		     tests/test-qmp-commands.h tests/test-qapi-event.h
+	tests/test-qmp-commands.h tests/test-qapi-event.h \
+	tests/test-qmp-introspect.h
 
 test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/check-qlist.o tests/check-qfloat.o tests/check-qjson.o \
@@ -270,47 +369,53 @@ test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/test-opts-visitor.o tests/test-qmp-event.o \
 	tests/rcutorture.o tests/test-rcu-list.o
 
-test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \
-		  tests/test-qapi-event.o
-
 $(test-obj-y): QEMU_INCLUDES += -Itests
 QEMU_CFLAGS += -I$(SRC_PATH)/tests
-qom-core-obj = qom/object.o qom/qom-qobject.o qom/container.o qom/object_interfaces.o
-
-tests/check-qint$(EXESUF): tests/check-qint.o libqemuutil.a
-tests/check-qstring$(EXESUF): tests/check-qstring.o libqemuutil.a
-tests/check-qdict$(EXESUF): tests/check-qdict.o libqemuutil.a
-tests/check-qlist$(EXESUF): tests/check-qlist.o libqemuutil.a
-tests/check-qfloat$(EXESUF): tests/check-qfloat.o libqemuutil.a
-tests/check-qjson$(EXESUF): tests/check-qjson.o libqemuutil.a libqemustub.a
-tests/check-qom-interface$(EXESUF): tests/check-qom-interface.o $(qom-core-obj) libqemuutil.a libqemustub.a
-tests/check-qom-proplist$(EXESUF): tests/check-qom-proplist.o $(qom-core-obj) libqemuutil.a libqemustub.a
-tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-aio$(EXESUF): tests/test-aio.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-rfifolock$(EXESUF): tests/test-rfifolock.o libqemuutil.a libqemustub.a
-tests/test-throttle$(EXESUF): tests/test-throttle.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-iov$(EXESUF): tests/test-iov.o libqemuutil.a
-tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o libqemuutil.a libqemustub.a
+
+
+# Deps that are common to various different sets of tests below
+test-util-obj-y = libqemuutil.a libqemustub.a
+test-qom-obj-y = $(qom-obj-y) $(test-util-obj-y)
+test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \
+	tests/test-qapi-event.o tests/test-qmp-introspect.o \
+	$(test-qom-obj-y)
+test-crypto-obj-y = $(crypto-obj-y) $(test-qom-obj-y)
+test-block-obj-y = $(block-obj-y) $(test-crypto-obj-y)
+
+tests/check-qint$(EXESUF): tests/check-qint.o $(test-util-obj-y)
+tests/check-qstring$(EXESUF): tests/check-qstring.o $(test-util-obj-y)
+tests/check-qdict$(EXESUF): tests/check-qdict.o $(test-util-obj-y)
+tests/check-qlist$(EXESUF): tests/check-qlist.o $(test-util-obj-y)
+tests/check-qfloat$(EXESUF): tests/check-qfloat.o $(test-util-obj-y)
+tests/check-qjson$(EXESUF): tests/check-qjson.o $(test-util-obj-y)
+tests/check-qom-interface$(EXESUF): tests/check-qom-interface.o $(test-qom-obj-y)
+tests/check-qom-proplist$(EXESUF): tests/check-qom-proplist.o $(test-qom-obj-y)
+tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(test-block-obj-y)
+tests/test-aio$(EXESUF): tests/test-aio.o $(test-block-obj-y)
+tests/test-rfifolock$(EXESUF): tests/test-rfifolock.o $(test-util-obj-y)
+tests/test-throttle$(EXESUF): tests/test-throttle.o $(test-block-obj-y)
+tests/test-blockjob-txn$(EXESUF): tests/test-blockjob-txn.o $(test-block-obj-y) $(test-util-obj-y)
+tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(test-block-obj-y)
+tests/test-iov$(EXESUF): tests/test-iov.o $(test-util-obj-y)
+tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o $(test-util-obj-y)
 tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o
-tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o migration/xbzrle.o page_cache.o libqemuutil.a
+tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o migration/xbzrle.o page_cache.o $(test-util-obj-y)
 tests/test-cutils$(EXESUF): tests/test-cutils.o util/cutils.o
 tests/test-int128$(EXESUF): tests/test-int128.o
-tests/rcutorture$(EXESUF): tests/rcutorture.o libqemuutil.a libqemustub.a
-tests/test-rcu-list$(EXESUF): tests/test-rcu-list.o libqemuutil.a libqemustub.a
+tests/rcutorture$(EXESUF): tests/rcutorture.o $(test-util-obj-y)
+tests/test-rcu-list$(EXESUF): tests/test-rcu-list.o $(test-util-obj-y)
 
 tests/test-qdev-global-props$(EXESUF): tests/test-qdev-global-props.o \
 	hw/core/qdev.o hw/core/qdev-properties.o hw/core/hotplug.o\
 	hw/core/irq.o \
 	hw/core/fw-path-provider.o \
-	$(qom-core-obj) \
-	$(test-qapi-obj-y) \
-	libqemuutil.a libqemustub.a
+	$(test-qapi-obj-y)
 tests/test-vmstate$(EXESUF): tests/test-vmstate.o \
 	migration/vmstate.o migration/qemu-file.o migration/qemu-file-buf.o \
         migration/qemu-file-unix.o qjson.o \
-	$(qom-core-obj) \
-	libqemuutil.a libqemustub.a
+	$(test-qom-obj-y)
+tests/test-timed-average$(EXESUF): tests/test-timed-average.o qemu-timer.o \
+	$(test-util-obj-y)
 
 tests/test-qapi-types.c tests/test-qapi-types.h :\
 $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
@@ -332,21 +437,38 @@ $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-eve
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-event.py \
 		$(gen-out-type) -o tests -p "test-" $<, \
 		"  GEN   $@")
+tests/test-qmp-introspect.c tests/test-qmp-introspect.h :\
+$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-introspect.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-introspect.py \
+		$(gen-out-type) -o tests -p "test-" $<, \
+		"  GEN   $@")
 
-tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-event$(EXESUF): tests/test-qmp-event.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-opts-visitor$(EXESUF): tests/test-opts-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-
-tests/test-mul64$(EXESUF): tests/test-mul64.o libqemuutil.a
-tests/test-bitops$(EXESUF): tests/test-bitops.o libqemuutil.a
-tests/test-crypto-hash$(EXESUF): tests/test-crypto-hash.o libqemuutil.a libqemustub.a
-tests/test-crypto-cipher$(EXESUF): tests/test-crypto-cipher.o libqemuutil.a libqemustub.a
+tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y)
+tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-event$(EXESUF): tests/test-qmp-event.o $(test-qapi-obj-y)
+tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y)
+tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y)
+tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y)
+tests/test-opts-visitor$(EXESUF): tests/test-opts-visitor.o $(test-qapi-obj-y)
+
+tests/test-mul64$(EXESUF): tests/test-mul64.o $(test-util-obj-y)
+tests/test-bitops$(EXESUF): tests/test-bitops.o $(test-util-obj-y)
+tests/test-crypto-hash$(EXESUF): tests/test-crypto-hash.o $(test-crypto-obj-y)
+tests/test-crypto-cipher$(EXESUF): tests/test-crypto-cipher.o $(test-crypto-obj-y)
+
+tests/crypto-tls-x509-helpers.o-cflags := $(TASN1_CFLAGS)
+tests/crypto-tls-x509-helpers.o-libs := $(TASN1_LIBS)
+tests/pkix_asn1_tab.o-cflags := $(TASN1_CFLAGS)
+
+tests/test-crypto-tlscredsx509.o-cflags := $(TASN1_CFLAGS)
+tests/test-crypto-tlscredsx509$(EXESUF): tests/test-crypto-tlscredsx509.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o $(test-crypto-obj-y)
+
+tests/test-crypto-tlssession.o-cflags := $(TASN1_CFLAGS)
+tests/test-crypto-tlssession$(EXESUF): tests/test-crypto-tlssession.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o $(test-crypto-obj-y)
 
 libqos-obj-y = tests/libqos/pci.o tests/libqos/fw_cfg.o tests/libqos/malloc.o
 libqos-obj-y += tests/libqos/i2c.o tests/libqos/libqos.o
@@ -354,6 +476,7 @@ libqos-pc-obj-y = $(libqos-obj-y) tests/libqos/pci-pc.o
 libqos-pc-obj-y += tests/libqos/malloc-pc.o tests/libqos/libqos-pc.o
 libqos-pc-obj-y += tests/libqos/ahci.o
 libqos-omap-obj-y = $(libqos-obj-y) tests/libqos/i2c-omap.o
+libqos-imx-obj-y = $(libqos-obj-y) tests/libqos/i2c-imx.o
 libqos-usb-obj-y = $(libqos-pc-obj-y) tests/libqos/usb.o
 libqos-virtio-obj-y = $(libqos-pc-obj-y) tests/libqos/virtio.o tests/libqos/virtio-pci.o tests/libqos/virtio-mmio.o tests/libqos/malloc-generic.o
 
@@ -369,6 +492,7 @@ tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o
 tests/boot-order-test$(EXESUF): tests/boot-order-test.o $(libqos-obj-y)
 tests/bios-tables-test$(EXESUF): tests/bios-tables-test.o $(libqos-obj-y)
 tests/tmp105-test$(EXESUF): tests/tmp105-test.o $(libqos-omap-obj-y)
+tests/ds1338-test$(EXESUF): tests/ds1338-test.o $(libqos-imx-obj-y)
 tests/i440fx-test$(EXESUF): tests/i440fx-test.o $(libqos-pc-obj-y)
 tests/q35-test$(EXESUF): tests/q35-test.o $(libqos-pc-obj-y)
 tests/fw_cfg-test$(EXESUF): tests/fw_cfg-test.o $(libqos-pc-obj-y)
@@ -382,7 +506,7 @@ tests/wdt_ib700-test$(EXESUF): tests/wdt_ib700-test.o
 tests/tco-test$(EXESUF): tests/tco-test.o $(libqos-pc-obj-y)
 tests/virtio-balloon-test$(EXESUF): tests/virtio-balloon-test.o
 tests/virtio-blk-test$(EXESUF): tests/virtio-blk-test.o $(libqos-virtio-obj-y)
-tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o $(libqos-pc-obj-y)
+tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o $(libqos-pc-obj-y) $(libqos-virtio-obj-y)
 tests/virtio-rng-test$(EXESUF): tests/virtio-rng-test.o $(libqos-pc-obj-y)
 tests/virtio-scsi-test$(EXESUF): tests/virtio-scsi-test.o $(libqos-virtio-obj-y)
 tests/virtio-9p-test$(EXESUF): tests/virtio-9p-test.o
@@ -408,8 +532,11 @@ tests/usb-hcd-xhci-test$(EXESUF): tests/usb-hcd-xhci-test.o $(libqos-usb-obj-y)
 tests/pc-cpu-test$(EXESUF): tests/pc-cpu-test.o
 tests/vhost-user-test$(EXESUF): tests/vhost-user-test.o qemu-char.o qemu-timer.o $(qtest-obj-y)
 tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o
-tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o libqemuutil.a libqemustub.a
-tests/test-write-threshold$(EXESUF): tests/test-write-threshold.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o $(test-util-obj-y)
+tests/test-write-threshold$(EXESUF): tests/test-write-threshold.o $(test-block-obj-y)
+tests/test-netfilter$(EXESUF): tests/test-netfilter.o $(qtest-obj-y)
+tests/ivshmem-test$(EXESUF): tests/ivshmem-test.o contrib/ivshmem-server/ivshmem-server.o $(libqos-pc-obj-y)
+tests/vhost-user-bridge$(EXESUF): tests/vhost-user-bridge.o
 
 ifeq ($(CONFIG_POSIX),y)
 LIBS += -lutil
@@ -426,9 +553,11 @@ else
 QTEST_TARGETS =
 endif
 
-qtest-obj-y = tests/libqtest.o libqemuutil.a libqemustub.a
+qtest-obj-y = tests/libqtest.o $(test-util-obj-y)
 $(check-qtest-y): $(qtest-obj-y)
 
+tests/test-qga: tests/test-qga.o $(qtest-obj-y)
+
 .PHONY: check-help
 check-help:
 	@echo "Regression testing targets:"
@@ -483,7 +612,7 @@ $(patsubst %, check-%, $(check-unit-y)): check-%: %
 $(patsubst %, check-report-qtest-%.xml, $(QTEST_TARGETS)): check-report-qtest-%.xml: $(check-qtest-y)
 	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
 		QTEST_QEMU_IMG=qemu-img$(EXESUF) \
-	  gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $(check-qtest-$*-y),"GTESTER $@")
+	  gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $(check-qtest-$*-y) $(check-qtest-generic-y),"GTESTER $@")
 
 check-report-unit.xml: $(check-unit-y)
 	$(call quiet-command,gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $^, "GTESTER $@")
@@ -531,7 +660,7 @@ check: check-qapi-schema check-unit check-qtest
 check-clean:
 	$(MAKE) -C tests/tcg clean
 	rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y)
-	rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)))
+	rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)) $(check-qtest-generic-y))
 
 clean: check-clean
 
diff --git a/tests/ahci-test.c b/tests/ahci-test.c
index 87d769186..088850642 100644
--- a/tests/ahci-test.c
+++ b/tests/ahci-test.c
@@ -39,15 +39,17 @@
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/pci_regs.h"
 
-/* Test-specific defines -- in MiB */
-#define TEST_IMAGE_SIZE_MB (200 * 1024)
-#define TEST_IMAGE_SECTORS ((TEST_IMAGE_SIZE_MB / AHCI_SECTOR_SIZE)     \
-                            * 1024 * 1024)
+/* Test images sizes in MB */
+#define TEST_IMAGE_SIZE_MB_LARGE (200 * 1024)
+#define TEST_IMAGE_SIZE_MB_SMALL 64
 
 /*** Globals ***/
 static char tmp_path[] = "/tmp/qtest.XXXXXX";
 static char debug_path[] = "/tmp/qtest-blkdebug.XXXXXX";
+static char mig_socket[] = "/tmp/qtest-migration.XXXXXX";
 static bool ahci_pedantic;
+static const char *imgfmt;
+static unsigned test_image_size_mb;
 
 /*** Function Declarations ***/
 static void ahci_test_port_spec(AHCIQState *ahci, uint8_t port);
@@ -60,6 +62,11 @@ static void ahci_test_pmcap(AHCIQState *ahci, uint8_t offset);
 
 /*** Utilities ***/
 
+static uint64_t mb_to_sectors(uint64_t image_size_mb)
+{
+    return (image_size_mb * 1024 * 1024) / AHCI_SECTOR_SIZE;
+}
+
 static void string_bswap16(uint16_t *s, size_t bytes)
 {
     g_assert_cmphex((bytes & 1), ==, 0);
@@ -71,32 +78,6 @@ static void string_bswap16(uint16_t *s, size_t bytes)
     }
 }
 
-static void generate_pattern(void *buffer, size_t len, size_t cycle_len)
-{
-    int i, j;
-    unsigned char *tx = (unsigned char *)buffer;
-    unsigned char p;
-    size_t *sx;
-
-    /* Write an indicative pattern that varies and is unique per-cycle */
-    p = rand() % 256;
-    for (i = j = 0; i < len; i++, j++) {
-        tx[i] = p;
-        if (j % cycle_len == 0) {
-            p = rand() % 256;
-        }
-    }
-
-    /* force uniqueness by writing an id per-cycle */
-    for (i = 0; i < len / cycle_len; i++) {
-        j = i * cycle_len;
-        if (j + sizeof(*sx) <= len) {
-            sx = (size_t *)&tx[j];
-            *sx = i;
-        }
-    }
-}
-
 /**
  * Verify that the transfer did not corrupt our state at all.
  */
@@ -140,8 +121,11 @@ static void ahci_migrate(AHCIQState *from, AHCIQState *to, const char *uri)
 {
     QOSState *tmp = to->parent;
     QPCIDevice *dev = to->dev;
+    char *uri_local = NULL;
+
     if (uri == NULL) {
-        uri = "tcp:127.0.0.1:1234";
+        uri_local = g_strdup_printf("%s%s", "unix:", mig_socket);
+        uri = uri_local;
     }
 
     /* context will be 'to' after completion. */
@@ -161,6 +145,7 @@ static void ahci_migrate(AHCIQState *from, AHCIQState *to, const char *uri)
     from->dev = dev;
 
     verify_state(to);
+    g_free(uri_local);
 }
 
 /*** Test Setup & Teardown ***/
@@ -196,11 +181,11 @@ static AHCIQState *ahci_boot(const char *cli, ...)
         va_end(ap);
     } else {
         cli = "-drive if=none,id=drive0,file=%s,cache=writeback,serial=%s"
-            ",format=qcow2"
+            ",format=%s"
             " -M q35 "
             "-device ide-hd,drive=drive0 "
             "-global ide-hd.ver=%s";
-        s = ahci_boot(cli, tmp_path, "testdisk", "version");
+        s = ahci_boot(cli, tmp_path, "testdisk", imgfmt, "version");
     }
 
     return s;
@@ -926,7 +911,7 @@ static void ahci_test_max(AHCIQState *ahci)
     uint64_t nsect;
     uint8_t port;
     uint8_t cmd;
-    uint64_t config_sect = TEST_IMAGE_SECTORS - 1;
+    uint64_t config_sect = mb_to_sectors(test_image_size_mb) - 1;
 
     if (config_sect > 0xFFFFFF) {
         cmd = CMD_READ_MAX_EXT;
@@ -1099,12 +1084,12 @@ static void test_flush_retry(void)
 
     prepare_blkdebug_script(debug_path, "flush_to_disk");
     ahci = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                                "format=qcow2,cache=writeback,"
+                                "format=%s,cache=writeback,"
                                 "rerror=stop,werror=stop "
                                 "-M q35 "
                                 "-device ide-hd,drive=drive0 ",
                                 debug_path,
-                                tmp_path);
+                                tmp_path, imgfmt);
 
     /* Issue Flush Command and wait for error */
     port = ahci_port_select(ahci);
@@ -1131,18 +1116,19 @@ static void test_flush_retry(void)
 static void test_migrate_sanity(void)
 {
     AHCIQState *src, *dst;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     src = ahci_boot("-m 1024 -M q35 "
-                    "-hda %s ", tmp_path);
+                    "-drive if=ide,file=%s,format=%s ", tmp_path, imgfmt);
     dst = ahci_boot("-m 1024 -M q35 "
-                    "-hda %s "
-                    "-incoming %s", tmp_path, uri);
+                    "-drive if=ide,file=%s,format=%s "
+                    "-incoming %s", tmp_path, imgfmt, uri);
 
     ahci_migrate(src, dst, uri);
 
     ahci_shutdown(src);
     ahci_shutdown(dst);
+    g_free(uri);
 }
 
 /**
@@ -1155,14 +1141,14 @@ static void ahci_migrate_simple(uint8_t cmd_read, uint8_t cmd_write)
     size_t bufsize = 4096;
     unsigned char *tx = g_malloc(bufsize);
     unsigned char *rx = g_malloc0(bufsize);
-    unsigned i;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     src = ahci_boot_and_enable("-m 1024 -M q35 "
-                               "-hda %s ", tmp_path);
+                               "-drive if=ide,format=%s,file=%s ",
+                               imgfmt, tmp_path);
     dst = ahci_boot("-m 1024 -M q35 "
-                    "-hda %s "
-                    "-incoming %s", tmp_path, uri);
+                    "-drive if=ide,format=%s,file=%s "
+                    "-incoming %s", imgfmt, tmp_path, uri);
 
     set_context(src->parent);
 
@@ -1171,9 +1157,7 @@ static void ahci_migrate_simple(uint8_t cmd_read, uint8_t cmd_write)
     ahci_port_clear(src, px);
 
     /* create pattern */
-    for (i = 0; i < bufsize; i++) {
-        tx[i] = (bufsize - i);
-    }
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
 
     /* Write, migrate, then read. */
     ahci_io(src, px, cmd_write, tx, bufsize, 0);
@@ -1187,6 +1171,7 @@ static void ahci_migrate_simple(uint8_t cmd_read, uint8_t cmd_write)
     ahci_shutdown(dst);
     g_free(rx);
     g_free(tx);
+    g_free(uri);
 }
 
 static void test_migrate_dma(void)
@@ -1213,29 +1198,25 @@ static void ahci_halted_io_test(uint8_t cmd_read, uint8_t cmd_write)
     size_t bufsize = 4096;
     unsigned char *tx = g_malloc(bufsize);
     unsigned char *rx = g_malloc0(bufsize);
-    unsigned i;
     uint64_t ptr;
     AHCICommand *cmd;
 
     prepare_blkdebug_script(debug_path, "write_aio");
 
     ahci = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                                "format=qcow2,cache=writeback,"
+                                "format=%s,cache=writeback,"
                                 "rerror=stop,werror=stop "
                                 "-M q35 "
                                 "-device ide-hd,drive=drive0 ",
                                 debug_path,
-                                tmp_path);
+                                tmp_path, imgfmt);
 
     /* Initialize and prepare */
     port = ahci_port_select(ahci);
     ahci_port_clear(ahci, port);
 
-    for (i = 0; i < bufsize; i++) {
-        tx[i] = (bufsize - i);
-    }
-
     /* create DMA source buffer and write pattern */
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
     ptr = ahci_alloc(ahci, bufsize);
     g_assert(ptr);
     memwrite(ptr, tx, bufsize);
@@ -1282,38 +1263,34 @@ static void ahci_migrate_halted_io(uint8_t cmd_read, uint8_t cmd_write)
     size_t bufsize = 4096;
     unsigned char *tx = g_malloc(bufsize);
     unsigned char *rx = g_malloc0(bufsize);
-    unsigned i;
     uint64_t ptr;
     AHCICommand *cmd;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     prepare_blkdebug_script(debug_path, "write_aio");
 
     src = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                               "format=qcow2,cache=writeback,"
+                               "format=%s,cache=writeback,"
                                "rerror=stop,werror=stop "
                                "-M q35 "
                                "-device ide-hd,drive=drive0 ",
                                debug_path,
-                               tmp_path);
+                               tmp_path, imgfmt);
 
     dst = ahci_boot("-drive file=%s,if=none,id=drive0,"
-                    "format=qcow2,cache=writeback,"
+                    "format=%s,cache=writeback,"
                     "rerror=stop,werror=stop "
                     "-M q35 "
                     "-device ide-hd,drive=drive0 "
                     "-incoming %s",
-                    tmp_path, uri);
+                    tmp_path, imgfmt, uri);
 
     set_context(src->parent);
 
     /* Initialize and prepare */
     port = ahci_port_select(src);
     ahci_port_clear(src, port);
-
-    for (i = 0; i < bufsize; i++) {
-        tx[i] = (bufsize - i);
-    }
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
 
     /* create DMA source buffer and write pattern */
     ptr = ahci_alloc(src, bufsize);
@@ -1338,6 +1315,7 @@ static void ahci_migrate_halted_io(uint8_t cmd_read, uint8_t cmd_write)
     ahci_shutdown(dst);
     g_free(rx);
     g_free(tx);
+    g_free(uri);
 }
 
 static void test_migrate_halted_dma(void)
@@ -1359,20 +1337,22 @@ static void test_flush_migrate(void)
     AHCICommand *cmd;
     uint8_t px;
     const char *s;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     prepare_blkdebug_script(debug_path, "flush_to_disk");
 
     src = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                               "cache=writeback,rerror=stop,werror=stop "
+                               "cache=writeback,rerror=stop,werror=stop,"
+                               "format=%s "
                                "-M q35 "
                                "-device ide-hd,drive=drive0 ",
-                               debug_path, tmp_path);
+                               debug_path, tmp_path, imgfmt);
     dst = ahci_boot("-drive file=%s,if=none,id=drive0,"
-                    "cache=writeback,rerror=stop,werror=stop "
+                    "cache=writeback,rerror=stop,werror=stop,"
+                    "format=%s "
                     "-M q35 "
                     "-device ide-hd,drive=drive0 "
-                    "-incoming %s", tmp_path, uri);
+                    "-incoming %s", tmp_path, imgfmt, uri);
 
     set_context(src->parent);
 
@@ -1397,6 +1377,7 @@ static void test_flush_migrate(void)
     ahci_command_free(cmd);
     ahci_shutdown(src);
     ahci_shutdown(dst);
+    g_free(uri);
 }
 
 static void test_max(void)
@@ -1513,7 +1494,7 @@ static uint64_t offset_sector(enum OffsetType ofst,
         return 1;
     case OFFSET_HIGH:
         ceil = (addr_type == ADDR_MODE_LBA28) ? 0xfffffff : 0xffffffffffff;
-        ceil = MIN(ceil, TEST_IMAGE_SECTORS - 1);
+        ceil = MIN(ceil, mb_to_sectors(test_image_size_mb) - 1);
         nsectors = buffsize / AHCI_SECTOR_SIZE;
         return ceil - nsectors + 1;
     default:
@@ -1595,8 +1576,9 @@ static void create_ahci_io_test(enum IOMode type, enum AddrMode addr,
                                 enum BuffLen len, enum OffsetType offset)
 {
     char *name;
-    AHCIIOTestOptions *opts = g_malloc(sizeof(AHCIIOTestOptions));
+    AHCIIOTestOptions *opts;
 
+    opts = g_malloc(sizeof(AHCIIOTestOptions));
     opts->length = len;
     opts->address_type = addr;
     opts->io_type = type;
@@ -1608,6 +1590,13 @@ static void create_ahci_io_test(enum IOMode type, enum AddrMode addr,
                            buff_len_str[len],
                            offset_str[offset]);
 
+    if ((addr == ADDR_MODE_LBA48) && (offset == OFFSET_HIGH) &&
+        (mb_to_sectors(test_image_size_mb) <= 0xFFFFFFF)) {
+        g_test_message("%s: skipped; test image too small", name);
+        g_free(name);
+        return;
+    }
+
     qtest_add_data_func(name, opts, test_io_interface);
     g_free(name);
 }
@@ -1654,15 +1643,33 @@ int main(int argc, char **argv)
         return 0;
     }
 
-    /* Create a temporary qcow2 image */
-    close(mkstemp(tmp_path));
-    mkqcow2(tmp_path, TEST_IMAGE_SIZE_MB);
+    /* Create a temporary image */
+    fd = mkstemp(tmp_path);
+    g_assert(fd >= 0);
+    if (have_qemu_img()) {
+        imgfmt = "qcow2";
+        test_image_size_mb = TEST_IMAGE_SIZE_MB_LARGE;
+        mkqcow2(tmp_path, TEST_IMAGE_SIZE_MB_LARGE);
+    } else {
+        g_test_message("QTEST_QEMU_IMG not set or qemu-img missing; "
+                       "skipping LBA48 high-sector tests");
+        imgfmt = "raw";
+        test_image_size_mb = TEST_IMAGE_SIZE_MB_SMALL;
+        ret = ftruncate(fd, test_image_size_mb * 1024 * 1024);
+        g_assert(ret == 0);
+    }
+    close(fd);
 
     /* Create temporary blkdebug instructions */
     fd = mkstemp(debug_path);
     g_assert(fd >= 0);
     close(fd);
 
+    /* Reserve a hollow file to use as a socket for migration tests */
+    fd = mkstemp(mig_socket);
+    g_assert(fd >= 0);
+    close(fd);
+
     /* Run the tests */
     qtest_add_func("/ahci/sanity",     test_sanity);
     qtest_add_func("/ahci/pci_spec",   test_pci_spec);
@@ -1705,6 +1712,7 @@ int main(int argc, char **argv)
     /* Cleanup */
     unlink(tmp_path);
     unlink(debug_path);
+    unlink(mig_socket);
 
     return ret;
 }
diff --git a/tests/bios-tables-test.c b/tests/bios-tables-test.c
index 0de1742d7..6d37332a6 100644
--- a/tests/bios-tables-test.c
+++ b/tests/bios-tables-test.c
@@ -18,7 +18,7 @@
 #include "libqtest.h"
 #include "qemu/compiler.h"
 #include "hw/acpi/acpi-defs.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "qemu/bitmap.h"
 
 #define MACHINE_PC "pc"
@@ -50,7 +50,7 @@ typedef struct {
     int rsdt_tables_nr;
     GArray *tables;
     uint32_t smbios_ep_addr;
-    struct smbios_entry_point smbios_ep_table;
+    struct smbios_21_entry_point smbios_ep_table;
 } test_data;
 
 #define LOW(x) ((x) & 0xff)
@@ -161,31 +161,23 @@ static void free_test_data(test_data *data)
     AcpiSdtTable *temp;
     int i;
 
-    if (data->rsdt_tables_addr) {
-        g_free(data->rsdt_tables_addr);
-    }
+    g_free(data->rsdt_tables_addr);
 
     for (i = 0; i < data->tables->len; ++i) {
         temp = &g_array_index(data->tables, AcpiSdtTable, i);
-        if (temp->aml) {
-            g_free(temp->aml);
-        }
-        if (temp->aml_file) {
-            if (!temp->tmp_files_retain &&
-                g_strstr_len(temp->aml_file, -1, "aml-")) {
-                unlink(temp->aml_file);
-            }
-            g_free(temp->aml_file);
+        g_free(temp->aml);
+        if (temp->aml_file &&
+            !temp->tmp_files_retain &&
+            g_strstr_len(temp->aml_file, -1, "aml-")) {
+            unlink(temp->aml_file);
         }
-        if (temp->asl) {
-            g_free(temp->asl);
-        }
-        if (temp->asl_file) {
-            if (!temp->tmp_files_retain) {
-                unlink(temp->asl_file);
-            }
-            g_free(temp->asl_file);
+        g_free(temp->aml_file);
+        g_free(temp->asl);
+        if (temp->asl_file &&
+            !temp->tmp_files_retain) {
+            unlink(temp->asl_file);
         }
+        g_free(temp->asl_file);
     }
 
     g_array_free(data->tables, false);
@@ -420,9 +412,7 @@ static void dump_aml_files(test_data *data, bool rebuild)
 
         close(fd);
 
-        if (aml_file) {
-            g_free(aml_file);
-        }
+        g_free(aml_file);
     }
 }
 
@@ -601,7 +591,7 @@ static void test_acpi_asl(test_data *data)
 
 static bool smbios_ep_table_ok(test_data *data)
 {
-    struct smbios_entry_point *ep_table = &data->smbios_ep_table;
+    struct smbios_21_entry_point *ep_table = &data->smbios_ep_table;
     uint32_t addr = data->smbios_ep_addr;
 
     ACPI_READ_ARRAY(ep_table->anchor_string, addr);
@@ -681,7 +671,7 @@ static inline bool smbios_single_instance(uint8_t type)
 static void test_smbios_structs(test_data *data)
 {
     DECLARE_BITMAP(struct_bitmap, SMBIOS_MAX_TYPE+1) = { 0 };
-    struct smbios_entry_point *ep_table = &data->smbios_ep_table;
+    struct smbios_21_entry_point *ep_table = &data->smbios_ep_table;
     uint32_t addr = ep_table->structure_table_address;
     int i, len, max_len = 0;
     uint8_t type, prv, crt;
diff --git a/tests/check-qjson.c b/tests/check-qjson.c
index 1cfffa593..61e9bfbab 100644
--- a/tests/check-qjson.c
+++ b/tests/check-qjson.c
@@ -1484,6 +1484,30 @@ static void unterminated_literal(void)
     g_assert(obj == NULL);
 }
 
+static char *make_nest(char *buf, size_t cnt)
+{
+    memset(buf, '[', cnt - 1);
+    buf[cnt - 1] = '{';
+    buf[cnt] = '}';
+    memset(buf + cnt + 1, ']', cnt - 1);
+    buf[2 * cnt] = 0;
+    return buf;
+}
+
+static void limits_nesting(void)
+{
+    enum { max_nesting = 1024 }; /* see qobject/json-streamer.c */
+    char buf[2 * (max_nesting + 1) + 1];
+    QObject *obj;
+
+    obj = qobject_from_json(make_nest(buf, max_nesting));
+    g_assert(obj != NULL);
+    qobject_decref(obj);
+
+    obj = qobject_from_json(make_nest(buf, max_nesting + 1));
+    g_assert(obj == NULL);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
@@ -1519,6 +1543,7 @@ int main(int argc, char **argv)
     g_test_add_func("/errors/invalid_array_comma", invalid_array_comma);
     g_test_add_func("/errors/invalid_dict_comma", invalid_dict_comma);
     g_test_add_func("/errors/unterminated/literal", unterminated_literal);
+    g_test_add_func("/errors/limits/nesting", limits_nesting);
 
     return g_test_run();
 }
diff --git a/tests/check-qom-proplist.c b/tests/check-qom-proplist.c
index 7400b1fce..e674c0fa8 100644
--- a/tests/check-qom-proplist.c
+++ b/tests/check-qom-proplist.c
@@ -152,6 +152,148 @@ static const TypeInfo dummy_info = {
     .class_size = sizeof(DummyObjectClass),
 };
 
+
+/*
+ * The following 3 object classes are used to
+ * simulate the kind of relationships seen in
+ * qdev, which result in complex object
+ * property destruction ordering.
+ *
+ * DummyDev has a 'bus' child to a DummyBus
+ * DummyBus has a 'backend' child to a DummyBackend
+ * DummyDev has a 'backend' link to DummyBackend
+ *
+ * When DummyDev is finalized, it unparents the
+ * DummyBackend, which unparents the DummyDev
+ * which deletes the 'backend' link from DummyDev
+ * to DummyBackend. This illustrates that the
+ * object_property_del_all() method needs to
+ * cope with the list of properties being changed
+ * while it iterates over them.
+ */
+typedef struct DummyDev DummyDev;
+typedef struct DummyDevClass DummyDevClass;
+typedef struct DummyBus DummyBus;
+typedef struct DummyBusClass DummyBusClass;
+typedef struct DummyBackend DummyBackend;
+typedef struct DummyBackendClass DummyBackendClass;
+
+#define TYPE_DUMMY_DEV "qemu-dummy-dev"
+#define TYPE_DUMMY_BUS "qemu-dummy-bus"
+#define TYPE_DUMMY_BACKEND "qemu-dummy-backend"
+
+#define DUMMY_DEV(obj)                               \
+    OBJECT_CHECK(DummyDev, (obj), TYPE_DUMMY_DEV)
+#define DUMMY_BUS(obj)                               \
+    OBJECT_CHECK(DummyBus, (obj), TYPE_DUMMY_BUS)
+#define DUMMY_BACKEND(obj)                               \
+    OBJECT_CHECK(DummyBackend, (obj), TYPE_DUMMY_BACKEND)
+
+struct DummyDev {
+    Object parent_obj;
+
+    DummyBus *bus;
+};
+
+struct DummyDevClass {
+    ObjectClass parent_class;
+};
+
+struct DummyBus {
+    Object parent_obj;
+
+    DummyBackend *backend;
+};
+
+struct DummyBusClass {
+    ObjectClass parent_class;
+};
+
+struct DummyBackend {
+    Object parent_obj;
+};
+
+struct DummyBackendClass {
+    ObjectClass parent_class;
+};
+
+
+static void dummy_dev_init(Object *obj)
+{
+    DummyDev *dev = DUMMY_DEV(obj);
+    DummyBus *bus = DUMMY_BUS(object_new(TYPE_DUMMY_BUS));
+    DummyBackend *backend = DUMMY_BACKEND(object_new(TYPE_DUMMY_BACKEND));
+
+    object_property_add_child(obj, "bus", OBJECT(bus), NULL);
+    dev->bus = bus;
+    object_property_add_child(OBJECT(bus), "backend", OBJECT(backend), NULL);
+    bus->backend = backend;
+
+    object_property_add_link(obj, "backend", TYPE_DUMMY_BACKEND,
+                             (Object **)&bus->backend, NULL, 0, NULL);
+}
+
+static void dummy_dev_unparent(Object *obj)
+{
+    DummyDev *dev = DUMMY_DEV(obj);
+    object_unparent(OBJECT(dev->bus));
+}
+
+static void dummy_dev_class_init(ObjectClass *klass, void *opaque)
+{
+    klass->unparent = dummy_dev_unparent;
+}
+
+
+static void dummy_bus_init(Object *obj)
+{
+}
+
+static void dummy_bus_unparent(Object *obj)
+{
+    DummyBus *bus = DUMMY_BUS(obj);
+    object_property_del(obj->parent, "backend", NULL);
+    object_unparent(OBJECT(bus->backend));
+}
+
+static void dummy_bus_class_init(ObjectClass *klass, void *opaque)
+{
+    klass->unparent = dummy_bus_unparent;
+}
+
+static void dummy_backend_init(Object *obj)
+{
+}
+
+
+static const TypeInfo dummy_dev_info = {
+    .name          = TYPE_DUMMY_DEV,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyDev),
+    .instance_init = dummy_dev_init,
+    .class_size = sizeof(DummyDevClass),
+    .class_init = dummy_dev_class_init,
+};
+
+static const TypeInfo dummy_bus_info = {
+    .name          = TYPE_DUMMY_BUS,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyBus),
+    .instance_init = dummy_bus_init,
+    .class_size = sizeof(DummyBusClass),
+    .class_init = dummy_bus_class_init,
+};
+
+static const TypeInfo dummy_backend_info = {
+    .name          = TYPE_DUMMY_BACKEND,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyBackend),
+    .instance_init = dummy_backend_init,
+    .class_size = sizeof(DummyBackendClass),
+};
+
+
+
 static void test_dummy_createv(void)
 {
     Error *err = NULL;
@@ -283,20 +425,83 @@ static void test_dummy_getenum(void)
                                    &err);
     g_assert(err != NULL);
     error_free(err);
+
+    object_unparent(OBJECT(dobj));
 }
 
 
+static void test_dummy_iterator(void)
+{
+    Object *parent = object_get_objects_root();
+    DummyObject *dobj = DUMMY_OBJECT(
+        object_new_with_props(TYPE_DUMMY,
+                              parent,
+                              "dummy0",
+                              &error_abort,
+                              "bv", "yes",
+                              "sv", "Hiss hiss hiss",
+                              "av", "platypus",
+                              NULL));
+
+    ObjectProperty *prop;
+    ObjectPropertyIterator *iter;
+    bool seenbv = false, seensv = false, seenav = false, seentype;
+
+    iter = object_property_iter_init(OBJECT(dobj));
+    while ((prop = object_property_iter_next(iter))) {
+        if (g_str_equal(prop->name, "bv")) {
+            seenbv = true;
+        } else if (g_str_equal(prop->name, "sv")) {
+            seensv = true;
+        } else if (g_str_equal(prop->name, "av")) {
+            seenav = true;
+        } else if (g_str_equal(prop->name, "type")) {
+            /* This prop comes from the base Object class */
+            seentype = true;
+        } else {
+            g_printerr("Found prop '%s'\n", prop->name);
+            g_assert_not_reached();
+        }
+    }
+    object_property_iter_free(iter);
+    g_assert(seenbv);
+    g_assert(seenav);
+    g_assert(seensv);
+    g_assert(seentype);
+
+    object_unparent(OBJECT(dobj));
+}
+
+
+static void test_dummy_delchild(void)
+{
+    Object *parent = object_get_objects_root();
+    DummyDev *dev = DUMMY_DEV(
+        object_new_with_props(TYPE_DUMMY_DEV,
+                              parent,
+                              "dev0",
+                              &error_abort,
+                              NULL));
+
+    object_unparent(OBJECT(dev));
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
 
     module_call_init(MODULE_INIT_QOM);
     type_register_static(&dummy_info);
+    type_register_static(&dummy_dev_info);
+    type_register_static(&dummy_bus_info);
+    type_register_static(&dummy_backend_info);
 
     g_test_add_func("/qom/proplist/createlist", test_dummy_createlist);
     g_test_add_func("/qom/proplist/createv", test_dummy_createv);
     g_test_add_func("/qom/proplist/badenum", test_dummy_badenum);
     g_test_add_func("/qom/proplist/getenum", test_dummy_getenum);
+    g_test_add_func("/qom/proplist/iterator", test_dummy_iterator);
+    g_test_add_func("/qom/proplist/delchild", test_dummy_delchild);
 
     return g_test_run();
 }
diff --git a/tests/crypto-tls-x509-helpers.c b/tests/crypto-tls-x509-helpers.c
new file mode 100644
index 000000000..47b4c7ba5
--- /dev/null
+++ b/tests/crypto-tls-x509-helpers.c
@@ -0,0 +1,487 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include <stdlib.h>
+#include <fcntl.h>
+
+#include "config-host.h"
+#include "crypto-tls-x509-helpers.h"
+#include "qemu/sockets.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+/*
+ * This stores some static data that is needed when
+ * encoding extensions in the x509 certs
+ */
+ASN1_TYPE pkix_asn1;
+
+/*
+ * To avoid consuming random entropy to generate keys,
+ * here's one we prepared earlier :-)
+ */
+gnutls_x509_privkey_t privkey;
+# define PRIVATE_KEY                                              \
+    "-----BEGIN PRIVATE KEY-----\n"                               \
+    "MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBALVcr\n"     \
+    "BL40Tm6yq88FBhJNw1aaoCjmtg0l4dWQZ/e9Fimx4ARxFpT+ji4FE\n"     \
+    "Cgl9s/SGqC+1nvlkm9ViSo0j7MKDbnDB+VRHDvMAzQhA2X7e8M0n9\n"     \
+    "rPolUY2lIVC83q0BBaOBkCj2RSmT2xTEbbC2xLukSrg2WP/ihVOxc\n"     \
+    "kXRuyFtzAgMBAAECgYB7slBexDwXrtItAMIH6m/U+LUpNe0Xx48OL\n"     \
+    "IOn4a4whNgO/o84uIwygUK27ZGFZT0kAGAk8CdF9hA6ArcbQ62s1H\n"     \
+    "myxrUbF9/mrLsQw1NEqpuUk9Ay2Tx5U/wPx35S3W/X2AvR/ZpTnCn\n"     \
+    "2q/7ym9fyiSoj86drD7BTvmKXlOnOwQJBAPOFMp4mMa9NGpGuEssO\n"     \
+    "m3Uwbp6lhcP0cA9MK+iOmeANpoKWfBdk5O34VbmeXnGYWEkrnX+9J\n"     \
+    "bM4wVhnnBWtgBMCQQC+qAEmvwcfhauERKYznMVUVksyeuhxhCe7EK\n"     \
+    "mPh+U2+g0WwdKvGDgO0PPt1gq0ILEjspMDeMHVdTwkaVBo/uMhAkA\n"     \
+    "Z5SsZyCP2aTOPFDypXRdI4eqRcjaEPOUBq27r3uYb/jeboVb2weLa\n"     \
+    "L1MmVuHiIHoa5clswPdWVI2y0em2IGoDAkBPSp/v9VKJEZabk9Frd\n"     \
+    "a+7u4fanrM9QrEjY3KhduslSilXZZSxrWjjAJPyPiqFb3M8XXA26W\n"     \
+    "nz1KYGnqYKhLcBAkB7dt57n9xfrhDpuyVEv+Uv1D3VVAhZlsaZ5Pp\n"     \
+    "dcrhrkJn2sa/+O8OKvdrPSeeu/N5WwYhJf61+CPoenMp7IFci\n"         \
+    "-----END PRIVATE KEY-----\n"
+
+/*
+ * This loads the private key we defined earlier
+ */
+static gnutls_x509_privkey_t test_tls_load_key(void)
+{
+    gnutls_x509_privkey_t key;
+    const gnutls_datum_t data = { (unsigned char *)PRIVATE_KEY,
+                                  strlen(PRIVATE_KEY) };
+    int err;
+
+    err = gnutls_x509_privkey_init(&key);
+    if (err < 0) {
+        g_critical("Failed to init key %s", gnutls_strerror(err));
+        abort();
+    }
+
+    err = gnutls_x509_privkey_import(key, &data,
+                                     GNUTLS_X509_FMT_PEM);
+    if (err < 0) {
+        if (err != GNUTLS_E_BASE64_UNEXPECTED_HEADER_ERROR &&
+            err != GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+            g_critical("Failed to import key %s", gnutls_strerror(err));
+            abort();
+        }
+
+        err = gnutls_x509_privkey_import_pkcs8(
+            key, &data, GNUTLS_X509_FMT_PEM, NULL, 0);
+        if (err < 0) {
+            g_critical("Failed to import PKCS8 key %s", gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    return key;
+}
+
+
+void test_tls_init(const char *keyfile)
+{
+    gnutls_global_init();
+
+    if (asn1_array2tree(pkix_asn1_tab, &pkix_asn1, NULL) != ASN1_SUCCESS) {
+        abort();
+    }
+
+    privkey = test_tls_load_key();
+    if (!g_file_set_contents(keyfile, PRIVATE_KEY, -1, NULL)) {
+        abort();
+    }
+}
+
+
+void test_tls_cleanup(const char *keyfile)
+{
+    asn1_delete_structure(&pkix_asn1);
+    unlink(keyfile);
+}
+
+/*
+ * Turns an ASN1 object into a DER encoded byte array
+ */
+static void test_tls_der_encode(ASN1_TYPE src,
+                                const char *src_name,
+                                gnutls_datum_t *res)
+{
+  int size;
+  char *data = NULL;
+
+  size = 0;
+  asn1_der_coding(src, src_name, NULL, &size, NULL);
+
+  data = g_new0(char, size);
+
+  asn1_der_coding(src, src_name, data, &size, NULL);
+
+  res->data = (unsigned char *)data;
+  res->size = size;
+}
+
+
+static void
+test_tls_get_ipaddr(const char *addrstr,
+                    char **data,
+                    int *datalen)
+{
+    struct addrinfo *res;
+    struct addrinfo hints;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_flags = AI_NUMERICHOST;
+    g_assert(getaddrinfo(addrstr, NULL, &hints, &res) == 0);
+
+    *datalen = res->ai_addrlen;
+    *data = g_new(char, *datalen);
+    memcpy(*data, res->ai_addr, *datalen);
+    freeaddrinfo(res);
+}
+
+/*
+ * This is a fairly lame x509 certificate generator.
+ *
+ * Do not copy/use this code for generating real certificates
+ * since it leaves out many things that you would want in
+ * certificates for real world usage.
+ *
+ * This is good enough only for doing tests of the QEMU
+ * TLS certificate code
+ */
+void
+test_tls_generate_cert(QCryptoTLSTestCertReq *req,
+                       gnutls_x509_crt_t ca)
+{
+    gnutls_x509_crt_t crt;
+    int err;
+    static char buffer[1024 * 1024];
+    size_t size = sizeof(buffer);
+    char serial[5] = { 1, 2, 3, 4, 0 };
+    gnutls_datum_t der;
+    time_t start = time(NULL) + (60 * 60 * req->start_offset);
+    time_t expire = time(NULL) + (60 * 60 * (req->expire_offset
+                                             ? req->expire_offset : 24));
+
+    /*
+     * Prepare our new certificate object
+     */
+    err = gnutls_x509_crt_init(&crt);
+    if (err < 0) {
+        g_critical("Failed to initialize certificate %s", gnutls_strerror(err));
+        abort();
+    }
+    err = gnutls_x509_crt_set_key(crt, privkey);
+    if (err < 0) {
+        g_critical("Failed to set certificate key %s", gnutls_strerror(err));
+        abort();
+    }
+
+    /*
+     * A v3 certificate is required in order to be able
+     * set any of the basic constraints, key purpose and
+     * key usage data
+     */
+    gnutls_x509_crt_set_version(crt, 3);
+
+    if (req->country) {
+        err = gnutls_x509_crt_set_dn_by_oid(
+            crt, GNUTLS_OID_X520_COUNTRY_NAME, 0,
+            req->country, strlen(req->country));
+        if (err < 0) {
+            g_critical("Failed to set certificate country name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+    if (req->cn) {
+        err = gnutls_x509_crt_set_dn_by_oid(
+            crt, GNUTLS_OID_X520_COMMON_NAME, 0,
+            req->cn, strlen(req->cn));
+        if (err < 0) {
+            g_critical("Failed to set certificate common name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    /*
+     * Setup the subject altnames, which are used
+     * for hostname checks in live sessions
+     */
+    if (req->altname1) {
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_DNSNAME,
+            req->altname1,
+            strlen(req->altname1),
+            GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+    if (req->altname2) {
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_DNSNAME,
+            req->altname2,
+            strlen(req->altname2),
+            GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate %s alt name",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    /*
+     * IP address need to be put into the cert in their
+     * raw byte form, not strings, hence this is a little
+     * more complicated
+     */
+    if (req->ipaddr1) {
+        char *data;
+        int len;
+
+        test_tls_get_ipaddr(req->ipaddr1, &data, &len);
+
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_IPADDRESS,
+            data, len, GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+        g_free(data);
+    }
+    if (req->ipaddr2) {
+        char *data;
+        int len;
+
+        test_tls_get_ipaddr(req->ipaddr2, &data, &len);
+
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_IPADDRESS,
+            data, len, GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+        g_free(data);
+    }
+
+
+    /*
+     * Basic constraints are used to decide if the cert
+     * is for a CA or not. We can't use the convenient
+     * gnutls API for setting this, since it hardcodes
+     * the 'critical' field which we want control over
+     */
+    if (req->basicConstraintsEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+
+        asn1_create_element(pkix_asn1, "PKIX1.BasicConstraints", &ext);
+        asn1_write_value(ext, "cA",
+                         req->basicConstraintsIsCA ? "TRUE" : "FALSE", 1);
+        asn1_write_value(ext, "pathLenConstraint", NULL, 0);
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.19",
+            der.data, der.size,
+            req->basicConstraintsCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate basic constraints %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Next up the key usage extension. Again we can't
+     * use the gnutls API since it hardcodes the extension
+     * to be 'critical'
+     */
+    if (req->keyUsageEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+        char str[2];
+
+        str[0] = req->keyUsageValue & 0xff;
+        str[1] = (req->keyUsageValue >> 8) & 0xff;
+
+        asn1_create_element(pkix_asn1, "PKIX1.KeyUsage", &ext);
+        asn1_write_value(ext, "", str, 9);
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.15",
+            der.data, der.size,
+            req->keyUsageCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate key usage %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Finally the key purpose extension. This time
+     * gnutls has the opposite problem, always hardcoding
+     * it to be non-critical. So once again we have to
+     * set this the hard way building up ASN1 data ourselves
+     */
+    if (req->keyPurposeEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+
+        asn1_create_element(pkix_asn1, "PKIX1.ExtKeyUsageSyntax", &ext);
+        if (req->keyPurposeOID1) {
+            asn1_write_value(ext, "", "NEW", 1);
+            asn1_write_value(ext, "?LAST", req->keyPurposeOID1, 1);
+        }
+        if (req->keyPurposeOID2) {
+            asn1_write_value(ext, "", "NEW", 1);
+            asn1_write_value(ext, "?LAST", req->keyPurposeOID2, 1);
+        }
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.37",
+            der.data, der.size,
+            req->keyPurposeCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate key purpose %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Any old serial number will do, so lets pick 5
+     */
+    err = gnutls_x509_crt_set_serial(crt, serial, 5);
+    if (err < 0) {
+        g_critical("Failed to set certificate serial %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+    err = gnutls_x509_crt_set_activation_time(crt, start);
+    if (err < 0) {
+        g_critical("Failed to set certificate activation %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+    err = gnutls_x509_crt_set_expiration_time(crt, expire);
+    if (err < 0) {
+        g_critical("Failed to set certificate expiration %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+
+    /*
+     * If no 'ca' is set then we are self signing
+     * the cert. This is done for the root CA certs
+     */
+    err = gnutls_x509_crt_sign(crt, ca ? ca : crt, privkey);
+    if (err < 0) {
+        g_critical("Failed to sign certificate %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+    /*
+     * Finally write the new cert out to disk
+     */
+    err = gnutls_x509_crt_export(
+        crt, GNUTLS_X509_FMT_PEM, buffer, &size);
+    if (err < 0) {
+        g_critical("Failed to export certificate %s: %d",
+                   gnutls_strerror(err), err);
+        abort();
+    }
+
+    if (!g_file_set_contents(req->filename, buffer, -1, NULL)) {
+        g_critical("Failed to write certificate %s",
+                   req->filename);
+        abort();
+    }
+
+    req->crt = crt;
+}
+
+
+void test_tls_write_cert_chain(const char *filename,
+                               gnutls_x509_crt_t *certs,
+                               size_t ncerts)
+{
+    size_t i;
+    size_t capacity = 1024, offset = 0;
+    char *buffer = g_new0(char, capacity);
+    int err;
+
+    for (i = 0; i < ncerts; i++) {
+        size_t len = capacity - offset;
+    retry:
+        err = gnutls_x509_crt_export(certs[i], GNUTLS_X509_FMT_PEM,
+                                     buffer + offset, &len);
+        if (err < 0) {
+            if (err == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+                buffer = g_renew(char, buffer, offset + len);
+                capacity = offset + len;
+                goto retry;
+            }
+            g_critical("Failed to export certificate chain %s: %d",
+                       gnutls_strerror(err), err);
+            abort();
+        }
+        offset += len;
+    }
+
+    if (!g_file_set_contents(filename, buffer, offset, NULL)) {
+        abort();
+    }
+    g_free(buffer);
+}
+
+
+void test_tls_discard_cert(QCryptoTLSTestCertReq *req)
+{
+    if (!req->crt) {
+        return;
+    }
+
+    gnutls_x509_crt_deinit(req->crt);
+    req->crt = NULL;
+
+    if (getenv("QEMU_TEST_DEBUG_CERTS") == NULL) {
+        unlink(req->filename);
+    }
+}
+
+#endif /* QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/crypto-tls-x509-helpers.h b/tests/crypto-tls-x509-helpers.h
new file mode 100644
index 000000000..356b49cd5
--- /dev/null
+++ b/tests/crypto-tls-x509-helpers.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
+#if !(defined WIN32) && \
+    defined(CONFIG_TASN1) && \
+    defined(LIBGNUTLS_VERSION_NUMBER) && \
+    (LIBGNUTLS_VERSION_NUMBER >= 0x020600)
+# define QCRYPTO_HAVE_TLS_TEST_SUPPORT
+#endif
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+# include <libtasn1.h>
+
+# include "qemu-common.h"
+
+/*
+ * This contains parameter about how to generate
+ * certificates.
+ */
+typedef struct QCryptoTLSTestCertReq QCryptoTLSTestCertReq;
+struct QCryptoTLSTestCertReq {
+    gnutls_x509_crt_t crt;
+
+    const char *filename;
+
+    /* Identifying information */
+    const char *country;
+    const char *cn;
+    const char *altname1;
+    const char *altname2;
+    const char *ipaddr1;
+    const char *ipaddr2;
+
+    /* Basic constraints */
+    bool basicConstraintsEnable;
+    bool basicConstraintsCritical;
+    bool basicConstraintsIsCA;
+
+    /* Key usage */
+    bool keyUsageEnable;
+    bool keyUsageCritical;
+    int keyUsageValue;
+
+    /* Key purpose (aka Extended key usage) */
+    bool keyPurposeEnable;
+    bool keyPurposeCritical;
+    const char *keyPurposeOID1;
+    const char *keyPurposeOID2;
+
+    /* zero for current time, or non-zero for hours from now */
+    int start_offset;
+    /* zero for 24 hours from now, or non-zero for hours from now */
+    int expire_offset;
+};
+
+void test_tls_generate_cert(QCryptoTLSTestCertReq *req,
+                            gnutls_x509_crt_t ca);
+void test_tls_write_cert_chain(const char *filename,
+                               gnutls_x509_crt_t *certs,
+                               size_t ncerts);
+void test_tls_discard_cert(QCryptoTLSTestCertReq *req);
+
+void test_tls_init(const char *keyfile);
+void test_tls_cleanup(const char *keyfile);
+
+# define TLS_CERT_REQ(varname, cavarname,                               \
+                      country, commonname,                              \
+                      altname1, altname2,                               \
+                      ipaddr1, ipaddr2,                                 \
+                      basicconsenable, basicconscritical, basicconsca,  \
+                      keyusageenable, keyusagecritical, keyusagevalue,  \
+                      keypurposeenable, keypurposecritical,             \
+                      keypurposeoid1, keypurposeoid2,                   \
+                      startoffset, endoffset)                           \
+    static QCryptoTLSTestCertReq varname = {                            \
+        NULL, WORKDIR #varname "-ctx.pem",                              \
+        country, commonname, altname1, altname2,                        \
+        ipaddr1, ipaddr2,                                               \
+        basicconsenable, basicconscritical, basicconsca,                \
+        keyusageenable, keyusagecritical, keyusagevalue,                \
+        keypurposeenable, keypurposecritical,                           \
+        keypurposeoid1, keypurposeoid2,                                 \
+        startoffset, endoffset                                          \
+    };                                                                  \
+    test_tls_generate_cert(&varname, cavarname.crt)
+
+# define TLS_ROOT_REQ(varname,                                          \
+                      country, commonname,                              \
+                      altname1, altname2,                               \
+                      ipaddr1, ipaddr2,                                 \
+                      basicconsenable, basicconscritical, basicconsca,  \
+                      keyusageenable, keyusagecritical, keyusagevalue,  \
+                      keypurposeenable, keypurposecritical,             \
+                      keypurposeoid1, keypurposeoid2,                   \
+                      startoffset, endoffset)                           \
+    static QCryptoTLSTestCertReq varname = {                            \
+        NULL, WORKDIR #varname "-ctx.pem",                              \
+        country, commonname, altname1, altname2,                        \
+        ipaddr1, ipaddr2,                                               \
+        basicconsenable, basicconscritical, basicconsca,                \
+        keyusageenable, keyusagecritical, keyusagevalue,                \
+        keypurposeenable, keypurposecritical,                           \
+        keypurposeoid1, keypurposeoid2,                                 \
+        startoffset, endoffset                                          \
+    };                                                                  \
+    test_tls_generate_cert(&varname, NULL)
+
+extern const ASN1_ARRAY_TYPE pkix_asn1_tab[];
+
+#endif /* QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/device-introspect-test.c b/tests/device-introspect-test.c
index f240b5c82..11d5fea3e 100644
--- a/tests/device-introspect-test.c
+++ b/tests/device-introspect-test.c
@@ -45,17 +45,10 @@ static void test_one_device(const char *type)
     QDict *resp;
     char *help, *qom_tree;
 
-    /*
-     * Skip this part for the abstract device test case, because
-     * device-list-properties crashes for such devices.
-     * FIXME fix it not to crash
-     */
-    if (strcmp(type, "device")) {
-        resp = qmp("{'execute': 'device-list-properties',"
-                   " 'arguments': {'typename': %s}}",
-                   type);
-        QDECREF(resp);
-    }
+    resp = qmp("{'execute': 'device-list-properties',"
+               " 'arguments': {'typename': %s}}",
+               type);
+    QDECREF(resp);
 
     help = hmp("device_add \"%s,help\"", type);
     g_free(help);
@@ -98,30 +91,6 @@ static void test_device_intro_abstract(void)
     qtest_end();
 }
 
-static bool blacklisted(const char *type)
-{
-    static const char *blacklist[] = {
-        /* hang in object_unref(): */
-        "realview_pci", "versatile_pci",
-        /* create a CPU, thus use after free (see below): */
-        "allwinner-a10", "digic", "fsl,imx25", "fsl,imx31", "xlnx,zynqmp",
-    };
-    size_t len = strlen(type);
-    int i;
-
-    if (len >= 4 && !strcmp(type + len - 4, "-cpu")) {
-        /* use after free: cpu_exec_init() saves CPUState in cpus */
-        return true;
-    }
-
-    for (i = 0; i < ARRAY_SIZE(blacklist); i++) {
-        if (!strcmp(blacklist[i], type)) {
-            return true;
-        }
-    }
-    return false;
-}
-
 static void test_device_intro_concrete(void)
 {
     QList *types;
@@ -135,9 +104,6 @@ static void test_device_intro_concrete(void)
         type = qdict_get_try_str(qobject_to_qdict(qlist_entry_obj(entry)),
                                 "name");
         g_assert(type);
-        if (blacklisted(type)) {
-            continue;           /* FIXME broken device, skip */
-        }
         test_one_device(type);
     }
 
diff --git a/tests/ds1338-test.c b/tests/ds1338-test.c
new file mode 100644
index 000000000..7d513d897
--- /dev/null
+++ b/tests/ds1338-test.c
@@ -0,0 +1,78 @@
+/*
+ * QTest testcase for the DS1338 RTC
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "libqtest.h"
+#include "libqos/i2c.h"
+
+#include <glib.h>
+
+#define IMX25_I2C_0_BASE 0x43F80000
+
+#define DS1338_ADDR 0x68
+
+static I2CAdapter *i2c;
+static uint8_t addr;
+
+static inline uint8_t bcd2bin(uint8_t x)
+{
+    return ((x) & 0x0f) + ((x) >> 4) * 10;
+}
+
+static void send_and_receive(void)
+{
+    uint8_t cmd[1];
+    uint8_t resp[7];
+    time_t now = time(NULL);
+    struct tm *tm_ptr = gmtime(&now);
+
+    /* reset the index in the RTC memory */
+    cmd[0] = 0;
+    i2c_send(i2c, addr, cmd, 1);
+
+    /* retrieve the date */
+    i2c_recv(i2c, addr, resp, 7);
+
+    /* check retrieved time againt local time */
+    g_assert_cmpuint(bcd2bin(resp[4]), == , tm_ptr->tm_mday);
+    g_assert_cmpuint(bcd2bin(resp[5]), == , 1 + tm_ptr->tm_mon);
+    g_assert_cmpuint(2000 + bcd2bin(resp[6]), == , 1900 + tm_ptr->tm_year);
+}
+
+int main(int argc, char **argv)
+{
+    QTestState *s = NULL;
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    s = qtest_start("-display none -machine imx25-pdk");
+    i2c = imx_i2c_create(IMX25_I2C_0_BASE);
+    addr = DS1338_ADDR;
+
+    qtest_add_func("/ds1338/tx-rx", send_and_receive);
+
+    ret = g_test_run();
+
+    if (s) {
+        qtest_quit(s);
+    }
+    g_free(i2c);
+
+    return ret;
+}
diff --git a/tests/fdc-test.c b/tests/fdc-test.c
index 416394fc7..b5a4696d8 100644
--- a/tests/fdc-test.c
+++ b/tests/fdc-test.c
@@ -304,9 +304,7 @@ static void test_media_insert(void)
     qmp_discard_response("{'execute':'change', 'arguments':{"
                          " 'device':'floppy0', 'target': %s, 'arg': 'raw' }}",
                          test_image);
-    qmp_discard_response(""); /* ignore event
-                                 (FIXME open -> open transition?!) */
-    qmp_discard_response(""); /* ignore event */
+    qmp_discard_response(""); /* ignore event (open -> close) */
 
     dir = inb(FLOPPY_BASE + reg_dir);
     assert_bit_set(dir, DSKCHG);
diff --git a/tests/fw_cfg-test.c b/tests/fw_cfg-test.c
index 9be78e956..b7d4007e3 100644
--- a/tests/fw_cfg-test.c
+++ b/tests/fw_cfg-test.c
@@ -37,7 +37,9 @@ static void test_fw_cfg_signature(void)
 
 static void test_fw_cfg_id(void)
 {
-    g_assert_cmpint(qfw_cfg_get_u32(fw_cfg, FW_CFG_ID), ==, 1);
+    uint32_t id = qfw_cfg_get_u32(fw_cfg, FW_CFG_ID);
+    g_assert((id == 1) ||
+             (id == 3));
 }
 
 static void test_fw_cfg_uuid(void)
diff --git a/tests/i440fx-test.c b/tests/i440fx-test.c
index d0bc8de25..7fa170990 100644
--- a/tests/i440fx-test.c
+++ b/tests/i440fx-test.c
@@ -191,7 +191,7 @@ static void write_area(uint32_t start, uint32_t end, uint8_t value)
     uint32_t size = end - start + 1;
     uint8_t *data;
 
-    data = g_malloc0(size);
+    data = g_malloc(size);
     memset(data, value, size);
     memwrite(start, data, size);
 
diff --git a/tests/ide-test.c b/tests/ide-test.c
index ef0a47343..b86470135 100644
--- a/tests/ide-test.c
+++ b/tests/ide-test.c
@@ -45,8 +45,15 @@
 #define IDE_BASE 0x1f0
 #define IDE_PRIMARY_IRQ 14
 
+#define ATAPI_BLOCK_SIZE 2048
+
+/* How many bytes to receive via ATAPI PIO at one time.
+ * Must be less than 0xFFFF. */
+#define BYTE_COUNT_LIMIT 5120
+
 enum {
     reg_data        = 0x0,
+    reg_feature     = 0x1,
     reg_nsectors    = 0x2,
     reg_lba_low     = 0x3,
     reg_lba_middle  = 0x4,
@@ -80,6 +87,7 @@ enum {
     CMD_WRITE_DMA   = 0xca,
     CMD_FLUSH_CACHE = 0xe7,
     CMD_IDENTIFY    = 0xec,
+    CMD_PACKET      = 0xa0,
 
     CMDF_ABORT      = 0x100,
     CMDF_NO_BM      = 0x200,
@@ -172,7 +180,8 @@ typedef struct PrdtEntry {
 #define assert_bit_clear(data, mask) g_assert_cmphex((data) & (mask), ==, 0)
 
 static int send_dma_request(int cmd, uint64_t sector, int nb_sectors,
-                            PrdtEntry *prdt, int prdt_entries)
+                            PrdtEntry *prdt, int prdt_entries,
+                            void(*post_exec)(uint64_t sector, int nb_sectors))
 {
     QPCIDevice *dev;
     uint16_t bmdma_base;
@@ -189,6 +198,9 @@ static int send_dma_request(int cmd, uint64_t sector, int nb_sectors,
 
     switch (cmd) {
     case CMD_READ_DMA:
+    case CMD_PACKET:
+        /* Assuming we only test data reads w/ ATAPI, otherwise we need to know
+         * the SCSI command being sent in the packet, too. */
         from_dev = true;
         break;
     case CMD_WRITE_DMA:
@@ -217,14 +229,22 @@ static int send_dma_request(int cmd, uint64_t sector, int nb_sectors,
     outl(bmdma_base + bmreg_prdt, guest_prdt);
 
     /* ATA DMA command */
-    outb(IDE_BASE + reg_nsectors, nb_sectors);
-
-    outb(IDE_BASE + reg_lba_low,    sector & 0xff);
-    outb(IDE_BASE + reg_lba_middle, (sector >> 8) & 0xff);
-    outb(IDE_BASE + reg_lba_high,   (sector >> 16) & 0xff);
+    if (cmd == CMD_PACKET) {
+        /* Enables ATAPI DMA; otherwise PIO is attempted */
+        outb(IDE_BASE + reg_feature, 0x01);
+    } else {
+        outb(IDE_BASE + reg_nsectors, nb_sectors);
+        outb(IDE_BASE + reg_lba_low,    sector & 0xff);
+        outb(IDE_BASE + reg_lba_middle, (sector >> 8) & 0xff);
+        outb(IDE_BASE + reg_lba_high,   (sector >> 16) & 0xff);
+    }
 
     outb(IDE_BASE + reg_command, cmd);
 
+    if (post_exec) {
+        post_exec(sector, nb_sectors);
+    }
+
     /* Start DMA transfer */
     outb(bmdma_base + bmreg_cmd, BM_CMD_START | (from_dev ? BM_CMD_WRITE : 0));
 
@@ -278,7 +298,8 @@ static void test_bmdma_simple_rw(void)
     memset(buf, 0x55, len);
     memwrite(guest_buf, buf, len);
 
-    status = send_dma_request(CMD_WRITE_DMA, 0, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_WRITE_DMA, 0, 1, prdt,
+                              ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -286,14 +307,15 @@ static void test_bmdma_simple_rw(void)
     memset(buf, 0xaa, len);
     memwrite(guest_buf, buf, len);
 
-    status = send_dma_request(CMD_WRITE_DMA, 1, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_WRITE_DMA, 1, 1, prdt,
+                              ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Read and verify 0x55 pattern in sector 0 */
     memset(cmpbuf, 0x55, len);
 
-    status = send_dma_request(CMD_READ_DMA, 0, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_READ_DMA, 0, 1, prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -303,7 +325,7 @@ static void test_bmdma_simple_rw(void)
     /* Read and verify 0xaa pattern in sector 1 */
     memset(cmpbuf, 0xaa, len);
 
-    status = send_dma_request(CMD_READ_DMA, 1, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_READ_DMA, 1, 1, prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -328,13 +350,13 @@ static void test_bmdma_short_prdt(void)
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -353,13 +375,13 @@ static void test_bmdma_one_sector_short_prdt(void)
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 2,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 2,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -377,13 +399,13 @@ static void test_bmdma_long_prdt(void)
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_ACTIVE | BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -399,7 +421,7 @@ static void test_bmdma_no_busmaster(void)
     PrdtEntry prdt[4096] = { };
 
     status = send_dma_request(CMD_READ_DMA | CMDF_NO_BM, 0, 512,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
 
     /* Not entirely clear what the expected result is, but this is what we get
      * in practice. At least we want to be aware of any changes. */
@@ -571,16 +593,215 @@ static void test_flush_nodev(void)
     ide_test_quit();
 }
 
-static void test_pci_retry_flush(const char *machine)
+static void test_pci_retry_flush(void)
 {
     test_retry_flush("pc");
 }
 
-static void test_isa_retry_flush(const char *machine)
+static void test_isa_retry_flush(void)
 {
     test_retry_flush("isapc");
 }
 
+typedef struct Read10CDB {
+    uint8_t opcode;
+    uint8_t flags;
+    uint32_t lba;
+    uint8_t reserved;
+    uint16_t nblocks;
+    uint8_t control;
+    uint16_t padding;
+} __attribute__((__packed__)) Read10CDB;
+
+static void send_scsi_cdb_read10(uint64_t lba, int nblocks)
+{
+    Read10CDB pkt = { .padding = 0 };
+    int i;
+
+    g_assert_cmpint(lba, <=, UINT32_MAX);
+    g_assert_cmpint(nblocks, <=, UINT16_MAX);
+    g_assert_cmpint(nblocks, >=, 0);
+
+    /* Construct SCSI CDB packet */
+    pkt.opcode = 0x28;
+    pkt.lba = cpu_to_be32(lba);
+    pkt.nblocks = cpu_to_be16(nblocks);
+
+    /* Send Packet */
+    for (i = 0; i < sizeof(Read10CDB)/2; i++) {
+        outw(IDE_BASE + reg_data, cpu_to_le16(((uint16_t *)&pkt)[i]));
+    }
+}
+
+static void nsleep(int64_t nsecs)
+{
+    const struct timespec val = { .tv_nsec = nsecs };
+    nanosleep(&val, NULL);
+    clock_set(nsecs);
+}
+
+static uint8_t ide_wait_clear(uint8_t flag)
+{
+    uint8_t data;
+    time_t st;
+
+    /* Wait with a 5 second timeout */
+    time(&st);
+    while (true) {
+        data = inb(IDE_BASE + reg_status);
+        if (!(data & flag)) {
+            return data;
+        }
+        if (difftime(time(NULL), st) > 5.0) {
+            break;
+        }
+        nsleep(400);
+    }
+    g_assert_not_reached();
+}
+
+static void ide_wait_intr(int irq)
+{
+    time_t st;
+    bool intr;
+
+    time(&st);
+    while (true) {
+        intr = get_irq(irq);
+        if (intr) {
+            return;
+        }
+        if (difftime(time(NULL), st) > 5.0) {
+            break;
+        }
+        nsleep(400);
+    }
+
+    g_assert_not_reached();
+}
+
+static void cdrom_pio_impl(int nblocks)
+{
+    FILE *fh;
+    int patt_blocks = MAX(16, nblocks);
+    size_t patt_len = ATAPI_BLOCK_SIZE * patt_blocks;
+    char *pattern = g_malloc(patt_len);
+    size_t rxsize = ATAPI_BLOCK_SIZE * nblocks;
+    uint16_t *rx = g_malloc0(rxsize);
+    int i, j;
+    uint8_t data;
+    uint16_t limit;
+
+    /* Prepopulate the CDROM with an interesting pattern */
+    generate_pattern(pattern, patt_len, ATAPI_BLOCK_SIZE);
+    fh = fopen(tmp_path, "w+");
+    fwrite(pattern, ATAPI_BLOCK_SIZE, patt_blocks, fh);
+    fclose(fh);
+
+    ide_test_start("-drive if=none,file=%s,media=cdrom,format=raw,id=sr0,index=0 "
+                   "-device ide-cd,drive=sr0,bus=ide.0", tmp_path);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    /* PACKET command on device 0 */
+    outb(IDE_BASE + reg_device, 0);
+    outb(IDE_BASE + reg_lba_middle, BYTE_COUNT_LIMIT & 0xFF);
+    outb(IDE_BASE + reg_lba_high, (BYTE_COUNT_LIMIT >> 8 & 0xFF));
+    outb(IDE_BASE + reg_command, CMD_PACKET);
+    /* HP0: Check_Status_A State */
+    nsleep(400);
+    data = ide_wait_clear(BSY);
+    /* HP1: Send_Packet State */
+    assert_bit_set(data, DRQ | DRDY);
+    assert_bit_clear(data, ERR | DF | BSY);
+
+    /* SCSI CDB (READ10) -- read n*2048 bytes from block 0 */
+    send_scsi_cdb_read10(0, nblocks);
+
+    /* Read data back: occurs in bursts of 'BYTE_COUNT_LIMIT' bytes.
+     * If BYTE_COUNT_LIMIT is odd, we transfer BYTE_COUNT_LIMIT - 1 bytes.
+     * We allow an odd limit only when the remaining transfer size is
+     * less than BYTE_COUNT_LIMIT. However, SCSI's read10 command can only
+     * request n blocks, so our request size is always even.
+     * For this reason, we assume there is never a hanging byte to fetch. */
+    g_assert(!(rxsize & 1));
+    limit = BYTE_COUNT_LIMIT & ~1;
+    for (i = 0; i < DIV_ROUND_UP(rxsize, limit); i++) {
+        size_t offset = i * (limit / 2);
+        size_t rem = (rxsize / 2) - offset;
+
+        /* HP3: INTRQ_Wait */
+        ide_wait_intr(IDE_PRIMARY_IRQ);
+
+        /* HP2: Check_Status_B (and clear IRQ) */
+        data = ide_wait_clear(BSY);
+        assert_bit_set(data, DRQ | DRDY);
+        assert_bit_clear(data, ERR | DF | BSY);
+
+        /* HP4: Transfer_Data */
+        for (j = 0; j < MIN((limit / 2), rem); j++) {
+            rx[offset + j] = le16_to_cpu(inw(IDE_BASE + reg_data));
+        }
+    }
+
+    /* Check for final completion IRQ */
+    ide_wait_intr(IDE_PRIMARY_IRQ);
+
+    /* Sanity check final state */
+    data = ide_wait_clear(DRQ);
+    assert_bit_set(data, DRDY);
+    assert_bit_clear(data, DRQ | ERR | DF | BSY);
+
+    g_assert_cmpint(memcmp(pattern, rx, rxsize), ==, 0);
+    g_free(pattern);
+    g_free(rx);
+    test_bmdma_teardown();
+}
+
+static void test_cdrom_pio(void)
+{
+    cdrom_pio_impl(1);
+}
+
+static void test_cdrom_pio_large(void)
+{
+    /* Test a few loops of the PIO DRQ mechanism. */
+    cdrom_pio_impl(BYTE_COUNT_LIMIT * 4 / ATAPI_BLOCK_SIZE);
+}
+
+
+static void test_cdrom_dma(void)
+{
+    static const size_t len = ATAPI_BLOCK_SIZE;
+    char *pattern = g_malloc(ATAPI_BLOCK_SIZE * 16);
+    char *rx = g_malloc0(len);
+    uintptr_t guest_buf;
+    PrdtEntry prdt[1];
+    FILE *fh;
+
+    ide_test_start("-drive if=none,file=%s,media=cdrom,format=raw,id=sr0,index=0 "
+                   "-device ide-cd,drive=sr0,bus=ide.0", tmp_path);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    guest_buf = guest_alloc(guest_malloc, len);
+    prdt[0].addr = cpu_to_le32(guest_buf);
+    prdt[0].size = cpu_to_le32(len | PRDT_EOT);
+
+    generate_pattern(pattern, ATAPI_BLOCK_SIZE * 16, ATAPI_BLOCK_SIZE);
+    fh = fopen(tmp_path, "w+");
+    fwrite(pattern, ATAPI_BLOCK_SIZE, 16, fh);
+    fclose(fh);
+
+    send_dma_request(CMD_PACKET, 0, 1, prdt, 1, send_scsi_cdb_read10);
+
+    /* Read back data from guest memory into local qtest memory */
+    memread(guest_buf, rx, len);
+    g_assert_cmpint(memcmp(pattern, rx, len), ==, 0);
+
+    g_free(pattern);
+    g_free(rx);
+    test_bmdma_teardown();
+}
+
 int main(int argc, char **argv)
 {
     const char *arch = qtest_get_arch();
@@ -624,6 +845,10 @@ int main(int argc, char **argv)
     qtest_add_func("/ide/flush/retry_pci", test_pci_retry_flush);
     qtest_add_func("/ide/flush/retry_isa", test_isa_retry_flush);
 
+    qtest_add_func("/ide/cdrom/pio", test_cdrom_pio);
+    qtest_add_func("/ide/cdrom/pio_large", test_cdrom_pio_large);
+    qtest_add_func("/ide/cdrom/dma", test_cdrom_dma);
+
     ret = g_test_run();
 
     /* Cleanup */
diff --git a/tests/image-fuzzer/runner.py b/tests/image-fuzzer/runner.py
index 0a8743ef4..be7e283dd 100755
--- a/tests/image-fuzzer/runner.py
+++ b/tests/image-fuzzer/runner.py
@@ -301,7 +301,7 @@ if __name__ == '__main__':
         JSON:
 
         '--command' accepts a JSON array of commands. Each command presents
-        an application under test with all its paramaters as a list of strings,
+        an application under test with all its parameters as a list of strings,
         e.g. ["qemu-io", "$test_img", "-c", "write $off $len"].
 
         Supported application aliases: 'qemu-img' and 'qemu-io'.
diff --git a/tests/ivshmem-test.c b/tests/ivshmem-test.c
new file mode 100644
index 000000000..03c7b962a
--- /dev/null
+++ b/tests/ivshmem-test.c
@@ -0,0 +1,494 @@
+/*
+ * QTest testcase for ivshmem
+ *
+ * Copyright (c) 2014 SUSE LINUX Products GmbH
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <glib.h>
+#include <glib/gstdio.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include "contrib/ivshmem-server/ivshmem-server.h"
+#include "libqos/pci-pc.h"
+#include "libqtest.h"
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#define TMPSHMSIZE (1 << 20)
+static char *tmpshm;
+static void *tmpshmem;
+static char *tmpdir;
+static char *tmpserver;
+
+static void save_fn(QPCIDevice *dev, int devfn, void *data)
+{
+    QPCIDevice **pdev = (QPCIDevice **) data;
+
+    *pdev = dev;
+}
+
+static QPCIDevice *get_device(void)
+{
+    QPCIDevice *dev;
+    QPCIBus *pcibus;
+
+    pcibus = qpci_init_pc();
+    dev = NULL;
+    qpci_device_foreach(pcibus, 0x1af4, 0x1110, save_fn, &dev);
+    g_assert(dev != NULL);
+
+    return dev;
+}
+
+typedef struct _IVState {
+    QTestState *qtest;
+    void *reg_base, *mem_base;
+    QPCIDevice *dev;
+} IVState;
+
+enum Reg {
+    INTRMASK = 0,
+    INTRSTATUS = 4,
+    IVPOSITION = 8,
+    DOORBELL = 12,
+};
+
+static const char* reg2str(enum Reg reg) {
+    switch (reg) {
+    case INTRMASK:
+        return "IntrMask";
+    case INTRSTATUS:
+        return "IntrStatus";
+    case IVPOSITION:
+        return "IVPosition";
+    case DOORBELL:
+        return "DoorBell";
+    default:
+        return NULL;
+    }
+}
+
+static inline unsigned in_reg(IVState *s, enum Reg reg)
+{
+    const char *name = reg2str(reg);
+    QTestState *qtest = global_qtest;
+    unsigned res;
+
+    global_qtest = s->qtest;
+    res = qpci_io_readl(s->dev, s->reg_base + reg);
+    g_test_message("*%s -> %x\n", name, res);
+    global_qtest = qtest;
+
+    return res;
+}
+
+static inline void out_reg(IVState *s, enum Reg reg, unsigned v)
+{
+    const char *name = reg2str(reg);
+    QTestState *qtest = global_qtest;
+
+    global_qtest = s->qtest;
+    g_test_message("%x -> *%s\n", v, name);
+    qpci_io_writel(s->dev, s->reg_base + reg, v);
+    global_qtest = qtest;
+}
+
+static void setup_vm_cmd(IVState *s, const char *cmd, bool msix)
+{
+    uint64_t barsize;
+
+    s->qtest = qtest_start(cmd);
+
+    s->dev = get_device();
+
+    /* FIXME: other bar order fails, mappings changes */
+    s->mem_base = qpci_iomap(s->dev, 2, &barsize);
+    g_assert_nonnull(s->mem_base);
+    g_assert_cmpuint(barsize, ==, TMPSHMSIZE);
+
+    if (msix) {
+        qpci_msix_enable(s->dev);
+    }
+
+    s->reg_base = qpci_iomap(s->dev, 0, &barsize);
+    g_assert_nonnull(s->reg_base);
+    g_assert_cmpuint(barsize, ==, 256);
+
+    qpci_device_enable(s->dev);
+}
+
+static void setup_vm(IVState *s)
+{
+    char *cmd = g_strdup_printf("-device ivshmem,shm=%s,size=1M", tmpshm);
+
+    setup_vm_cmd(s, cmd, false);
+
+    g_free(cmd);
+}
+
+static void test_ivshmem_single(void)
+{
+    IVState state, *s;
+    uint32_t data[1024];
+    int i;
+
+    setup_vm(&state);
+    s = &state;
+
+    /* valid io */
+    out_reg(s, INTRMASK, 0);
+    in_reg(s, INTRSTATUS);
+    in_reg(s, IVPOSITION);
+
+    out_reg(s, INTRMASK, 0xffffffff);
+    g_assert_cmpuint(in_reg(s, INTRMASK), ==, 0xffffffff);
+    out_reg(s, INTRSTATUS, 1);
+    /* XXX: intercept IRQ, not seen in resp */
+    g_assert_cmpuint(in_reg(s, INTRSTATUS), ==, 1);
+
+    /* invalid io */
+    out_reg(s, IVPOSITION, 1);
+    out_reg(s, DOORBELL, 8 << 16);
+
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        data[i] = i;
+    }
+    qtest_memwrite(s->qtest, (uintptr_t)s->mem_base, data, sizeof(data));
+
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        g_assert_cmpuint(((uint32_t *)tmpshmem)[i], ==, i);
+    }
+
+    memset(data, 0, sizeof(data));
+
+    qtest_memread(s->qtest, (uintptr_t)s->mem_base, data, sizeof(data));
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        g_assert_cmpuint(data[i], ==, i);
+    }
+
+    qtest_quit(s->qtest);
+}
+
+static void test_ivshmem_pair(void)
+{
+    IVState state1, state2, *s1, *s2;
+    char *data;
+    int i;
+
+    setup_vm(&state1);
+    s1 = &state1;
+    setup_vm(&state2);
+    s2 = &state2;
+
+    data = g_malloc0(TMPSHMSIZE);
+
+    /* host write, guest 1 & 2 read */
+    memset(tmpshmem, 0x42, TMPSHMSIZE);
+    qtest_memread(s1->qtest, (uintptr_t)s1->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x42);
+    }
+    qtest_memread(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x42);
+    }
+
+    /* guest 1 write, guest 2 read */
+    memset(data, 0x43, TMPSHMSIZE);
+    qtest_memwrite(s1->qtest, (uintptr_t)s1->mem_base, data, TMPSHMSIZE);
+    memset(data, 0, TMPSHMSIZE);
+    qtest_memread(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x43);
+    }
+
+    /* guest 2 write, guest 1 read */
+    memset(data, 0x44, TMPSHMSIZE);
+    qtest_memwrite(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    memset(data, 0, TMPSHMSIZE);
+    qtest_memread(s1->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x44);
+    }
+
+    qtest_quit(s1->qtest);
+    qtest_quit(s2->qtest);
+    g_free(data);
+}
+
+typedef struct ServerThread {
+    GThread *thread;
+    IvshmemServer *server;
+    int pipe[2]; /* to handle quit */
+} ServerThread;
+
+static void *server_thread(void *data)
+{
+    ServerThread *t = data;
+    IvshmemServer *server = t->server;
+
+    while (true) {
+        fd_set fds;
+        int maxfd, ret;
+
+        FD_ZERO(&fds);
+        FD_SET(t->pipe[0], &fds);
+        maxfd = t->pipe[0] + 1;
+
+        ivshmem_server_get_fds(server, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            g_critical("select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (FD_ISSET(t->pipe[0], &fds)) {
+            break;
+        }
+
+        if (ivshmem_server_handle_fds(server, &fds, maxfd) < 0) {
+            g_critical("ivshmem_server_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return NULL;
+}
+
+static void setup_vm_with_server(IVState *s, int nvectors)
+{
+    char *cmd = g_strdup_printf("-chardev socket,id=chr0,path=%s,nowait "
+                                "-device ivshmem,size=1M,chardev=chr0,vectors=%d",
+                                tmpserver, nvectors);
+
+    setup_vm_cmd(s, cmd, true);
+
+    g_free(cmd);
+}
+
+static void test_ivshmem_server(void)
+{
+    IVState state1, state2, *s1, *s2;
+    ServerThread thread;
+    IvshmemServer server;
+    int ret, vm1, vm2;
+    int nvectors = 2;
+    guint64 end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+
+    memset(tmpshmem, 0x42, TMPSHMSIZE);
+    ret = ivshmem_server_init(&server, tmpserver, tmpshm,
+                              TMPSHMSIZE, nvectors,
+                              g_test_verbose());
+    g_assert_cmpint(ret, ==, 0);
+
+    ret = ivshmem_server_start(&server);
+    g_assert_cmpint(ret, ==, 0);
+
+    setup_vm_with_server(&state1, nvectors);
+    s1 = &state1;
+    setup_vm_with_server(&state2, nvectors);
+    s2 = &state2;
+
+    g_assert_cmpuint(in_reg(s1, IVPOSITION), ==, 0xffffffff);
+    g_assert_cmpuint(in_reg(s2, IVPOSITION), ==, 0xffffffff);
+
+    g_assert_cmpuint(qtest_readb(s1->qtest, (uintptr_t)s1->mem_base), ==, 0x00);
+
+    thread.server = &server;
+    ret = pipe(thread.pipe);
+    g_assert_cmpint(ret, ==, 0);
+    thread.thread = g_thread_new("ivshmem-server", server_thread, &thread);
+    g_assert(thread.thread != NULL);
+
+    /* waiting until mapping is done */
+    while (g_get_monotonic_time() < end_time) {
+        g_usleep(1000);
+
+        if (qtest_readb(s1->qtest, (uintptr_t)s1->mem_base) == 0x42 &&
+            qtest_readb(s2->qtest, (uintptr_t)s2->mem_base) == 0x42) {
+            break;
+        }
+    }
+
+    /* check got different VM ids */
+    vm1 = in_reg(s1, IVPOSITION);
+    vm2 = in_reg(s2, IVPOSITION);
+    g_assert_cmpuint(vm1, !=, vm2);
+
+    global_qtest = s1->qtest;
+    ret = qpci_msix_table_size(s1->dev);
+    g_assert_cmpuint(ret, ==, nvectors);
+
+    /* ping vm2 -> vm1 */
+    ret = qpci_msix_pending(s1->dev, 0);
+    g_assert_cmpuint(ret, ==, 0);
+    out_reg(s2, DOORBELL, vm1 << 16);
+    do {
+        g_usleep(10000);
+        ret = qpci_msix_pending(s1->dev, 0);
+    } while (ret == 0 && g_get_monotonic_time() < end_time);
+    g_assert_cmpuint(ret, !=, 0);
+
+    /* ping vm1 -> vm2 */
+    global_qtest = s2->qtest;
+    ret = qpci_msix_pending(s2->dev, 0);
+    g_assert_cmpuint(ret, ==, 0);
+    out_reg(s1, DOORBELL, vm2 << 16);
+    do {
+        g_usleep(10000);
+        ret = qpci_msix_pending(s2->dev, 0);
+    } while (ret == 0 && g_get_monotonic_time() < end_time);
+    g_assert_cmpuint(ret, !=, 0);
+
+    qtest_quit(s2->qtest);
+    qtest_quit(s1->qtest);
+
+    if (qemu_write_full(thread.pipe[1], "q", 1) != 1) {
+        g_error("qemu_write_full: %s", g_strerror(errno));
+    }
+
+    g_thread_join(thread.thread);
+
+    ivshmem_server_close(&server);
+    close(thread.pipe[1]);
+    close(thread.pipe[0]);
+}
+
+#define PCI_SLOT_HP             0x06
+
+static void test_ivshmem_hotplug(void)
+{
+    gchar *opts;
+
+    qtest_start("");
+
+    opts = g_strdup_printf("'shm': '%s', 'size': '1M'", tmpshm);
+
+    qpci_plug_device_test("ivshmem", "iv1", PCI_SLOT_HP, opts);
+    qpci_unplug_acpi_device_test("iv1", PCI_SLOT_HP);
+
+    qtest_end();
+    g_free(opts);
+}
+
+static void test_ivshmem_memdev(void)
+{
+    IVState state;
+
+    /* just for the sake of checking memory-backend property */
+    setup_vm_cmd(&state, "-object memory-backend-ram,size=1M,id=mb1"
+                 " -device ivshmem,x-memdev=mb1", false);
+
+    qtest_quit(state.qtest);
+}
+
+static void cleanup(void)
+{
+    if (tmpshmem) {
+        munmap(tmpshmem, TMPSHMSIZE);
+        tmpshmem = NULL;
+    }
+
+    if (tmpshm) {
+        shm_unlink(tmpshm);
+        g_free(tmpshm);
+        tmpshm = NULL;
+    }
+
+    if (tmpserver) {
+        g_unlink(tmpserver);
+        g_free(tmpserver);
+        tmpserver = NULL;
+    }
+
+    if (tmpdir) {
+        g_rmdir(tmpdir);
+        tmpdir = NULL;
+    }
+}
+
+static void abrt_handler(void *data)
+{
+    cleanup();
+}
+
+static gchar *mktempshm(int size, int *fd)
+{
+    while (true) {
+        gchar *name;
+
+        name = g_strdup_printf("/qtest-%u-%u", getpid(), g_random_int());
+        *fd = shm_open(name, O_CREAT|O_RDWR|O_EXCL,
+                       S_IRWXU|S_IRWXG|S_IRWXO);
+        if (*fd > 0) {
+            g_assert(ftruncate(*fd, size) == 0);
+            return name;
+        }
+
+        g_free(name);
+
+        if (errno != EEXIST) {
+            perror("shm_open");
+            return NULL;
+        }
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int ret, fd;
+    gchar dir[] = "/tmp/ivshmem-test.XXXXXX";
+
+#if !GLIB_CHECK_VERSION(2, 31, 0)
+    if (!g_thread_supported()) {
+        g_thread_init(NULL);
+    }
+#endif
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_abrt_handler(abrt_handler, NULL);
+    /* shm */
+    tmpshm = mktempshm(TMPSHMSIZE, &fd);
+    if (!tmpshm) {
+        return 0;
+    }
+    tmpshmem = mmap(0, TMPSHMSIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+    g_assert(tmpshmem != MAP_FAILED);
+    /* server */
+    if (mkdtemp(dir) == NULL) {
+        g_error("mkdtemp: %s", g_strerror(errno));
+    }
+    tmpdir = dir;
+    tmpserver = g_strconcat(tmpdir, "/server", NULL);
+
+    qtest_add_func("/ivshmem/single", test_ivshmem_single);
+    qtest_add_func("/ivshmem/hotplug", test_ivshmem_hotplug);
+    qtest_add_func("/ivshmem/memdev", test_ivshmem_memdev);
+    if (g_test_slow()) {
+        qtest_add_func("/ivshmem/pair", test_ivshmem_pair);
+        qtest_add_func("/ivshmem/server", test_ivshmem_server);
+    }
+
+    ret = g_test_run();
+
+    cleanup();
+
+    return ret;
+}
diff --git a/tests/libqos/ahci.c b/tests/libqos/ahci.c
index cf66b3e32..adb2665c6 100644
--- a/tests/libqos/ahci.c
+++ b/tests/libqos/ahci.c
@@ -742,7 +742,7 @@ AHCICommand *ahci_command_create(uint8_t command_name)
     g_assert(!(props->lba28 && props->lba48));
     g_assert(!(props->read && props->write));
     g_assert(!props->size || props->data);
-    g_assert(!props->ncq || (props->ncq && props->lba48));
+    g_assert(!props->ncq || props->lba48);
 
     /* Defaults and book-keeping */
     cmd->props = props;
diff --git a/tests/libqos/i2c-imx.c b/tests/libqos/i2c-imx.c
new file mode 100644
index 000000000..b5cef66b9
--- /dev/null
+++ b/tests/libqos/i2c-imx.c
@@ -0,0 +1,209 @@
+/*
+ * QTest i.MX I2C driver
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "libqos/i2c.h"
+
+#include <glib.h>
+#include <string.h>
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+
+#include "hw/i2c/imx_i2c.h"
+
+enum IMXI2CDirection {
+    IMX_I2C_READ,
+    IMX_I2C_WRITE,
+};
+
+typedef struct IMXI2C {
+    I2CAdapter parent;
+
+    uint64_t addr;
+} IMXI2C;
+
+
+static void imx_i2c_set_slave_addr(IMXI2C *s, uint8_t addr,
+                                   enum IMXI2CDirection direction)
+{
+    writeb(s->addr + I2DR_ADDR, (addr << 1) |
+           (direction == IMX_I2C_READ ? 1 : 0));
+}
+
+static void imx_i2c_send(I2CAdapter *i2c, uint8_t addr,
+                         const uint8_t *buf, uint16_t len)
+{
+    IMXI2C *s = (IMXI2C *)i2c;
+    uint8_t data;
+    uint8_t status;
+    uint16_t size = 0;
+
+    if (!len) {
+        return;
+    }
+
+    /* set the bus for write */
+    data = I2CR_IEN |
+           I2CR_IIEN |
+           I2CR_MSTA |
+           I2CR_MTX |
+           I2CR_TXAK;
+
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* set the slave address */
+    imx_i2c_set_slave_addr(s, addr, IMX_I2C_WRITE);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+    g_assert((status & I2SR_RXAK) == 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    while (size < len) {
+        /* check we are still busy */
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IBB) != 0);
+
+        /* write the data */
+        writeb(s->addr + I2DR_ADDR, buf[size]);
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) != 0);
+        g_assert((status & I2SR_RXAK) == 0);
+
+        /* ack the interrupt */
+        writeb(s->addr + I2SR_ADDR, 0);
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) == 0);
+
+        size++;
+    }
+
+    /* release the bus */
+    data &= ~(I2CR_MSTA | I2CR_MTX);
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) == 0);
+}
+
+static void imx_i2c_recv(I2CAdapter *i2c, uint8_t addr,
+                         uint8_t *buf, uint16_t len)
+{
+    IMXI2C *s = (IMXI2C *)i2c;
+    uint8_t data;
+    uint8_t status;
+    uint16_t size = 0;
+
+    if (!len) {
+        return;
+    }
+
+    /* set the bus for write */
+    data = I2CR_IEN |
+           I2CR_IIEN |
+           I2CR_MSTA |
+           I2CR_MTX |
+           I2CR_TXAK;
+
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* set the slave address */
+    imx_i2c_set_slave_addr(s, addr, IMX_I2C_READ);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+    g_assert((status & I2SR_RXAK) == 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    /* set the bus for read */
+    data &= ~I2CR_MTX;
+    /* if only one byte don't ack */
+    if (len != 1) {
+        data &= ~I2CR_TXAK;
+    }
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* dummy read */
+    readb(s->addr + I2DR_ADDR);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    while (size < len) {
+        /* check we are still busy */
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IBB) != 0);
+
+        if (size == (len - 1)) {
+            /* stop the read transaction */
+            data &= ~(I2CR_MSTA | I2CR_MTX);
+        } else {
+            /* ack the data read */
+            data |= I2CR_TXAK;
+        }
+        writeb(s->addr + I2CR_ADDR, data);
+
+        /* read the data */
+        buf[size] = readb(s->addr + I2DR_ADDR);
+
+        if (size != (len - 1)) {
+            status = readb(s->addr + I2SR_ADDR);
+            g_assert((status & I2SR_IIF) != 0);
+
+            /* ack the interrupt */
+            writeb(s->addr + I2SR_ADDR, 0);
+        }
+
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) == 0);
+
+        size++;
+    }
+
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) == 0);
+}
+
+I2CAdapter *imx_i2c_create(uint64_t addr)
+{
+    IMXI2C *s = g_malloc0(sizeof(*s));
+    I2CAdapter *i2c = (I2CAdapter *)s;
+
+    s->addr = addr;
+
+    i2c->send = imx_i2c_send;
+    i2c->recv = imx_i2c_recv;
+
+    return i2c;
+}
diff --git a/tests/libqos/i2c.h b/tests/libqos/i2c.h
index 1ce9af405..c21f1dcbd 100644
--- a/tests/libqos/i2c.h
+++ b/tests/libqos/i2c.h
@@ -27,4 +27,7 @@ void i2c_recv(I2CAdapter *i2c, uint8_t addr,
 /* libi2c-omap.c */
 I2CAdapter *omap_i2c_create(uint64_t addr);
 
+/* libi2c-imx.c */
+I2CAdapter *imx_i2c_create(uint64_t addr);
+
 #endif
diff --git a/tests/libqos/libqos.c b/tests/libqos/libqos.c
index fce625b18..2d1a802db 100644
--- a/tests/libqos/libqos.c
+++ b/tests/libqos/libqos.c
@@ -147,6 +147,23 @@ void migrate(QOSState *from, QOSState *to, const char *uri)
     set_context(to);
 }
 
+bool have_qemu_img(void)
+{
+    char *rpath;
+    const char *path = getenv("QTEST_QEMU_IMG");
+    if (!path) {
+        return false;
+    }
+
+    rpath = realpath(path, NULL);
+    if (!rpath) {
+        return false;
+    } else {
+        free(rpath);
+        return true;
+    }
+}
+
 void mkimg(const char *file, const char *fmt, unsigned size_mb)
 {
     gchar *cli;
@@ -155,13 +172,14 @@ void mkimg(const char *file, const char *fmt, unsigned size_mb)
     GError *err = NULL;
     char *qemu_img_path;
     gchar *out, *out2;
-    char *abs_path;
+    char *qemu_img_abs_path;
 
     qemu_img_path = getenv("QTEST_QEMU_IMG");
-    abs_path = realpath(qemu_img_path, NULL);
-    assert(qemu_img_path);
+    g_assert(qemu_img_path);
+    qemu_img_abs_path = realpath(qemu_img_path, NULL);
+    g_assert(qemu_img_abs_path);
 
-    cli = g_strdup_printf("%s create -f %s %s %uM", abs_path,
+    cli = g_strdup_printf("%s create -f %s %s %uM", qemu_img_abs_path,
                           fmt, file, size_mb);
     ret = g_spawn_command_line_sync(cli, &out, &out2, &rc, &err);
     if (err) {
@@ -183,7 +201,7 @@ void mkimg(const char *file, const char *fmt, unsigned size_mb)
     g_free(out);
     g_free(out2);
     g_free(cli);
-    free(abs_path);
+    free(qemu_img_abs_path);
 }
 
 void mkqcow2(const char *file, unsigned size_mb)
@@ -212,3 +230,29 @@ void prepare_blkdebug_script(const char *debug_fn, const char *event)
     ret = fclose(debug_file);
     g_assert(ret == 0);
 }
+
+void generate_pattern(void *buffer, size_t len, size_t cycle_len)
+{
+    int i, j;
+    unsigned char *tx = (unsigned char *)buffer;
+    unsigned char p;
+    size_t *sx;
+
+    /* Write an indicative pattern that varies and is unique per-cycle */
+    p = rand() % 256;
+    for (i = 0; i < len; i++) {
+        tx[i] = p++ % 256;
+        if (i % cycle_len == 0) {
+            p = rand() % 256;
+        }
+    }
+
+    /* force uniqueness by writing an id per-cycle */
+    for (i = 0; i < len / cycle_len; i++) {
+        j = i * cycle_len;
+        if (j + sizeof(*sx) <= len) {
+            sx = (size_t *)&tx[j];
+            *sx = i;
+        }
+    }
+}
diff --git a/tests/libqos/libqos.h b/tests/libqos/libqos.h
index e1f14ea6f..ca14d2e9f 100644
--- a/tests/libqos/libqos.h
+++ b/tests/libqos/libqos.h
@@ -19,11 +19,13 @@ typedef struct QOSState {
 QOSState *qtest_vboot(QOSOps *ops, const char *cmdline_fmt, va_list ap);
 QOSState *qtest_boot(QOSOps *ops, const char *cmdline_fmt, ...);
 void qtest_shutdown(QOSState *qs);
+bool have_qemu_img(void);
 void mkimg(const char *file, const char *fmt, unsigned size_mb);
 void mkqcow2(const char *file, unsigned size_mb);
 void set_context(QOSState *s);
 void migrate(QOSState *from, QOSState *to, const char *uri);
 void prepare_blkdebug_script(const char *debug_fn, const char *event);
+void generate_pattern(void *buffer, size_t len, size_t cycle_len);
 
 static inline uint64_t qmalloc(QOSState *q, size_t bytes)
 {
diff --git a/tests/libqtest.c b/tests/libqtest.c
index f1b8206c3..fa314e1ee 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -46,9 +46,9 @@ struct QTestState
     bool irq_level[MAX_IRQ];
     GString *rx;
     pid_t qemu_pid;  /* our child QEMU process */
-    struct sigaction sigact_old; /* restored on exit */
 };
 
+static GHookList abrt_hooks;
 static GList *qtest_instances;
 static struct sigaction sigact_old;
 
@@ -110,12 +110,14 @@ static void kill_qemu(QTestState *s)
     }
 }
 
+static void kill_qemu_hook_func(void *s)
+{
+    kill_qemu(s);
+}
+
 static void sigabrt_handler(int signo)
 {
-    GList *elem;
-    for (elem = qtest_instances; elem; elem = elem->next) {
-        kill_qemu(elem->data);
-    }
+    g_hook_list_invoke(&abrt_hooks, FALSE);
 }
 
 static void setup_sigabrt_handler(void)
@@ -136,6 +138,23 @@ static void cleanup_sigabrt_handler(void)
     sigaction(SIGABRT, &sigact_old, NULL);
 }
 
+void qtest_add_abrt_handler(GHookFunc fn, const void *data)
+{
+    GHook *hook;
+
+    /* Only install SIGABRT handler once */
+    if (!abrt_hooks.is_setup) {
+        g_hook_list_init(&abrt_hooks, sizeof(GHook));
+        setup_sigabrt_handler();
+    }
+
+    hook = g_hook_alloc(&abrt_hooks);
+    hook->func = fn;
+    hook->data = (void *)data;
+
+    g_hook_prepend(&abrt_hooks, hook);
+}
+
 QTestState *qtest_init(const char *extra_args)
 {
     QTestState *s;
@@ -156,12 +175,7 @@ QTestState *qtest_init(const char *extra_args)
     sock = init_socket(socket_path);
     qmpsock = init_socket(qmp_socket_path);
 
-    /* Only install SIGABRT handler once */
-    if (!qtest_instances) {
-        setup_sigabrt_handler();
-    }
-
-    qtest_instances = g_list_prepend(qtest_instances, s);
+    qtest_add_abrt_handler(kill_qemu_hook_func, s);
 
     s->qemu_pid = fork();
     if (s->qemu_pid == 0) {
@@ -209,13 +223,14 @@ QTestState *qtest_init(const char *extra_args)
 
 void qtest_quit(QTestState *s)
 {
+    qtest_instances = g_list_remove(qtest_instances, s);
+    g_hook_destroy_link(&abrt_hooks, g_hook_find_data(&abrt_hooks, TRUE, s));
+
     /* Uninstall SIGABRT handler on last instance */
-    if (qtest_instances && !qtest_instances->next) {
+    if (!qtest_instances) {
         cleanup_sigabrt_handler();
     }
 
-    qtest_instances = g_list_remove(qtest_instances, s);
-
     kill_qemu(s);
     close(s->fd);
     close(s->qmp_fd);
@@ -341,7 +356,7 @@ typedef struct {
     QDict *response;
 } QMPResponseParser;
 
-static void qmp_response(JSONMessageParser *parser, QList *tokens)
+static void qmp_response(JSONMessageParser *parser, GQueue *tokens)
 {
     QMPResponseParser *qmp = container_of(parser, QMPResponseParser, parser);
     QObject *obj;
@@ -357,7 +372,7 @@ static void qmp_response(JSONMessageParser *parser, QList *tokens)
     qmp->response = (QDict *)obj;
 }
 
-QDict *qtest_qmp_receive(QTestState *s)
+QDict *qmp_fd_receive(int fd)
 {
     QMPResponseParser qmp;
     bool log = getenv("QTEST_LOG") != NULL;
@@ -368,7 +383,7 @@ QDict *qtest_qmp_receive(QTestState *s)
         ssize_t len;
         char c;
 
-        len = read(s->qmp_fd, &c, 1);
+        len = read(fd, &c, 1);
         if (len == -1 && errno == EINTR) {
             continue;
         }
@@ -388,12 +403,17 @@ QDict *qtest_qmp_receive(QTestState *s)
     return qmp.response;
 }
 
+QDict *qtest_qmp_receive(QTestState *s)
+{
+    return qmp_fd_receive(s->qmp_fd);
+}
+
 /**
  * Allow users to send a message without waiting for the reply,
  * in the case that they choose to discard all replies up until
  * a particular EVENT is received.
  */
-void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap)
+void qmp_fd_sendv(int fd, const char *fmt, va_list ap)
 {
     va_list ap_copy;
     QObject *qobj;
@@ -417,13 +437,25 @@ void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap)
             fprintf(stderr, "%s", str);
         }
         /* Send QMP request */
-        socket_send(s->qmp_fd, str, size);
+        socket_send(fd, str, size);
 
         QDECREF(qstr);
         qobject_decref(qobj);
     }
 }
 
+void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap)
+{
+    qmp_fd_sendv(s->qmp_fd, fmt, ap);
+}
+
+QDict *qmp_fdv(int fd, const char *fmt, va_list ap)
+{
+    qmp_fd_sendv(fd, fmt, ap);
+
+    return qmp_fd_receive(fd);
+}
+
 QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap)
 {
     qtest_async_qmpv(s, fmt, ap);
@@ -432,6 +464,26 @@ QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap)
     return qtest_qmp_receive(s);
 }
 
+QDict *qmp_fd(int fd, const char *fmt, ...)
+{
+    va_list ap;
+    QDict *response;
+
+    va_start(ap, fmt);
+    response = qmp_fdv(fd, fmt, ap);
+    va_end(ap);
+    return response;
+}
+
+void qmp_fd_send(int fd, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    qmp_fd_sendv(fd, fmt, ap);
+    va_end(ap);
+}
+
 QDict *qtest_qmp(QTestState *s, const char *fmt, ...)
 {
     va_list ap;
@@ -708,14 +760,15 @@ void qtest_memread(QTestState *s, uint64_t addr, void *data, size_t size)
     g_strfreev(args);
 }
 
-void qtest_add_func(const char *str, void (*fn))
+void qtest_add_func(const char *str, void (*fn)(void))
 {
     gchar *path = g_strdup_printf("/%s/%s", qtest_get_arch(), str);
     g_test_add_func(path, fn);
     g_free(path);
 }
 
-void qtest_add_data_func(const char *str, const void *data, void (*fn))
+void qtest_add_data_func(const char *str, const void *data,
+                         void (*fn)(const void *))
 {
     gchar *path = g_strdup_printf("/%s/%s", qtest_get_arch(), str);
     g_test_add_data_func(path, data, fn);
diff --git a/tests/libqtest.h b/tests/libqtest.h
index 55bccbf0e..ebdd5bbe5 100644
--- a/tests/libqtest.h
+++ b/tests/libqtest.h
@@ -416,7 +416,7 @@ const char *qtest_get_arch(void);
  * The path is prefixed with the architecture under test, as
  * returned by qtest_get_arch().
  */
-void qtest_add_func(const char *str, void (*fn));
+void qtest_add_func(const char *str, void (*fn)(void));
 
 /**
  * qtest_add_data_func:
@@ -428,7 +428,8 @@ void qtest_add_func(const char *str, void (*fn));
  * The path is prefixed with the architecture under test, as
  * returned by qtest_get_arch().
  */
-void qtest_add_data_func(const char *str, const void *data, void (*fn));
+void qtest_add_data_func(const char *str, const void *data,
+                         void (*fn)(const void *));
 
 /**
  * qtest_add:
@@ -450,6 +451,8 @@ void qtest_add_data_func(const char *str, const void *data, void (*fn));
         g_free(path); \
     } while (0)
 
+void qtest_add_abrt_handler(GHookFunc fn, const void *data);
+
 /**
  * qtest_start:
  * @args: other arguments to pass to QEMU
@@ -851,4 +854,11 @@ static inline int64_t clock_set(int64_t val)
  */
 bool qtest_big_endian(void);
 
+
+QDict *qmp_fd_receive(int fd);
+void qmp_fd_sendv(int fd, const char *fmt, va_list ap);
+void qmp_fd_send(int fd, const char *fmt, ...);
+QDict *qmp_fdv(int fd, const char *fmt, va_list ap);
+QDict *qmp_fd(int fd, const char *fmt, ...);
+
 #endif
diff --git a/tests/pkix_asn1_tab.c b/tests/pkix_asn1_tab.c
new file mode 100644
index 000000000..5b4e6b9e2
--- /dev/null
+++ b/tests/pkix_asn1_tab.c
@@ -0,0 +1,1104 @@
+/*
+ * This file is taken from gnutls 1.6.3 under the GPLv2+
+ * and is under copyright of various GNUTLS contributors.
+ */
+
+#include <libtasn1.h>
+
+const ASN1_ARRAY_TYPE pkix_asn1_tab[] = {
+  {"PKIX1", 536875024, 0},
+  {0, 1073741836, 0},
+  {"id-ce", 1879048204, 0},
+  {"joint-iso-ccitt", 1073741825, "2"},
+  {"ds", 1073741825, "5"},
+  {0, 1, "29"},
+  {"id-ce-authorityKeyIdentifier", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "35"},
+  {"AuthorityKeyIdentifier", 1610612741, 0},
+  {"keyIdentifier", 1610637314, "KeyIdentifier"},
+  {0, 4104, "0"},
+  {"authorityCertIssuer", 1610637314, "GeneralNames"},
+  {0, 4104, "1"},
+  {"authorityCertSerialNumber", 536895490, "CertificateSerialNumber"},
+  {0, 4104, "2"},
+  {"KeyIdentifier", 1073741831, 0},
+  {"id-ce-subjectKeyIdentifier", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "14"},
+  {"SubjectKeyIdentifier", 1073741826, "KeyIdentifier"},
+  {"id-ce-keyUsage", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "15"},
+  {"KeyUsage", 1610874886, 0},
+  {"digitalSignature", 1073741825, "0"},
+  {"nonRepudiation", 1073741825, "1"},
+  {"keyEncipherment", 1073741825, "2"},
+  {"dataEncipherment", 1073741825, "3"},
+  {"keyAgreement", 1073741825, "4"},
+  {"keyCertSign", 1073741825, "5"},
+  {"cRLSign", 1073741825, "6"},
+  {"encipherOnly", 1073741825, "7"},
+  {"decipherOnly", 1, "8"},
+  {"id-ce-privateKeyUsagePeriod", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "16"},
+  {"PrivateKeyUsagePeriod", 1610612741, 0},
+  {"notBefore", 1619025937, 0},
+  {0, 4104, "0"},
+  {"notAfter", 545284113, 0},
+  {0, 4104, "1"},
+  {"id-ce-certificatePolicies", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "32"},
+  {"CertificatePolicies", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "PolicyInformation"},
+  {"PolicyInformation", 1610612741, 0},
+  {"policyIdentifier", 1073741826, "CertPolicyId"},
+  {"policyQualifiers", 538984459, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "PolicyQualifierInfo"},
+  {"CertPolicyId", 1073741836, 0},
+  {"PolicyQualifierInfo", 1610612741, 0},
+  {"policyQualifierId", 1073741826, "PolicyQualifierId"},
+  {"qualifier", 541065229, 0},
+  {"policyQualifierId", 1, 0},
+  {"PolicyQualifierId", 1073741836, 0},
+  {"CPSuri", 1073741826, "IA5String"},
+  {"UserNotice", 1610612741, 0},
+  {"noticeRef", 1073758210, "NoticeReference"},
+  {"explicitText", 16386, "DisplayText"},
+  {"NoticeReference", 1610612741, 0},
+  {"organization", 1073741826, "DisplayText"},
+  {"noticeNumbers", 536870923, 0},
+  {0, 3, 0},
+  {"DisplayText", 1610612754, 0},
+  {"visibleString", 1612709890, "VisibleString"},
+  {"200", 524298, "1"},
+  {"bmpString", 1612709890, "BMPString"},
+  {"200", 524298, "1"},
+  {"utf8String", 538968066, "UTF8String"},
+  {"200", 524298, "1"},
+  {"id-ce-policyMappings", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "33"},
+  {"PolicyMappings", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 536870917, 0},
+  {"issuerDomainPolicy", 1073741826, "CertPolicyId"},
+  {"subjectDomainPolicy", 2, "CertPolicyId"},
+  {"DirectoryString", 1610612754, 0},
+  {"teletexString", 1612709890, "TeletexString"},
+  {"MAX", 524298, "1"},
+  {"printableString", 1612709890, "PrintableString"},
+  {"MAX", 524298, "1"},
+  {"universalString", 1612709890, "UniversalString"},
+  {"MAX", 524298, "1"},
+  {"utf8String", 1612709890, "UTF8String"},
+  {"MAX", 524298, "1"},
+  {"bmpString", 1612709890, "BMPString"},
+  {"MAX", 524298, "1"},
+  {"ia5String", 538968066, "IA5String"},
+  {"MAX", 524298, "1"},
+  {"id-ce-subjectAltName", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "17"},
+  {"SubjectAltName", 1073741826, "GeneralNames"},
+  {"GeneralNames", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "GeneralName"},
+  {"GeneralName", 1610612754, 0},
+  {"otherName", 1610620930, "AnotherName"},
+  {0, 4104, "0"},
+  {"rfc822Name", 1610620930, "IA5String"},
+  {0, 4104, "1"},
+  {"dNSName", 1610620930, "IA5String"},
+  {0, 4104, "2"},
+  {"x400Address", 1610620930, "ORAddress"},
+  {0, 4104, "3"},
+  {"directoryName", 1610620930, "RDNSequence"},
+  {0, 2056, "4"},
+  {"ediPartyName", 1610620930, "EDIPartyName"},
+  {0, 4104, "5"},
+  {"uniformResourceIdentifier", 1610620930, "IA5String"},
+  {0, 4104, "6"},
+  {"iPAddress", 1610620935, 0},
+  {0, 4104, "7"},
+  {"registeredID", 536879116, 0},
+  {0, 4104, "8"},
+  {"AnotherName", 1610612741, 0},
+  {"type-id", 1073741836, 0},
+  {"value", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"type-id", 1, 0},
+  {"EDIPartyName", 1610612741, 0},
+  {"nameAssigner", 1610637314, "DirectoryString"},
+  {0, 4104, "0"},
+  {"partyName", 536879106, "DirectoryString"},
+  {0, 4104, "1"},
+  {"id-ce-issuerAltName", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "18"},
+  {"IssuerAltName", 1073741826, "GeneralNames"},
+  {"id-ce-subjectDirectoryAttributes", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "9"},
+  {"SubjectDirectoryAttributes", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Attribute"},
+  {"id-ce-basicConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "19"},
+  {"BasicConstraints", 1610612741, 0},
+  {"cA", 1610645508, 0},
+  {0, 131081, 0},
+  {"pathLenConstraint", 537411587, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-nameConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "30"},
+  {"NameConstraints", 1610612741, 0},
+  {"permittedSubtrees", 1610637314, "GeneralSubtrees"},
+  {0, 4104, "0"},
+  {"excludedSubtrees", 536895490, "GeneralSubtrees"},
+  {0, 4104, "1"},
+  {"GeneralSubtrees", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "GeneralSubtree"},
+  {"GeneralSubtree", 1610612741, 0},
+  {"base", 1073741826, "GeneralName"},
+  {"minimum", 1610653698, "BaseDistance"},
+  {0, 1073741833, "0"},
+  {0, 4104, "0"},
+  {"maximum", 536895490, "BaseDistance"},
+  {0, 4104, "1"},
+  {"BaseDistance", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-policyConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "36"},
+  {"PolicyConstraints", 1610612741, 0},
+  {"requireExplicitPolicy", 1610637314, "SkipCerts"},
+  {0, 4104, "0"},
+  {"inhibitPolicyMapping", 536895490, "SkipCerts"},
+  {0, 4104, "1"},
+  {"SkipCerts", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-cRLDistributionPoints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "31"},
+  {"CRLDistributionPoints", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "DistributionPoint"},
+  {"DistributionPoint", 1610612741, 0},
+  {"distributionPoint", 1610637314, "DistributionPointName"},
+  {0, 2056, "0"},
+  {"reasons", 1610637314, "ReasonFlags"},
+  {0, 4104, "1"},
+  {"cRLIssuer", 536895490, "GeneralNames"},
+  {0, 4104, "2"},
+  {"DistributionPointName", 1610612754, 0},
+  {"fullName", 1610620930, "GeneralNames"},
+  {0, 4104, "0"},
+  {"nameRelativeToCRLIssuer", 536879106, "RelativeDistinguishedName"},
+  {0, 4104, "1"},
+  {"ReasonFlags", 1610874886, 0},
+  {"unused", 1073741825, "0"},
+  {"keyCompromise", 1073741825, "1"},
+  {"cACompromise", 1073741825, "2"},
+  {"affiliationChanged", 1073741825, "3"},
+  {"superseded", 1073741825, "4"},
+  {"cessationOfOperation", 1073741825, "5"},
+  {"certificateHold", 1073741825, "6"},
+  {"privilegeWithdrawn", 1073741825, "7"},
+  {"aACompromise", 1, "8"},
+  {"id-ce-extKeyUsage", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "37"},
+  {"ExtKeyUsageSyntax", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "KeyPurposeId"},
+  {"KeyPurposeId", 1073741836, 0},
+  {"id-kp-serverAuth", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "1"},
+  {"id-kp-clientAuth", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "2"},
+  {"id-kp-codeSigning", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "3"},
+  {"id-kp-emailProtection", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "4"},
+  {"id-kp-ipsecEndSystem", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "5"},
+  {"id-kp-ipsecTunnel", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "6"},
+  {"id-kp-ipsecUser", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "7"},
+  {"id-kp-timeStamping", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "8"},
+  {"id-pe-authorityInfoAccess", 1879048204, 0},
+  {0, 1073741825, "id-pe"},
+  {0, 1, "1"},
+  {"AuthorityInfoAccessSyntax", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "AccessDescription"},
+  {"AccessDescription", 1610612741, 0},
+  {"accessMethod", 1073741836, 0},
+  {"accessLocation", 2, "GeneralName"},
+  {"id-ce-cRLNumber", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "20"},
+  {"CRLNumber", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-issuingDistributionPoint", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "28"},
+  {"IssuingDistributionPoint", 1610612741, 0},
+  {"distributionPoint", 1610637314, "DistributionPointName"},
+  {0, 4104, "0"},
+  {"onlyContainsUserCerts", 1610653700, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "1"},
+  {"onlyContainsCACerts", 1610653700, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "2"},
+  {"onlySomeReasons", 1610637314, "ReasonFlags"},
+  {0, 4104, "3"},
+  {"indirectCRL", 536911876, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "4"},
+  {"id-ce-deltaCRLIndicator", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "27"},
+  {"BaseCRLNumber", 1073741826, "CRLNumber"},
+  {"id-ce-cRLReasons", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "21"},
+  {"CRLReason", 1610874901, 0},
+  {"unspecified", 1073741825, "0"},
+  {"keyCompromise", 1073741825, "1"},
+  {"cACompromise", 1073741825, "2"},
+  {"affiliationChanged", 1073741825, "3"},
+  {"superseded", 1073741825, "4"},
+  {"cessationOfOperation", 1073741825, "5"},
+  {"certificateHold", 1073741825, "6"},
+  {"removeFromCRL", 1, "8"},
+  {"id-ce-certificateIssuer", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "29"},
+  {"CertificateIssuer", 1073741826, "GeneralNames"},
+  {"id-ce-holdInstructionCode", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "23"},
+  {"HoldInstructionCode", 1073741836, 0},
+  {"holdInstruction", 1879048204, 0},
+  {"joint-iso-itu-t", 1073741825, "2"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9cm", 1073741825, "10040"},
+  {0, 1, "2"},
+  {"id-holdinstruction-none", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "1"},
+  {"id-holdinstruction-callissuer", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "2"},
+  {"id-holdinstruction-reject", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "3"},
+  {"id-ce-invalidityDate", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "24"},
+  {"InvalidityDate", 1082130449, 0},
+  {"VisibleString", 1610620935, 0},
+  {0, 4360, "26"},
+  {"NumericString", 1610620935, 0},
+  {0, 4360, "18"},
+  {"IA5String", 1610620935, 0},
+  {0, 4360, "22"},
+  {"TeletexString", 1610620935, 0},
+  {0, 4360, "20"},
+  {"PrintableString", 1610620935, 0},
+  {0, 4360, "19"},
+  {"UniversalString", 1610620935, 0},
+  {0, 4360, "28"},
+  {"BMPString", 1610620935, 0},
+  {0, 4360, "30"},
+  {"UTF8String", 1610620935, 0},
+  {0, 4360, "12"},
+  {"id-pkix", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"identified-organization", 1073741825, "3"},
+  {"dod", 1073741825, "6"},
+  {"internet", 1073741825, "1"},
+  {"security", 1073741825, "5"},
+  {"mechanisms", 1073741825, "5"},
+  {"pkix", 1, "7"},
+  {"id-pe", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "1"},
+  {"id-qt", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "2"},
+  {"id-kp", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "3"},
+  {"id-ad", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "48"},
+  {"id-qt-cps", 1879048204, 0},
+  {0, 1073741825, "id-qt"},
+  {0, 1, "1"},
+  {"id-qt-unotice", 1879048204, 0},
+  {0, 1073741825, "id-qt"},
+  {0, 1, "2"},
+  {"id-ad-ocsp", 1879048204, 0},
+  {0, 1073741825, "id-ad"},
+  {0, 1, "1"},
+  {"id-ad-caIssuers", 1879048204, 0},
+  {0, 1073741825, "id-ad"},
+  {0, 1, "2"},
+  {"Attribute", 1610612741, 0},
+  {"type", 1073741826, "AttributeType"},
+  {"values", 536870927, 0},
+  {0, 2, "AttributeValue"},
+  {"AttributeType", 1073741836, 0},
+  {"AttributeValue", 1614807053, 0},
+  {"type", 1, 0},
+  {"AttributeTypeAndValue", 1610612741, 0},
+  {"type", 1073741826, "AttributeType"},
+  {"value", 2, "AttributeValue"},
+  {"id-at", 1879048204, 0},
+  {"joint-iso-ccitt", 1073741825, "2"},
+  {"ds", 1073741825, "5"},
+  {0, 1, "4"},
+  {"id-at-initials", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "43"},
+  {"X520initials", 1073741826, "DirectoryString"},
+  {"id-at-generationQualifier", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "44"},
+  {"X520generationQualifier", 1073741826, "DirectoryString"},
+  {"id-at-surname", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "4"},
+  {"X520surName", 1073741826, "DirectoryString"},
+  {"id-at-givenName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "42"},
+  {"X520givenName", 1073741826, "DirectoryString"},
+  {"id-at-name", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "41"},
+  {"X520name", 1073741826, "DirectoryString"},
+  {"id-at-commonName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "3"},
+  {"X520CommonName", 1073741826, "DirectoryString"},
+  {"id-at-localityName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "7"},
+  {"X520LocalityName", 1073741826, "DirectoryString"},
+  {"id-at-stateOrProvinceName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "8"},
+  {"X520StateOrProvinceName", 1073741826, "DirectoryString"},
+  {"id-at-organizationName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "10"},
+  {"X520OrganizationName", 1073741826, "DirectoryString"},
+  {"id-at-organizationalUnitName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "11"},
+  {"X520OrganizationalUnitName", 1073741826, "DirectoryString"},
+  {"id-at-title", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "12"},
+  {"X520Title", 1073741826, "DirectoryString"},
+  {"id-at-description", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "13"},
+  {"X520Description", 1073741826, "DirectoryString"},
+  {"id-at-dnQualifier", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "46"},
+  {"X520dnQualifier", 1073741826, "PrintableString"},
+  {"id-at-countryName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "6"},
+  {"X520countryName", 1612709890, "PrintableString"},
+  {0, 1048586, "2"},
+  {"id-at-serialNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "5"},
+  {"X520serialNumber", 1073741826, "PrintableString"},
+  {"id-at-telephoneNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "20"},
+  {"X520telephoneNumber", 1073741826, "PrintableString"},
+  {"id-at-facsimileTelephoneNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "23"},
+  {"X520facsimileTelephoneNumber", 1073741826, "PrintableString"},
+  {"id-at-pseudonym", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "65"},
+  {"X520pseudonym", 1073741826, "DirectoryString"},
+  {"id-at-name", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "41"},
+  {"X520name", 1073741826, "DirectoryString"},
+  {"id-at-streetAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "9"},
+  {"X520streetAddress", 1073741826, "DirectoryString"},
+  {"id-at-postalAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "16"},
+  {"X520postalAddress", 1073741826, "PostalAddress"},
+  {"PostalAddress", 1610612747, 0},
+  {0, 2, "DirectoryString"},
+  {"pkcs", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1, "1"},
+  {"pkcs-9", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "9"},
+  {"emailAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "1"},
+  {"Pkcs9email", 1612709890, "IA5String"},
+  {"ub-emailaddress-length", 524298, "1"},
+  {"Name", 1610612754, 0},
+  {"rdnSequence", 2, "RDNSequence"},
+  {"RDNSequence", 1610612747, 0},
+  {0, 2, "RelativeDistinguishedName"},
+  {"DistinguishedName", 1073741826, "RDNSequence"},
+  {"RelativeDistinguishedName", 1612709903, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "AttributeTypeAndValue"},
+  {"Certificate", 1610612741, 0},
+  {"tbsCertificate", 1073741826, "TBSCertificate"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"TBSCertificate", 1610612741, 0},
+  {"version", 1610653698, "Version"},
+  {0, 1073741833, "v1"},
+  {0, 2056, "0"},
+  {"serialNumber", 1073741826, "CertificateSerialNumber"},
+  {"signature", 1073741826, "AlgorithmIdentifier"},
+  {"issuer", 1073741826, "Name"},
+  {"validity", 1073741826, "Validity"},
+  {"subject", 1073741826, "Name"},
+  {"subjectPublicKeyInfo", 1073741826, "SubjectPublicKeyInfo"},
+  {"issuerUniqueID", 1610637314, "UniqueIdentifier"},
+  {0, 4104, "1"},
+  {"subjectUniqueID", 1610637314, "UniqueIdentifier"},
+  {0, 4104, "2"},
+  {"extensions", 536895490, "Extensions"},
+  {0, 2056, "3"},
+  {"Version", 1610874883, 0},
+  {"v1", 1073741825, "0"},
+  {"v2", 1073741825, "1"},
+  {"v3", 1, "2"},
+  {"CertificateSerialNumber", 1073741827, 0},
+  {"Validity", 1610612741, 0},
+  {"notBefore", 1073741826, "Time"},
+  {"notAfter", 2, "Time"},
+  {"Time", 1610612754, 0},
+  {"utcTime", 1090519057, 0},
+  {"generalTime", 8388625, 0},
+  {"UniqueIdentifier", 1073741830, 0},
+  {"SubjectPublicKeyInfo", 1610612741, 0},
+  {"algorithm", 1073741826, "AlgorithmIdentifier"},
+  {"subjectPublicKey", 6, 0},
+  {"Extensions", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Extension"},
+  {"Extension", 1610612741, 0},
+  {"extnID", 1073741836, 0},
+  {"critical", 1610645508, 0},
+  {0, 131081, 0},
+  {"extnValue", 7, 0},
+  {"CertificateList", 1610612741, 0},
+  {"tbsCertList", 1073741826, "TBSCertList"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"TBSCertList", 1610612741, 0},
+  {"version", 1073758210, "Version"},
+  {"signature", 1073741826, "AlgorithmIdentifier"},
+  {"issuer", 1073741826, "Name"},
+  {"thisUpdate", 1073741826, "Time"},
+  {"nextUpdate", 1073758210, "Time"},
+  {"revokedCertificates", 1610629131, 0},
+  {0, 536870917, 0},
+  {"userCertificate", 1073741826, "CertificateSerialNumber"},
+  {"revocationDate", 1073741826, "Time"},
+  {"crlEntryExtensions", 16386, "Extensions"},
+  {"crlExtensions", 536895490, "Extensions"},
+  {0, 2056, "0"},
+  {"AlgorithmIdentifier", 1610612741, 0},
+  {"algorithm", 1073741836, 0},
+  {"parameters", 541081613, 0},
+  {"algorithm", 1, 0},
+  {"pkcs-1", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "1"},
+  {"rsaEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "1"},
+  {"md2WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "2"},
+  {"md5WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "4"},
+  {"sha1WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "5"},
+  {"id-dsa-with-sha1", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9-57", 1073741825, "10040"},
+  {"x9algorithm", 1073741825, "4"},
+  {0, 1, "3"},
+  {"Dss-Sig-Value", 1610612741, 0},
+  {"r", 1073741827, 0},
+  {"s", 3, 0},
+  {"dhpublicnumber", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"ansi-x942", 1073741825, "10046"},
+  {"number-type", 1073741825, "2"},
+  {0, 1, "1"},
+  {"DomainParameters", 1610612741, 0},
+  {"p", 1073741827, 0},
+  {"g", 1073741827, 0},
+  {"q", 1073741827, 0},
+  {"j", 1073758211, 0},
+  {"validationParms", 16386, "ValidationParms"},
+  {"ValidationParms", 1610612741, 0},
+  {"seed", 1073741830, 0},
+  {"pgenCounter", 3, 0},
+  {"id-dsa", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9-57", 1073741825, "10040"},
+  {"x9algorithm", 1073741825, "4"},
+  {0, 1, "1"},
+  {"Dss-Parms", 1610612741, 0},
+  {"p", 1073741827, 0},
+  {"q", 1073741827, 0},
+  {"g", 3, 0},
+  {"ORAddress", 1610612741, 0},
+  {"built-in-standard-attributes", 1073741826, "BuiltInStandardAttributes"},
+  {"built-in-domain-defined-attributes", 1073758210,
+   "BuiltInDomainDefinedAttributes"},
+  {"extension-attributes", 16386, "ExtensionAttributes"},
+  {"BuiltInStandardAttributes", 1610612741, 0},
+  {"country-name", 1073758210, "CountryName"},
+  {"administration-domain-name", 1073758210, "AdministrationDomainName"},
+  {"network-address", 1610637314, "NetworkAddress"},
+  {0, 2056, "0"},
+  {"terminal-identifier", 1610637314, "TerminalIdentifier"},
+  {0, 2056, "1"},
+  {"private-domain-name", 1610637314, "PrivateDomainName"},
+  {0, 2056, "2"},
+  {"organization-name", 1610637314, "OrganizationName"},
+  {0, 2056, "3"},
+  {"numeric-user-identifier", 1610637314, "NumericUserIdentifier"},
+  {0, 2056, "4"},
+  {"personal-name", 1610637314, "PersonalName"},
+  {0, 2056, "5"},
+  {"organizational-unit-names", 536895490, "OrganizationalUnitNames"},
+  {0, 2056, "6"},
+  {"CountryName", 1610620946, 0},
+  {0, 1073746952, "1"},
+  {"x121-dcc-code", 1612709890, "NumericString"},
+  {0, 1048586, "ub-country-name-numeric-length"},
+  {"iso-3166-alpha2-code", 538968066, "PrintableString"},
+  {0, 1048586, "ub-country-name-alpha-length"},
+  {"AdministrationDomainName", 1610620946, 0},
+  {0, 1073744904, "2"},
+  {"numeric", 1612709890, "NumericString"},
+  {"ub-domain-name-length", 524298, "0"},
+  {"printable", 538968066, "PrintableString"},
+  {"ub-domain-name-length", 524298, "0"},
+  {"NetworkAddress", 1073741826, "X121Address"},
+  {"X121Address", 1612709890, "NumericString"},
+  {"ub-x121-address-length", 524298, "1"},
+  {"TerminalIdentifier", 1612709890, "PrintableString"},
+  {"ub-terminal-id-length", 524298, "1"},
+  {"PrivateDomainName", 1610612754, 0},
+  {"numeric", 1612709890, "NumericString"},
+  {"ub-domain-name-length", 524298, "1"},
+  {"printable", 538968066, "PrintableString"},
+  {"ub-domain-name-length", 524298, "1"},
+  {"OrganizationName", 1612709890, "PrintableString"},
+  {"ub-organization-name-length", 524298, "1"},
+  {"NumericUserIdentifier", 1612709890, "NumericString"},
+  {"ub-numeric-user-id-length", 524298, "1"},
+  {"PersonalName", 1610612750, 0},
+  {"surname", 1814044674, "PrintableString"},
+  {0, 1073745928, "0"},
+  {"ub-surname-length", 524298, "1"},
+  {"given-name", 1814061058, "PrintableString"},
+  {0, 1073745928, "1"},
+  {"ub-given-name-length", 524298, "1"},
+  {"initials", 1814061058, "PrintableString"},
+  {0, 1073745928, "2"},
+  {"ub-initials-length", 524298, "1"},
+  {"generation-qualifier", 740319234, "PrintableString"},
+  {0, 1073745928, "3"},
+  {"ub-generation-qualifier-length", 524298, "1"},
+  {"OrganizationalUnitNames", 1612709899, 0},
+  {"ub-organizational-units", 1074266122, "1"},
+  {0, 2, "OrganizationalUnitName"},
+  {"OrganizationalUnitName", 1612709890, "PrintableString"},
+  {"ub-organizational-unit-name-length", 524298, "1"},
+  {"BuiltInDomainDefinedAttributes", 1612709899, 0},
+  {"ub-domain-defined-attributes", 1074266122, "1"},
+  {0, 2, "BuiltInDomainDefinedAttribute"},
+  {"BuiltInDomainDefinedAttribute", 1610612741, 0},
+  {"type", 1612709890, "PrintableString"},
+  {"ub-domain-defined-attribute-type-length", 524298, "1"},
+  {"value", 538968066, "PrintableString"},
+  {"ub-domain-defined-attribute-value-length", 524298, "1"},
+  {"ExtensionAttributes", 1612709903, 0},
+  {"ub-extension-attributes", 1074266122, "1"},
+  {0, 2, "ExtensionAttribute"},
+  {"ExtensionAttribute", 1610612741, 0},
+  {"extension-attribute-type", 1611145219, 0},
+  {0, 1073743880, "0"},
+  {"0", 10, "ub-extension-attributes"},
+  {"extension-attribute-value", 541073421, 0},
+  {0, 1073743880, "1"},
+  {"extension-attribute-type", 1, 0},
+  {"common-name", 1342177283, "1"},
+  {"CommonName", 1612709890, "PrintableString"},
+  {"ub-common-name-length", 524298, "1"},
+  {"teletex-common-name", 1342177283, "2"},
+  {"TeletexCommonName", 1612709890, "TeletexString"},
+  {"ub-common-name-length", 524298, "1"},
+  {"teletex-organization-name", 1342177283, "3"},
+  {"TeletexOrganizationName", 1612709890, "TeletexString"},
+  {"ub-organization-name-length", 524298, "1"},
+  {"teletex-personal-name", 1342177283, "4"},
+  {"TeletexPersonalName", 1610612750, 0},
+  {"surname", 1814044674, "TeletexString"},
+  {0, 1073743880, "0"},
+  {"ub-surname-length", 524298, "1"},
+  {"given-name", 1814061058, "TeletexString"},
+  {0, 1073743880, "1"},
+  {"ub-given-name-length", 524298, "1"},
+  {"initials", 1814061058, "TeletexString"},
+  {0, 1073743880, "2"},
+  {"ub-initials-length", 524298, "1"},
+  {"generation-qualifier", 740319234, "TeletexString"},
+  {0, 1073743880, "3"},
+  {"ub-generation-qualifier-length", 524298, "1"},
+  {"teletex-organizational-unit-names", 1342177283, "5"},
+  {"TeletexOrganizationalUnitNames", 1612709899, 0},
+  {"ub-organizational-units", 1074266122, "1"},
+  {0, 2, "TeletexOrganizationalUnitName"},
+  {"TeletexOrganizationalUnitName", 1612709890, "TeletexString"},
+  {"ub-organizational-unit-name-length", 524298, "1"},
+  {"pds-name", 1342177283, "7"},
+  {"PDSName", 1612709890, "PrintableString"},
+  {"ub-pds-name-length", 524298, "1"},
+  {"physical-delivery-country-name", 1342177283, "8"},
+  {"PhysicalDeliveryCountryName", 1610612754, 0},
+  {"x121-dcc-code", 1612709890, "NumericString"},
+  {0, 1048586, "ub-country-name-numeric-length"},
+  {"iso-3166-alpha2-code", 538968066, "PrintableString"},
+  {0, 1048586, "ub-country-name-alpha-length"},
+  {"postal-code", 1342177283, "9"},
+  {"PostalCode", 1610612754, 0},
+  {"numeric-code", 1612709890, "NumericString"},
+  {"ub-postal-code-length", 524298, "1"},
+  {"printable-code", 538968066, "PrintableString"},
+  {"ub-postal-code-length", 524298, "1"},
+  {"physical-delivery-office-name", 1342177283, "10"},
+  {"PhysicalDeliveryOfficeName", 1073741826, "PDSParameter"},
+  {"physical-delivery-office-number", 1342177283, "11"},
+  {"PhysicalDeliveryOfficeNumber", 1073741826, "PDSParameter"},
+  {"extension-OR-address-components", 1342177283, "12"},
+  {"ExtensionORAddressComponents", 1073741826, "PDSParameter"},
+  {"physical-delivery-personal-name", 1342177283, "13"},
+  {"PhysicalDeliveryPersonalName", 1073741826, "PDSParameter"},
+  {"physical-delivery-organization-name", 1342177283, "14"},
+  {"PhysicalDeliveryOrganizationName", 1073741826, "PDSParameter"},
+  {"extension-physical-delivery-address-components", 1342177283, "15"},
+  {"ExtensionPhysicalDeliveryAddressComponents", 1073741826, "PDSParameter"},
+  {"unformatted-postal-address", 1342177283, "16"},
+  {"UnformattedPostalAddress", 1610612750, 0},
+  {"printable-address", 1814052875, 0},
+  {"ub-pds-physical-address-lines", 1074266122, "1"},
+  {0, 538968066, "PrintableString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"teletex-string", 740311042, "TeletexString"},
+  {"ub-unformatted-address-length", 524298, "1"},
+  {"street-address", 1342177283, "17"},
+  {"StreetAddress", 1073741826, "PDSParameter"},
+  {"post-office-box-address", 1342177283, "18"},
+  {"PostOfficeBoxAddress", 1073741826, "PDSParameter"},
+  {"poste-restante-address", 1342177283, "19"},
+  {"PosteRestanteAddress", 1073741826, "PDSParameter"},
+  {"unique-postal-name", 1342177283, "20"},
+  {"UniquePostalName", 1073741826, "PDSParameter"},
+  {"local-postal-attributes", 1342177283, "21"},
+  {"LocalPostalAttributes", 1073741826, "PDSParameter"},
+  {"PDSParameter", 1610612750, 0},
+  {"printable-string", 1814052866, "PrintableString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"teletex-string", 740311042, "TeletexString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"extended-network-address", 1342177283, "22"},
+  {"ExtendedNetworkAddress", 1610612754, 0},
+  {"e163-4-address", 1610612741, 0},
+  {"number", 1612718082, "NumericString"},
+  {0, 1073743880, "0"},
+  {"ub-e163-4-number-length", 524298, "1"},
+  {"sub-address", 538992642, "NumericString"},
+  {0, 1073743880, "1"},
+  {"ub-e163-4-sub-address-length", 524298, "1"},
+  {"psap-address", 536879106, "PresentationAddress"},
+  {0, 2056, "0"},
+  {"PresentationAddress", 1610612741, 0},
+  {"pSelector", 1610637319, 0},
+  {0, 2056, "0"},
+  {"sSelector", 1610637319, 0},
+  {0, 2056, "1"},
+  {"tSelector", 1610637319, 0},
+  {0, 2056, "2"},
+  {"nAddresses", 538976271, 0},
+  {0, 1073743880, "3"},
+  {"MAX", 1074266122, "1"},
+  {0, 7, 0},
+  {"terminal-type", 1342177283, "23"},
+  {"TerminalType", 1610874883, 0},
+  {"telex", 1073741825, "3"},
+  {"teletex", 1073741825, "4"},
+  {"g3-facsimile", 1073741825, "5"},
+  {"g4-facsimile", 1073741825, "6"},
+  {"ia5-terminal", 1073741825, "7"},
+  {"videotex", 1, "8"},
+  {"teletex-domain-defined-attributes", 1342177283, "6"},
+  {"TeletexDomainDefinedAttributes", 1612709899, 0},
+  {"ub-domain-defined-attributes", 1074266122, "1"},
+  {0, 2, "TeletexDomainDefinedAttribute"},
+  {"TeletexDomainDefinedAttribute", 1610612741, 0},
+  {"type", 1612709890, "TeletexString"},
+  {"ub-domain-defined-attribute-type-length", 524298, "1"},
+  {"value", 538968066, "TeletexString"},
+  {"ub-domain-defined-attribute-value-length", 524298, "1"},
+  {"ub-name", 1342177283, "32768"},
+  {"ub-common-name", 1342177283, "64"},
+  {"ub-locality-name", 1342177283, "128"},
+  {"ub-state-name", 1342177283, "128"},
+  {"ub-organization-name", 1342177283, "64"},
+  {"ub-organizational-unit-name", 1342177283, "64"},
+  {"ub-title", 1342177283, "64"},
+  {"ub-match", 1342177283, "128"},
+  {"ub-emailaddress-length", 1342177283, "128"},
+  {"ub-common-name-length", 1342177283, "64"},
+  {"ub-country-name-alpha-length", 1342177283, "2"},
+  {"ub-country-name-numeric-length", 1342177283, "3"},
+  {"ub-domain-defined-attributes", 1342177283, "4"},
+  {"ub-domain-defined-attribute-type-length", 1342177283, "8"},
+  {"ub-domain-defined-attribute-value-length", 1342177283, "128"},
+  {"ub-domain-name-length", 1342177283, "16"},
+  {"ub-extension-attributes", 1342177283, "256"},
+  {"ub-e163-4-number-length", 1342177283, "15"},
+  {"ub-e163-4-sub-address-length", 1342177283, "40"},
+  {"ub-generation-qualifier-length", 1342177283, "3"},
+  {"ub-given-name-length", 1342177283, "16"},
+  {"ub-initials-length", 1342177283, "5"},
+  {"ub-integer-options", 1342177283, "256"},
+  {"ub-numeric-user-id-length", 1342177283, "32"},
+  {"ub-organization-name-length", 1342177283, "64"},
+  {"ub-organizational-unit-name-length", 1342177283, "32"},
+  {"ub-organizational-units", 1342177283, "4"},
+  {"ub-pds-name-length", 1342177283, "16"},
+  {"ub-pds-parameter-length", 1342177283, "30"},
+  {"ub-pds-physical-address-lines", 1342177283, "6"},
+  {"ub-postal-code-length", 1342177283, "16"},
+  {"ub-surname-length", 1342177283, "40"},
+  {"ub-terminal-id-length", 1342177283, "24"},
+  {"ub-unformatted-address-length", 1342177283, "180"},
+  {"ub-x121-address-length", 1342177283, "16"},
+  {"pkcs-7-ContentInfo", 1610612741, 0},
+  {"contentType", 1073741826, "pkcs-7-ContentType"},
+  {"content", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"contentType", 1, 0},
+  {"pkcs-7-DigestInfo", 1610612741, 0},
+  {"digestAlgorithm", 1073741826, "pkcs-7-DigestAlgorithmIdentifier"},
+  {"digest", 2, "pkcs-7-Digest"},
+  {"pkcs-7-Digest", 1073741831, 0},
+  {"pkcs-7-ContentType", 1073741836, 0},
+  {"pkcs-7-SignedData", 1610612741, 0},
+  {"version", 1073741826, "pkcs-7-CMSVersion"},
+  {"digestAlgorithms", 1073741826, "pkcs-7-DigestAlgorithmIdentifiers"},
+  {"encapContentInfo", 1073741826, "pkcs-7-EncapsulatedContentInfo"},
+  {"certificates", 1610637314, "pkcs-7-CertificateSet"},
+  {0, 4104, "0"},
+  {"crls", 1610637314, "pkcs-7-CertificateRevocationLists"},
+  {0, 4104, "1"},
+  {"signerInfos", 2, "pkcs-7-SignerInfos"},
+  {"pkcs-7-CMSVersion", 1610874883, 0},
+  {"v0", 1073741825, "0"},
+  {"v1", 1073741825, "1"},
+  {"v2", 1073741825, "2"},
+  {"v3", 1073741825, "3"},
+  {"v4", 1, "4"},
+  {"pkcs-7-DigestAlgorithmIdentifiers", 1610612751, 0},
+  {0, 2, "pkcs-7-DigestAlgorithmIdentifier"},
+  {"pkcs-7-DigestAlgorithmIdentifier", 1073741826, "AlgorithmIdentifier"},
+  {"pkcs-7-EncapsulatedContentInfo", 1610612741, 0},
+  {"eContentType", 1073741826, "pkcs-7-ContentType"},
+  {"eContent", 536895495, 0},
+  {0, 2056, "0"},
+  {"pkcs-7-CertificateRevocationLists", 1610612751, 0},
+  {0, 13, 0},
+  {"pkcs-7-CertificateChoices", 1610612754, 0},
+  {"certificate", 13, 0},
+  {"pkcs-7-CertificateSet", 1610612751, 0},
+  {0, 2, "pkcs-7-CertificateChoices"},
+  {"pkcs-7-SignerInfos", 1610612751, 0},
+  {0, 13, 0},
+  {"pkcs-10-CertificationRequestInfo", 1610612741, 0},
+  {"version", 1610874883, 0},
+  {"v1", 1, "0"},
+  {"subject", 1073741826, "Name"},
+  {"subjectPKInfo", 1073741826, "SubjectPublicKeyInfo"},
+  {"attributes", 536879106, "Attributes"},
+  {0, 4104, "0"},
+  {"Attributes", 1610612751, 0},
+  {0, 2, "Attribute"},
+  {"pkcs-10-CertificationRequest", 1610612741, 0},
+  {"certificationRequestInfo", 1073741826, "pkcs-10-CertificationRequestInfo"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"pkcs-9-ub-challengePassword", 1342177283, "255"},
+  {"pkcs-9-certTypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "22"},
+  {"pkcs-9-crlTypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "23"},
+  {"pkcs-9-at-challengePassword", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "7"},
+  {"pkcs-9-challengePassword", 1610612754, 0},
+  {"printableString", 1612709890, "PrintableString"},
+  {"pkcs-9-ub-challengePassword", 524298, "1"},
+  {"utf8String", 538968066, "UTF8String"},
+  {"pkcs-9-ub-challengePassword", 524298, "1"},
+  {"pkcs-9-at-localKeyId", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "21"},
+  {"pkcs-9-localKeyId", 1073741831, 0},
+  {"pkcs-9-at-friendlyName", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "20"},
+  {"pkcs-9-friendlyName", 1612709890, "BMPString"},
+  {"255", 524298, "1"},
+  {"pkcs-8-PrivateKeyInfo", 1610612741, 0},
+  {"version", 1073741826, "pkcs-8-Version"},
+  {"privateKeyAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"privateKey", 1073741826, "pkcs-8-PrivateKey"},
+  {"attributes", 536895490, "Attributes"},
+  {0, 4104, "0"},
+  {"pkcs-8-Version", 1610874883, 0},
+  {"v1", 1, "0"},
+  {"pkcs-8-PrivateKey", 1073741831, 0},
+  {"pkcs-8-Attributes", 1610612751, 0},
+  {0, 2, "Attribute"},
+  {"pkcs-8-EncryptedPrivateKeyInfo", 1610612741, 0},
+  {"encryptionAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"encryptedData", 2, "pkcs-8-EncryptedData"},
+  {"pkcs-8-EncryptedData", 1073741831, 0},
+  {"pkcs-5", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "5"},
+  {"pkcs-5-encryptionAlgorithm", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {0, 1, "3"},
+  {"pkcs-5-des-EDE3-CBC", 1879048204, 0},
+  {0, 1073741825, "pkcs-5-encryptionAlgorithm"},
+  {0, 1, "7"},
+  {"pkcs-5-des-EDE3-CBC-params", 1612709895, 0},
+  {0, 1048586, "8"},
+  {"pkcs-5-id-PBES2", 1879048204, 0},
+  {0, 1073741825, "pkcs-5"},
+  {0, 1, "13"},
+  {"pkcs-5-PBES2-params", 1610612741, 0},
+  {"keyDerivationFunc", 1073741826, "AlgorithmIdentifier"},
+  {"encryptionScheme", 2, "AlgorithmIdentifier"},
+  {"pkcs-5-id-PBKDF2", 1879048204, 0},
+  {0, 1073741825, "pkcs-5"},
+  {0, 1, "12"},
+  {"pkcs-5-PBKDF2-params", 1610612741, 0},
+  {"salt", 1610612754, 0},
+  {"specified", 1073741831, 0},
+  {"otherSource", 2, "AlgorithmIdentifier"},
+  {"iterationCount", 1611137027, 0},
+  {"1", 10, "MAX"},
+  {"keyLength", 1611153411, 0},
+  {"1", 10, "MAX"},
+  {"prf", 16386, "AlgorithmIdentifier"},
+  {"pkcs-12", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "12"},
+  {"pkcs-12-PFX", 1610612741, 0},
+  {"version", 1610874883, 0},
+  {"v3", 1, "3"},
+  {"authSafe", 1073741826, "pkcs-7-ContentInfo"},
+  {"macData", 16386, "pkcs-12-MacData"},
+  {"pkcs-12-PbeParams", 1610612741, 0},
+  {"salt", 1073741831, 0},
+  {"iterations", 3, 0},
+  {"pkcs-12-MacData", 1610612741, 0},
+  {"mac", 1073741826, "pkcs-7-DigestInfo"},
+  {"macSalt", 1073741831, 0},
+  {"iterations", 536903683, 0},
+  {0, 9, "1"},
+  {"pkcs-12-AuthenticatedSafe", 1610612747, 0},
+  {0, 2, "pkcs-7-ContentInfo"},
+  {"pkcs-12-SafeContents", 1610612747, 0},
+  {0, 2, "pkcs-12-SafeBag"},
+  {"pkcs-12-SafeBag", 1610612741, 0},
+  {"bagId", 1073741836, 0},
+  {"bagValue", 1614815245, 0},
+  {0, 1073743880, "0"},
+  {"badId", 1, 0},
+  {"bagAttributes", 536887311, 0},
+  {0, 2, "pkcs-12-PKCS12Attribute"},
+  {"pkcs-12-bagtypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-12"},
+  {0, 1073741825, "10"},
+  {0, 1, "1"},
+  {"pkcs-12-keyBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "1"},
+  {"pkcs-12-pkcs8ShroudedKeyBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "2"},
+  {"pkcs-12-certBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "3"},
+  {"pkcs-12-crlBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "4"},
+  {"pkcs-12-KeyBag", 1073741826, "pkcs-8-PrivateKeyInfo"},
+  {"pkcs-12-PKCS8ShroudedKeyBag", 1073741826, "pkcs-8-EncryptedPrivateKeyInfo"},
+  {"pkcs-12-CertBag", 1610612741, 0},
+  {"certId", 1073741836, 0},
+  {"certValue", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"certId", 1, 0},
+  {"pkcs-12-CRLBag", 1610612741, 0},
+  {"crlId", 1073741836, 0},
+  {"crlValue", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"crlId", 1, 0},
+  {"pkcs-12-PKCS12Attribute", 1073741826, "Attribute"},
+  {"pkcs-7-data", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1073741825, "1"},
+  {"pkcs7", 1073741825, "7"},
+  {0, 1, "1"},
+  {"pkcs-7-encryptedData", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1073741825, "1"},
+  {"pkcs7", 1073741825, "7"},
+  {0, 1, "6"},
+  {"pkcs-7-Data", 1073741831, 0},
+  {"pkcs-7-EncryptedData", 1610612741, 0},
+  {"version", 1073741826, "pkcs-7-CMSVersion"},
+  {"encryptedContentInfo", 1073741826, "pkcs-7-EncryptedContentInfo"},
+  {"unprotectedAttrs", 536895490, "pkcs-7-UnprotectedAttributes"},
+  {0, 4104, "1"},
+  {"pkcs-7-EncryptedContentInfo", 1610612741, 0},
+  {"contentType", 1073741826, "pkcs-7-ContentType"},
+  {"contentEncryptionAlgorithm", 1073741826,
+   "pkcs-7-ContentEncryptionAlgorithmIdentifier"},
+  {"encryptedContent", 536895490, "pkcs-7-EncryptedContent"},
+  {0, 4104, "0"},
+  {"pkcs-7-ContentEncryptionAlgorithmIdentifier", 1073741826,
+   "AlgorithmIdentifier"},
+  {"pkcs-7-EncryptedContent", 1073741831, 0},
+  {"pkcs-7-UnprotectedAttributes", 1612709903, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Attribute"},
+  {"id-at-ldap-DC", 1880096780, "AttributeType"},
+  {0, 1073741825, "0"},
+  {0, 1073741825, "9"},
+  {0, 1073741825, "2342"},
+  {0, 1073741825, "19200300"},
+  {0, 1073741825, "100"},
+  {0, 1073741825, "1"},
+  {0, 1, "25"},
+  {"ldap-DC", 1073741826, "IA5String"},
+  {"id-at-ldap-UID", 1880096780, "AttributeType"},
+  {0, 1073741825, "0"},
+  {0, 1073741825, "9"},
+  {0, 1073741825, "2342"},
+  {0, 1073741825, "19200300"},
+  {0, 1073741825, "100"},
+  {0, 1073741825, "1"},
+  {0, 1, "1"},
+  {"ldap-UID", 1073741826, "DirectoryString"},
+  {"id-pda", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "9"},
+  {"id-pda-dateOfBirth", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "1"},
+  {"DateOfBirth", 1082130449, 0},
+  {"id-pda-placeOfBirth", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "2"},
+  {"PlaceOfBirth", 1073741826, "DirectoryString"},
+  {"id-pda-gender", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "3"},
+  {"Gender", 1612709890, "PrintableString"},
+  {0, 1048586, "1"},
+  {"id-pda-countryOfCitizenship", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "4"},
+  {"CountryOfCitizenship", 1612709890, "PrintableString"},
+  {0, 1048586, "2"},
+  {"id-pda-countryOfResidence", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "5"},
+  {"CountryOfResidence", 538968066, "PrintableString"},
+  {0, 1048586, "2"},
+  {0, 0, 0}
+};
diff --git a/tests/qapi-schema/alternate-clash.err b/tests/qapi-schema/alternate-clash.err
index 51bea3e27..a475ab634 100644
--- a/tests/qapi-schema/alternate-clash.err
+++ b/tests/qapi-schema/alternate-clash.err
@@ -1 +1 @@
-tests/qapi-schema/alternate-clash.json:2: Alternate 'Alt1' member 'ONE' clashes with 'one'
+tests/qapi-schema/alternate-clash.json:7: Alternate 'Alt1' member 'a_b' clashes with 'a-b'
diff --git a/tests/qapi-schema/alternate-clash.json b/tests/qapi-schema/alternate-clash.json
index 39479353b..6d73bc527 100644
--- a/tests/qapi-schema/alternate-clash.json
+++ b/tests/qapi-schema/alternate-clash.json
@@ -1,3 +1,8 @@
-# we detect C enum collisions in an alternate
+# Alternate branch name collision
+# Reject an alternate that would result in a collision in generated C
+# names (this would try to generate two enum values 'ALT1_KIND_A_B').
+# TODO: In the future, if alternates are simplified to not generate
+# the implicit Alt1Kind enum, we would still have a collision with the
+# resulting C union trying to have two members named 'a_b'.
 { 'alternate': 'Alt1',
-  'data': { 'one': 'str', 'ONE': 'int' } }
+  'data': { 'a-b': 'str', 'a_b': 'int' } }
diff --git a/tests/qapi-schema/enum-empty.err b/tests/qapi-schema/alternate-empty.err
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/enum-empty.err
+++ b/tests/qapi-schema/alternate-empty.err
diff --git a/tests/qapi-schema/alternate-good.exit b/tests/qapi-schema/alternate-empty.exit
index 573541ac9..573541ac9 100644
--- a/tests/qapi-schema/alternate-good.exit
+++ b/tests/qapi-schema/alternate-empty.exit
diff --git a/tests/qapi-schema/alternate-empty.json b/tests/qapi-schema/alternate-empty.json
new file mode 100644
index 000000000..db3820f84
--- /dev/null
+++ b/tests/qapi-schema/alternate-empty.json
@@ -0,0 +1,2 @@
+# FIXME - alternates should list at least two types to be useful
+{ 'alternate': 'Alt', 'data': { 'i': 'int' } }
diff --git a/tests/qapi-schema/alternate-empty.out b/tests/qapi-schema/alternate-empty.out
new file mode 100644
index 000000000..0f153b6f6
--- /dev/null
+++ b/tests/qapi-schema/alternate-empty.out
@@ -0,0 +1,4 @@
+object :empty
+alternate Alt
+    case i: int
+enum AltKind ['i']
diff --git a/tests/qapi-schema/alternate-good.json b/tests/qapi-schema/alternate-good.json
deleted file mode 100644
index 33717704c..000000000
--- a/tests/qapi-schema/alternate-good.json
+++ /dev/null
@@ -1,9 +0,0 @@
-# Working example of alternate
-{ 'struct': 'Data',
-  'data': { '*number': 'int', '*name': 'str' } }
-{ 'enum': 'Enum',
-  'data': [ 'hello', 'world' ] }
-{ 'alternate': 'Alt',
-  'data': { 'value': 'int',
-            'string': 'Enum',
-            'struct': 'Data' } }
diff --git a/tests/qapi-schema/alternate-good.out b/tests/qapi-schema/alternate-good.out
deleted file mode 100644
index 99848eefb..000000000
--- a/tests/qapi-schema/alternate-good.out
+++ /dev/null
@@ -1,6 +0,0 @@
-[OrderedDict([('struct', 'Data'), ('data', OrderedDict([('*number', 'int'), ('*name', 'str')]))]),
- OrderedDict([('enum', 'Enum'), ('data', ['hello', 'world'])]),
- OrderedDict([('alternate', 'Alt'), ('data', OrderedDict([('value', 'int'), ('string', 'Enum'), ('struct', 'Data')]))])]
-[{'enum_name': 'Enum', 'enum_values': ['hello', 'world']},
- {'enum_name': 'AltKind', 'enum_values': None}]
-[OrderedDict([('struct', 'Data'), ('data', OrderedDict([('*number', 'int'), ('*name', 'str')]))])]
diff --git a/tests/qapi-schema/alternate-nested.json b/tests/qapi-schema/alternate-nested.json
index c4233b9f3..8e2218649 100644
--- a/tests/qapi-schema/alternate-nested.json
+++ b/tests/qapi-schema/alternate-nested.json
@@ -2,4 +2,4 @@
 { 'alternate': 'Alt1',
   'data': { 'name': 'str', 'value': 'int' } }
 { 'alternate': 'Alt2',
-  'data': { 'nested': 'Alt1' } }
+  'data': { 'nested': 'Alt1', 'b': 'bool' } }
diff --git a/tests/qapi-schema/alternate-unknown.json b/tests/qapi-schema/alternate-unknown.json
index ad5c10302..08c80dced 100644
--- a/tests/qapi-schema/alternate-unknown.json
+++ b/tests/qapi-schema/alternate-unknown.json
@@ -1,3 +1,3 @@
 # we reject an alternate with unknown type in branch
 { 'alternate': 'Alt',
-  'data': { 'unknown': 'MissingType' } }
+  'data': { 'unknown': 'MissingType', 'i': 'int' } }
diff --git a/tests/qapi-schema/args-alternate.err b/tests/qapi-schema/args-alternate.err
new file mode 100644
index 000000000..3086eae56
--- /dev/null
+++ b/tests/qapi-schema/args-alternate.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-alternate.json:3: 'data' for command 'oops' cannot use alternate type 'Alt'
diff --git a/tests/qapi-schema/data-array-empty.exit b/tests/qapi-schema/args-alternate.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/data-array-empty.exit
+++ b/tests/qapi-schema/args-alternate.exit
diff --git a/tests/qapi-schema/args-alternate.json b/tests/qapi-schema/args-alternate.json
new file mode 100644
index 000000000..69e94d481
--- /dev/null
+++ b/tests/qapi-schema/args-alternate.json
@@ -0,0 +1,3 @@
+# we do not allow alternate arguments
+{ 'alternate': 'Alt', 'data': { 'case1': 'int', 'case2': 'str' } }
+{ 'command': 'oops', 'data': 'Alt' }
diff --git a/tests/qapi-schema/enum-union-clash.out b/tests/qapi-schema/args-alternate.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/enum-union-clash.out
+++ b/tests/qapi-schema/args-alternate.out
diff --git a/tests/qapi-schema/args-any.err b/tests/qapi-schema/args-any.err
new file mode 100644
index 000000000..bf9b5e073
--- /dev/null
+++ b/tests/qapi-schema/args-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-any.json:2: 'data' for command 'oops' cannot use built-in type 'any'
diff --git a/tests/qapi-schema/data-array-unknown.exit b/tests/qapi-schema/args-any.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/data-array-unknown.exit
+++ b/tests/qapi-schema/args-any.exit
diff --git a/tests/qapi-schema/args-any.json b/tests/qapi-schema/args-any.json
new file mode 100644
index 000000000..58fe5e470
--- /dev/null
+++ b/tests/qapi-schema/args-any.json
@@ -0,0 +1,2 @@
+# we do not allow an 'any' argument
+{ 'command': 'oops', 'data': 'any' }
diff --git a/tests/qapi-schema/flat-union-base-star.out b/tests/qapi-schema/args-any.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/flat-union-base-star.out
+++ b/tests/qapi-schema/args-any.out
diff --git a/tests/qapi-schema/data-array-empty.err b/tests/qapi-schema/args-array-empty.err
index f713f1489..cb7ed33b3 100644
--- a/tests/qapi-schema/data-array-empty.err
+++ b/tests/qapi-schema/args-array-empty.err
@@ -1 +1 @@
-tests/qapi-schema/data-array-empty.json:2: Member 'empty' of 'data' for command 'oops': array type must contain single type name
+tests/qapi-schema/args-array-empty.json:2: Member 'empty' of 'data' for command 'oops': array type must contain single type name
diff --git a/tests/qapi-schema/data-int.exit b/tests/qapi-schema/args-array-empty.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/data-int.exit
+++ b/tests/qapi-schema/args-array-empty.exit
diff --git a/tests/qapi-schema/data-array-empty.json b/tests/qapi-schema/args-array-empty.json
index 652dcfb24..652dcfb24 100644
--- a/tests/qapi-schema/data-array-empty.json
+++ b/tests/qapi-schema/args-array-empty.json
diff --git a/tests/qapi-schema/flat-union-branch-clash.out b/tests/qapi-schema/args-array-empty.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/flat-union-branch-clash.out
+++ b/tests/qapi-schema/args-array-empty.out
diff --git a/tests/qapi-schema/args-array-unknown.err b/tests/qapi-schema/args-array-unknown.err
new file mode 100644
index 000000000..cd7a0f98d
--- /dev/null
+++ b/tests/qapi-schema/args-array-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-array-unknown.json:2: Member 'array' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/data-member-array-bad.exit b/tests/qapi-schema/args-array-unknown.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/data-member-array-bad.exit
+++ b/tests/qapi-schema/args-array-unknown.exit
diff --git a/tests/qapi-schema/data-array-unknown.json b/tests/qapi-schema/args-array-unknown.json
index 6f3e88331..6f3e88331 100644
--- a/tests/qapi-schema/data-array-unknown.json
+++ b/tests/qapi-schema/args-array-unknown.json
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-array-unknown.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/flat-union-reverse-define.err
+++ b/tests/qapi-schema/args-array-unknown.out
diff --git a/tests/qapi-schema/args-int.err b/tests/qapi-schema/args-int.err
new file mode 100644
index 000000000..dc1d2504f
--- /dev/null
+++ b/tests/qapi-schema/args-int.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-int.json:2: 'data' for command 'oops' cannot use built-in type 'int'
diff --git a/tests/qapi-schema/data-member-unknown.exit b/tests/qapi-schema/args-int.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/data-member-unknown.exit
+++ b/tests/qapi-schema/args-int.exit
diff --git a/tests/qapi-schema/data-int.json b/tests/qapi-schema/args-int.json
index a334d92e8..a334d92e8 100644
--- a/tests/qapi-schema/data-int.json
+++ b/tests/qapi-schema/args-int.json
diff --git a/tests/qapi-schema/nested-struct-returns.out b/tests/qapi-schema/args-int.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/nested-struct-returns.out
+++ b/tests/qapi-schema/args-int.out
diff --git a/tests/qapi-schema/args-invalid.err b/tests/qapi-schema/args-invalid.err
new file mode 100644
index 000000000..fe1e94975
--- /dev/null
+++ b/tests/qapi-schema/args-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-invalid.json:1: 'data' for command 'foo' should be a dictionary or type name
diff --git a/tests/qapi-schema/data-unknown.exit b/tests/qapi-schema/args-invalid.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/data-unknown.exit
+++ b/tests/qapi-schema/args-invalid.exit
diff --git a/tests/qapi-schema/args-invalid.json b/tests/qapi-schema/args-invalid.json
new file mode 100644
index 000000000..db0981341
--- /dev/null
+++ b/tests/qapi-schema/args-invalid.json
@@ -0,0 +1,2 @@
+{ 'command': 'foo',
+  'data': false }
diff --git a/tests/qapi-schema/returns-int.err b/tests/qapi-schema/args-invalid.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/returns-int.err
+++ b/tests/qapi-schema/args-invalid.out
diff --git a/tests/qapi-schema/data-member-array-bad.err b/tests/qapi-schema/args-member-array-bad.err
index 2c072d598..881b4d954 100644
--- a/tests/qapi-schema/data-member-array-bad.err
+++ b/tests/qapi-schema/args-member-array-bad.err
@@ -1 +1 @@
-tests/qapi-schema/data-member-array-bad.json:2: Member 'member' of 'data' for command 'oops': array type must contain single type name
+tests/qapi-schema/args-member-array-bad.json:2: Member 'member' of 'data' for command 'oops': array type must contain single type name
diff --git a/tests/qapi-schema/enum-union-clash.exit b/tests/qapi-schema/args-member-array-bad.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/enum-union-clash.exit
+++ b/tests/qapi-schema/args-member-array-bad.exit
diff --git a/tests/qapi-schema/data-member-array-bad.json b/tests/qapi-schema/args-member-array-bad.json
index b2ff144ec..b2ff144ec 100644
--- a/tests/qapi-schema/data-member-array-bad.json
+++ b/tests/qapi-schema/args-member-array-bad.json
diff --git a/tests/qapi-schema/type-bypass-no-gen.out b/tests/qapi-schema/args-member-array-bad.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/type-bypass-no-gen.out
+++ b/tests/qapi-schema/args-member-array-bad.out
diff --git a/tests/qapi-schema/args-member-unknown.err b/tests/qapi-schema/args-member-unknown.err
new file mode 100644
index 000000000..f6f82828c
--- /dev/null
+++ b/tests/qapi-schema/args-member-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-member-unknown.json:2: Member 'member' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/flat-union-base-star.exit b/tests/qapi-schema/args-member-unknown.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/flat-union-base-star.exit
+++ b/tests/qapi-schema/args-member-unknown.exit
diff --git a/tests/qapi-schema/data-member-unknown.json b/tests/qapi-schema/args-member-unknown.json
index 342a41ec9..342a41ec9 100644
--- a/tests/qapi-schema/data-member-unknown.json
+++ b/tests/qapi-schema/args-member-unknown.json
diff --git a/tests/qapi-schema/type-bypass.err b/tests/qapi-schema/args-member-unknown.out
index e69de29bb..e69de29bb 100644
--- a/tests/qapi-schema/type-bypass.err
+++ b/tests/qapi-schema/args-member-unknown.out
diff --git a/tests/qapi-schema/args-name-clash.err b/tests/qapi-schema/args-name-clash.err
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/args-name-clash.err
diff --git a/tests/qapi-schema/data-member-array.exit b/tests/qapi-schema/args-name-clash.exit
index 573541ac9..573541ac9 100644
--- a/tests/qapi-schema/data-member-array.exit
+++ b/tests/qapi-schema/args-name-clash.exit
diff --git a/tests/qapi-schema/args-name-clash.json b/tests/qapi-schema/args-name-clash.json
new file mode 100644
index 000000000..9e8f88916
--- /dev/null
+++ b/tests/qapi-schema/args-name-clash.json
@@ -0,0 +1,5 @@
+# C member name collision
+# FIXME - This parses, but fails to compile, because the C struct is given
+# two 'a_b' members.  Either reject this at parse time, or munge the C names
+# to avoid the collision.
+{ 'command': 'oops', 'data': { 'a-b': 'str', 'a_b': 'str' } }
diff --git a/tests/qapi-schema/args-name-clash.out b/tests/qapi-schema/args-name-clash.out
new file mode 100644
index 000000000..9b2f6e4d5
--- /dev/null
+++ b/tests/qapi-schema/args-name-clash.out
@@ -0,0 +1,6 @@
+object :empty
+object :obj-oops-arg
+    member a-b: str optional=False
+    member a_b: str optional=False
+command oops :obj-oops-arg -> None
+   gen=True success_response=True
diff --git a/tests/qapi-schema/args-union.err b/tests/qapi-schema/args-union.err
new file mode 100644
index 000000000..1d693d74d
--- /dev/null
+++ b/tests/qapi-schema/args-union.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-union.json:4: 'data' for command 'oops' cannot use union type 'Uni'
diff --git a/tests/qapi-schema/flat-union-branch-clash.exit b/tests/qapi-schema/args-union.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/flat-union-branch-clash.exit
+++ b/tests/qapi-schema/args-union.exit
diff --git a/tests/qapi-schema/args-union.json b/tests/qapi-schema/args-union.json
new file mode 100644
index 000000000..7bdcbb7f0
--- /dev/null
+++ b/tests/qapi-schema/args-union.json
@@ -0,0 +1,4 @@
+# we do not allow union arguments
+# TODO should we support this?
+{ 'union': 'Uni', 'data': { 'case1': 'int', 'case2': 'str' } }
+{ 'command': 'oops', 'data': 'Uni' }
diff --git a/tests/qapi-schema/args-union.out b/tests/qapi-schema/args-union.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/args-union.out
diff --git a/tests/qapi-schema/args-unknown.err b/tests/qapi-schema/args-unknown.err
new file mode 100644
index 000000000..4d91ec869
--- /dev/null
+++ b/tests/qapi-schema/args-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-unknown.json:2: 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/nested-struct-returns.exit b/tests/qapi-schema/args-unknown.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/nested-struct-returns.exit
+++ b/tests/qapi-schema/args-unknown.exit
diff --git a/tests/qapi-schema/data-unknown.json b/tests/qapi-schema/args-unknown.json
index 32aba43b3..32aba43b3 100644
--- a/tests/qapi-schema/data-unknown.json
+++ b/tests/qapi-schema/args-unknown.json
diff --git a/tests/qapi-schema/args-unknown.out b/tests/qapi-schema/args-unknown.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/args-unknown.out
diff --git a/tests/qapi-schema/command-int.json b/tests/qapi-schema/command-int.json
index c90d408ab..9a62554fc 100644
--- a/tests/qapi-schema/command-int.json
+++ b/tests/qapi-schema/command-int.json
@@ -1,3 +1,2 @@
 # we reject collisions between commands and types
-{ 'command': 'int', 'data': { 'character': 'str' },
-  'returns': { 'value': 'int' } }
+{ 'command': 'int', 'data': { 'character': 'str' } }
diff --git a/tests/qapi-schema/comments.out b/tests/qapi-schema/comments.out
index 4ce3dcf12..9e2c656fa 100644
--- a/tests/qapi-schema/comments.out
+++ b/tests/qapi-schema/comments.out
@@ -1,3 +1,2 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+object :empty
+enum Status ['good', 'bad', 'ugly']
diff --git a/tests/qapi-schema/data-array-unknown.err b/tests/qapi-schema/data-array-unknown.err
deleted file mode 100644
index 8b731bbcc..000000000
--- a/tests/qapi-schema/data-array-unknown.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-array-unknown.json:2: Member 'array' of 'data' for command 'oops' uses unknown type 'array of NoSuchType'
diff --git a/tests/qapi-schema/data-int.err b/tests/qapi-schema/data-int.err
deleted file mode 100644
index 1a9b077c0..000000000
--- a/tests/qapi-schema/data-int.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-int.json:2: 'data' for command 'oops' cannot use built-in type 'int'
diff --git a/tests/qapi-schema/data-member-array.json b/tests/qapi-schema/data-member-array.json
deleted file mode 100644
index e6f7f5da1..000000000
--- a/tests/qapi-schema/data-member-array.json
+++ /dev/null
@@ -1,4 +0,0 @@
-# valid array members
-{ 'enum': 'abc', 'data': [ 'a', 'b', 'c' ] }
-{ 'struct': 'def', 'data': { 'array': [ 'abc' ] } }
-{ 'command': 'okay', 'data': { 'member1': [ 'int' ], 'member2': [ 'def' ] } }
diff --git a/tests/qapi-schema/data-member-array.out b/tests/qapi-schema/data-member-array.out
deleted file mode 100644
index c39fa2548..000000000
--- a/tests/qapi-schema/data-member-array.out
+++ /dev/null
@@ -1,5 +0,0 @@
-[OrderedDict([('enum', 'abc'), ('data', ['a', 'b', 'c'])]),
- OrderedDict([('struct', 'def'), ('data', OrderedDict([('array', ['abc'])]))]),
- OrderedDict([('command', 'okay'), ('data', OrderedDict([('member1', ['int']), ('member2', ['def'])]))])]
-[{'enum_name': 'abc', 'enum_values': ['a', 'b', 'c']}]
-[OrderedDict([('struct', 'def'), ('data', OrderedDict([('array', ['abc'])]))])]
diff --git a/tests/qapi-schema/data-member-unknown.err b/tests/qapi-schema/data-member-unknown.err
deleted file mode 100644
index ab905db80..000000000
--- a/tests/qapi-schema/data-member-unknown.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-member-unknown.json:2: Member 'member' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/data-unknown.err b/tests/qapi-schema/data-unknown.err
deleted file mode 100644
index 5b07277a9..000000000
--- a/tests/qapi-schema/data-unknown.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-unknown.json:2: 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/duplicate-key.err b/tests/qapi-schema/duplicate-key.err
index 768b276f8..6d02f8353 100644
--- a/tests/qapi-schema/duplicate-key.err
+++ b/tests/qapi-schema/duplicate-key.err
@@ -1 +1 @@
-tests/qapi-schema/duplicate-key.json:2:10: Duplicate key "key"
+tests/qapi-schema/duplicate-key.json:3:10: Duplicate key "key"
diff --git a/tests/qapi-schema/duplicate-key.json b/tests/qapi-schema/duplicate-key.json
index 1b55d8810..14ac0e8a4 100644
--- a/tests/qapi-schema/duplicate-key.json
+++ b/tests/qapi-schema/duplicate-key.json
@@ -1,2 +1,3 @@
+# QAPI cannot include the same key more than once in any {}
 { 'key': 'value',
   'key': 'value' }
diff --git a/tests/qapi-schema/empty.out b/tests/qapi-schema/empty.out
index b7f89a45c..272b1616f 100644
--- a/tests/qapi-schema/empty.out
+++ b/tests/qapi-schema/empty.out
@@ -1,3 +1 @@
-[]
-[]
-[]
+object :empty
diff --git a/tests/qapi-schema/enum-bad-prefix.err b/tests/qapi-schema/enum-bad-prefix.err
new file mode 100644
index 000000000..399f5f7af
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-prefix.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-bad-prefix.json:2: Enum 'MyEnum' requires a string for 'prefix'
diff --git a/tests/qapi-schema/type-bypass-no-gen.exit b/tests/qapi-schema/enum-bad-prefix.exit
index d00491fd7..d00491fd7 100644
--- a/tests/qapi-schema/type-bypass-no-gen.exit
+++ b/tests/qapi-schema/enum-bad-prefix.exit
diff --git a/tests/qapi-schema/enum-bad-prefix.json b/tests/qapi-schema/enum-bad-prefix.json
new file mode 100644
index 000000000..996f628f6
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-prefix.json
@@ -0,0 +1,2 @@
+# The prefix must be a string type
+{ 'enum': 'MyEnum', 'data': [ 'one' ], 'prefix': [ 'fish' ] }
diff --git a/tests/qapi-schema/enum-bad-prefix.out b/tests/qapi-schema/enum-bad-prefix.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-prefix.out
diff --git a/tests/qapi-schema/enum-empty.json b/tests/qapi-schema/enum-empty.json
deleted file mode 100644
index 40d4e85a2..000000000
--- a/tests/qapi-schema/enum-empty.json
+++ /dev/null
@@ -1,2 +0,0 @@
-# An empty enum, although unusual, is currently acceptable
-{ 'enum': 'MyEnum', 'data': [ ] }
diff --git a/tests/qapi-schema/enum-empty.out b/tests/qapi-schema/enum-empty.out
deleted file mode 100644
index 3b75c1613..000000000
--- a/tests/qapi-schema/enum-empty.out
+++ /dev/null
@@ -1,3 +0,0 @@
-[OrderedDict([('enum', 'MyEnum'), ('data', [])])]
-[{'enum_name': 'MyEnum', 'enum_values': []}]
-[]
diff --git a/tests/qapi-schema/enum-union-clash.err b/tests/qapi-schema/enum-union-clash.err
deleted file mode 100644
index c04e1a806..000000000
--- a/tests/qapi-schema/enum-union-clash.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/enum-union-clash.json:2: enum 'UnionKind' should not end in 'Kind'
diff --git a/tests/qapi-schema/event-case.out b/tests/qapi-schema/event-case.out
index 3764bc781..cdfd264f9 100644
--- a/tests/qapi-schema/event-case.out
+++ b/tests/qapi-schema/event-case.out
@@ -1,3 +1,2 @@
-[OrderedDict([('event', 'oops')])]
-[]
-[]
+object :empty
+event oops None
diff --git a/tests/qapi-schema/flat-union-bad-base.err b/tests/qapi-schema/flat-union-bad-base.err
index f9c31b2bf..79b8a71eb 100644
--- a/tests/qapi-schema/flat-union-bad-base.err
+++ b/tests/qapi-schema/flat-union-bad-base.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-bad-base.json:9: Flat union 'TestUnion' must have a string base field
+tests/qapi-schema/flat-union-bad-base.json:9: 'base' for union 'TestUnion' should be a type name
diff --git a/tests/qapi-schema/flat-union-base-any.err b/tests/qapi-schema/flat-union-base-any.err
new file mode 100644
index 000000000..646f1c9cd
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-base-any.json:8: 'base' for union 'TestUnion' cannot use built-in type 'any'
diff --git a/tests/qapi-schema/flat-union-base-any.exit b/tests/qapi-schema/flat-union-base-any.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-any.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-base-star.json b/tests/qapi-schema/flat-union-base-any.json
index 5099439a9..fe66b713e 100644
--- a/tests/qapi-schema/flat-union-base-star.json
+++ b/tests/qapi-schema/flat-union-base-any.json
@@ -6,7 +6,7 @@
 { 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
 { 'union': 'TestUnion',
-  'base': '**',
+  'base': 'any',
   'discriminator': 'enum1',
   'data': { 'value1': 'TestTypeA',
             'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-base-any.out b/tests/qapi-schema/flat-union-base-any.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-any.out
diff --git a/tests/qapi-schema/flat-union-base-star.err b/tests/qapi-schema/flat-union-base-star.err
deleted file mode 100644
index b7748f08b..000000000
--- a/tests/qapi-schema/flat-union-base-star.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/flat-union-base-star.json:8: Base '**' is not a valid struct
diff --git a/tests/qapi-schema/flat-union-base-union.err b/tests/qapi-schema/flat-union-base-union.err
index ede9859a3..f138395e4 100644
--- a/tests/qapi-schema/flat-union-base-union.err
+++ b/tests/qapi-schema/flat-union-base-union.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-base-union.json:11: Base 'UnionBase' is not a valid struct
+tests/qapi-schema/flat-union-base-union.json:14: 'base' for union 'TestUnion' cannot use union type 'UnionBase'
diff --git a/tests/qapi-schema/flat-union-base-union.json b/tests/qapi-schema/flat-union-base-union.json
index 6a8ea687a..98b4eba18 100644
--- a/tests/qapi-schema/flat-union-base-union.json
+++ b/tests/qapi-schema/flat-union-base-union.json
@@ -1,4 +1,7 @@
-# we require the base to be a struct
+# For now, we require the base to be a struct without variants
+# TODO: It would be possible to allow a union as a base, as long as all
+# permutations of QMP names exposed by base do not clash with any QMP
+# member names added by local variants.
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 { 'struct': 'TestTypeA',
diff --git a/tests/qapi-schema/flat-union-branch-clash.err b/tests/qapi-schema/flat-union-branch-clash.err
deleted file mode 100644
index f11276688..000000000
--- a/tests/qapi-schema/flat-union-branch-clash.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/flat-union-branch-clash.json:10: Member name 'name' of branch 'value1' clashes with base 'Base'
diff --git a/tests/qapi-schema/flat-union-clash-branch.err b/tests/qapi-schema/flat-union-clash-branch.err
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-branch.err
diff --git a/tests/qapi-schema/enum-empty.exit b/tests/qapi-schema/flat-union-clash-branch.exit
index 573541ac9..573541ac9 100644
--- a/tests/qapi-schema/enum-empty.exit
+++ b/tests/qapi-schema/flat-union-clash-branch.exit
diff --git a/tests/qapi-schema/flat-union-clash-branch.json b/tests/qapi-schema/flat-union-clash-branch.json
new file mode 100644
index 000000000..e59333603
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-branch.json
@@ -0,0 +1,18 @@
+# Flat union branch name collision
+# FIXME: this parses, but then fails to compile due to a duplicate 'c_d'
+# (one from the base member, the other from the branch name).  We should
+# either reject the collision at parse time, or munge the generated branch
+# name to allow this to compile.
+{ 'enum': 'TestEnum',
+  'data': [ 'base', 'c-d' ] }
+{ 'struct': 'Base',
+  'data': { 'enum1': 'TestEnum', '*c_d': 'str' } }
+{ 'struct': 'Branch1',
+  'data': { 'string': 'str' } }
+{ 'struct': 'Branch2',
+  'data': { 'value': 'int' } }
+{ 'union': 'TestUnion',
+  'base': 'Base',
+  'discriminator': 'enum1',
+  'data': { 'base': 'Branch1',
+            'c-d': 'Branch2' } }
diff --git a/tests/qapi-schema/flat-union-clash-branch.out b/tests/qapi-schema/flat-union-clash-branch.out
new file mode 100644
index 000000000..8e0da7360
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-branch.out
@@ -0,0 +1,14 @@
+object :empty
+object Base
+    member enum1: TestEnum optional=False
+    member c_d: str optional=True
+object Branch1
+    member string: str optional=False
+object Branch2
+    member value: int optional=False
+enum TestEnum ['base', 'c-d']
+object TestUnion
+    base Base
+    tag enum1
+    case base: Branch1
+    case c-d: Branch2
diff --git a/tests/qapi-schema/flat-union-clash-member.err b/tests/qapi-schema/flat-union-clash-member.err
new file mode 100644
index 000000000..2f0397a8a
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-member.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-clash-member.json:11: Member name 'name' of branch 'value1' clashes with base 'Base'
diff --git a/tests/qapi-schema/flat-union-clash-member.exit b/tests/qapi-schema/flat-union-clash-member.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-member.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-branch-clash.json b/tests/qapi-schema/flat-union-clash-member.json
index 8fb054f00..9efc7719b 100644
--- a/tests/qapi-schema/flat-union-branch-clash.json
+++ b/tests/qapi-schema/flat-union-clash-member.json
@@ -1,4 +1,5 @@
-# we check for no duplicate keys between branches and base
+# We check for no duplicate keys between branch members and base
+# base's member 'name' clashes with Branch1's
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 { 'struct': 'Base',
diff --git a/tests/qapi-schema/flat-union-clash-member.out b/tests/qapi-schema/flat-union-clash-member.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-member.out
diff --git a/tests/qapi-schema/flat-union-clash-type.err b/tests/qapi-schema/flat-union-clash-type.err
new file mode 100644
index 000000000..b44dd4005
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-type.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-clash-type.json:11: Discriminator name 'type' collides with enum value in 'TestEnum'
diff --git a/tests/qapi-schema/flat-union-clash-type.exit b/tests/qapi-schema/flat-union-clash-type.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-type.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-clash-type.json b/tests/qapi-schema/flat-union-clash-type.json
new file mode 100644
index 000000000..8f710f08a
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-type.json
@@ -0,0 +1,14 @@
+# Flat union branch 'type'
+# Reject this, because we would have a clash in generated C, between the
+# outer tag 'type' and the branch name 'type' within the union.
+# TODO: We could munge the generated C branch name to let it compile.
+{ 'enum': 'TestEnum',
+  'data': [ 'type' ] }
+{ 'struct': 'Base',
+  'data': { 'type': 'TestEnum' } }
+{ 'struct': 'Branch1',
+  'data': { 'string': 'str' } }
+{ 'union': 'TestUnion',
+  'base': 'Base',
+  'discriminator': 'type',
+  'data': { 'type': 'Branch1' } }
diff --git a/tests/qapi-schema/flat-union-clash-type.out b/tests/qapi-schema/flat-union-clash-type.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-type.out
diff --git a/tests/qapi-schema/flat-union-empty.err b/tests/qapi-schema/flat-union-empty.err
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/flat-union-empty.err
diff --git a/tests/qapi-schema/flat-union-reverse-define.exit b/tests/qapi-schema/flat-union-empty.exit
index 573541ac9..573541ac9 100644
--- a/tests/qapi-schema/flat-union-reverse-define.exit
+++ b/tests/qapi-schema/flat-union-empty.exit
diff --git a/tests/qapi-schema/flat-union-empty.json b/tests/qapi-schema/flat-union-empty.json
new file mode 100644
index 000000000..67dd2978e
--- /dev/null
+++ b/tests/qapi-schema/flat-union-empty.json
@@ -0,0 +1,4 @@
+# FIXME - flat unions should not be empty
+{ 'enum': 'Empty', 'data': [ ] }
+{ 'struct': 'Base', 'data': { 'type': 'Empty' } }
+{ 'union': 'Union', 'base': 'Base', 'discriminator': 'type', 'data': { } }
diff --git a/tests/qapi-schema/flat-union-empty.out b/tests/qapi-schema/flat-union-empty.out
new file mode 100644
index 000000000..0e0665af3
--- /dev/null
+++ b/tests/qapi-schema/flat-union-empty.out
@@ -0,0 +1,7 @@
+object :empty
+object Base
+    member type: Empty optional=False
+enum Empty []
+object Union
+    base Base
+    tag type
diff --git a/tests/qapi-schema/flat-union-inline.err b/tests/qapi-schema/flat-union-inline.err
index ec586277b..2333358d2 100644
--- a/tests/qapi-schema/flat-union-inline.err
+++ b/tests/qapi-schema/flat-union-inline.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-inline.json:7: Flat union 'TestUnion' must have a string base field
+tests/qapi-schema/flat-union-inline.json:7: Member 'value1' of union 'TestUnion' should be a type name
diff --git a/tests/qapi-schema/flat-union-inline.json b/tests/qapi-schema/flat-union-inline.json
index 6bfdd6581..62c7cda61 100644
--- a/tests/qapi-schema/flat-union-inline.json
+++ b/tests/qapi-schema/flat-union-inline.json
@@ -1,11 +1,11 @@
 # we require branches to be a struct name
-# TODO: should we allow anonymous inline types?
+# TODO: should we allow anonymous inline branch types?
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 { 'struct': 'Base',
   'data': { 'enum1': 'TestEnum', 'kind': 'str' } }
 { 'union': 'TestUnion',
-  'base': { 'enum1': 'TestEnum', 'kind': 'str' },
+  'base': 'Base',
   'discriminator': 'enum1',
   'data': { 'value1': { 'string': 'str' },
             'value2': { 'integer': 'int' } } }
diff --git a/tests/qapi-schema/flat-union-no-base.err b/tests/qapi-schema/flat-union-no-base.err
index bb3f70874..841c93b55 100644
--- a/tests/qapi-schema/flat-union-no-base.err
+++ b/tests/qapi-schema/flat-union-no-base.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-no-base.json:9: Flat union 'TestUnion' must have a string base field
+tests/qapi-schema/flat-union-no-base.json:9: Flat union 'TestUnion' must have a base
diff --git a/tests/qapi-schema/flat-union-reverse-define.json b/tests/qapi-schema/flat-union-reverse-define.json
deleted file mode 100644
index 648bbfe2b..000000000
--- a/tests/qapi-schema/flat-union-reverse-define.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{ 'union': 'TestUnion',
-  'base': 'TestBase',
-  'discriminator': 'enum1',
-  'data': { 'value1': 'TestTypeA',
-            'value2': 'TestTypeB' } }
-
-{ 'struct': 'TestBase',
-  'data': { 'enum1': 'TestEnum' } }
-
-{ 'enum': 'TestEnum',
-  'data': [ 'value1', 'value2' ] }
-
-{ 'struct': 'TestTypeA',
-  'data': { 'string': 'str' } }
-
-{ 'struct': 'TestTypeB',
-  'data': { 'integer': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.out b/tests/qapi-schema/flat-union-reverse-define.out
deleted file mode 100644
index 1ed7b8a51..000000000
--- a/tests/qapi-schema/flat-union-reverse-define.out
+++ /dev/null
@@ -1,9 +0,0 @@
-[OrderedDict([('union', 'TestUnion'), ('base', 'TestBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'TestTypeA'), ('value2', 'TestTypeB')]))]),
- OrderedDict([('struct', 'TestBase'), ('data', OrderedDict([('enum1', 'TestEnum')]))]),
- OrderedDict([('enum', 'TestEnum'), ('data', ['value1', 'value2'])]),
- OrderedDict([('struct', 'TestTypeA'), ('data', OrderedDict([('string', 'str')]))]),
- OrderedDict([('struct', 'TestTypeB'), ('data', OrderedDict([('integer', 'int')]))])]
-[{'enum_name': 'TestEnum', 'enum_values': ['value1', 'value2']}]
-[OrderedDict([('struct', 'TestBase'), ('data', OrderedDict([('enum1', 'TestEnum')]))]),
- OrderedDict([('struct', 'TestTypeA'), ('data', OrderedDict([('string', 'str')]))]),
- OrderedDict([('struct', 'TestTypeB'), ('data', OrderedDict([('integer', 'int')]))])]
diff --git a/tests/qapi-schema/ident-with-escape.out b/tests/qapi-schema/ident-with-escape.out
index 402843081..f4542b12d 100644
--- a/tests/qapi-schema/ident-with-escape.out
+++ b/tests/qapi-schema/ident-with-escape.out
@@ -1,3 +1,5 @@
-[OrderedDict([('command', 'fooA'), ('data', OrderedDict([('bar1', 'str')]))])]
-[]
-[]
+object :empty
+object :obj-fooA-arg
+    member bar1: str optional=False
+command fooA :obj-fooA-arg -> None
+   gen=True success_response=True
diff --git a/tests/qapi-schema/include-non-file.err b/tests/qapi-schema/include-non-file.err
index 9658c7880..faae1eacf 100644
--- a/tests/qapi-schema/include-non-file.err
+++ b/tests/qapi-schema/include-non-file.err
@@ -1 +1 @@
-tests/qapi-schema/include-non-file.json:1: Expected a file name (string), got: ['foo', 'bar']
+tests/qapi-schema/include-non-file.json:1: Value of 'include' must be a string
diff --git a/tests/qapi-schema/include-non-file.json b/tests/qapi-schema/include-non-file.json
index cd43c3f9d..4711aa42e 100644
--- a/tests/qapi-schema/include-non-file.json
+++ b/tests/qapi-schema/include-non-file.json
@@ -1 +1 @@
-{ 'include': [ 'foo', 'bar' ] }
+{ 'include': {} }
diff --git a/tests/qapi-schema/include-relpath.out b/tests/qapi-schema/include-relpath.out
index 4ce3dcf12..9e2c656fa 100644
--- a/tests/qapi-schema/include-relpath.out
+++ b/tests/qapi-schema/include-relpath.out
@@ -1,3 +1,2 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+object :empty
+enum Status ['good', 'bad', 'ugly']
diff --git a/tests/qapi-schema/include-repetition.out b/tests/qapi-schema/include-repetition.out
index 4ce3dcf12..9e2c656fa 100644
--- a/tests/qapi-schema/include-repetition.out
+++ b/tests/qapi-schema/include-repetition.out
@@ -1,3 +1,2 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+object :empty
+enum Status ['good', 'bad', 'ugly']
diff --git a/tests/qapi-schema/include-simple.out b/tests/qapi-schema/include-simple.out
index 4ce3dcf12..9e2c656fa 100644
--- a/tests/qapi-schema/include-simple.out
+++ b/tests/qapi-schema/include-simple.out
@@ -1,3 +1,2 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+object :empty
+enum Status ['good', 'bad', 'ugly']
diff --git a/tests/qapi-schema/indented-expr.out b/tests/qapi-schema/indented-expr.out
index b5ce9151b..226d30079 100644
--- a/tests/qapi-schema/indented-expr.out
+++ b/tests/qapi-schema/indented-expr.out
@@ -1,3 +1,5 @@
-[OrderedDict([('command', 'eins')]), OrderedDict([('command', 'zwei')])]
-[]
-[]
+object :empty
+command eins None -> None
+   gen=True success_response=True
+command zwei None -> None
+   gen=True success_response=True
diff --git a/tests/qapi-schema/leading-comma-list.err b/tests/qapi-schema/leading-comma-list.err
new file mode 100644
index 000000000..f5c870bb9
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-list.err
@@ -0,0 +1 @@
+tests/qapi-schema/leading-comma-list.json:2:13: Expected "{", "[", "]", string, boolean or "null"
diff --git a/tests/qapi-schema/leading-comma-list.exit b/tests/qapi-schema/leading-comma-list.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-list.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/leading-comma-list.json b/tests/qapi-schema/leading-comma-list.json
new file mode 100644
index 000000000..c5ba50159
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-list.json
@@ -0,0 +1,2 @@
+{ 'enum': 'Status',
+  'data': [ , 'good', 'bad', 'ugly' ] }
diff --git a/tests/qapi-schema/leading-comma-list.out b/tests/qapi-schema/leading-comma-list.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-list.out
diff --git a/tests/qapi-schema/leading-comma-object.err b/tests/qapi-schema/leading-comma-object.err
new file mode 100644
index 000000000..f767b9554
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-object.err
@@ -0,0 +1 @@
+tests/qapi-schema/leading-comma-object.json:1:3: Expected string or "}"
diff --git a/tests/qapi-schema/leading-comma-object.exit b/tests/qapi-schema/leading-comma-object.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-object.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/leading-comma-object.json b/tests/qapi-schema/leading-comma-object.json
new file mode 100644
index 000000000..c89023ff3
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-object.json
@@ -0,0 +1,2 @@
+{ , 'enum': 'Status',
+  'data': [ 'good', 'bad', 'ugly' ] }
diff --git a/tests/qapi-schema/leading-comma-object.out b/tests/qapi-schema/leading-comma-object.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-object.out
diff --git a/tests/qapi-schema/nested-struct-data.json b/tests/qapi-schema/nested-struct-data.json
index 3d52d2b39..efbe773de 100644
--- a/tests/qapi-schema/nested-struct-data.json
+++ b/tests/qapi-schema/nested-struct-data.json
@@ -1,4 +1,3 @@
 # inline subtypes collide with our desired future use of defaults
 { 'command': 'foo',
-  'data': { 'a' : { 'string' : 'str', 'integer': 'int' }, 'b' : 'str' },
-  'returns': {} }
+  'data': { 'a' : { 'string' : 'str', 'integer': 'int' }, 'b' : 'str' } }
diff --git a/tests/qapi-schema/nested-struct-returns.err b/tests/qapi-schema/nested-struct-returns.err
deleted file mode 100644
index 5238d075b..000000000
--- a/tests/qapi-schema/nested-struct-returns.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/nested-struct-returns.json:2: Member 'a' of 'returns' for command 'foo' should be a type name
diff --git a/tests/qapi-schema/nested-struct-returns.json b/tests/qapi-schema/nested-struct-returns.json
deleted file mode 100644
index d2cd047f0..000000000
--- a/tests/qapi-schema/nested-struct-returns.json
+++ /dev/null
@@ -1,3 +0,0 @@
-# inline subtypes collide with our desired future use of defaults
-{ 'command': 'foo',
-  'returns': { 'a' : { 'string' : 'str', 'integer': 'int' }, 'b' : 'str' } }
diff --git a/tests/qapi-schema/qapi-schema-test.json b/tests/qapi-schema/qapi-schema-test.json
index c7eaa865d..44638da94 100644
--- a/tests/qapi-schema/qapi-schema-test.json
+++ b/tests/qapi-schema/qapi-schema-test.json
@@ -1,19 +1,40 @@
 # *-*- Mode: Python -*-*
 
+# This file is a stress test of supported qapi constructs that must
+# parse and compile correctly.
+
+{ 'struct': 'TestStruct',
+  'data': { 'integer': 'int', 'boolean': 'bool', 'string': 'str' } }
+
 # for testing enums
-{ 'enum': 'EnumOne',
-  'data': [ 'value1', 'value2', 'value3' ] }
 { 'struct': 'NestedEnumsOne',
-  'data': { 'enum1': 'EnumOne', '*enum2': 'EnumOne', 'enum3': 'EnumOne', '*enum4': 'EnumOne' } }
+  'data': { 'enum1': 'EnumOne',   # Intentional forward reference
+            '*enum2': 'EnumOne', 'enum3': 'EnumOne', '*enum4': 'EnumOne' } }
+
+# An empty enum, although unusual, is currently acceptable
+{ 'enum': 'MyEnum', 'data': [ ] }
+
+# Likewise for an empty struct, including an empty base
+{ 'struct': 'Empty1', 'data': { } }
+{ 'struct': 'Empty2', 'base': 'Empty1', 'data': { } }
+
+# for testing override of default naming heuristic
+{ 'enum': 'QEnumTwo',
+  'prefix': 'QENUM_TWO',
+  'data': [ 'value1', 'value2' ] }
 
 # for testing nested structs
+{ 'struct': 'UserDefOne',
+  'base': 'UserDefZero',        # intentional forward reference
+  'data': { 'string': 'str',
+            '*enum1': 'EnumOne' } }   # intentional forward reference
+
+{ 'enum': 'EnumOne',
+  'data': [ 'value1', 'value2', 'value3' ] }
+
 { 'struct': 'UserDefZero',
   'data': { 'integer': 'int' } }
 
-{ 'struct': 'UserDefOne',
-  'base': 'UserDefZero',
-  'data': { 'string': 'str', '*enum1': 'EnumOne' } }
-
 { 'struct': 'UserDefTwoDictDict',
   'data': { 'userdef': 'UserDefOne', 'string': 'str' } }
 
@@ -26,36 +47,55 @@
   'data': { 'string0': 'str',
             'dict1': 'UserDefTwoDict' } }
 
+# dummy struct to force generation of array types not otherwise mentioned
+{ 'struct': 'ForceArrays',
+  'data': { 'unused1':['UserDefOne'], 'unused2':['UserDefTwo'],
+            'unused3':['TestStruct'] } }
+
 # for testing unions
+# Among other things, test that a name collision between branches does
+# not cause any problems (since only one branch can be in use at a time),
+# by intentionally using two branches that both have a C member 'a_b'
 { 'struct': 'UserDefA',
-  'data': { 'boolean': 'bool' } }
+  'data': { 'boolean': 'bool', '*a_b': 'int' } }
 
 { 'struct': 'UserDefB',
-  'data': { 'integer': 'int' } }
+  'data': { 'intb': 'int', '*a-b': 'bool' } }
 
-{ 'struct': 'UserDefC',
-  'data': { 'string1': 'str', 'string2': 'str' } }
+{ 'union': 'UserDefFlatUnion',
+  'base': 'UserDefUnionBase',   # intentional forward reference
+  'discriminator': 'enum1',
+  'data': { 'value1' : 'UserDefA',
+            'value2' : 'UserDefB',
+            'value3' : 'UserDefB' } }
 
 { 'struct': 'UserDefUnionBase',
+  'base': 'UserDefZero',
   'data': { 'string': 'str', 'enum1': 'EnumOne' } }
 
-{ 'union': 'UserDefFlatUnion',
-  'base': 'UserDefUnionBase',
-  'discriminator': 'enum1',
-  'data': { 'value1' : 'UserDefA', 'value2' : 'UserDefB', 'value3' : 'UserDefB' } }
-# FIXME generated struct UserDefFlatUnion has members for direct base
-# UserDefOne, but lacks members for indirect base UserDefZero
-
 # this variant of UserDefFlatUnion defaults to a union that uses fields with
 # allocated types to test corner cases in the cleanup/dealloc visitor
 { 'union': 'UserDefFlatUnion2',
   'base': 'UserDefUnionBase',
   'discriminator': 'enum1',
-  'data': { 'value1' : 'UserDefC', 'value2' : 'UserDefB', 'value3' : 'UserDefA' } }
+  'data': { 'value1' : 'UserDefC', # intentional forward reference
+            'value2' : 'UserDefB',
+            'value3' : 'UserDefA' } }
 
 { 'alternate': 'UserDefAlternate',
   'data': { 'uda': 'UserDefA', 's': 'str', 'i': 'int' } }
 
+{ 'struct': 'UserDefC',
+  'data': { 'string1': 'str', 'string2': 'str' } }
+
+# for testing use of 'number' within alternates
+{ 'alternate': 'AltStrBool', 'data': { 's': 'str', 'b': 'bool' } }
+{ 'alternate': 'AltStrNum', 'data': { 's': 'str', 'n': 'number' } }
+{ 'alternate': 'AltNumStr', 'data': { 'n': 'number', 's': 'str' } }
+{ 'alternate': 'AltStrInt', 'data': { 's': 'str', 'i': 'int' } }
+{ 'alternate': 'AltIntNum', 'data': { 'i': 'int', 'n': 'number' } }
+{ 'alternate': 'AltNumInt', 'data': { 'n': 'number', 'i': 'int' } }
+
 # for testing native lists
 { 'union': 'UserDefNativeListUnion',
   'data': { 'integer': ['int'],
@@ -70,7 +110,8 @@
             'number': ['number'],
             'boolean': ['bool'],
             'string': ['str'],
-            'sizes': ['size'] } }
+            'sizes': ['size'],
+            'any': ['any'] } }
 
 # testing commands
 { 'command': 'user_def_cmd', 'data': {} }
@@ -78,8 +119,11 @@
 { 'command': 'user_def_cmd2',
   'data': {'ud1a': 'UserDefOne', '*ud1b': 'UserDefOne'},
   'returns': 'UserDefTwo' }
-{ 'command': 'user_def_cmd3', 'data': {'a': 'int', '*b': 'int' },
+
+# Returning a non-dictionary requires a name from the whitelist
+{ 'command': 'guest-get-time', 'data': {'a': 'int', '*b': 'int' },
   'returns': 'int' }
+{ 'command': 'guest-sync', 'data': { 'arg': 'any' }, 'returns': 'any' }
 
 # For testing integer range flattening in opts-visitor. The following schema
 # corresponds to the option format:
diff --git a/tests/qapi-schema/qapi-schema-test.out b/tests/qapi-schema/qapi-schema-test.out
index cf0ccc402..e20a8239a 100644
--- a/tests/qapi-schema/qapi-schema-test.out
+++ b/tests/qapi-schema/qapi-schema-test.out
@@ -1,55 +1,207 @@
-[OrderedDict([('enum', 'EnumOne'), ('data', ['value1', 'value2', 'value3'])]),
- OrderedDict([('struct', 'NestedEnumsOne'), ('data', OrderedDict([('enum1', 'EnumOne'), ('*enum2', 'EnumOne'), ('enum3', 'EnumOne'), ('*enum4', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefZero'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefOne'), ('base', 'UserDefZero'), ('data', OrderedDict([('string', 'str'), ('*enum1', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefTwoDictDict'), ('data', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')]))]),
- OrderedDict([('struct', 'UserDefTwoDict'), ('data', OrderedDict([('string1', 'str'), ('dict2', 'UserDefTwoDictDict'), ('*dict3', 'UserDefTwoDictDict')]))]),
- OrderedDict([('struct', 'UserDefTwo'), ('data', OrderedDict([('string0', 'str'), ('dict1', 'UserDefTwoDict')]))]),
- OrderedDict([('struct', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
- OrderedDict([('struct', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
- OrderedDict([('struct', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
- OrderedDict([('union', 'UserDefFlatUnion'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefA'), ('value2', 'UserDefB'), ('value3', 'UserDefB')]))]),
- OrderedDict([('union', 'UserDefFlatUnion2'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefC'), ('value2', 'UserDefB'), ('value3', 'UserDefA')]))]),
- OrderedDict([('alternate', 'UserDefAlternate'), ('data', OrderedDict([('uda', 'UserDefA'), ('s', 'str'), ('i', 'int')]))]),
- OrderedDict([('union', 'UserDefNativeListUnion'), ('data', OrderedDict([('integer', ['int']), ('s8', ['int8']), ('s16', ['int16']), ('s32', ['int32']), ('s64', ['int64']), ('u8', ['uint8']), ('u16', ['uint16']), ('u32', ['uint32']), ('u64', ['uint64']), ('number', ['number']), ('boolean', ['bool']), ('string', ['str']), ('sizes', ['size'])]))]),
- OrderedDict([('command', 'user_def_cmd'), ('data', OrderedDict())]),
- OrderedDict([('command', 'user_def_cmd1'), ('data', OrderedDict([('ud1a', 'UserDefOne')]))]),
- OrderedDict([('command', 'user_def_cmd2'), ('data', OrderedDict([('ud1a', 'UserDefOne'), ('*ud1b', 'UserDefOne')])), ('returns', 'UserDefTwo')]),
- OrderedDict([('command', 'user_def_cmd3'), ('data', OrderedDict([('a', 'int'), ('*b', 'int')])), ('returns', 'int')]),
- OrderedDict([('struct', 'UserDefOptions'), ('data', OrderedDict([('*i64', ['int']), ('*u64', ['uint64']), ('*u16', ['uint16']), ('*i64x', 'int'), ('*u64x', 'uint64')]))]),
- OrderedDict([('struct', 'EventStructOne'), ('data', OrderedDict([('struct1', 'UserDefOne'), ('string', 'str'), ('*enum2', 'EnumOne')]))]),
- OrderedDict([('event', 'EVENT_A')]),
- OrderedDict([('event', 'EVENT_B'), ('data', OrderedDict())]),
- OrderedDict([('event', 'EVENT_C'), ('data', OrderedDict([('*a', 'int'), ('*b', 'UserDefOne'), ('c', 'str')]))]),
- OrderedDict([('event', 'EVENT_D'), ('data', OrderedDict([('a', 'EventStructOne'), ('b', 'str'), ('*c', 'str'), ('*enum3', 'EnumOne')]))]),
- OrderedDict([('enum', '__org.qemu_x-Enum'), ('data', ['__org.qemu_x-value'])]),
- OrderedDict([('struct', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member1', '__org.qemu_x-Enum')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct'), ('base', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member2', 'str')]))]),
- OrderedDict([('union', '__org.qemu_x-Union1'), ('data', OrderedDict([('__org.qemu_x-branch', 'str')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct2'), ('data', OrderedDict([('array', ['__org.qemu_x-Union1'])]))]),
- OrderedDict([('union', '__org.qemu_x-Union2'), ('base', '__org.qemu_x-Base'), ('discriminator', '__org.qemu_x-member1'), ('data', OrderedDict([('__org.qemu_x-value', '__org.qemu_x-Struct2')]))]),
- OrderedDict([('alternate', '__org.qemu_x-Alt'), ('data', OrderedDict([('__org.qemu_x-branch', 'str'), ('b', '__org.qemu_x-Base')]))]),
- OrderedDict([('event', '__ORG.QEMU_X-EVENT'), ('data', '__org.qemu_x-Struct')]),
- OrderedDict([('command', '__org.qemu_x-command'), ('data', OrderedDict([('a', ['__org.qemu_x-Enum']), ('b', ['__org.qemu_x-Struct']), ('c', '__org.qemu_x-Union2'), ('d', '__org.qemu_x-Alt')])), ('returns', '__org.qemu_x-Union1')])]
-[{'enum_name': 'EnumOne', 'enum_values': ['value1', 'value2', 'value3']},
- {'enum_name': '__org.qemu_x-Enum', 'enum_values': ['__org.qemu_x-value']},
- {'enum_name': 'UserDefAlternateKind', 'enum_values': None},
- {'enum_name': 'UserDefNativeListUnionKind', 'enum_values': None},
- {'enum_name': '__org.qemu_x-Union1Kind', 'enum_values': None},
- {'enum_name': '__org.qemu_x-AltKind', 'enum_values': None}]
-[OrderedDict([('struct', 'NestedEnumsOne'), ('data', OrderedDict([('enum1', 'EnumOne'), ('*enum2', 'EnumOne'), ('enum3', 'EnumOne'), ('*enum4', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefZero'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefOne'), ('base', 'UserDefZero'), ('data', OrderedDict([('string', 'str'), ('*enum1', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefTwoDictDict'), ('data', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')]))]),
- OrderedDict([('struct', 'UserDefTwoDict'), ('data', OrderedDict([('string1', 'str'), ('dict2', 'UserDefTwoDictDict'), ('*dict3', 'UserDefTwoDictDict')]))]),
- OrderedDict([('struct', 'UserDefTwo'), ('data', OrderedDict([('string0', 'str'), ('dict1', 'UserDefTwoDict')]))]),
- OrderedDict([('struct', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
- OrderedDict([('struct', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
- OrderedDict([('struct', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefOptions'), ('data', OrderedDict([('*i64', ['int']), ('*u64', ['uint64']), ('*u16', ['uint16']), ('*i64x', 'int'), ('*u64x', 'uint64')]))]),
- OrderedDict([('struct', 'EventStructOne'), ('data', OrderedDict([('struct1', 'UserDefOne'), ('string', 'str'), ('*enum2', 'EnumOne')]))]),
- OrderedDict([('struct', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member1', '__org.qemu_x-Enum')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct'), ('base', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member2', 'str')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct2'), ('data', OrderedDict([('array', ['__org.qemu_x-Union1'])]))])]
+object :empty
+object :obj-EVENT_C-arg
+    member a: int optional=True
+    member b: UserDefOne optional=True
+    member c: str optional=False
+object :obj-EVENT_D-arg
+    member a: EventStructOne optional=False
+    member b: str optional=False
+    member c: str optional=True
+    member enum3: EnumOne optional=True
+object :obj-__org.qemu_x-command-arg
+    member a: __org.qemu_x-EnumList optional=False
+    member b: __org.qemu_x-StructList optional=False
+    member c: __org.qemu_x-Union2 optional=False
+    member d: __org.qemu_x-Alt optional=False
+object :obj-anyList-wrapper
+    member data: anyList optional=False
+object :obj-boolList-wrapper
+    member data: boolList optional=False
+object :obj-guest-get-time-arg
+    member a: int optional=False
+    member b: int optional=True
+object :obj-guest-sync-arg
+    member arg: any optional=False
+object :obj-int16List-wrapper
+    member data: int16List optional=False
+object :obj-int32List-wrapper
+    member data: int32List optional=False
+object :obj-int64List-wrapper
+    member data: int64List optional=False
+object :obj-int8List-wrapper
+    member data: int8List optional=False
+object :obj-intList-wrapper
+    member data: intList optional=False
+object :obj-numberList-wrapper
+    member data: numberList optional=False
+object :obj-sizeList-wrapper
+    member data: sizeList optional=False
+object :obj-str-wrapper
+    member data: str optional=False
+object :obj-strList-wrapper
+    member data: strList optional=False
+object :obj-uint16List-wrapper
+    member data: uint16List optional=False
+object :obj-uint32List-wrapper
+    member data: uint32List optional=False
+object :obj-uint64List-wrapper
+    member data: uint64List optional=False
+object :obj-uint8List-wrapper
+    member data: uint8List optional=False
+object :obj-user_def_cmd1-arg
+    member ud1a: UserDefOne optional=False
+object :obj-user_def_cmd2-arg
+    member ud1a: UserDefOne optional=False
+    member ud1b: UserDefOne optional=True
+alternate AltIntNum
+    case i: int
+    case n: number
+enum AltIntNumKind ['i', 'n']
+alternate AltNumInt
+    case n: number
+    case i: int
+enum AltNumIntKind ['n', 'i']
+alternate AltNumStr
+    case n: number
+    case s: str
+enum AltNumStrKind ['n', 's']
+alternate AltStrBool
+    case s: str
+    case b: bool
+enum AltStrBoolKind ['s', 'b']
+alternate AltStrInt
+    case s: str
+    case i: int
+enum AltStrIntKind ['s', 'i']
+alternate AltStrNum
+    case s: str
+    case n: number
+enum AltStrNumKind ['s', 'n']
+event EVENT_A None
+event EVENT_B None
+event EVENT_C :obj-EVENT_C-arg
+event EVENT_D :obj-EVENT_D-arg
+object Empty1
+object Empty2
+    base Empty1
+enum EnumOne ['value1', 'value2', 'value3']
+object EventStructOne
+    member struct1: UserDefOne optional=False
+    member string: str optional=False
+    member enum2: EnumOne optional=True
+object ForceArrays
+    member unused1: UserDefOneList optional=False
+    member unused2: UserDefTwoList optional=False
+    member unused3: TestStructList optional=False
+enum MyEnum []
+object NestedEnumsOne
+    member enum1: EnumOne optional=False
+    member enum2: EnumOne optional=True
+    member enum3: EnumOne optional=False
+    member enum4: EnumOne optional=True
+enum QEnumTwo ['value1', 'value2']
+    prefix QENUM_TWO
+object TestStruct
+    member integer: int optional=False
+    member boolean: bool optional=False
+    member string: str optional=False
+object UserDefA
+    member boolean: bool optional=False
+    member a_b: int optional=True
+alternate UserDefAlternate
+    case uda: UserDefA
+    case s: str
+    case i: int
+enum UserDefAlternateKind ['uda', 's', 'i']
+object UserDefB
+    member intb: int optional=False
+    member a-b: bool optional=True
+object UserDefC
+    member string1: str optional=False
+    member string2: str optional=False
+object UserDefFlatUnion
+    base UserDefUnionBase
+    tag enum1
+    case value1: UserDefA
+    case value2: UserDefB
+    case value3: UserDefB
+object UserDefFlatUnion2
+    base UserDefUnionBase
+    tag enum1
+    case value1: UserDefC
+    case value2: UserDefB
+    case value3: UserDefA
+object UserDefNativeListUnion
+    case integer: :obj-intList-wrapper
+    case s8: :obj-int8List-wrapper
+    case s16: :obj-int16List-wrapper
+    case s32: :obj-int32List-wrapper
+    case s64: :obj-int64List-wrapper
+    case u8: :obj-uint8List-wrapper
+    case u16: :obj-uint16List-wrapper
+    case u32: :obj-uint32List-wrapper
+    case u64: :obj-uint64List-wrapper
+    case number: :obj-numberList-wrapper
+    case boolean: :obj-boolList-wrapper
+    case string: :obj-strList-wrapper
+    case sizes: :obj-sizeList-wrapper
+    case any: :obj-anyList-wrapper
+enum UserDefNativeListUnionKind ['integer', 's8', 's16', 's32', 's64', 'u8', 'u16', 'u32', 'u64', 'number', 'boolean', 'string', 'sizes', 'any']
+object UserDefOne
+    base UserDefZero
+    member string: str optional=False
+    member enum1: EnumOne optional=True
+object UserDefOptions
+    member i64: intList optional=True
+    member u64: uint64List optional=True
+    member u16: uint16List optional=True
+    member i64x: int optional=True
+    member u64x: uint64 optional=True
+object UserDefTwo
+    member string0: str optional=False
+    member dict1: UserDefTwoDict optional=False
+object UserDefTwoDict
+    member string1: str optional=False
+    member dict2: UserDefTwoDictDict optional=False
+    member dict3: UserDefTwoDictDict optional=True
+object UserDefTwoDictDict
+    member userdef: UserDefOne optional=False
+    member string: str optional=False
+object UserDefUnionBase
+    base UserDefZero
+    member string: str optional=False
+    member enum1: EnumOne optional=False
+object UserDefZero
+    member integer: int optional=False
+event __ORG.QEMU_X-EVENT __org.qemu_x-Struct
+alternate __org.qemu_x-Alt
+    case __org.qemu_x-branch: str
+    case b: __org.qemu_x-Base
+enum __org.qemu_x-AltKind ['__org.qemu_x-branch', 'b']
+object __org.qemu_x-Base
+    member __org.qemu_x-member1: __org.qemu_x-Enum optional=False
+enum __org.qemu_x-Enum ['__org.qemu_x-value']
+object __org.qemu_x-Struct
+    base __org.qemu_x-Base
+    member __org.qemu_x-member2: str optional=False
+object __org.qemu_x-Struct2
+    member array: __org.qemu_x-Union1List optional=False
+object __org.qemu_x-Union1
+    case __org.qemu_x-branch: :obj-str-wrapper
+enum __org.qemu_x-Union1Kind ['__org.qemu_x-branch']
+object __org.qemu_x-Union2
+    base __org.qemu_x-Base
+    tag __org.qemu_x-member1
+    case __org.qemu_x-value: __org.qemu_x-Struct2
+command __org.qemu_x-command :obj-__org.qemu_x-command-arg -> __org.qemu_x-Union1
+   gen=True success_response=True
+command guest-get-time :obj-guest-get-time-arg -> int
+   gen=True success_response=True
+command guest-sync :obj-guest-sync-arg -> any
+   gen=True success_response=True
+command user_def_cmd None -> None
+   gen=True success_response=True
+command user_def_cmd1 :obj-user_def_cmd1-arg -> None
+   gen=True success_response=True
+command user_def_cmd2 :obj-user_def_cmd2-arg -> UserDefTwo
+   gen=True success_response=True
diff --git a/tests/qapi-schema/reserved-command-q.err b/tests/qapi-schema/reserved-command-q.err
new file mode 100644
index 000000000..f939e044e
--- /dev/null
+++ b/tests/qapi-schema/reserved-command-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-command-q.json:5: 'command' uses invalid name 'q-unix'
diff --git a/tests/qapi-schema/reserved-command-q.exit b/tests/qapi-schema/reserved-command-q.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/reserved-command-q.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-command-q.json b/tests/qapi-schema/reserved-command-q.json
new file mode 100644
index 000000000..99f8aae31
--- /dev/null
+++ b/tests/qapi-schema/reserved-command-q.json
@@ -0,0 +1,5 @@
+# C entity name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'command': 'unix' }
+{ 'command': 'q-unix' }
diff --git a/tests/qapi-schema/reserved-command-q.out b/tests/qapi-schema/reserved-command-q.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/reserved-command-q.out
diff --git a/tests/qapi-schema/reserved-member-has.err b/tests/qapi-schema/reserved-member-has.err
new file mode 100644
index 000000000..e75577144
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-has.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-has.json:5: Member of 'data' for command 'oops' uses reserved name 'has-a'
diff --git a/tests/qapi-schema/reserved-member-has.exit b/tests/qapi-schema/reserved-member-has.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-has.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-member-has.json b/tests/qapi-schema/reserved-member-has.json
new file mode 100644
index 000000000..45b9109bd
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-has.json
@@ -0,0 +1,5 @@
+# C member name collision
+# We reject names like 'has-a', because they can collide with the flag
+# for an optional 'a' in generated C.
+# TODO we could munge the optional flag name to avoid the collision.
+{ 'command': 'oops', 'data': { '*a': 'str', 'has-a': 'str' } }
diff --git a/tests/qapi-schema/reserved-member-has.out b/tests/qapi-schema/reserved-member-has.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-has.out
diff --git a/tests/qapi-schema/reserved-member-q.err b/tests/qapi-schema/reserved-member-q.err
new file mode 100644
index 000000000..f3d5dd781
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-q.json:4: Member of 'data' for struct 'Foo' uses invalid name 'q-unix'
diff --git a/tests/qapi-schema/reserved-member-q.exit b/tests/qapi-schema/reserved-member-q.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-q.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-member-q.json b/tests/qapi-schema/reserved-member-q.json
new file mode 100644
index 000000000..62fed8fdd
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-q.json
@@ -0,0 +1,4 @@
+# C member name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'struct': 'Foo', 'data': { 'unix':'int', 'q-unix':'bool' } }
diff --git a/tests/qapi-schema/reserved-member-q.out b/tests/qapi-schema/reserved-member-q.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-q.out
diff --git a/tests/qapi-schema/reserved-member-u.err b/tests/qapi-schema/reserved-member-u.err
new file mode 100644
index 000000000..87d42296c
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-u.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-u.json:7: Member of 'data' for struct 'Oops' uses reserved name 'u'
diff --git a/tests/qapi-schema/reserved-member-u.exit b/tests/qapi-schema/reserved-member-u.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-u.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-member-u.json b/tests/qapi-schema/reserved-member-u.json
new file mode 100644
index 000000000..1eaf0f301
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-u.json
@@ -0,0 +1,7 @@
+# Potential C member name collision
+# We reject use of 'u' as a member name, to allow it for internal use in
+# putting union branch members in a separate namespace from QMP members.
+# This is true even for non-unions, because it is possible to convert a
+# struct to flat union while remaining backwards compatible in QMP.
+# TODO - we could munge the member name to 'q_u' to avoid the collision
+{ 'struct': 'Oops', 'data': { 'u': 'str' } }
diff --git a/tests/qapi-schema/reserved-member-u.out b/tests/qapi-schema/reserved-member-u.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-u.out
diff --git a/tests/qapi-schema/reserved-type-kind.err b/tests/qapi-schema/reserved-type-kind.err
new file mode 100644
index 000000000..0a38efaad
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-kind.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-type-kind.json:2: enum 'UnionKind' should not end in 'Kind'
diff --git a/tests/qapi-schema/reserved-type-kind.exit b/tests/qapi-schema/reserved-type-kind.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-kind.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-union-clash.json b/tests/qapi-schema/reserved-type-kind.json
index 593282b6c..9ecaba12b 100644
--- a/tests/qapi-schema/enum-union-clash.json
+++ b/tests/qapi-schema/reserved-type-kind.json
@@ -1,4 +1,2 @@
 # we reject types that would conflict with implicit union enum
 { 'enum': 'UnionKind', 'data': [ 'oops' ] }
-{ 'union': 'Union',
-  'data': { 'a': 'int' } }
diff --git a/tests/qapi-schema/reserved-type-kind.out b/tests/qapi-schema/reserved-type-kind.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-kind.out
diff --git a/tests/qapi-schema/reserved-type-list.err b/tests/qapi-schema/reserved-type-list.err
new file mode 100644
index 000000000..4510fa6d9
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-list.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-type-list.json:5: struct 'FooList' should not end in 'List'
diff --git a/tests/qapi-schema/reserved-type-list.exit b/tests/qapi-schema/reserved-type-list.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-list.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-type-list.json b/tests/qapi-schema/reserved-type-list.json
new file mode 100644
index 000000000..98d53bf80
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-list.json
@@ -0,0 +1,5 @@
+# Potential C name collision
+# We reserve names ending in 'List' for use by array types.
+# TODO - we could choose array names to avoid collision with user types,
+# in order to let this compile
+{ 'struct': 'FooList', 'data': { 's': 'str' } }
diff --git a/tests/qapi-schema/reserved-type-list.out b/tests/qapi-schema/reserved-type-list.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-list.out
diff --git a/tests/qapi-schema/returns-dict.err b/tests/qapi-schema/returns-dict.err
new file mode 100644
index 000000000..eb2d0c466
--- /dev/null
+++ b/tests/qapi-schema/returns-dict.err
@@ -0,0 +1 @@
+tests/qapi-schema/returns-dict.json:2: 'returns' for command 'oops' should be a type name
diff --git a/tests/qapi-schema/returns-dict.exit b/tests/qapi-schema/returns-dict.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/returns-dict.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/returns-dict.json b/tests/qapi-schema/returns-dict.json
new file mode 100644
index 000000000..1cfef3ede
--- /dev/null
+++ b/tests/qapi-schema/returns-dict.json
@@ -0,0 +1,2 @@
+# we reject inline struct return type
+{ 'command': 'oops', 'returns': { 'a': 'str' } }
diff --git a/tests/qapi-schema/returns-dict.out b/tests/qapi-schema/returns-dict.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/returns-dict.out
diff --git a/tests/qapi-schema/returns-int.json b/tests/qapi-schema/returns-int.json
deleted file mode 100644
index 870ec6366..000000000
--- a/tests/qapi-schema/returns-int.json
+++ /dev/null
@@ -1,3 +0,0 @@
-# It is okay (although not extensible) to return a non-dictionary
-# But to make it work, the name must be in a whitelist
-{ 'command': 'guest-get-time', 'returns': 'int' }
diff --git a/tests/qapi-schema/returns-int.out b/tests/qapi-schema/returns-int.out
deleted file mode 100644
index 70b3ac5e6..000000000
--- a/tests/qapi-schema/returns-int.out
+++ /dev/null
@@ -1,3 +0,0 @@
-[OrderedDict([('command', 'guest-get-time'), ('returns', 'int')])]
-[]
-[]
diff --git a/tests/qapi-schema/returns-whitelist.err b/tests/qapi-schema/returns-whitelist.err
index a41f019a5..f47c1ee7c 100644
--- a/tests/qapi-schema/returns-whitelist.err
+++ b/tests/qapi-schema/returns-whitelist.err
@@ -1 +1 @@
-tests/qapi-schema/returns-whitelist.json:10: 'returns' for command 'no-way-this-will-get-whitelisted' cannot use built-in type 'array of int'
+tests/qapi-schema/returns-whitelist.json:10: 'returns' for command 'no-way-this-will-get-whitelisted' cannot use built-in type 'int'
diff --git a/tests/qapi-schema/struct-base-clash-deep.err b/tests/qapi-schema/struct-base-clash-deep.err
index e3e9f8d28..f7a25a3b3 100644
--- a/tests/qapi-schema/struct-base-clash-deep.err
+++ b/tests/qapi-schema/struct-base-clash-deep.err
@@ -1 +1 @@
-tests/qapi-schema/struct-base-clash-deep.json:7: Member name 'name' clashes with base 'Base'
+tests/qapi-schema/struct-base-clash-deep.json:10: Member name 'name' clashes with base 'Base'
diff --git a/tests/qapi-schema/struct-base-clash-deep.json b/tests/qapi-schema/struct-base-clash-deep.json
index 552fe9431..fa873ab5d 100644
--- a/tests/qapi-schema/struct-base-clash-deep.json
+++ b/tests/qapi-schema/struct-base-clash-deep.json
@@ -1,4 +1,7 @@
-# we check for no duplicate keys with indirect base
+# Reject attempts to duplicate QMP members
+# Here, 'name' would have to appear twice on the wire, locally and
+# indirectly for the grandparent base; the collision doesn't care that
+# one instance is optional.
 { 'struct': 'Base',
   'data': { 'name': 'str' } }
 { 'struct': 'Mid',
diff --git a/tests/qapi-schema/struct-base-clash.err b/tests/qapi-schema/struct-base-clash.err
index 3ac37fb26..3a9f66b04 100644
--- a/tests/qapi-schema/struct-base-clash.err
+++ b/tests/qapi-schema/struct-base-clash.err
@@ -1 +1 @@
-tests/qapi-schema/struct-base-clash.json:4: Member name 'name' clashes with base 'Base'
+tests/qapi-schema/struct-base-clash.json:5: Member name 'name' clashes with base 'Base'
diff --git a/tests/qapi-schema/struct-base-clash.json b/tests/qapi-schema/struct-base-clash.json
index f2afc9b6f..11aec80fe 100644
--- a/tests/qapi-schema/struct-base-clash.json
+++ b/tests/qapi-schema/struct-base-clash.json
@@ -1,4 +1,5 @@
-# we check for no duplicate keys with base
+# Reject attempts to duplicate QMP members
+# Here, 'name' would have to appear twice on the wire, locally and for base.
 { 'struct': 'Base',
   'data': { 'name': 'str' } }
 { 'struct': 'Sub',
diff --git a/tests/qapi-schema/struct-data-invalid.err b/tests/qapi-schema/struct-data-invalid.err
new file mode 100644
index 000000000..6644f4c2a
--- /dev/null
+++ b/tests/qapi-schema/struct-data-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-data-invalid.json:1: 'data' for struct 'foo' should be a dictionary or type name
diff --git a/tests/qapi-schema/struct-data-invalid.exit b/tests/qapi-schema/struct-data-invalid.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/struct-data-invalid.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/struct-data-invalid.json b/tests/qapi-schema/struct-data-invalid.json
new file mode 100644
index 000000000..9adbc3bb6
--- /dev/null
+++ b/tests/qapi-schema/struct-data-invalid.json
@@ -0,0 +1,2 @@
+{ 'struct': 'foo',
+  'data': false }
diff --git a/tests/qapi-schema/struct-data-invalid.out b/tests/qapi-schema/struct-data-invalid.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/struct-data-invalid.out
diff --git a/tests/qapi-schema/struct-member-invalid.err b/tests/qapi-schema/struct-member-invalid.err
new file mode 100644
index 000000000..69a326d45
--- /dev/null
+++ b/tests/qapi-schema/struct-member-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-member-invalid.json:1: Member 'a' of 'data' for struct 'foo' should be a type name
diff --git a/tests/qapi-schema/struct-member-invalid.exit b/tests/qapi-schema/struct-member-invalid.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/struct-member-invalid.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/struct-member-invalid.json b/tests/qapi-schema/struct-member-invalid.json
new file mode 100644
index 000000000..8f172f7a8
--- /dev/null
+++ b/tests/qapi-schema/struct-member-invalid.json
@@ -0,0 +1,2 @@
+{ 'struct': 'foo',
+  'data': { 'a': false } }
diff --git a/tests/qapi-schema/struct-member-invalid.out b/tests/qapi-schema/struct-member-invalid.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/struct-member-invalid.out
diff --git a/tests/qapi-schema/test-qapi.py b/tests/qapi-schema/test-qapi.py
index 634ef2d00..649677e01 100644
--- a/tests/qapi-schema/test-qapi.py
+++ b/tests/qapi-schema/test-qapi.py
@@ -15,11 +15,42 @@ from pprint import pprint
 import os
 import sys
 
-try:
-    exprs = parse_schema(sys.argv[1])
-except SystemExit:
-    raise
-
-pprint(exprs)
-pprint(enum_types)
-pprint(struct_types)
+
+class QAPISchemaTestVisitor(QAPISchemaVisitor):
+    def visit_enum_type(self, name, info, values, prefix):
+        print 'enum %s %s' % (name, values)
+        if prefix:
+            print '    prefix %s' % prefix
+
+    def visit_object_type(self, name, info, base, members, variants):
+        print 'object %s' % name
+        if base:
+            print '    base %s' % base.name
+        for m in members:
+            print '    member %s: %s optional=%s' % \
+                (m.name, m.type.name, m.optional)
+        self._print_variants(variants)
+
+    def visit_alternate_type(self, name, info, variants):
+        print 'alternate %s' % name
+        self._print_variants(variants)
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        print 'command %s %s -> %s' % \
+            (name, arg_type and arg_type.name, ret_type and ret_type.name)
+        print '   gen=%s success_response=%s' % (gen, success_response)
+
+    def visit_event(self, name, info, arg_type):
+        print 'event %s %s' % (name, arg_type and arg_type.name)
+
+    @staticmethod
+    def _print_variants(variants):
+        if variants:
+            if variants.tag_name:
+                print '    tag %s' % variants.tag_name
+            for v in variants.variants:
+                print '    case %s: %s' % (v.name, v.type.name)
+
+schema = QAPISchema(sys.argv[1])
+schema.visit(QAPISchemaTestVisitor())
diff --git a/tests/qapi-schema/type-bypass-no-gen.err b/tests/qapi-schema/type-bypass-no-gen.err
deleted file mode 100644
index 20cef0a8a..000000000
--- a/tests/qapi-schema/type-bypass-no-gen.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/type-bypass-no-gen.json:2: Member 'arg' of 'data' for command 'unsafe' uses '**' but did not request 'gen':false
diff --git a/tests/qapi-schema/type-bypass-no-gen.json b/tests/qapi-schema/type-bypass-no-gen.json
deleted file mode 100644
index 4feae3719..000000000
--- a/tests/qapi-schema/type-bypass-no-gen.json
+++ /dev/null
@@ -1,2 +0,0 @@
-# type bypass only works with 'gen':false
-{ 'command': 'unsafe', 'data': { 'arg': '**' }, 'returns': '**' }
diff --git a/tests/qapi-schema/type-bypass.json b/tests/qapi-schema/type-bypass.json
deleted file mode 100644
index 48b213783..000000000
--- a/tests/qapi-schema/type-bypass.json
+++ /dev/null
@@ -1,2 +0,0 @@
-# Use of 'gen':false allows bypassing type system
-{ 'command': 'unsafe', 'data': { 'arg': '**' }, 'returns': '**', 'gen': false }
diff --git a/tests/qapi-schema/type-bypass.out b/tests/qapi-schema/type-bypass.out
deleted file mode 100644
index eaf20f834..000000000
--- a/tests/qapi-schema/type-bypass.out
+++ /dev/null
@@ -1,3 +0,0 @@
-[OrderedDict([('command', 'unsafe'), ('data', OrderedDict([('arg', '**')])), ('returns', '**'), ('gen', False)])]
-[]
-[]
diff --git a/tests/qapi-schema/union-base-no-discriminator.err b/tests/qapi-schema/union-base-no-discriminator.err
index fc8b79c45..8b7a24260 100644
--- a/tests/qapi-schema/union-base-no-discriminator.err
+++ b/tests/qapi-schema/union-base-no-discriminator.err
@@ -1 +1 @@
-tests/qapi-schema/union-base-no-discriminator.json:11: Union 'TestUnion' requires a discriminator to go along with base
+tests/qapi-schema/union-base-no-discriminator.json:11: Simple union 'TestUnion' must not have a base
diff --git a/tests/qapi-schema/union-clash-branches.err b/tests/qapi-schema/union-clash-branches.err
new file mode 100644
index 000000000..005c48d90
--- /dev/null
+++ b/tests/qapi-schema/union-clash-branches.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-clash-branches.json:4: Union 'TestUnion' member 'a_b' clashes with 'a-b'
diff --git a/tests/qapi-schema/union-clash-branches.exit b/tests/qapi-schema/union-clash-branches.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/union-clash-branches.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-clash-branches.json b/tests/qapi-schema/union-clash-branches.json
new file mode 100644
index 000000000..31d135fb1
--- /dev/null
+++ b/tests/qapi-schema/union-clash-branches.json
@@ -0,0 +1,5 @@
+# Union branch name collision
+# Reject a union that would result in a collision in generated C names (this
+# would try to generate two enum values 'TEST_UNION_KIND_A_B').
+{ 'union': 'TestUnion',
+  'data': { 'a-b': 'int', 'a_b': 'str' } }
diff --git a/tests/qapi-schema/union-clash-branches.out b/tests/qapi-schema/union-clash-branches.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/union-clash-branches.out
diff --git a/tests/qapi-schema/union-clash-data.err b/tests/qapi-schema/union-clash-data.err
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/union-clash-data.err
diff --git a/tests/qapi-schema/returns-int.exit b/tests/qapi-schema/union-clash-data.exit
index 573541ac9..573541ac9 100644
--- a/tests/qapi-schema/returns-int.exit
+++ b/tests/qapi-schema/union-clash-data.exit
diff --git a/tests/qapi-schema/union-clash-data.json b/tests/qapi-schema/union-clash-data.json
new file mode 100644
index 000000000..7308e69f9
--- /dev/null
+++ b/tests/qapi-schema/union-clash-data.json
@@ -0,0 +1,7 @@
+# Union branch 'data'
+# FIXME: this parses, but then fails to compile due to a duplicate 'data'
+# (one from the branch name, another as a filler to avoid an empty union).
+# we should either detect the collision at parse time, or change the
+# generated struct to allow this to compile.
+{ 'union': 'TestUnion',
+  'data': { 'data': 'int' } }
diff --git a/tests/qapi-schema/union-clash-data.out b/tests/qapi-schema/union-clash-data.out
new file mode 100644
index 000000000..6277239d4
--- /dev/null
+++ b/tests/qapi-schema/union-clash-data.out
@@ -0,0 +1,6 @@
+object :empty
+object :obj-int-wrapper
+    member data: int optional=False
+object TestUnion
+    case data: :obj-int-wrapper
+enum TestUnionKind ['data']
diff --git a/tests/qapi-schema/union-clash-type.err b/tests/qapi-schema/union-clash-type.err
new file mode 100644
index 000000000..a5dead128
--- /dev/null
+++ b/tests/qapi-schema/union-clash-type.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-clash-type.json:8: Union 'TestUnion' member 'kind' clashes with '(automatic)'
diff --git a/tests/qapi-schema/union-clash-type.exit b/tests/qapi-schema/union-clash-type.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/qapi-schema/union-clash-type.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-clash-type.json b/tests/qapi-schema/union-clash-type.json
new file mode 100644
index 000000000..cfc256b04
--- /dev/null
+++ b/tests/qapi-schema/union-clash-type.json
@@ -0,0 +1,9 @@
+# Union branch 'type'
+# Reject this, because we would have a clash in generated C, between the
+# simple union's implicit tag member 'kind' and the branch name 'kind'
+# within the union.
+# TODO: Even when the generated C is switched to use 'type' rather than
+# 'kind', to match the QMP spelling, the collision should still be detected.
+# Or, we could munge the branch name to allow compilation.
+{ 'union': 'TestUnion',
+  'data': { 'kind': 'int', 'type': 'str' } }
diff --git a/tests/qapi-schema/union-clash-type.out b/tests/qapi-schema/union-clash-type.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/union-clash-type.out
diff --git a/tests/qapi-schema/union-empty.err b/tests/qapi-schema/union-empty.err
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/qapi-schema/union-empty.err
diff --git a/tests/qapi-schema/type-bypass.exit b/tests/qapi-schema/union-empty.exit
index 573541ac9..573541ac9 100644
--- a/tests/qapi-schema/type-bypass.exit
+++ b/tests/qapi-schema/union-empty.exit
diff --git a/tests/qapi-schema/union-empty.json b/tests/qapi-schema/union-empty.json
new file mode 100644
index 000000000..178500711
--- /dev/null
+++ b/tests/qapi-schema/union-empty.json
@@ -0,0 +1,2 @@
+# FIXME - unions should not be empty
+{ 'union': 'Union', 'data': { } }
diff --git a/tests/qapi-schema/union-empty.out b/tests/qapi-schema/union-empty.out
new file mode 100644
index 000000000..8b5a7bf58
--- /dev/null
+++ b/tests/qapi-schema/union-empty.out
@@ -0,0 +1,3 @@
+object :empty
+object Union
+enum UnionKind []
diff --git a/tests/qapi-schema/union-invalid-base.err b/tests/qapi-schema/union-invalid-base.err
index 9f637963e..03d7b97a9 100644
--- a/tests/qapi-schema/union-invalid-base.err
+++ b/tests/qapi-schema/union-invalid-base.err
@@ -1 +1 @@
-tests/qapi-schema/union-invalid-base.json:8: Base 'int' is not a valid struct
+tests/qapi-schema/union-invalid-base.json:8: 'base' for union 'TestUnion' cannot use built-in type 'int'
diff --git a/tests/qemu-iotests/017.out b/tests/qemu-iotests/017.out
index 7c409fc5b..8fc924194 100644
--- a/tests/qemu-iotests/017.out
+++ b/tests/qemu-iotests/017.out
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4295032832
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/tests/qemu-iotests/018.out b/tests/qemu-iotests/018.out
index 39a6011d2..d66bd6343 100644
--- a/tests/qemu-iotests/018.out
+++ b/tests/qemu-iotests/018.out
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4295032832
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/tests/qemu-iotests/019.out b/tests/qemu-iotests/019.out
index 4695b972d..615450a53 100644
--- a/tests/qemu-iotests/019.out
+++ b/tests/qemu-iotests/019.out
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4296015872
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 43
diff --git a/tests/qemu-iotests/020.out b/tests/qemu-iotests/020.out
index 71aab1c74..134aa293e 100644
--- a/tests/qemu-iotests/020.out
+++ b/tests/qemu-iotests/020.out
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4295032832
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/tests/qemu-iotests/024.out b/tests/qemu-iotests/024.out
index 521d46942..9b9ef3a34 100644
--- a/tests/qemu-iotests/024.out
+++ b/tests/qemu-iotests/024.out
@@ -33,7 +33,7 @@ wrote 131072/131072 bytes at offset 786432
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Creating COW image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 backing_file='TEST_DIR/t.IMGFMT.base_old'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 backing_file=TEST_DIR/t.IMGFMT.base_old
 === IO: pattern 0x33
 wrote 262144/262144 bytes at offset 0
 256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/qemu-iotests/028.out b/tests/qemu-iotests/028.out
index 5db167ce7..29c9972e2 100644
--- a/tests/qemu-iotests/028.out
+++ b/tests/qemu-iotests/028.out
@@ -70,7 +70,7 @@ wrote 512/512 bytes at offset 3221225984
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294968832 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294968832 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 196
@@ -468,7 +468,7 @@ No errors were found on the image.
 
 block-backup
 
-Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file='TEST_DIR/t.IMGFMT.base' backing_fmt='IMGFMT'
+Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT
 (qemu)
 (qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block-info block-jinfo block-joinfo block-jobinfo block-jobs
 Type backup, device disk: Completed 0 of 4294968832 bytes, speed limit 0 bytes/s
diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index 952a524ec..32469efd7 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -245,6 +245,7 @@ class TestEIO(TestErrors):
         while not completed:
             for event in self.vm.get_qmp_events(wait=True):
                 if event['event'] == 'BLOCK_JOB_ERROR':
+                    error = True
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/operation', 'read')
 
@@ -257,9 +258,11 @@ class TestEIO(TestErrors):
                     self.assert_qmp(result, 'return', {})
 
                     result = self.vm.qmp('query-block-jobs')
+                    if result == {'return': []}:
+                        # Race; likely already finished. Check.
+                        continue
                     self.assert_qmp(result, 'return[0]/paused', False)
                     self.assert_qmp(result, 'return[0]/io-status', 'ok')
-                    error = True
                 elif event['event'] == 'BLOCK_JOB_COMPLETED':
                     self.assertTrue(error, 'job completed unexpectedly')
                     self.assert_qmp(event, 'data/type', 'stream')
diff --git a/tests/qemu-iotests/034.out b/tests/qemu-iotests/034.out
index d12daf206..34fda80c8 100644
--- a/tests/qemu-iotests/034.out
+++ b/tests/qemu-iotests/034.out
@@ -4,7 +4,7 @@ QA output created by 034
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == zero write with backing file ==
 wrote 196608/196608 bytes at offset 65536
diff --git a/tests/qemu-iotests/037.out b/tests/qemu-iotests/037.out
index dc40a021a..55b30fdc6 100644
--- a/tests/qemu-iotests/037.out
+++ b/tests/qemu-iotests/037.out
@@ -514,7 +514,7 @@ wrote 512/512 bytes at offset 130048
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 130560
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == COW in a single cluster ==
 wrote 2048/2048 bytes at offset 0
diff --git a/tests/qemu-iotests/038.out b/tests/qemu-iotests/038.out
index e1a7e9441..ecb656e08 100644
--- a/tests/qemu-iotests/038.out
+++ b/tests/qemu-iotests/038.out
@@ -514,7 +514,7 @@ wrote 65536/65536 bytes at offset 16646144
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 16711680
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == Some concurrent requests touching the same cluster ==
 wrote 65536/65536 bytes at offset XXX
diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
index 859705f84..9e9b379ba 100755
--- a/tests/qemu-iotests/039
+++ b/tests/qemu-iotests/039
@@ -47,13 +47,6 @@ _supported_os Linux
 _default_cache_mode "writethrough"
 _supported_cache_modes "writethrough"
 
-_subshell_exec()
-{
-    # Executing crashing commands in a subshell prevents information like the
-    # "Killed" line from being lost
-    (exec "$@")
-}
-
 size=128M
 
 echo
@@ -74,8 +67,8 @@ echo "== Creating a dirty image file =="
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \
-                        -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
     | _filter_qemu_io
 
 # The dirty bit must be set
@@ -109,8 +102,8 @@ echo "== Opening a dirty image read/write should repair it =="
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \
-                        -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
     | _filter_qemu_io
 
 # The dirty bit must be set
@@ -127,8 +120,8 @@ echo "== Creating an image file with lazy_refcounts=off =="
 IMGOPTS="compat=1.1,lazy_refcounts=off"
 _make_test_img $size
 
-_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \
-                        -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
     | _filter_qemu_io
 
 # The dirty bit must not be set since lazy_refcounts=off
@@ -154,6 +147,33 @@ $PYTHON qcow2.py "$TEST_IMG".base dump-header | grep incompatible_features
 _check_test_img
 TEST_IMG="$TEST_IMG".base _check_test_img
 
+echo
+echo "== Changing lazy_refcounts setting at runtime =="
+
+IMGOPTS="compat=1.1,lazy_refcounts=off"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=on" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
+IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=off" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out
index d09751f9c..32c884694 100644
--- a/tests/qemu-iotests/039.out
+++ b/tests/qemu-iotests/039.out
@@ -11,7 +11,11 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Killed                  ( exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 incompatible_features     0x1
 ERROR cluster 5 refcount=0 reference=1
 ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
@@ -46,7 +50,11 @@ read 512/512 bytes at offset 0
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Killed                  ( exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 incompatible_features     0x1
 ERROR cluster 5 refcount=0 reference=1
 Rebuilding refcount structure
@@ -60,13 +68,17 @@ incompatible_features     0x0
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Killed                  ( exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 incompatible_features     0x0
 No errors were found on the image.
 
 == Committing to a backing file with lazy_refcounts=on ==
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Image committed.
@@ -74,4 +86,30 @@ incompatible_features     0x0
 incompatible_features     0x0
 No errors were found on the image.
 No errors were found on the image.
+
+== Changing lazy_refcounts setting at runtime ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
+incompatible_features     0x1
+ERROR cluster 5 refcount=0 reference=1
+ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
+
+2 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
+incompatible_features     0x0
+No errors were found on the image.
 *** done
diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
index ea2f98e51..5bdaf3d48 100755
--- a/tests/qemu-iotests/040
+++ b/tests/qemu-iotests/040
@@ -41,6 +41,7 @@ class ImageCommitTestCase(iotests.QMPTestCase):
         while not completed:
             for event in self.vm.get_qmp_events(wait=True):
                 if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp_absent(event, 'data/error')
                     self.assert_qmp(event, 'data/type', 'commit')
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/offset', event['data']['len'])
@@ -251,5 +252,34 @@ class TestSetSpeed(ImageCommitTestCase):
 class TestActiveZeroLengthImage(TestSingleDrive):
     image_len = 0
 
+class TestReopenOverlay(ImageCommitTestCase):
+    image_len = 1024 * 1024
+    img0 = os.path.join(iotests.test_dir, '0.img')
+    img1 = os.path.join(iotests.test_dir, '1.img')
+    img2 = os.path.join(iotests.test_dir, '2.img')
+    img3 = os.path.join(iotests.test_dir, '3.img')
+
+    def setUp(self):
+        iotests.create_image(self.img0, self.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img0, self.img1)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img1, self.img2)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img2, self.img3)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xab 0 128K', self.img1)
+        self.vm = iotests.VM().add_drive(self.img3)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.img0)
+        os.remove(self.img1)
+        os.remove(self.img2)
+        os.remove(self.img3)
+
+    # This tests what happens when the overlay image of the 'top' node
+    # needs to be reopened in read-write mode in order to update the
+    # backing image string.
+    def test_reopen_overlay(self):
+        self.run_commit_test(self.img1, self.img0)
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/040.out b/tests/qemu-iotests/040.out
index 42314e9c0..4fd1c2dcd 100644
--- a/tests/qemu-iotests/040.out
+++ b/tests/qemu-iotests/040.out
@@ -1,5 +1,5 @@
-........................
+.........................
 ----------------------------------------------------------------------
-Ran 24 tests
+Ran 25 tests
 
 OK
diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041
index 3d46ed705..05b5962ce 100755
--- a/tests/qemu-iotests/041
+++ b/tests/qemu-iotests/041
@@ -42,6 +42,8 @@ class TestSingleDrive(iotests.QMPTestCase):
         iotests.create_image(backing_img, self.image_len)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
         self.vm = iotests.VM().add_drive(test_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
@@ -167,8 +169,11 @@ class TestSingleDrive(iotests.QMPTestCase):
                         'target image does not match source after mirroring')
 
     def test_medium_not_found(self):
-        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
-                             target=target_img)
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('drive-mirror', device='drive1', # CD-ROM
+                             sync='full', target=target_img)
         self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_image_not_found(self):
@@ -707,6 +712,9 @@ class TestRepairQuorum(iotests.QMPTestCase):
     def setUp(self):
         self.vm = iotests.VM()
 
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
+
         # Add each individual quorum images
         for i in self.IMAGES:
             qemu_img('create', '-f', iotests.imgfmt, i,
@@ -772,7 +780,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
         # here we check that the last registered quorum file has not been
         # swapped out and unref
         result = self.vm.qmp('query-named-block-nodes')
-        self.assert_qmp(result, 'return[0]/file', quorum_img3)
+        self.assert_qmp(result, 'return[1]/file', quorum_img3)
         self.vm.shutdown()
 
     def test_cancel_after_ready(self):
@@ -791,7 +799,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
         result = self.vm.qmp('query-named-block-nodes')
         # here we check that the last registered quorum file has not been
         # swapped out and unref
-        self.assert_qmp(result, 'return[0]/file', quorum_img3)
+        self.assert_qmp(result, 'return[1]/file', quorum_img3)
         self.vm.shutdown()
         self.assertTrue(iotests.compare_images(quorum_img2, quorum_repair_img),
                         'target image does not match source after mirroring')
@@ -831,7 +839,11 @@ class TestRepairQuorum(iotests.QMPTestCase):
         if not self.has_quorum():
             return
 
-        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('drive-mirror', device='drive0', # CD-ROM
+                             sync='full',
                              node_name='repair0',
                              replaces='img1',
                              target=quorum_repair_img, format=iotests.imgfmt)
@@ -877,7 +889,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
                              target=quorum_repair_img, format=iotests.imgfmt)
         self.assert_qmp(result, 'error/class', 'GenericError')
 
-    def test_unexistant_replaces(self):
+    def test_nonexistent_replaces(self):
         if not self.has_quorum():
             return
 
diff --git a/tests/qemu-iotests/043.out b/tests/qemu-iotests/043.out
index 012cc008e..33f8cc37d 100644
--- a/tests/qemu-iotests/043.out
+++ b/tests/qemu-iotests/043.out
@@ -4,20 +4,20 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 == backing file references self ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 
 == parent references self ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.3.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.1.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.2.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.3.base
 
 == ancestor references another ancestor ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT.2.base' creates an infinite loop.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.1.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.2.base
 
 == finite chain of length 3 (human) ==
 image: TEST_DIR/t.IMGFMT
diff --git a/tests/qemu-iotests/046.out b/tests/qemu-iotests/046.out
index 9d18af53a..ca2c7404a 100644
--- a/tests/qemu-iotests/046.out
+++ b/tests/qemu-iotests/046.out
@@ -66,7 +66,7 @@ wrote 65536/65536 bytes at offset 1966080
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 2031616
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == Some concurrent requests touching the same cluster ==
 blkdebug: Suspended request 'A'
diff --git a/tests/qemu-iotests/049.out b/tests/qemu-iotests/049.out
index 9f93666c5..a2b670395 100644
--- a/tests/qemu-iotests/049.out
+++ b/tests/qemu-iotests/049.out
@@ -95,17 +95,15 @@ qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1024
 qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1024 TEST_DIR/t.qcow2
-qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+qemu-img: Parameter 'size' expects a non-negative number below 2^64
+qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1k
 qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1k TEST_DIR/t.qcow2
-qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+qemu-img: Parameter 'size' expects a non-negative number below 2^64
+qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- 1kilobyte
 qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for
@@ -120,6 +118,7 @@ qemu-img: kilobytes, megabytes, gigabytes, terabytes, petabytes and exabytes.
 
 qemu-img create -f qcow2 -o size=foobar TEST_DIR/t.qcow2
 qemu-img: Parameter 'size' expects a size
+You may use k, M, G or T suffixes for kilobytes, megabytes, gigabytes and terabytes.
 qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 == Check correct interpretation of suffixes for cluster size ==
@@ -157,30 +156,30 @@ Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_si
 == Check compat level option ==
 
 qemu-img create -f qcow2 -o compat=0.10 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=1.1 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.42 TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: '0.42'
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.42' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.42 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=foobar TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: 'foobar'
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='foobar' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=foobar encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 == Check preallocation option ==
 
 qemu-img create -f qcow2 -o preallocation=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='off' lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=off lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o preallocation=metadata TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='metadata' lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=metadata lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o preallocation=1234 TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: invalid parameter value: 1234
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='1234' lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=1234 lazy_refcounts=off refcount_bits=16
 
 == Check encryption option ==
 
@@ -199,16 +198,16 @@ Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=on cluster_siz
 == Check lazy_refcounts option (only with v3) ==
 
 qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=on TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=on TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
 
 *** done
diff --git a/tests/qemu-iotests/050.out b/tests/qemu-iotests/050.out
index a6cb2e686..397cf7f39 100644
--- a/tests/qemu-iotests/050.out
+++ b/tests/qemu-iotests/050.out
@@ -7,7 +7,7 @@ wrote 1048576/1048576 bytes at offset 0
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 backing_file='TEST_DIR/t.IMGFMT.old'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 backing_file=TEST_DIR/t.IMGFMT.old
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/tests/qemu-iotests/051 b/tests/qemu-iotests/051
index 4a8055b67..17dbf04af 100755
--- a/tests/qemu-iotests/051
+++ b/tests/qemu-iotests/051
@@ -108,7 +108,8 @@ echo
 echo === Overriding backing file ===
 echo
 
-echo "info block" | run_qemu -drive file="$TEST_IMG",driver=qcow2,backing.file.filename="$TEST_IMG.orig" -nodefaults
+echo "info block" | run_qemu -drive file="$TEST_IMG",driver=qcow2,backing.file.filename="$TEST_IMG.orig" -nodefaults\
+                  | _filter_generated_node_ids
 
 # Drivers that don't support backing files
 run_qemu -drive file="$TEST_IMG",driver=raw,backing.file.filename="$TEST_IMG.orig"
diff --git a/tests/qemu-iotests/051.out b/tests/qemu-iotests/051.out
index 23c282357..7765aa0bb 100644
--- a/tests/qemu-iotests/051.out
+++ b/tests/qemu-iotests/051.out
@@ -1,6 +1,6 @@
 QA output created by 051
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 
 === Unknown option ===
 
@@ -59,7 +59,7 @@ QEMU X.Y.Z monitor - type 'help' for more information
 Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig -nodefaults
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
-ide0-hd0: TEST_DIR/t.qcow2 (qcow2)
+ide0-hd0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
     Cache mode:       writeback
     Backing file:     TEST_DIR/t.qcow2.orig (chain depth: 1)
 (qemu) qququiquit
diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055
index 017a609f3..c8e357870 100755
--- a/tests/qemu-iotests/055
+++ b/tests/qemu-iotests/055
@@ -42,6 +42,8 @@ class TestSingleDrive(iotests.QMPTestCase):
         qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len))
 
         self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
@@ -104,12 +106,18 @@ class TestSingleDrive(iotests.QMPTestCase):
         self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img)
 
     def test_medium_not_found(self):
-        result = self.vm.qmp('drive-backup', device='ide1-cd0',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('drive-backup', device='drive2', # CD-ROM
                              target=target_img, sync='full')
         self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_medium_not_found_blockdev_backup(self):
-        result = self.vm.qmp('blockdev-backup', device='ide1-cd0',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('blockdev-backup', device='drive2', # CD-ROM
                              target='drive1', sync='full')
         self.assert_qmp(result, 'error/class', 'GenericError')
 
@@ -249,6 +257,8 @@ class TestSingleTransaction(iotests.QMPTestCase):
         qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len))
 
         self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
@@ -323,9 +333,12 @@ class TestSingleTransaction(iotests.QMPTestCase):
         self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img)
 
     def do_test_medium_not_found(self, cmd, target):
+        if iotests.qemu_default_machine != 'pc':
+            return
+
         result = self.vm.qmp('transaction', actions=[{
                 'type': cmd,
-                'data': { 'device': 'ide1-cd0',
+                'data': { 'device': 'drive2', # CD-ROM
                           'target': target,
                           'sync': 'full' },
             }
diff --git a/tests/qemu-iotests/056 b/tests/qemu-iotests/056
index 54e4bd069..04f2c3c84 100755
--- a/tests/qemu-iotests/056
+++ b/tests/qemu-iotests/056
@@ -82,6 +82,31 @@ class TestSyncModesNoneAndTop(iotests.QMPTestCase):
         time.sleep(1)
         self.assertEqual(-1, qemu_io('-c', 'read -P0x41 0 512', target_img).find("verification failed"))
 
+class TestBeforeWriteNotifier(iotests.QMPTestCase):
+    def setUp(self):
+        self.vm = iotests.VM().add_drive_raw("file=blkdebug::null-co://,id=drive0,align=65536,driver=blkdebug")
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(target_img)
+
+    def test_before_write_notifier(self):
+        self.vm.pause_drive("drive0")
+        result = self.vm.qmp('drive-backup', device='drive0',
+                             sync='full', target=target_img,
+                             format="file", speed=1)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('block-job-pause', device="drive0")
+        self.assert_qmp(result, 'return', {})
+        # Speed is low enough that this must be an uncopied range, which will
+        # trigger the before write notifier
+        self.vm.hmp_qemu_io('drive0', 'aio_write -P 1 512512 512')
+        self.vm.resume_drive("drive0")
+        result = self.vm.qmp('block-job-resume', device="drive0")
+        self.assert_qmp(result, 'return', {})
+        event = self.cancel_and_wait()
+        self.assert_qmp(event, 'data/type', 'backup')
 
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/056.out b/tests/qemu-iotests/056.out
index fbc63e62f..8d7e99670 100644
--- a/tests/qemu-iotests/056.out
+++ b/tests/qemu-iotests/056.out
@@ -1,5 +1,5 @@
-..
+...
 ----------------------------------------------------------------------
-Ran 2 tests
+Ran 3 tests
 
 OK
diff --git a/tests/qemu-iotests/058 b/tests/qemu-iotests/058
index f2bdd0bff..63a659878 100755
--- a/tests/qemu-iotests/058
+++ b/tests/qemu-iotests/058
@@ -32,11 +32,17 @@ status=1	# failure is the default!
 
 nbd_unix_socket=$TEST_DIR/test_qemu_nbd_socket
 nbd_snapshot_img="nbd:unix:$nbd_unix_socket"
+rm -f "${TEST_DIR}/qemu-nbd.pid"
 
 _cleanup_nbd()
 {
-    if [ -n "$NBD_SNAPSHOT_PID" ]; then
-        kill "$NBD_SNAPSHOT_PID"
+    local NBD_SNAPSHOT_PID
+    if [ -f "${TEST_DIR}/qemu-nbd.pid" ]; then
+        read NBD_SNAPSHOT_PID < "${TEST_DIR}/qemu-nbd.pid"
+        rm -f "${TEST_DIR}/qemu-nbd.pid"
+        if [ -n "$NBD_SNAPSHOT_PID" ]; then
+            kill "$NBD_SNAPSHOT_PID"
+        fi
     fi
     rm -f "$nbd_unix_socket"
 }
@@ -60,7 +66,6 @@ _export_nbd_snapshot()
 {
     _cleanup_nbd
     $QEMU_NBD -v -t -k "$nbd_unix_socket" "$TEST_IMG" -l $1 &
-    NBD_SNAPSHOT_PID=$!
     _wait_for_nbd
 }
 
@@ -68,7 +73,6 @@ _export_nbd_snapshot1()
 {
     _cleanup_nbd
     $QEMU_NBD -v -t -k "$nbd_unix_socket" "$TEST_IMG" -l snapshot.name=$1 &
-    NBD_SNAPSHOT_PID=$!
     _wait_for_nbd
 }
 
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index 67e3cf57e..00057fef9 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -16,17 +16,17 @@ qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big
 no file open, try 'help open'
 
 === Testing monolithicFlat creation and opening ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 2.0G (2147483648 bytes)
 
 === Testing monolithicFlat with zeroed_grain ===
 qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
 
 === Testing big twoGbMaxExtentFlat ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat
 image: TEST_DIR/t.vmdk
 file format: vmdk
 virtual size: 1.0T (1073741824000 bytes)
@@ -2043,7 +2043,7 @@ RW 12582912 VMFS "dummy.IMGFMT" 1
 
 
 === Testing truncated sparse ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes
 
 === Converting to streamOptimized from image with small cluster size===
@@ -2054,7 +2054,7 @@ wrote 512/512 bytes at offset 10240
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing monolithicFlat with internally generated JSON file name ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat
 qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
 
 === Testing version 3 ===
@@ -2264,7 +2264,7 @@ read 512/512 bytes at offset 64931328
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing 4TB monolithicFlat creation and IO ===
-Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104
+Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
 image: TEST_DIR/iotest-version3.IMGFMT
 file format: IMGFMT
 virtual size: 4.0T (4398046511104 bytes)
diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061
index 8d37f8a65..e191e65d5 100755
--- a/tests/qemu-iotests/061
+++ b/tests/qemu-iotests/061
@@ -58,7 +58,8 @@ echo
 echo "=== Testing dirty version downgrade ==="
 echo
 IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
-$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
 $PYTHON qcow2.py "$TEST_IMG" dump-header
@@ -91,7 +92,8 @@ echo
 echo "=== Testing dirty lazy_refcounts=off ==="
 echo
 IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
-$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "lazy_refcounts=off" "$TEST_IMG"
 $PYTHON qcow2.py "$TEST_IMG" dump-header
diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out
index 5ec248f79..f2598a8f9 100644
--- a/tests/qemu-iotests/061.out
+++ b/tests/qemu-iotests/061.out
@@ -57,6 +57,11 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -214,6 +219,11 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -334,7 +344,7 @@ read 131072/131072 bytes at offset 0
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 read 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 0
@@ -350,7 +360,7 @@ read 65536/65536 bytes at offset 65536
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 0
@@ -369,7 +379,7 @@ read 65536/65536 bytes at offset 65536
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
@@ -393,7 +403,7 @@ read 67108864/67108864 bytes at offset 0
 === Testing progress report without snapshot ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4294967296
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 1073741824
@@ -408,7 +418,7 @@ No errors were found on the image.
 === Testing progress report with snapshot ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4294967296
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 1073741824
diff --git a/tests/qemu-iotests/067 b/tests/qemu-iotests/067
index 83eefa394..3788534d6 100755
--- a/tests/qemu-iotests/067
+++ b/tests/qemu-iotests/067
@@ -48,7 +48,8 @@ function do_run_qemu()
 function run_qemu()
 {
     do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | _filter_qemu \
-                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
+                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g' \
+                          | _filter_generated_node_ids
 }
 
 size=128M
@@ -59,7 +60,7 @@ echo
 echo === -drive/-device and device_del ===
 echo
 
-run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0 <<EOF
+run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk,drive=disk,id=virtio0 <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "query-block" }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
@@ -76,7 +77,7 @@ run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
@@ -94,7 +95,7 @@ run_qemu <<EOF
   "arguments": { "command-line": "drive_add 0 file=$TEST_IMG,format=$IMGFMT,if=none,id=disk" } }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
@@ -122,7 +123,7 @@ run_qemu <<EOF
   }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
diff --git a/tests/qemu-iotests/067.out b/tests/qemu-iotests/067.out
index 6ff41bc7a..27ad56fe2 100644
--- a/tests/qemu-iotests/067.out
+++ b/tests/qemu-iotests/067.out
@@ -3,7 +3,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 === -drive/-device and device_del ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk,drive=disk,id=virtio0
 {
     QMP_VERSION
 }
@@ -40,6 +40,7 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virti
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -57,28 +58,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virti
                 "encryption_key_missing": false
             },
             "type": "unknown"
-        },
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
         }
     ]
 }
@@ -120,28 +99,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virti
 }
 {
     "return": [
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        }
     ]
 }
 {
@@ -155,28 +112,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virti
     },
     "event": "SHUTDOWN"
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
 
 
 === -drive/device_add and device_del ===
@@ -217,6 +152,7 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -235,28 +171,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
             },
             "tray_open": false,
             "type": "unknown"
-        },
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
         }
     ]
 }
@@ -302,28 +216,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
 }
 {
     "return": [
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        }
     ]
 }
 {
@@ -337,28 +229,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
     },
     "event": "SHUTDOWN"
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
 
 
 === drive_add/device_add and device_del ===
@@ -377,28 +247,6 @@ Testing:
 {
     "return": [
         {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
             "device": "disk",
             "locked": false,
             "removable": true,
@@ -424,6 +272,7 @@ Testing:
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -487,28 +336,6 @@ Testing:
 }
 {
     "return": [
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        }
     ]
 }
 {
@@ -522,28 +349,6 @@ Testing:
     },
     "event": "SHUTDOWN"
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
 
 
 === blockdev_add/device_add and device_del ===
@@ -563,28 +368,6 @@ Testing:
 {
     "return": [
         {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
             "device": "disk",
             "locked": false,
             "removable": true,
@@ -610,6 +393,7 @@ Testing:
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -675,28 +459,6 @@ Testing:
     "return": [
         {
             "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "io-status": "ok",
             "device": "disk",
             "locked": false,
             "removable": true,
@@ -722,6 +484,7 @@ Testing:
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -754,27 +517,5 @@ Testing:
     },
     "event": "SHUTDOWN"
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
 
 *** done
diff --git a/tests/qemu-iotests/069.out b/tests/qemu-iotests/069.out
index 4d7e63cf8..c78e8c2b7 100644
--- a/tests/qemu-iotests/069.out
+++ b/tests/qemu-iotests/069.out
@@ -3,6 +3,6 @@ QA output created by 069
 === Creating an image with a backing file and deleting that file ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=131072
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 backing_file=TEST_DIR/t.IMGFMT.base
 qemu-io: can't open device TEST_DIR/t.IMGFMT: Could not open backing file: Could not open 'TEST_DIR/t.IMGFMT.base': No such file or directory
 *** done
diff --git a/tests/qemu-iotests/071 b/tests/qemu-iotests/071
index 9eaa49b41..92ab99145 100755
--- a/tests/qemu-iotests/071
+++ b/tests/qemu-iotests/071
@@ -104,11 +104,20 @@ echo
 echo "=== Testing blkdebug on existing block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "$IMGFMT",
             "id": "drive0-debug",
             "file": {
@@ -133,20 +142,29 @@ echo
 echo "=== Testing blkverify on existing block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=$IMGFMT,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "$IMGFMT",
+            "file": {
+                "driver": "file",
+                "filename": "$TEST_IMG"
+            }
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "blkverify",
             "id": "drive0-verify",
             "test": "drive0",
             "raw": {
-                "driver": "raw",
-                "file": {
-                    "driver": "file",
-                    "filename": "$TEST_IMG.base"
-                }
+                "driver": "file",
+                "filename": "$TEST_IMG.base"
             }
         }
     }
@@ -163,11 +181,20 @@ echo
 echo "=== Testing blkverify on existing raw block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG.base,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG.base"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "blkverify",
             "id": "drive0-verify",
             "test": {
@@ -193,11 +220,20 @@ echo
 echo "=== Testing blkdebug's set-state through QMP ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "$IMGFMT",
             "id": "drive0-debug",
             "file": {
diff --git a/tests/qemu-iotests/071.out b/tests/qemu-iotests/071.out
index 9205ce251..2b40eadae 100644
--- a/tests/qemu-iotests/071.out
+++ b/tests/qemu-iotests/071.out
@@ -42,44 +42,46 @@ read failed: Input/output error
 
 === Testing blkdebug on existing block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 read failed: Input/output error
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 QEMU_PROG: Failed to flush the L2 table cache: Input/output error
 QEMU_PROG: Failed to flush the refcount block cache: Input/output error
 
 
 === Testing blkverify on existing block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=IMGFMT,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 
 === Testing blkverify on existing raw block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT.base,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 
 === Testing blkdebug's set-state through QMP ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 {"return": ""}
@@ -90,8 +92,6 @@ read failed: Input/output error
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 QEMU_PROG: Failed to flush the L2 table cache: Input/output error
 QEMU_PROG: Failed to flush the refcount block cache: Input/output error
 
diff --git a/tests/qemu-iotests/073.out b/tests/qemu-iotests/073.out
index 733d79ca9..de5452492 100644
--- a/tests/qemu-iotests/073.out
+++ b/tests/qemu-iotests/073.out
@@ -2,7 +2,7 @@ QA output created by 073
 
 == creating backing file ==
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 134217728/134217728 bytes at offset 0
 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/tests/qemu-iotests/079.out b/tests/qemu-iotests/079.out
index 6dc5d5763..aab922fb3 100644
--- a/tests/qemu-iotests/079.out
+++ b/tests/qemu-iotests/079.out
@@ -1,14 +1,14 @@
 QA output created by 079
 === Check option preallocation and cluster_size ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
 qemu-img: TEST_DIR/t.IMGFMT: Cluster size must be a power of two between 512 and 2048k
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
 *** done
diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081
index d9b042cfc..e4b4c6cbf 100755
--- a/tests/qemu-iotests/081
+++ b/tests/qemu-iotests/081
@@ -53,7 +53,8 @@ function do_run_qemu()
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_qmp | _filter_qemu_io
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_qmp\
+                          | _filter_qemu_io | _filter_generated_node_ids
 }
 
 test_quorum=$($QEMU_IMG --help|grep quorum)
@@ -101,17 +102,29 @@ $QEMU_IO -c "open -o $quorum" -c "read -P 0x32 0 $size" | _filter_qemu_io
 echo
 echo "== checking mixed reference/option specification =="
 
-run_qemu -drive "file=$TEST_DIR/2.raw,format=$IMGFMT,if=none,id=drive2" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive2",
+            "driver": "$IMGFMT",
+            "file": {
+                "driver": "file",
+                "filename": "$TEST_DIR/2.raw"
+            }
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "quorum",
             "id": "drive0-quorum",
             "vote-threshold": 2,
             "children": [
                 {
-                    "driver": "raw",
+                    "driver": "$IMGFMT",
                     "file": {
                         "driver": "file",
                         "filename": "$TEST_DIR/1.raw"
@@ -119,7 +132,7 @@ run_qemu -drive "file=$TEST_DIR/2.raw,format=$IMGFMT,if=none,id=drive2" <<EOF
                 },
                 "drive2",
                 {
-                    "driver": "raw",
+                    "driver": "$IMGFMT",
                     "file": {
                         "driver": "file",
                         "filename": "$TEST_DIR/3.raw"
diff --git a/tests/qemu-iotests/081.out b/tests/qemu-iotests/081.out
index 9f57d9d3b..70632314c 100644
--- a/tests/qemu-iotests/081.out
+++ b/tests/qemu-iotests/081.out
@@ -26,18 +26,17 @@ read 10485760/10485760 bytes at offset 0
 10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == checking mixed reference/option specification ==
-Testing: -drive file=TEST_DIR/2.IMGFMT,format=IMGFMT,if=none,id=drive2
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "QUORUM_REPORT_BAD", "data": {"node-name": "", "sectors-count": 20480, "sector-num": 0}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "QUORUM_REPORT_BAD", "data": {"node-name": "drive2", "sectors-count": 20480, "sector-num": 0}}
 read 10485760/10485760 bytes at offset 0
 10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 == using quorum rewrite corrupted mode ==
diff --git a/tests/qemu-iotests/082.out b/tests/qemu-iotests/082.out
index 3a749b825..a952330ba 100644
--- a/tests/qemu-iotests/082.out
+++ b/tests/qemu-iotests/082.out
@@ -147,10 +147,10 @@ refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2,,help TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2,help' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2,,help encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2,,? TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2,?' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2,,? encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2, -o help TEST_DIR/t.qcow2 128M
 qemu-img: Invalid option list: backing_file=TEST_DIR/t.qcow2,
diff --git a/tests/qemu-iotests/085 b/tests/qemu-iotests/085
index 56cd6f89b..aa77eca77 100755
--- a/tests/qemu-iotests/085
+++ b/tests/qemu-iotests/085
@@ -7,6 +7,7 @@
 # snapshots are performed.
 #
 # Copyright (C) 2014 Red Hat, Inc.
+# Copyright (C) 2015 Igalia, S.L.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -34,17 +35,17 @@ status=1	# failure is the default!
 snapshot_virt0="snapshot-v0.qcow2"
 snapshot_virt1="snapshot-v1.qcow2"
 
-MAX_SNAPSHOTS=10
+SNAPSHOTS=10
 
 _cleanup()
 {
     _cleanup_qemu
-    for i in $(seq 1 ${MAX_SNAPSHOTS})
+    for i in $(seq 1 ${SNAPSHOTS})
     do
         rm -f "${TEST_DIR}/${i}-${snapshot_virt0}"
         rm -f "${TEST_DIR}/${i}-${snapshot_virt1}"
     done
-	_cleanup_test_img
+    rm -f "${TEST_IMG}.1" "${TEST_IMG}.2"
 
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -64,7 +65,7 @@ function create_single_snapshot()
 {
     cmd="{ 'execute': 'blockdev-snapshot-sync',
                       'arguments': { 'device': 'virtio0',
-                                     'snapshot-file':'"${TEST_DIR}/${1}-${snapshot_virt0}"',
+                                     'snapshot-file':'${TEST_DIR}/${1}-${snapshot_virt0}',
                                      'format': 'qcow2' } }"
     _send_qemu_cmd $h "${cmd}" "return"
 }
@@ -76,27 +77,60 @@ function create_group_snapshot()
            {'actions': [
                { 'type': 'blockdev-snapshot-sync', 'data' :
                    { 'device': 'virtio0',
-                      'snapshot-file': '"${TEST_DIR}/${1}-${snapshot_virt0}"' } },
+                      'snapshot-file': '${TEST_DIR}/${1}-${snapshot_virt0}' } },
                { 'type': 'blockdev-snapshot-sync', 'data' :
                    { 'device': 'virtio1',
-                       'snapshot-file': '"${TEST_DIR}/${1}-${snapshot_virt1}"' } } ]
+                       'snapshot-file': '${TEST_DIR}/${1}-${snapshot_virt1}' } } ]
              } }"
 
     _send_qemu_cmd $h "${cmd}" "return"
 }
 
+# ${1}: unique identifier for the snapshot filename
+# ${2}: true: open backing images; false: don't open them (default)
+function add_snapshot_image()
+{
+    if [ "${2}" = "true" ]; then
+        extra_params=""
+    else
+        extra_params="'backing': '', "
+    fi
+    base_image="${TEST_DIR}/$((${1}-1))-${snapshot_virt0}"
+    snapshot_file="${TEST_DIR}/${1}-${snapshot_virt0}"
+    _make_test_img -b "${base_image}" "$size"
+    mv "${TEST_IMG}" "${snapshot_file}"
+    cmd="{ 'execute': 'blockdev-add', 'arguments':
+           { 'options':
+             { 'driver': 'qcow2', 'node-name': 'snap_${1}', ${extra_params}
+               'file':
+               { 'driver': 'file', 'filename': '${snapshot_file}',
+                 'node-name': 'file_${1}' } } } }"
+    _send_qemu_cmd $h "${cmd}" "return"
+}
+
+# ${1}: unique identifier for the snapshot filename
+# ${2}: expected response, defaults to 'return'
+function blockdev_snapshot()
+{
+    cmd="{ 'execute': 'blockdev-snapshot',
+                      'arguments': { 'node': 'virtio0',
+                                     'overlay':'snap_${1}' } }"
+    _send_qemu_cmd $h "${cmd}" "${2:-return}"
+}
+
 size=128M
 
 _make_test_img $size
-mv "${TEST_IMG}" "${TEST_IMG}.orig"
+mv "${TEST_IMG}" "${TEST_IMG}.1"
 _make_test_img $size
+mv "${TEST_IMG}" "${TEST_IMG}.2"
 
 echo
 echo === Running QEMU ===
 echo
 
 qemu_comm_method="qmp"
-_launch_qemu -drive file="${TEST_IMG}.orig",if=virtio -drive file="${TEST_IMG}",if=virtio
+_launch_qemu -drive file="${TEST_IMG}.1",if=virtio -drive file="${TEST_IMG}.2",if=virtio
 h=$QEMU_HANDLE
 
 echo
@@ -105,6 +139,8 @@ echo
 
 _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" "return"
 
+# Tests for the blockdev-snapshot-sync command
+
 echo
 echo === Create a single snapshot on virtio0 ===
 echo
@@ -117,7 +153,7 @@ echo === Invalid command - missing device and nodename ===
 echo
 
 _send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
-                         'arguments': { 'snapshot-file':'"${TEST_DIR}/1-${snapshot_virt0}"',
+                         'arguments': { 'snapshot-file':'${TEST_DIR}/1-${snapshot_virt0}',
                                      'format': 'qcow2' } }" "error"
 
 echo
@@ -132,11 +168,75 @@ echo
 echo === Create several transactional group snapshots ===
 echo
 
-for i in $(seq 2 ${MAX_SNAPSHOTS})
+for i in $(seq 2 ${SNAPSHOTS})
 do
     create_group_snapshot ${i}
 done
 
+# Tests for the blockdev-snapshot command
+
+echo
+echo === Create a couple of snapshots using blockdev-snapshot ===
+echo
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS}
+blockdev_snapshot ${SNAPSHOTS}
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS}
+blockdev_snapshot ${SNAPSHOTS}
+
+echo
+echo === Invalid command - cannot create a snapshot using a file BDS ===
+echo
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'file_${SNAPSHOTS}' }
+                   }" "error"
+
+echo
+echo === Invalid command - snapshot node used as active layer ===
+echo
+
+blockdev_snapshot ${SNAPSHOTS} error
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'virtio0' }
+                   }" "error"
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'virtio1' }
+                   }" "error"
+
+echo
+echo === Invalid command - snapshot node used as backing hd ===
+echo
+
+blockdev_snapshot $((${SNAPSHOTS}-1)) error
+
+echo
+echo === Invalid command - snapshot node has a backing image ===
+echo
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS} true
+blockdev_snapshot ${SNAPSHOTS} error
+
+echo
+echo === Invalid command - The node does not exist ===
+echo
+
+blockdev_snapshot $((${SNAPSHOTS}+1)) error
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'nodevice',
+                                    'overlay':'snap_${SNAPSHOTS}' }
+                   }" "error"
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/085.out b/tests/qemu-iotests/085.out
index 5eb8b947c..01c78d689 100644
--- a/tests/qemu-iotests/085.out
+++ b/tests/qemu-iotests/085.out
@@ -11,7 +11,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 === Create a single snapshot on virtio0 ===
 
-Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2.orig' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2.1 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
 
 === Invalid command - missing device and nodename ===
@@ -25,31 +25,65 @@ Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file
 
 === Create several transactional group snapshots ===
 
-Formatting 'TEST_DIR/2-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/1-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/2-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/2-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/1-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/2-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2.2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/3-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/2-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/3-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/2-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/3-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/2-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/3-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/2-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/4-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/3-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/4-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/3-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/4-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/3-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/4-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/3-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/5-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/4-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/5-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/4-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/5-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/4-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/5-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/4-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/6-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/5-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/6-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/5-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/6-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/5-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/6-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/5-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/7-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/6-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/7-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/6-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/7-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/6-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/7-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/6-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/8-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/7-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/8-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/7-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/8-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/7-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/8-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/7-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/9-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/8-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/9-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/8-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/9-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/8-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/9-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/8-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/10-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/9-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/10-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/9-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/10-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/9-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/10-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/9-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
+
+=== Create a couple of snapshots using blockdev-snapshot ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/10-snapshot-v0.IMGFMT
+{"return": {}}
+{"return": {}}
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/11-snapshot-v0.IMGFMT
+{"return": {}}
+{"return": {}}
+
+=== Invalid command - cannot create a snapshot using a file BDS ===
+
+{"error": {"class": "GenericError", "desc": "The snapshot does not support backing images"}}
+
+=== Invalid command - snapshot node used as active layer ===
+
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio0"}}
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio0"}}
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio1"}}
+
+=== Invalid command - snapshot node used as backing hd ===
+
+{"error": {"class": "GenericError", "desc": "Node 'snap_11' is busy: node is used as backing hd of 'virtio0'"}}
+
+=== Invalid command - snapshot node has a backing image ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/12-snapshot-v0.IMGFMT
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "The snapshot already has a backing image"}}
+
+=== Invalid command - The node does not exist ===
+
+{"error": {"class": "GenericError", "desc": "Cannot find device=snap_14 nor node_name=snap_14"}}
+{"error": {"class": "GenericError", "desc": "Cannot find device=nodevice nor node_name=nodevice"}}
 *** done
diff --git a/tests/qemu-iotests/087 b/tests/qemu-iotests/087
index 869474994..af44299e0 100755
--- a/tests/qemu-iotests/087
+++ b/tests/qemu-iotests/087
@@ -54,7 +54,7 @@ size=128M
 _make_test_img $size
 
 echo
-echo === Missing ID ===
+echo === Missing ID and node-name ===
 echo
 
 run_qemu <<EOF
diff --git a/tests/qemu-iotests/087.out b/tests/qemu-iotests/087.out
index c71bb3aa4..7d62cd584 100644
--- a/tests/qemu-iotests/087.out
+++ b/tests/qemu-iotests/087.out
@@ -1,16 +1,14 @@
 QA output created by 087
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
-=== Missing ID ===
+=== Missing ID and node-name ===
 
 Testing:
 QMP_VERSION
 {"return": {}}
-{"error": {"class": "GenericError", "desc": "Block device needs an ID"}}
+{"error": {"class": "GenericError", "desc": "'id' and/or 'node-name' need to be specified for the root node"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Duplicate ID ===
@@ -26,8 +24,6 @@ QMP_VERSION
 {"error": {"class": "GenericError", "desc": "node-name=disk3 is conflicting with a device id"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === aio=native without O_DIRECT ===
@@ -38,8 +34,6 @@ QMP_VERSION
 {"error": {"class": "GenericError", "desc": "aio=native requires cache.direct=true"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Encrypted image ===
@@ -60,8 +54,6 @@ You can use 'qemu-img convert' to convert your image to an unencrypted one.
 {"error": {"class": "GenericError", "desc": "blockdev-add doesn't support encrypted devices"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 Testing:
 QMP_VERSION
@@ -72,8 +64,6 @@ You can use 'qemu-img convert' to convert your image to an unencrypted one.
 {"error": {"class": "GenericError", "desc": "Guest must be stopped for opening of encrypted image"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Missing driver ===
@@ -91,7 +81,5 @@ QMP_VERSION
 {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'driver', expected: string"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 *** done
diff --git a/tests/qemu-iotests/095.out b/tests/qemu-iotests/095.out
index 267c4836e..236006162 100644
--- a/tests/qemu-iotests/095.out
+++ b/tests/qemu-iotests/095.out
@@ -1,7 +1,7 @@
 QA output created by 095
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=5242880
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR/t.IMGFMT.snp1'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file=TEST_DIR/t.IMGFMT.snp1
 
 === Base image info before commit and resize ===
 image: TEST_DIR/t.IMGFMT.base
diff --git a/tests/qemu-iotests/096 b/tests/qemu-iotests/096
new file mode 100644
index 000000000..e34204b8f
--- /dev/null
+++ b/tests/qemu-iotests/096
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+#
+# Test that snapshots move the throttling configuration to the active
+# layer
+#
+# Copyright (C) 2015 Igalia, S.L.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+import os
+
+class TestLiveSnapshot(iotests.QMPTestCase):
+    base_img = os.path.join(iotests.test_dir, 'base.img')
+    target_img = os.path.join(iotests.test_dir, 'target.img')
+    group = 'mygroup'
+    iops = 6000
+    iops_size = 1024
+
+    def setUp(self):
+        opts = []
+        opts.append('node-name=base')
+        opts.append('throttling.group=%s' % self.group)
+        opts.append('throttling.iops-total=%d' % self.iops)
+        opts.append('throttling.iops-size=%d' % self.iops_size)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, self.base_img, '100M')
+        self.vm = iotests.VM().add_drive(self.base_img, ','.join(opts))
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.base_img)
+        os.remove(self.target_img)
+
+    def checkConfig(self, active_layer):
+        result = self.vm.qmp('query-named-block-nodes')
+        for r in result['return']:
+            if r['node-name'] == active_layer:
+                self.assertEqual(r['group'], self.group)
+                self.assertEqual(r['iops'], self.iops)
+                self.assertEqual(r['iops_size'], self.iops_size)
+            else:
+                self.assertFalse(r.has_key('group'))
+                self.assertEqual(r['iops'], 0)
+                self.assertFalse(r.has_key('iops_size'))
+
+    def testSnapshot(self):
+        self.checkConfig('base')
+        self.vm.qmp('blockdev-snapshot-sync',
+                    node_name = 'base',
+                    snapshot_node_name = 'target',
+                    snapshot_file = self.target_img,
+                    format = iotests.imgfmt)
+        self.checkConfig('target')
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/096.out b/tests/qemu-iotests/096.out
new file mode 100644
index 000000000..ae1213e6f
--- /dev/null
+++ b/tests/qemu-iotests/096.out
@@ -0,0 +1,5 @@
+.
+----------------------------------------------------------------------
+Ran 1 tests
+
+OK
diff --git a/tests/qemu-iotests/097.out b/tests/qemu-iotests/097.out
index 81651f496..48abd2e64 100644
--- a/tests/qemu-iotests/097.out
+++ b/tests/qemu-iotests/097.out
@@ -3,8 +3,8 @@ QA output created by 097
 === Test pass 0 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -32,8 +32,8 @@ Offset          Length          File
 === Test pass 1 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -62,8 +62,8 @@ Offset          Length          File
 === Test pass 2 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -92,8 +92,8 @@ Offset          Length          File
 === Test pass 3 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
diff --git a/tests/qemu-iotests/098.out b/tests/qemu-iotests/098.out
index e08a189b4..7634d0e8b 100644
--- a/tests/qemu-iotests/098.out
+++ b/tests/qemu-iotests/098.out
@@ -3,7 +3,7 @@ QA output created by 098
 === l1_update ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -12,7 +12,7 @@ No errors were found on the image.
 === empty_image_prepare ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -25,7 +25,7 @@ No errors were found on the image.
 === reftable_update ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -39,7 +39,7 @@ No errors were found on the image.
 === refblock_alloc ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
diff --git a/tests/qemu-iotests/110.out b/tests/qemu-iotests/110.out
index 152bacf41..0270980c2 100644
--- a/tests/qemu-iotests/110.out
+++ b/tests/qemu-iotests/110.out
@@ -3,7 +3,7 @@ QA output created by 110
 === Reconstructable filename ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=t.IMGFMT.base
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
@@ -15,5 +15,5 @@ qemu-img: Cannot use relative backing file names for 'json:{"driver": "IMGFMT",
 
 === Backing name is always relative to the backed image ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=t.IMGFMT.base
 *** done
diff --git a/tests/qemu-iotests/112.out b/tests/qemu-iotests/112.out
index 9a98633f6..8dd3df087 100644
--- a/tests/qemu-iotests/112.out
+++ b/tests/qemu-iotests/112.out
@@ -21,9 +21,9 @@ refcount bits: 16
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 refcount bits: 16
-qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use compat=1.1 or greater)
+qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use or greater)
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use compat=1.1 or greater)
+qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use or greater)
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 
 === Snapshot limit on refcount_bits=1 ===
diff --git a/tests/qemu-iotests/114.out b/tests/qemu-iotests/114.out
index 6c6b21085..6a2c75056 100644
--- a/tests/qemu-iotests/114.out
+++ b/tests/qemu-iotests/114.out
@@ -1,6 +1,6 @@
 QA output created by 114
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base 
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
diff --git a/tests/qemu-iotests/115.out b/tests/qemu-iotests/115.out
index 7b2c5e02f..dbdad17b2 100644
--- a/tests/qemu-iotests/115.out
+++ b/tests/qemu-iotests/115.out
@@ -2,7 +2,7 @@ QA output created by 115
 
 === Testing large refcount and L1 table ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=268435456 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=268435456 preallocation=metadata
 No errors were found on the image.
 100.00% allocated clusters
 *** done
diff --git a/tests/qemu-iotests/118 b/tests/qemu-iotests/118
new file mode 100755
index 000000000..114d0e2de
--- /dev/null
+++ b/tests/qemu-iotests/118
@@ -0,0 +1,720 @@
+#!/usr/bin/env python
+#
+# Test case for the QMP 'change' command and all other associated
+# commands
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import stat
+import time
+import iotests
+from iotests import qemu_img
+
+old_img = os.path.join(iotests.test_dir, 'test0.img')
+new_img = os.path.join(iotests.test_dir, 'test1.img')
+
+class ChangeBaseClass(iotests.QMPTestCase):
+    has_opened = False
+    has_closed = False
+
+    def process_events(self):
+        for event in self.vm.get_qmp_events(wait=False):
+            if (event['event'] == 'DEVICE_TRAY_MOVED' and
+                event['data']['device'] == 'drive0'):
+                if event['data']['tray-open'] == False:
+                    self.has_closed = True
+                else:
+                    self.has_opened = True
+
+    def wait_for_open(self):
+        timeout = time.clock() + 3
+        while not self.has_opened and time.clock() < timeout:
+            self.process_events()
+        if not self.has_opened:
+            self.fail('Timeout while waiting for the tray to open')
+
+    def wait_for_close(self):
+        timeout = time.clock() + 3
+        while not self.has_closed and time.clock() < timeout:
+            self.process_events()
+        if not self.has_opened:
+            self.fail('Timeout while waiting for the tray to close')
+
+class GeneralChangeTestsBaseClass(ChangeBaseClass):
+    def test_change(self):
+        result = self.vm.qmp('change', device='drive0', target=new_img,
+                                       arg=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_blockdev_change_medium(self):
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_eject(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+    def test_tray_eject_change(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_tray_open_close(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        if self.has_real_tray or not self.was_empty:
+            self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray or not self.was_empty:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        else:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_tray_eject_close(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        if self.has_real_tray:
+            self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        else:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+    def test_tray_open_change(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_cycle(self):
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_close_on_closed(self):
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        # Should be a no-op
+        self.assert_qmp(result, 'return', {})
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+    def test_remove_on_closed(self):
+        if self.has_opened:
+            # Empty floppy drive
+            return
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_insert_on_closed(self):
+        if self.has_opened:
+            # Empty floppy drive
+            return
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+class TestInitiallyFilled(GeneralChangeTestsBaseClass):
+    was_empty = False
+
+    def setUp(self, media, interface):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1440k')
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM().add_drive(old_img, 'media=%s' % media, interface)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(old_img)
+        os.remove(new_img)
+
+    def test_insert_on_filled(self):
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-open-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+class TestInitiallyEmpty(GeneralChangeTestsBaseClass):
+    was_empty = True
+
+    def setUp(self, media, interface):
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM().add_drive(None, 'media=%s' % media, interface)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(new_img)
+
+    def test_remove_on_empty(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        # Should be a no-op
+        self.assert_qmp(result, 'return', {})
+
+class TestCDInitiallyFilled(TestInitiallyFilled):
+    TestInitiallyFilled = TestInitiallyFilled
+    has_real_tray = True
+
+    def setUp(self):
+        self.TestInitiallyFilled.setUp(self, 'cdrom', 'ide')
+
+class TestCDInitiallyEmpty(TestInitiallyEmpty):
+    TestInitiallyEmpty = TestInitiallyEmpty
+    has_real_tray = True
+
+    def setUp(self):
+        self.TestInitiallyEmpty.setUp(self, 'cdrom', 'ide')
+
+class TestFloppyInitiallyFilled(TestInitiallyFilled):
+    TestInitiallyFilled = TestInitiallyFilled
+    has_real_tray = False
+
+    def setUp(self):
+        self.TestInitiallyFilled.setUp(self, 'disk', 'floppy')
+
+class TestFloppyInitiallyEmpty(TestInitiallyEmpty):
+    TestInitiallyEmpty = TestInitiallyEmpty
+    has_real_tray = False
+
+    def setUp(self):
+        self.TestInitiallyEmpty.setUp(self, 'disk', 'floppy')
+        # FDDs not having a real tray and there not being a medium inside the
+        # tray at startup means the tray will be considered open
+        self.has_opened = True
+
+class TestChangeReadOnly(ChangeBaseClass):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1440k')
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.chmod(old_img, 0666)
+        os.chmod(new_img, 0666)
+        os.remove(old_img)
+        os.remove(new_img)
+
+    def test_ro_ro_retain(self):
+        os.chmod(old_img, 0444)
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_ro_rw_retain(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_rw_ro_retain(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_ro_rw(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-write')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_rw_ro(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-only')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_rw_ro(self):
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-only')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_ro_rw(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-write')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_make_rw_ro_by_retain(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_ro_rw_by_retain(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_rw_ro_cycle(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'read-only': True,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+GeneralChangeTestsBaseClass = None
+TestInitiallyFilled = None
+TestInitiallyEmpty = None
+
+
+class TestBlockJobsAfterCycle(ChangeBaseClass):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1M')
+
+        self.vm = iotests.VM()
+        self.vm.launch()
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'id': 'drive0',
+                                      'driver': 'null-co'})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/format', 'null-co')
+
+        # For device-less BBs, calling blockdev-open-tray or blockdev-close-tray
+        # is not necessary
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'node0',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': old_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='node0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(old_img)
+        try:
+            os.remove(new_img)
+        except OSError:
+            pass
+
+    def test_snapshot_and_commit(self):
+        # We need backing file support
+        if iotests.imgfmt != 'qcow2' and iotests.imgfmt != 'qed':
+            return
+
+        result = self.vm.qmp('blockdev-snapshot-sync', device='drive0',
+                                                       snapshot_file=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+        self.assert_qmp(result,
+                        'return[0]/inserted/image/backing-image/filename',
+                        old_img)
+
+        result = self.vm.qmp('block-commit', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait(name='BLOCK_JOB_READY')
+
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+
+        result = self.vm.qmp('block-job-complete', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait(name='BLOCK_JOB_COMPLETED')
+
+
+if __name__ == '__main__':
+    if iotests.qemu_default_machine != 'pc':
+        # We need floppy and IDE CD-ROM
+        iotests.notrun('not suitable for this machine type: %s' %
+                       iotests.qemu_default_machine)
+    iotests.main()
diff --git a/tests/qemu-iotests/118.out b/tests/qemu-iotests/118.out
new file mode 100644
index 000000000..6a917130b
--- /dev/null
+++ b/tests/qemu-iotests/118.out
@@ -0,0 +1,5 @@
+...........................................................
+----------------------------------------------------------------------
+Ran 59 tests
+
+OK
diff --git a/tests/qemu-iotests/119 b/tests/qemu-iotests/119
index 9a11f1b92..cc6ec0770 100755
--- a/tests/qemu-iotests/119
+++ b/tests/qemu-iotests/119
@@ -49,7 +49,7 @@ echo "{'execute': 'qmp_capabilities'}
       {'execute': 'human-monitor-command',
        'arguments': {'command-line': 'qemu-io drv \"read -P 0 0 64k\"'}}
       {'execute': 'quit'}" \
-    | $QEMU -drive id=drv,if=none,file="$TEST_IMG",driver=nbd \
+    | $QEMU -nographic -drive id=drv,if=none,file="$TEST_IMG",driver=nbd \
             -qmp stdio -nodefaults \
     | _filter_qmp | _filter_qemu_io
 
diff --git a/tests/qemu-iotests/120 b/tests/qemu-iotests/120
index 9f1307876..d899a3f52 100755
--- a/tests/qemu-iotests/120
+++ b/tests/qemu-iotests/120
@@ -49,7 +49,7 @@ echo "{'execute': 'qmp_capabilities'}
       {'execute': 'human-monitor-command',
        'arguments': {'command-line': 'qemu-io drv \"write -P 42 0 64k\"'}}
       {'execute': 'quit'}" \
-    | $QEMU -qmp stdio -nodefaults \
+    | $QEMU -qmp stdio -nographic -nodefaults \
             -drive id=drv,if=none,file="$TEST_IMG",driver=raw,file.driver=$IMGFMT \
     | _filter_qmp | _filter_qemu_io
 $QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
diff --git a/tests/qemu-iotests/121.out b/tests/qemu-iotests/121.out
index ff18e2c61..5961a44cd 100644
--- a/tests/qemu-iotests/121.out
+++ b/tests/qemu-iotests/121.out
@@ -4,7 +4,7 @@ QA output created by 121
 
 --- Test 1 ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66060288 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66060288 preallocation=metadata
 Image resized.
 wrote 1049600/1049600 bytes at offset 65011712
 1.001 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -14,7 +14,7 @@ No errors were found on the image.
 
 --- Test 2 ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66061312 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66061312 preallocation=metadata
 Image resized.
 wrote 133120/133120 bytes at offset 66060288
 130 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/qemu-iotests/122.out b/tests/qemu-iotests/122.out
index 1f853b9e9..0068e9674 100644
--- a/tests/qemu-iotests/122.out
+++ b/tests/qemu-iotests/122.out
@@ -5,7 +5,7 @@ wrote 67108864/67108864 bytes at offset 0
 
 === Check allocation status regression with -B ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Offset          Length          File
@@ -14,7 +14,7 @@ Offset          Length          File
 
 === Check that zero clusters are kept in overlay ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 3145728/3145728 bytes at offset 0
@@ -125,7 +125,7 @@ read 63963136/63963136 bytes at offset 3145728
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 33554432/33554432 bytes at offset 0
 32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/tests/qemu-iotests/124 b/tests/qemu-iotests/124
index 9ccd11809..c928f0101 100644
--- a/tests/qemu-iotests/124
+++ b/tests/qemu-iotests/124
@@ -36,6 +36,23 @@ def try_remove(img):
         pass
 
 
+def transaction_action(action, **kwargs):
+    return {
+        'type': action,
+        'data': dict((k.replace('_', '-'), v) for k, v in kwargs.iteritems())
+    }
+
+
+def transaction_bitmap_clear(node, name, **kwargs):
+    return transaction_action('block-dirty-bitmap-clear',
+                              node=node, name=name, **kwargs)
+
+
+def transaction_drive_backup(device, target, **kwargs):
+    return transaction_action('drive-backup', device=device, target=target,
+                              **kwargs)
+
+
 class Bitmap:
     def __init__(self, name, drive):
         self.name = name
@@ -122,9 +139,12 @@ class TestIncrementalBackup(iotests.QMPTestCase):
     def do_qmp_backup(self, error='Input/output error', **kwargs):
         res = self.vm.qmp('drive-backup', **kwargs)
         self.assert_qmp(res, 'return', {})
+        return self.wait_qmp_backup(kwargs['device'], error)
+
 
+    def wait_qmp_backup(self, device, error='Input/output error'):
         event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
-                                   match={'data': {'device': kwargs['device']}})
+                                   match={'data': {'device': device}})
         self.assertNotEqual(event, None)
 
         try:
@@ -139,6 +159,12 @@ class TestIncrementalBackup(iotests.QMPTestCase):
             return False
 
 
+    def wait_qmp_backup_cancelled(self, device):
+        event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
+                                   match={'data': {'device': device}})
+        self.assertNotEqual(event, None)
+
+
     def create_anchor_backup(self, drive=None):
         if drive is None:
             drive = self.drives[-1]
@@ -264,6 +290,43 @@ class TestIncrementalBackup(iotests.QMPTestCase):
         return self.do_incremental_simple(granularity=131072)
 
 
+    def test_incremental_transaction(self):
+        '''Test: Verify backups made from transactionally created bitmaps.
+
+        Create a bitmap "before" VM execution begins, then create a second
+        bitmap AFTER writes have already occurred. Use transactions to create
+        a full backup and synchronize both bitmaps to this backup.
+        Create an incremental backup through both bitmaps and verify that
+        both backups match the current drive0 image.
+        '''
+
+        drive0 = self.drives[0]
+        bitmap0 = self.add_bitmap('bitmap0', drive0)
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+        bitmap1 = self.add_bitmap('bitmap1', drive0)
+
+        result = self.vm.qmp('transaction', actions=[
+            transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
+            transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
+            transaction_drive_backup(drive0['id'], drive0['backup'],
+                                     sync='full', format=drive0['fmt'])
+        ])
+        self.assert_qmp(result, 'return', {})
+        self.wait_until_completed(drive0['id'])
+        self.files.append(drive0['backup'])
+
+        self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
+                                          ('0x55', '8M', '352k'),
+                                          ('0x78', '15872k', '1M')))
+        # Both bitmaps should be correctly in sync.
+        self.create_incremental(bitmap0)
+        self.create_incremental(bitmap1)
+        self.vm.shutdown()
+        self.check_backups()
+
+
     def test_incremental_failure(self):
         '''Test: Verify backups made after a failure are correct.
 
@@ -321,6 +384,123 @@ class TestIncrementalBackup(iotests.QMPTestCase):
         self.check_backups()
 
 
+    def test_transaction_failure(self):
+        '''Test: Verify backups made from a transaction that partially fails.
+
+        Add a second drive with its own unique pattern, and add a bitmap to each
+        drive. Use blkdebug to interfere with the backup on just one drive and
+        attempt to create a coherent incremental backup across both drives.
+
+        verify a failure in one but not both, then delete the failed stubs and
+        re-run the same transaction.
+
+        verify that both incrementals are created successfully.
+        '''
+
+        # Create a second drive, with pattern:
+        drive1 = self.add_node('drive1')
+        self.img_create(drive1['file'], drive1['fmt'])
+        io_write_patterns(drive1['file'], (('0x14', 0, 512),
+                                           ('0x5d', '1M', '32k'),
+                                           ('0xcd', '32M', '124k')))
+
+        # Create a blkdebug interface to this img as 'drive1'
+        result = self.vm.qmp('blockdev-add', options={
+            'id': drive1['id'],
+            'driver': drive1['fmt'],
+            'file': {
+                'driver': 'blkdebug',
+                'image': {
+                    'driver': 'file',
+                    'filename': drive1['file']
+                },
+                'set-state': [{
+                    'event': 'flush_to_disk',
+                    'state': 1,
+                    'new_state': 2
+                }],
+                'inject-error': [{
+                    'event': 'read_aio',
+                    'errno': 5,
+                    'state': 2,
+                    'immediately': False,
+                    'once': True
+                }],
+            }
+        })
+        self.assert_qmp(result, 'return', {})
+
+        # Create bitmaps and full backups for both drives
+        drive0 = self.drives[0]
+        dr0bm0 = self.add_bitmap('bitmap0', drive0)
+        dr1bm0 = self.add_bitmap('bitmap0', drive1)
+        self.create_anchor_backup(drive0)
+        self.create_anchor_backup(drive1)
+        self.assert_no_active_block_jobs()
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+
+        # Emulate some writes
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+        self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
+                                          ('0xef', '16M', '256k'),
+                                          ('0x46', '32736k', '64k')))
+
+        # Create incremental backup targets
+        target0 = self.prepare_backup(dr0bm0)
+        target1 = self.prepare_backup(dr1bm0)
+
+        # Ask for a new incremental backup per-each drive,
+        # expecting drive1's backup to fail:
+        transaction = [
+            transaction_drive_backup(drive0['id'], target0, sync='incremental',
+                                     format=drive0['fmt'], mode='existing',
+                                     bitmap=dr0bm0.name),
+            transaction_drive_backup(drive1['id'], target1, sync='incremental',
+                                     format=drive1['fmt'], mode='existing',
+                                     bitmap=dr1bm0.name)
+        ]
+        result = self.vm.qmp('transaction', actions=transaction,
+                             properties={'completion-mode': 'grouped'} )
+        self.assert_qmp(result, 'return', {})
+
+        # Observe that drive0's backup is cancelled and drive1 completes with
+        # an error.
+        self.wait_qmp_backup_cancelled(drive0['id'])
+        self.assertFalse(self.wait_qmp_backup(drive1['id']))
+        error = self.vm.event_wait('BLOCK_JOB_ERROR')
+        self.assert_qmp(error, 'data', {'device': drive1['id'],
+                                        'action': 'report',
+                                        'operation': 'read'})
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+        self.assert_no_active_block_jobs()
+
+        # Delete drive0's successful target and eliminate our record of the
+        # unsuccessful drive1 target. Then re-run the same transaction.
+        dr0bm0.del_target()
+        dr1bm0.del_target()
+        target0 = self.prepare_backup(dr0bm0)
+        target1 = self.prepare_backup(dr1bm0)
+
+        # Re-run the exact same transaction.
+        result = self.vm.qmp('transaction', actions=transaction,
+                             properties={'completion-mode':'grouped'})
+        self.assert_qmp(result, 'return', {})
+
+        # Both should complete successfully this time.
+        self.assertTrue(self.wait_qmp_backup(drive0['id']))
+        self.assertTrue(self.wait_qmp_backup(drive1['id']))
+        self.make_reference_backup(dr0bm0)
+        self.make_reference_backup(dr1bm0)
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+        self.assert_no_active_block_jobs()
+
+        # And the images should of course validate.
+        self.vm.shutdown()
+        self.check_backups()
+
+
     def test_sync_dirty_bitmap_missing(self):
         self.assert_no_active_block_jobs()
         self.files.append(self.err_img)
diff --git a/tests/qemu-iotests/124.out b/tests/qemu-iotests/124.out
index 2f7d3902f..dae404e27 100644
--- a/tests/qemu-iotests/124.out
+++ b/tests/qemu-iotests/124.out
@@ -1,5 +1,5 @@
-.......
+.........
 ----------------------------------------------------------------------
-Ran 7 tests
+Ran 9 tests
 
 OK
diff --git a/tests/qemu-iotests/128 b/tests/qemu-iotests/128
index e2a0f2f89..3d8107d2a 100755
--- a/tests/qemu-iotests/128
+++ b/tests/qemu-iotests/128
@@ -31,6 +31,11 @@ status=1	# failure is the default!
 devname="eiodev$$"
 sudo=""
 
+_sudo_qemu_io_wrapper()
+{
+    (exec $sudo "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@")
+}
+
 _setup_eiodev()
 {
 	# This test should either be run as root or with passwordless sudo
@@ -76,7 +81,9 @@ TEST_IMG="/dev/mapper/$devname"
 echo
 echo "== reading from error device =="
 # Opening image should succeed but the read operation should fail
-$sudo $QEMU_IO --format "$IMGFMT" --nocache -c "read 0 65536" "$TEST_IMG" | _filter_qemu_io
+_sudo_qemu_io_wrapper --format "$IMGFMT" --nocache \
+                      -c "read 0 65536" "$TEST_IMG" \
+    | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/130 b/tests/qemu-iotests/130
index bc26247e3..9209992da 100755
--- a/tests/qemu-iotests/130
+++ b/tests/qemu-iotests/130
@@ -59,8 +59,8 @@ echo
 # bdrv_make_empty() involves a header update for qcow2
 
 # Test that a backing file isn't written
-_launch_qemu -drive file="$TEST_IMG",backing.file.filename="$TEST_IMG.base"
-_send_qemu_cmd $QEMU_HANDLE "commit ide0-hd0" "(qemu)"
+_launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base"
+_send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
 _send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
 _cleanup_qemu
 _img_info | _filter_img_info
@@ -68,8 +68,8 @@ _img_info | _filter_img_info
 # Make sure that if there was a backing file that was just overridden on the
 # command line, that backing file is retained, with the right format
 _make_test_img -F raw -b "$TEST_IMG.orig" 64M
-_launch_qemu -drive file="$TEST_IMG",backing.file.filename="$TEST_IMG.base",backing.driver=$IMGFMT
-_send_qemu_cmd $QEMU_HANDLE "commit ide0-hd0" "(qemu)"
+_launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base",backing.driver=$IMGFMT
+_send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
 _send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
 _cleanup_qemu
 _img_info | _filter_img_info
diff --git a/tests/qemu-iotests/130.out b/tests/qemu-iotests/130.out
index ea68b5d28..ae95b5027 100644
--- a/tests/qemu-iotests/130.out
+++ b/tests/qemu-iotests/130.out
@@ -9,14 +9,14 @@ virtual size: 64M (67108864 bytes)
 === HMP commit ===
 
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) ccocomcommcommicommitcommit commit icommit idcommit idecommit ide0commit ide0-commit ide0-hcommit ide0-hdcommit ide0-hd0
+(qemu) ccocomcommcommicommitcommit commit tcommit tecommit tescommit testcommit testdcommit testdicommit testdiscommit testdisk
 (qemu) 
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.orig' backing_fmt='raw'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.orig backing_fmt=raw
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) ccocomcommcommicommitcommit commit icommit idcommit idecommit ide0commit ide0-commit ide0-hcommit ide0-hdcommit ide0-hd0
+(qemu) ccocomcommcommicommitcommit commit tcommit tecommit tescommit testcommit testdcommit testdicommit testdiscommit testdisk
 (qemu) 
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
@@ -32,7 +32,7 @@ wrote 4096/4096 bytes at offset 0
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.orig' backing_fmt='raw'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.orig backing_fmt=raw
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 image: TEST_DIR/t.IMGFMT
diff --git a/tests/qemu-iotests/136 b/tests/qemu-iotests/136
new file mode 100644
index 000000000..e8c6937fc
--- /dev/null
+++ b/tests/qemu-iotests/136
@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+#
+# Tests for block device statistics
+#
+# Copyright (C) 2015 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+import os
+
+interval_length = 10
+nsec_per_sec = 1000000000
+op_latency = nsec_per_sec / 1000 # See qtest_latency_ns in accounting.c
+bad_sector = 8192
+bad_offset = bad_sector * 512
+blkdebug_file = os.path.join(iotests.test_dir, 'blkdebug.conf')
+
+class BlockDeviceStatsTestCase(iotests.QMPTestCase):
+    test_img = "null-aio://"
+    total_rd_bytes = 0
+    total_rd_ops = 0
+    total_wr_bytes = 0
+    total_wr_ops = 0
+    total_wr_merged = 0
+    total_flush_ops = 0
+    failed_rd_ops = 0
+    failed_wr_ops = 0
+    invalid_rd_ops = 0
+    invalid_wr_ops = 0
+    wr_highest_offset = 0
+    account_invalid = False
+    account_failed = False
+
+    def blockstats(self, device):
+        result = self.vm.qmp("query-blockstats")
+        for r in result['return']:
+            if r['device'] == device:
+                return r['stats']
+        raise Exception("Device not found for blockstats: %s" % device)
+
+    def create_blkdebug_file(self):
+        file = open(blkdebug_file, 'w')
+        file.write('''
+[inject-error]
+event = "read_aio"
+errno = "5"
+sector = "%d"
+
+[inject-error]
+event = "write_aio"
+errno = "5"
+sector = "%d"
+''' % (bad_sector, bad_sector))
+        file.close()
+
+    def setUp(self):
+        drive_args = []
+        drive_args.append("stats-intervals.0=%d" % interval_length)
+        drive_args.append("stats-account-invalid=%s" %
+                          (self.account_invalid and "on" or "off"))
+        drive_args.append("stats-account-failed=%s" %
+                          (self.account_failed and "on" or "off"))
+        self.create_blkdebug_file()
+        self.vm = iotests.VM().add_drive('blkdebug:%s:%s ' %
+                                         (blkdebug_file, self.test_img),
+                                         ','.join(drive_args))
+        self.vm.launch()
+        # Set an initial value for the clock
+        self.vm.qtest("clock_step %d" % nsec_per_sec)
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(blkdebug_file)
+
+    def accounted_ops(self, read = False, write = False, flush = False):
+        ops = 0
+        if write:
+            ops += self.total_wr_ops
+            if self.account_failed:
+                ops += self.failed_wr_ops
+            if self.account_invalid:
+                ops += self.invalid_wr_ops
+        if read:
+            ops += self.total_rd_ops
+            if self.account_failed:
+                ops += self.failed_rd_ops
+            if self.account_invalid:
+                ops += self.invalid_rd_ops
+        if flush:
+            ops += self.total_flush_ops
+        return ops
+
+    def accounted_latency(self, read = False, write = False, flush = False):
+        latency = 0
+        if write:
+            latency += self.total_wr_ops * op_latency
+            if self.account_failed:
+                latency += self.failed_wr_ops * op_latency
+        if read:
+            latency += self.total_rd_ops * op_latency
+            if self.account_failed:
+                latency += self.failed_rd_ops * op_latency
+        if flush:
+            latency += self.total_flush_ops * op_latency
+        return latency
+
+    def check_values(self):
+        stats = self.blockstats('drive0')
+
+        # Check that the totals match with what we have calculated
+        self.assertEqual(self.total_rd_bytes, stats['rd_bytes'])
+        self.assertEqual(self.total_wr_bytes, stats['wr_bytes'])
+        self.assertEqual(self.total_rd_ops, stats['rd_operations'])
+        self.assertEqual(self.total_wr_ops, stats['wr_operations'])
+        self.assertEqual(self.total_flush_ops, stats['flush_operations'])
+        self.assertEqual(self.wr_highest_offset, stats['wr_highest_offset'])
+        self.assertEqual(self.failed_rd_ops, stats['failed_rd_operations'])
+        self.assertEqual(self.failed_wr_ops, stats['failed_wr_operations'])
+        self.assertEqual(self.invalid_rd_ops, stats['invalid_rd_operations'])
+        self.assertEqual(self.invalid_wr_ops, stats['invalid_wr_operations'])
+        self.assertEqual(self.account_invalid, stats['account_invalid'])
+        self.assertEqual(self.account_failed, stats['account_failed'])
+        self.assertEqual(self.total_wr_merged, stats['wr_merged'])
+
+        # Check that there's exactly one interval with the length we defined
+        self.assertEqual(1, len(stats['timed_stats']))
+        timed_stats = stats['timed_stats'][0]
+        self.assertEqual(interval_length, timed_stats['interval_length'])
+
+        total_rd_latency = self.accounted_latency(read = True)
+        if (total_rd_latency != 0):
+            self.assertEqual(total_rd_latency, stats['rd_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_rd_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_rd_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_rd_latency_ns'])
+            self.assertLess(0, timed_stats['avg_rd_queue_depth'])
+        else:
+            self.assertEqual(0, stats['rd_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['max_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_rd_queue_depth'])
+
+        # min read latency <= avg read latency <= max read latency
+        self.assertLessEqual(timed_stats['min_rd_latency_ns'],
+                             timed_stats['avg_rd_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_rd_latency_ns'],
+                             timed_stats['max_rd_latency_ns'])
+
+        total_wr_latency = self.accounted_latency(write = True)
+        if (total_wr_latency != 0):
+            self.assertEqual(total_wr_latency, stats['wr_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_wr_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_wr_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_wr_latency_ns'])
+            self.assertLess(0, timed_stats['avg_wr_queue_depth'])
+        else:
+            self.assertEqual(0, stats['wr_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['max_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_wr_queue_depth'])
+
+        # min write latency <= avg write latency <= max write latency
+        self.assertLessEqual(timed_stats['min_wr_latency_ns'],
+                             timed_stats['avg_wr_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_wr_latency_ns'],
+                             timed_stats['max_wr_latency_ns'])
+
+        total_flush_latency = self.accounted_latency(flush = True)
+        if (total_flush_latency != 0):
+            self.assertEqual(total_flush_latency, stats['flush_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_flush_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_flush_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_flush_latency_ns'])
+        else:
+            self.assertEqual(0, stats['flush_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_flush_latency_ns'])
+            self.assertEqual(0, timed_stats['max_flush_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_flush_latency_ns'])
+
+        # min flush latency <= avg flush latency <= max flush latency
+        self.assertLessEqual(timed_stats['min_flush_latency_ns'],
+                             timed_stats['avg_flush_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_flush_latency_ns'],
+                             timed_stats['max_flush_latency_ns'])
+
+        # idle_time_ns must be > 0 if we have performed any operation
+        if (self.accounted_ops(read = True, write = True, flush = True) != 0):
+            self.assertLess(0, stats['idle_time_ns'])
+        else:
+            self.assertFalse(stats.has_key('idle_time_ns'))
+
+        # This test does not alter these, so they must be all 0
+        self.assertEqual(0, stats['rd_merged'])
+        self.assertEqual(0, stats['failed_flush_operations'])
+        self.assertEqual(0, stats['invalid_flush_operations'])
+
+    def do_test_stats(self, rd_size = 0, rd_ops = 0, wr_size = 0, wr_ops = 0,
+                      flush_ops = 0, invalid_rd_ops = 0, invalid_wr_ops = 0,
+                      failed_rd_ops = 0, failed_wr_ops = 0, wr_merged = 0):
+        # The 'ops' list will contain all the requested I/O operations
+        ops = []
+        for i in range(rd_ops):
+            ops.append("aio_read %d %d" % (i * rd_size, rd_size))
+
+        for i in range(wr_ops):
+            ops.append("aio_write %d %d" % (i * wr_size, wr_size))
+
+        for i in range(flush_ops):
+            ops.append("aio_flush")
+
+        highest_offset = wr_ops * wr_size
+
+        # Two types of invalid operations: unaligned length and unaligned offset
+        for i in range(invalid_rd_ops / 2):
+            ops.append("aio_read 0 511")
+
+        for i in range(invalid_rd_ops / 2, invalid_rd_ops):
+            ops.append("aio_read 13 512")
+
+        for i in range(invalid_wr_ops / 2):
+            ops.append("aio_write 0 511")
+
+        for i in range(invalid_wr_ops / 2, invalid_wr_ops):
+            ops.append("aio_write 13 512")
+
+        for i in range(failed_rd_ops):
+            ops.append("aio_read %d 512" % bad_offset)
+
+        for i in range(failed_wr_ops):
+            ops.append("aio_write %d 512" % bad_offset)
+
+        if failed_wr_ops > 0:
+            highest_offset = max(highest_offset, bad_offset + 512)
+
+        for i in range(wr_merged):
+            first = i * wr_size * 2
+            second = first + wr_size
+            ops.append("multiwrite %d %d ; %d %d" %
+                       (first, wr_size, second, wr_size))
+
+        highest_offset = max(highest_offset, wr_merged * wr_size * 2)
+
+        # Now perform all operations
+        for op in ops:
+            self.vm.hmp_qemu_io("drive0", op)
+
+        # Update the expected totals
+        self.total_rd_bytes += rd_ops * rd_size
+        self.total_rd_ops += rd_ops
+        self.total_wr_bytes += wr_ops * wr_size
+        self.total_wr_ops += wr_ops
+        self.total_wr_merged += wr_merged
+        self.total_flush_ops += flush_ops
+        self.invalid_rd_ops += invalid_rd_ops
+        self.invalid_wr_ops += invalid_wr_ops
+        self.failed_rd_ops += failed_rd_ops
+        self.failed_wr_ops += failed_wr_ops
+
+        self.wr_highest_offset = max(self.wr_highest_offset, highest_offset)
+
+        # Advance the clock so idle_time_ns has a meaningful value
+        self.vm.qtest("clock_step %d" % nsec_per_sec)
+
+        # And check that the actual statistics match the expected ones
+        self.check_values()
+
+    def test_read_only(self):
+        test_values = [[512,    1],
+                       [65536,  1],
+                       [512,   12],
+                       [65536, 12]]
+        for i in test_values:
+            self.do_test_stats(rd_size = i[0], rd_ops = i[1])
+
+    def test_write_only(self):
+        test_values = [[512,    1],
+                       [65536,  1],
+                       [512,   12],
+                       [65536, 12]]
+        for i in test_values:
+            self.do_test_stats(wr_size = i[0], wr_ops = i[1])
+
+    def test_invalid(self):
+        self.do_test_stats(invalid_rd_ops = 7)
+        self.do_test_stats(invalid_wr_ops = 3)
+        self.do_test_stats(invalid_rd_ops = 4, invalid_wr_ops = 5)
+
+    def test_failed(self):
+        self.do_test_stats(failed_rd_ops = 8)
+        self.do_test_stats(failed_wr_ops = 6)
+        self.do_test_stats(failed_rd_ops = 5, failed_wr_ops = 12)
+
+    def test_flush(self):
+        self.do_test_stats(flush_ops = 8)
+
+    def test_merged(self):
+        for i in range(5):
+            self.do_test_stats(wr_merged = i * 3)
+
+    def test_all(self):
+        # rd_size, rd_ops, wr_size, wr_ops, flush_ops
+        # invalid_rd_ops,  invalid_wr_ops,
+        # failed_rd_ops,   failed_wr_ops
+        # wr_merged
+        test_values = [[512,    1, 512,   1, 1, 4, 7, 5, 2, 1],
+                       [65536,  1, 2048, 12, 7, 7, 5, 2, 5, 5],
+                       [32768,  9, 8192,  1, 4, 3, 2, 4, 6, 4],
+                       [16384, 11, 3584, 16, 9, 8, 6, 7, 3, 4]]
+        for i in test_values:
+            self.do_test_stats(*i)
+
+    def test_no_op(self):
+        # All values must be sane before doing any I/O
+        self.check_values()
+
+
+class BlockDeviceStatsTestAccountInvalid(BlockDeviceStatsTestCase):
+    account_invalid = True
+    account_failed = False
+
+class BlockDeviceStatsTestAccountFailed(BlockDeviceStatsTestCase):
+    account_invalid = False
+    account_failed = True
+
+class BlockDeviceStatsTestAccountBoth(BlockDeviceStatsTestCase):
+    account_invalid = True
+    account_failed = True
+
+class BlockDeviceStatsTestCoroutine(BlockDeviceStatsTestCase):
+    test_img = "null-co://"
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["raw"])
diff --git a/tests/qemu-iotests/136.out b/tests/qemu-iotests/136.out
new file mode 100644
index 000000000..0a5e9583a
--- /dev/null
+++ b/tests/qemu-iotests/136.out
@@ -0,0 +1,5 @@
+........................................
+----------------------------------------------------------------------
+Ran 40 tests
+
+OK
diff --git a/tests/qemu-iotests/137 b/tests/qemu-iotests/137
new file mode 100755
index 000000000..9a6597cf9
--- /dev/null
+++ b/tests/qemu-iotests/137
@@ -0,0 +1,145 @@
+#!/bin/bash
+#
+# Test qcow2 reopen
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+
+_make_test_img 64M
+
+echo === Try setting valid values for all options ===
+echo
+
+# Try all options and then check that all of the basic I/O operations still
+# work on this image.
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,pass-discard-request=on" \
+    -c "reopen -o lazy-refcounts=off,pass-discard-request=off" \
+    -c "reopen -o pass-discard-snapshot=on,pass-discard-other=on" \
+    -c "reopen -o pass-discard-snapshot=off,pass-discard-other=off" \
+    -c "reopen -o overlap-check=all" \
+    -c "reopen -o overlap-check=none" \
+    -c "reopen -o overlap-check=cached" \
+    -c "reopen -o overlap-check=constant" \
+    -c "reopen -o overlap-check.template=all" \
+    -c "reopen -o overlap-check.template=none" \
+    -c "reopen -o overlap-check.template=cached" \
+    -c "reopen -o overlap-check.template=constant" \
+    -c "reopen -o overlap-check.main-header=on" \
+    -c "reopen -o overlap-check.main-header=off" \
+    -c "reopen -o overlap-check.active-l1=on" \
+    -c "reopen -o overlap-check.active-l1=off" \
+    -c "reopen -o overlap-check.active-l2=on" \
+    -c "reopen -o overlap-check.active-l2=off" \
+    -c "reopen -o overlap-check.refcount-table=on" \
+    -c "reopen -o overlap-check.refcount-table=off" \
+    -c "reopen -o overlap-check.refcount-block=on" \
+    -c "reopen -o overlap-check.refcount-block=off" \
+    -c "reopen -o overlap-check.snapshot-table=on" \
+    -c "reopen -o overlap-check.snapshot-table=off" \
+    -c "reopen -o overlap-check.inactive-l1=on" \
+    -c "reopen -o overlap-check.inactive-l1=off" \
+    -c "reopen -o overlap-check.inactive-l2=on" \
+    -c "reopen -o overlap-check.inactive-l2=off" \
+    -c "reopen -o cache-size=1M" \
+    -c "reopen -o l2-cache-size=512k" \
+    -c "reopen -o refcount-cache-size=128k" \
+    -c "reopen -o cache-clean-interval=5" \
+    -c "reopen -o cache-clean-interval=0" \
+    -c "reopen -o cache-clean-interval=10" \
+    \
+    -c "write -P 55 0 32M" \
+    -c "read -P 55 0 32M" \
+    -c "discard 0 32M" \
+    -c "write -z 0 32M" \
+    -c "read -P 0 0 32M" \
+    \
+    "$TEST_IMG" | _filter_qemu_io
+
+
+echo
+echo === Try setting some invalid values ===
+echo
+
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=42" \
+    -c "reopen -o cache-size=1M,l2-cache-size=64k,refcount-cache-size=64k" \
+    -c "reopen -o cache-size=1M,l2-cache-size=2M" \
+    -c "reopen -o cache-size=1M,refcount-cache-size=2M" \
+    -c "reopen -o l2-cache-size=256T" \
+    -c "reopen -o refcount-cache-size=256T" \
+    -c "reopen -o overlap-check=constant,overlap-check.template=all" \
+    -c "reopen -o overlap-check=blubb" \
+    -c "reopen -o overlap-check.template=blubb" \
+    -c "reopen -o cache-clean-interval=-1" \
+    "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo === Test transaction semantics ===
+echo
+
+# Whether lazy-refcounts was actually enabled can easily be tested: Check if
+# the dirty bit is set after a crash
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,overlap-check=blubb" \
+    -c "write -P 0x5a 0 512" \
+    -c "sigraise $(kill -l KILL)" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+# Similarly we can test whether corruption detection has been enabled:
+# Create L1/L2, overwrite first entry in refcount block, allocate something.
+# Disabling the checks should fail, so the corruption must be detected.
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$((0x20000))" "\x00\x00"
+$QEMU_IO \
+    -c "reopen -o overlap-check=none,lazy-refcounts=42" \
+    -c "write 64k 64k" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/137.out b/tests/qemu-iotests/137.out
new file mode 100644
index 000000000..88c702cf7
--- /dev/null
+++ b/tests/qemu-iotests/137.out
@@ -0,0 +1,46 @@
+QA output created by 137
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+=== Try setting valid values for all options ===
+
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+discard 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Try setting some invalid values ===
+
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+l2-cache-size may not exceed cache-size
+refcount-cache-size may not exceed cache-size
+L2 cache size too big
+L2 cache size too big
+Conflicting values for qcow2 options 'overlap-check' ('constant') and 'overlap-check.template' ('all')
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Cache clean interval too big
+
+=== Test transaction semantics ===
+
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
+incompatible_features     0x0
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with qcow2_header); further corruption events will be suppressed
+write failed: Input/output error
+*** done
diff --git a/tests/qemu-iotests/138 b/tests/qemu-iotests/138
new file mode 100755
index 000000000..a5c3464d5
--- /dev/null
+++ b/tests/qemu-iotests/138
@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# General test case for qcow2's image check
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+echo
+echo '=== Check on an image with a multiple of 2^32 clusters ==='
+echo
+
+IMGOPTS=$(_optstr_add "$IMGOPTS" "cluster_size=512") \
+    _make_test_img 512
+
+# Allocate L2 table
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Put the data cluster at a multiple of 2 TB, resulting in the image apparently
+# having a multiple of 2^32 clusters
+# (To be more specific: It is at 32 PB)
+poke_file "$TEST_IMG" 2048 "\x80\x80\x00\x00\x00\x00\x00\x00"
+
+# An offset of 32 PB results in qemu-img check having to allocate an in-memory
+# refcount table of 128 TB (16 bit refcounts, 512 byte clusters).
+# This should be generally too much for any system and thus fail.
+# What this test is checking is that the qcow2 driver actually tries to allocate
+# such a large amount of memory (and is consequently aborting) instead of having
+# truncated the cluster count somewhere (which would result in much less memory
+# being allocated and then a segfault occurring).
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/138.out b/tests/qemu-iotests/138.out
new file mode 100644
index 000000000..3fe911f85
--- /dev/null
+++ b/tests/qemu-iotests/138.out
@@ -0,0 +1,9 @@
+QA output created by 138
+
+=== Check on an image with a multiple of 2^32 clusters ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=512
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Check failed: Cannot allocate memory
+*** done
diff --git a/tests/qemu-iotests/139 b/tests/qemu-iotests/139
new file mode 100644
index 000000000..a4b969499
--- /dev/null
+++ b/tests/qemu-iotests/139
@@ -0,0 +1,416 @@
+#!/usr/bin/env python
+#
+# Test cases for the QMP 'x-blockdev-del' command
+#
+# Copyright (C) 2015 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+import time
+
+base_img = os.path.join(iotests.test_dir, 'base.img')
+new_img = os.path.join(iotests.test_dir, 'new.img')
+
+class TestBlockdevDel(iotests.QMPTestCase):
+
+    def setUp(self):
+        iotests.qemu_img('create', '-f', iotests.imgfmt, base_img, '1M')
+        self.vm = iotests.VM()
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(base_img)
+        if os.path.isfile(new_img):
+            os.remove(new_img)
+
+    # Check whether a BlockBackend exists
+    def checkBlockBackend(self, backend, node, must_exist = True):
+        result = self.vm.qmp('query-block')
+        backends = filter(lambda x: x['device'] == backend, result['return'])
+        self.assertLessEqual(len(backends), 1)
+        self.assertEqual(must_exist, len(backends) == 1)
+        if must_exist:
+            if node:
+                self.assertEqual(backends[0]['inserted']['node-name'], node)
+            else:
+                self.assertFalse(backends[0].has_key('inserted'))
+
+    # Check whether a BlockDriverState exists
+    def checkBlockDriverState(self, node, must_exist = True):
+        result = self.vm.qmp('query-named-block-nodes')
+        nodes = filter(lambda x: x['node-name'] == node, result['return'])
+        self.assertLessEqual(len(nodes), 1)
+        self.assertEqual(must_exist, len(nodes) == 1)
+
+    # Add a new BlockBackend (with its attached BlockDriverState)
+    def addBlockBackend(self, backend, node):
+        file_node = '%s_file' % node
+        self.checkBlockBackend(backend, node, False)
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(file_node, False)
+        opts = {'driver': iotests.imgfmt,
+                'id': backend,
+                'node-name': node,
+                'file': {'driver': 'file',
+                         'node-name': file_node,
+                         'filename': base_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(file_node)
+
+    # Add a BlockDriverState without a BlockBackend
+    def addBlockDriverState(self, node):
+        file_node = '%s_file' % node
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(file_node, False)
+        opts = {'driver': iotests.imgfmt,
+                'node-name': node,
+                'file': {'driver': 'file',
+                         'node-name': file_node,
+                         'filename': base_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(file_node)
+
+    # Add a BlockDriverState that will be used as overlay for the base_img BDS
+    def addBlockDriverStateOverlay(self, node):
+        self.checkBlockDriverState(node, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt,
+                         '-b', base_img, new_img, '1M')
+        opts = {'driver': iotests.imgfmt,
+                'node-name': node,
+                'backing': '',
+                'file': {'driver': 'file',
+                         'filename': new_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+
+    # Delete a BlockBackend
+    def delBlockBackend(self, backend, node, expect_error = False,
+                        destroys_media = True):
+        self.checkBlockBackend(backend, node)
+        if node:
+            self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-del', id = backend)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+            if node:
+                self.checkBlockDriverState(node)
+        else:
+            self.assert_qmp(result, 'return', {})
+            if node:
+                self.checkBlockDriverState(node, not destroys_media)
+        self.checkBlockBackend(backend, node, must_exist = expect_error)
+
+    # Delete a BlockDriverState
+    def delBlockDriverState(self, node, expect_error = False):
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-del', node_name = node)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+        else:
+            self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node, expect_error)
+
+    # Add a device model
+    def addDeviceModel(self, device, backend):
+        result = self.vm.qmp('device_add', id = device,
+                             driver = 'virtio-blk-pci', drive = backend)
+        self.assert_qmp(result, 'return', {})
+
+    # Delete a device model
+    def delDeviceModel(self, device):
+        result = self.vm.qmp('device_del', id = device)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('system_reset')
+        self.assert_qmp(result, 'return', {})
+
+        device_path = '/machine/peripheral/%s/virtio-backend' % device
+        event = self.vm.event_wait(name="DEVICE_DELETED",
+                                   match={'data': {'path': device_path}})
+        self.assertNotEqual(event, None)
+
+        event = self.vm.event_wait(name="DEVICE_DELETED",
+                                   match={'data': {'device': device}})
+        self.assertNotEqual(event, None)
+
+    # Remove a BlockDriverState
+    def ejectDrive(self, backend, node, expect_error = False,
+                   destroys_media = True):
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('eject', device = backend)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+            self.checkBlockDriverState(node)
+            self.checkBlockBackend(backend, node)
+        else:
+            self.assert_qmp(result, 'return', {})
+            self.checkBlockDriverState(node, not destroys_media)
+            self.checkBlockBackend(backend, None)
+
+    # Insert a BlockDriverState
+    def insertDrive(self, backend, node):
+        self.checkBlockBackend(backend, None)
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-insert-medium',
+                             device = backend, node_name = node)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+
+    # Create a snapshot using 'blockdev-snapshot-sync'
+    def createSnapshotSync(self, node, overlay):
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay, False)
+        opts = {'node-name': node,
+                'snapshot-file': new_img,
+                'snapshot-node-name': overlay,
+                'format': iotests.imgfmt}
+        result = self.vm.qmp('blockdev-snapshot-sync', conv_keys=False, **opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+
+    # Create a snapshot using 'blockdev-snapshot'
+    def createSnapshot(self, node, overlay):
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+        result = self.vm.qmp('blockdev-snapshot',
+                             node = node, overlay = overlay)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+
+    # Create a mirror
+    def createMirror(self, backend, node, new_node):
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(new_node, False)
+        opts = {'device': backend,
+                'target': new_img,
+                'node-name': new_node,
+                'sync': 'top',
+                'format': iotests.imgfmt}
+        result = self.vm.qmp('drive-mirror', conv_keys=False, **opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(new_node)
+
+    # Complete an existing block job
+    def completeBlockJob(self, backend, node_before, node_after):
+        self.checkBlockBackend(backend, node_before)
+        result = self.vm.qmp('block-job-complete', device=backend)
+        self.assert_qmp(result, 'return', {})
+        self.wait_until_completed(backend)
+        self.checkBlockBackend(backend, node_after)
+
+    # Add a BlkDebug node
+    # Note that the purpose of this is to test the x-blockdev-del
+    # sanity checks, not to create a usable blkdebug drive
+    def addBlkDebug(self, debug, node):
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(debug, False)
+        image = {'driver': iotests.imgfmt,
+                 'node-name': node,
+                 'file': {'driver': 'file',
+                          'filename': base_img}}
+        opts = {'driver': 'blkdebug',
+                'node-name': debug,
+                'image': image}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(debug)
+
+    # Add a BlkVerify node
+    # Note that the purpose of this is to test the x-blockdev-del
+    # sanity checks, not to create a usable blkverify drive
+    def addBlkVerify(self, blkverify, test, raw):
+        self.checkBlockDriverState(test, False)
+        self.checkBlockDriverState(raw, False)
+        self.checkBlockDriverState(blkverify, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, new_img, '1M')
+        node_0 = {'driver': iotests.imgfmt,
+                  'node-name': test,
+                  'file': {'driver': 'file',
+                           'filename': base_img}}
+        node_1 = {'driver': iotests.imgfmt,
+                  'node-name': raw,
+                  'file': {'driver': 'file',
+                           'filename': new_img}}
+        opts = {'driver': 'blkverify',
+                'node-name': blkverify,
+                'test': node_0,
+                'raw': node_1}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(test)
+        self.checkBlockDriverState(raw)
+        self.checkBlockDriverState(blkverify)
+
+    # Add a Quorum node
+    def addQuorum(self, quorum, child0, child1):
+        self.checkBlockDriverState(child0, False)
+        self.checkBlockDriverState(child1, False)
+        self.checkBlockDriverState(quorum, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, new_img, '1M')
+        child_0 = {'driver': iotests.imgfmt,
+                   'node-name': child0,
+                   'file': {'driver': 'file',
+                            'filename': base_img}}
+        child_1 = {'driver': iotests.imgfmt,
+                   'node-name': child1,
+                   'file': {'driver': 'file',
+                            'filename': new_img}}
+        opts = {'driver': 'quorum',
+                'node-name': quorum,
+                'vote-threshold': 1,
+                'children': [ child_0, child_1 ]}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(child0)
+        self.checkBlockDriverState(child1)
+        self.checkBlockDriverState(quorum)
+
+    ########################
+    # The tests start here #
+    ########################
+
+    def testWrongParameters(self):
+        self.addBlockBackend('drive0', 'node0')
+        result = self.vm.qmp('x-blockdev-del')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        result = self.vm.qmp('x-blockdev-del', id='drive0', node_name='node0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.delBlockBackend('drive0', 'node0')
+
+    def testBlockBackend(self):
+        self.addBlockBackend('drive0', 'node0')
+        # You cannot delete a BDS that is attached to a backend
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockBackend('drive0', 'node0')
+
+    def testBlockDriverState(self):
+        self.addBlockDriverState('node0')
+        # You cannot delete a file BDS directly
+        self.delBlockDriverState('node0_file', expect_error = True)
+        self.delBlockDriverState('node0')
+
+    def testEject(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.ejectDrive('drive0', 'node0')
+        self.delBlockBackend('drive0', None)
+
+    def testDeviceModel(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.addDeviceModel('device0', 'drive0')
+        self.ejectDrive('drive0', 'node0', expect_error = True)
+        self.delBlockBackend('drive0', 'node0', expect_error = True)
+        self.delDeviceModel('device0')
+        self.delBlockBackend('drive0', 'node0')
+
+    def testAttachMedia(self):
+        # This creates a BlockBackend and removes its media
+        self.addBlockBackend('drive0', 'node0')
+        self.ejectDrive('drive0', 'node0')
+        # This creates a new BlockDriverState and inserts it into the backend
+        self.addBlockDriverState('node1')
+        self.insertDrive('drive0', 'node1')
+        # The backend can't be removed: the new BDS has an extra reference
+        self.delBlockBackend('drive0', 'node1', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # The BDS still exists after being ejected, but now it can be removed
+        self.ejectDrive('drive0', 'node1', destroys_media = False)
+        self.delBlockDriverState('node1')
+        self.delBlockBackend('drive0', None)
+
+    def testSnapshotSync(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.createSnapshotSync('node0', 'overlay0')
+        # This fails because node0 is now being used as a backing image
+        self.delBlockDriverState('node0', expect_error = True)
+        # This succeeds because overlay0 only has the backend reference
+        self.delBlockBackend('drive0', 'overlay0')
+        self.checkBlockDriverState('node0', False)
+
+    def testSnapshot(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.addBlockDriverStateOverlay('overlay0')
+        self.createSnapshot('node0', 'overlay0')
+        self.delBlockBackend('drive0', 'overlay0', expect_error = True)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('overlay0', expect_error = True)
+        self.ejectDrive('drive0', 'overlay0', destroys_media = False)
+        self.delBlockBackend('drive0', None)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('overlay0')
+        self.checkBlockDriverState('node0', False)
+
+    def testMirror(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.createMirror('drive0', 'node0', 'mirror0')
+        # The block job prevents removing the device
+        self.delBlockBackend('drive0', 'node0', expect_error = True)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('mirror0', expect_error = True)
+        self.wait_ready('drive0')
+        self.completeBlockJob('drive0', 'node0', 'mirror0')
+        self.assert_no_active_block_jobs()
+        self.checkBlockDriverState('node0', False)
+        # This succeeds because the backend now points to mirror0
+        self.delBlockBackend('drive0', 'mirror0')
+
+    def testBlkDebug(self):
+        self.addBlkDebug('debug0', 'node0')
+        # 'node0' is used by the blkdebug node
+        self.delBlockDriverState('node0', expect_error = True)
+        # But we can remove the blkdebug node directly
+        self.delBlockDriverState('debug0')
+        self.checkBlockDriverState('node0', False)
+
+    def testBlkVerify(self):
+        self.addBlkVerify('verify0', 'node0', 'node1')
+        # We cannot remove the children of a blkverify device
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # But we can remove the blkverify node directly
+        self.delBlockDriverState('verify0')
+        self.checkBlockDriverState('node0', False)
+        self.checkBlockDriverState('node1', False)
+
+    def testQuorum(self):
+        if not 'quorum' in iotests.qemu_img_pipe('--help'):
+            return
+        self.addQuorum('quorum0', 'node0', 'node1')
+        # We cannot remove the children of a Quorum device
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # But we can remove the Quorum node directly
+        self.delBlockDriverState('quorum0')
+        self.checkBlockDriverState('node0', False)
+        self.checkBlockDriverState('node1', False)
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["qcow2"])
diff --git a/tests/qemu-iotests/139.out b/tests/qemu-iotests/139.out
new file mode 100644
index 000000000..281b69efe
--- /dev/null
+++ b/tests/qemu-iotests/139.out
@@ -0,0 +1,5 @@
+............
+----------------------------------------------------------------------
+Ran 12 tests
+
+OK
diff --git a/tests/qemu-iotests/144 b/tests/qemu-iotests/144
new file mode 100755
index 000000000..00de3c33c
--- /dev/null
+++ b/tests/qemu-iotests/144
@@ -0,0 +1,114 @@
+#!/bin/bash
+# Check live snapshot, followed by active commit, and another snapshot.
+#
+# This test is to catch the error case of BZ #1300209:
+# https://bugzilla.redhat.com/show_bug.cgi?id=1300209
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=jcody@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+TMP_SNAP1=${TEST_DIR}/tmp.qcow2
+TMP_SNAP2=${TEST_DIR}/tmp2.qcow2
+
+_cleanup()
+{
+    _cleanup_qemu
+    rm -f "${TEST_IMG}" "${TMP_SNAP1}" "${TMP_SNAP2}"
+}
+
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+size=512M
+
+_make_test_img $size
+
+echo
+echo === Launching QEMU ===
+echo
+
+qemu_comm_method="qmp"
+_launch_qemu -drive file="${TEST_IMG}",if=virtio
+h=$QEMU_HANDLE
+
+
+echo
+echo === Performing Live Snapshot 1 ===
+echo
+
+_send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" "return"
+
+
+# First live snapshot, new overlay as active layer
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
+                                'arguments': {
+                                             'device': 'virtio0',
+                                             'snapshot-file':'${TMP_SNAP1}',
+                                             'format': 'qcow2'
+                                             }
+                    }" "return"
+
+echo
+echo === Performing block-commit on active layer ===
+echo
+
+# Block commit on active layer, push the new overlay into base
+_send_qemu_cmd $h "{ 'execute': 'block-commit',
+                                'arguments': {
+                                                 'device': 'virtio0'
+                                              }
+                    }" "READY"
+
+_send_qemu_cmd $h "{ 'execute': 'block-job-complete',
+                                'arguments': {
+                                                'device': 'virtio0'
+                                              }
+                   }" "COMPLETED"
+
+echo
+echo === Performing Live Snapshot 2 ===
+echo
+
+# New live snapshot, new overlays as active layer
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
+                                'arguments': {
+                                                'device': 'virtio0',
+                                                'snapshot-file':'${TMP_SNAP2}',
+                                                'format': 'qcow2'
+                                              }
+                   }" "return"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/144.out b/tests/qemu-iotests/144.out
new file mode 100644
index 000000000..410d74180
--- /dev/null
+++ b/tests/qemu-iotests/144.out
@@ -0,0 +1,24 @@
+QA output created by 144
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=536870912
+
+=== Launching QEMU ===
+
+
+=== Performing Live Snapshot 1 ===
+
+{"return": {}}
+Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+{"return": {}}
+
+=== Performing block-commit on active layer ===
+
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+
+=== Performing Live Snapshot 2 ===
+
+Formatting 'TEST_DIR/tmp2.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+{"return": {}}
+*** done
diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check
index 1fa63193b..c350f16b6 100755
--- a/tests/qemu-iotests/check
+++ b/tests/qemu-iotests/check
@@ -231,10 +231,10 @@ FULL_HOST_DETAILS=`_full_platform_details`
 #FULL_MOUNT_OPTIONS=`_scratch_mount_options`
 
 cat <<EOF
-QEMU          -- $QEMU
-QEMU_IMG      -- $QEMU_IMG
-QEMU_IO       -- $QEMU_IO
-QEMU_NBD      -- $QEMU_NBD
+QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
+QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
+QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
+QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
 IMGFMT        -- $FULL_IMGFMT_DETAILS
 IMGPROTO      -- $FULL_IMGPROTO_DETAILS
 PLATFORM      -- $FULL_HOST_DETAILS
@@ -330,6 +330,11 @@ do
             fi
 
             reference="$source_iotests/$seq.out"
+            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
+            if [ -f "$reference_machine" ]; then
+                reference="$reference_machine"
+            fi
+
             if [ "$CACHEMODE" = "none" ]; then
                 [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
             fi
diff --git a/tests/qemu-iotests/common b/tests/qemu-iotests/common
index 1030aaf25..ff84f4b0d 100644
--- a/tests/qemu-iotests/common
+++ b/tests/qemu-iotests/common
@@ -41,7 +41,6 @@ sortme=false
 expunge=true
 have_test_arg=false
 randomize=false
-valgrind=false
 cachemode=false
 rm -f $tmp.list $tmp.tmp $tmp.sed
 
@@ -52,6 +51,8 @@ export IMGOPTS=""
 export CACHEMODE="writeback"
 export QEMU_IO_OPTIONS=""
 export CACHEMODE_IS_DEFAULT=true
+export QEMU_OPTIONS="-nodefaults"
+export VALGRIND_QEMU=
 
 for r
 do
@@ -277,7 +278,7 @@ testlist options
             ;;
 
         -valgrind)
-            valgrind=true
+            VALGRIND_QEMU='y'
             xpand=false
             ;;
 
@@ -435,8 +436,3 @@ fi
 if [ "$IMGPROTO" = "nbd" ] ; then
     [ "$QEMU_NBD" = "" ] && _fatal "qemu-nbd not found"
 fi
-
-if $valgrind; then
-    export REAL_QEMU_IO="$QEMU_IO_PROG"
-    export QEMU_IO_PROG=valgrind_qemu_io
-fi
diff --git a/tests/qemu-iotests/common.config b/tests/qemu-iotests/common.config
index a1973ad9d..3ed51b8ba 100644
--- a/tests/qemu-iotests/common.config
+++ b/tests/qemu-iotests/common.config
@@ -44,6 +44,8 @@ export HOST_OPTIONS=${HOST_OPTIONS:=local.config}
 export CHECK_OPTIONS=${CHECK_OPTIONS:="-g auto"}
 export PWD=`pwd`
 
+export _QEMU_HANDLE=0
+
 # $1 = prog to look for, $2* = default pathnames if not found in $PATH
 set_prog_path()
 {
@@ -103,10 +105,64 @@ if [ -z "$QEMU_NBD_PROG" ]; then
     export QEMU_NBD_PROG="`set_prog_path qemu-nbd`"
 fi
 
-export QEMU=$QEMU_PROG
-export QEMU_IMG=$QEMU_IMG_PROG
-export QEMU_IO="$QEMU_IO_PROG $QEMU_IO_OPTIONS"
-export QEMU_NBD=$QEMU_NBD_PROG
+_qemu_wrapper()
+{
+    (
+        if [ -n "${QEMU_NEED_PID}" ]; then
+            echo $BASHPID > "${TEST_DIR}/qemu-${_QEMU_HANDLE}.pid"
+        fi
+        exec "$QEMU_PROG" $QEMU_OPTIONS "$@"
+    )
+}
+
+_qemu_img_wrapper()
+{
+    (exec "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS "$@")
+}
+
+_qemu_io_wrapper()
+{
+    local VALGRIND_LOGFILE=/tmp/$$.valgrind
+    local RETVAL
+    (
+        if [ "${VALGRIND_QEMU}" == "y" ]; then
+            exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@"
+        else
+            exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@"
+        fi
+    )
+    RETVAL=$?
+    if [ "${VALGRIND_QEMU}" == "y" ]; then
+        if [ $RETVAL == 99 ]; then
+            cat "${VALGRIND_LOGFILE}"
+        fi
+        rm -f "${VALGRIND_LOGFILE}"
+    fi
+    (exit $RETVAL)
+}
+
+_qemu_nbd_wrapper()
+{
+    (
+        echo $BASHPID > "${TEST_DIR}/qemu-nbd.pid"
+        exec "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS "$@"
+    )
+}
+
+export QEMU=_qemu_wrapper
+export QEMU_IMG=_qemu_img_wrapper
+export QEMU_IO=_qemu_io_wrapper
+export QEMU_NBD=_qemu_nbd_wrapper
+
+default_machine=$($QEMU -machine \? | awk '/(default)/{print $1}')
+default_alias_machine=$($QEMU -machine \? |\
+    awk -v var_default_machine="$default_machine"\)\
+    '{if ($(NF-2)=="(alias"&&$(NF-1)=="of"&&$(NF)==var_default_machine){print $1}}')
+if [ ! -z "$default_alias_machine" ]; then
+    default_machine="$default_alias_machine"
+fi
+
+export QEMU_DEFAULT_MACHINE="$default_machine"
 
 [ -f /etc/qemu-iotest.config ]       && . /etc/qemu-iotest.config
 
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index 012a8122d..cfdb6338a 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -128,6 +128,11 @@ _filter_date()
         -e 's/[A-Z][a-z][a-z] [A-z][a-z][a-z]  *[0-9][0-9]* [0-9][0-9]:[0-9][0-9]:[0-9][0-9] [0-9][0-9][0-9][0-9]$/DATE/'
 }
 
+_filter_generated_node_ids()
+{
+     sed -re 's/\#block[0-9]{3,}/NODE_NAME/'
+}
+
 # replace occurrences of the actual TEST_DIR value with TEST_DIR
 _filter_testdir()
 {
@@ -182,7 +187,7 @@ _filter_img_create()
         -e "s# encryption=off##g" \
         -e "s# cluster_size=[0-9]\\+##g" \
         -e "s# table_size=[0-9]\\+##g" \
-        -e "s# compat='[^']*'##g" \
+        -e "s# compat=[^ ]*##g" \
         -e "s# compat6=\\(on\\|off\\)##g" \
         -e "s# static=\\(on\\|off\\)##g" \
         -e "s# zeroed_grain=\\(on\\|off\\)##g" \
diff --git a/tests/qemu-iotests/common.qemu b/tests/qemu-iotests/common.qemu
index 4e1996c3e..8bf396941 100644
--- a/tests/qemu-iotests/common.qemu
+++ b/tests/qemu-iotests/common.qemu
@@ -30,8 +30,6 @@ QEMU_COMM_TIMEOUT=10
 QEMU_FIFO_IN="${TEST_DIR}/qmp-in-$$"
 QEMU_FIFO_OUT="${TEST_DIR}/qmp-out-$$"
 
-QEMU_PID=
-_QEMU_HANDLE=0
 QEMU_HANDLE=0
 
 # If bash version is >= 4.1, these will be overwritten and dynamic
@@ -153,11 +151,11 @@ function _launch_qemu()
     mkfifo "${fifo_out}"
     mkfifo "${fifo_in}"
 
-    "${QEMU}" -nographic -serial none ${comm} -machine accel=qtest "${@}" \
+    QEMU_NEED_PID='y'\
+    ${QEMU} -nographic -serial none ${comm} -machine accel=qtest "${@}" \
                                                                 >"${fifo_out}" \
                                                                 2>&1 \
                                                                 <"${fifo_in}" &
-    QEMU_PID[${_QEMU_HANDLE}]=$!
 
     if [[ "${BASH_VERSINFO[0]}" -ge "5" ||
         ("${BASH_VERSINFO[0]}" -ge "4"  &&  "${BASH_VERSINFO[1]}" -ge "1") ]]
@@ -196,10 +194,18 @@ function _cleanup_qemu()
     # QEMU_PID[], QEMU_IN[], QEMU_OUT[] all use same indices
     for i in "${!QEMU_OUT[@]}"
     do
-        if [ -z "${wait}" ]; then
-            kill -KILL ${QEMU_PID[$i]} 2>/dev/null
+        local QEMU_PID
+        if [ -f "${TEST_DIR}/qemu-${i}.pid" ]; then
+            read QEMU_PID < "${TEST_DIR}/qemu-${i}.pid"
+            rm -f "${TEST_DIR}/qemu-${i}.pid"
+            if [ -z "${wait}" ] && [ -n "${QEMU_PID}" ]; then
+                kill -KILL ${QEMU_PID} 2>/dev/null
+            fi
+            if [ -n "${QEMU_PID}" ]; then
+                wait ${QEMU_PID} 2>/dev/null # silent kill
+            fi
         fi
-        wait ${QEMU_PID[$i]} 2>/dev/null # silent kill
+
         if [ -n "${wait}" ]; then
             cat <&${QEMU_OUT[$i]} | _filter_testdir | _filter_qemu \
                                   | _filter_qemu_io | _filter_qmp
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index 22d351404..d9913f849 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -70,16 +70,6 @@ else
     TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
 fi
 
-function valgrind_qemu_io()
-{
-    valgrind --log-file=/tmp/$$.valgrind --error-exitcode=99 $REAL_QEMU_IO "$@"
-    if [ $? != 0 ]; then
-        cat /tmp/$$.valgrind
-    fi
-    rm -f /tmp/$$.valgrind
-}
-
-
 _optstr_add()
 {
     if [ -n "$1" ]; then
@@ -154,7 +144,6 @@ _make_test_img()
     # Start an NBD server on the image file, which is what we'll be talking to
     if [ $IMGPROTO = "nbd" ]; then
         eval "$QEMU_NBD -v -t -b 127.0.0.1 -p 10810 -f $IMGFMT  $TEST_IMG_FILE &"
-        QEMU_NBD_PID=$!
         sleep 1 # FIXME: qemu-nbd needs to be listening before we continue
     fi
 }
@@ -175,8 +164,11 @@ _cleanup_test_img()
     case "$IMGPROTO" in
 
         nbd)
-            if [ -n "$QEMU_NBD_PID" ]; then
-                kill $QEMU_NBD_PID
+            if [ -f "${TEST_DIR}/qemu-nbd.pid" ]; then
+                local QEMU_NBD_PID
+                read QEMU_NBD_PID < "${TEST_DIR}/qemu-nbd.pid"
+                kill ${QEMU_NBD_PID}
+                rm -f "${TEST_DIR}/qemu-nbd.pid"
             fi
             rm -f "$TEST_IMG_FILE"
             ;;
@@ -439,7 +431,17 @@ _unsupported_imgopts()
 #
 _require_command()
 {
-    eval c=\$$1
+    if [ "$1" = "QEMU" ]; then
+        c=$QEMU_PROG
+    elif [ "$1" = "QEMU_IMG" ]; then
+        c=$QEMU_IMG_PROG
+    elif [ "$1" = "QEMU_IO" ]; then
+        c=$QEMU_IO_PROG
+    elif [ "$1" = "QEMU_NBD" ]; then
+        c=$QEMU_NBD_PROG
+    else
+        eval c=\$$1
+    fi
     [ -x "$c" ] || _notrun "$1 utility required, skipped this test"
 }
 
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index c430b6c23..d90629fc3 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -102,6 +102,7 @@
 093 auto
 094 rw auto quick
 095 rw auto quick
+096 rw auto quick
 097 rw auto backing
 098 rw auto backing quick
 099 rw auto quick
@@ -121,6 +122,7 @@
 114 rw auto quick
 115 rw auto
 116 rw auto quick
+118 rw auto
 119 rw auto quick
 120 rw auto quick
 121 rw auto
@@ -134,3 +136,8 @@
 132 rw auto quick
 134 rw auto quick
 135 rw auto
+136 rw auto
+137 rw auto
+138 rw auto quick
+139 rw auto quick
+144 rw auto quick
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 8615b1075..e02245ed0 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -31,37 +31,60 @@ import struct
 __all__ = ['imgfmt', 'imgproto', 'test_dir' 'qemu_img', 'qemu_io',
            'VM', 'QMPTestCase', 'notrun', 'main']
 
-# This will not work if arguments or path contain spaces but is necessary if we
+# This will not work if arguments contain spaces but is necessary if we
 # want to support the override options that ./check supports.
-qemu_img_args = os.environ.get('QEMU_IMG', 'qemu-img').strip().split(' ')
-qemu_io_args = os.environ.get('QEMU_IO', 'qemu-io').strip().split(' ')
-qemu_args = os.environ.get('QEMU', 'qemu').strip().split(' ')
+qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')]
+if os.environ.get('QEMU_IMG_OPTIONS'):
+    qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ')
+
+qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')]
+if os.environ.get('QEMU_IO_OPTIONS'):
+    qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ')
+
+qemu_args = [os.environ.get('QEMU_PROG', 'qemu')]
+if os.environ.get('QEMU_OPTIONS'):
+    qemu_args += os.environ['QEMU_OPTIONS'].strip().split(' ')
 
 imgfmt = os.environ.get('IMGFMT', 'raw')
 imgproto = os.environ.get('IMGPROTO', 'file')
 test_dir = os.environ.get('TEST_DIR', '/var/tmp')
 output_dir = os.environ.get('OUTPUT_DIR', '.')
 cachemode = os.environ.get('CACHEMODE')
+qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE')
 
 socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
 
 def qemu_img(*args):
     '''Run qemu-img and return the exit code'''
     devnull = open('/dev/null', 'r+')
-    return subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
+    exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return exitcode
 
 def qemu_img_verbose(*args):
     '''Run qemu-img without suppressing its output and return the exit code'''
-    return subprocess.call(qemu_img_args + list(args))
+    exitcode = subprocess.call(qemu_img_args + list(args))
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return exitcode
 
 def qemu_img_pipe(*args):
     '''Run qemu-img and return its output'''
-    return subprocess.Popen(qemu_img_args + list(args), stdout=subprocess.PIPE).communicate()[0]
+    subp = subprocess.Popen(qemu_img_args + list(args), stdout=subprocess.PIPE)
+    exitcode = subp.wait()
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return subp.communicate()[0]
 
 def qemu_io(*args):
     '''Run qemu-io and return the stdout data'''
     args = qemu_io_args + list(args)
-    return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
+    subp = subprocess.Popen(args, stdout=subprocess.PIPE)
+    exitcode = subp.wait()
+    if exitcode < 0:
+        sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args)))
+    return subp.communicate()[0]
 
 def compare_images(img1, img2):
     '''Return True if two image files are identical'''
@@ -117,13 +140,21 @@ class VM(object):
         self._args.append('-monitor')
         self._args.append(args)
 
-    def add_drive(self, path, opts=''):
+    def add_drive_raw(self, opts):
+        self._args.append('-drive')
+        self._args.append(opts)
+        return self
+
+    def add_drive(self, path, opts='', interface='virtio'):
         '''Add a virtio-blk drive to the VM'''
-        options = ['if=virtio',
+        options = ['if=%s' % interface,
                    'format=%s' % imgfmt,
                    'cache=%s' % cachemode,
-                   'file=%s' % path,
                    'id=drive%d' % self._num_drives]
+
+        if path is not None:
+            options.append('file=%s' % path)
+
         if opts:
             options.append(opts)
 
@@ -196,7 +227,9 @@ class VM(object):
         '''Terminate the VM and clean up'''
         if not self._popen is None:
             self._qmp.cmd('quit')
-            self._popen.wait()
+            exitcode = self._popen.wait()
+            if exitcode < 0:
+                sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, ' '.join(self._args)))
             os.remove(self._monitor_path)
             os.remove(self._qtest_path)
             os.remove(self._qemu_log_path)
diff --git a/tests/qom-test.c b/tests/qom-test.c
index fde04e7a1..3e5e8730e 100644
--- a/tests/qom-test.c
+++ b/tests/qom-test.c
@@ -47,7 +47,7 @@ static bool is_blacklisted(const char *arch, const char *mach)
 static void test_properties(const char *path, bool recurse)
 {
     char *child_path;
-    QDict *response, *tuple;
+    QDict *response, *tuple, *tmp;
     QList *list;
     QListEntry *entry;
 
@@ -57,6 +57,7 @@ static void test_properties(const char *path, bool recurse)
     g_assert(response);
 
     if (!recurse) {
+        QDECREF(response);
         return;
     }
 
@@ -75,14 +76,16 @@ static void test_properties(const char *path, bool recurse)
         } else {
             const char *prop = qdict_get_str(tuple, "name");
             g_test_message("Testing property %s.%s", path, prop);
-            response = qmp("{ 'execute': 'qom-get',"
-                           "  'arguments': { 'path': %s,"
-                           "                 'property': %s } }",
-                           path, prop);
+            tmp = qmp("{ 'execute': 'qom-get',"
+                      "  'arguments': { 'path': %s,"
+                      "                 'property': %s } }",
+                      path, prop);
             /* qom-get may fail but should not, e.g., segfault. */
-            g_assert(response);
+            g_assert(tmp);
+            QDECREF(tmp);
         }
     }
+    QDECREF(response);
 }
 
 static void test_machine(gconstpointer data)
@@ -98,9 +101,11 @@ static void test_machine(gconstpointer data)
 
     response = qmp("{ 'execute': 'quit' }");
     g_assert(qdict_haskey(response, "return"));
+    QDECREF(response);
 
     qtest_end();
     g_free(args);
+    g_free((void *)machine);
 }
 
 static void add_machine_test_cases(void)
@@ -129,10 +134,12 @@ static void add_machine_test_cases(void)
         mname = qstring_get_str(qstr);
         if (!is_blacklisted(arch, mname)) {
             path = g_strdup_printf("qom/%s", mname);
-            qtest_add_data_func(path, mname, test_machine);
+            qtest_add_data_func(path, g_strdup(mname), test_machine);
         }
     }
+
     qtest_end();
+    QDECREF(response);
 }
 
 int main(int argc, char **argv)
diff --git a/tests/rtl8139-test.c b/tests/rtl8139-test.c
index e749be38e..ba62851ca 100644
--- a/tests/rtl8139-test.c
+++ b/tests/rtl8139-test.c
@@ -20,7 +20,7 @@ static void nop(void)
 {
 }
 
-#define CLK 33000000
+#define CLK 33333333
 
 static QPCIBus *pcibus;
 static QPCIDevice *dev;
diff --git a/tests/tcg/testthread.c b/tests/tcg/testthread.c
index 2679af119..810ba5de6 100644
--- a/tests/tcg/testthread.c
+++ b/tests/tcg/testthread.c
@@ -2,7 +2,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <signal.h>
 #include <unistd.h>
 #include <inttypes.h>
 #include <pthread.h>
diff --git a/tests/test-aio.c b/tests/test-aio.c
index 217e33772..e188d8c13 100644
--- a/tests/test-aio.c
+++ b/tests/test-aio.c
@@ -118,6 +118,12 @@ static void *test_acquire_thread(void *opaque)
     return NULL;
 }
 
+static void set_event_notifier(AioContext *ctx, EventNotifier *notifier,
+                               EventNotifierHandler *handler)
+{
+    aio_set_event_notifier(ctx, notifier, false, handler);
+}
+
 static void dummy_notifier_read(EventNotifier *unused)
 {
     g_assert(false); /* should never be invoked */
@@ -131,7 +137,7 @@ static void test_acquire(void)
 
     /* Dummy event notifier ensures aio_poll() will block */
     event_notifier_init(&notifier, false);
-    aio_set_event_notifier(ctx, &notifier, dummy_notifier_read);
+    set_event_notifier(ctx, &notifier, dummy_notifier_read);
     g_assert(!aio_poll(ctx, false)); /* consume aio_notify() */
 
     qemu_mutex_init(&data.start_lock);
@@ -149,7 +155,7 @@ static void test_acquire(void)
     aio_context_release(ctx);
 
     qemu_thread_join(&thread);
-    aio_set_event_notifier(ctx, &notifier, NULL);
+    set_event_notifier(ctx, &notifier, NULL);
     event_notifier_cleanup(&notifier);
 
     g_assert(data.thread_acquired);
@@ -308,11 +314,11 @@ static void test_set_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 0 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     event_notifier_cleanup(&data.e);
@@ -322,7 +328,7 @@ static void test_wait_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 1 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 1);
@@ -336,7 +342,7 @@ static void test_wait_event_notifier(void)
     g_assert_cmpint(data.n, ==, 1);
     g_assert_cmpint(data.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 1);
 
@@ -347,7 +353,7 @@ static void test_flush_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 10);
@@ -363,18 +369,42 @@ static void test_flush_event_notifier(void)
     g_assert_cmpint(data.active, ==, 0);
     g_assert(!aio_poll(ctx, false));
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     event_notifier_cleanup(&data.e);
 }
 
+static void test_aio_external_client(void)
+{
+    int i, j;
+
+    for (i = 1; i < 3; i++) {
+        EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
+        event_notifier_init(&data.e, false);
+        aio_set_event_notifier(ctx, &data.e, true, event_ready_cb);
+        event_notifier_set(&data.e);
+        for (j = 0; j < i; j++) {
+            aio_disable_external(ctx);
+        }
+        for (j = 0; j < i; j++) {
+            assert(!aio_poll(ctx, false));
+            assert(event_notifier_test_and_clear(&data.e));
+            event_notifier_set(&data.e);
+            aio_enable_external(ctx);
+        }
+        assert(aio_poll(ctx, false));
+        set_event_notifier(ctx, &data.e, NULL);
+        event_notifier_cleanup(&data.e);
+    }
+}
+
 static void test_wait_event_notifier_noflush(void)
 {
     EventNotifierTestData data = { .n = 0 };
     EventNotifierTestData dummy = { .n = 0, .active = 1 };
 
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
 
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
@@ -387,7 +417,7 @@ static void test_wait_event_notifier_noflush(void)
 
     /* An active event notifier forces aio_poll to look at EventNotifiers.  */
     event_notifier_init(&dummy.e, false);
-    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb);
+    set_event_notifier(ctx, &dummy.e, event_ready_cb);
 
     event_notifier_set(&data.e);
     g_assert(aio_poll(ctx, false));
@@ -407,10 +437,10 @@ static void test_wait_event_notifier_noflush(void)
     g_assert_cmpint(dummy.n, ==, 1);
     g_assert_cmpint(dummy.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &dummy.e, NULL);
+    set_event_notifier(ctx, &dummy.e, NULL);
     event_notifier_cleanup(&dummy.e);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
@@ -428,7 +458,7 @@ static void test_timer_schedule(void)
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
     event_notifier_init(&e, false);
-    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
+    set_event_notifier(ctx, &e, dummy_io_handler_read);
     aio_poll(ctx, false);
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -467,7 +497,7 @@ static void test_timer_schedule(void)
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
-    aio_set_event_notifier(ctx, &e, NULL);
+    set_event_notifier(ctx, &e, NULL);
     event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
@@ -638,11 +668,11 @@ static void test_source_set_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 0 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     event_notifier_cleanup(&data.e);
@@ -652,7 +682,7 @@ static void test_source_wait_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 1 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 1);
@@ -666,7 +696,7 @@ static void test_source_wait_event_notifier(void)
     g_assert_cmpint(data.n, ==, 1);
     g_assert_cmpint(data.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 1);
 
@@ -677,7 +707,7 @@ static void test_source_flush_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 10);
@@ -693,7 +723,7 @@ static void test_source_flush_event_notifier(void)
     g_assert_cmpint(data.active, ==, 0);
     g_assert(!g_main_context_iteration(NULL, false));
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     event_notifier_cleanup(&data.e);
 }
@@ -704,7 +734,7 @@ static void test_source_wait_event_notifier_noflush(void)
     EventNotifierTestData dummy = { .n = 0, .active = 1 };
 
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
 
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
@@ -717,7 +747,7 @@ static void test_source_wait_event_notifier_noflush(void)
 
     /* An active event notifier forces aio_poll to look at EventNotifiers.  */
     event_notifier_init(&dummy.e, false);
-    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb);
+    set_event_notifier(ctx, &dummy.e, event_ready_cb);
 
     event_notifier_set(&data.e);
     g_assert(g_main_context_iteration(NULL, false));
@@ -737,10 +767,10 @@ static void test_source_wait_event_notifier_noflush(void)
     g_assert_cmpint(dummy.n, ==, 1);
     g_assert_cmpint(dummy.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &dummy.e, NULL);
+    set_event_notifier(ctx, &dummy.e, NULL);
     event_notifier_cleanup(&dummy.e);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 2);
 
@@ -759,7 +789,7 @@ static void test_source_timer_schedule(void)
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
     event_notifier_init(&e, false);
-    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
+    set_event_notifier(ctx, &e, dummy_io_handler_read);
     do {} while (g_main_context_iteration(NULL, false));
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -784,7 +814,7 @@ static void test_source_timer_schedule(void)
     g_assert_cmpint(data.n, ==, 2);
     g_assert(qemu_clock_get_ns(data.clock_type) > expiry);
 
-    aio_set_event_notifier(ctx, &e, NULL);
+    set_event_notifier(ctx, &e, NULL);
     event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
@@ -826,6 +856,7 @@ int main(int argc, char **argv)
     g_test_add_func("/aio/event/wait",              test_wait_event_notifier);
     g_test_add_func("/aio/event/wait/no-flush-cb",  test_wait_event_notifier_noflush);
     g_test_add_func("/aio/event/flush",             test_flush_event_notifier);
+    g_test_add_func("/aio/external-client",         test_aio_external_client);
     g_test_add_func("/aio/timer/schedule",          test_timer_schedule);
 
     g_test_add_func("/aio-gsource/flush",                   test_source_flush);
diff --git a/tests/test-blockjob-txn.c b/tests/test-blockjob-txn.c
new file mode 100644
index 000000000..34747e924
--- /dev/null
+++ b/tests/test-blockjob-txn.c
@@ -0,0 +1,250 @@
+/*
+ * Blockjob transactions tests
+ *
+ * Copyright Red Hat, Inc. 2015
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include <glib.h>
+#include "qapi/error.h"
+#include "qemu/main-loop.h"
+#include "block/blockjob.h"
+
+typedef struct {
+    BlockJob common;
+    unsigned int iterations;
+    bool use_timer;
+    int rc;
+    int *result;
+} TestBlockJob;
+
+static const BlockJobDriver test_block_job_driver = {
+    .instance_size = sizeof(TestBlockJob),
+};
+
+static void test_block_job_complete(BlockJob *job, void *opaque)
+{
+    BlockDriverState *bs = job->bs;
+    int rc = (intptr_t)opaque;
+
+    if (block_job_is_cancelled(job)) {
+        rc = -ECANCELED;
+    }
+
+    block_job_completed(job, rc);
+    bdrv_unref(bs);
+}
+
+static void coroutine_fn test_block_job_run(void *opaque)
+{
+    TestBlockJob *s = opaque;
+    BlockJob *job = &s->common;
+
+    while (s->iterations--) {
+        if (s->use_timer) {
+            block_job_sleep_ns(job, QEMU_CLOCK_REALTIME, 0);
+        } else {
+            block_job_yield(job);
+        }
+
+        if (block_job_is_cancelled(job)) {
+            break;
+        }
+    }
+
+    block_job_defer_to_main_loop(job, test_block_job_complete,
+                                 (void *)(intptr_t)s->rc);
+}
+
+typedef struct {
+    TestBlockJob *job;
+    int *result;
+} TestBlockJobCBData;
+
+static void test_block_job_cb(void *opaque, int ret)
+{
+    TestBlockJobCBData *data = opaque;
+    if (!ret && block_job_is_cancelled(&data->job->common)) {
+        ret = -ECANCELED;
+    }
+    *data->result = ret;
+    g_free(data);
+}
+
+/* Create a block job that completes with a given return code after a given
+ * number of event loop iterations.  The return code is stored in the given
+ * result pointer.
+ *
+ * The event loop iterations can either be handled automatically with a 0 delay
+ * timer, or they can be stepped manually by entering the coroutine.
+ */
+static BlockJob *test_block_job_start(unsigned int iterations,
+                                      bool use_timer,
+                                      int rc, int *result)
+{
+    BlockDriverState *bs;
+    TestBlockJob *s;
+    TestBlockJobCBData *data;
+
+    data = g_new0(TestBlockJobCBData, 1);
+    bs = bdrv_new();
+    s = block_job_create(&test_block_job_driver, bs, 0, test_block_job_cb,
+                         data, &error_abort);
+    s->iterations = iterations;
+    s->use_timer = use_timer;
+    s->rc = rc;
+    s->result = result;
+    s->common.co = qemu_coroutine_create(test_block_job_run);
+    data->job = s;
+    data->result = result;
+    qemu_coroutine_enter(s->common.co, s);
+    return &s->common;
+}
+
+static void test_single_job(int expected)
+{
+    BlockJob *job;
+    BlockJobTxn *txn;
+    int result = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job = test_block_job_start(1, true, expected, &result);
+    block_job_txn_add_job(txn, job);
+
+    if (expected == -ECANCELED) {
+        block_job_cancel(job);
+    }
+
+    while (result == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+    g_assert_cmpint(result, ==, expected);
+
+    block_job_txn_unref(txn);
+}
+
+static void test_single_job_success(void)
+{
+    test_single_job(0);
+}
+
+static void test_single_job_failure(void)
+{
+    test_single_job(-EIO);
+}
+
+static void test_single_job_cancel(void)
+{
+    test_single_job(-ECANCELED);
+}
+
+static void test_pair_jobs(int expected1, int expected2)
+{
+    BlockJob *job1;
+    BlockJob *job2;
+    BlockJobTxn *txn;
+    int result1 = -EINPROGRESS;
+    int result2 = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job1 = test_block_job_start(1, true, expected1, &result1);
+    block_job_txn_add_job(txn, job1);
+    job2 = test_block_job_start(2, true, expected2, &result2);
+    block_job_txn_add_job(txn, job2);
+
+    if (expected1 == -ECANCELED) {
+        block_job_cancel(job1);
+    }
+    if (expected2 == -ECANCELED) {
+        block_job_cancel(job2);
+    }
+
+    while (result1 == -EINPROGRESS || result2 == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+
+    /* Failure or cancellation of one job cancels the other job */
+    if (expected1 != 0) {
+        expected2 = -ECANCELED;
+    } else if (expected2 != 0) {
+        expected1 = -ECANCELED;
+    }
+
+    g_assert_cmpint(result1, ==, expected1);
+    g_assert_cmpint(result2, ==, expected2);
+
+    block_job_txn_unref(txn);
+}
+
+static void test_pair_jobs_success(void)
+{
+    test_pair_jobs(0, 0);
+}
+
+static void test_pair_jobs_failure(void)
+{
+    /* Test both orderings.  The two jobs run for a different number of
+     * iterations so the code path is different depending on which job fails
+     * first.
+     */
+    test_pair_jobs(-EIO, 0);
+    test_pair_jobs(0, -EIO);
+}
+
+static void test_pair_jobs_cancel(void)
+{
+    test_pair_jobs(-ECANCELED, 0);
+    test_pair_jobs(0, -ECANCELED);
+}
+
+static void test_pair_jobs_fail_cancel_race(void)
+{
+    BlockJob *job1;
+    BlockJob *job2;
+    BlockJobTxn *txn;
+    int result1 = -EINPROGRESS;
+    int result2 = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job1 = test_block_job_start(1, true, -ECANCELED, &result1);
+    block_job_txn_add_job(txn, job1);
+    job2 = test_block_job_start(2, false, 0, &result2);
+    block_job_txn_add_job(txn, job2);
+
+    block_job_cancel(job1);
+
+    /* Now make job2 finish before the main loop kicks jobs.  This simulates
+     * the race between a pending kick and another job completing.
+     */
+    block_job_enter(job2);
+    block_job_enter(job2);
+
+    while (result1 == -EINPROGRESS || result2 == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+
+    g_assert_cmpint(result1, ==, -ECANCELED);
+    g_assert_cmpint(result2, ==, -ECANCELED);
+
+    block_job_txn_unref(txn);
+}
+
+int main(int argc, char **argv)
+{
+    qemu_init_main_loop(&error_abort);
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/single/success", test_single_job_success);
+    g_test_add_func("/single/failure", test_single_job_failure);
+    g_test_add_func("/single/cancel", test_single_job_cancel);
+    g_test_add_func("/pair/success", test_pair_jobs_success);
+    g_test_add_func("/pair/failure", test_pair_jobs_failure);
+    g_test_add_func("/pair/cancel", test_pair_jobs_cancel);
+    g_test_add_func("/pair/fail-cancel-race", test_pair_jobs_fail_cancel_race);
+    return g_test_run();
+}
diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c
index b552d9f5e..f5951cb1f 100644
--- a/tests/test-coroutine.c
+++ b/tests/test-coroutine.c
@@ -12,8 +12,8 @@
  */
 
 #include <glib.h>
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
 
 /*
  * Check that qemu_in_coroutine() works
diff --git a/tests/test-crypto-cipher.c b/tests/test-crypto-cipher.c
index 9d38d2640..f4946a0af 100644
--- a/tests/test-crypto-cipher.c
+++ b/tests/test-crypto-cipher.c
@@ -287,6 +287,79 @@ static void test_cipher(const void *opaque)
     qcrypto_cipher_free(cipher);
 }
 
+
+static void test_cipher_null_iv(void)
+{
+    QCryptoCipher *cipher;
+    uint8_t key[32] = { 0 };
+    uint8_t plaintext[32] = { 0 };
+    uint8_t ciphertext[32] = { 0 };
+
+    cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_256,
+        QCRYPTO_CIPHER_MODE_CBC,
+        key, sizeof(key),
+        &error_abort);
+    g_assert(cipher != NULL);
+
+    /* Don't call qcrypto_cipher_setiv */
+
+    qcrypto_cipher_encrypt(cipher,
+                           plaintext,
+                           ciphertext,
+                           sizeof(plaintext),
+                           &error_abort);
+
+    qcrypto_cipher_free(cipher);
+}
+
+static void test_cipher_short_plaintext(void)
+{
+    Error *err = NULL;
+    QCryptoCipher *cipher;
+    uint8_t key[32] = { 0 };
+    uint8_t plaintext1[20] = { 0 };
+    uint8_t ciphertext1[20] = { 0 };
+    uint8_t plaintext2[40] = { 0 };
+    uint8_t ciphertext2[40] = { 0 };
+    int ret;
+
+    cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_256,
+        QCRYPTO_CIPHER_MODE_CBC,
+        key, sizeof(key),
+        &error_abort);
+    g_assert(cipher != NULL);
+
+    /* Should report an error as plaintext is shorter
+     * than block size
+     */
+    ret = qcrypto_cipher_encrypt(cipher,
+                                 plaintext1,
+                                 ciphertext1,
+                                 sizeof(plaintext1),
+                                 &err);
+    g_assert(ret == -1);
+    g_assert(err != NULL);
+
+    error_free(err);
+    err = NULL;
+
+    /* Should report an error as plaintext is larger than
+     * block size, but not a multiple of block size
+     */
+    ret = qcrypto_cipher_encrypt(cipher,
+                                 plaintext2,
+                                 ciphertext2,
+                                 sizeof(plaintext2),
+                                 &err);
+    g_assert(ret == -1);
+    g_assert(err != NULL);
+
+    error_free(err);
+    qcrypto_cipher_free(cipher);
+}
+
 int main(int argc, char **argv)
 {
     size_t i;
@@ -298,5 +371,12 @@ int main(int argc, char **argv)
     for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
         g_test_add_data_func(test_data[i].path, &test_data[i], test_cipher);
     }
+
+    g_test_add_func("/crypto/cipher/null-iv",
+                    test_cipher_null_iv);
+
+    g_test_add_func("/crypto/cipher/short-plaintext",
+                    test_cipher_short_plaintext);
+
     return g_test_run();
 }
diff --git a/tests/test-crypto-tlscredsx509.c b/tests/test-crypto-tlscredsx509.c
new file mode 100644
index 000000000..c70aa5563
--- /dev/null
+++ b/tests/test-crypto-tlscredsx509.c
@@ -0,0 +1,731 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include <stdlib.h>
+#include <fcntl.h>
+
+#include "config-host.h"
+#include "crypto-tls-x509-helpers.h"
+#include "crypto/tlscredsx509.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-crypto-tlscredsx509-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QCryptoTLSCredsTestData {
+    bool isServer;
+    const char *cacrt;
+    const char *crt;
+    bool expectFail;
+};
+
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        "testtlscreds",
+        errp,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        "sanity-check", "yes",
+        NULL);
+
+    if (*errp) {
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+/*
+ * This tests sanity checking of our own certificates
+ *
+ * The code being tested is used when TLS creds are created,
+ * and aim to ensure QMEU has been configured with sane
+ * certificates. This allows us to give much much much
+ * clearer error messages to the admin when they misconfigure
+ * things.
+ */
+static void test_tls_creds(const void *opaque)
+{
+    struct QCryptoTLSCredsTestData *data =
+        (struct QCryptoTLSCredsTestData *)opaque;
+    QCryptoTLSCreds *creds;
+    Error *err = NULL;
+
+#define CERT_DIR "tests/test-crypto-tlscredsx509-certs/"
+    mkdir(CERT_DIR, 0700);
+
+    unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    if (data->isServer) {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+    } else {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+    }
+
+    if (access(data->cacrt, R_OK) == 0) {
+        g_assert(link(data->cacrt,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    }
+    if (data->isServer) {
+        if (access(data->crt, R_OK) == 0) {
+            g_assert(link(data->crt,
+                          CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+        }
+        g_assert(link(KEYFILE,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+    } else {
+        if (access(data->crt, R_OK) == 0) {
+            g_assert(link(data->crt,
+                          CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+        }
+        g_assert(link(KEYFILE,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+    }
+
+    creds = test_tls_creds_create(
+        (data->isServer ?
+         QCRYPTO_TLS_CREDS_ENDPOINT_SERVER :
+         QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT),
+        CERT_DIR,
+        &err);
+
+    if (data->expectFail) {
+        error_free(err);
+        g_assert(creds == NULL);
+    } else {
+        if (err) {
+            g_printerr("Failed to generate creds: %s\n",
+                       error_get_pretty(err));
+            error_free(err);
+        }
+        g_assert(creds != NULL);
+    }
+
+    unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    if (data->isServer) {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+    } else {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+    }
+    rmdir(CERT_DIR);
+    if (creds) {
+        object_unparent(OBJECT(creds));
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TLS_TEST_REG(name, isServer, caCrt, crt, expectFail)           \
+    struct QCryptoTLSCredsTestData name = {                             \
+        isServer, caCrt, crt, expectFail                                \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlscredsx509/" # name,               \
+                         &name, test_tls_creds);                        \
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(perfectserver, true,
+                 cacertreq.filename, servercertreq.filename, false);
+    TLS_TEST_REG(perfectclient, false,
+                 cacertreq.filename, clientcertreq.filename, false);
+
+
+    /* Some other CAs which are good */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacert1req,
+                 "UK", "qemu CA 1", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert1req, cacert1req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Basic:CA:not-critical */
+    TLS_ROOT_REQ(cacert2req,
+                 "UK", "qemu CA 2", NULL, NULL, NULL, NULL,
+                 true, false, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert2req, cacert2req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Key usage:cert-sign:critical */
+    TLS_ROOT_REQ(cacert3req,
+                 "UK", "qemu CA 3", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert3req, cacert3req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(goodca1, true,
+                 cacert1req.filename, servercert1req.filename, false);
+    TLS_TEST_REG(goodca2, true,
+                 cacert2req.filename, servercert2req.filename, false);
+    TLS_TEST_REG(goodca3, true,
+                 cacert3req.filename, servercert3req.filename, false);
+
+    /* Now some bad certs */
+
+    /* Key usage:dig-sig:not-critical */
+    TLS_ROOT_REQ(cacert4req,
+                 "UK", "qemu CA 4", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, false, GNUTLS_KEY_DIGITAL_SIGNATURE,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert4req, cacert4req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* no-basic */
+    TLS_ROOT_REQ(cacert5req,
+                 "UK", "qemu CA 5", NULL, NULL, NULL, NULL,
+                 false, false, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert5req, cacert5req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* Key usage:dig-sig:critical */
+    TLS_ROOT_REQ(cacert6req,
+                 "UK", "qemu CA 6", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_DIGITAL_SIGNATURE,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert6req, cacert6req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Technically a CA cert with basic constraints
+     * key purpose == key signing + non-critical should
+     * be rejected. GNUTLS < 3.1 does not reject it and
+     * we don't anticipate them changing this behaviour
+     */
+    TLS_TEST_REG(badca1, true, cacert4req.filename, servercert4req.filename,
+                (GNUTLS_VERSION_MAJOR == 3 && GNUTLS_VERSION_MINOR >= 1) ||
+                GNUTLS_VERSION_MAJOR > 3);
+    TLS_TEST_REG(badca2, true,
+                 cacert5req.filename, servercert5req.filename, true);
+    TLS_TEST_REG(badca3, true,
+                 cacert6req.filename, servercert6req.filename, true);
+
+
+    /* Various good servers */
+    /* no usage or purpose */
+    TLS_CERT_REQ(servercert7req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign+dig-sig+encipher:critical */
+    TLS_CERT_REQ(servercert8req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT |
+                 GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign:not-critical */
+    TLS_CERT_REQ(servercert9req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, false, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:server:critical */
+    TLS_CERT_REQ(servercert10req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* purpose:server:not-critical */
+    TLS_CERT_REQ(servercert11req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* purpose:client+server:critical */
+    TLS_CERT_REQ(servercert12req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+    /* purpose:client+server:not-critical */
+    TLS_CERT_REQ(servercert13req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+
+    TLS_TEST_REG(goodserver1, true,
+                 cacertreq.filename, servercert7req.filename, false);
+    TLS_TEST_REG(goodserver2, true,
+                 cacertreq.filename, servercert8req.filename, false);
+    TLS_TEST_REG(goodserver3, true,
+                 cacertreq.filename, servercert9req.filename, false);
+    TLS_TEST_REG(goodserver4, true,
+                 cacertreq.filename, servercert10req.filename, false);
+    TLS_TEST_REG(goodserver5, true,
+                 cacertreq.filename, servercert11req.filename, false);
+    TLS_TEST_REG(goodserver6, true,
+                 cacertreq.filename, servercert12req.filename, false);
+    TLS_TEST_REG(goodserver7, true,
+                 cacertreq.filename, servercert13req.filename, false);
+
+    /* Bad servers */
+
+    /* usage:cert-sign:critical */
+    TLS_CERT_REQ(servercert14req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(servercert15req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* usage: none:critical */
+    TLS_CERT_REQ(servercert16req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(badserver1, true,
+                 cacertreq.filename, servercert14req.filename, true);
+    TLS_TEST_REG(badserver2, true,
+                 cacertreq.filename, servercert15req.filename, true);
+    TLS_TEST_REG(badserver3, true,
+                 cacertreq.filename, servercert16req.filename, true);
+
+
+
+    /* Various good clients */
+    /* no usage or purpose */
+    TLS_CERT_REQ(clientcert1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign+dig-sig+encipher:critical */
+    TLS_CERT_REQ(clientcert2req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT |
+                 GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign:not-critical */
+    TLS_CERT_REQ(clientcert3req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, false, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(clientcert4req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* purpose:client:not-critical */
+    TLS_CERT_REQ(clientcert5req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* purpose:client+client:critical */
+    TLS_CERT_REQ(clientcert6req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+    /* purpose:client+client:not-critical */
+    TLS_CERT_REQ(clientcert7req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+
+    TLS_TEST_REG(goodclient1, false,
+                 cacertreq.filename, clientcert1req.filename, false);
+    TLS_TEST_REG(goodclient2, false,
+                 cacertreq.filename, clientcert2req.filename, false);
+    TLS_TEST_REG(goodclient3, false,
+                 cacertreq.filename, clientcert3req.filename, false);
+    TLS_TEST_REG(goodclient4, false,
+                 cacertreq.filename, clientcert4req.filename, false);
+    TLS_TEST_REG(goodclient5, false,
+                 cacertreq.filename, clientcert5req.filename, false);
+    TLS_TEST_REG(goodclient6, false,
+                 cacertreq.filename, clientcert6req.filename, false);
+    TLS_TEST_REG(goodclient7, false,
+                 cacertreq.filename, clientcert7req.filename, false);
+
+    /* Bad clients */
+
+    /* usage:cert-sign:critical */
+    TLS_CERT_REQ(clientcert8req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(clientcert9req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* usage: none:critical */
+    TLS_CERT_REQ(clientcert10req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(badclient1, false,
+                 cacertreq.filename, clientcert8req.filename, true);
+    TLS_TEST_REG(badclient2, false,
+                 cacertreq.filename, clientcert9req.filename, true);
+    TLS_TEST_REG(badclient3, false,
+                 cacertreq.filename, clientcert10req.filename, true);
+
+
+
+    /* Expired stuff */
+
+    TLS_ROOT_REQ(cacertexpreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, -1);
+    TLS_CERT_REQ(servercertexpreq, cacertexpreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertexp1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, -1);
+    TLS_CERT_REQ(clientcertexp1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, -1);
+
+    TLS_TEST_REG(expired1, true,
+                 cacertexpreq.filename, servercertexpreq.filename, true);
+    TLS_TEST_REG(expired2, true,
+                 cacertreq.filename, servercertexp1req.filename, true);
+    TLS_TEST_REG(expired3, false,
+                 cacertreq.filename, clientcertexp1req.filename, true);
+
+
+    /* Not activated stuff */
+
+    TLS_ROOT_REQ(cacertnewreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 1, 2);
+    TLS_CERT_REQ(servercertnewreq, cacertnewreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertnew1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 1, 2);
+    TLS_CERT_REQ(clientcertnew1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 1, 2);
+
+    TLS_TEST_REG(inactive1, true,
+                 cacertnewreq.filename, servercertnewreq.filename, true);
+    TLS_TEST_REG(inactive2, true,
+                 cacertreq.filename, servercertnew1req.filename, true);
+    TLS_TEST_REG(inactive3, false,
+                 cacertreq.filename, clientcertnew1req.filename, true);
+
+    TLS_ROOT_REQ(cacertrootreq,
+                 "UK", "qemu root", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1areq, cacertrootreq,
+                 "UK", "qemu level 1a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1breq, cacertrootreq,
+                 "UK", "qemu level 1b", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel2areq, cacertlevel1areq,
+                 "UK", "qemu level 2a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertlevel3areq, cacertlevel2areq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertlevel2breq, cacertlevel1breq,
+                 "UK", "qemu client level 2b", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    gnutls_x509_crt_t certchain[] = {
+        cacertrootreq.crt,
+        cacertlevel1areq.crt,
+        cacertlevel1breq.crt,
+        cacertlevel2areq.crt,
+    };
+
+    test_tls_write_cert_chain(WORKDIR "cacertchain-ctx.pem",
+                              certchain,
+                              G_N_ELEMENTS(certchain));
+
+    TLS_TEST_REG(chain1, true,
+                 WORKDIR "cacertchain-ctx.pem",
+                 servercertlevel3areq.filename, false);
+    TLS_TEST_REG(chain2, false,
+                 WORKDIR "cacertchain-ctx.pem",
+                 clientcertlevel2breq.filename, false);
+
+    /* Some missing certs - first two are fatal, the last
+     * is ok
+     */
+    TLS_TEST_REG(missingca, true,
+                 "cacertdoesnotexist.pem",
+                 servercert1req.filename, true);
+    TLS_TEST_REG(missingserver, true,
+                 cacert1req.filename,
+                 "servercertdoesnotexist.pem", true);
+    TLS_TEST_REG(missingclient, false,
+                 cacert1req.filename,
+                 "clientcertdoesnotexist.pem", false);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&cacertreq);
+    test_tls_discard_cert(&cacert1req);
+    test_tls_discard_cert(&cacert2req);
+    test_tls_discard_cert(&cacert3req);
+    test_tls_discard_cert(&cacert4req);
+    test_tls_discard_cert(&cacert5req);
+    test_tls_discard_cert(&cacert6req);
+
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&servercert1req);
+    test_tls_discard_cert(&servercert2req);
+    test_tls_discard_cert(&servercert3req);
+    test_tls_discard_cert(&servercert4req);
+    test_tls_discard_cert(&servercert5req);
+    test_tls_discard_cert(&servercert6req);
+    test_tls_discard_cert(&servercert7req);
+    test_tls_discard_cert(&servercert8req);
+    test_tls_discard_cert(&servercert9req);
+    test_tls_discard_cert(&servercert10req);
+    test_tls_discard_cert(&servercert11req);
+    test_tls_discard_cert(&servercert12req);
+    test_tls_discard_cert(&servercert13req);
+    test_tls_discard_cert(&servercert14req);
+    test_tls_discard_cert(&servercert15req);
+    test_tls_discard_cert(&servercert16req);
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&clientcert1req);
+    test_tls_discard_cert(&clientcert2req);
+    test_tls_discard_cert(&clientcert3req);
+    test_tls_discard_cert(&clientcert4req);
+    test_tls_discard_cert(&clientcert5req);
+    test_tls_discard_cert(&clientcert6req);
+    test_tls_discard_cert(&clientcert7req);
+    test_tls_discard_cert(&clientcert8req);
+    test_tls_discard_cert(&clientcert9req);
+    test_tls_discard_cert(&clientcert10req);
+
+    test_tls_discard_cert(&cacertexpreq);
+    test_tls_discard_cert(&servercertexpreq);
+    test_tls_discard_cert(&servercertexp1req);
+    test_tls_discard_cert(&clientcertexp1req);
+
+    test_tls_discard_cert(&cacertnewreq);
+    test_tls_discard_cert(&servercertnewreq);
+    test_tls_discard_cert(&servercertnew1req);
+    test_tls_discard_cert(&clientcertnew1req);
+
+    test_tls_discard_cert(&cacertrootreq);
+    test_tls_discard_cert(&cacertlevel1areq);
+    test_tls_discard_cert(&cacertlevel1breq);
+    test_tls_discard_cert(&cacertlevel2areq);
+    test_tls_discard_cert(&servercertlevel3areq);
+    test_tls_discard_cert(&clientcertlevel2breq);
+    unlink(WORKDIR "cacertchain-ctx.pem");
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/test-crypto-tlssession.c b/tests/test-crypto-tlssession.c
new file mode 100644
index 000000000..4524128fc
--- /dev/null
+++ b/tests/test-crypto-tlssession.c
@@ -0,0 +1,535 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include <stdlib.h>
+#include <fcntl.h>
+
+#include "config-host.h"
+#include "crypto-tls-x509-helpers.h"
+#include "crypto/tlscredsx509.h"
+#include "crypto/tlssession.h"
+#include "qom/object_interfaces.h"
+#include "qemu/sockets.h"
+#include "qemu/acl.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-crypto-tlssession-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QCryptoTLSSessionTestData {
+    const char *servercacrt;
+    const char *clientcacrt;
+    const char *servercrt;
+    const char *clientcrt;
+    bool expectServerFail;
+    bool expectClientFail;
+    const char *hostname;
+    const char *const *wildcards;
+};
+
+
+static ssize_t testWrite(const char *buf, size_t len, void *opaque)
+{
+    int *fd = opaque;
+
+    return write(*fd, buf, len);
+}
+
+static ssize_t testRead(char *buf, size_t len, void *opaque)
+{
+    int *fd = opaque;
+
+    return read(*fd, buf, len);
+}
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Error *err = NULL;
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+         "testtlscredsserver" : "testtlscredsclient"),
+        &err,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        /* We skip initial sanity checks here because we
+         * want to make sure that problems are being
+         * detected at the TLS session validation stage,
+         * and the test-crypto-tlscreds test already
+         * validate the sanity check code.
+         */
+        "sanity-check", "no",
+        NULL
+        );
+
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
+/*
+ * This tests validation checking of peer certificates
+ *
+ * This is replicating the checks that are done for an
+ * active TLS session after handshake completes. To
+ * simulate that we create our TLS contexts, skipping
+ * sanity checks. We then get a socketpair, and
+ * initiate a TLS session across them. Finally do
+ * do actual cert validation tests
+ */
+static void test_crypto_tls_session(const void *opaque)
+{
+    struct QCryptoTLSSessionTestData *data =
+        (struct QCryptoTLSSessionTestData *)opaque;
+    QCryptoTLSCreds *clientCreds;
+    QCryptoTLSCreds *serverCreds;
+    QCryptoTLSSession *clientSess = NULL;
+    QCryptoTLSSession *serverSess = NULL;
+    qemu_acl *acl;
+    const char * const *wildcards;
+    int channel[2];
+    bool clientShake = false;
+    bool serverShake = false;
+    Error *err = NULL;
+    int ret;
+
+    /* We'll use this for our fake client-server connection */
+    ret = socketpair(AF_UNIX, SOCK_STREAM, 0, channel);
+    g_assert(ret == 0);
+
+    /*
+     * We have an evil loop to do the handshake in a single
+     * thread, so we need these non-blocking to avoid deadlock
+     * of ourselves
+     */
+    qemu_set_nonblock(channel[0]);
+    qemu_set_nonblock(channel[1]);
+
+#define CLIENT_CERT_DIR "tests/test-crypto-tlssession-client/"
+#define SERVER_CERT_DIR "tests/test-crypto-tlssession-server/"
+    mkdir(CLIENT_CERT_DIR, 0700);
+    mkdir(SERVER_CERT_DIR, 0700);
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    g_assert(link(data->servercacrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->servercrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+
+    g_assert(link(data->clientcacrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->clientcrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+
+    clientCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+        CLIENT_CERT_DIR,
+        &err);
+    g_assert(clientCreds != NULL);
+
+    serverCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+        SERVER_CERT_DIR,
+        &err);
+    g_assert(serverCreds != NULL);
+
+    acl = qemu_acl_init("tlssessionacl");
+    qemu_acl_reset(acl);
+    wildcards = data->wildcards;
+    while (wildcards && *wildcards) {
+        qemu_acl_append(acl, 0, *wildcards);
+        wildcards++;
+    }
+
+    /* Now the real part of the test, setup the sessions */
+    clientSess = qcrypto_tls_session_new(
+        clientCreds, data->hostname, NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT, &err);
+    serverSess = qcrypto_tls_session_new(
+        serverCreds, NULL,
+        data->wildcards ? "tlssessionacl" : NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER, &err);
+
+    g_assert(clientSess != NULL);
+    g_assert(serverSess != NULL);
+
+    /* For handshake to work, we need to set the I/O callbacks
+     * to read/write over the socketpair
+     */
+    qcrypto_tls_session_set_callbacks(serverSess,
+                                      testWrite, testRead,
+                                      &channel[0]);
+    qcrypto_tls_session_set_callbacks(clientSess,
+                                      testWrite, testRead,
+                                      &channel[1]);
+
+    /*
+     * Finally we loop around & around doing handshake on each
+     * session until we get an error, or the handshake completes.
+     * This relies on the socketpair being nonblocking to avoid
+     * deadlocking ourselves upon handshake
+     */
+    do {
+        int rv;
+        if (!serverShake) {
+            rv = qcrypto_tls_session_handshake(serverSess,
+                                               &err);
+            g_assert(rv >= 0);
+            if (qcrypto_tls_session_get_handshake_status(serverSess) ==
+                QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+                serverShake = true;
+            }
+        }
+        if (!clientShake) {
+            rv = qcrypto_tls_session_handshake(clientSess,
+                                               &err);
+            g_assert(rv >= 0);
+            if (qcrypto_tls_session_get_handshake_status(clientSess) ==
+                QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+                clientShake = true;
+            }
+        }
+    } while (!clientShake && !serverShake);
+
+
+    /* Finally make sure the server validation does what
+     * we were expecting
+     */
+    if (qcrypto_tls_session_check_credentials(serverSess, &err) < 0) {
+        g_assert(data->expectServerFail);
+        error_free(err);
+        err = NULL;
+    } else {
+        g_assert(!data->expectServerFail);
+    }
+
+    /*
+     * And the same for the client validation check
+     */
+    if (qcrypto_tls_session_check_credentials(clientSess, &err) < 0) {
+        g_assert(data->expectClientFail);
+        error_free(err);
+        err = NULL;
+    } else {
+        g_assert(!data->expectClientFail);
+    }
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    rmdir(CLIENT_CERT_DIR);
+    rmdir(SERVER_CERT_DIR);
+
+    object_unparent(OBJECT(serverCreds));
+    object_unparent(OBJECT(clientCreds));
+
+    qcrypto_tls_session_free(serverSess);
+    qcrypto_tls_session_free(clientSess);
+
+    close(channel[0]);
+    close(channel[1]);
+}
+
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TEST_SESS_REG(name, caCrt,                                     \
+                       serverCrt, clientCrt,                            \
+                       expectServerFail, expectClientFail,              \
+                       hostname, wildcards)                             \
+    struct QCryptoTLSSessionTestData name = {                           \
+        caCrt, caCrt, serverCrt, clientCrt,                             \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlssession/" # name,                 \
+                         &name, test_crypto_tls_session);               \
+
+
+# define TEST_SESS_REG_EXT(name, serverCaCrt, clientCaCrt,              \
+                           serverCrt, clientCrt,                        \
+                           expectServerFail, expectClientFail,          \
+                           hostname, wildcards)                         \
+    struct QCryptoTLSSessionTestData name = {                           \
+        serverCaCrt, clientCaCrt, serverCrt, clientCrt,                 \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlssession/" # name,                 \
+                         &name, test_crypto_tls_session);               \
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_ROOT_REQ(altcacertreq,
+                 "UK", "qemu CA 1", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(clientcertaltreq, altcacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TEST_SESS_REG(basicca, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", NULL);
+    TEST_SESS_REG_EXT(differentca, cacertreq.filename,
+                      altcacertreq.filename, servercertreq.filename,
+                      clientcertaltreq.filename, true, true, "qemu.org", NULL);
+
+
+    /* When an altname is set, the CN is ignored, so it must be duplicated
+     * as an altname for it to match */
+    TLS_CERT_REQ(servercertalt1req, cacertreq,
+                 "UK", "qemu.org", "www.qemu.org", "qemu.org",
+                 "192.168.122.1", "fec0::dead:beaf",
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* This intentionally doesn't replicate */
+    TLS_CERT_REQ(servercertalt2req, cacertreq,
+                 "UK", "qemu.org", "www.qemu.org", "wiki.qemu.org",
+                 "192.168.122.1", "fec0::dead:beaf",
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    TEST_SESS_REG(altname1, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, false, "qemu.org", NULL);
+    TEST_SESS_REG(altname2, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, false, "www.qemu.org", NULL);
+    TEST_SESS_REG(altname3, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, true, "wiki.qemu.org", NULL);
+
+    TEST_SESS_REG(altname4, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, true, "qemu.org", NULL);
+    TEST_SESS_REG(altname5, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, false, "www.qemu.org", NULL);
+    TEST_SESS_REG(altname6, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, false, "wiki.qemu.org", NULL);
+
+    const char *const wildcards1[] = {
+        "C=UK,CN=dogfood",
+        NULL,
+    };
+    const char *const wildcards2[] = {
+        "C=UK,CN=qemu",
+        NULL,
+    };
+    const char *const wildcards3[] = {
+        "C=UK,CN=dogfood",
+        "C=UK,CN=qemu",
+        NULL,
+    };
+    const char *const wildcards4[] = {
+        "C=UK,CN=qemustuff",
+        NULL,
+    };
+    const char *const wildcards5[] = {
+        "C=UK,CN=qemu*",
+        NULL,
+    };
+    const char *const wildcards6[] = {
+        "C=UK,CN=*emu*",
+        NULL,
+    };
+
+    TEST_SESS_REG(wildcard1, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  true, false, "qemu.org", wildcards1);
+    TEST_SESS_REG(wildcard2, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards2);
+    TEST_SESS_REG(wildcard3, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards3);
+    TEST_SESS_REG(wildcard4, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  true, false, "qemu.org", wildcards4);
+    TEST_SESS_REG(wildcard5, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards5);
+    TEST_SESS_REG(wildcard6, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards6);
+
+    TLS_ROOT_REQ(cacertrootreq,
+                 "UK", "qemu root", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1areq, cacertrootreq,
+                 "UK", "qemu level 1a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1breq, cacertrootreq,
+                 "UK", "qemu level 1b", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel2areq, cacertlevel1areq,
+                 "UK", "qemu level 2a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertlevel3areq, cacertlevel2areq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertlevel2breq, cacertlevel1breq,
+                 "UK", "qemu client level 2b", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    gnutls_x509_crt_t certchain[] = {
+        cacertrootreq.crt,
+        cacertlevel1areq.crt,
+        cacertlevel1breq.crt,
+        cacertlevel2areq.crt,
+    };
+
+    test_tls_write_cert_chain(WORKDIR "cacertchain-sess.pem",
+                              certchain,
+                              G_N_ELEMENTS(certchain));
+
+    TEST_SESS_REG(cachain, WORKDIR "cacertchain-sess.pem",
+                  servercertlevel3areq.filename, clientcertlevel2breq.filename,
+                  false, false, "qemu.org", NULL);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&clientcertaltreq);
+
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&servercertalt1req);
+    test_tls_discard_cert(&servercertalt2req);
+
+    test_tls_discard_cert(&cacertreq);
+    test_tls_discard_cert(&altcacertreq);
+
+    test_tls_discard_cert(&cacertrootreq);
+    test_tls_discard_cert(&cacertlevel1areq);
+    test_tls_discard_cert(&cacertlevel1breq);
+    test_tls_discard_cert(&cacertlevel2areq);
+    test_tls_discard_cert(&servercertlevel3areq);
+    test_tls_discard_cert(&clientcertlevel2breq);
+    unlink(WORKDIR "cacertchain-sess.pem");
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/test-cutils.c b/tests/test-cutils.c
index 2a4556d3a..a3de6ab87 100644
--- a/tests/test-cutils.c
+++ b/tests/test-cutils.c
@@ -226,6 +226,1212 @@ static void test_parse_uint_full_correct(void)
     g_assert_cmpint(i, ==, 123);
 }
 
+static void test_qemu_strtol_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtol_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtol_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtol_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_max(void)
+{
+    const char *str = g_strdup_printf("%ld", LONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err  = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LONG_MIN);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_full_correct(void)
+{
+    const char *str = "123";
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtol_full_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtol_full_empty(void)
+{
+    const char *str = "";
+    long res = 999L;
+    int err;
+
+    err =  qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_full_negative(void)
+{
+    const char *str = " \t -321";
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+}
+
+static void test_qemu_strtol_full_trailing(void)
+{
+    const char *str = "123xxx";
+    long res;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_full_max(void)
+{
+    const char *str = g_strdup_printf("%ld", LONG_MAX);
+    long res;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LONG_MAX);
+}
+
+static void test_qemu_strtoul_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoul_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoul_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoul_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_max(void)
+{
+    const char *str = g_strdup_printf("%lu", ULONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err  = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, -1ul);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321ul);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_full_correct(void)
+{
+    const char *str = "123";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtoul_full_null(void)
+{
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_full_empty(void)
+{
+    const char *str = "";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+static void test_qemu_strtoul_full_negative(void)
+{
+    const char *str = " \t -321";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321ul);
+}
+
+static void test_qemu_strtoul_full_trailing(void)
+{
+    const char *str = "123xxx";
+    unsigned long res;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_full_max(void)
+{
+    const char *str = g_strdup_printf("%lu", ULONG_MAX);
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+}
+
+static void test_qemu_strtoll_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoll_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoll_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoll_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_max(void)
+{
+    const char *str = g_strdup_printf("%lld", LLONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err  = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LLONG_MIN);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_full_correct(void)
+{
+    const char *str = "123";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtoll_full_null(void)
+{
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_empty(void)
+{
+    const char *str = "";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_negative(void)
+{
+    const char *str = " \t -321";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+}
+
+static void test_qemu_strtoll_full_trailing(void)
+{
+    const char *str = "123xxx";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_max(void)
+{
+
+    const char *str = g_strdup_printf("%lld", LLONG_MAX);
+    int64_t res;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+}
+
+static void test_qemu_strtoull_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoull_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoull_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoull_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_max(void)
+{
+    const char *str = g_strdup_printf("%llu", ULLONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err  = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, -1);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_full_correct(void)
+{
+    const char *str = "18446744073709551614";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 18446744073709551614LLU);
+}
+
+static void test_qemu_strtoull_full_null(void)
+{
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_empty(void)
+{
+    const char *str = "";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_negative(void)
+{
+    const char *str = " \t -321";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 18446744073709551295LLU);
+}
+
+static void test_qemu_strtoull_full_trailing(void)
+{
+    const char *str = "18446744073709551614xxxxxx";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_max(void)
+{
+    const char *str = g_strdup_printf("%lld", ULLONG_MAX);
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+}
+
+static void test_qemu_strtosz_simple(void)
+{
+    const char *str = "12345M";
+    char *endptr = NULL;
+    int64_t res;
+
+    res = qemu_strtosz(str, &endptr);
+    g_assert_cmpint(res, ==, 12345 * M_BYTE);
+    g_assert(endptr == str + 6);
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, 12345 * M_BYTE);
+}
+
+static void test_qemu_strtosz_units(void)
+{
+    const char *none = "1";
+    const char *b = "1B";
+    const char *k = "1K";
+    const char *m = "1M";
+    const char *g = "1G";
+    const char *t = "1T";
+    const char *p = "1P";
+    const char *e = "1E";
+    int64_t res;
+
+    /* default is M */
+    res = qemu_strtosz(none, NULL);
+    g_assert_cmpint(res, ==, M_BYTE);
+
+    res = qemu_strtosz(b, NULL);
+    g_assert_cmpint(res, ==, 1);
+
+    res = qemu_strtosz(k, NULL);
+    g_assert_cmpint(res, ==, K_BYTE);
+
+    res = qemu_strtosz(m, NULL);
+    g_assert_cmpint(res, ==, M_BYTE);
+
+    res = qemu_strtosz(g, NULL);
+    g_assert_cmpint(res, ==, G_BYTE);
+
+    res = qemu_strtosz(t, NULL);
+    g_assert_cmpint(res, ==, T_BYTE);
+
+    res = qemu_strtosz(p, NULL);
+    g_assert_cmpint(res, ==, P_BYTE);
+
+    res = qemu_strtosz(e, NULL);
+    g_assert_cmpint(res, ==, E_BYTE);
+}
+
+static void test_qemu_strtosz_float(void)
+{
+    const char *str = "12.345M";
+    int64_t res;
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, 12.345 * M_BYTE);
+}
+
+static void test_qemu_strtosz_erange(void)
+{
+    const char *str = "10E";
+    int64_t res;
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, -ERANGE);
+}
+
+static void test_qemu_strtosz_suffix_unit(void)
+{
+    const char *str = "12345";
+    int64_t res;
+
+    res = qemu_strtosz_suffix_unit(str, NULL,
+                                   QEMU_STRTOSZ_DEFSUFFIX_KB, 1000);
+    g_assert_cmpint(res, ==, 12345000);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
@@ -247,5 +1453,145 @@ int main(int argc, char **argv)
     g_test_add_func("/cutils/parse_uint_full/correct",
                     test_parse_uint_full_correct);
 
+    /* qemu_strtol() tests */
+    g_test_add_func("/cutils/qemu_strtol/correct", test_qemu_strtol_correct);
+    g_test_add_func("/cutils/qemu_strtol/null", test_qemu_strtol_null);
+    g_test_add_func("/cutils/qemu_strtol/empty", test_qemu_strtol_empty);
+    g_test_add_func("/cutils/qemu_strtol/whitespace",
+                    test_qemu_strtol_whitespace);
+    g_test_add_func("/cutils/qemu_strtol/invalid", test_qemu_strtol_invalid);
+    g_test_add_func("/cutils/qemu_strtol/trailing", test_qemu_strtol_trailing);
+    g_test_add_func("/cutils/qemu_strtol/octal", test_qemu_strtol_octal);
+    g_test_add_func("/cutils/qemu_strtol/decimal", test_qemu_strtol_decimal);
+    g_test_add_func("/cutils/qemu_strtol/hex", test_qemu_strtol_hex);
+    g_test_add_func("/cutils/qemu_strtol/max", test_qemu_strtol_max);
+    g_test_add_func("/cutils/qemu_strtol/overflow", test_qemu_strtol_overflow);
+    g_test_add_func("/cutils/qemu_strtol/underflow",
+                    test_qemu_strtol_underflow);
+    g_test_add_func("/cutils/qemu_strtol/negative", test_qemu_strtol_negative);
+    g_test_add_func("/cutils/qemu_strtol_full/correct",
+                    test_qemu_strtol_full_correct);
+    g_test_add_func("/cutils/qemu_strtol_full/null",
+                    test_qemu_strtol_full_null);
+    g_test_add_func("/cutils/qemu_strtol_full/empty",
+                    test_qemu_strtol_full_empty);
+    g_test_add_func("/cutils/qemu_strtol_full/negative",
+                    test_qemu_strtol_full_negative);
+    g_test_add_func("/cutils/qemu_strtol_full/trailing",
+                    test_qemu_strtol_full_trailing);
+    g_test_add_func("/cutils/qemu_strtol_full/max",
+                    test_qemu_strtol_full_max);
+
+    /* qemu_strtoul() tests */
+    g_test_add_func("/cutils/qemu_strtoul/correct", test_qemu_strtoul_correct);
+    g_test_add_func("/cutils/qemu_strtoul/null", test_qemu_strtoul_null);
+    g_test_add_func("/cutils/qemu_strtoul/empty", test_qemu_strtoul_empty);
+    g_test_add_func("/cutils/qemu_strtoul/whitespace",
+                    test_qemu_strtoul_whitespace);
+    g_test_add_func("/cutils/qemu_strtoul/invalid", test_qemu_strtoul_invalid);
+    g_test_add_func("/cutils/qemu_strtoul/trailing",
+                    test_qemu_strtoul_trailing);
+    g_test_add_func("/cutils/qemu_strtoul/octal", test_qemu_strtoul_octal);
+    g_test_add_func("/cutils/qemu_strtoul/decimal", test_qemu_strtoul_decimal);
+    g_test_add_func("/cutils/qemu_strtoul/hex", test_qemu_strtoul_hex);
+    g_test_add_func("/cutils/qemu_strtoul/max", test_qemu_strtoul_max);
+    g_test_add_func("/cutils/qemu_strtoul/overflow",
+                    test_qemu_strtoul_overflow);
+    g_test_add_func("/cutils/qemu_strtoul/underflow",
+                    test_qemu_strtoul_underflow);
+    g_test_add_func("/cutils/qemu_strtoul/negative",
+                    test_qemu_strtoul_negative);
+    g_test_add_func("/cutils/qemu_strtoul_full/correct",
+                    test_qemu_strtoul_full_correct);
+    g_test_add_func("/cutils/qemu_strtoul_full/null",
+                    test_qemu_strtoul_full_null);
+    g_test_add_func("/cutils/qemu_strtoul_full/empty",
+                    test_qemu_strtoul_full_empty);
+    g_test_add_func("/cutils/qemu_strtoul_full/negative",
+                    test_qemu_strtoul_full_negative);
+    g_test_add_func("/cutils/qemu_strtoul_full/trailing",
+                    test_qemu_strtoul_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoul_full/max",
+                    test_qemu_strtoul_full_max);
+
+    /* qemu_strtoll() tests */
+    g_test_add_func("/cutils/qemu_strtoll/correct", test_qemu_strtoll_correct);
+    g_test_add_func("/cutils/qemu_strtoll/null", test_qemu_strtoll_null);
+    g_test_add_func("/cutils/qemu_strtoll/empty", test_qemu_strtoll_empty);
+    g_test_add_func("/cutils/qemu_strtoll/whitespace",
+                    test_qemu_strtoll_whitespace);
+    g_test_add_func("/cutils/qemu_strtoll/invalid", test_qemu_strtoll_invalid);
+    g_test_add_func("/cutils/qemu_strtoll/trailing",
+                    test_qemu_strtoll_trailing);
+    g_test_add_func("/cutils/qemu_strtoll/octal", test_qemu_strtoll_octal);
+    g_test_add_func("/cutils/qemu_strtoll/decimal", test_qemu_strtoll_decimal);
+    g_test_add_func("/cutils/qemu_strtoll/hex", test_qemu_strtoll_hex);
+    g_test_add_func("/cutils/qemu_strtoll/max", test_qemu_strtoll_max);
+    g_test_add_func("/cutils/qemu_strtoll/overflow",
+                    test_qemu_strtoll_overflow);
+    g_test_add_func("/cutils/qemu_strtoll/underflow",
+                    test_qemu_strtoll_underflow);
+    g_test_add_func("/cutils/qemu_strtoll/negative",
+                    test_qemu_strtoll_negative);
+    g_test_add_func("/cutils/qemu_strtoll_full/correct",
+                    test_qemu_strtoll_full_correct);
+    g_test_add_func("/cutils/qemu_strtoll_full/null",
+                    test_qemu_strtoll_full_null);
+    g_test_add_func("/cutils/qemu_strtoll_full/empty",
+                    test_qemu_strtoll_full_empty);
+    g_test_add_func("/cutils/qemu_strtoll_full/negative",
+                    test_qemu_strtoll_full_negative);
+    g_test_add_func("/cutils/qemu_strtoll_full/trailing",
+                    test_qemu_strtoll_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoll_full/max",
+                    test_qemu_strtoll_full_max);
+
+    /* qemu_strtoull() tests */
+    g_test_add_func("/cutils/qemu_strtoull/correct",
+                    test_qemu_strtoull_correct);
+    g_test_add_func("/cutils/qemu_strtoull/null",
+                    test_qemu_strtoull_null);
+    g_test_add_func("/cutils/qemu_strtoull/empty", test_qemu_strtoull_empty);
+    g_test_add_func("/cutils/qemu_strtoull/whitespace",
+                    test_qemu_strtoull_whitespace);
+    g_test_add_func("/cutils/qemu_strtoull/invalid",
+                    test_qemu_strtoull_invalid);
+    g_test_add_func("/cutils/qemu_strtoull/trailing",
+                    test_qemu_strtoull_trailing);
+    g_test_add_func("/cutils/qemu_strtoull/octal", test_qemu_strtoull_octal);
+    g_test_add_func("/cutils/qemu_strtoull/decimal",
+                    test_qemu_strtoull_decimal);
+    g_test_add_func("/cutils/qemu_strtoull/hex", test_qemu_strtoull_hex);
+    g_test_add_func("/cutils/qemu_strtoull/max", test_qemu_strtoull_max);
+    g_test_add_func("/cutils/qemu_strtoull/overflow",
+                    test_qemu_strtoull_overflow);
+    g_test_add_func("/cutils/qemu_strtoull/underflow",
+                    test_qemu_strtoull_underflow);
+    g_test_add_func("/cutils/qemu_strtoull/negative",
+                    test_qemu_strtoull_negative);
+    g_test_add_func("/cutils/qemu_strtoull_full/correct",
+                    test_qemu_strtoull_full_correct);
+    g_test_add_func("/cutils/qemu_strtoull_full/null",
+                    test_qemu_strtoull_full_null);
+    g_test_add_func("/cutils/qemu_strtoull_full/empty",
+                    test_qemu_strtoull_full_empty);
+    g_test_add_func("/cutils/qemu_strtoull_full/negative",
+                    test_qemu_strtoull_full_negative);
+    g_test_add_func("/cutils/qemu_strtoull_full/trailing",
+                    test_qemu_strtoull_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoull_full/max",
+                    test_qemu_strtoull_full_max);
+
+    g_test_add_func("/cutils/strtosz/simple",
+                    test_qemu_strtosz_simple);
+    g_test_add_func("/cutils/strtosz/units",
+                    test_qemu_strtosz_units);
+    g_test_add_func("/cutils/strtosz/float",
+                    test_qemu_strtosz_float);
+    g_test_add_func("/cutils/strtosz/erange",
+                    test_qemu_strtosz_erange);
+    g_test_add_func("/cutils/strtosz/suffix-unit",
+                    test_qemu_strtosz_suffix_unit);
+
     return g_test_run();
 }
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index 161eeb496..abcea0cda 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -139,10 +139,8 @@ static void hbitmap_test_teardown(TestHBitmapData *data,
         hbitmap_free(data->hb);
         data->hb = NULL;
     }
-    if (data->bits) {
-        g_free(data->bits);
-        data->bits = NULL;
-    }
+    g_free(data->bits);
+    data->bits = NULL;
 }
 
 /* Set a range in the HBitmap and in the shadow "simple" bitmap.
diff --git a/tests/test-netfilter.c b/tests/test-netfilter.c
new file mode 100644
index 000000000..303deb7e0
--- /dev/null
+++ b/tests/test-netfilter.c
@@ -0,0 +1,200 @@
+/*
+ * QTest testcase for netfilter
+ *
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include "libqtest.h"
+
+/* add a netfilter to a netdev and then remove it */
+static void add_one_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add a netfilter to a netdev and then remove the netdev */
+static void remove_netdev_with_one_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'netdev_del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* add back the netdev */
+    response = qmp("{'execute': 'netdev_add',"
+                   " 'arguments': {"
+                   "   'type': 'user',"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add multi(2) netfilters to a netdev and then remove them */
+static void add_multi_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f1',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f1'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add multi(2) netfilters to a netdev and then remove the netdev */
+static void remove_netdev_with_multi_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f1',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'netdev_del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* add back the netdev */
+    response = qmp("{'execute': 'netdev_add',"
+                   " 'arguments': {"
+                   "   'type': 'user',"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+    qtest_add_func("/netfilter/addremove_one", add_one_netfilter);
+    qtest_add_func("/netfilter/remove_netdev_one",
+                   remove_netdev_with_one_netfilter);
+    qtest_add_func("/netfilter/addremove_multi", add_multi_netfilter);
+    qtest_add_func("/netfilter/remove_netdev_multi",
+                   remove_netdev_with_multi_netfilter);
+
+    qtest_start("-netdev user,id=qtest-bn0 -device e1000,netdev=qtest-bn0");
+    ret = g_test_run();
+
+    qtest_end();
+
+    return ret;
+}
diff --git a/tests/test-qga.c b/tests/test-qga.c
new file mode 100644
index 000000000..e6a84d17f
--- /dev/null
+++ b/tests/test-qga.c
@@ -0,0 +1,877 @@
+#include <locale.h>
+#include <glib.h>
+#include <glib/gstdio.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+#include <inttypes.h>
+
+#include "libqtest.h"
+#include "config-host.h"
+#include "qga/guest-agent-core.h"
+
+typedef struct {
+    char *test_dir;
+    GMainLoop *loop;
+    int fd;
+    GPid pid;
+} TestFixture;
+
+static int connect_qga(char *path)
+{
+    int s, ret, len, i = 0;
+    struct sockaddr_un remote;
+
+    s = socket(AF_UNIX, SOCK_STREAM, 0);
+    g_assert(s != -1);
+
+    remote.sun_family = AF_UNIX;
+    do {
+        strcpy(remote.sun_path, path);
+        len = strlen(remote.sun_path) + sizeof(remote.sun_family);
+        ret = connect(s, (struct sockaddr *)&remote, len);
+        if (ret == -1) {
+            g_usleep(G_USEC_PER_SEC);
+        }
+        if (i++ == 10) {
+            return -1;
+        }
+    } while (ret == -1);
+
+    return s;
+}
+
+static void qga_watch(GPid pid, gint status, gpointer user_data)
+{
+    TestFixture *fixture = user_data;
+
+    g_assert_cmpint(status, ==, 0);
+    g_main_loop_quit(fixture->loop);
+}
+
+static void
+fixture_setup(TestFixture *fixture, gconstpointer data)
+{
+    const gchar *extra_arg = data;
+    GError *error = NULL;
+    gchar *cwd, *path, *cmd, **argv = NULL;
+
+    fixture->loop = g_main_loop_new(NULL, FALSE);
+
+    fixture->test_dir = g_strdup("/tmp/qgatest.XXXXXX");
+    g_assert_nonnull(mkdtemp(fixture->test_dir));
+
+    path = g_build_filename(fixture->test_dir, "sock", NULL);
+    cwd = g_get_current_dir();
+    cmd = g_strdup_printf("%s%cqemu-ga -m unix-listen -t %s -p %s %s %s",
+                          cwd, G_DIR_SEPARATOR,
+                          fixture->test_dir, path,
+                          getenv("QTEST_LOG") ? "-v" : "",
+                          extra_arg ?: "");
+    g_shell_parse_argv(cmd, NULL, &argv, &error);
+    g_assert_no_error(error);
+
+    g_spawn_async(fixture->test_dir, argv, NULL,
+                  G_SPAWN_SEARCH_PATH|G_SPAWN_DO_NOT_REAP_CHILD,
+                  NULL, NULL, &fixture->pid, &error);
+    g_assert_no_error(error);
+
+    g_child_watch_add(fixture->pid, qga_watch, fixture);
+
+    fixture->fd = connect_qga(path);
+    g_assert_cmpint(fixture->fd, !=, -1);
+
+    g_strfreev(argv);
+    g_free(cmd);
+    g_free(cwd);
+    g_free(path);
+}
+
+static void
+fixture_tear_down(TestFixture *fixture, gconstpointer data)
+{
+    gchar *tmp;
+
+    kill(fixture->pid, SIGTERM);
+
+    g_main_loop_run(fixture->loop);
+    g_main_loop_unref(fixture->loop);
+
+    g_spawn_close_pid(fixture->pid);
+
+    tmp = g_build_filename(fixture->test_dir, "foo", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    tmp = g_build_filename(fixture->test_dir, "qga.state", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    tmp = g_build_filename(fixture->test_dir, "sock", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    g_rmdir(fixture->test_dir);
+    g_free(fixture->test_dir);
+}
+
+static void qmp_assertion_message_error(const char     *domain,
+                                        const char     *file,
+                                        int             line,
+                                        const char     *func,
+                                        const char     *expr,
+                                        QDict          *dict)
+{
+    const char *class, *desc;
+    char *s;
+    QDict *error;
+
+    error = qdict_get_qdict(dict, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+
+    s = g_strdup_printf("assertion failed %s: %s %s", expr, class, desc);
+    g_assertion_message(domain, file, line, func, s);
+    g_free(s);
+}
+
+#define qmp_assert_no_error(err) do {                                   \
+    if (qdict_haskey(err, "error")) {                                   \
+        qmp_assertion_message_error(G_LOG_DOMAIN, __FILE__, __LINE__,   \
+                                    G_STRFUNC, #err, err);              \
+    }                                                                   \
+} while (0)
+
+static void test_qga_sync_delimited(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    guint32 v, r = g_random_int();
+    unsigned char c;
+    QDict *ret;
+    gchar *cmd;
+
+    cmd = g_strdup_printf("%c{'execute': 'guest-sync-delimited',"
+                          " 'arguments': {'id': %u } }", 0xff, r);
+    qmp_fd_send(fixture->fd, cmd);
+    g_free(cmd);
+
+    v = read(fixture->fd, &c, 1);
+    g_assert_cmpint(v, ==, 1);
+    g_assert_cmpint(c, ==, 0xff);
+
+    ret = qmp_fd_receive(fixture->fd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    v = qdict_get_int(ret, "return");
+    g_assert_cmpint(r, ==, v);
+
+    QDECREF(ret);
+}
+
+static void test_qga_sync(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    guint32 v, r = g_random_int();
+    QDict *ret;
+    gchar *cmd;
+
+    cmd = g_strdup_printf("%c{'execute': 'guest-sync',"
+                          " 'arguments': {'id': %u } }", 0xff, r);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_free(cmd);
+
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    v = qdict_get_int(ret, "return");
+    g_assert_cmpint(r, ==, v);
+
+    QDECREF(ret);
+}
+
+static void test_qga_ping(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-ping'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    QDECREF(ret);
+}
+
+static void test_qga_invalid_cmd(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *error;
+    const gchar *class, *desc;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-invalid-cmd'}");
+    g_assert_nonnull(ret);
+
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+
+    g_assert_cmpstr(class, ==, "CommandNotFound");
+    g_assert_cmpint(strlen(desc), >, 0);
+
+    QDECREF(ret);
+}
+
+static void test_qga_info(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *val;
+    const gchar *version;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-info'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    version = qdict_get_try_str(val, "version");
+    g_assert_cmpstr(version, ==, QEMU_VERSION);
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_vcpus(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-vcpus'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* check there is at least a cpu */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "online"));
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "logical-id"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_fsinfo(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-fsinfo'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* sanity-check the response if there are any filesystems */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    if (entry) {
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "name"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "mountpoint"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "type"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "disk"));
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_memory_block_info(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *val;
+    int64_t size;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-memory-block-info'}");
+    g_assert_nonnull(ret);
+
+    /* some systems might not expose memory block info in sysfs */
+    if (!qdict_haskey(ret, "error")) {
+        /* check there is at least some memory */
+        val = qdict_get_qdict(ret, "return");
+        size = qdict_get_int(val, "size");
+        g_assert_cmpint(size, >, 0);
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_memory_blocks(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-memory-blocks'}");
+    g_assert_nonnull(ret);
+
+    /* some systems might not expose memory block info in sysfs */
+    if (!qdict_haskey(ret, "error")) {
+        list = qdict_get_qlist(ret, "return");
+        entry = qlist_first(list);
+        /* newer versions of qga may return empty list without error */
+        if (entry) {
+            g_assert(qdict_haskey(qobject_to_qdict(entry->value), "phys-index"));
+            g_assert(qdict_haskey(qobject_to_qdict(entry->value), "online"));
+        }
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_network_get_interfaces(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-network-get-interfaces'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* check there is at least an interface */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "name"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_file_ops(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    const unsigned char helloworld[] = "Hello World!\n";
+    const char *b64;
+    gchar *cmd, *path, *enc;
+    unsigned char *dec;
+    QDict *ret, *val;
+    int64_t id, eof;
+    gsize count;
+    FILE *f;
+    char tmp[100];
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'w+' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    enc = g_base64_encode(helloworld, sizeof(helloworld));
+    /* write */
+    cmd = g_strdup_printf("{'execute': 'guest-file-write',"
+                          " 'arguments': { 'handle': %" PRId64 ","
+                          " 'buf-b64': '%s' } }", id, enc);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert_cmpint(eof, ==, 0);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* flush */
+    cmd = g_strdup_printf("{'execute': 'guest-file-flush',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* check content */
+    path = g_build_filename(fixture->test_dir, "foo", NULL);
+    f = fopen(path, "r");
+    g_assert_nonnull(f);
+    count = fread(tmp, 1, sizeof(tmp), f);
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    tmp[count] = 0;
+    g_assert_cmpstr(tmp, ==, (char *)helloworld);
+    fclose(f);
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'r' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    /* read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, enc);
+
+    QDECREF(ret);
+    g_free(cmd);
+    g_free(enc);
+
+    /* read eof */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, "");
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* seek */
+    cmd = g_strdup_printf("{'execute': 'guest-file-seek',"
+                          " 'arguments': { 'handle': %" PRId64 ", "
+                          " 'offset': %d, 'whence': %d } }",
+                          id, 6, QGA_SEEK_SET);
+    ret = qmp_fd(fixture->fd, cmd);
+    qmp_assert_no_error(ret);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "position");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, 6);
+    g_assert(!eof);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* partial read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld) - 6);
+    g_assert(eof);
+    dec = g_base64_decode(b64, &count);
+    g_assert_cmpint(count, ==, sizeof(helloworld) - 6);
+    g_assert_cmpmem(dec, count, helloworld + 6, sizeof(helloworld) - 6);
+    g_free(dec);
+
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+}
+
+static void test_qga_file_write_read(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    const unsigned char helloworld[] = "Hello World!\n";
+    const char *b64;
+    gchar *cmd, *enc;
+    QDict *ret, *val;
+    int64_t id, eof;
+    gsize count;
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'w+' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    enc = g_base64_encode(helloworld, sizeof(helloworld));
+    /* write */
+    cmd = g_strdup_printf("{'execute': 'guest-file-write',"
+                          " 'arguments': { 'handle': %" PRId64 ","
+                          " 'buf-b64': '%s' } }", id, enc);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert_cmpint(eof, ==, 0);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* read (check implicit flush) */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, "");
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* seek to 0 */
+    cmd = g_strdup_printf("{'execute': 'guest-file-seek',"
+                          " 'arguments': { 'handle': %" PRId64 ", "
+                          " 'offset': %d, 'whence': %d } }",
+                          id, 0, QGA_SEEK_SET);
+    ret = qmp_fd(fixture->fd, cmd);
+    qmp_assert_no_error(ret);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "position");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(!eof);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, enc);
+    QDECREF(ret);
+    g_free(cmd);
+    g_free(enc);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+}
+
+static void test_qga_get_time(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    int64_t time;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    time = qdict_get_int(ret, "return");
+    g_assert_cmpint(time, >, 0);
+
+    QDECREF(ret);
+}
+
+static void test_qga_set_time(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    int64_t current, time;
+    gchar *cmd;
+
+    /* get current time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    current = qdict_get_int(ret, "return");
+    g_assert_cmpint(current, >, 0);
+    QDECREF(ret);
+
+    /* set some old time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-set-time',"
+                 " 'arguments': { 'time': 1000 } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    /* check old time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    time = qdict_get_int(ret, "return");
+    g_assert_cmpint(time / 1000, <, G_USEC_PER_SEC * 10);
+    QDECREF(ret);
+
+    /* set back current time */
+    cmd = g_strdup_printf("{'execute': 'guest-set-time',"
+                          " 'arguments': { 'time': %" PRId64 " } }",
+                          current + time * 1000);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_free(cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+}
+
+static void test_qga_fstrim(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fstrim',"
+                 " arguments: { minimum: 4194304 } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "paths"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_blacklist(gconstpointer data)
+{
+    TestFixture fix;
+    QDict *ret, *error;
+    const gchar *class, *desc;
+
+    fixture_setup(&fix, "-b guest-ping,guest-get-time");
+
+    /* check blacklist */
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-ping'}");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_nonnull(g_strstr_len(desc, -1, "has been disabled"));
+    QDECREF(ret);
+
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_nonnull(g_strstr_len(desc, -1, "has been disabled"));
+    QDECREF(ret);
+
+    /* check something work */
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-get-fsinfo'}");
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    fixture_tear_down(&fix, NULL);
+}
+
+static void test_qga_config(gconstpointer data)
+{
+    GError *error = NULL;
+    char *cwd, *cmd, *out, *err, *str, **strv, *conf, **argv = NULL;
+    char *env[2];
+    int status, tmp;
+    gsize n;
+    GKeyFile *kf;
+    const char *qga_config =
+        "[general]\n"
+        "daemon=false\n"
+        "method=virtio-serial\n"
+        "path=/path/to/org.qemu.guest_agent.0\n"
+        "pidfile=/var/foo/qemu-ga.pid\n"
+        "statedir=/var/state\n"
+        "verbose=true\n"
+        "blacklist=guest-ping;guest-get-time\n";
+
+    tmp = g_file_open_tmp(NULL, &conf, &error);
+    g_assert_no_error(error);
+    g_assert_cmpint(tmp, >=, 0);
+    g_assert_cmpstr(conf, !=, "");
+
+    g_file_set_contents(conf, qga_config, -1, &error);
+    g_assert_no_error(error);
+
+    cwd = g_get_current_dir();
+    cmd = g_strdup_printf("%s%cqemu-ga -D",
+                          cwd, G_DIR_SEPARATOR);
+    g_shell_parse_argv(cmd, NULL, &argv, &error);
+    g_assert_no_error(error);
+
+    env[0] = g_strdup_printf("QGA_CONF=%s", conf);
+    env[1] = NULL;
+    g_spawn_sync(NULL, argv, env, 0,
+                 NULL, NULL, &out, &err, &status, &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(err, ==, "");
+    g_assert_cmpint(status, ==, 0);
+
+    kf = g_key_file_new();
+    g_key_file_load_from_data(kf, out, -1, G_KEY_FILE_NONE, &error);
+    g_assert_no_error(error);
+
+    str = g_key_file_get_start_group(kf);
+    g_assert_cmpstr(str, ==, "general");
+    g_free(str);
+
+    g_assert_false(g_key_file_get_boolean(kf, "general", "daemon", &error));
+    g_assert_no_error(error);
+
+    str = g_key_file_get_string(kf, "general", "method", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "virtio-serial");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "path", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/path/to/org.qemu.guest_agent.0");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "pidfile", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/var/foo/qemu-ga.pid");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "statedir", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/var/state");
+    g_free(str);
+
+    g_assert_true(g_key_file_get_boolean(kf, "general", "verbose", &error));
+    g_assert_no_error(error);
+
+    strv = g_key_file_get_string_list(kf, "general", "blacklist", &n, &error);
+    g_assert_cmpint(n, ==, 2);
+#if GLIB_CHECK_VERSION(2, 44, 0)
+    g_assert_true(g_strv_contains((const char * const *)strv,
+                                  "guest-ping"));
+    g_assert_true(g_strv_contains((const char * const *)strv,
+                                  "guest-get-time"));
+#endif
+    g_assert_no_error(error);
+    g_strfreev(strv);
+
+    g_free(out);
+    g_free(err);
+    g_free(conf);
+    g_free(env[0]);
+    g_key_file_free(kf);
+
+    close(tmp);
+}
+
+static void test_qga_fsfreeze_status(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    const gchar *status;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-status'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    status = qdict_get_try_str(ret, "return");
+    g_assert_cmpstr(status, ==, "thawed");
+
+    QDECREF(ret);
+}
+
+static void test_qga_fsfreeze_and_thaw(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    const gchar *status;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-freeze'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-status'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    status = qdict_get_try_str(ret, "return");
+    g_assert_cmpstr(status, ==, "frozen");
+    QDECREF(ret);
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-thaw'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+}
+
+int main(int argc, char **argv)
+{
+    TestFixture fix;
+    int ret;
+
+    setlocale (LC_ALL, "");
+    g_test_init(&argc, &argv, NULL);
+    fixture_setup(&fix, NULL);
+
+    g_test_add_data_func("/qga/sync-delimited", &fix, test_qga_sync_delimited);
+    g_test_add_data_func("/qga/sync", &fix, test_qga_sync);
+    g_test_add_data_func("/qga/ping", &fix, test_qga_ping);
+    g_test_add_data_func("/qga/info", &fix, test_qga_info);
+    g_test_add_data_func("/qga/network-get-interfaces", &fix,
+                         test_qga_network_get_interfaces);
+    g_test_add_data_func("/qga/get-vcpus", &fix, test_qga_get_vcpus);
+    g_test_add_data_func("/qga/get-fsinfo", &fix, test_qga_get_fsinfo);
+    g_test_add_data_func("/qga/get-memory-block-info", &fix,
+                         test_qga_get_memory_block_info);
+    g_test_add_data_func("/qga/get-memory-blocks", &fix,
+                         test_qga_get_memory_blocks);
+    g_test_add_data_func("/qga/file-ops", &fix, test_qga_file_ops);
+    g_test_add_data_func("/qga/file-write-read", &fix, test_qga_file_write_read);
+    g_test_add_data_func("/qga/get-time", &fix, test_qga_get_time);
+    g_test_add_data_func("/qga/invalid-cmd", &fix, test_qga_invalid_cmd);
+    g_test_add_data_func("/qga/fsfreeze-status", &fix,
+                         test_qga_fsfreeze_status);
+
+    g_test_add_data_func("/qga/blacklist", NULL, test_qga_blacklist);
+    g_test_add_data_func("/qga/config", NULL, test_qga_config);
+
+    if (g_getenv("QGA_TEST_SIDE_EFFECTING")) {
+        g_test_add_data_func("/qga/fsfreeze-and-thaw", &fix,
+                             test_qga_fsfreeze_and_thaw);
+        g_test_add_data_func("/qga/set-time", &fix, test_qga_set_time);
+        g_test_add_data_func("/qga/fstrim", &fix, test_qga_fstrim);
+    }
+
+    ret = g_test_run();
+
+    fixture_tear_down(&fix, NULL);
+
+    return ret;
+}
diff --git a/tests/test-qmp-commands.c b/tests/test-qmp-commands.c
index 9918f2306..888fb5ffe 100644
--- a/tests/test-qmp-commands.c
+++ b/tests/test-qmp-commands.c
@@ -25,11 +25,9 @@ UserDefTwo *qmp_user_def_cmd2(UserDefOne *ud1a,
     UserDefOne *ud1d = g_malloc0(sizeof(UserDefOne));
 
     ud1c->string = strdup(ud1a->string);
-    ud1c->base = g_new0(UserDefZero, 1);
-    ud1c->base->integer = ud1a->base->integer;
+    ud1c->integer = ud1a->integer;
     ud1d->string = strdup(has_udb1 ? ud1b->string : "blah0");
-    ud1d->base = g_new0(UserDefZero, 1);
-    ud1d->base->integer = has_udb1 ? ud1b->base->integer : 0;
+    ud1d->integer = has_udb1 ? ud1b->integer : 0;
 
     ret = g_new0(UserDefTwo, 1);
     ret->string0 = strdup("blah1");
@@ -46,11 +44,16 @@ UserDefTwo *qmp_user_def_cmd2(UserDefOne *ud1a,
     return ret;
 }
 
-int64_t qmp_user_def_cmd3(int64_t a, bool has_b, int64_t b, Error **errp)
+int64_t qmp_guest_get_time(int64_t a, bool has_b, int64_t b, Error **errp)
 {
     return a + (has_b ? b : 0);
 }
 
+QObject *qmp_guest_sync(QObject *arg, Error **errp)
+{
+    return arg;
+}
+
 __org_qemu_x_Union1 *qmp___org_qemu_x_command(__org_qemu_x_EnumList *a,
                                               __org_qemu_x_StructList *b,
                                               __org_qemu_x_Union2 *c,
@@ -59,8 +62,8 @@ __org_qemu_x_Union1 *qmp___org_qemu_x_command(__org_qemu_x_EnumList *a,
 {
     __org_qemu_x_Union1 *ret = g_new0(__org_qemu_x_Union1, 1);
 
-    ret->kind = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
-    ret->__org_qemu_x_branch = strdup("blah1");
+    ret->type = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
+    ret->u.__org_qemu_x_branch = strdup("blah1");
 
     return ret;
 }
@@ -155,7 +158,7 @@ static void test_dispatch_cmd_io(void)
 
     qdict_put(args3, "a", qint_from_int(66));
     qdict_put(req, "arguments", args3);
-    qdict_put(req, "execute", qstring_from_str("user_def_cmd3"));
+    qdict_put(req, "execute", qstring_from_str("guest-get-time"));
 
     ret3 = qobject_to_qint(test_qmp_dispatch(req));
     assert(qint_get_int(ret3) == 66);
@@ -171,20 +174,17 @@ static void test_dealloc_types(void)
     UserDefOneList *ud1list;
 
     ud1test = g_malloc0(sizeof(UserDefOne));
-    ud1test->base = g_new0(UserDefZero, 1);
-    ud1test->base->integer = 42;
+    ud1test->integer = 42;
     ud1test->string = g_strdup("hi there 42");
 
     qapi_free_UserDefOne(ud1test);
 
     ud1a = g_malloc0(sizeof(UserDefOne));
-    ud1a->base = g_new0(UserDefZero, 1);
-    ud1a->base->integer = 43;
+    ud1a->integer = 43;
     ud1a->string = g_strdup("hi there 43");
 
     ud1b = g_malloc0(sizeof(UserDefOne));
-    ud1b->base = g_new0(UserDefZero, 1);
-    ud1b->base->integer = 44;
+    ud1b->integer = 44;
     ud1b->string = g_strdup("hi there 44");
 
     ud1list = g_malloc0(sizeof(UserDefOneList));
@@ -225,8 +225,7 @@ static void test_dealloc_partial(void)
     assert(ud2->dict1 == NULL);
 
     /* confirm & release construction error */
-    assert(err != NULL);
-    error_free(err);
+    error_free_or_abort(&err);
 
     /* tear down partial object */
     qapi_free_UserDefTwo(ud2);
diff --git a/tests/test-qmp-event.c b/tests/test-qmp-event.c
index 1ee40e148..035c65cfd 100644
--- a/tests/test-qmp-event.c
+++ b/tests/test-qmp-event.c
@@ -94,7 +94,7 @@ static bool qdict_cmp_simple(QDict *a, QDict *b)
 
 /* This function is hooked as final emit function, which can verify the
    correctness. */
-static void event_test_emit(TEST_QAPIEvent event, QDict *d, Error **errp)
+static void event_test_emit(test_QAPIEvent event, QDict *d, Error **errp)
 {
     QObject *obj;
     QDict *t;
@@ -179,9 +179,7 @@ static void test_event_c(TestEventData *data,
     QDict *d, *d_data, *d_b;
 
     UserDefOne b;
-    UserDefZero z;
-    z.integer = 2;
-    b.base = &z;
+    b.integer = 2;
     b.string = g_strdup("test1");
     b.has_enum1 = false;
 
@@ -209,11 +207,9 @@ static void test_event_d(TestEventData *data,
 {
     UserDefOne struct1;
     EventStructOne a;
-    UserDefZero z;
     QDict *d, *d_data, *d_a, *d_struct1;
 
-    z.integer = 2;
-    struct1.base = &z;
+    struct1.integer = 2;
     struct1.string = g_strdup("test1");
     struct1.has_enum1 = true;
     struct1.enum1 = ENUM_ONE_VALUE1;
diff --git a/tests/test-qmp-input-strict.c b/tests/test-qmp-input-strict.c
index 68f855bdf..f1c2e3ba6 100644
--- a/tests/test-qmp-input-strict.c
+++ b/tests/test-qmp-input-strict.c
@@ -19,6 +19,9 @@
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
 #include "qapi/qmp/types.h"
+#include "test-qmp-introspect.h"
+#include "qmp-introspect.h"
+#include "qapi-visit.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
@@ -37,78 +40,65 @@ static void validate_teardown(TestInputVisitorData *data,
     }
 }
 
-/* This is provided instead of a test setup function so that the JSON
-   string used by the tests are kept in the test functions (and not
-   int main()) */
-static GCC_FMT_ATTR(2, 3)
-Visitor *validate_test_init(TestInputVisitorData *data,
-                             const char *json_string, ...)
+/* The various test_init functions are provided instead of a test setup
+   function so that the JSON string used by the tests are kept in the test
+   functions (and not in main()). */
+static Visitor *validate_test_init_internal(TestInputVisitorData *data,
+                                            const char *json_string,
+                                            va_list *ap)
 {
     Visitor *v;
-    va_list ap;
 
-    va_start(ap, json_string);
-    data->obj = qobject_from_jsonv(json_string, &ap);
-    va_end(ap);
+    validate_teardown(data, NULL);
 
-    g_assert(data->obj != NULL);
+    data->obj = qobject_from_jsonv(json_string, ap);
+    g_assert(data->obj);
 
     data->qiv = qmp_input_visitor_new_strict(data->obj);
-    g_assert(data->qiv != NULL);
+    g_assert(data->qiv);
 
     v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
+    g_assert(v);
 
     return v;
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
+static GCC_FMT_ATTR(2, 3)
+Visitor *validate_test_init(TestInputVisitorData *data,
+                             const char *json_string, ...)
 {
-    Error *err = NULL;
+    Visitor *v;
+    va_list ap;
 
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
+    va_start(ap, json_string);
+    v = validate_test_init_internal(data, json_string, &ap);
+    va_end(ap);
+    return v;
+}
 
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
+/* similar to validate_test_init(), but does not expect a string
+ * literal/format json_string argument and so can be used for
+ * programatically generated strings (and we can't pass in programatically
+ * generated strings via %s format parameters since qobject_from_jsonv()
+ * will wrap those in double-quotes and treat the entire object as a
+ * string)
+ */
+static Visitor *validate_test_init_raw(TestInputVisitorData *data,
+                                       const char *json_string)
+{
+    return validate_test_init_internal(data, json_string, NULL);
 }
 
+
 static void test_validate_struct(TestInputVisitorData *data,
                                   const void *unused)
 {
     TestStruct *p = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(v, &p, NULL, &error_abort);
     g_free(p->string);
     g_free(p);
 }
@@ -117,7 +107,6 @@ static void test_validate_struct_nested(TestInputVisitorData *data,
                                          const void *unused)
 {
     UserDefTwo *udp = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "{ 'string0': 'string0', "
@@ -125,8 +114,7 @@ static void test_validate_struct_nested(TestInputVisitorData *data,
                            "'dict2': { 'userdef': { 'integer': 42, "
                            "'string': 'string' }, 'string': 'string2'}}}");
 
-    visit_type_UserDefTwo(v, &udp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefTwo(v, &udp, NULL, &error_abort);
     qapi_free_UserDefTwo(udp);
 }
 
@@ -134,13 +122,11 @@ static void test_validate_list(TestInputVisitorData *data,
                                 const void *unused)
 {
     UserDefOneList *head = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44 } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefOneList(v, &head, NULL, &error_abort);
     qapi_free_UserDefOneList(head);
 }
 
@@ -149,12 +135,10 @@ static void test_validate_union_native_list(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data, "{ 'type': 'integer', 'data' : [ 1, 2 ] }");
 
-    visit_type_UserDefNativeListUnion(v, &tmp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefNativeListUnion(v, &tmp, NULL, &error_abort);
     qapi_free_UserDefNativeListUnion(tmp);
 }
 
@@ -163,16 +147,14 @@ static void test_validate_union_flat(TestInputVisitorData *data,
 {
     UserDefFlatUnion *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data,
                            "{ 'enum1': 'value1', "
+                           "'integer': 41, "
                            "'string': 'str', "
                            "'boolean': true }");
-    /* TODO when generator bug is fixed, add 'integer': 41 */
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefFlatUnion(v, &tmp, NULL, &error_abort);
     qapi_free_UserDefFlatUnion(tmp);
 }
 
@@ -181,12 +163,10 @@ static void test_validate_alternate(TestInputVisitorData *data,
 {
     UserDefAlternate *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data, "42");
 
-    visit_type_UserDefAlternate(v, &tmp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefAlternate(v, &tmp, NULL, &error_abort);
     qapi_free_UserDefAlternate(tmp);
 }
 
@@ -200,7 +180,7 @@ static void test_validate_fail_struct(TestInputVisitorData *data,
     v = validate_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo', 'extra': 42 }");
 
     visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
     if (p) {
         g_free(p->string);
     }
@@ -217,7 +197,7 @@ static void test_validate_fail_struct_nested(TestInputVisitorData *data,
     v = validate_test_init(data, "{ 'string0': 'string0', 'dict1': { 'string1': 'string1', 'dict2': { 'userdef1': { 'integer': 42, 'string': 'string', 'extra': [42, 23, {'foo':'bar'}] }, 'string2': 'string2'}}}");
 
     visit_type_UserDefTwo(v, &udp, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
     qapi_free_UserDefTwo(udp);
 }
 
@@ -231,7 +211,7 @@ static void test_validate_fail_list(TestInputVisitorData *data,
     v = validate_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44, 'extra': 'ggg' } ]");
 
     visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
     qapi_free_UserDefOneList(head);
 }
 
@@ -246,7 +226,7 @@ static void test_validate_fail_union_native_list(TestInputVisitorData *data,
                            "{ 'type': 'integer', 'data' : [ 'string' ] }");
 
     visit_type_UserDefNativeListUnion(v, &tmp, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
     qapi_free_UserDefNativeListUnion(tmp);
 }
 
@@ -260,7 +240,7 @@ static void test_validate_fail_union_flat(TestInputVisitorData *data,
     v = validate_test_init(data, "{ 'string': 'c', 'integer': 41, 'boolean': true }");
 
     visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
     qapi_free_UserDefFlatUnion(tmp);
 }
 
@@ -272,10 +252,10 @@ static void test_validate_fail_union_flat_no_discrim(TestInputVisitorData *data,
     Visitor *v;
 
     /* test situation where discriminator field ('enum1' here) is missing */
-    v = validate_test_init(data, "{ 'string': 'c', 'string1': 'd', 'string2': 'e' }");
+    v = validate_test_init(data, "{ 'integer': 42, 'string': 'c', 'string1': 'd', 'string2': 'e' }");
 
     visit_type_UserDefFlatUnion2(v, &tmp, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
     qapi_free_UserDefFlatUnion2(tmp);
 }
 
@@ -289,10 +269,31 @@ static void test_validate_fail_alternate(TestInputVisitorData *data,
     v = validate_test_init(data, "3.14");
 
     visit_type_UserDefAlternate(v, &tmp, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
     qapi_free_UserDefAlternate(tmp);
 }
 
+static void do_test_validate_qmp_introspect(TestInputVisitorData *data,
+                                            const char *schema_json)
+{
+    SchemaInfoList *schema = NULL;
+    Visitor *v;
+
+    v = validate_test_init_raw(data, schema_json);
+
+    visit_type_SchemaInfoList(v, &schema, NULL, &error_abort);
+    g_assert(schema);
+
+    qapi_free_SchemaInfoList(schema);
+}
+
+static void test_validate_qmp_introspect(TestInputVisitorData *data,
+                                           const void *unused)
+{
+    do_test_validate_qmp_introspect(data, test_qmp_schema_json);
+    do_test_validate_qmp_introspect(data, qmp_schema_json);
+}
+
 static void validate_test_add(const char *testpath,
                                TestInputVisitorData *data,
                                void (*test_func)(TestInputVisitorData *data, const void *user_data))
@@ -333,6 +334,8 @@ int main(int argc, char **argv)
                       &testdata, test_validate_fail_alternate);
     validate_test_add("/visitor/input-strict/fail/union-native-list",
                       &testdata, test_validate_fail_union_native_list);
+    validate_test_add("/visitor/input-strict/pass/qmp-introspect",
+                      &testdata, test_validate_qmp_introspect);
 
     g_test_run();
 
diff --git a/tests/test-qmp-input-visitor.c b/tests/test-qmp-input-visitor.c
index b96195309..d48ebdd62 100644
--- a/tests/test-qmp-input-visitor.c
+++ b/tests/test-qmp-input-visitor.c
@@ -36,28 +36,39 @@ static void visitor_input_teardown(TestInputVisitorData *data,
     }
 }
 
-/* This is provided instead of a test setup function so that the JSON
-   string used by the tests are kept in the test functions (and not
-   int main()) */
-static GCC_FMT_ATTR(2, 3)
-Visitor *visitor_input_test_init(TestInputVisitorData *data,
-                                 const char *json_string, ...)
+/* The various test_init functions are provided instead of a test setup
+   function so that the JSON string used by the tests are kept in the test
+   functions (and not in main()). */
+static Visitor *visitor_input_test_init_internal(TestInputVisitorData *data,
+                                                 const char *json_string,
+                                                 va_list *ap)
 {
     Visitor *v;
-    va_list ap;
 
-    va_start(ap, json_string);
-    data->obj = qobject_from_jsonv(json_string, &ap);
-    va_end(ap);
+    visitor_input_teardown(data, NULL);
 
-    g_assert(data->obj != NULL);
+    data->obj = qobject_from_jsonv(json_string, ap);
+    g_assert(data->obj);
 
     data->qiv = qmp_input_visitor_new(data->obj);
-    g_assert(data->qiv != NULL);
+    g_assert(data->qiv);
 
     v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
+    g_assert(v);
+
+    return v;
+}
+
+static GCC_FMT_ATTR(2, 3)
+Visitor *visitor_input_test_init(TestInputVisitorData *data,
+                                 const char *json_string, ...)
+{
+    Visitor *v;
+    va_list ap;
 
+    va_start(ap, json_string);
+    v = visitor_input_test_init_internal(data, json_string, &ap);
+    va_end(ap);
     return v;
 }
 
@@ -71,32 +82,18 @@ Visitor *visitor_input_test_init(TestInputVisitorData *data,
 static Visitor *visitor_input_test_init_raw(TestInputVisitorData *data,
                                             const char *json_string)
 {
-    Visitor *v;
-
-    data->obj = qobject_from_json(json_string);
-
-    g_assert(data->obj != NULL);
-
-    data->qiv = qmp_input_visitor_new(data->obj);
-    g_assert(data->qiv != NULL);
-
-    v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
-
-    return v;
+    return visitor_input_test_init_internal(data, json_string, NULL);
 }
 
 static void test_visitor_in_int(TestInputVisitorData *data,
                                 const void *unused)
 {
     int64_t res = 0, value = -42;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%" PRId64, value);
 
-    visit_type_int(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_int(v, &res, NULL, &error_abort);
     g_assert_cmpint(res, ==, value);
 }
 
@@ -114,21 +111,18 @@ static void test_visitor_in_int_overflow(TestInputVisitorData *data,
     v = visitor_input_test_init(data, "%f", DBL_MAX);
 
     visit_type_int(v, &res, NULL, &err);
-    g_assert(err);
-    error_free(err);
+    error_free_or_abort(&err);
 }
 
 static void test_visitor_in_bool(TestInputVisitorData *data,
                                  const void *unused)
 {
-    Error *err = NULL;
     bool res = false;
     Visitor *v;
 
     v = visitor_input_test_init(data, "true");
 
-    visit_type_bool(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_bool(v, &res, NULL, &error_abort);
     g_assert_cmpint(res, ==, true);
 }
 
@@ -136,13 +130,11 @@ static void test_visitor_in_number(TestInputVisitorData *data,
                                    const void *unused)
 {
     double res = 0, value = 3.14;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%f", value);
 
-    visit_type_number(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_number(v, &res, NULL, &error_abort);
     g_assert_cmpfloat(res, ==, value);
 }
 
@@ -150,13 +142,11 @@ static void test_visitor_in_string(TestInputVisitorData *data,
                                    const void *unused)
 {
     char *res = NULL, *value = (char *) "Q E M U";
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%s", value);
 
-    visit_type_str(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_str(v, &res, NULL, &error_abort);
     g_assert_cmpstr(res, ==, value);
 
     g_free(res);
@@ -165,7 +155,6 @@ static void test_visitor_in_string(TestInputVisitorData *data,
 static void test_visitor_in_enum(TestInputVisitorData *data,
                                  const void *unused)
 {
-    Error *err = NULL;
     Visitor *v;
     EnumOne i;
 
@@ -174,63 +163,21 @@ static void test_visitor_in_enum(TestInputVisitorData *data,
 
         v = visitor_input_test_init(data, "%s", EnumOne_lookup[i]);
 
-        visit_type_EnumOne(v, &res, NULL, &err);
-        g_assert(!err);
+        visit_type_EnumOne(v, &res, NULL, &error_abort);
         g_assert_cmpint(i, ==, res);
-
-        visitor_input_teardown(data, NULL);
     }
-
-    data->obj = NULL;
-    data->qiv = NULL;
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static void test_visitor_in_struct(TestInputVisitorData *data,
                                    const void *unused)
 {
     TestStruct *p = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(v, &p, NULL, &error_abort);
     g_assert_cmpint(p->integer, ==, -42);
     g_assert(p->boolean == true);
     g_assert_cmpstr(p->string, ==, "foo");
@@ -239,17 +186,10 @@ static void test_visitor_in_struct(TestInputVisitorData *data,
     g_free(p);
 }
 
-static void check_and_free_str(char *str, const char *cmp)
-{
-    g_assert_cmpstr(str, ==, cmp);
-    g_free(str);
-}
-
 static void test_visitor_in_struct_nested(TestInputVisitorData *data,
                                           const void *unused)
 {
     UserDefTwo *udp = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "{ 'string0': 'string0', "
@@ -257,34 +197,28 @@ static void test_visitor_in_struct_nested(TestInputVisitorData *data,
                                 "'dict2': { 'userdef': { 'integer': 42, "
                                 "'string': 'string' }, 'string': 'string2'}}}");
 
-    visit_type_UserDefTwo(v, &udp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefTwo(v, &udp, NULL, &error_abort);
 
-    check_and_free_str(udp->string0, "string0");
-    check_and_free_str(udp->dict1->string1, "string1");
-    g_assert_cmpint(udp->dict1->dict2->userdef->base->integer, ==, 42);
-    check_and_free_str(udp->dict1->dict2->userdef->string, "string");
-    check_and_free_str(udp->dict1->dict2->string, "string2");
+    g_assert_cmpstr(udp->string0, ==, "string0");
+    g_assert_cmpstr(udp->dict1->string1, ==, "string1");
+    g_assert_cmpint(udp->dict1->dict2->userdef->integer, ==, 42);
+    g_assert_cmpstr(udp->dict1->dict2->userdef->string, ==, "string");
+    g_assert_cmpstr(udp->dict1->dict2->string, ==, "string2");
     g_assert(udp->dict1->has_dict3 == false);
 
-    g_free(udp->dict1->dict2->userdef);
-    g_free(udp->dict1->dict2);
-    g_free(udp->dict1);
-    g_free(udp);
+    qapi_free_UserDefTwo(udp);
 }
 
 static void test_visitor_in_list(TestInputVisitorData *data,
                                  const void *unused)
 {
     UserDefOneList *item, *head = NULL;
-    Error *err = NULL;
     Visitor *v;
     int i;
 
     v = visitor_input_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44 } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefOneList(v, &head, NULL, &error_abort);
     g_assert(head != NULL);
 
     for (i = 0, item = head; item; item = item->next, i++) {
@@ -292,31 +226,80 @@ static void test_visitor_in_list(TestInputVisitorData *data,
 
         snprintf(string, sizeof(string), "string%d", i);
         g_assert_cmpstr(item->value->string, ==, string);
-        g_assert_cmpint(item->value->base->integer, ==, 42 + i);
+        g_assert_cmpint(item->value->integer, ==, 42 + i);
     }
 
     qapi_free_UserDefOneList(head);
+    head = NULL;
+
+    /* An empty list is valid */
+    v = visitor_input_test_init(data, "[]");
+    visit_type_UserDefOneList(v, &head, NULL, &error_abort);
+    g_assert(!head);
+}
+
+static void test_visitor_in_any(TestInputVisitorData *data,
+                                const void *unused)
+{
+    QObject *res = NULL;
+    Visitor *v;
+    QInt *qint;
+    QBool *qbool;
+    QString *qstring;
+    QDict *qdict;
+    QObject *qobj;
+
+    v = visitor_input_test_init(data, "-42");
+    visit_type_any(v, &res, NULL, &error_abort);
+    qint = qobject_to_qint(res);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobject_decref(res);
+
+    v = visitor_input_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
+    visit_type_any(v, &res, NULL, &error_abort);
+    qdict = qobject_to_qdict(res);
+    g_assert(qdict && qdict_size(qdict) == 3);
+    qobj = qdict_get(qdict, "integer");
+    g_assert(qobj);
+    qint = qobject_to_qint(qobj);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobj = qdict_get(qdict, "boolean");
+    g_assert(qobj);
+    qbool = qobject_to_qbool(qobj);
+    g_assert(qbool);
+    g_assert(qbool_get_bool(qbool) == true);
+    qobj = qdict_get(qdict, "string");
+    g_assert(qobj);
+    qstring = qobject_to_qstring(qobj);
+    g_assert(qstring);
+    g_assert_cmpstr(qstring_get_str(qstring), ==, "foo");
+    qobject_decref(res);
 }
 
 static void test_visitor_in_union_flat(TestInputVisitorData *data,
                                        const void *unused)
 {
     Visitor *v;
-    Error *err = NULL;
     UserDefFlatUnion *tmp;
+    UserDefUnionBase *base;
 
     v = visitor_input_test_init(data,
                                 "{ 'enum1': 'value1', "
+                                "'integer': 41, "
                                 "'string': 'str', "
                                 "'boolean': true }");
-    /* TODO when generator bug is fixed, add 'integer': 41 */
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    g_assert_cmpint(tmp->kind, ==, ENUM_ONE_VALUE1);
+    visit_type_UserDefFlatUnion(v, &tmp, NULL, &error_abort);
+    g_assert_cmpint(tmp->enum1, ==, ENUM_ONE_VALUE1);
     g_assert_cmpstr(tmp->string, ==, "str");
-    /* TODO g_assert_cmpint(tmp->integer, ==, 41); */
-    g_assert_cmpint(tmp->value1->boolean, ==, true);
+    g_assert_cmpint(tmp->integer, ==, 41);
+    g_assert_cmpint(tmp->u.value1->boolean, ==, true);
+
+    base = qapi_UserDefFlatUnion_base(tmp);
+    g_assert(&base->enum1 == &tmp->enum1);
+
     qapi_free_UserDefFlatUnion(tmp);
 }
 
@@ -328,20 +311,117 @@ static void test_visitor_in_alternate(TestInputVisitorData *data,
     UserDefAlternate *tmp;
 
     v = visitor_input_test_init(data, "42");
+    visit_type_UserDefAlternate(v, &tmp, NULL, &error_abort);
+    g_assert_cmpint(tmp->type, ==, USER_DEF_ALTERNATE_KIND_I);
+    g_assert_cmpint(tmp->u.i, ==, 42);
+    qapi_free_UserDefAlternate(tmp);
 
+    v = visitor_input_test_init(data, "'string'");
+    visit_type_UserDefAlternate(v, &tmp, NULL, &error_abort);
+    g_assert_cmpint(tmp->type, ==, USER_DEF_ALTERNATE_KIND_S);
+    g_assert_cmpstr(tmp->u.s, ==, "string");
+    qapi_free_UserDefAlternate(tmp);
+
+    v = visitor_input_test_init(data, "false");
     visit_type_UserDefAlternate(v, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    g_assert_cmpint(tmp->kind, ==, USER_DEF_ALTERNATE_KIND_I);
-    g_assert_cmpint(tmp->i, ==, 42);
+    error_free_or_abort(&err);
     qapi_free_UserDefAlternate(tmp);
 }
 
+static void test_visitor_in_alternate_number(TestInputVisitorData *data,
+                                             const void *unused)
+{
+    Visitor *v;
+    Error *err = NULL;
+    AltStrBool *asb;
+    AltStrNum *asn;
+    AltNumStr *ans;
+    AltStrInt *asi;
+    AltIntNum *ain;
+    AltNumInt *ani;
+
+    /* Parsing an int */
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrBool(v, &asb, NULL, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrBool(asb);
+
+    /* FIXME: Order of alternate should not affect semantics; asn should
+     * parse the same as ans */
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrNum(v, &asn, NULL, &err);
+    /* FIXME g_assert_cmpint(asn->type, == ALT_STR_NUM_KIND_N); */
+    /* FIXME g_assert_cmpfloat(asn->u.n, ==, 42); */
+    error_free_or_abort(&err);
+    qapi_free_AltStrNum(asn);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltNumStr(v, &ans, NULL, &error_abort);
+    g_assert_cmpint(ans->type, ==, ALT_NUM_STR_KIND_N);
+    g_assert_cmpfloat(ans->u.n, ==, 42);
+    qapi_free_AltNumStr(ans);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrInt(v, &asi, NULL, &error_abort);
+    g_assert_cmpint(asi->type, ==, ALT_STR_INT_KIND_I);
+    g_assert_cmpint(asi->u.i, ==, 42);
+    qapi_free_AltStrInt(asi);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltIntNum(v, &ain, NULL, &error_abort);
+    g_assert_cmpint(ain->type, ==, ALT_INT_NUM_KIND_I);
+    g_assert_cmpint(ain->u.i, ==, 42);
+    qapi_free_AltIntNum(ain);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltNumInt(v, &ani, NULL, &error_abort);
+    g_assert_cmpint(ani->type, ==, ALT_NUM_INT_KIND_I);
+    g_assert_cmpint(ani->u.i, ==, 42);
+    qapi_free_AltNumInt(ani);
+
+    /* Parsing a double */
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrBool(v, &asb, NULL, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrBool(asb);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrNum(v, &asn, NULL, &error_abort);
+    g_assert_cmpint(asn->type, ==, ALT_STR_NUM_KIND_N);
+    g_assert_cmpfloat(asn->u.n, ==, 42.5);
+    qapi_free_AltStrNum(asn);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltNumStr(v, &ans, NULL, &error_abort);
+    g_assert_cmpint(ans->type, ==, ALT_NUM_STR_KIND_N);
+    g_assert_cmpfloat(ans->u.n, ==, 42.5);
+    qapi_free_AltNumStr(ans);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrInt(v, &asi, NULL, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrInt(asi);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltIntNum(v, &ain, NULL, &error_abort);
+    g_assert_cmpint(ain->type, ==, ALT_INT_NUM_KIND_N);
+    g_assert_cmpfloat(ain->u.n, ==, 42.5);
+    qapi_free_AltIntNum(ain);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltNumInt(v, &ani, NULL, &error_abort);
+    g_assert_cmpint(ani->type, ==, ALT_NUM_INT_KIND_N);
+    g_assert_cmpfloat(ani->u.n, ==, 42.5);
+    qapi_free_AltNumInt(ani);
+}
+
 static void test_native_list_integer_helper(TestInputVisitorData *data,
                                             const void *unused,
                                             UserDefNativeListUnionKind kind)
 {
     UserDefNativeListUnion *cvalue = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -358,71 +438,70 @@ static void test_native_list_integer_helper(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, kind);
+    g_assert_cmpint(cvalue->type, ==, kind);
 
     switch (kind) {
     case USER_DEF_NATIVE_LIST_UNION_KIND_INTEGER: {
         intList *elem = NULL;
-        for (i = 0, elem = cvalue->integer; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.integer; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S8: {
         int8List *elem = NULL;
-        for (i = 0, elem = cvalue->s8; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s8; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S16: {
         int16List *elem = NULL;
-        for (i = 0, elem = cvalue->s16; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s16; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S32: {
         int32List *elem = NULL;
-        for (i = 0, elem = cvalue->s32; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s32; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S64: {
         int64List *elem = NULL;
-        for (i = 0, elem = cvalue->s64; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s64; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U8: {
         uint8List *elem = NULL;
-        for (i = 0, elem = cvalue->u8; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u8; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U16: {
         uint16List *elem = NULL;
-        for (i = 0, elem = cvalue->u16; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u16; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U32: {
         uint32List *elem = NULL;
-        for (i = 0, elem = cvalue->u32; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u32; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U64: {
         uint64List *elem = NULL;
-        for (i = 0, elem = cvalue->u64; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u64; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
@@ -504,7 +583,6 @@ static void test_visitor_in_native_list_bool(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *cvalue = NULL;
     boolList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -521,12 +599,11 @@ static void test_visitor_in_native_list_bool(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN);
 
-    for (i = 0, elem = cvalue->boolean; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.boolean; elem; elem = elem->next, i++) {
         g_assert_cmpint(elem->value, ==, (i % 3 == 0) ? 1 : 0);
     }
 
@@ -540,7 +617,6 @@ static void test_visitor_in_native_list_string(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *cvalue = NULL;
     strList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -556,12 +632,11 @@ static void test_visitor_in_native_list_string(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_STRING);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_STRING);
 
-    for (i = 0, elem = cvalue->string; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.string; elem; elem = elem->next, i++) {
         gchar str[8];
         sprintf(str, "%d", i);
         g_assert_cmpstr(elem->value, ==, str);
@@ -579,7 +654,6 @@ static void test_visitor_in_native_list_number(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *cvalue = NULL;
     numberList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -595,12 +669,11 @@ static void test_visitor_in_native_list_number(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER);
 
-    for (i = 0, elem = cvalue->number; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.number; elem; elem = elem->next, i++) {
         GString *double_expected = g_string_new("");
         GString *double_actual = g_string_new("");
 
@@ -631,16 +704,69 @@ static void test_visitor_in_errors(TestInputVisitorData *data,
     TestStruct *p = NULL;
     Error *err = NULL;
     Visitor *v;
+    strList *q = NULL;
 
-    v = visitor_input_test_init(data, "{ 'integer': false, 'boolean': 'foo', 'string': -42 }");
+    v = visitor_input_test_init(data, "{ 'integer': false, 'boolean': 'foo', "
+                                "'string': -42 }");
 
     visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(err);
+    error_free_or_abort(&err);
+    /* FIXME - a failed parse should not leave a partially-allocated p
+     * for us to clean up; this could cause callers to leak memory. */
     g_assert(p->string == NULL);
 
-    error_free(err);
     g_free(p->string);
     g_free(p);
+
+    v = visitor_input_test_init(data, "[ '1', '2', false, '3' ]");
+    visit_type_strList(v, &q, NULL, &err);
+    error_free_or_abort(&err);
+    assert(q);
+    qapi_free_strList(q);
+}
+
+static void test_visitor_in_wrong_type(TestInputVisitorData *data,
+                                       const void *unused)
+{
+    TestStruct *p = NULL;
+    Visitor *v;
+    strList *q = NULL;
+    int64_t i;
+    Error *err = NULL;
+
+    /* Make sure arrays and structs cannot be confused */
+
+    v = visitor_input_test_init(data, "[]");
+    visit_type_TestStruct(v, &p, NULL, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
+
+    v = visitor_input_test_init(data, "{}");
+    visit_type_strList(v, &q, NULL, &err);
+    error_free_or_abort(&err);
+    assert(!q);
+
+    /* Make sure primitives and struct cannot be confused */
+
+    v = visitor_input_test_init(data, "1");
+    visit_type_TestStruct(v, &p, NULL, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
+
+    v = visitor_input_test_init(data, "{}");
+    visit_type_int(v, &i, NULL, &err);
+    error_free_or_abort(&err);
+
+    /* Make sure primitives and arrays cannot be confused */
+
+    v = visitor_input_test_init(data, "1");
+    visit_type_strList(v, &q, NULL, &err);
+    error_free_or_abort(&err);
+    assert(!q);
+
+    v = visitor_input_test_init(data, "[]");
+    visit_type_int(v, &i, NULL, &err);
+    error_free_or_abort(&err);
 }
 
 int main(int argc, char **argv)
@@ -667,12 +793,18 @@ int main(int argc, char **argv)
                            &in_visitor_data, test_visitor_in_struct_nested);
     input_visitor_test_add("/visitor/input/list",
                            &in_visitor_data, test_visitor_in_list);
+    input_visitor_test_add("/visitor/input/any",
+                           &in_visitor_data, test_visitor_in_any);
     input_visitor_test_add("/visitor/input/union-flat",
                            &in_visitor_data, test_visitor_in_union_flat);
     input_visitor_test_add("/visitor/input/alternate",
                            &in_visitor_data, test_visitor_in_alternate);
     input_visitor_test_add("/visitor/input/errors",
                            &in_visitor_data, test_visitor_in_errors);
+    input_visitor_test_add("/visitor/input/wrong-type",
+                           &in_visitor_data, test_visitor_in_wrong_type);
+    input_visitor_test_add("/visitor/input/alternate-number",
+                           &in_visitor_data, test_visitor_in_alternate_number);
     input_visitor_test_add("/visitor/input/native_list/int",
                            &in_visitor_data,
                            test_visitor_in_native_list_int);
diff --git a/tests/test-qmp-output-visitor.c b/tests/test-qmp-output-visitor.c
index 87ba350b4..5609bb8df 100644
--- a/tests/test-qmp-output-visitor.c
+++ b/tests/test-qmp-output-visitor.c
@@ -45,11 +45,9 @@ static void test_visitor_out_int(TestOutputVisitorData *data,
                                  const void *unused)
 {
     int64_t value = -42;
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_int(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_int(data->ov, &value, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -62,12 +60,10 @@ static void test_visitor_out_int(TestOutputVisitorData *data,
 static void test_visitor_out_bool(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    Error *err = NULL;
     bool value = true;
     QObject *obj;
 
-    visit_type_bool(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_bool(data->ov, &value, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -81,11 +77,9 @@ static void test_visitor_out_number(TestOutputVisitorData *data,
                                     const void *unused)
 {
     double value = 3.14;
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_number(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_number(data->ov, &value, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -99,11 +93,9 @@ static void test_visitor_out_string(TestOutputVisitorData *data,
                                     const void *unused)
 {
     char *string = (char *) "Q E M U";
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_str(data->ov, &string, NULL, &err);
-    g_assert(!err);
+    visit_type_str(data->ov, &string, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -117,12 +109,10 @@ static void test_visitor_out_no_string(TestOutputVisitorData *data,
                                        const void *unused)
 {
     char *string = NULL;
-    Error *err = NULL;
     QObject *obj;
 
     /* A null string should return "" */
-    visit_type_str(data->ov, &string, NULL, &err);
-    g_assert(!err);
+    visit_type_str(data->ov, &string, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -135,13 +125,11 @@ static void test_visitor_out_no_string(TestOutputVisitorData *data,
 static void test_visitor_out_enum(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    Error *err = NULL;
     QObject *obj;
     EnumOne i;
 
     for (i = 0; i < ENUM_ONE_MAX; i++) {
-        visit_type_EnumOne(data->ov, &i, "unused", &err);
-        g_assert(!err);
+        visit_type_EnumOne(data->ov, &i, "unused", &error_abort);
 
         obj = qmp_output_get_qobject(data->qov);
         g_assert(obj != NULL);
@@ -166,41 +154,6 @@ static void test_visitor_out_enum_errors(TestOutputVisitorData *data,
     }
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
-
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static void test_visitor_out_struct(TestOutputVisitorData *data,
                                     const void *unused)
@@ -209,12 +162,10 @@ static void test_visitor_out_struct(TestOutputVisitorData *data,
                                .boolean = false,
                                .string = (char *) "foo"};
     TestStruct *p = &test_struct;
-    Error *err = NULL;
     QObject *obj;
     QDict *qdict;
 
-    visit_type_TestStruct(data->ov, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(data->ov, &p, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -233,7 +184,6 @@ static void test_visitor_out_struct_nested(TestOutputVisitorData *data,
                                            const void *unused)
 {
     int64_t value = 42;
-    Error *err = NULL;
     UserDefTwo *ud2;
     QObject *obj;
     QDict *qdict, *dict1, *dict2, *dict3, *userdef;
@@ -250,20 +200,17 @@ static void test_visitor_out_struct_nested(TestOutputVisitorData *data,
     ud2->dict1->dict2 = g_malloc0(sizeof(*ud2->dict1->dict2));
     ud2->dict1->dict2->userdef = g_new0(UserDefOne, 1);
     ud2->dict1->dict2->userdef->string = g_strdup(string);
-    ud2->dict1->dict2->userdef->base = g_new0(UserDefZero, 1);
-    ud2->dict1->dict2->userdef->base->integer = value;
+    ud2->dict1->dict2->userdef->integer = value;
     ud2->dict1->dict2->string = g_strdup(strings[2]);
 
     ud2->dict1->dict3 = g_malloc0(sizeof(*ud2->dict1->dict3));
     ud2->dict1->has_dict3 = true;
     ud2->dict1->dict3->userdef = g_new0(UserDefOne, 1);
     ud2->dict1->dict3->userdef->string = g_strdup(string);
-    ud2->dict1->dict3->userdef->base = g_new0(UserDefZero, 1);
-    ud2->dict1->dict3->userdef->base->integer = value;
+    ud2->dict1->dict3->userdef->integer = value;
     ud2->dict1->dict3->string = g_strdup(strings[3]);
 
-    visit_type_UserDefTwo(data->ov, &ud2, "unused", &err);
-    g_assert(!err);
+    visit_type_UserDefTwo(data->ov, &ud2, "unused", &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -301,8 +248,8 @@ static void test_visitor_out_struct_errors(TestOutputVisitorData *data,
                                            const void *unused)
 {
     EnumOne bad_values[] = { ENUM_ONE_MAX, -1 };
-    UserDefZero b;
-    UserDefOne u = { .base = &b }, *pu = &u;
+    UserDefOne u = {0};
+    UserDefOne *pu = &u;
     Error *err;
     int i;
 
@@ -316,57 +263,33 @@ static void test_visitor_out_struct_errors(TestOutputVisitorData *data,
     }
 }
 
-typedef struct TestStructList
-{
-    union {
-        TestStruct *value;
-        uint64_t padding;
-    };
-    struct TestStructList *next;
-} TestStructList;
-
-static void visit_type_TestStructList(Visitor *v, TestStructList **obj,
-                                      const char *name, Error **errp)
-{
-    GenericList *i, **head = (GenericList **)obj;
-
-    visit_start_list(v, name, errp);
-
-    for (*head = i = visit_next_list(v, head, errp); i; i = visit_next_list(v, &i, errp)) {
-        TestStructList *native_i = (TestStructList *)i;
-        visit_type_TestStruct(v, &native_i->value, NULL, errp);
-    }
-
-    visit_end_list(v, errp);
-}
 
 static void test_visitor_out_list(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    char *value_str = (char *) "list value";
+    const char *value_str = "list value";
     TestStructList *p, *head = NULL;
     const int max_items = 10;
     bool value_bool = true;
     int value_int = 10;
-    Error *err = NULL;
     QListEntry *entry;
     QObject *obj;
     QList *qlist;
     int i;
 
+    /* Build the list in reverse order... */
     for (i = 0; i < max_items; i++) {
         p = g_malloc0(sizeof(*p));
         p->value = g_malloc0(sizeof(*p->value));
-        p->value->integer = value_int;
+        p->value->integer = value_int + (max_items - i - 1);
         p->value->boolean = value_bool;
-        p->value->string = value_str;
+        p->value->string = g_strdup(value_str);
 
         p->next = head;
         head = p;
     }
 
-    visit_type_TestStructList(data->ov, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStructList(data->ov, &head, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -375,6 +298,7 @@ static void test_visitor_out_list(TestOutputVisitorData *data,
     qlist = qobject_to_qlist(obj);
     g_assert(!qlist_empty(qlist));
 
+    /* ...and ensure that the visitor sees it in order */
     i = 0;
     QLIST_FOREACH_ENTRY(qlist, entry) {
         QDict *qdict;
@@ -382,7 +306,7 @@ static void test_visitor_out_list(TestOutputVisitorData *data,
         g_assert(qobject_type(entry->value) == QTYPE_QDICT);
         qdict = qobject_to_qdict(entry->value);
         g_assert_cmpint(qdict_size(qdict), ==, 3);
-        g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, value_int);
+        g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, value_int + i);
         g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, value_bool);
         g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, value_str);
         i++;
@@ -390,13 +314,7 @@ static void test_visitor_out_list(TestOutputVisitorData *data,
     g_assert_cmpint(i, ==, max_items);
 
     QDECREF(qlist);
-
-    for (p = head; p;) {
-        TestStructList *tmp = p->next;
-        g_free(p->value);
-        g_free(p);
-        p = tmp;
-    }
+    qapi_free_TestStructList(head);
 }
 
 static void test_visitor_out_list_qapi_free(TestOutputVisitorData *data,
@@ -416,8 +334,7 @@ static void test_visitor_out_list_qapi_free(TestOutputVisitorData *data,
         p->value->dict1->dict2 = g_new0(UserDefTwoDictDict, 1);
         p->value->dict1->dict2->userdef = g_new0(UserDefOne, 1);
         p->value->dict1->dict2->userdef->string = g_strdup(string);
-        p->value->dict1->dict2->userdef->base = g_new0(UserDefZero, 1);
-        p->value->dict1->dict2->userdef->base->integer = 42;
+        p->value->dict1->dict2->userdef->integer = 42;
         p->value->dict1->dict2->string = g_strdup(string);
         p->value->dict1->has_dict3 = false;
 
@@ -428,23 +345,68 @@ static void test_visitor_out_list_qapi_free(TestOutputVisitorData *data,
     qapi_free_UserDefTwoList(head);
 }
 
+static void test_visitor_out_any(TestOutputVisitorData *data,
+                                 const void *unused)
+{
+    QObject *qobj;
+    QInt *qint;
+    QBool *qbool;
+    QString *qstring;
+    QDict *qdict;
+    QObject *obj;
+
+    qobj = QOBJECT(qint_from_int(-42));
+    visit_type_any(data->ov, &qobj, NULL, &error_abort);
+    obj = qmp_output_get_qobject(data->qov);
+    g_assert(obj != NULL);
+    g_assert(qobject_type(obj) == QTYPE_QINT);
+    g_assert_cmpint(qint_get_int(qobject_to_qint(obj)), ==, -42);
+    qobject_decref(obj);
+    qobject_decref(qobj);
+
+    qdict = qdict_new();
+    qdict_put(qdict, "integer", qint_from_int(-42));
+    qdict_put(qdict, "boolean", qbool_from_bool(true));
+    qdict_put(qdict, "string", qstring_from_str("foo"));
+    qobj = QOBJECT(qdict);
+    visit_type_any(data->ov, &qobj, NULL, &error_abort);
+    qobject_decref(qobj);
+    obj = qmp_output_get_qobject(data->qov);
+    g_assert(obj != NULL);
+    qdict = qobject_to_qdict(obj);
+    g_assert(qdict);
+    qobj = qdict_get(qdict, "integer");
+    g_assert(qobj);
+    qint = qobject_to_qint(qobj);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobj = qdict_get(qdict, "boolean");
+    g_assert(qobj);
+    qbool = qobject_to_qbool(qobj);
+    g_assert(qbool);
+    g_assert(qbool_get_bool(qbool) == true);
+    qobj = qdict_get(qdict, "string");
+    g_assert(qobj);
+    qstring = qobject_to_qstring(qobj);
+    g_assert(qstring);
+    g_assert_cmpstr(qstring_get_str(qstring), ==, "foo");
+    qobject_decref(obj);
+}
+
 static void test_visitor_out_union_flat(TestOutputVisitorData *data,
                                         const void *unused)
 {
     QObject *arg;
     QDict *qdict;
 
-    Error *err = NULL;
-
     UserDefFlatUnion *tmp = g_malloc0(sizeof(UserDefFlatUnion));
-    tmp->kind = ENUM_ONE_VALUE1;
+    tmp->enum1 = ENUM_ONE_VALUE1;
     tmp->string = g_strdup("str");
-    tmp->value1 = g_malloc0(sizeof(UserDefA));
-    /* TODO when generator bug is fixed: tmp->integer = 41; */
-    tmp->value1->boolean = true;
+    tmp->u.value1 = g_malloc0(sizeof(UserDefA));
+    tmp->integer = 41;
+    tmp->u.value1->boolean = true;
 
-    visit_type_UserDefFlatUnion(data->ov, &tmp, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefFlatUnion(data->ov, &tmp, NULL, &error_abort);
     arg = qmp_output_get_qobject(data->qov);
 
     g_assert(qobject_type(arg) == QTYPE_QDICT);
@@ -452,7 +414,7 @@ static void test_visitor_out_union_flat(TestOutputVisitorData *data,
 
     g_assert_cmpstr(qdict_get_str(qdict, "enum1"), ==, "value1");
     g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, "str");
-    /* TODO g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 41); */
+    g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 41);
     g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, true);
 
     qapi_free_UserDefFlatUnion(tmp);
@@ -463,20 +425,33 @@ static void test_visitor_out_alternate(TestOutputVisitorData *data,
                                        const void *unused)
 {
     QObject *arg;
-    Error *err = NULL;
+    UserDefAlternate *tmp;
 
-    UserDefAlternate *tmp = g_malloc0(sizeof(UserDefAlternate));
-    tmp->kind = USER_DEF_ALTERNATE_KIND_I;
-    tmp->i = 42;
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = USER_DEF_ALTERNATE_KIND_I;
+    tmp->u.i = 42;
 
-    visit_type_UserDefAlternate(data->ov, &tmp, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefAlternate(data->ov, &tmp, NULL, &error_abort);
     arg = qmp_output_get_qobject(data->qov);
 
     g_assert(qobject_type(arg) == QTYPE_QINT);
     g_assert_cmpint(qint_get_int(qobject_to_qint(arg)), ==, 42);
 
     qapi_free_UserDefAlternate(tmp);
+    qobject_decref(arg);
+
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = USER_DEF_ALTERNATE_KIND_S;
+    tmp->u.s = g_strdup("hello");
+
+    visit_type_UserDefAlternate(data->ov, &tmp, NULL, &error_abort);
+    arg = qmp_output_get_qobject(data->qov);
+
+    g_assert(qobject_type(arg) == QTYPE_QSTRING);
+    g_assert_cmpstr(qstring_get_str(qobject_to_qstring(arg)), ==, "hello");
+
+    qapi_free_UserDefAlternate(tmp);
+    qobject_decref(arg);
 }
 
 static void test_visitor_out_empty(TestOutputVisitorData *data,
@@ -485,15 +460,18 @@ static void test_visitor_out_empty(TestOutputVisitorData *data,
     QObject *arg;
 
     arg = qmp_output_get_qobject(data->qov);
-    g_assert(!arg);
+    g_assert(qobject_type(arg) == QTYPE_QNULL);
+    /* Check that qnull reference counting is sane */
+    g_assert(arg->refcnt == 2);
+    qobject_decref(arg);
 }
 
 static void init_native_list(UserDefNativeListUnion *cvalue)
 {
     int i;
-    switch (cvalue->kind) {
+    switch (cvalue->type) {
     case USER_DEF_NATIVE_LIST_UNION_KIND_INTEGER: {
-        intList **list = &cvalue->integer;
+        intList **list = &cvalue->u.integer;
         for (i = 0; i < 32; i++) {
             *list = g_new0(intList, 1);
             (*list)->value = i;
@@ -503,7 +481,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S8: {
-        int8List **list = &cvalue->s8;
+        int8List **list = &cvalue->u.s8;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int8List, 1);
             (*list)->value = i;
@@ -513,7 +491,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S16: {
-        int16List **list = &cvalue->s16;
+        int16List **list = &cvalue->u.s16;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int16List, 1);
             (*list)->value = i;
@@ -523,7 +501,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S32: {
-        int32List **list = &cvalue->s32;
+        int32List **list = &cvalue->u.s32;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int32List, 1);
             (*list)->value = i;
@@ -533,7 +511,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S64: {
-        int64List **list = &cvalue->s64;
+        int64List **list = &cvalue->u.s64;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int64List, 1);
             (*list)->value = i;
@@ -543,7 +521,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U8: {
-        uint8List **list = &cvalue->u8;
+        uint8List **list = &cvalue->u.u8;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint8List, 1);
             (*list)->value = i;
@@ -553,7 +531,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U16: {
-        uint16List **list = &cvalue->u16;
+        uint16List **list = &cvalue->u.u16;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint16List, 1);
             (*list)->value = i;
@@ -563,7 +541,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U32: {
-        uint32List **list = &cvalue->u32;
+        uint32List **list = &cvalue->u.u32;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint32List, 1);
             (*list)->value = i;
@@ -573,7 +551,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U64: {
-        uint64List **list = &cvalue->u64;
+        uint64List **list = &cvalue->u.u64;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint64List, 1);
             (*list)->value = i;
@@ -583,7 +561,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN: {
-        boolList **list = &cvalue->boolean;
+        boolList **list = &cvalue->u.boolean;
         for (i = 0; i < 32; i++) {
             *list = g_new0(boolList, 1);
             (*list)->value = (i % 3 == 0);
@@ -593,7 +571,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_STRING: {
-        strList **list = &cvalue->string;
+        strList **list = &cvalue->u.string;
         for (i = 0; i < 32; i++) {
             *list = g_new0(strList, 1);
             (*list)->value = g_strdup_printf("%d", i);
@@ -603,7 +581,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER: {
-        numberList **list = &cvalue->number;
+        numberList **list = &cvalue->u.number;
         for (i = 0; i < 32; i++) {
             *list = g_new0(numberList, 1);
             (*list)->value = (double)i / 3;
@@ -709,17 +687,15 @@ static void test_native_list(TestOutputVisitorData *data,
                              UserDefNativeListUnionKind kind)
 {
     UserDefNativeListUnion *cvalue = g_new0(UserDefNativeListUnion, 1);
-    Error *err = NULL;
     QObject *obj;
 
-    cvalue->kind = kind;
+    cvalue->type = kind;
     init_native_list(cvalue);
 
-    visit_type_UserDefNativeListUnion(data->ov, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(data->ov, &cvalue, NULL, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
-    check_native_list(obj, cvalue->kind);
+    check_native_list(obj, cvalue->type);
     qapi_free_UserDefNativeListUnion(cvalue);
     qobject_decref(obj);
 }
@@ -832,6 +808,8 @@ int main(int argc, char **argv)
                             &out_visitor_data, test_visitor_out_struct_errors);
     output_visitor_test_add("/visitor/output/list",
                             &out_visitor_data, test_visitor_out_list);
+    output_visitor_test_add("/visitor/output/any",
+                            &out_visitor_data, test_visitor_out_any);
     output_visitor_test_add("/visitor/output/list-qapi-free",
                             &out_visitor_data, test_visitor_out_list_qapi_free);
     output_visitor_test_add("/visitor/output/union-flat",
diff --git a/tests/test-string-output-visitor.c b/tests/test-string-output-visitor.c
index 101fb27dd..fd5e67be6 100644
--- a/tests/test-string-output-visitor.c
+++ b/tests/test-string-output-visitor.c
@@ -248,39 +248,39 @@ int main(int argc, char **argv)
 
     output_visitor_test_add("/string-visitor/output/int",
                             &out_visitor_data, test_visitor_out_int, false);
-    output_visitor_test_add("/string-visitor/output/int",
+    output_visitor_test_add("/string-visitor/output/int-human",
                             &out_visitor_data, test_visitor_out_int, true);
     output_visitor_test_add("/string-visitor/output/bool",
                             &out_visitor_data, test_visitor_out_bool, false);
-    output_visitor_test_add("/string-visitor/output/bool",
+    output_visitor_test_add("/string-visitor/output/bool-human",
                             &out_visitor_data, test_visitor_out_bool, true);
     output_visitor_test_add("/string-visitor/output/number",
                             &out_visitor_data, test_visitor_out_number, false);
-    output_visitor_test_add("/string-visitor/output/number",
+    output_visitor_test_add("/string-visitor/output/number-human",
                             &out_visitor_data, test_visitor_out_number, true);
     output_visitor_test_add("/string-visitor/output/string",
                             &out_visitor_data, test_visitor_out_string, false);
-    output_visitor_test_add("/string-visitor/output/string",
+    output_visitor_test_add("/string-visitor/output/string-human",
                             &out_visitor_data, test_visitor_out_string, true);
     output_visitor_test_add("/string-visitor/output/no-string",
                             &out_visitor_data, test_visitor_out_no_string,
                             false);
-    output_visitor_test_add("/string-visitor/output/no-string",
+    output_visitor_test_add("/string-visitor/output/no-string-human",
                             &out_visitor_data, test_visitor_out_no_string,
                             true);
     output_visitor_test_add("/string-visitor/output/enum",
                             &out_visitor_data, test_visitor_out_enum, false);
-    output_visitor_test_add("/string-visitor/output/enum",
+    output_visitor_test_add("/string-visitor/output/enum-human",
                             &out_visitor_data, test_visitor_out_enum, true);
     output_visitor_test_add("/string-visitor/output/enum-errors",
                             &out_visitor_data, test_visitor_out_enum_errors,
                             false);
-    output_visitor_test_add("/string-visitor/output/enum-errors",
+    output_visitor_test_add("/string-visitor/output/enum-errors-human",
                             &out_visitor_data, test_visitor_out_enum_errors,
                             true);
     output_visitor_test_add("/string-visitor/output/intList",
                             &out_visitor_data, test_visitor_out_intList, false);
-    output_visitor_test_add("/string-visitor/output/intList",
+    output_visitor_test_add("/string-visitor/output/intList-human",
                             &out_visitor_data, test_visitor_out_intList, true);
 
     g_test_run();
diff --git a/tests/test-throttle.c b/tests/test-throttle.c
index 016844546..85c9b6cee 100644
--- a/tests/test-throttle.c
+++ b/tests/test-throttle.c
@@ -329,6 +329,26 @@ static void test_is_valid(void)
     test_is_valid_for_value(1, true);
 }
 
+static void test_max_is_missing_limit(void)
+{
+    int i;
+
+    for (i = 0; i < BUCKETS_COUNT; i++) {
+        memset(&cfg, 0, sizeof(cfg));
+        cfg.buckets[i].max = 100;
+        cfg.buckets[i].avg = 0;
+        g_assert(throttle_max_is_missing_limit(&cfg));
+
+        cfg.buckets[i].max = 0;
+        cfg.buckets[i].avg = 0;
+        g_assert(!throttle_max_is_missing_limit(&cfg));
+
+        cfg.buckets[i].max = 0;
+        cfg.buckets[i].avg = 100;
+        g_assert(!throttle_max_is_missing_limit(&cfg));
+    }
+}
+
 static void test_have_timer(void)
 {
     /* zero structures */
@@ -591,6 +611,7 @@ int main(int argc, char **argv)
     g_test_add_func("/throttle/config/enabled",     test_enabled);
     g_test_add_func("/throttle/config/conflicting", test_conflicting_config);
     g_test_add_func("/throttle/config/is_valid",    test_is_valid);
+    g_test_add_func("/throttle/config/max",         test_max_is_missing_limit);
     g_test_add_func("/throttle/config_functions",   test_config_functions);
     g_test_add_func("/throttle/accounting",         test_accounting);
     g_test_add_func("/throttle/groups",             test_groups);
diff --git a/tests/test-timed-average.c b/tests/test-timed-average.c
new file mode 100644
index 000000000..a049799b8
--- /dev/null
+++ b/tests/test-timed-average.c
@@ -0,0 +1,90 @@
+/*
+ * Timed average computation tests
+ *
+ * Copyright Nodalink, EURL. 2014
+ *
+ * Authors:
+ *  Benoît Canet     <benoit.canet@nodalink.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <unistd.h>
+
+#include "qemu/timed-average.h"
+
+/* This is the clock for QEMU_CLOCK_VIRTUAL */
+static int64_t my_clock_value;
+
+int64_t cpu_get_clock(void)
+{
+    return my_clock_value;
+}
+
+static void account(TimedAverage *ta)
+{
+    timed_average_account(ta, 1);
+    timed_average_account(ta, 5);
+    timed_average_account(ta, 2);
+    timed_average_account(ta, 4);
+    timed_average_account(ta, 3);
+}
+
+static void test_average(void)
+{
+    TimedAverage ta;
+    uint64_t result;
+    int i;
+
+    /* we will compute some average on a period of 1 second */
+    timed_average_init(&ta, QEMU_CLOCK_VIRTUAL, NANOSECONDS_PER_SECOND);
+
+    result = timed_average_min(&ta);
+    g_assert(result == 0);
+    result = timed_average_avg(&ta);
+    g_assert(result == 0);
+    result = timed_average_max(&ta);
+    g_assert(result == 0);
+
+    for (i = 0; i < 100; i++) {
+        account(&ta);
+        result = timed_average_min(&ta);
+        g_assert(result == 1);
+        result = timed_average_avg(&ta);
+        g_assert(result == 3);
+        result = timed_average_max(&ta);
+        g_assert(result == 5);
+        my_clock_value += NANOSECONDS_PER_SECOND / 10;
+    }
+
+    my_clock_value += NANOSECONDS_PER_SECOND * 100;
+
+    result = timed_average_min(&ta);
+    g_assert(result == 0);
+    result = timed_average_avg(&ta);
+    g_assert(result == 0);
+    result = timed_average_max(&ta);
+    g_assert(result == 0);
+
+    for (i = 0; i < 100; i++) {
+        account(&ta);
+        result = timed_average_min(&ta);
+        g_assert(result == 1);
+        result = timed_average_avg(&ta);
+        g_assert(result == 3);
+        result = timed_average_max(&ta);
+        g_assert(result == 5);
+        my_clock_value += NANOSECONDS_PER_SECOND / 10;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    /* tests in the same order as the header function declarations */
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/timed-average/average", test_average);
+    return g_test_run();
+}
+
diff --git a/tests/test-visitor-serialization.c b/tests/test-visitor-serialization.c
index fa86cae88..9f67f9e00 100644
--- a/tests/test-visitor-serialization.c
+++ b/tests/test-visitor-serialization.c
@@ -186,40 +186,6 @@ static void visit_primitive_list(Visitor *v, void **native, Error **errp)
     }
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, NULL, name, sizeof(TestStruct), &err);
-    if (err) {
-        goto out;
-    }
-
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static TestStruct *struct_create(void)
 {
@@ -258,15 +224,13 @@ static UserDefTwo *nested_struct_create(void)
     udnp->dict1->string1 = strdup("test_string1");
     udnp->dict1->dict2 = g_malloc0(sizeof(*udnp->dict1->dict2));
     udnp->dict1->dict2->userdef = g_new0(UserDefOne, 1);
-    udnp->dict1->dict2->userdef->base = g_new0(UserDefZero, 1);
-    udnp->dict1->dict2->userdef->base->integer = 42;
+    udnp->dict1->dict2->userdef->integer = 42;
     udnp->dict1->dict2->userdef->string = strdup("test_string");
     udnp->dict1->dict2->string = strdup("test_string2");
     udnp->dict1->dict3 = g_malloc0(sizeof(*udnp->dict1->dict3));
     udnp->dict1->has_dict3 = true;
     udnp->dict1->dict3->userdef = g_new0(UserDefOne, 1);
-    udnp->dict1->dict3->userdef->base = g_new0(UserDefZero, 1);
-    udnp->dict1->dict3->userdef->base->integer = 43;
+    udnp->dict1->dict3->userdef->integer = 43;
     udnp->dict1->dict3->userdef->string = strdup("test_string");
     udnp->dict1->dict3->string = strdup("test_string3");
     return udnp;
@@ -278,15 +242,15 @@ static void nested_struct_compare(UserDefTwo *udnp1, UserDefTwo *udnp2)
     g_assert(udnp2);
     g_assert_cmpstr(udnp1->string0, ==, udnp2->string0);
     g_assert_cmpstr(udnp1->dict1->string1, ==, udnp2->dict1->string1);
-    g_assert_cmpint(udnp1->dict1->dict2->userdef->base->integer, ==,
-                    udnp2->dict1->dict2->userdef->base->integer);
+    g_assert_cmpint(udnp1->dict1->dict2->userdef->integer, ==,
+                    udnp2->dict1->dict2->userdef->integer);
     g_assert_cmpstr(udnp1->dict1->dict2->userdef->string, ==,
                     udnp2->dict1->dict2->userdef->string);
     g_assert_cmpstr(udnp1->dict1->dict2->string, ==,
                     udnp2->dict1->dict2->string);
     g_assert(udnp1->dict1->has_dict3 == udnp2->dict1->has_dict3);
-    g_assert_cmpint(udnp1->dict1->dict3->userdef->base->integer, ==,
-                    udnp2->dict1->dict3->userdef->base->integer);
+    g_assert_cmpint(udnp1->dict1->dict3->userdef->integer, ==,
+                    udnp2->dict1->dict3->userdef->integer);
     g_assert_cmpstr(udnp1->dict1->dict3->userdef->string, ==,
                     udnp2->dict1->dict3->userdef->string);
     g_assert_cmpstr(udnp1->dict1->dict3->string, ==,
@@ -338,14 +302,13 @@ static void test_primitives(gconstpointer opaque)
     const SerializeOps *ops = args->ops;
     PrimitiveType *pt = args->test_data;
     PrimitiveType *pt_copy = g_malloc0(sizeof(*pt_copy));
-    Error *err = NULL;
     void *serialize_data;
 
     pt_copy->type = pt->type;
-    ops->serialize(pt, &serialize_data, visit_primitive_type, &err);
-    ops->deserialize((void **)&pt_copy, serialize_data, visit_primitive_type, &err);
+    ops->serialize(pt, &serialize_data, visit_primitive_type, &error_abort);
+    ops->deserialize((void **)&pt_copy, serialize_data, visit_primitive_type,
+                     &error_abort);
 
-    g_assert(err == NULL);
     g_assert(pt_copy != NULL);
     if (pt->type == PTYPE_STRING) {
         g_assert_cmpstr(pt->value.string, ==, pt_copy->value.string);
@@ -381,7 +344,6 @@ static void test_primitive_lists(gconstpointer opaque)
     PrimitiveList pl = { .value = { NULL } };
     PrimitiveList pl_copy = { .value = { NULL } };
     PrimitiveList *pl_copy_ptr = &pl_copy;
-    Error *err = NULL;
     void *serialize_data;
     void *cur_head = NULL;
     int i;
@@ -528,10 +490,11 @@ static void test_primitive_lists(gconstpointer opaque)
         }
     }
 
-    ops->serialize((void **)&pl, &serialize_data, visit_primitive_list, &err);
-    ops->deserialize((void **)&pl_copy_ptr, serialize_data, visit_primitive_list, &err);
+    ops->serialize((void **)&pl, &serialize_data, visit_primitive_list,
+                   &error_abort);
+    ops->deserialize((void **)&pl_copy_ptr, serialize_data,
+                     visit_primitive_list, &error_abort);
 
-    g_assert(err == NULL);
     i = 0;
 
     /* compare our deserialized list of primitives to the original */
@@ -688,10 +651,8 @@ static void test_primitive_lists(gconstpointer opaque)
     g_assert_cmpint(i, ==, 33);
 
     ops->cleanup(serialize_data);
-    dealloc_helper(&pl, visit_primitive_list, &err);
-    g_assert(!err);
-    dealloc_helper(&pl_copy, visit_primitive_list, &err);
-    g_assert(!err);
+    dealloc_helper(&pl, visit_primitive_list, &error_abort);
+    dealloc_helper(&pl_copy, visit_primitive_list, &error_abort);
     g_free(args);
 }
 
@@ -701,13 +662,12 @@ static void test_struct(gconstpointer opaque)
     const SerializeOps *ops = args->ops;
     TestStruct *ts = struct_create();
     TestStruct *ts_copy = NULL;
-    Error *err = NULL;
     void *serialize_data;
 
-    ops->serialize(ts, &serialize_data, visit_struct, &err);
-    ops->deserialize((void **)&ts_copy, serialize_data, visit_struct, &err); 
+    ops->serialize(ts, &serialize_data, visit_struct, &error_abort);
+    ops->deserialize((void **)&ts_copy, serialize_data, visit_struct,
+                     &error_abort);
 
-    g_assert(err == NULL);
     struct_compare(ts, ts_copy);
 
     struct_cleanup(ts);
@@ -723,14 +683,12 @@ static void test_nested_struct(gconstpointer opaque)
     const SerializeOps *ops = args->ops;
     UserDefTwo *udnp = nested_struct_create();
     UserDefTwo *udnp_copy = NULL;
-    Error *err = NULL;
     void *serialize_data;
 
-    ops->serialize(udnp, &serialize_data, visit_nested_struct, &err);
+    ops->serialize(udnp, &serialize_data, visit_nested_struct, &error_abort);
     ops->deserialize((void **)&udnp_copy, serialize_data, visit_nested_struct,
-                     &err);
+                     &error_abort);
 
-    g_assert(err == NULL);
     nested_struct_compare(udnp, udnp_copy);
 
     nested_struct_cleanup(udnp);
@@ -745,7 +703,6 @@ static void test_nested_struct_list(gconstpointer opaque)
     TestArgs *args = (TestArgs *) opaque;
     const SerializeOps *ops = args->ops;
     UserDefTwoList *listp = NULL, *tmp, *tmp_copy, *listp_copy = NULL;
-    Error *err = NULL;
     void *serialize_data;
     int i = 0;
 
@@ -756,11 +713,10 @@ static void test_nested_struct_list(gconstpointer opaque)
         listp = tmp;
     }
 
-    ops->serialize(listp, &serialize_data, visit_nested_struct_list, &err);
+    ops->serialize(listp, &serialize_data, visit_nested_struct_list,
+                   &error_abort);
     ops->deserialize((void **)&listp_copy, serialize_data,
-                     visit_nested_struct_list, &err); 
-
-    g_assert(err == NULL);
+                     visit_nested_struct_list, &error_abort);
 
     tmp = listp;
     tmp_copy = listp_copy;
diff --git a/tests/test-vmstate.c b/tests/test-vmstate.c
index 1d620e04f..4d13bd09b 100644
--- a/tests/test-vmstate.c
+++ b/tests/test-vmstate.c
@@ -27,7 +27,7 @@
 #include "qemu-common.h"
 #include "migration/migration.h"
 #include "migration/vmstate.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 static char temp_file[] = "/tmp/vmst.test.XXXXXX";
 static int temp_fd;
diff --git a/tests/test-xbzrle.c b/tests/test-xbzrle.c
index db93b0a3d..1cd8cb7d7 100644
--- a/tests/test-xbzrle.c
+++ b/tests/test-xbzrle.c
@@ -13,10 +13,8 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <strings.h>
 #include <string.h>
 #include <sys/time.h>
-#include <assert.h>
 #include "qemu-common.h"
 #include "include/migration/migration.h"
 
diff --git a/tests/vhost-user-bridge.c b/tests/vhost-user-bridge.c
new file mode 100644
index 000000000..9fb09f1df
--- /dev/null
+++ b/tests/vhost-user-bridge.c
@@ -0,0 +1,1398 @@
+/*
+ * Vhost User Bridge
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Victor Kaplansky <victork@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+/*
+ * TODO:
+ *     - main should get parameters from the command line.
+ *     - implement all request handlers. Still not implemented:
+ *          vubr_get_queue_num_exec()
+ *          vubr_send_rarp_exec()
+ *     - test for broken requests and virtqueue.
+ *     - implement features defined by Virtio 1.0 spec.
+ *     - support mergeable buffers and indirect descriptors.
+ *     - implement clean shutdown.
+ *     - implement non-blocking writes to UDP backend.
+ *     - implement polling strategy.
+ *     - implement clean starting/stopping of vq processing
+ *     - implement clean starting/stopping of used and buffers
+ *       dirty page logging.
+ */
+
+#define _FILE_OFFSET_BITS 64
+
+#include <stddef.h>
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/unistd.h>
+#include <sys/mman.h>
+#include <sys/eventfd.h>
+#include <arpa/inet.h>
+#include <ctype.h>
+#include <netdb.h>
+
+#include <linux/vhost.h>
+
+#include "qemu/atomic.h"
+#include "standard-headers/linux/virtio_net.h"
+#include "standard-headers/linux/virtio_ring.h"
+
+#define VHOST_USER_BRIDGE_DEBUG 1
+
+#define DPRINT(...) \
+    do { \
+        if (VHOST_USER_BRIDGE_DEBUG) { \
+            printf(__VA_ARGS__); \
+        } \
+    } while (0)
+
+typedef void (*CallbackFunc)(int sock, void *ctx);
+
+typedef struct Event {
+    void *ctx;
+    CallbackFunc callback;
+} Event;
+
+typedef struct Dispatcher {
+    int max_sock;
+    fd_set fdset;
+    Event events[FD_SETSIZE];
+} Dispatcher;
+
+static void
+vubr_die(const char *s)
+{
+    perror(s);
+    exit(1);
+}
+
+static int
+dispatcher_init(Dispatcher *dispr)
+{
+    FD_ZERO(&dispr->fdset);
+    dispr->max_sock = -1;
+    return 0;
+}
+
+static int
+dispatcher_add(Dispatcher *dispr, int sock, void *ctx, CallbackFunc cb)
+{
+    if (sock >= FD_SETSIZE) {
+        fprintf(stderr,
+                "Error: Failed to add new event. sock %d should be less than %d\n",
+                sock, FD_SETSIZE);
+        return -1;
+    }
+
+    dispr->events[sock].ctx = ctx;
+    dispr->events[sock].callback = cb;
+
+    FD_SET(sock, &dispr->fdset);
+    if (sock > dispr->max_sock) {
+        dispr->max_sock = sock;
+    }
+    DPRINT("Added sock %d for watching. max_sock: %d\n",
+           sock, dispr->max_sock);
+    return 0;
+}
+
+/* dispatcher_remove() is not currently in use but may be useful
+ * in the future. */
+static int
+dispatcher_remove(Dispatcher *dispr, int sock)
+{
+    if (sock >= FD_SETSIZE) {
+        fprintf(stderr,
+                "Error: Failed to remove event. sock %d should be less than %d\n",
+                sock, FD_SETSIZE);
+        return -1;
+    }
+
+    FD_CLR(sock, &dispr->fdset);
+    DPRINT("Sock %d removed from dispatcher watch.\n", sock);
+    return 0;
+}
+
+/* timeout in us */
+static int
+dispatcher_wait(Dispatcher *dispr, uint32_t timeout)
+{
+    struct timeval tv;
+    tv.tv_sec = timeout / 1000000;
+    tv.tv_usec = timeout % 1000000;
+
+    fd_set fdset = dispr->fdset;
+
+    /* wait until some of sockets become readable. */
+    int rc = select(dispr->max_sock + 1, &fdset, 0, 0, &tv);
+
+    if (rc == -1) {
+        vubr_die("select");
+    }
+
+    /* Timeout */
+    if (rc == 0) {
+        return 0;
+    }
+
+    /* Now call callback for every ready socket. */
+
+    int sock;
+    for (sock = 0; sock < dispr->max_sock + 1; sock++) {
+        /* The callback on a socket can remove other sockets from the
+         * dispatcher, thus we have to check that the socket is
+         * still not removed from dispatcher's list
+         */
+        if (FD_ISSET(sock, &fdset) && FD_ISSET(sock, &dispr->fdset)) {
+            Event *e = &dispr->events[sock];
+            e->callback(sock, e->ctx);
+        }
+    }
+
+    return 0;
+}
+
+typedef struct VubrVirtq {
+    int call_fd;
+    int kick_fd;
+    uint32_t size;
+    uint16_t last_avail_index;
+    uint16_t last_used_index;
+    struct vring_desc *desc;
+    struct vring_avail *avail;
+    struct vring_used *used;
+    uint64_t log_guest_addr;
+    int enable;
+} VubrVirtq;
+
+/* Based on qemu/hw/virtio/vhost-user.c */
+
+#define VHOST_MEMORY_MAX_NREGIONS    8
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+
+#define VHOST_LOG_PAGE 4096
+
+enum VhostUserProtocolFeature {
+    VHOST_USER_PROTOCOL_F_MQ = 0,
+    VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
+    VHOST_USER_PROTOCOL_F_RARP = 2,
+
+    VHOST_USER_PROTOCOL_F_MAX
+};
+
+#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
+
+typedef enum VhostUserRequest {
+    VHOST_USER_NONE = 0,
+    VHOST_USER_GET_FEATURES = 1,
+    VHOST_USER_SET_FEATURES = 2,
+    VHOST_USER_SET_OWNER = 3,
+    VHOST_USER_RESET_OWNER = 4,
+    VHOST_USER_SET_MEM_TABLE = 5,
+    VHOST_USER_SET_LOG_BASE = 6,
+    VHOST_USER_SET_LOG_FD = 7,
+    VHOST_USER_SET_VRING_NUM = 8,
+    VHOST_USER_SET_VRING_ADDR = 9,
+    VHOST_USER_SET_VRING_BASE = 10,
+    VHOST_USER_GET_VRING_BASE = 11,
+    VHOST_USER_SET_VRING_KICK = 12,
+    VHOST_USER_SET_VRING_CALL = 13,
+    VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_GET_QUEUE_NUM = 17,
+    VHOST_USER_SET_VRING_ENABLE = 18,
+    VHOST_USER_SEND_RARP = 19,
+    VHOST_USER_MAX
+} VhostUserRequest;
+
+typedef struct VhostUserMemoryRegion {
+    uint64_t guest_phys_addr;
+    uint64_t memory_size;
+    uint64_t userspace_addr;
+    uint64_t mmap_offset;
+} VhostUserMemoryRegion;
+
+typedef struct VhostUserMemory {
+    uint32_t nregions;
+    uint32_t padding;
+    VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
+} VhostUserMemory;
+
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
+typedef struct VhostUserMsg {
+    VhostUserRequest request;
+
+#define VHOST_USER_VERSION_MASK     (0x3)
+#define VHOST_USER_REPLY_MASK       (0x1<<2)
+    uint32_t flags;
+    uint32_t size; /* the following payload size */
+    union {
+#define VHOST_USER_VRING_IDX_MASK   (0xff)
+#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
+        uint64_t u64;
+        struct vhost_vring_state state;
+        struct vhost_vring_addr addr;
+        VhostUserMemory memory;
+        VhostUserLog log;
+    } payload;
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    int fd_num;
+} QEMU_PACKED VhostUserMsg;
+
+#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
+
+/* The version of the protocol we support */
+#define VHOST_USER_VERSION    (0x1)
+
+#define MAX_NR_VIRTQUEUE (8)
+
+typedef struct VubrDevRegion {
+    /* Guest Physical address. */
+    uint64_t gpa;
+    /* Memory region size. */
+    uint64_t size;
+    /* QEMU virtual address (userspace). */
+    uint64_t qva;
+    /* Starting offset in our mmaped space. */
+    uint64_t mmap_offset;
+    /* Start address of mmaped space. */
+    uint64_t mmap_addr;
+} VubrDevRegion;
+
+typedef struct VubrDev {
+    int sock;
+    Dispatcher dispatcher;
+    uint32_t nregions;
+    VubrDevRegion regions[VHOST_MEMORY_MAX_NREGIONS];
+    VubrVirtq vq[MAX_NR_VIRTQUEUE];
+    int log_call_fd;
+    uint64_t log_size;
+    uint8_t *log_table;
+    int backend_udp_sock;
+    struct sockaddr_in backend_udp_dest;
+    int ready;
+    uint64_t features;
+} VubrDev;
+
+static const char *vubr_request_str[] = {
+    [VHOST_USER_NONE]                   =  "VHOST_USER_NONE",
+    [VHOST_USER_GET_FEATURES]           =  "VHOST_USER_GET_FEATURES",
+    [VHOST_USER_SET_FEATURES]           =  "VHOST_USER_SET_FEATURES",
+    [VHOST_USER_SET_OWNER]              =  "VHOST_USER_SET_OWNER",
+    [VHOST_USER_RESET_OWNER]           =  "VHOST_USER_RESET_OWNER",
+    [VHOST_USER_SET_MEM_TABLE]          =  "VHOST_USER_SET_MEM_TABLE",
+    [VHOST_USER_SET_LOG_BASE]           =  "VHOST_USER_SET_LOG_BASE",
+    [VHOST_USER_SET_LOG_FD]             =  "VHOST_USER_SET_LOG_FD",
+    [VHOST_USER_SET_VRING_NUM]          =  "VHOST_USER_SET_VRING_NUM",
+    [VHOST_USER_SET_VRING_ADDR]         =  "VHOST_USER_SET_VRING_ADDR",
+    [VHOST_USER_SET_VRING_BASE]         =  "VHOST_USER_SET_VRING_BASE",
+    [VHOST_USER_GET_VRING_BASE]         =  "VHOST_USER_GET_VRING_BASE",
+    [VHOST_USER_SET_VRING_KICK]         =  "VHOST_USER_SET_VRING_KICK",
+    [VHOST_USER_SET_VRING_CALL]         =  "VHOST_USER_SET_VRING_CALL",
+    [VHOST_USER_SET_VRING_ERR]          =  "VHOST_USER_SET_VRING_ERR",
+    [VHOST_USER_GET_PROTOCOL_FEATURES]  =  "VHOST_USER_GET_PROTOCOL_FEATURES",
+    [VHOST_USER_SET_PROTOCOL_FEATURES]  =  "VHOST_USER_SET_PROTOCOL_FEATURES",
+    [VHOST_USER_GET_QUEUE_NUM]          =  "VHOST_USER_GET_QUEUE_NUM",
+    [VHOST_USER_SET_VRING_ENABLE]       =  "VHOST_USER_SET_VRING_ENABLE",
+    [VHOST_USER_SEND_RARP]              =  "VHOST_USER_SEND_RARP",
+    [VHOST_USER_MAX]                    =  "VHOST_USER_MAX",
+};
+
+static void
+print_buffer(uint8_t *buf, size_t len)
+{
+    int i;
+    printf("Raw buffer:\n");
+    for (i = 0; i < len; i++) {
+        if (i % 16 == 0) {
+            printf("\n");
+        }
+        if (i % 4 == 0) {
+            printf("   ");
+        }
+        printf("%02x ", buf[i]);
+    }
+    printf("\n............................................................\n");
+}
+
+/* Translate guest physical address to our virtual address.  */
+static uint64_t
+gpa_to_va(VubrDev *dev, uint64_t guest_addr)
+{
+    int i;
+
+    /* Find matching memory region.  */
+    for (i = 0; i < dev->nregions; i++) {
+        VubrDevRegion *r = &dev->regions[i];
+
+        if ((guest_addr >= r->gpa) && (guest_addr < (r->gpa + r->size))) {
+            return guest_addr - r->gpa + r->mmap_addr + r->mmap_offset;
+        }
+    }
+
+    assert(!"address not found in regions");
+    return 0;
+}
+
+/* Translate qemu virtual address to our virtual address.  */
+static uint64_t
+qva_to_va(VubrDev *dev, uint64_t qemu_addr)
+{
+    int i;
+
+    /* Find matching memory region.  */
+    for (i = 0; i < dev->nregions; i++) {
+        VubrDevRegion *r = &dev->regions[i];
+
+        if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
+            return qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
+        }
+    }
+
+    assert(!"address not found in regions");
+    return 0;
+}
+
+static void
+vubr_message_read(int conn_fd, VhostUserMsg *vmsg)
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+
+    rc = recvmsg(conn_fd, &msg, 0);
+
+    if (rc == 0) {
+        vubr_die("recvmsg");
+        fprintf(stderr, "Peer disconnected.\n");
+        exit(1);
+    }
+    if (rc < 0) {
+        vubr_die("recvmsg");
+    }
+
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }
+
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        fprintf(stderr,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %lu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        exit(1);
+    }
+
+    if (vmsg->size) {
+        rc = read(conn_fd, &vmsg->payload, vmsg->size);
+        if (rc == 0) {
+            vubr_die("recvmsg");
+            fprintf(stderr, "Peer disconnected.\n");
+            exit(1);
+        }
+        if (rc < 0) {
+            vubr_die("recvmsg");
+        }
+
+        assert(rc == vmsg->size);
+    }
+}
+
+static void
+vubr_message_write(int conn_fd, VhostUserMsg *vmsg)
+{
+    int rc;
+
+    do {
+        rc = write(conn_fd, vmsg, VHOST_USER_HDR_SIZE + vmsg->size);
+    } while (rc < 0 && errno == EINTR);
+
+    if (rc < 0) {
+        vubr_die("write");
+    }
+}
+
+static void
+vubr_backend_udp_sendbuf(VubrDev *dev, uint8_t *buf, size_t len)
+{
+    int slen = sizeof(struct sockaddr_in);
+
+    if (sendto(dev->backend_udp_sock, buf, len, 0,
+               (struct sockaddr *) &dev->backend_udp_dest, slen) == -1) {
+        vubr_die("sendto()");
+    }
+}
+
+static int
+vubr_backend_udp_recvbuf(VubrDev *dev, uint8_t *buf, size_t buflen)
+{
+    int slen = sizeof(struct sockaddr_in);
+    int rc;
+
+    rc = recvfrom(dev->backend_udp_sock, buf, buflen, 0,
+                  (struct sockaddr *) &dev->backend_udp_dest,
+                  (socklen_t *)&slen);
+    if (rc == -1) {
+        vubr_die("recvfrom()");
+    }
+
+    return rc;
+}
+
+static void
+vubr_consume_raw_packet(VubrDev *dev, uint8_t *buf, uint32_t len)
+{
+    int hdrlen = sizeof(struct virtio_net_hdr_v1);
+
+    if (VHOST_USER_BRIDGE_DEBUG) {
+        print_buffer(buf, len);
+    }
+    vubr_backend_udp_sendbuf(dev, buf + hdrlen, len - hdrlen);
+}
+
+/* Kick the log_call_fd if required. */
+static void
+vubr_log_kick(VubrDev *dev)
+{
+    if (dev->log_call_fd != -1) {
+        DPRINT("Kicking the QEMU's log...\n");
+        eventfd_write(dev->log_call_fd, 1);
+    }
+}
+
+/* Kick the guest if necessary. */
+static void
+vubr_virtqueue_kick(VubrVirtq *vq)
+{
+    if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
+        DPRINT("Kicking the guest...\n");
+        eventfd_write(vq->call_fd, 1);
+    }
+}
+
+static void
+vubr_log_page(uint8_t *log_table, uint64_t page)
+{
+    DPRINT("Logged dirty guest page: %"PRId64"\n", page);
+    atomic_or(&log_table[page / 8], 1 << (page % 8));
+}
+
+static void
+vubr_log_write(VubrDev *dev, uint64_t address, uint64_t length)
+{
+    uint64_t page;
+
+    if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) ||
+        !dev->log_table || !length) {
+        return;
+    }
+
+    assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
+
+    page = address / VHOST_LOG_PAGE;
+    while (page * VHOST_LOG_PAGE < address + length) {
+        vubr_log_page(dev->log_table, page);
+        page += VHOST_LOG_PAGE;
+    }
+    vubr_log_kick(dev);
+}
+
+static void
+vubr_post_buffer(VubrDev *dev, VubrVirtq *vq, uint8_t *buf, int32_t len)
+{
+    struct vring_desc *desc = vq->desc;
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+
+    unsigned int size = vq->size;
+
+    uint16_t avail_index = atomic_mb_read(&avail->idx);
+
+    /* We check the available descriptors before posting the
+     * buffer, so here we assume that enough available
+     * descriptors. */
+    assert(vq->last_avail_index != avail_index);
+    uint16_t a_index = vq->last_avail_index % size;
+    uint16_t u_index = vq->last_used_index % size;
+    uint16_t d_index = avail->ring[a_index];
+
+    int i = d_index;
+
+    DPRINT("Post packet to guest on vq:\n");
+    DPRINT("    size             = %d\n", vq->size);
+    DPRINT("    last_avail_index = %d\n", vq->last_avail_index);
+    DPRINT("    last_used_index  = %d\n", vq->last_used_index);
+    DPRINT("    a_index = %d\n", a_index);
+    DPRINT("    u_index = %d\n", u_index);
+    DPRINT("    d_index = %d\n", d_index);
+    DPRINT("    desc[%d].addr    = 0x%016"PRIx64"\n", i, desc[i].addr);
+    DPRINT("    desc[%d].len     = %d\n", i, desc[i].len);
+    DPRINT("    desc[%d].flags   = %d\n", i, desc[i].flags);
+    DPRINT("    avail->idx = %d\n", avail_index);
+    DPRINT("    used->idx  = %d\n", used->idx);
+
+    if (!(desc[i].flags & VRING_DESC_F_WRITE)) {
+        /* FIXME: we should find writable descriptor. */
+        fprintf(stderr, "Error: descriptor is not writable. Exiting.\n");
+        exit(1);
+    }
+
+    void *chunk_start = (void *)gpa_to_va(dev, desc[i].addr);
+    uint32_t chunk_len = desc[i].len;
+
+    if (len <= chunk_len) {
+        memcpy(chunk_start, buf, len);
+        vubr_log_write(dev, desc[i].addr, len);
+    } else {
+        fprintf(stderr,
+                "Received too long packet from the backend. Dropping...\n");
+        return;
+    }
+
+    /* Add descriptor to the used ring. */
+    used->ring[u_index].id = d_index;
+    used->ring[u_index].len = len;
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, ring[u_index]),
+                   sizeof(used->ring[u_index]));
+
+    vq->last_avail_index++;
+    vq->last_used_index++;
+
+    atomic_mb_set(&used->idx, vq->last_used_index);
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, idx),
+                   sizeof(used->idx));
+
+    /* Kick the guest if necessary. */
+    vubr_virtqueue_kick(vq);
+}
+
+static int
+vubr_process_desc(VubrDev *dev, VubrVirtq *vq)
+{
+    struct vring_desc *desc = vq->desc;
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+
+    unsigned int size = vq->size;
+
+    uint16_t a_index = vq->last_avail_index % size;
+    uint16_t u_index = vq->last_used_index % size;
+    uint16_t d_index = avail->ring[a_index];
+
+    uint32_t i, len = 0;
+    size_t buf_size = 4096;
+    uint8_t buf[4096];
+
+    DPRINT("Chunks: ");
+    i = d_index;
+    do {
+        void *chunk_start = (void *)gpa_to_va(dev, desc[i].addr);
+        uint32_t chunk_len = desc[i].len;
+
+        assert(!(desc[i].flags & VRING_DESC_F_WRITE));
+
+        if (len + chunk_len < buf_size) {
+            memcpy(buf + len, chunk_start, chunk_len);
+            DPRINT("%d ", chunk_len);
+        } else {
+            fprintf(stderr, "Error: too long packet. Dropping...\n");
+            break;
+        }
+
+        len += chunk_len;
+
+        if (!(desc[i].flags & VRING_DESC_F_NEXT)) {
+            break;
+        }
+
+        i = desc[i].next;
+    } while (1);
+    DPRINT("\n");
+
+    if (!len) {
+        return -1;
+    }
+
+    /* Add descriptor to the used ring. */
+    used->ring[u_index].id = d_index;
+    used->ring[u_index].len = len;
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, ring[u_index]),
+                   sizeof(used->ring[u_index]));
+
+    vubr_consume_raw_packet(dev, buf, len);
+
+    return 0;
+}
+
+static void
+vubr_process_avail(VubrDev *dev, VubrVirtq *vq)
+{
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+
+    while (vq->last_avail_index != atomic_mb_read(&avail->idx)) {
+        vubr_process_desc(dev, vq);
+        vq->last_avail_index++;
+        vq->last_used_index++;
+    }
+
+    atomic_mb_set(&used->idx, vq->last_used_index);
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, idx),
+                   sizeof(used->idx));
+}
+
+static void
+vubr_backend_recv_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    VubrVirtq *rx_vq = &dev->vq[0];
+    uint8_t buf[4096];
+    struct virtio_net_hdr_v1 *hdr = (struct virtio_net_hdr_v1 *)buf;
+    int hdrlen = sizeof(struct virtio_net_hdr_v1);
+    int buflen = sizeof(buf);
+    int len;
+
+    if (!dev->ready) {
+        return;
+    }
+
+    DPRINT("\n\n   ***   IN UDP RECEIVE CALLBACK    ***\n\n");
+
+    uint16_t avail_index = atomic_mb_read(&rx_vq->avail->idx);
+
+    /* If there is no available descriptors, just do nothing.
+     * The buffer will be handled by next arrived UDP packet,
+     * or next kick on receive virtq. */
+    if (rx_vq->last_avail_index == avail_index) {
+        DPRINT("Got UDP packet, but no available descriptors on RX virtq.\n");
+        return;
+    }
+
+    len = vubr_backend_udp_recvbuf(dev, buf + hdrlen, buflen - hdrlen);
+
+    *hdr = (struct virtio_net_hdr_v1) { };
+    hdr->num_buffers = 1;
+    vubr_post_buffer(dev, rx_vq, buf, len + hdrlen);
+}
+
+static void
+vubr_kick_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        vubr_die("eventfd_read()");
+    } else {
+        DPRINT("Got kick_data: %016"PRIx64"\n", kick_data);
+        vubr_process_avail(dev, &dev->vq[1]);
+    }
+}
+
+static int
+vubr_none_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_get_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vmsg->payload.u64 =
+            ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
+             (1ULL << VHOST_F_LOG_ALL) |
+             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) |
+             (1ULL << VHOST_USER_F_PROTOCOL_FEATURES));
+
+    vmsg->size = sizeof(vmsg->payload.u64);
+
+    DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    dev->features = vmsg->payload.u64;
+    return 0;
+}
+
+static int
+vubr_set_owner_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    return 0;
+}
+
+static void
+vubr_close_log(VubrDev *dev)
+{
+    if (dev->log_table) {
+        if (munmap(dev->log_table, dev->log_size) != 0) {
+            vubr_die("munmap()");
+        }
+
+        dev->log_table = 0;
+    }
+    if (dev->log_call_fd != -1) {
+        close(dev->log_call_fd);
+        dev->log_call_fd = -1;
+    }
+}
+
+static int
+vubr_reset_device_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vubr_close_log(dev);
+    dev->ready = 0;
+    dev->features = 0;
+    return 0;
+}
+
+static int
+vubr_set_mem_table_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    int i;
+    VhostUserMemory *memory = &vmsg->payload.memory;
+    dev->nregions = memory->nregions;
+
+    DPRINT("Nregions: %d\n", memory->nregions);
+    for (i = 0; i < dev->nregions; i++) {
+        void *mmap_addr;
+        VhostUserMemoryRegion *msg_region = &memory->regions[i];
+        VubrDevRegion *dev_region = &dev->regions[i];
+
+        DPRINT("Region %d\n", i);
+        DPRINT("    guest_phys_addr: 0x%016"PRIx64"\n",
+               msg_region->guest_phys_addr);
+        DPRINT("    memory_size:     0x%016"PRIx64"\n",
+               msg_region->memory_size);
+        DPRINT("    userspace_addr   0x%016"PRIx64"\n",
+               msg_region->userspace_addr);
+        DPRINT("    mmap_offset      0x%016"PRIx64"\n",
+               msg_region->mmap_offset);
+
+        dev_region->gpa = msg_region->guest_phys_addr;
+        dev_region->size = msg_region->memory_size;
+        dev_region->qva = msg_region->userspace_addr;
+        dev_region->mmap_offset = msg_region->mmap_offset;
+
+        /* We don't use offset argument of mmap() since the
+         * mapped address has to be page aligned, and we use huge
+         * pages.  */
+        mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
+                         PROT_READ | PROT_WRITE, MAP_SHARED,
+                         vmsg->fds[i], 0);
+
+        if (mmap_addr == MAP_FAILED) {
+            vubr_die("mmap");
+        }
+        dev_region->mmap_addr = (uint64_t) mmap_addr;
+        DPRINT("    mmap_addr:       0x%016"PRIx64"\n", dev_region->mmap_addr);
+
+        close(vmsg->fds[i]);
+    }
+
+    return 0;
+}
+
+static int
+vubr_set_log_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    int fd;
+    uint64_t log_mmap_size, log_mmap_offset;
+    void *rc;
+
+    assert(vmsg->fd_num == 1);
+    fd = vmsg->fds[0];
+
+    assert(vmsg->size == sizeof(vmsg->payload.log));
+    log_mmap_offset = vmsg->payload.log.mmap_offset;
+    log_mmap_size = vmsg->payload.log.mmap_size;
+    DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
+    DPRINT("Log mmap_size:   %"PRId64"\n", log_mmap_size);
+
+    rc = mmap(0, log_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+              log_mmap_offset);
+    if (rc == MAP_FAILED) {
+        vubr_die("mmap");
+    }
+    dev->log_table = rc;
+    dev->log_size = log_mmap_size;
+
+    vmsg->size = sizeof(vmsg->payload.u64);
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_log_fd_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    assert(vmsg->fd_num == 1);
+    dev->log_call_fd = vmsg->fds[0];
+    DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
+    return 0;
+}
+
+static int
+vubr_set_vring_num_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int num = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.num:   %d\n", num);
+    dev->vq[index].size = num;
+    return 0;
+}
+
+static int
+vubr_set_vring_addr_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    struct vhost_vring_addr *vra = &vmsg->payload.addr;
+    unsigned int index = vra->index;
+    VubrVirtq *vq = &dev->vq[index];
+
+    DPRINT("vhost_vring_addr:\n");
+    DPRINT("    index:  %d\n", vra->index);
+    DPRINT("    flags:  %d\n", vra->flags);
+    DPRINT("    desc_user_addr:   0x%016llx\n", vra->desc_user_addr);
+    DPRINT("    used_user_addr:   0x%016llx\n", vra->used_user_addr);
+    DPRINT("    avail_user_addr:  0x%016llx\n", vra->avail_user_addr);
+    DPRINT("    log_guest_addr:   0x%016llx\n", vra->log_guest_addr);
+
+    vq->desc = (struct vring_desc *)qva_to_va(dev, vra->desc_user_addr);
+    vq->used = (struct vring_used *)qva_to_va(dev, vra->used_user_addr);
+    vq->avail = (struct vring_avail *)qva_to_va(dev, vra->avail_user_addr);
+    vq->log_guest_addr = vra->log_guest_addr;
+
+    DPRINT("Setting virtq addresses:\n");
+    DPRINT("    vring_desc  at %p\n", vq->desc);
+    DPRINT("    vring_used  at %p\n", vq->used);
+    DPRINT("    vring_avail at %p\n", vq->avail);
+
+    vq->last_used_index = vq->used->idx;
+    return 0;
+}
+
+static int
+vubr_set_vring_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int num = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.num:   %d\n", num);
+    dev->vq[index].last_avail_index = num;
+
+    return 0;
+}
+
+static int
+vubr_get_vring_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+
+    DPRINT("State.index: %d\n", index);
+    vmsg->payload.state.num = dev->vq[index].last_avail_index;
+    vmsg->size = sizeof(vmsg->payload.state);
+    /* FIXME: this is a work-around for a bug in QEMU enabling
+     * too early vrings. When protocol features are enabled,
+     * we have to respect * VHOST_USER_SET_VRING_ENABLE request. */
+    dev->ready = 0;
+
+    if (dev->vq[index].call_fd != -1) {
+        close(dev->vq[index].call_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].call_fd);
+        dev->vq[index].call_fd = -1;
+    }
+    if (dev->vq[index].kick_fd != -1) {
+        close(dev->vq[index].kick_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].kick_fd);
+        dev->vq[index].kick_fd = -1;
+    }
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_vring_kick_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    uint64_t u64_arg = vmsg->payload.u64;
+    int index = u64_arg & VHOST_USER_VRING_IDX_MASK;
+
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    assert((u64_arg & VHOST_USER_VRING_NOFD_MASK) == 0);
+    assert(vmsg->fd_num == 1);
+
+    if (dev->vq[index].kick_fd != -1) {
+        close(dev->vq[index].kick_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].kick_fd);
+    }
+    dev->vq[index].kick_fd = vmsg->fds[0];
+    DPRINT("Got kick_fd: %d for vq: %d\n", vmsg->fds[0], index);
+
+    if (index % 2 == 1) {
+        /* TX queue. */
+        dispatcher_add(&dev->dispatcher, dev->vq[index].kick_fd,
+                       dev, vubr_kick_cb);
+
+        DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
+               dev->vq[index].kick_fd, index);
+    }
+    /* We temporarily use this hack to determine that both TX and RX
+     * queues are set up and ready for processing.
+     * FIXME: we need to rely in VHOST_USER_SET_VRING_ENABLE and
+     * actual kicks. */
+    if (dev->vq[0].kick_fd != -1 &&
+        dev->vq[1].kick_fd != -1) {
+        dev->ready = 1;
+        DPRINT("vhost-user-bridge is ready for processing queues.\n");
+    }
+    return 0;
+
+}
+
+static int
+vubr_set_vring_call_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    uint64_t u64_arg = vmsg->payload.u64;
+    int index = u64_arg & VHOST_USER_VRING_IDX_MASK;
+
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    assert((u64_arg & VHOST_USER_VRING_NOFD_MASK) == 0);
+    assert(vmsg->fd_num == 1);
+
+    if (dev->vq[index].call_fd != -1) {
+        close(dev->vq[index].call_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].call_fd);
+    }
+    dev->vq[index].call_fd = vmsg->fds[0];
+    DPRINT("Got call_fd: %d for vq: %d\n", vmsg->fds[0], index);
+
+    return 0;
+}
+
+static int
+vubr_set_vring_err_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    return 0;
+}
+
+static int
+vubr_get_protocol_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vmsg->payload.u64 = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD;
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    vmsg->size = sizeof(vmsg->payload.u64);
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_protocol_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    /* FIXME: unimplented */
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    return 0;
+}
+
+static int
+vubr_get_queue_num_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_set_vring_enable_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int enable = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.enable:   %d\n", enable);
+    dev->vq[index].enable = enable;
+    return 0;
+}
+
+static int
+vubr_send_rarp_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_execute_request(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    /* Print out generic part of the request. */
+    DPRINT(
+           "==================   Vhost user message from QEMU   ==================\n");
+    DPRINT("Request: %s (%d)\n", vubr_request_str[vmsg->request],
+           vmsg->request);
+    DPRINT("Flags:   0x%x\n", vmsg->flags);
+    DPRINT("Size:    %d\n", vmsg->size);
+
+    if (vmsg->fd_num) {
+        int i;
+        DPRINT("Fds:");
+        for (i = 0; i < vmsg->fd_num; i++) {
+            DPRINT(" %d", vmsg->fds[i]);
+        }
+        DPRINT("\n");
+    }
+
+    switch (vmsg->request) {
+    case VHOST_USER_NONE:
+        return vubr_none_exec(dev, vmsg);
+    case VHOST_USER_GET_FEATURES:
+        return vubr_get_features_exec(dev, vmsg);
+    case VHOST_USER_SET_FEATURES:
+        return vubr_set_features_exec(dev, vmsg);
+    case VHOST_USER_SET_OWNER:
+        return vubr_set_owner_exec(dev, vmsg);
+    case VHOST_USER_RESET_OWNER:
+        return vubr_reset_device_exec(dev, vmsg);
+    case VHOST_USER_SET_MEM_TABLE:
+        return vubr_set_mem_table_exec(dev, vmsg);
+    case VHOST_USER_SET_LOG_BASE:
+        return vubr_set_log_base_exec(dev, vmsg);
+    case VHOST_USER_SET_LOG_FD:
+        return vubr_set_log_fd_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_NUM:
+        return vubr_set_vring_num_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ADDR:
+        return vubr_set_vring_addr_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_BASE:
+        return vubr_set_vring_base_exec(dev, vmsg);
+    case VHOST_USER_GET_VRING_BASE:
+        return vubr_get_vring_base_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_KICK:
+        return vubr_set_vring_kick_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_CALL:
+        return vubr_set_vring_call_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ERR:
+        return vubr_set_vring_err_exec(dev, vmsg);
+    case VHOST_USER_GET_PROTOCOL_FEATURES:
+        return vubr_get_protocol_features_exec(dev, vmsg);
+    case VHOST_USER_SET_PROTOCOL_FEATURES:
+        return vubr_set_protocol_features_exec(dev, vmsg);
+    case VHOST_USER_GET_QUEUE_NUM:
+        return vubr_get_queue_num_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ENABLE:
+        return vubr_set_vring_enable_exec(dev, vmsg);
+    case VHOST_USER_SEND_RARP:
+        return vubr_send_rarp_exec(dev, vmsg);
+
+    case VHOST_USER_MAX:
+        assert(vmsg->request != VHOST_USER_MAX);
+    }
+    return 0;
+}
+
+static void
+vubr_receive_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    VhostUserMsg vmsg;
+    int reply_requested;
+
+    vubr_message_read(sock, &vmsg);
+    reply_requested = vubr_execute_request(dev, &vmsg);
+    if (reply_requested) {
+        /* Set the version in the flags when sending the reply */
+        vmsg.flags &= ~VHOST_USER_VERSION_MASK;
+        vmsg.flags |= VHOST_USER_VERSION;
+        vmsg.flags |= VHOST_USER_REPLY_MASK;
+        vubr_message_write(sock, &vmsg);
+    }
+}
+
+static void
+vubr_accept_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *)ctx;
+    int conn_fd;
+    struct sockaddr_un un;
+    socklen_t len = sizeof(un);
+
+    conn_fd = accept(sock, (struct sockaddr *) &un, &len);
+    if (conn_fd == -1) {
+        vubr_die("accept()");
+    }
+    DPRINT("Got connection from remote peer on sock %d\n", conn_fd);
+    dispatcher_add(&dev->dispatcher, conn_fd, ctx, vubr_receive_cb);
+}
+
+static VubrDev *
+vubr_new(const char *path)
+{
+    VubrDev *dev = (VubrDev *) calloc(1, sizeof(VubrDev));
+    dev->nregions = 0;
+    int i;
+    struct sockaddr_un un;
+    size_t len;
+
+    for (i = 0; i < MAX_NR_VIRTQUEUE; i++) {
+        dev->vq[i] = (VubrVirtq) {
+            .call_fd = -1, .kick_fd = -1,
+            .size = 0,
+            .last_avail_index = 0, .last_used_index = 0,
+            .desc = 0, .avail = 0, .used = 0,
+            .enable = 0,
+        };
+    }
+
+    /* Init log */
+    dev->log_call_fd = -1;
+    dev->log_size = 0;
+    dev->log_table = 0;
+    dev->ready = 0;
+    dev->features = 0;
+
+    /* Get a UNIX socket. */
+    dev->sock = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (dev->sock == -1) {
+        vubr_die("socket");
+    }
+
+    un.sun_family = AF_UNIX;
+    strcpy(un.sun_path, path);
+    len = sizeof(un.sun_family) + strlen(path);
+    unlink(path);
+
+    if (bind(dev->sock, (struct sockaddr *) &un, len) == -1) {
+        vubr_die("bind");
+    }
+
+    if (listen(dev->sock, 1) == -1) {
+        vubr_die("listen");
+    }
+
+    dispatcher_init(&dev->dispatcher);
+    dispatcher_add(&dev->dispatcher, dev->sock, (void *)dev,
+                   vubr_accept_cb);
+
+    DPRINT("Waiting for connections on UNIX socket %s ...\n", path);
+    return dev;
+}
+
+static void
+vubr_set_host(struct sockaddr_in *saddr, const char *host)
+{
+    if (isdigit(host[0])) {
+        if (!inet_aton(host, &saddr->sin_addr)) {
+            fprintf(stderr, "inet_aton() failed.\n");
+            exit(1);
+        }
+    } else {
+        struct hostent *he = gethostbyname(host);
+
+        if (!he) {
+            fprintf(stderr, "gethostbyname() failed.\n");
+            exit(1);
+        }
+        saddr->sin_addr = *(struct in_addr *)he->h_addr;
+    }
+}
+
+static void
+vubr_backend_udp_setup(VubrDev *dev,
+                       const char *local_host,
+                       const char *local_port,
+                       const char *remote_host,
+                       const char *remote_port)
+{
+    int sock;
+    const char *r;
+
+    int lport, rport;
+
+    lport = strtol(local_port, (char **)&r, 0);
+    if (r == local_port) {
+        fprintf(stderr, "lport parsing failed.\n");
+        exit(1);
+    }
+
+    rport = strtol(remote_port, (char **)&r, 0);
+    if (r == remote_port) {
+        fprintf(stderr, "rport parsing failed.\n");
+        exit(1);
+    }
+
+    struct sockaddr_in si_local = {
+        .sin_family = AF_INET,
+        .sin_port = htons(lport),
+    };
+
+    vubr_set_host(&si_local, local_host);
+
+    /* setup destination for sends */
+    dev->backend_udp_dest = (struct sockaddr_in) {
+        .sin_family = AF_INET,
+        .sin_port = htons(rport),
+    };
+    vubr_set_host(&dev->backend_udp_dest, remote_host);
+
+    sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+    if (sock == -1) {
+        vubr_die("socket");
+    }
+
+    if (bind(sock, (struct sockaddr *)&si_local, sizeof(si_local)) == -1) {
+        vubr_die("bind");
+    }
+
+    dev->backend_udp_sock = sock;
+    dispatcher_add(&dev->dispatcher, sock, dev, vubr_backend_recv_cb);
+    DPRINT("Waiting for data from udp backend on %s:%d...\n",
+           local_host, lport);
+}
+
+static void
+vubr_run(VubrDev *dev)
+{
+    while (1) {
+        /* timeout 200ms */
+        dispatcher_wait(&dev->dispatcher, 200000);
+        /* Here one can try polling strategy. */
+    }
+}
+
+static int
+vubr_parse_host_port(const char **host, const char **port, const char *buf)
+{
+    char *p = strchr(buf, ':');
+
+    if (!p) {
+        return -1;
+    }
+    *p = '\0';
+    *host = strdup(buf);
+    *port = strdup(p + 1);
+    return 0;
+}
+
+#define DEFAULT_UD_SOCKET "/tmp/vubr.sock"
+#define DEFAULT_LHOST "127.0.0.1"
+#define DEFAULT_LPORT "4444"
+#define DEFAULT_RHOST "127.0.0.1"
+#define DEFAULT_RPORT "5555"
+
+static const char *ud_socket_path = DEFAULT_UD_SOCKET;
+static const char *lhost = DEFAULT_LHOST;
+static const char *lport = DEFAULT_LPORT;
+static const char *rhost = DEFAULT_RHOST;
+static const char *rport = DEFAULT_RPORT;
+
+int
+main(int argc, char *argv[])
+{
+    VubrDev *dev;
+    int opt;
+
+    while ((opt = getopt(argc, argv, "l:r:u:")) != -1) {
+
+        switch (opt) {
+        case 'l':
+            if (vubr_parse_host_port(&lhost, &lport, optarg) < 0) {
+                goto out;
+            }
+            break;
+        case 'r':
+            if (vubr_parse_host_port(&rhost, &rport, optarg) < 0) {
+                goto out;
+            }
+            break;
+        case 'u':
+            ud_socket_path = strdup(optarg);
+            break;
+        default:
+            goto out;
+        }
+    }
+
+    DPRINT("ud socket: %s\n", ud_socket_path);
+    DPRINT("local:     %s:%s\n", lhost, lport);
+    DPRINT("remote:    %s:%s\n", rhost, rport);
+
+    dev = vubr_new(ud_socket_path);
+    if (!dev) {
+        return 1;
+    }
+
+    vubr_backend_udp_setup(dev, lhost, lport, rhost, rport);
+    vubr_run(dev);
+    return 0;
+
+out:
+    fprintf(stderr, "Usage: %s ", argv[0]);
+    fprintf(stderr, "[-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n");
+    fprintf(stderr, "\t-u path to unix doman socket. default: %s\n",
+            DEFAULT_UD_SOCKET);
+    fprintf(stderr, "\t-l local host and port. default: %s:%s\n",
+            DEFAULT_LHOST, DEFAULT_LPORT);
+    fprintf(stderr, "\t-r remote host and port. default: %s:%s\n",
+            DEFAULT_RHOST, DEFAULT_RPORT);
+
+    return 1;
+}
diff --git a/tests/vhost-user-test.c b/tests/vhost-user-test.c
index 75fedf097..991fd85c7 100644
--- a/tests/vhost-user-test.c
+++ b/tests/vhost-user-test.c
@@ -8,11 +8,11 @@
  *
  */
 
-#define QEMU_GLIB_COMPAT_H
 #include <glib.h>
 
 #include "libqtest.h"
 #include "qemu/option.h"
+#include "qemu/range.h"
 #include "sysemu/char.h"
 #include "sysemu/sysemu.h"
 
@@ -30,17 +30,11 @@
 #define HAVE_MONOTONIC_TIME
 #endif
 
-#if GLIB_CHECK_VERSION(2, 32, 0)
-#define HAVE_MUTEX_INIT
-#define HAVE_COND_INIT
-#define HAVE_THREAD_NEW
-#endif
-
 #define QEMU_CMD_ACCEL  " -machine accel=tcg"
-#define QEMU_CMD_MEM    " -m 512 -object memory-backend-file,id=mem,size=512M,"\
+#define QEMU_CMD_MEM    " -m %d -object memory-backend-file,id=mem,size=%dM,"\
                         "mem-path=%s,share=on -numa node,memdev=mem"
-#define QEMU_CMD_CHR    " -chardev socket,id=chr0,path=%s"
-#define QEMU_CMD_NETDEV " -netdev vhost-user,id=net0,chardev=chr0,vhostforce"
+#define QEMU_CMD_CHR    " -chardev socket,id=%s,path=%s"
+#define QEMU_CMD_NETDEV " -netdev vhost-user,id=net0,chardev=%s,vhostforce"
 #define QEMU_CMD_NET    " -device virtio-net-pci,netdev=net0 "
 #define QEMU_CMD_ROM    " -option-rom ../pc-bios/pxe-virtio.rom"
 
@@ -53,6 +47,11 @@
 
 #define VHOST_MEMORY_MAX_NREGIONS    8
 
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1
+
+#define VHOST_LOG_PAGE 0x1000
+
 typedef enum VhostUserRequest {
     VHOST_USER_NONE = 0,
     VHOST_USER_GET_FEATURES = 1,
@@ -69,6 +68,9 @@ typedef enum VhostUserRequest {
     VHOST_USER_SET_VRING_KICK = 12,
     VHOST_USER_SET_VRING_CALL = 13,
     VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_SET_VRING_ENABLE = 18,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -85,6 +87,11 @@ typedef struct VhostUserMemory {
     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 } VhostUserMemory;
 
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
 typedef struct VhostUserMsg {
     VhostUserRequest request;
 
@@ -93,11 +100,14 @@ typedef struct VhostUserMsg {
     uint32_t flags;
     uint32_t size; /* the following payload size */
     union {
+#define VHOST_USER_VRING_IDX_MASK   (0xff)
+#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
         uint64_t u64;
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
-    };
+        VhostUserLog log;
+    } payload;
 } QEMU_PACKED VhostUserMsg;
 
 static VhostUserMsg m __attribute__ ((unused));
@@ -111,154 +121,104 @@ static VhostUserMsg m __attribute__ ((unused));
 #define VHOST_USER_VERSION    (0x1)
 /*****************************************************************************/
 
-int fds_num = 0, fds[VHOST_MEMORY_MAX_NREGIONS];
-static VhostUserMemory memory;
-static GMutex *data_mutex;
-static GCond *data_cond;
-
-static gint64 _get_time(void)
-{
-#ifdef HAVE_MONOTONIC_TIME
-    return g_get_monotonic_time();
-#else
-    GTimeVal time;
-    g_get_current_time(&time);
-
-    return time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
-#endif
-}
-
-static GMutex *_mutex_new(void)
-{
-    GMutex *mutex;
-
-#ifdef HAVE_MUTEX_INIT
-    mutex = g_new(GMutex, 1);
-    g_mutex_init(mutex);
-#else
-    mutex = g_mutex_new();
-#endif
-
-    return mutex;
-}
-
-static void _mutex_free(GMutex *mutex)
-{
-#ifdef HAVE_MUTEX_INIT
-    g_mutex_clear(mutex);
-    g_free(mutex);
-#else
-    g_mutex_free(mutex);
-#endif
-}
-
-static GCond *_cond_new(void)
-{
-    GCond *cond;
-
-#ifdef HAVE_COND_INIT
-    cond = g_new(GCond, 1);
-    g_cond_init(cond);
-#else
-    cond = g_cond_new();
-#endif
-
-    return cond;
-}
-
-static gboolean _cond_wait_until(GCond *cond, GMutex *mutex, gint64 end_time)
+typedef struct TestServer {
+    gchar *socket_path;
+    gchar *mig_path;
+    gchar *chr_name;
+    CharDriverState *chr;
+    int fds_num;
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    VhostUserMemory memory;
+    GMutex data_mutex;
+    GCond data_cond;
+    int log_fd;
+    uint64_t rings;
+} TestServer;
+
+#if !GLIB_CHECK_VERSION(2, 32, 0)
+static gboolean g_cond_wait_until(CompatGCond cond, CompatGMutex mutex,
+                                  gint64 end_time)
 {
     gboolean ret = FALSE;
-#ifdef HAVE_COND_INIT
-    ret = g_cond_wait_until(cond, mutex, end_time);
-#else
+    end_time -= g_get_monotonic_time();
     GTimeVal time = { end_time / G_TIME_SPAN_SECOND,
                       end_time % G_TIME_SPAN_SECOND };
     ret = g_cond_timed_wait(cond, mutex, &time);
-#endif
     return ret;
 }
-
-static void _cond_free(GCond *cond)
-{
-#ifdef HAVE_COND_INIT
-    g_cond_clear(cond);
-    g_free(cond);
-#else
-    g_cond_free(cond);
 #endif
-}
 
-static GThread *_thread_new(const gchar *name, GThreadFunc func, gpointer data)
-{
-    GThread *thread = NULL;
-    GError *error = NULL;
-#ifdef HAVE_THREAD_NEW
-    thread = g_thread_try_new(name, func, data, &error);
-#else
-    thread = g_thread_create(func, data, TRUE, &error);
-#endif
-    return thread;
-}
+static const char *tmpfs;
+static const char *root;
 
-static void read_guest_mem(void)
+static void wait_for_fds(TestServer *s)
 {
-    uint32_t *guest_mem;
     gint64 end_time;
-    int i, j;
-    size_t size;
 
-    g_mutex_lock(data_mutex);
+    g_mutex_lock(&s->data_mutex);
 
-    end_time = _get_time() + 5 * G_TIME_SPAN_SECOND;
-    while (!fds_num) {
-        if (!_cond_wait_until(data_cond, data_mutex, end_time)) {
+    end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+    while (!s->fds_num) {
+        if (!g_cond_wait_until(&s->data_cond, &s->data_mutex, end_time)) {
             /* timeout has passed */
-            g_assert(fds_num);
+            g_assert(s->fds_num);
             break;
         }
     }
 
     /* check for sanity */
-    g_assert_cmpint(fds_num, >, 0);
-    g_assert_cmpint(fds_num, ==, memory.nregions);
+    g_assert_cmpint(s->fds_num, >, 0);
+    g_assert_cmpint(s->fds_num, ==, s->memory.nregions);
+
+    g_mutex_unlock(&s->data_mutex);
+}
+
+static void read_guest_mem(const void *data)
+{
+    TestServer *s = (void *)data;
+    uint32_t *guest_mem;
+    int i, j;
+    size_t size;
+
+    wait_for_fds(s);
+
+    g_mutex_lock(&s->data_mutex);
 
     /* iterate all regions */
-    for (i = 0; i < fds_num; i++) {
+    for (i = 0; i < s->fds_num; i++) {
 
         /* We'll check only the region statring at 0x0*/
-        if (memory.regions[i].guest_phys_addr != 0x0) {
+        if (s->memory.regions[i].guest_phys_addr != 0x0) {
             continue;
         }
 
-        g_assert_cmpint(memory.regions[i].memory_size, >, 1024);
+        g_assert_cmpint(s->memory.regions[i].memory_size, >, 1024);
 
-        size =  memory.regions[i].memory_size + memory.regions[i].mmap_offset;
+        size = s->memory.regions[i].memory_size +
+            s->memory.regions[i].mmap_offset;
 
         guest_mem = mmap(0, size, PROT_READ | PROT_WRITE,
-                         MAP_SHARED, fds[i], 0);
+                         MAP_SHARED, s->fds[i], 0);
 
         g_assert(guest_mem != MAP_FAILED);
-        guest_mem += (memory.regions[i].mmap_offset / sizeof(*guest_mem));
+        guest_mem += (s->memory.regions[i].mmap_offset / sizeof(*guest_mem));
 
         for (j = 0; j < 256; j++) {
-            uint32_t a = readl(memory.regions[i].guest_phys_addr + j*4);
+            uint32_t a = readl(s->memory.regions[i].guest_phys_addr + j*4);
             uint32_t b = guest_mem[j];
 
             g_assert_cmpint(a, ==, b);
         }
 
-        munmap(guest_mem, memory.regions[i].memory_size);
+        munmap(guest_mem, s->memory.regions[i].memory_size);
     }
 
-    g_assert_cmpint(1, ==, 1);
-    g_mutex_unlock(data_mutex);
+    g_mutex_unlock(&s->data_mutex);
 }
 
 static void *thread_function(void *data)
 {
-    GMainLoop *loop;
-    loop = g_main_loop_new(NULL, FALSE);
+    GMainLoop *loop = data;
     g_main_loop_run(loop);
     return NULL;
 }
@@ -270,7 +230,8 @@ static int chr_can_read(void *opaque)
 
 static void chr_read(void *opaque, const uint8_t *buf, int size)
 {
-    CharDriverState *chr = opaque;
+    TestServer *s = opaque;
+    CharDriverState *chr = s->chr;
     VhostUserMsg msg;
     uint8_t *p = (uint8_t *) &msg;
     int fd;
@@ -280,20 +241,35 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
         return;
     }
 
-    g_mutex_lock(data_mutex);
+    g_mutex_lock(&s->data_mutex);
     memcpy(p, buf, VHOST_USER_HDR_SIZE);
 
     if (msg.size) {
         p += VHOST_USER_HDR_SIZE;
-        qemu_chr_fe_read_all(chr, p, msg.size);
+        g_assert_cmpint(qemu_chr_fe_read_all(chr, p, msg.size), ==, msg.size);
     }
 
     switch (msg.request) {
     case VHOST_USER_GET_FEATURES:
         /* send back features to qemu */
         msg.flags |= VHOST_USER_REPLY_MASK;
-        msg.size = sizeof(m.u64);
-        msg.u64 = 0;
+        msg.size = sizeof(m.payload.u64);
+        msg.payload.u64 = 0x1ULL << VHOST_F_LOG_ALL |
+            0x1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
+        p = (uint8_t *) &msg;
+        qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
+        break;
+
+    case VHOST_USER_SET_FEATURES:
+	g_assert_cmpint(msg.payload.u64 & (0x1ULL << VHOST_USER_F_PROTOCOL_FEATURES),
+			!=, 0ULL);
+        break;
+
+    case VHOST_USER_GET_PROTOCOL_FEATURES:
+        /* send back features to qemu */
+        msg.flags |= VHOST_USER_REPLY_MASK;
+        msg.size = sizeof(m.payload.u64);
+        msg.payload.u64 = 1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD;
         p = (uint8_t *) &msg;
         qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
         break;
@@ -301,19 +277,22 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
     case VHOST_USER_GET_VRING_BASE:
         /* send back vring base to qemu */
         msg.flags |= VHOST_USER_REPLY_MASK;
-        msg.size = sizeof(m.state);
-        msg.state.num = 0;
+        msg.size = sizeof(m.payload.state);
+        msg.payload.state.num = 0;
         p = (uint8_t *) &msg;
         qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
+
+        assert(msg.payload.state.index < 2);
+        s->rings &= ~(0x1ULL << msg.payload.state.index);
         break;
 
     case VHOST_USER_SET_MEM_TABLE:
         /* received the mem table */
-        memcpy(&memory, &msg.memory, sizeof(msg.memory));
-        fds_num = qemu_chr_fe_get_msgfds(chr, fds, sizeof(fds) / sizeof(int));
+        memcpy(&s->memory, &msg.payload.memory, sizeof(msg.payload.memory));
+        s->fds_num = qemu_chr_fe_get_msgfds(chr, s->fds, G_N_ELEMENTS(s->fds));
 
         /* signal the test that it can continue */
-        g_cond_signal(data_cond);
+        g_cond_signal(&s->data_cond);
         break;
 
     case VHOST_USER_SET_VRING_KICK:
@@ -327,23 +306,38 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
          */
         qemu_set_nonblock(fd);
         break;
+
+    case VHOST_USER_SET_LOG_BASE:
+        if (s->log_fd != -1) {
+            close(s->log_fd);
+            s->log_fd = -1;
+        }
+        qemu_chr_fe_get_msgfds(chr, &s->log_fd, 1);
+        msg.flags |= VHOST_USER_REPLY_MASK;
+        msg.size = 0;
+        p = (uint8_t *) &msg;
+        qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE);
+
+        g_cond_signal(&s->data_cond);
+        break;
+
+    case VHOST_USER_SET_VRING_BASE:
+        assert(msg.payload.state.index < 2);
+        s->rings |= 0x1ULL << msg.payload.state.index;
+        break;
+
     default:
         break;
     }
-    g_mutex_unlock(data_mutex);
+
+    g_mutex_unlock(&s->data_mutex);
 }
 
-static const char *init_hugepagefs(void)
+static const char *init_hugepagefs(const char *path)
 {
-    const char *path;
     struct statfs fs;
     int ret;
 
-    path = getenv("QTEST_HUGETLBFS_PATH");
-    if (!path) {
-        path = "/hugetlbfs";
-    }
-
     if (access(path, R_OK | W_OK | X_OK)) {
         g_test_message("access on path (%s): %s\n", path, strerror(errno));
         return NULL;
@@ -366,44 +360,294 @@ static const char *init_hugepagefs(void)
     return path;
 }
 
+static TestServer *test_server_new(const gchar *name)
+{
+    TestServer *server = g_new0(TestServer, 1);
+    gchar *chr_path;
+
+    server->socket_path = g_strdup_printf("%s/%s.sock", tmpfs, name);
+    server->mig_path = g_strdup_printf("%s/%s.mig", tmpfs, name);
+
+    chr_path = g_strdup_printf("unix:%s,server,nowait", server->socket_path);
+    server->chr_name = g_strdup_printf("chr-%s", name);
+    server->chr = qemu_chr_new(server->chr_name, chr_path, NULL);
+    g_free(chr_path);
+
+    qemu_chr_add_handlers(server->chr, chr_can_read, chr_read, NULL, server);
+
+    g_mutex_init(&server->data_mutex);
+    g_cond_init(&server->data_cond);
+
+    server->log_fd = -1;
+
+    return server;
+}
+
+#define GET_QEMU_CMD(s)                                                        \
+    g_strdup_printf(QEMU_CMD, 512, 512, (root), (s)->chr_name,                 \
+                    (s)->socket_path, (s)->chr_name)
+
+#define GET_QEMU_CMDE(s, mem, extra, ...)                                      \
+    g_strdup_printf(QEMU_CMD extra, (mem), (mem), (root), (s)->chr_name,       \
+                    (s)->socket_path, (s)->chr_name, ##__VA_ARGS__)
+
+static gboolean _test_server_free(TestServer *server)
+{
+    int i;
+
+    qemu_chr_delete(server->chr);
+
+    for (i = 0; i < server->fds_num; i++) {
+        close(server->fds[i]);
+    }
+
+    if (server->log_fd != -1) {
+        close(server->log_fd);
+    }
+
+    unlink(server->socket_path);
+    g_free(server->socket_path);
+
+    unlink(server->mig_path);
+    g_free(server->mig_path);
+
+    g_free(server->chr_name);
+    g_free(server);
+
+    return FALSE;
+}
+
+static void test_server_free(TestServer *server)
+{
+    g_idle_add((GSourceFunc)_test_server_free, server);
+}
+
+static void wait_for_log_fd(TestServer *s)
+{
+    gint64 end_time;
+
+    g_mutex_lock(&s->data_mutex);
+    end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+    while (s->log_fd == -1) {
+        if (!g_cond_wait_until(&s->data_cond, &s->data_mutex, end_time)) {
+            /* timeout has passed */
+            g_assert(s->log_fd != -1);
+            break;
+        }
+    }
+
+    g_mutex_unlock(&s->data_mutex);
+}
+
+static void write_guest_mem(TestServer *s, uint32 seed)
+{
+    uint32_t *guest_mem;
+    int i, j;
+    size_t size;
+
+    wait_for_fds(s);
+
+    /* iterate all regions */
+    for (i = 0; i < s->fds_num; i++) {
+
+        /* We'll write only the region statring at 0x0 */
+        if (s->memory.regions[i].guest_phys_addr != 0x0) {
+            continue;
+        }
+
+        g_assert_cmpint(s->memory.regions[i].memory_size, >, 1024);
+
+        size = s->memory.regions[i].memory_size +
+            s->memory.regions[i].mmap_offset;
+
+        guest_mem = mmap(0, size, PROT_READ | PROT_WRITE,
+                         MAP_SHARED, s->fds[i], 0);
+
+        g_assert(guest_mem != MAP_FAILED);
+        guest_mem += (s->memory.regions[i].mmap_offset / sizeof(*guest_mem));
+
+        for (j = 0; j < 256; j++) {
+            guest_mem[j] = seed + j;
+        }
+
+        munmap(guest_mem, s->memory.regions[i].memory_size);
+        break;
+    }
+}
+
+static guint64 get_log_size(TestServer *s)
+{
+    guint64 log_size = 0;
+    int i;
+
+    for (i = 0; i < s->memory.nregions; ++i) {
+        VhostUserMemoryRegion *reg = &s->memory.regions[i];
+        guint64 last = range_get_last(reg->guest_phys_addr,
+                                       reg->memory_size);
+        log_size = MAX(log_size, last / (8 * VHOST_LOG_PAGE) + 1);
+    }
+
+    return log_size;
+}
+
+typedef struct TestMigrateSource {
+    GSource source;
+    TestServer *src;
+    TestServer *dest;
+} TestMigrateSource;
+
+static gboolean
+test_migrate_source_check(GSource *source)
+{
+    TestMigrateSource *t = (TestMigrateSource *)source;
+    gboolean overlap = t->src->rings && t->dest->rings;
+
+    g_assert(!overlap);
+
+    return FALSE;
+}
+
+#if !GLIB_CHECK_VERSION(2,36,0)
+/* this callback is unnecessary with glib >2.36, the default
+ * prepare for the source does the same */
+static gboolean
+test_migrate_source_prepare(GSource *source, gint *timeout)
+{
+    *timeout = -1;
+    return FALSE;
+}
+#endif
+
+GSourceFuncs test_migrate_source_funcs = {
+#if !GLIB_CHECK_VERSION(2,36,0)
+    .prepare = test_migrate_source_prepare,
+#endif
+    .check = test_migrate_source_check,
+};
+
+static void test_migrate(void)
+{
+    TestServer *s = test_server_new("src");
+    TestServer *dest = test_server_new("dest");
+    char *uri = g_strdup_printf("%s%s", "unix:", dest->mig_path);
+    QTestState *global = global_qtest, *from, *to;
+    GSource *source;
+    gchar *cmd;
+    QDict *rsp;
+    guint8 *log;
+    guint64 size;
+
+    cmd = GET_QEMU_CMDE(s, 2, "");
+    from = qtest_start(cmd);
+    g_free(cmd);
+
+    wait_for_fds(s);
+    size = get_log_size(s);
+    g_assert_cmpint(size, ==, (2 * 1024 * 1024) / (VHOST_LOG_PAGE * 8));
+
+    cmd = GET_QEMU_CMDE(dest, 2, " -incoming %s", uri);
+    to = qtest_init(cmd);
+    g_free(cmd);
+
+    source = g_source_new(&test_migrate_source_funcs,
+                          sizeof(TestMigrateSource));
+    ((TestMigrateSource *)source)->src = s;
+    ((TestMigrateSource *)source)->dest = dest;
+    g_source_attach(source, NULL);
+
+    /* slow down migration to have time to fiddle with log */
+    /* TODO: qtest could learn to break on some places */
+    rsp = qmp("{ 'execute': 'migrate_set_speed',"
+              "'arguments': { 'value': 10 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    cmd = g_strdup_printf("{ 'execute': 'migrate',"
+                          "'arguments': { 'uri': '%s' } }",
+                          uri);
+    rsp = qmp(cmd);
+    g_free(cmd);
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    wait_for_log_fd(s);
+
+    log = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, s->log_fd, 0);
+    g_assert(log != MAP_FAILED);
+
+    /* modify first page */
+    write_guest_mem(s, 0x42);
+    log[0] = 1;
+    munmap(log, size);
+
+    /* speed things up */
+    rsp = qmp("{ 'execute': 'migrate_set_speed',"
+              "'arguments': { 'value': 0 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    qmp_eventwait("STOP");
+
+    global_qtest = to;
+    qmp_eventwait("RESUME");
+
+    read_guest_mem(dest);
+
+    g_source_destroy(source);
+    g_source_unref(source);
+
+    qtest_quit(to);
+    test_server_free(dest);
+    qtest_quit(from);
+    test_server_free(s);
+    g_free(uri);
+
+    global_qtest = global;
+}
+
 int main(int argc, char **argv)
 {
     QTestState *s = NULL;
-    CharDriverState *chr = NULL;
-    const char *hugefs = 0;
-    char *socket_path = 0;
-    char *qemu_cmd = 0;
-    char *chr_path = 0;
+    TestServer *server = NULL;
+    const char *hugefs;
+    char *qemu_cmd = NULL;
     int ret;
+    char template[] = "/tmp/vhost-test-XXXXXX";
+    GMainLoop *loop;
+    GThread *thread;
 
     g_test_init(&argc, &argv, NULL);
 
     module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_chardev_opts);
 
-    hugefs = init_hugepagefs();
-    if (!hugefs) {
-        return 0;
+    tmpfs = mkdtemp(template);
+    if (!tmpfs) {
+        g_test_message("mkdtemp on path (%s): %s\n", template, strerror(errno));
+    }
+    g_assert(tmpfs);
+
+    hugefs = getenv("QTEST_HUGETLBFS_PATH");
+    if (hugefs) {
+        root = init_hugepagefs(hugefs);
+        g_assert(root);
+    } else {
+        root = tmpfs;
     }
 
-    socket_path = g_strdup_printf("/tmp/vhost-%d.sock", getpid());
-
-    /* create char dev and add read handlers */
-    qemu_add_opts(&qemu_chardev_opts);
-    chr_path = g_strdup_printf("unix:%s,server,nowait", socket_path);
-    chr = qemu_chr_new("chr0", chr_path, NULL);
-    g_free(chr_path);
-    qemu_chr_add_handlers(chr, chr_can_read, chr_read, NULL, chr);
+    server = test_server_new("test");
 
+    loop = g_main_loop_new(NULL, FALSE);
     /* run the main loop thread so the chardev may operate */
-    data_mutex = _mutex_new();
-    data_cond = _cond_new();
-    _thread_new(NULL, thread_function, NULL);
+    thread = g_thread_new(NULL, thread_function, loop);
+
+    qemu_cmd = GET_QEMU_CMD(server);
 
-    qemu_cmd = g_strdup_printf(QEMU_CMD, hugefs, socket_path);
     s = qtest_start(qemu_cmd);
     g_free(qemu_cmd);
 
-    qtest_add_func("/vhost-user/read-guest-mem", read_guest_mem);
+    qtest_add_data_func("/vhost-user/read-guest-mem", server, read_guest_mem);
+    qtest_add_func("/vhost-user/migrate", test_migrate);
 
     ret = g_test_run();
 
@@ -412,10 +656,22 @@ int main(int argc, char **argv)
     }
 
     /* cleanup */
-    unlink(socket_path);
-    g_free(socket_path);
-    _cond_free(data_cond);
-    _mutex_free(data_mutex);
+    test_server_free(server);
+
+    /* finish the helper thread and dispatch pending sources */
+    g_main_loop_quit(loop);
+    g_thread_join(thread);
+    while (g_main_context_pending(NULL)) {
+        g_main_context_iteration (NULL, TRUE);
+    }
+    g_main_loop_unref(loop);
+
+    ret = rmdir(tmpfs);
+    if (ret != 0) {
+        g_test_message("unable to rmdir: path (%s): %s\n",
+                       tmpfs, strerror(errno));
+    }
+    g_assert_cmpint(ret, ==, 0);
 
     return ret;
 }
diff --git a/tests/virtio-net-test.c b/tests/virtio-net-test.c
index ea7478c27..982d77a14 100644
--- a/tests/virtio-net-test.c
+++ b/tests/virtio-net-test.c
@@ -10,20 +10,242 @@
 #include <glib.h>
 #include <string.h>
 #include "libqtest.h"
+#include "qemu-common.h"
+#include "qemu/sockets.h"
 #include "qemu/osdep.h"
-#include "libqos/pci.h"
+#include "qemu/iov.h"
+#include "libqos/pci-pc.h"
+#include "libqos/virtio.h"
+#include "libqos/virtio-pci.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-pc.h"
+#include "libqos/malloc-generic.h"
+#include "qemu/bswap.h"
+#include "hw/virtio/virtio-net.h"
 
 #define PCI_SLOT_HP             0x06
+#define PCI_SLOT                0x04
+#define PCI_FN                  0x00
 
-/* Tests only initialization so far. TODO: Replace with functional tests */
-static void pci_nop(void)
+#define QVIRTIO_NET_TIMEOUT_US (30 * 1000 * 1000)
+#define VNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf)
+
+static void test_end(void)
+{
+    qtest_end();
+}
+
+#ifndef _WIN32
+
+static QVirtioPCIDevice *virtio_net_pci_init(QPCIBus *bus, int slot)
+{
+    QVirtioPCIDevice *dev;
+
+    dev = qvirtio_pci_device_find(bus, QVIRTIO_NET_DEVICE_ID);
+    g_assert(dev != NULL);
+    g_assert_cmphex(dev->vdev.device_type, ==, QVIRTIO_NET_DEVICE_ID);
+
+    qvirtio_pci_device_enable(dev);
+    qvirtio_reset(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_acknowledge(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_driver(&qvirtio_pci, &dev->vdev);
+
+    return dev;
+}
+
+static QPCIBus *pci_test_start(int socket)
+{
+    char *cmdline;
+
+    cmdline = g_strdup_printf("-netdev socket,fd=%d,id=hs0 -device "
+                              "virtio-net-pci,netdev=hs0", socket);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    return qpci_init_pc();
+}
+
+static void driver_init(const QVirtioBus *bus, QVirtioDevice *dev)
+{
+    uint32_t features;
+
+    features = qvirtio_get_features(bus, dev);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE |
+                            QVIRTIO_F_RING_INDIRECT_DESC |
+                            QVIRTIO_F_RING_EVENT_IDX);
+    qvirtio_set_features(bus, dev, features);
+
+    qvirtio_set_driver_ok(bus, dev);
+}
+
+static void rx_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                    QGuestAllocator *alloc, QVirtQueue *vq,
+                    int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    char test[] = "TEST";
+    char buffer[64];
+    int len = htonl(sizeof(test));
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len = sizeof(len),
+        }, {
+            .iov_base = test,
+            .iov_len = sizeof(test),
+        },
+    };
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, true, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    ret = iov_send(socket, iov, 2, 0, sizeof(len) + sizeof(test));
+    g_assert_cmpint(ret, ==, sizeof(test) + sizeof(len));
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
+    g_assert_cmpstr(buffer, ==, "TEST");
+
+    guest_free(alloc, req_addr);
+}
+
+static void tx_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                    QGuestAllocator *alloc, QVirtQueue *vq,
+                    int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    uint32_t len;
+    char buffer[64];
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+    memwrite(req_addr + VNET_HDR_SIZE, "TEST", 4);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, false, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    guest_free(alloc, req_addr);
+
+    ret = qemu_recv(socket, &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    ret = qemu_recv(socket, buffer, len, 0);
+    g_assert_cmpstr(buffer, ==, "TEST");
+}
+
+static void rx_stop_cont_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                              QGuestAllocator *alloc, QVirtQueue *vq,
+                              int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    char test[] = "TEST";
+    char buffer[64];
+    int len = htonl(sizeof(test));
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len = sizeof(len),
+        }, {
+            .iov_base = test,
+            .iov_len = sizeof(test),
+        },
+    };
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, true, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    qmp("{ 'execute' : 'stop'}");
+
+    ret = iov_send(socket, iov, 2, 0, sizeof(len) + sizeof(test));
+    g_assert_cmpint(ret, ==, sizeof(test) + sizeof(len));
+
+    /* We could check the status, but this command is more importantly to
+     * ensure the packet data gets queued in QEMU, before we do 'cont'.
+     */
+    qmp("{ 'execute' : 'query-status'}");
+    qmp("{ 'execute' : 'cont'}");
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
+    g_assert_cmpstr(buffer, ==, "TEST");
+
+    guest_free(alloc, req_addr);
+}
+
+static void send_recv_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                           QGuestAllocator *alloc, QVirtQueue *rvq,
+                           QVirtQueue *tvq, int socket)
 {
+    rx_test(bus, dev, alloc, rvq, socket);
+    tx_test(bus, dev, alloc, tvq, socket);
 }
 
+static void stop_cont_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                           QGuestAllocator *alloc, QVirtQueue *rvq,
+                           QVirtQueue *tvq, int socket)
+{
+    rx_stop_cont_test(bus, dev, alloc, rvq, socket);
+}
+
+static void pci_basic(gconstpointer data)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *tx, *rx;
+    QGuestAllocator *alloc;
+    void (*func) (const QVirtioBus *bus,
+                  QVirtioDevice *dev,
+                  QGuestAllocator *alloc,
+                  QVirtQueue *rvq,
+                  QVirtQueue *tvq,
+                  int socket) = data;
+    int sv[2], ret;
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, sv);
+    g_assert_cmpint(ret, !=, -1);
+
+    bus = pci_test_start(sv[1]);
+    dev = virtio_net_pci_init(bus, PCI_SLOT);
+
+    alloc = pc_alloc_init();
+    rx = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                           alloc, 0);
+    tx = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                           alloc, 1);
+
+    driver_init(&qvirtio_pci, &dev->vdev);
+    func(&qvirtio_pci, &dev->vdev, alloc, &rx->vq, &tx->vq, sv[0]);
+
+    /* End test */
+    close(sv[0]);
+    guest_free(alloc, tx->vq.desc);
+    pc_alloc_uninit(alloc);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+    qpci_free_pc(bus);
+    test_end();
+}
+#endif
+
 static void hotplug(void)
 {
+    qtest_start("-device virtio-net-pci");
+
     qpci_plug_device_test("virtio-net-pci", "net1", PCI_SLOT_HP, NULL);
     qpci_unplug_acpi_device_test("net1", PCI_SLOT_HP);
+
+    test_end();
 }
 
 int main(int argc, char **argv)
@@ -31,13 +253,14 @@ int main(int argc, char **argv)
     int ret;
 
     g_test_init(&argc, &argv, NULL);
-    qtest_add_func("/virtio/net/pci/nop", pci_nop);
+#ifndef _WIN32
+    qtest_add_data_func("/virtio/net/pci/basic", send_recv_test, pci_basic);
+    qtest_add_data_func("/virtio/net/pci/rx_stop_cont",
+                        stop_cont_test, pci_basic);
+#endif
     qtest_add_func("/virtio/net/pci/hotplug", hotplug);
 
-    qtest_start("-device virtio-net-pci");
     ret = g_test_run();
 
-    qtest_end();
-
     return ret;
 }
diff --git a/tests/virtio-scsi-test.c b/tests/virtio-scsi-test.c
index 11ccdd632..66d8491e9 100644
--- a/tests/virtio-scsi-test.c
+++ b/tests/virtio-scsi-test.c
@@ -13,6 +13,7 @@
 #include "libqtest.h"
 #include "qemu/osdep.h"
 #include <stdio.h>
+#include "block/scsi.h"
 #include "libqos/virtio.h"
 #include "libqos/virtio-pci.h"
 #include "libqos/pci-pc.h"
@@ -71,40 +72,6 @@ static void qvirtio_scsi_stop(void)
     qtest_end();
 }
 
-static QVirtIOSCSI *qvirtio_scsi_pci_init(int slot)
-{
-    QVirtIOSCSI *vs;
-    QVirtioPCIDevice *dev;
-    void *addr;
-    int i;
-
-    vs = g_new0(QVirtIOSCSI, 1);
-    vs->alloc = pc_alloc_init();
-    vs->bus = qpci_init_pc();
-
-    dev = qvirtio_pci_device_find(vs->bus, QVIRTIO_SCSI_DEVICE_ID);
-    vs->dev = (QVirtioDevice *)dev;
-    g_assert(dev != NULL);
-    g_assert_cmphex(vs->dev->device_type, ==, QVIRTIO_SCSI_DEVICE_ID);
-
-    qvirtio_pci_device_enable(dev);
-    qvirtio_reset(&qvirtio_pci, vs->dev);
-    qvirtio_set_acknowledge(&qvirtio_pci, vs->dev);
-    qvirtio_set_driver(&qvirtio_pci, vs->dev);
-
-    addr = dev->addr + QVIRTIO_PCI_DEVICE_SPECIFIC_NO_MSIX;
-    vs->num_queues = qvirtio_config_readl(&qvirtio_pci, vs->dev,
-                                          (uint64_t)(uintptr_t)addr);
-
-    g_assert_cmpint(vs->num_queues, <, MAX_NUM_QUEUES);
-
-    for (i = 0; i < vs->num_queues + 2; i++) {
-        vs->vq[i] = qvirtqueue_setup(&qvirtio_pci, vs->dev, vs->alloc, i);
-    }
-
-    return vs;
-}
-
 static void qvirtio_scsi_pci_free(QVirtIOSCSI *vs)
 {
     int i;
@@ -134,7 +101,8 @@ static uint64_t qvirtio_scsi_alloc(QVirtIOSCSI *vs, size_t alloc_size,
 static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
                                       const uint8_t *data_in,
                                       size_t data_in_len,
-                                      uint8_t *data_out, size_t data_out_len)
+                                      uint8_t *data_out, size_t data_out_len,
+                                      QVirtIOSCSICmdResp *resp_out)
 {
     QVirtQueue *vq;
     QVirtIOSCSICmdReq req = { { 0 } };
@@ -174,6 +142,10 @@ static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
 
     response = readb(resp_addr + offsetof(QVirtIOSCSICmdResp, response));
 
+    if (resp_out) {
+        memread(resp_addr, resp_out, sizeof(*resp_out));
+    }
+
     guest_free(vs->alloc, req_addr);
     guest_free(vs->alloc, resp_addr);
     guest_free(vs->alloc, data_in_addr);
@@ -181,6 +153,52 @@ static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
     return response;
 }
 
+static QVirtIOSCSI *qvirtio_scsi_pci_init(int slot)
+{
+    const uint8_t test_unit_ready_cdb[CDB_SIZE] = {};
+    QVirtIOSCSI *vs;
+    QVirtioPCIDevice *dev;
+    QVirtIOSCSICmdResp resp;
+    void *addr;
+    int i;
+
+    vs = g_new0(QVirtIOSCSI, 1);
+    vs->alloc = pc_alloc_init();
+    vs->bus = qpci_init_pc();
+
+    dev = qvirtio_pci_device_find(vs->bus, QVIRTIO_SCSI_DEVICE_ID);
+    vs->dev = (QVirtioDevice *)dev;
+    g_assert(dev != NULL);
+    g_assert_cmphex(vs->dev->device_type, ==, QVIRTIO_SCSI_DEVICE_ID);
+
+    qvirtio_pci_device_enable(dev);
+    qvirtio_reset(&qvirtio_pci, vs->dev);
+    qvirtio_set_acknowledge(&qvirtio_pci, vs->dev);
+    qvirtio_set_driver(&qvirtio_pci, vs->dev);
+
+    addr = dev->addr + QVIRTIO_PCI_DEVICE_SPECIFIC_NO_MSIX;
+    vs->num_queues = qvirtio_config_readl(&qvirtio_pci, vs->dev,
+                                          (uint64_t)(uintptr_t)addr);
+
+    g_assert_cmpint(vs->num_queues, <, MAX_NUM_QUEUES);
+
+    for (i = 0; i < vs->num_queues + 2; i++) {
+        vs->vq[i] = qvirtqueue_setup(&qvirtio_pci, vs->dev, vs->alloc, i);
+    }
+
+    /* Clear the POWER ON OCCURRED unit attention */
+    g_assert_cmpint(virtio_scsi_do_command(vs, test_unit_ready_cdb,
+                                           NULL, 0, NULL, 0, &resp),
+                    ==, 0);
+    g_assert_cmpint(resp.status, ==, CHECK_CONDITION);
+    g_assert_cmpint(resp.sense[0], ==, 0x70); /* Fixed format sense buffer */
+    g_assert_cmpint(resp.sense[2], ==, UNIT_ATTENTION);
+    g_assert_cmpint(resp.sense[12], ==, 0x29); /* POWER ON */
+    g_assert_cmpint(resp.sense[13], ==, 0x00);
+
+    return vs;
+}
+
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
 {
@@ -221,9 +239,12 @@ static void hotplug(void)
 static void test_unaligned_write_same(void)
 {
     QVirtIOSCSI *vs;
-    uint8_t buf[512] = { 0 };
-    const uint8_t write_same_cdb[CDB_SIZE] = { 0x41, 0x00, 0x00, 0x00, 0x00,
+    uint8_t buf1[512] = { 0 };
+    uint8_t buf2[512] = { 1 };
+    const uint8_t write_same_cdb_1[CDB_SIZE] = { 0x41, 0x00, 0x00, 0x00, 0x00,
                                                0x01, 0x00, 0x00, 0x02, 0x00 };
+    const uint8_t write_same_cdb_2[CDB_SIZE] = { 0x41, 0x00, 0x00, 0x00, 0x00,
+                                               0x01, 0x00, 0x33, 0x00, 0x00 };
 
     qvirtio_scsi_start("-drive file=blkdebug::null-co://,if=none,id=dr1"
                        ",format=raw,file.align=4k "
@@ -231,7 +252,10 @@ static void test_unaligned_write_same(void)
     vs = qvirtio_scsi_pci_init(PCI_SLOT);
 
     g_assert_cmphex(0, ==,
-        virtio_scsi_do_command(vs, write_same_cdb, NULL, 0, buf, 512));
+        virtio_scsi_do_command(vs, write_same_cdb_1, NULL, 0, buf1, 512, NULL));
+
+    g_assert_cmphex(0, ==,
+        virtio_scsi_do_command(vs, write_same_cdb_2, NULL, 0, buf2, 512, NULL));
 
     qvirtio_scsi_pci_free(vs);
     qvirtio_scsi_stop();
diff --git a/thread-pool.c b/thread-pool.c
index ac909f498..402c778b4 100644
--- a/thread-pool.c
+++ b/thread-pool.c
@@ -18,7 +18,7 @@
 #include "qemu/queue.h"
 #include "qemu/thread.h"
 #include "qemu/osdep.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "trace.h"
 #include "block/thread-pool.h"
 #include "qemu/main-loop.h"
diff --git a/tpm.c b/tpm.c
index 4e9b109fb..f2c59d1f7 100644
--- a/tpm.c
+++ b/tpm.c
@@ -260,9 +260,9 @@ static TPMInfo *qmp_query_tpm_inst(TPMBackend *drv)
 
     switch (drv->ops->type) {
     case TPM_TYPE_PASSTHROUGH:
-        res->options->kind = TPM_TYPE_OPTIONS_KIND_PASSTHROUGH;
+        res->options->type = TPM_TYPE_OPTIONS_KIND_PASSTHROUGH;
         tpo = g_new0(TPMPassthroughOptions, 1);
-        res->options->passthrough = tpo;
+        res->options->u.passthrough = tpo;
         if (drv->path) {
             tpo->path = g_strdup(drv->path);
             tpo->has_path = true;
diff --git a/trace-events b/trace-events
index e19873078..2fce98e76 100644
--- a/trace-events
+++ b/trace-events
@@ -69,6 +69,7 @@ bdrv_aio_write_zeroes(void *bs, int64_t sector_num, int nb_sectors, int flags, v
 bdrv_lock_medium(void *bs, bool locked) "bs %p locked %d"
 bdrv_co_readv(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
 bdrv_co_copy_on_readv(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
+bdrv_co_readv_no_serialising(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
 bdrv_co_writev(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
 bdrv_co_write_zeroes(void *bs, int64_t sector_num, int nb_sector, int flags) "bs %p sector_num %"PRId64" nb_sectors %d flags %#x"
 bdrv_co_io_em(void *bs, int64_t sector_num, int nb_sectors, int is_write, void *acb) "bs %p sector_num %"PRId64" nb_sectors %d is_write %d acb %p"
@@ -94,6 +95,7 @@ mirror_yield(void *s, int64_t cnt, int buf_free_count, int in_flight) "s %p dirt
 mirror_yield_in_flight(void *s, int64_t sector_num, int in_flight) "s %p sector_num %"PRId64" in_flight %d"
 mirror_yield_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
 mirror_break_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
+mirror_break_iov_max(void *s, int nb_chunks, int added_chunks) "s %p requested chunks %d added_chunks %d"
 
 # block/backup.c
 backup_do_cow_enter(void *job, int64_t start, int64_t sector_num, int nb_sectors) "job %p start %"PRId64" sector_num %"PRId64" nb_sectors %d"
@@ -137,8 +139,8 @@ paio_submit_co(int64_t sector_num, int nb_sectors, int type) "sector_num %"PRId6
 paio_submit(void *acb, void *opaque, int64_t sector_num, int nb_sectors, int type) "acb %p opaque %p sector_num %"PRId64" nb_sectors %d type %d"
 
 # ioport.c
-cpu_in(unsigned int addr, unsigned int val) "addr %#x value %u"
-cpu_out(unsigned int addr, unsigned int val) "addr %#x value %u"
+cpu_in(unsigned int addr, char size, unsigned int val) "addr %#x(%c) value %u"
+cpu_out(unsigned int addr, char size, unsigned int val) "addr %#x(%c) value %u"
 
 # balloon.c
 # Since requests are raised via monitor, not many tracepoints are needed.
@@ -932,6 +934,13 @@ xen_map_portio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %
 xen_unmap_portio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %u start: %#"PRIx64" end: %#"PRIx64
 xen_map_pcidev(uint32_t id, uint8_t bus, uint8_t dev, uint8_t func) "id: %u bdf: %02x.%02x.%02x"
 xen_unmap_pcidev(uint32_t id, uint8_t bus, uint8_t dev, uint8_t func) "id: %u bdf: %02x.%02x.%02x"
+handle_ioreq(void *req, uint32_t type, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p type=%d dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+handle_ioreq_read(void *req, uint32_t type, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p read type=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+handle_ioreq_write(void *req, uint32_t type, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p write type=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+cpu_ioreq_pio(void *req, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p pio dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+cpu_ioreq_pio_read_reg(void *req, uint64_t data, uint64_t addr, uint32_t size) "I/O=%p pio read reg data=%#"PRIx64" port=%#"PRIx64" size=%d"
+cpu_ioreq_pio_write_reg(void *req, uint64_t data, uint64_t addr, uint32_t size) "I/O=%p pio write reg data=%#"PRIx64" port=%#"PRIx64" size=%d"
+cpu_ioreq_move(void *req, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p copy dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
 
 # xen-mapcache.c
 xen_map_cache(uint64_t phys_addr) "want %#"PRIx64
@@ -1022,9 +1031,9 @@ esp_pci_sbac_write(uint32_t reg, uint32_t val) "sbac: 0x%8.8x -> 0x%8.8x"
 # monitor.c
 handle_qmp_command(void *mon, const char *cmd_name) "mon %p cmd_name \"%s\""
 monitor_protocol_emitter(void *mon) "mon %p"
-monitor_protocol_event_handler(uint32_t event, void *data, uint64_t last, uint64_t now) "event=%d data=%p last=%" PRId64 " now=%" PRId64
+monitor_protocol_event_handler(uint32_t event, void *qdict) "event=%d data=%p"
 monitor_protocol_event_emit(uint32_t event, void *data) "event=%d data=%p"
-monitor_protocol_event_queue(uint32_t event, void *data, uint64_t rate, uint64_t last, uint64_t now) "event=%d data=%p rate=%" PRId64 " last=%" PRId64 " now=%" PRId64
+monitor_protocol_event_queue(uint32_t event, void *qdict, uint64_t rate) "event=%d data=%p rate=%" PRId64
 monitor_protocol_event_throttle(uint32_t event, uint64_t rate) "event=%d rate=%" PRId64
 
 # hw/net/opencores_eth.c
@@ -1139,7 +1148,8 @@ ppm_save(const char *filename, void *display_surface) "%s surface=%p"
 gd_switch(const char *tab, int width, int height) "tab=%s, width=%d, height=%d"
 gd_update(const char *tab, int x, int y, int w, int h) "tab=%s, x=%d, y=%d, w=%d, h=%d"
 gd_key_event(const char *tab, int gdk_keycode, int qemu_keycode, const char *action) "tab=%s, translated GDK keycode %d to QEMU keycode %d (%s)"
-gd_grab(const char *tab, const char *device, bool on) "tab=%s, %s %d"
+gd_grab(const char *tab, const char *device, const char *reason) "tab=%s, dev=%s, reason=%s"
+gd_ungrab(const char *tab, const char *device) "tab=%s, dev=%s"
 
 # ui/vnc.c
 vnc_key_guest_leds(bool caps, bool num, bool scroll) "caps %d, num %d, scroll %d"
@@ -1168,6 +1178,7 @@ vmware_scratch_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
 vmware_setmode(uint32_t w, uint32_t h, uint32_t bpp) "%dx%d @ %d bpp"
 
 # hw/display/virtio-gpu.c
+virtio_gpu_features(bool virgl) "virgl %d"
 virtio_gpu_cmd_get_display_info(void) ""
 virtio_gpu_cmd_get_caps(void) ""
 virtio_gpu_cmd_set_scanout(uint32_t id, uint32_t res, uint32_t w, uint32_t h, uint32_t x, uint32_t y) "id %d, res 0x%x, w %d, h %d, x %d, y %d"
@@ -1177,22 +1188,57 @@ virtio_gpu_cmd_res_unref(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_back_attach(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_back_detach(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_xfer_toh_2d(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_xfer_toh_3d(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_xfer_fromh_3d(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_flush(uint32_t res, uint32_t w, uint32_t h, uint32_t x, uint32_t y) "res 0x%x, w %d, h %d, x %d, y %d"
+virtio_gpu_cmd_ctx_create(uint32_t ctx, const char *name) "ctx 0x%x, name %s"
+virtio_gpu_cmd_ctx_destroy(uint32_t ctx) "ctx 0x%x"
+virtio_gpu_cmd_ctx_res_attach(uint32_t ctx, uint32_t res) "ctx 0x%x, res 0x%x"
+virtio_gpu_cmd_ctx_res_detach(uint32_t ctx, uint32_t res) "ctx 0x%x, res 0x%x"
+virtio_gpu_cmd_ctx_submit(uint32_t ctx, uint32_t size) "ctx 0x%x, size %d"
+virtio_gpu_update_cursor(uint32_t scanout, uint32_t x, uint32_t y, const char *type, uint32_t res) "scanout %d, x %d, y %d, %s, res 0x%x"
 virtio_gpu_fence_ctrl(uint64_t fence, uint32_t type) "fence 0x%" PRIx64 ", type 0x%x"
 virtio_gpu_fence_resp(uint64_t fence) "fence 0x%" PRIx64
 
 # migration/savevm.c
 qemu_loadvm_state_section(unsigned int section_type) "%d"
+qemu_loadvm_state_section_command(int ret) "%d"
 qemu_loadvm_state_section_partend(uint32_t section_id) "%u"
+qemu_loadvm_state_main(void) ""
+qemu_loadvm_state_main_quit_parent(void) ""
+qemu_loadvm_state_post_main(int ret) "%d"
 qemu_loadvm_state_section_startfull(uint32_t section_id, const char *idstr, uint32_t instance_id, uint32_t version_id) "%u(%s) %u %u"
+qemu_savevm_send_packaged(void) ""
+loadvm_handle_cmd_packaged(unsigned int length) "%u"
+loadvm_handle_cmd_packaged_main(int ret) "%d"
+loadvm_handle_cmd_packaged_received(int ret) "%d"
+loadvm_postcopy_handle_advise(void) ""
+loadvm_postcopy_handle_listen(void) ""
+loadvm_postcopy_handle_run(void) ""
+loadvm_postcopy_handle_run_cpu_sync(void) ""
+loadvm_postcopy_handle_run_vmstart(void) ""
+loadvm_postcopy_ram_handle_discard(void) ""
+loadvm_postcopy_ram_handle_discard_end(void) ""
+loadvm_postcopy_ram_handle_discard_header(const char *ramid, uint16_t len) "%s: %ud"
+loadvm_process_command(uint16_t com, uint16_t len) "com=0x%x len=%d"
+loadvm_process_command_ping(uint32_t val) "%x"
+postcopy_ram_listen_thread_exit(void) ""
+postcopy_ram_listen_thread_start(void) ""
+qemu_savevm_send_postcopy_advise(void) ""
+qemu_savevm_send_postcopy_ram_discard(const char *id, uint16_t len) "%s: %ud"
+savevm_command_send(uint16_t command, uint16_t len) "com=0x%x len=%d"
 savevm_section_start(const char *id, unsigned int section_id) "%s, section_id %u"
 savevm_section_end(const char *id, unsigned int section_id, int ret) "%s, section_id %u -> %d"
 savevm_section_skip(const char *id, unsigned int section_id) "%s, section_id %u"
+savevm_send_open_return_path(void) ""
+savevm_send_ping(uint32_t val) "%x"
+savevm_send_postcopy_listen(void) ""
+savevm_send_postcopy_run(void) ""
 savevm_state_begin(void) ""
 savevm_state_header(void) ""
 savevm_state_iterate(void) ""
-savevm_state_complete(void) ""
-savevm_state_cancel(void) ""
+savevm_state_cleanup(void) ""
+savevm_state_complete_precopy(void) ""
 vmstate_save(const char *idstr, const char *vmsd_name) "%s, %s"
 vmstate_load(const char *idstr, const char *vmsd_name) "%s, %s"
 qemu_announce_self_iter(const char *mac) "%s"
@@ -1210,9 +1256,14 @@ vmstate_subsection_load_good(const char *parent) "%s"
 qemu_file_fclose(void) ""
 
 # migration/ram.c
+get_queued_page(const char *block_name, uint64_t tmp_offset, uint64_t ram_addr) "%s/%" PRIx64 " ram_addr=%" PRIx64
+get_queued_page_not_dirty(const char *block_name, uint64_t tmp_offset, uint64_t ram_addr, int sent) "%s/%" PRIx64 " ram_addr=%" PRIx64 " (sent=%d)"
 migration_bitmap_sync_start(void) ""
 migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64""
 migration_throttle(void) ""
+ram_load_postcopy_loop(uint64_t addr, int flags) "@%" PRIx64 " %x"
+ram_postcopy_send_discard_bitmap(void) ""
+ram_save_queue_pages(const char *rbname, size_t start, size_t len) "%s: start: %zx len: %zx"
 
 # hw/display/qxl.c
 disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
@@ -1299,8 +1350,8 @@ spapr_pci_lsi_set(const char *busname, int pin, uint32_t irq) "%s PIN%d IRQ %u"
 spapr_pci_msi_retry(unsigned config_addr, unsigned req_num, unsigned max_irqs) "Guest device at %x asked %u, have only %u"
 
 # hw/pci/pci.c
-pci_update_mappings_del(void *d, uint32_t bus, uint32_t func, uint32_t slot, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
-pci_update_mappings_add(void *d, uint32_t bus, uint32_t func, uint32_t slot, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
+pci_update_mappings_del(void *d, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
+pci_update_mappings_add(void *d, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
 
 # hw/net/pcnet.c
 pcnet_s_reset(void *s) "s=%p"
@@ -1364,11 +1415,25 @@ spapr_iommu_new_table(uint64_t liobn, void *tcet, void *table, int fd) "liobn=%"
 # hw/ppc/ppc.c
 ppc_tb_adjust(uint64_t offs1, uint64_t offs2, int64_t diff, int64_t seconds) "adjusted from 0x%"PRIx64" to 0x%"PRIx64", diff %"PRId64" (%"PRId64"s)"
 
+# hw/ppc/prep.c
+prep_io_800_writeb(uint32_t addr, uint32_t val) "0x%08" PRIx32 " => 0x%02" PRIx32
+prep_io_800_readb(uint32_t addr, uint32_t retval) "0x%08" PRIx32 " <= 0x%02" PRIx32
+
+# io/buffer.c
+buffer_resize(const char *buf, size_t olen, size_t len) "%s: old %zd, new %zd"
+buffer_move_empty(const char *buf, size_t len, const char *from) "%s: %zd bytes from %s"
+buffer_move(const char *buf, size_t len, const char *from) "%s: %zd bytes from %s"
+buffer_free(const char *buf, size_t len) "%s: capacity %zd"
+
 # util/hbitmap.c
 hbitmap_iter_skip_words(const void *hb, void *hbi, uint64_t pos, unsigned long cur) "hb %p hbi %p pos %"PRId64" cur 0x%lx"
 hbitmap_reset(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
 hbitmap_set(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
 
+# target-s390x/mmu_helper.c
+get_skeys_nonzero(int rc) "SKEY: Call to get_skeys unexpectedly returned %d"
+set_skeys_nonzero(int rc) "SKEY: Call to set_skeys unexpectedly returned %d"
+
 # target-s390x/ioinst.c
 ioinst(const char *insn) "IOINST: %s"
 ioinst_sch_id(const char *insn, int cssid, int ssid, int schid) "IOINST: %s (%x.%x.%04x)"
@@ -1394,15 +1459,40 @@ flic_no_device_api(int err) "flic: no Device Contral API support %d"
 flic_reset_failed(int err) "flic: reset failed %d"
 
 # migration.c
+await_return_path_close_on_source_close(void) ""
+await_return_path_close_on_source_joining(void) ""
 migrate_set_state(int new_state) "new state %d"
 migrate_fd_cleanup(void) ""
 migrate_fd_error(void) ""
 migrate_fd_cancel(void) ""
-migrate_pending(uint64_t size, uint64_t max) "pending size %" PRIu64 " max %" PRIu64
-migrate_transferred(uint64_t tranferred, uint64_t time_spent, double bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " bandwidth %g max_size %" PRId64
-migrate_state_too_big(void) ""
+migrate_handle_rp_req_pages(const char *rbname, size_t start, size_t len) "in %s at %zx len %zx"
+migrate_pending(uint64_t size, uint64_t max, uint64_t post, uint64_t nonpost) "pending size %" PRIu64 " max %" PRIu64 " (post=%" PRIu64 " nonpost=%" PRIu64 ")"
+migrate_send_rp_message(int msg_type, uint16_t len) "%d: len %d"
+migration_completion_file_err(void) ""
+migration_completion_postcopy_end(void) ""
+migration_completion_postcopy_end_after_complete(void) ""
+migration_completion_postcopy_end_before_rp(void) ""
+migration_completion_postcopy_end_after_rp(int rp_error) "%d"
+migration_thread_after_loop(void) ""
+migration_thread_file_err(void) ""
+migration_thread_setup_complete(void) ""
+open_return_path_on_source(void) ""
+open_return_path_on_source_continue(void) ""
+postcopy_start(void) ""
+postcopy_start_set_run(void) ""
+source_return_path_thread_bad_end(void) ""
+source_return_path_thread_end(void) ""
+source_return_path_thread_entry(void) ""
+source_return_path_thread_loop_top(void) ""
+source_return_path_thread_pong(uint32_t val) "%x"
+source_return_path_thread_shut(uint32_t val) "%x"
 migrate_global_state_post_load(const char *state) "loaded state: %s"
 migrate_global_state_pre_save(const char *state) "saved state: %s"
+migration_thread_low_pending(uint64_t pending) "%" PRIu64
+migrate_state_too_big(void) ""
+migrate_transferred(uint64_t tranferred, uint64_t time_spent, double bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " bandwidth %g max_size %" PRId64
+process_incoming_migration_co_end(int ret, int ps) "ret=%d postcopy-state=%d"
+process_incoming_migration_co_postcopy_end_main(void) ""
 
 # migration/rdma.c
 qemu_rdma_accept_incoming_migration(void) ""
@@ -1423,7 +1513,7 @@ qemu_rdma_exchange_get_response_none(const char *desc, int type) "Surprise: got
 qemu_rdma_exchange_send_issue_callback(void) ""
 qemu_rdma_exchange_send_waiting(const char *desc) "Waiting for response %s"
 qemu_rdma_exchange_send_received(const char *desc) "Response %s received."
-qemu_rdma_fill(int64_t control_len, int size) "RDMA %" PRId64 " of %d bytes already in buffer"
+qemu_rdma_fill(size_t control_len, size_t size) "RDMA %zd of %zd bytes already in buffer"
 qemu_rdma_init_ram_blocks(int blocks) "Allocated %d local ram block structures"
 qemu_rdma_poll_recv(const char *compstr, int64_t comp, int64_t id, int sent) "completion %s #%" PRId64 " received (%" PRId64 ") left %d"
 qemu_rdma_poll_write(const char *compstr, int64_t comp, int left, uint64_t block, uint64_t chunk, void *local, void *remote) "completions %s (%" PRId64 ") left %d, block %" PRIu64 ", chunk: %" PRIu64 " %p %p"
@@ -1468,6 +1558,25 @@ rdma_start_incoming_migration_after_rdma_listen(void) ""
 rdma_start_outgoing_migration_after_rdma_connect(void) ""
 rdma_start_outgoing_migration_after_rdma_source_init(void) ""
 
+# migration/postcopy-ram.c
+postcopy_discard_send_finish(const char *ramblock, int nwords, int ncmds) "%s mask words sent=%d in %d commands"
+postcopy_discard_send_range(const char *ramblock, unsigned long start, unsigned long length) "%s:%lx/%lx"
+postcopy_ram_discard_range(void *start, size_t length) "%p,+%zx"
+postcopy_cleanup_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_init_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_nhp_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_place_page(void *host_addr) "host=%p"
+postcopy_place_page_zero(void *host_addr) "host=%p"
+postcopy_ram_enable_notify(void) ""
+postcopy_ram_fault_thread_entry(void) ""
+postcopy_ram_fault_thread_exit(void) ""
+postcopy_ram_fault_thread_quit(void) ""
+postcopy_ram_fault_thread_request(uint64_t hostaddr, const char *ramblock, size_t offset) "Request for HVA=%" PRIx64 " rb=%s offset=%zx"
+postcopy_ram_incoming_cleanup_closeuf(void) ""
+postcopy_ram_incoming_cleanup_entry(void) ""
+postcopy_ram_incoming_cleanup_exit(void) ""
+postcopy_ram_incoming_cleanup_join(void) ""
+
 # kvm-all.c
 kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
 kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
@@ -1506,56 +1615,30 @@ xen_pv_mmio_write(uint64_t addr) "WARNING: write to Xen PV Device MMIO space (ad
 pci_cfg_read(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x -> 0x%x"
 pci_cfg_write(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x <- 0x%x"
 
-# hw/vfio/vfio-pci.c
+# hw/vfio/pci.c
 vfio_intx_interrupt(const char *name, char line) " (%s) Pin %c"
-vfio_eoi(const char *name) " (%s) EOI"
-vfio_enable_intx_kvm(const char *name) " (%s) KVM INTx accel enabled"
-vfio_disable_intx_kvm(const char *name) " (%s) KVM INTx accel disabled"
-vfio_update_irq(const char *name, int new_irq, int target_irq) " (%s) IRQ moved %d -> %d"
-vfio_enable_intx(const char *name) " (%s)"
-vfio_disable_intx(const char *name) " (%s)"
+vfio_intx_eoi(const char *name) " (%s) EOI"
+vfio_intx_enable_kvm(const char *name) " (%s) KVM INTx accel enabled"
+vfio_intx_disable_kvm(const char *name) " (%s) KVM INTx accel disabled"
+vfio_intx_update(const char *name, int new_irq, int target_irq) " (%s) IRQ moved %d -> %d"
+vfio_intx_enable(const char *name) " (%s)"
+vfio_intx_disable(const char *name) " (%s)"
 vfio_msi_interrupt(const char *name, int index, uint64_t addr, int data) " (%s) vector %d 0x%"PRIx64"/0x%x"
 vfio_msix_vector_do_use(const char *name, int index) " (%s) vector %d used"
 vfio_msix_vector_release(const char *name, int index) " (%s) vector %d released"
-vfio_enable_msix(const char *name) " (%s)"
-vfio_enable_msi(const char *name, int nr_vectors) " (%s) Enabled %d MSI vectors"
-vfio_disable_msix(const char *name) " (%s)"
-vfio_disable_msi(const char *name) " (%s)"
+vfio_msix_enable(const char *name) " (%s)"
+vfio_msix_disable(const char *name) " (%s)"
+vfio_msi_enable(const char *name, int nr_vectors) " (%s) Enabled %d MSI vectors"
+vfio_msi_disable(const char *name) " (%s)"
 vfio_pci_load_rom(const char *name, unsigned long size, unsigned long offset, unsigned long flags) "Device %s ROM:\n  size: 0x%lx, offset: 0x%lx, flags: 0x%lx"
 vfio_rom_read(const char *name, uint64_t addr, int size, uint64_t data) " (%s, 0x%"PRIx64", 0x%x) = 0x%"PRIx64
 vfio_pci_size_rom(const char *name, int size) "%s ROM size 0x%x"
 vfio_vga_write(uint64_t addr, uint64_t data, int size) " (0x%"PRIx64", 0x%"PRIx64", %d)"
 vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%"PRIx64
-# remove ) =
-vfio_generic_window_quirk_read(const char * region_name, const char *name, int index, uint64_t addr, int size, uint64_t data) "%s read(%s:BAR%d+0x%"PRIx64", %d = 0x%"PRIx64
-## remove )
-vfio_generic_window_quirk_write(const char * region_name, const char *name, int index, uint64_t addr, uint64_t data, int size) "%s write(%s:BAR%d+0x%"PRIx64", 0x%"PRIx64", %d"
-# remove ) =
-vfio_generic_quirk_read(const char * region_name, const char *name, int index, uint64_t addr, int size, uint64_t data) "%s read(%s:BAR%d+0x%"PRIx64", %d = 0x%"PRIx64
-# remove )
-vfio_generic_quirk_write(const char * region_name, const char *name, int index, uint64_t addr, uint64_t data, int size) "%s write(%s:BAR%d+0x%"PRIx64", 0x%"PRIx64", %d"
-vfio_ati_3c3_quirk_read(uint64_t data) " (0x3c3, 1) = 0x%"PRIx64
-vfio_vga_probe_ati_3c3_quirk(const char *name) "Enabled ATI/AMD quirk 0x3c3 BAR4for device %s"
-vfio_probe_ati_bar4_window_quirk(const char *name) "Enabled ATI/AMD BAR4 window quirk for device %s"
-#issue with )
-vfio_rtl8168_window_quirk_read_fake(const char *region_name, const char *name) "%s fake read(%s"
-vfio_rtl8168_window_quirk_read_table(const char *region_name, const char *name) "%s MSI-X table read(%s"
-vfio_rtl8168_window_quirk_read_direct(const char *region_name, const char *name) "%s direct read(%s"
-vfio_rtl8168_window_quirk_write_table(const char *region_name, const char *name) "%s MSI-X table write(%s"
-vfio_rtl8168_window_quirk_write_direct(const char *region_name, const char *name) "%s direct write(%s"
-vfio_probe_rtl8168_bar2_window_quirk(const char *name) "Enabled RTL8168 BAR2 window quirk for device %s"
-vfio_probe_ati_bar2_4000_quirk(const char *name) "Enabled ATI/AMD BAR2 0x4000 quirk for device %s"
-vfio_nvidia_3d0_quirk_read(int size, uint64_t data) " (0x3d0, %d) = 0x%"PRIx64
-vfio_nvidia_3d0_quirk_write(uint64_t data, int size) " (0x3d0, 0x%"PRIx64", %d)"
-vfio_vga_probe_nvidia_3d0_quirk(const char *name) "Enabled NVIDIA VGA 0x3d0 quirk for device %s"
-vfio_probe_nvidia_bar5_window_quirk(const char *name) "Enabled NVIDIA BAR5 window quirk for device %s"
-vfio_probe_nvidia_bar0_88000_quirk(const char *name) "Enabled NVIDIA BAR0 0x88000 quirk for device %s"
-vfio_probe_nvidia_bar0_1800_quirk_id(int id) "Nvidia NV%02x"
-vfio_probe_nvidia_bar0_1800_quirk(const char *name) "Enabled NVIDIA BAR0 0x1800 quirk for device %s"
 vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) %x"
 vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)"
-vfio_setup_msi(const char *name, int pos) "%s PCI MSI CAP @0x%x"
-vfio_early_setup_msix(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
+vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
+vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
 vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
 vfio_check_pm_reset(const char *name) "%s Supports PM reset"
 vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"
@@ -1570,6 +1653,41 @@ vfio_initfn(const char *name, int group_id) " (%s) group %d"
 vfio_pci_reset(const char *name) " (%s)"
 vfio_pci_reset_flr(const char *name) "%s FLR/VFIO_DEVICE_RESET"
 vfio_pci_reset_pm(const char *name) "%s PCI PM Reset"
+vfio_pci_emulated_vendor_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_device_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_sub_vendor_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_sub_device_id(const char *name, uint16_t val) "%s %04x"
+
+# hw/vfio/pci-quirks.
+vfio_quirk_rom_blacklisted(const char *name, uint16_t vid, uint16_t did) "%s %04x:%04x"
+vfio_quirk_generic_window_address_write(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_window_data_read(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_window_data_write(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_mirror_read(const char *name, const char * region_name, uint64_t addr, uint64_t data) "%s %s 0x%"PRIx64": 0x%"PRIx64
+vfio_quirk_generic_mirror_write(const char *name, const char * region_name, uint64_t addr, uint64_t data) "%s %s 0x%"PRIx64": 0x%"PRIx64
+vfio_quirk_ati_3c3_read(const char *name, uint64_t data) "%s 0x%"PRIx64
+vfio_quirk_ati_3c3_probe(const char *name) "%s"
+vfio_quirk_ati_bar4_probe(const char *name) "%s"
+vfio_quirk_ati_bar2_probe(const char *name) "%s"
+vfio_quirk_nvidia_3d0_state(const char *name, const char *state) "%s %s"
+vfio_quirk_nvidia_3d0_read(const char *name, uint8_t offset, unsigned size, uint64_t val) " (%s, @0x%x, len=0x%x) %"PRIx64
+vfio_quirk_nvidia_3d0_write(const char *name, uint8_t offset, uint64_t data, unsigned size) "(%s, @0x%x, 0x%"PRIx64", len=0x%x)"
+vfio_quirk_nvidia_3d0_probe(const char *name) "%s"
+vfio_quirk_nvidia_bar5_state(const char *name, const char *state) "%s %s"
+vfio_quirk_nvidia_bar5_probe(const char *name) "%s"
+vfio_quirk_nvidia_bar0_msi_ack(const char *name) "%s"
+vfio_quirk_nvidia_bar0_probe(const char *name) "%s"
+vfio_quirk_rtl8168_fake_latch(const char *name, uint64_t val) "%s 0x%"PRIx64
+vfio_quirk_rtl8168_msix_write(const char *name, uint16_t offset, uint64_t val) "%s MSI-X table write[0x%x]: 0x%"PRIx64
+vfio_quirk_rtl8168_msix_read(const char *name, uint16_t offset, uint64_t val) "%s MSI-X table read[0x%x]: 0x%"PRIx64
+vfio_quirk_rtl8168_probe(const char *name) "%s"
+
+vfio_quirk_ati_bonaire_reset_skipped(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_no_smc(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_timeout(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_done(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset(const char *name) "%s"
+
 
 # hw/vfio/vfio-common.c
 vfio_region_write(const char *name, int index, uint64_t addr, uint64_t data, unsigned size) " (%s:region%d+0x%"PRIx64", 0x%"PRIx64 ", %d)"
@@ -1596,7 +1714,9 @@ vfio_platform_intp_interrupt(int pin, int fd) "Inject IRQ #%d (fd = %d)"
 vfio_platform_intp_inject_pending_lockheld(int pin, int fd) "Inject pending IRQ #%d (fd = %d)"
 vfio_platform_populate_interrupts(int pin, int count, int flags) "- IRQ index %d: count %d, flags=0x%x"
 vfio_intp_interrupt_set_pending(int index) "irq %d is set PENDING"
-vfio_platform_start_irqfd_injection(int index, int fd, int resamplefd) "IRQ index=%d, fd = %d, resamplefd = %d"
+vfio_platform_start_level_irqfd_injection(int index, int fd, int resamplefd) "IRQ index=%d, fd = %d, resamplefd = %d"
+vfio_platform_start_edge_irqfd_injection(int index, int fd) "IRQ index=%d, fd = %d"
+
 
 #hw/acpi/memory_hotplug.c
 mhp_acpi_invalid_slot_selected(uint32_t slot) "0x%"PRIx32
@@ -1651,3 +1771,24 @@ alsa_no_frames(int state) "No frames available and ALSA state is %d"
 # audio/ossaudio.c
 oss_version(int version) "OSS version = %#x"
 oss_invalid_available_size(int size, int bufsize) "Invalid available size, size=%d bufsize=%d"
+
+# crypto/tlscreds.c
+qcrypto_tls_creds_load_dh(void *creds, const char *filename) "TLS creds load DH creds=%p filename=%s"
+qcrypto_tls_creds_get_path(void *creds, const char *filename, const char *path) "TLS creds path creds=%p filename=%s path=%s"
+
+# crypto/tlscredsanon.c
+qcrypto_tls_creds_anon_load(void *creds, const char *dir) "TLS creds anon load creds=%p dir=%s"
+
+# crypto/tlscredsx509.c
+qcrypto_tls_creds_x509_load(void *creds, const char *dir) "TLS creds x509 load creds=%p dir=%s"
+qcrypto_tls_creds_x509_check_basic_constraints(void *creds, const char *file, int status) "TLS creds x509 check basic constraints creds=%p file=%s status=%d"
+qcrypto_tls_creds_x509_check_key_usage(void *creds, const char *file, int status, int usage, int critical) "TLS creds x509 check key usage creds=%p file=%s status=%d usage=%d critical=%d"
+qcrypto_tls_creds_x509_check_key_purpose(void *creds, const char *file, int status, const char *usage, int critical) "TLS creds x509 check key usage creds=%p file=%s status=%d usage=%s critical=%d"
+qcrypto_tls_creds_x509_load_cert(void *creds, int isServer, const char *file) "TLS creds x509 load cert creds=%p isServer=%d file=%s"
+qcrypto_tls_creds_x509_load_cert_list(void *creds, const char *file) "TLS creds x509 load cert list creds=%p file=%s"
+
+# crypto/tlssession.c
+qcrypto_tls_session_new(void *session, void *creds, const char *hostname, const char *aclname, int endpoint) "TLS session new session=%p creds=%p hostname=%s aclname=%s endpoint=%d"
+
+# net/vhost-user.c
+vhost_user_event(const char *chr, int event) "chr: %s got event: %d"
diff --git a/trace/simple.c b/trace/simple.c
index 11ad03093..56a624cac 100644
--- a/trace/simple.c
+++ b/trace/simple.c
@@ -329,7 +329,8 @@ bool st_set_trace_file(const char *file)
     g_free(trace_file_name);
 
     if (!file) {
-        trace_file_name = g_strdup_printf(CONFIG_TRACE_FILE, getpid());
+        /* Type cast needed for Windows where getpid() returns an int. */
+        trace_file_name = g_strdup_printf(CONFIG_TRACE_FILE, (pid_t)getpid());
     } else {
         trace_file_name = g_strdup_printf("%s", file);
     }
diff --git a/translate-all.c b/translate-all.c
index 60a3d8b2b..042a8576a 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -117,18 +117,48 @@ typedef struct PageDesc {
 
 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
 
-uintptr_t qemu_real_host_page_size;
-uintptr_t qemu_real_host_page_mask;
 uintptr_t qemu_host_page_size;
-uintptr_t qemu_host_page_mask;
+intptr_t qemu_host_page_mask;
 
-/* This is a multi-level map on the virtual address space.
-   The bottom level has pointers to PageDesc.  */
+/* The bottom level has pointers to PageDesc */
 static void *l1_map[V_L1_SIZE];
 
 /* code generation context */
 TCGContext tcg_ctx;
 
+/* translation block context */
+#ifdef CONFIG_USER_ONLY
+__thread int have_tb_lock;
+#endif
+
+void tb_lock(void)
+{
+#ifdef CONFIG_USER_ONLY
+    assert(!have_tb_lock);
+    qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
+    have_tb_lock++;
+#endif
+}
+
+void tb_unlock(void)
+{
+#ifdef CONFIG_USER_ONLY
+    assert(have_tb_lock);
+    have_tb_lock--;
+    qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+#endif
+}
+
+void tb_lock_reset(void)
+{
+#ifdef CONFIG_USER_ONLY
+    if (have_tb_lock) {
+        qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+        have_tb_lock = 0;
+    }
+#endif
+}
+
 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
                          tb_page_addr_t phys_page2);
 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
@@ -138,124 +168,137 @@ void cpu_gen_init(void)
     tcg_context_init(&tcg_ctx); 
 }
 
-/* return non zero if the very first instruction is invalid so that
-   the virtual CPU can trigger an exception.
+/* Encode VAL as a signed leb128 sequence at P.
+   Return P incremented past the encoded value.  */
+static uint8_t *encode_sleb128(uint8_t *p, target_long val)
+{
+    int more, byte;
+
+    do {
+        byte = val & 0x7f;
+        val >>= 7;
+        more = !((val == 0 && (byte & 0x40) == 0)
+                 || (val == -1 && (byte & 0x40) != 0));
+        if (more) {
+            byte |= 0x80;
+        }
+        *p++ = byte;
+    } while (more);
 
-   '*gen_code_size_ptr' contains the size of the generated code (host
-   code).
-*/
-int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
+    return p;
+}
+
+/* Decode a signed leb128 sequence at *PP; increment *PP past the
+   decoded value.  Return the decoded value.  */
+static target_long decode_sleb128(uint8_t **pp)
 {
-    TCGContext *s = &tcg_ctx;
-    tcg_insn_unit *gen_code_buf;
-    int gen_code_size;
-#ifdef CONFIG_PROFILER
-    int64_t ti;
-#endif
+    uint8_t *p = *pp;
+    target_long val = 0;
+    int byte, shift = 0;
 
-#ifdef CONFIG_PROFILER
-    s->tb_count1++; /* includes aborted translations because of
-                       exceptions */
-    ti = profile_getclock();
-#endif
-    tcg_func_start(s);
+    do {
+        byte = *p++;
+        val |= (target_ulong)(byte & 0x7f) << shift;
+        shift += 7;
+    } while (byte & 0x80);
+    if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
+        val |= -(target_ulong)1 << shift;
+    }
 
-    gen_intermediate_code(env, tb);
+    *pp = p;
+    return val;
+}
 
-    trace_translate_block(tb, tb->pc, tb->tc_ptr);
+/* Encode the data collected about the instructions while compiling TB.
+   Place the data at BLOCK, and return the number of bytes consumed.
 
-    /* generate machine code */
-    gen_code_buf = tb->tc_ptr;
-    tb->tb_next_offset[0] = 0xffff;
-    tb->tb_next_offset[1] = 0xffff;
-    s->tb_next_offset = tb->tb_next_offset;
-#ifdef USE_DIRECT_JUMP
-    s->tb_jmp_offset = tb->tb_jmp_offset;
-    s->tb_next = NULL;
-#else
-    s->tb_jmp_offset = NULL;
-    s->tb_next = tb->tb_next;
-#endif
+   The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
+   which come from the target's insn_start data, followed by a uintptr_t
+   which comes from the host pc of the end of the code implementing the insn.
 
-#ifdef CONFIG_PROFILER
-    s->tb_count++;
-    s->interm_time += profile_getclock() - ti;
-    s->code_time -= profile_getclock();
-#endif
-    gen_code_size = tcg_gen_code(s, gen_code_buf);
-    *gen_code_size_ptr = gen_code_size;
-#ifdef CONFIG_PROFILER
-    s->code_time += profile_getclock();
-    s->code_in_len += tb->size;
-    s->code_out_len += gen_code_size;
-#endif
+   Each line of the table is encoded as sleb128 deltas from the previous
+   line.  The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
+   That is, the first column is seeded with the guest pc, the last column
+   with the host pc, and the middle columns with zeros.  */
 
-#ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
-        qemu_log("OUT: [size=%d]\n", gen_code_size);
-        log_disas(tb->tc_ptr, gen_code_size);
-        qemu_log("\n");
-        qemu_log_flush();
+static int encode_search(TranslationBlock *tb, uint8_t *block)
+{
+    uint8_t *highwater = tcg_ctx.code_gen_highwater;
+    uint8_t *p = block;
+    int i, j, n;
+
+    tb->tc_search = block;
+
+    for (i = 0, n = tb->icount; i < n; ++i) {
+        target_ulong prev;
+
+        for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
+            if (i == 0) {
+                prev = (j == 0 ? tb->pc : 0);
+            } else {
+                prev = tcg_ctx.gen_insn_data[i - 1][j];
+            }
+            p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
+        }
+        prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
+        p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
+
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one row beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           encoding a row without having to check during encoding.  */
+        if (unlikely(p > highwater)) {
+            return -1;
+        }
     }
-#endif
-    return 0;
+
+    return p - block;
 }
 
-/* The cpu state corresponding to 'searched_pc' is restored.
- */
+/* The cpu state corresponding to 'searched_pc' is restored.  */
 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
                                      uintptr_t searched_pc)
 {
+    target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
+    uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
     CPUArchState *env = cpu->env_ptr;
-    TCGContext *s = &tcg_ctx;
-    int j;
-    uintptr_t tc_ptr;
+    uint8_t *p = tb->tc_search;
+    int i, j, num_insns = tb->icount;
 #ifdef CONFIG_PROFILER
-    int64_t ti;
+    int64_t ti = profile_getclock();
 #endif
 
-#ifdef CONFIG_PROFILER
-    ti = profile_getclock();
-#endif
-    tcg_func_start(s);
+    if (searched_pc < host_pc) {
+        return -1;
+    }
 
-    gen_intermediate_code_pc(env, tb);
+    /* Reconstruct the stored insn data while looking for the point at
+       which the end of the insn exceeds the searched_pc.  */
+    for (i = 0; i < num_insns; ++i) {
+        for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
+            data[j] += decode_sleb128(&p);
+        }
+        host_pc += decode_sleb128(&p);
+        if (host_pc > searched_pc) {
+            goto found;
+        }
+    }
+    return -1;
 
+ found:
     if (tb->cflags & CF_USE_ICOUNT) {
+        assert(use_icount);
         /* Reset the cycle counter to the start of the block.  */
-        cpu->icount_decr.u16.low += tb->icount;
+        cpu->icount_decr.u16.low += num_insns;
         /* Clear the IO flag.  */
         cpu->can_do_io = 0;
     }
-
-    /* find opc index corresponding to search_pc */
-    tc_ptr = (uintptr_t)tb->tc_ptr;
-    if (searched_pc < tc_ptr)
-        return -1;
-
-    s->tb_next_offset = tb->tb_next_offset;
-#ifdef USE_DIRECT_JUMP
-    s->tb_jmp_offset = tb->tb_jmp_offset;
-    s->tb_next = NULL;
-#else
-    s->tb_jmp_offset = NULL;
-    s->tb_next = tb->tb_next;
-#endif
-    j = tcg_gen_code_search_pc(s, (tcg_insn_unit *)tc_ptr,
-                               searched_pc - tc_ptr);
-    if (j < 0)
-        return -1;
-    /* now find start of instruction before */
-    while (s->gen_opc_instr_start[j] == 0) {
-        j--;
-    }
-    cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
-
-    restore_state_to_opc(env, tb, j);
+    cpu->icount_decr.u16.low -= i;
+    restore_state_to_opc(env, tb, data);
 
 #ifdef CONFIG_PROFILER
-    s->restore_time += profile_getclock() - ti;
-    s->restore_count++;
+    tcg_ctx.restore_time += profile_getclock() - ti;
+    tcg_ctx.restore_count++;
 #endif
     return 0;
 }
@@ -278,44 +321,19 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
     return false;
 }
 
-#ifdef _WIN32
-static __attribute__((unused)) void map_exec(void *addr, long size)
-{
-    DWORD old_protect;
-    VirtualProtect(addr, size,
-                   PAGE_EXECUTE_READWRITE, &old_protect);
-}
-#else
-static __attribute__((unused)) void map_exec(void *addr, long size)
-{
-    unsigned long start, end, page_size;
-
-    page_size = getpagesize();
-    start = (unsigned long)addr;
-    start &= ~(page_size - 1);
-
-    end = (unsigned long)addr + size;
-    end += page_size - 1;
-    end &= ~(page_size - 1);
-
-    mprotect((void *)start, end - start,
-             PROT_READ | PROT_WRITE | PROT_EXEC);
-}
-#endif
-
 void page_size_init(void)
 {
     /* NOTE: we can always suppose that qemu_host_page_size >=
        TARGET_PAGE_SIZE */
     qemu_real_host_page_size = getpagesize();
-    qemu_real_host_page_mask = ~(qemu_real_host_page_size - 1);
+    qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
     if (qemu_host_page_size == 0) {
         qemu_host_page_size = qemu_real_host_page_size;
     }
     if (qemu_host_page_size < TARGET_PAGE_SIZE) {
         qemu_host_page_size = TARGET_PAGE_SIZE;
     }
-    qemu_host_page_mask = ~(qemu_host_page_size - 1);
+    qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
 }
 
 static void page_init(void)
@@ -387,6 +405,9 @@ static void page_init(void)
 #endif
 }
 
+/* If alloc=1:
+ * Called with mmap_lock held for user-mode emulation.
+ */
 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
 {
     PageDesc *pd;
@@ -398,26 +419,26 @@ static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
 
     /* Level 2..N-1.  */
     for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
-        void **p = *lp;
+        void **p = atomic_rcu_read(lp);
 
         if (p == NULL) {
             if (!alloc) {
                 return NULL;
             }
             p = g_new0(void *, V_L2_SIZE);
-            *lp = p;
+            atomic_rcu_set(lp, p);
         }
 
         lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
     }
 
-    pd = *lp;
+    pd = atomic_rcu_read(lp);
     if (pd == NULL) {
         if (!alloc) {
             return NULL;
         }
         pd = g_new0(PageDesc, V_L2_SIZE);
-        *lp = pd;
+        atomic_rcu_set(lp, pd);
     }
 
     return pd + (index & (V_L2_SIZE - 1));
@@ -428,11 +449,6 @@ static inline PageDesc *page_find(tb_page_addr_t index)
     return page_find_alloc(index, 0);
 }
 
-#if !defined(CONFIG_USER_ONLY)
-#define mmap_lock() do { } while (0)
-#define mmap_unlock() do { } while (0)
-#endif
-
 #if defined(CONFIG_USER_ONLY)
 /* Currently it is not recommended to allocate big chunks of data in
    user mode. It will change when a dedicated libc will be used.  */
@@ -441,14 +457,6 @@ static inline PageDesc *page_find(tb_page_addr_t index)
 #define USE_STATIC_CODE_GEN_BUFFER
 #endif
 
-/* ??? Should configure for this, not list operating systems here.  */
-#if (defined(__linux__) \
-    || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
-    || defined(__DragonFly__) || defined(__OpenBSD__) \
-    || defined(__NetBSD__))
-# define USE_MMAP
-#endif
-
 /* Minimum size of the code gen buffer.  This number is randomly chosen,
    but not so small that we can't have a fair number of TB's live.  */
 #define MIN_CODE_GEN_BUFFER_SIZE     (1024u * 1024)
@@ -460,6 +468,8 @@ static inline PageDesc *page_find(tb_page_addr_t index)
 # define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
 #elif defined(__sparc__)
 # define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__powerpc64__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
 #elif defined(__aarch64__)
 # define MAX_CODE_GEN_BUFFER_SIZE  (128ul * 1024 * 1024)
 #elif defined(__arm__)
@@ -536,22 +546,102 @@ static inline void *split_cross_256mb(void *buf1, size_t size1)
 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
     __attribute__((aligned(CODE_GEN_ALIGN)));
 
+# ifdef _WIN32
+static inline void do_protect(void *addr, long size, int prot)
+{
+    DWORD old_protect;
+    VirtualProtect(addr, size, prot, &old_protect);
+}
+
+static inline void map_exec(void *addr, long size)
+{
+    do_protect(addr, size, PAGE_EXECUTE_READWRITE);
+}
+
+static inline void map_none(void *addr, long size)
+{
+    do_protect(addr, size, PAGE_NOACCESS);
+}
+# else
+static inline void do_protect(void *addr, long size, int prot)
+{
+    uintptr_t start, end;
+
+    start = (uintptr_t)addr;
+    start &= qemu_real_host_page_mask;
+
+    end = (uintptr_t)addr + size;
+    end = ROUND_UP(end, qemu_real_host_page_size);
+
+    mprotect((void *)start, end - start, prot);
+}
+
+static inline void map_exec(void *addr, long size)
+{
+    do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
+}
+
+static inline void map_none(void *addr, long size)
+{
+    do_protect(addr, size, PROT_NONE);
+}
+# endif /* WIN32 */
+
 static inline void *alloc_code_gen_buffer(void)
 {
     void *buf = static_code_gen_buffer;
+    size_t full_size, size;
+
+    /* The size of the buffer, rounded down to end on a page boundary.  */
+    full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
+                 & qemu_real_host_page_mask) - (uintptr_t)buf;
+
+    /* Reserve a guard page.  */
+    size = full_size - qemu_real_host_page_size;
+
+    /* Honor a command-line option limiting the size of the buffer.  */
+    if (size > tcg_ctx.code_gen_buffer_size) {
+        size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
+                & qemu_real_host_page_mask) - (uintptr_t)buf;
+    }
+    tcg_ctx.code_gen_buffer_size = size;
+
 #ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
-        buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
+    if (cross_256mb(buf, size)) {
+        buf = split_cross_256mb(buf, size);
+        size = tcg_ctx.code_gen_buffer_size;
     }
 #endif
-    map_exec(buf, tcg_ctx.code_gen_buffer_size);
+
+    map_exec(buf, size);
+    map_none(buf + size, qemu_real_host_page_size);
+    qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
+
     return buf;
 }
-#elif defined(USE_MMAP)
+#elif defined(_WIN32)
+static inline void *alloc_code_gen_buffer(void)
+{
+    size_t size = tcg_ctx.code_gen_buffer_size;
+    void *buf1, *buf2;
+
+    /* Perform the allocation in two steps, so that the guard page
+       is reserved but uncommitted.  */
+    buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
+                        MEM_RESERVE, PAGE_NOACCESS);
+    if (buf1 != NULL) {
+        buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+        assert(buf1 == buf2);
+    }
+
+    return buf1;
+}
+#else
 static inline void *alloc_code_gen_buffer(void)
 {
     int flags = MAP_PRIVATE | MAP_ANONYMOUS;
     uintptr_t start = 0;
+    size_t size = tcg_ctx.code_gen_buffer_size;
     void *buf;
 
     /* Constrain the position of the buffer based on the host cpu.
@@ -567,86 +657,70 @@ static inline void *alloc_code_gen_buffer(void)
        Leave the choice of exact location with the kernel.  */
     flags |= MAP_32BIT;
     /* Cannot expect to map more than 800MB in low memory.  */
-    if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
-        tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
+    if (size > 800u * 1024 * 1024) {
+        tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
     }
 # elif defined(__sparc__)
     start = 0x40000000ul;
 # elif defined(__s390x__)
     start = 0x90000000ul;
 # elif defined(__mips__)
-    /* ??? We ought to more explicitly manage layout for softmmu too.  */
-#  ifdef CONFIG_USER_ONLY
-    start = 0x68000000ul;
-#  elif _MIPS_SIM == _ABI64
+#  if _MIPS_SIM == _ABI64
     start = 0x128000000ul;
 #  else
     start = 0x08000000ul;
 #  endif
 # endif
 
-    buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
-               PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
+    buf = mmap((void *)start, size + qemu_real_host_page_size,
+               PROT_NONE, flags, -1, 0);
     if (buf == MAP_FAILED) {
         return NULL;
     }
 
 #ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
+    if (cross_256mb(buf, size)) {
         /* Try again, with the original still mapped, to avoid re-acquiring
            that 256mb crossing.  This time don't specify an address.  */
-        size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
-        void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
-                          flags, -1, 0);
-        if (buf2 != MAP_FAILED) {
-            if (!cross_256mb(buf2, size1)) {
+        size_t size2;
+        void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
+                          PROT_NONE, flags, -1, 0);
+        switch (buf2 != MAP_FAILED) {
+        case 1:
+            if (!cross_256mb(buf2, size)) {
                 /* Success!  Use the new buffer.  */
-                munmap(buf, size1);
-                return buf2;
+                munmap(buf, size);
+                break;
             }
             /* Failure.  Work with what we had.  */
-            munmap(buf2, size1);
+            munmap(buf2, size);
+            /* fallthru */
+        default:
+            /* Split the original buffer.  Free the smaller half.  */
+            buf2 = split_cross_256mb(buf, size);
+            size2 = tcg_ctx.code_gen_buffer_size;
+            if (buf == buf2) {
+                munmap(buf + size2 + qemu_real_host_page_size, size - size2);
+            } else {
+                munmap(buf, size - size2);
+            }
+            size = size2;
+            break;
         }
-
-        /* Split the original buffer.  Free the smaller half.  */
-        buf2 = split_cross_256mb(buf, size1);
-        size2 = tcg_ctx.code_gen_buffer_size;
-        munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
-        return buf2;
+        buf = buf2;
     }
 #endif
 
-    return buf;
-}
-#else
-static inline void *alloc_code_gen_buffer(void)
-{
-    void *buf = g_try_malloc(tcg_ctx.code_gen_buffer_size);
-
-    if (buf == NULL) {
-        return NULL;
-    }
+    /* Make the final buffer accessible.  The guard page at the end
+       will remain inaccessible with PROT_NONE.  */
+    mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
 
-#ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
-        void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
-        if (buf2 != NULL && !cross_256mb(buf2, size1)) {
-            /* Success!  Use the new buffer.  */
-            free(buf);
-            buf = buf2;
-        } else {
-            /* Failure.  Work with what we had.  Since this is malloc
-               and not mmap, we can't free the other half.  */
-            free(buf2);
-            buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
-        }
-    }
-#endif
+    /* Request large pages for the buffer.  */
+    qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
 
-    map_exec(buf, tcg_ctx.code_gen_buffer_size);
     return buf;
 }
-#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
+#endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
 
 static inline void code_gen_alloc(size_t tb_size)
 {
@@ -657,24 +731,14 @@ static inline void code_gen_alloc(size_t tb_size)
         exit(1);
     }
 
-    qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
-            QEMU_MADV_HUGEPAGE);
+    /* Estimate a good size for the number of TBs we can support.  We
+       still haven't deducted the prologue from the buffer size here,
+       but that's minimal and won't affect the estimate much.  */
+    tcg_ctx.code_gen_max_blocks
+        = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
+    tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
 
-    /* Steal room for the prologue at the end of the buffer.  This ensures
-       (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
-       from TB's to the prologue are going to be in range.  It also means
-       that we don't need to mark (additional) portions of the data segment
-       as executable.  */
-    tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
-            tcg_ctx.code_gen_buffer_size - 1024;
-    tcg_ctx.code_gen_buffer_size -= 1024;
-
-    tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
-        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
-    tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
-            CODE_GEN_AVG_BLOCK_SIZE;
-    tcg_ctx.tb_ctx.tbs =
-            g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
+    qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
 }
 
 /* Must be called before using the QEMU cpus. 'tb_size' is the size
@@ -683,11 +747,9 @@ static inline void code_gen_alloc(size_t tb_size)
 void tcg_exec_init(unsigned long tb_size)
 {
     cpu_gen_init();
-    code_gen_alloc(tb_size);
-    tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
-    tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
     page_init();
-#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
+    code_gen_alloc(tb_size);
+#if defined(CONFIG_SOFTMMU)
     /* There's no guest base to take into account, so go ahead and
        initialize the prologue now.  */
     tcg_prologue_init(&tcg_ctx);
@@ -705,9 +767,7 @@ static TranslationBlock *tb_alloc(target_ulong pc)
 {
     TranslationBlock *tb;
 
-    if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
-        (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
-         tcg_ctx.code_gen_buffer_max_size) {
+    if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
         return NULL;
     }
     tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
@@ -730,10 +790,8 @@ void tb_free(TranslationBlock *tb)
 
 static inline void invalidate_page_bitmap(PageDesc *p)
 {
-    if (p->code_bitmap) {
-        g_free(p->code_bitmap);
-        p->code_bitmap = NULL;
-    }
+    g_free(p->code_bitmap);
+    p->code_bitmap = NULL;
     p->code_write_count = 0;
 }
 
@@ -995,6 +1053,7 @@ static void build_page_bitmap(PageDesc *p)
     }
 }
 
+/* Called with mmap_lock held for user mode emulation.  */
 TranslationBlock *tb_gen_code(CPUState *cpu,
                               target_ulong pc, target_ulong cs_base,
                               int flags, int cflags)
@@ -1003,28 +1062,98 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
     TranslationBlock *tb;
     tb_page_addr_t phys_pc, phys_page2;
     target_ulong virt_page2;
-    int code_gen_size;
+    tcg_insn_unit *gen_code_buf;
+    int gen_code_size, search_size;
+#ifdef CONFIG_PROFILER
+    int64_t ti;
+#endif
 
     phys_pc = get_page_addr_code(env, pc);
-    if (use_icount) {
+    if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
         cflags |= CF_USE_ICOUNT;
     }
+
     tb = tb_alloc(pc);
-    if (!tb) {
+    if (unlikely(!tb)) {
+ buffer_overflow:
         /* flush must be done */
         tb_flush(cpu);
         /* cannot fail at this point */
         tb = tb_alloc(pc);
+        assert(tb != NULL);
         /* Don't forget to invalidate previous TB info.  */
         tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
     }
-    tb->tc_ptr = tcg_ctx.code_gen_ptr;
+
+    gen_code_buf = tcg_ctx.code_gen_ptr;
+    tb->tc_ptr = gen_code_buf;
     tb->cs_base = cs_base;
     tb->flags = flags;
     tb->cflags = cflags;
-    cpu_gen_code(env, tb, &code_gen_size);
-    tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
-            code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.tb_count1++; /* includes aborted translations because of
+                       exceptions */
+    ti = profile_getclock();
+#endif
+
+    tcg_func_start(&tcg_ctx);
+
+    gen_intermediate_code(env, tb);
+
+    trace_translate_block(tb, tb->pc, tb->tc_ptr);
+
+    /* generate machine code */
+    tb->tb_next_offset[0] = 0xffff;
+    tb->tb_next_offset[1] = 0xffff;
+    tcg_ctx.tb_next_offset = tb->tb_next_offset;
+#ifdef USE_DIRECT_JUMP
+    tcg_ctx.tb_jmp_offset = tb->tb_jmp_offset;
+    tcg_ctx.tb_next = NULL;
+#else
+    tcg_ctx.tb_jmp_offset = NULL;
+    tcg_ctx.tb_next = tb->tb_next;
+#endif
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.tb_count++;
+    tcg_ctx.interm_time += profile_getclock() - ti;
+    tcg_ctx.code_time -= profile_getclock();
+#endif
+
+    /* ??? Overflow could be handled better here.  In particular, we
+       don't need to re-do gen_intermediate_code, nor should we re-do
+       the tcg optimization currently hidden inside tcg_gen_code.  All
+       that should be required is to flush the TBs, allocate a new TB,
+       re-initialize it per above, and re-do the actual code generation.  */
+    gen_code_size = tcg_gen_code(&tcg_ctx, gen_code_buf);
+    if (unlikely(gen_code_size < 0)) {
+        goto buffer_overflow;
+    }
+    search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
+    if (unlikely(search_size < 0)) {
+        goto buffer_overflow;
+    }
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.code_time += profile_getclock();
+    tcg_ctx.code_in_len += tb->size;
+    tcg_ctx.code_out_len += gen_code_size;
+    tcg_ctx.search_out_len += search_size;
+#endif
+
+#ifdef DEBUG_DISAS
+    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
+        qemu_log("OUT: [size=%d]\n", gen_code_size);
+        log_disas(tb->tc_ptr, gen_code_size);
+        qemu_log("\n");
+        qemu_log_flush();
+    }
+#endif
+
+    tcg_ctx.code_gen_ptr = (void *)
+        ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
+                 CODE_GEN_ALIGN);
 
     /* check next page if needed */
     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
@@ -1042,6 +1171,8 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
  * 'is_cpu_write_access' should be true if called from a real cpu write
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
  */
 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
 {
@@ -1058,6 +1189,8 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
  * 'is_cpu_write_access' should be true if called from a real cpu write
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
  */
 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                    int is_cpu_write_access)
@@ -1206,6 +1339,7 @@ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
 }
 
 #if !defined(CONFIG_SOFTMMU)
+/* Called with mmap_lock held.  */
 static void tb_invalidate_phys_page(tb_page_addr_t addr,
                                     uintptr_t pc, void *puc,
                                     bool locked)
@@ -1275,7 +1409,10 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr,
 }
 #endif
 
-/* add the tb in the target page and protect it if necessary */
+/* add the tb in the target page and protect it if necessary
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
 static inline void tb_alloc_page(TranslationBlock *tb,
                                  unsigned int n, tb_page_addr_t page_addr)
 {
@@ -1331,16 +1468,16 @@ static inline void tb_alloc_page(TranslationBlock *tb,
 }
 
 /* add a new TB and link it to the physical page tables. phys_page2 is
-   (-1) to indicate that only one page contains the TB. */
+ * (-1) to indicate that only one page contains the TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
                          tb_page_addr_t phys_page2)
 {
     unsigned int h;
     TranslationBlock **ptb;
 
-    /* Grab the mmap lock to stop another thread invalidating this TB
-       before we are done.  */
-    mmap_lock();
     /* add in the physical hash table */
     h = tb_phys_hash_func(phys_pc);
     ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
@@ -1370,7 +1507,6 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
 #ifdef DEBUG_TB_CHECK
     tb_page_check();
 #endif
-    mmap_unlock();
 }
 
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
@@ -1451,36 +1587,6 @@ void tb_check_watchpoint(CPUState *cpu)
 }
 
 #ifndef CONFIG_USER_ONLY
-/* mask must never be zero, except for A20 change call */
-static void tcg_handle_interrupt(CPUState *cpu, int mask)
-{
-    int old_mask;
-
-    old_mask = cpu->interrupt_request;
-    cpu->interrupt_request |= mask;
-
-    /*
-     * If called from iothread context, wake the target cpu in
-     * case its halted.
-     */
-    if (!qemu_cpu_is_self(cpu)) {
-        qemu_cpu_kick(cpu);
-        return;
-    }
-
-    if (use_icount) {
-        cpu->icount_decr.u16.high = 0xffff;
-        if (!cpu_can_do_io(cpu)
-            && (mask & ~old_mask) != 0) {
-            cpu_abort(cpu, "Raised interrupt while not in I/O function");
-        }
-    } else {
-        cpu->tcg_exit_req = 1;
-    }
-}
-
-CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
-
 /* in deterministic execution mode, instructions doing device I/Os
    must be at the end of the TB */
 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
@@ -1533,6 +1639,14 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
     cs_base = tb->cs_base;
     flags = tb->flags;
     tb_phys_invalidate(tb, -1);
+    if (tb->cflags & CF_NOCACHE) {
+        if (tb->orig_tb) {
+            /* Invalidate original TB if this TB was generated in
+             * cpu_exec_nocache() */
+            tb_phys_invalidate(tb->orig_tb, -1);
+        }
+        tb_free(tb);
+    }
     /* FIXME: In theory this could raise an exception.  In practice
        we have already translated the block once so it's probably ok.  */
     tb_gen_code(cpu, pc, cs_base, flags, cflags);
@@ -1590,7 +1704,7 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
     cpu_fprintf(f, "Translation buffer state:\n");
     cpu_fprintf(f, "gen code size       %td/%zd\n",
                 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
-                tcg_ctx.code_gen_buffer_max_size);
+                tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
     cpu_fprintf(f, "TB count            %d/%d\n",
             tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
     cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
diff --git a/translate-common.c b/translate-common.c
new file mode 100644
index 000000000..171222d03
--- /dev/null
+++ b/translate-common.c
@@ -0,0 +1,56 @@
+/*
+ *  Host code generation common components
+ *
+ *  Copyright (c) 2015 Peter Crosthwaite <crosthwaite.peter@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu-common.h"
+#include "qom/cpu.h"
+
+uintptr_t qemu_real_host_page_size;
+intptr_t qemu_real_host_page_mask;
+
+#ifndef CONFIG_USER_ONLY
+/* mask must never be zero, except for A20 change call */
+static void tcg_handle_interrupt(CPUState *cpu, int mask)
+{
+    int old_mask;
+
+    old_mask = cpu->interrupt_request;
+    cpu->interrupt_request |= mask;
+
+    /*
+     * If called from iothread context, wake the target cpu in
+     * case its halted.
+     */
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
+        return;
+    }
+
+    if (use_icount) {
+        cpu->icount_decr.u16.high = 0xffff;
+        if (!cpu->can_do_io
+            && (mask & ~old_mask) != 0) {
+            cpu_abort(cpu, "Raised interrupt while not in I/O function");
+        }
+    } else {
+        cpu->tcg_exit_req = 1;
+    }
+}
+
+CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
+#endif
diff --git a/ui/Makefile.objs b/ui/Makefile.objs
index c62d4d972..728393c5e 100644
--- a/ui/Makefile.objs
+++ b/ui/Makefile.objs
@@ -2,7 +2,7 @@ vnc-obj-y += vnc.o
 vnc-obj-y += vnc-enc-zlib.o vnc-enc-hextile.o
 vnc-obj-y += vnc-enc-tight.o vnc-palette.o
 vnc-obj-y += vnc-enc-zrle.o
-vnc-obj-$(CONFIG_VNC_TLS) += vnc-tls.o vnc-auth-vencrypt.o
+vnc-obj-y += vnc-auth-vencrypt.o
 vnc-obj-$(CONFIG_VNC_SASL) += vnc-auth-sasl.o
 vnc-obj-y += vnc-ws.o
 vnc-obj-y += vnc-jobs.o
@@ -31,11 +31,17 @@ ifeq ($(CONFIG_OPENGL),y)
 common-obj-y += shader.o
 common-obj-y += console-gl.o
 common-obj-y += egl-helpers.o
+common-obj-y += egl-context.o
+ifeq ($(CONFIG_GTK_GL),y)
+common-obj-$(CONFIG_GTK) += gtk-gl-area.o
+else
 common-obj-$(CONFIG_GTK) += gtk-egl.o
 endif
+endif
 
 gtk.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS)
 gtk-egl.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS) $(OPENGL_CFLAGS)
+gtk-gl-area.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS) $(OPENGL_CFLAGS)
 shader.o-cflags += $(OPENGL_CFLAGS)
 console-gl.o-cflags += $(OPENGL_CFLAGS)
 egl-helpers.o-cflags += $(OPENGL_CFLAGS)
diff --git a/ui/cocoa.m b/ui/cocoa.m
index 334e6f666..d76b94273 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -304,6 +304,7 @@ static void handleAnyDeviceErrors(Error * err)
 - (float) cdx;
 - (float) cdy;
 - (QEMUScreen) gscreen;
+- (void) raiseAllKeys;
 @end
 
 QemuCocoaView *cocoaView;
@@ -723,7 +724,15 @@ QemuCocoaView *cocoaView;
     }
 
     if (mouse_event) {
-        if (last_buttons != buttons) {
+        /* Don't send button events to the guest unless we've got a
+         * mouse grab or window focus. If we have neither then this event
+         * is the user clicking on the background window to activate and
+         * bring us to the front, which will be done by the sendEvent
+         * call below. We definitely don't want to pass that click through
+         * to the guest.
+         */
+        if ((isMouseGrabbed || [[self window] isKeyWindow]) &&
+            (last_buttons != buttons)) {
             static uint32_t bmap[INPUT_BUTTON_MAX] = {
                 [INPUT_BUTTON_LEFT]       = MOUSE_EVENT_LBUTTON,
                 [INPUT_BUTTON_MIDDLE]     = MOUSE_EVENT_MBUTTON,
@@ -798,6 +807,24 @@ QemuCocoaView *cocoaView;
 - (float) cdx {return cdx;}
 - (float) cdy {return cdy;}
 - (QEMUScreen) gscreen {return screen;}
+
+/*
+ * Makes the target think all down keys are being released.
+ * This prevents a stuck key problem, since we will not see
+ * key up events for those keys after we have lost focus.
+ */
+- (void) raiseAllKeys
+{
+    int index;
+    const int max_index = ARRAY_SIZE(modifiers_state);
+
+   for (index = 0; index < max_index; index++) {
+       if (modifiers_state[index]) {
+           modifiers_state[index] = 0;
+           qemu_input_event_send_key_number(dcl->con, index, false);
+       }
+   }
+}
 @end
 
 
@@ -809,12 +836,11 @@ QemuCocoaView *cocoaView;
 */
 @interface QemuCocoaAppController : NSObject
 #if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
-                                             <NSApplicationDelegate>
+                                       <NSWindowDelegate, NSApplicationDelegate>
 #endif
 {
 }
 - (void)startEmulationWithArgc:(int)argc argv:(char**)argv;
-- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(NSInteger)returnCode contextInfo:(void *)contextInfo;
 - (void)doToggleFullScreen:(id)sender;
 - (void)toggleFullScreen:(id)sender;
 - (void)showQEMUDoc:(id)sender;
@@ -829,6 +855,7 @@ QemuCocoaView *cocoaView;
 - (void)powerDownQEMU:(id)sender;
 - (void)ejectDeviceMedia:(id)sender;
 - (void)changeDeviceMedia:(id)sender;
+- (BOOL)verifyQuit;
 @end
 
 @implementation QemuCocoaAppController
@@ -855,13 +882,14 @@ QemuCocoaView *cocoaView;
             exit(1);
         }
         [normalWindow setAcceptsMouseMovedEvents:YES];
-        [normalWindow setTitle:[NSString stringWithFormat:@"QEMU"]];
+        [normalWindow setTitle:@"QEMU"];
         [normalWindow setContentView:cocoaView];
 #if (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_10)
         [normalWindow useOptimizedDrawing:YES];
 #endif
         [normalWindow makeKeyAndOrderFront:self];
         [normalWindow center];
+        [normalWindow setDelegate: self];
         stretch_video = false;
 
         /* Used for displaying pause on the screen */
@@ -895,29 +923,8 @@ QemuCocoaView *cocoaView;
 - (void)applicationDidFinishLaunching: (NSNotification *) note
 {
     COCOA_DEBUG("QemuCocoaAppController: applicationDidFinishLaunching\n");
-
-    // Display an open dialog box if no arguments were passed or
-    // if qemu was launched from the finder ( the Finder passes "-psn" )
-    if( gArgc <= 1 || strncmp ((char *)gArgv[1], "-psn", 4) == 0) {
-        NSOpenPanel *op = [[NSOpenPanel alloc] init];
-        [op setPrompt:@"Boot image"];
-        [op setMessage:@"Select the disk image you want to boot.\n\nHit the \"Cancel\" button to quit"];
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
-        [op setAllowedFileTypes:supportedImageFileTypes];
-        [op beginSheetModalForWindow:normalWindow
-            completionHandler:^(NSInteger returnCode)
-            { [self openPanelDidEnd:op
-                  returnCode:returnCode contextInfo:NULL ]; } ];
-#else
-        // Compatibility code for pre-10.6, using deprecated method
-        [op beginSheetForDirectory:nil file:nil types:filetypes
-              modalForWindow:normalWindow modalDelegate:self
-              didEndSelector:@selector(openPanelDidEnd:returnCode:contextInfo:) contextInfo:NULL];
-#endif
-    } else {
-        // or launch QEMU, with the global args
-        [self startEmulationWithArgc:gArgc argv:(char **)gArgv];
-    }
+    // launch QEMU, with the global args
+    [self startEmulationWithArgc:gArgc argv:(char **)gArgv];
 }
 
 - (void)applicationWillTerminate:(NSNotification *)aNotification
@@ -933,43 +940,40 @@ QemuCocoaView *cocoaView;
     return YES;
 }
 
-- (void)startEmulationWithArgc:(int)argc argv:(char**)argv
+- (NSApplicationTerminateReply)applicationShouldTerminate:
+                                                         (NSApplication *)sender
 {
-    COCOA_DEBUG("QemuCocoaAppController: startEmulationWithArgc\n");
-
-    int status;
-    status = qemu_main(argc, argv, *_NSGetEnviron());
-    exit(status);
+    COCOA_DEBUG("QemuCocoaAppController: applicationShouldTerminate\n");
+    return [self verifyQuit];
 }
 
-- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(NSInteger)returnCode contextInfo:(void *)contextInfo
+/* Called when the user clicks on a window's close button */
+- (BOOL)windowShouldClose:(id)sender
 {
-    COCOA_DEBUG("QemuCocoaAppController: openPanelDidEnd\n");
-
-    /* The NSFileHandlingPanelOKButton/NSFileHandlingPanelCancelButton values for
-     * returnCode strictly only apply for the 10.6-and-up beginSheetModalForWindow
-     * API. For the legacy pre-10.6 beginSheetForDirectory API they are NSOKButton
-     * and NSCancelButton. However conveniently the values are the same.
-     * We use the non-legacy names because the others are deprecated in OSX 10.10.
+    COCOA_DEBUG("QemuCocoaAppController: windowShouldClose\n");
+    [NSApp terminate: sender];
+    /* If the user allows the application to quit then the call to
+     * NSApp terminate will never return. If we get here then the user
+     * cancelled the quit, so we should return NO to not permit the
+     * closing of this window.
      */
-    if (returnCode == NSFileHandlingPanelCancelButton) {
-        exit(0);
-    } else if (returnCode == NSFileHandlingPanelOKButton) {
-        char *img = (char*)[ [ [ sheet URL ] path ] cStringUsingEncoding:NSASCIIStringEncoding];
-
-        char **argv = g_new(char *, 4);
-
-        [sheet close];
+    return NO;
+}
 
-        argv[0] = g_strdup(gArgv[0]);
-        argv[1] = g_strdup("-hda");
-        argv[2] = g_strdup(img);
-        argv[3] = NULL;
+/* Called when QEMU goes into the background */
+- (void) applicationWillResignActive: (NSNotification *)aNotification
+{
+    COCOA_DEBUG("QemuCocoaAppController: applicationWillResignActive\n");
+    [cocoaView raiseAllKeys];
+}
 
-        // printf("Using argc %d argv %s -hda %s\n", 3, gArgv[0], img);
+- (void)startEmulationWithArgc:(int)argc argv:(char**)argv
+{
+    COCOA_DEBUG("QemuCocoaAppController: startEmulationWithArgc\n");
 
-        [self startEmulationWithArgc:3 argv:(char**)argv];
-    }
+    int status;
+    status = qemu_main(argc, argv, *_NSGetEnviron());
+    exit(status);
 }
 
 /* We abstract the method called by the Enter Fullscreen menu item
@@ -1117,14 +1121,32 @@ QemuCocoaView *cocoaView;
         }
 
         Error *err = NULL;
-        qmp_change_blockdev([drive cStringUsingEncoding: NSASCIIStringEncoding],
-                            [file cStringUsingEncoding: NSASCIIStringEncoding],
-                            "raw",
-                            &err);
+        qmp_blockdev_change_medium([drive cStringUsingEncoding:
+                                          NSASCIIStringEncoding],
+                                   [file cStringUsingEncoding:
+                                         NSASCIIStringEncoding],
+                                   true, "raw",
+                                   false, 0,
+                                   &err);
         handleAnyDeviceErrors(err);
     }
 }
 
+/* Verifies if the user really wants to quit */
+- (BOOL)verifyQuit
+{
+    NSAlert *alert = [NSAlert new];
+    [alert autorelease];
+    [alert setMessageText: @"Are you sure you want to quit QEMU?"];
+    [alert addButtonWithTitle: @"Cancel"];
+    [alert addButtonWithTitle: @"Quit"];
+    if([alert runModal] == NSAlertSecondButtonReturn) {
+        return YES;
+    } else {
+        return NO;
+    }
+}
+
 @end
 
 
@@ -1148,6 +1170,7 @@ int main (int argc, const char * argv[]) {
                 !strcmp(opt, "-nographic") ||
                 !strcmp(opt, "-version") ||
                 !strcmp(opt, "-curses") ||
+                !strcmp(opt, "-display") ||
                 !strcmp(opt, "-qtest")) {
                 return qemu_main(gArgc, gArgv, *_NSGetEnviron());
             }
@@ -1275,6 +1298,7 @@ static void cocoa_refresh(DisplayChangeListener *dcl)
     NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
 
     COCOA_DEBUG("qemu_cocoa: cocoa_refresh\n");
+    graphic_hw_update(NULL);
 
     if (qemu_input_is_absolute()) {
         if (![cocoaView isAbsoluteEnabled]) {
@@ -1295,7 +1319,6 @@ static void cocoa_refresh(DisplayChangeListener *dcl)
             [cocoaView handleEvent:event];
         }
     } while(event != nil);
-    graphic_hw_update(NULL);
     [pool release];
 }
 
@@ -1341,7 +1364,7 @@ static void add_console_menu_entries(void)
 /* Make menu items for all removable devices.
  * Each device is given an 'Eject' and 'Change' menu item.
  */
-static void addRemovableDevicesMenuItems()
+static void addRemovableDevicesMenuItems(void)
 {
     NSMenu *menu;
     NSMenuItem *menuItem;
diff --git a/ui/console-gl.c b/ui/console-gl.c
index cb45cf8a2..baf397b30 100644
--- a/ui/console-gl.c
+++ b/ui/console-gl.c
@@ -33,6 +33,7 @@
 
 struct ConsoleGLState {
     GLint texture_blit_prog;
+    GLint texture_blit_vao;
 };
 
 /* ---------------------------------------------------------------------- */
@@ -47,6 +48,9 @@ ConsoleGLState *console_gl_init_context(void)
         exit(1);
     }
 
+    gls->texture_blit_vao =
+        qemu_gl_init_texture_blit(gls->texture_blit_prog);
+
     return gls;
 }
 
@@ -131,7 +135,8 @@ void surface_gl_render_texture(ConsoleGLState *gls,
     glClearColor(0.1f, 0.1f, 0.1f, 0.0f);
     glClear(GL_COLOR_BUFFER_BIT);
 
-    qemu_gl_run_texture_blit(gls->texture_blit_prog);
+    qemu_gl_run_texture_blit(gls->texture_blit_prog,
+                             gls->texture_blit_vao);
 }
 
 void surface_gl_destroy_texture(ConsoleGLState *gls,
diff --git a/ui/console.c b/ui/console.c
index 75fc492f7..745c354f5 100644
--- a/ui/console.c
+++ b/ui/console.c
@@ -121,6 +121,7 @@ struct QemuConsole {
     DisplayState *ds;
     DisplaySurface *surface;
     int dcls;
+    DisplayChangeListener *gl;
 
     /* Graphic console state.  */
     Object *device;
@@ -449,7 +450,7 @@ static void text_console_resize(QemuConsole *s)
     if (s->width < w1)
         w1 = s->width;
 
-    cells = g_malloc(s->width * s->total_height * sizeof(TextCell));
+    cells = g_new(TextCell, s->width * s->total_height);
     for(y = 0; y < s->total_height; y++) {
         c = &cells[y * s->width];
         if (w1 > 0) {
@@ -1332,6 +1333,11 @@ void qemu_free_displaysurface(DisplaySurface *surface)
     g_free(surface);
 }
 
+bool console_has_gl(QemuConsole *con)
+{
+    return con->gl != NULL;
+}
+
 void register_displaychangelistener(DisplayChangeListener *dcl)
 {
     static const char nodev[] =
@@ -1339,6 +1345,17 @@ void register_displaychangelistener(DisplayChangeListener *dcl)
     static DisplaySurface *dummy;
     QemuConsole *con;
 
+    if (dcl->ops->dpy_gl_ctx_create) {
+        /* display has opengl support */
+        assert(dcl->con);
+        if (dcl->con->gl) {
+            fprintf(stderr, "can't register two opengl displays (%s, %s)\n",
+                    dcl->ops->dpy_name, dcl->con->gl->ops->dpy_name);
+            exit(1);
+        }
+        dcl->con->gl = dcl;
+    }
+
     trace_displaychangelistener_register(dcl, dcl->ops->dpy_name);
     dcl->ds = get_alloc_displaystate();
     QLIST_INSERT_HEAD(&dcl->ds->listeners, dcl, next);
@@ -1417,9 +1434,13 @@ void dpy_gfx_update(QemuConsole *con, int x, int y, int w, int h)
 {
     DisplayState *s = con->ds;
     DisplayChangeListener *dcl;
-    int width = surface_width(con->surface);
-    int height = surface_height(con->surface);
+    int width = w;
+    int height = h;
 
+    if (con->surface) {
+        width = surface_width(con->surface);
+        height = surface_height(con->surface);
+    }
     x = MAX(x, 0);
     y = MAX(y, 0);
     x = MIN(x, width);
@@ -1619,6 +1640,48 @@ bool dpy_cursor_define_supported(QemuConsole *con)
     return false;
 }
 
+QEMUGLContext dpy_gl_ctx_create(QemuConsole *con,
+                                struct QEMUGLParams *qparams)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_create(con->gl, qparams);
+}
+
+void dpy_gl_ctx_destroy(QemuConsole *con, QEMUGLContext ctx)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_ctx_destroy(con->gl, ctx);
+}
+
+int dpy_gl_ctx_make_current(QemuConsole *con, QEMUGLContext ctx)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_make_current(con->gl, ctx);
+}
+
+QEMUGLContext dpy_gl_ctx_get_current(QemuConsole *con)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_get_current(con->gl);
+}
+
+void dpy_gl_scanout(QemuConsole *con,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y, uint32_t width, uint32_t height)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_scanout(con->gl, backing_id,
+                                 backing_y_0_top,
+                                 x, y, width, height);
+}
+
+void dpy_gl_update(QemuConsole *con,
+                   uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_update(con->gl, x, y, w, h);
+}
+
 /***********************************************************/
 /* register display */
 
@@ -1899,7 +1962,7 @@ static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
         chr->init(chr);
 }
 
-static CharDriverState *text_console_init(ChardevVC *vc)
+static CharDriverState *text_console_init(ChardevVC *vc, Error **errp)
 {
     CharDriverState *chr;
     QemuConsole *s;
@@ -1930,6 +1993,7 @@ static CharDriverState *text_console_init(ChardevVC *vc)
 
     if (!s) {
         g_free(chr);
+        error_setg(errp, "cannot create text console");
         return NULL;
     }
 
@@ -1949,9 +2013,10 @@ static CharDriverState *text_console_init(ChardevVC *vc)
 
 static VcHandler *vc_handler = text_console_init;
 
-CharDriverState *vc_init(ChardevVC *vc)
+static CharDriverState *vc_init(const char *id, ChardevBackend *backend,
+                                ChardevReturn *ret, Error **errp)
 {
-    return vc_handler(vc);
+    return vc_handler(backend->u.vc, errp);
 }
 
 void register_vc_handler(VcHandler *handler)
@@ -1992,30 +2057,30 @@ static void qemu_chr_parse_vc(QemuOpts *opts, ChardevBackend *backend,
 {
     int val;
 
-    backend->vc = g_new0(ChardevVC, 1);
+    backend->u.vc = g_new0(ChardevVC, 1);
 
     val = qemu_opt_get_number(opts, "width", 0);
     if (val != 0) {
-        backend->vc->has_width = true;
-        backend->vc->width = val;
+        backend->u.vc->has_width = true;
+        backend->u.vc->width = val;
     }
 
     val = qemu_opt_get_number(opts, "height", 0);
     if (val != 0) {
-        backend->vc->has_height = true;
-        backend->vc->height = val;
+        backend->u.vc->has_height = true;
+        backend->u.vc->height = val;
     }
 
     val = qemu_opt_get_number(opts, "cols", 0);
     if (val != 0) {
-        backend->vc->has_cols = true;
-        backend->vc->cols = val;
+        backend->u.vc->has_cols = true;
+        backend->u.vc->cols = val;
     }
 
     val = qemu_opt_get_number(opts, "rows", 0);
     if (val != 0) {
-        backend->vc->has_rows = true;
-        backend->vc->rows = val;
+        backend->u.vc->has_rows = true;
+        backend->u.vc->rows = val;
     }
 }
 
@@ -2030,7 +2095,8 @@ static const TypeInfo qemu_console_info = {
 static void register_types(void)
 {
     type_register_static(&qemu_console_info);
-    register_char_driver("vc", CHARDEV_BACKEND_KIND_VC, qemu_chr_parse_vc);
+    register_char_driver("vc", CHARDEV_BACKEND_KIND_VC, qemu_chr_parse_vc,
+                         vc_init);
 }
 
 type_init(register_types);
diff --git a/ui/curses.c b/ui/curses.c
index 8edb038bb..7e7e4029e 100644
--- a/ui/curses.c
+++ b/ui/curses.c
@@ -42,6 +42,8 @@ static WINDOW *screenpad = NULL;
 static int width, height, gwidth, gheight, invalidate;
 static int px, py, sminx, sminy, smaxx, smaxy;
 
+chtype vga_to_curses[256];
+
 static void curses_update(DisplayChangeListener *dcl,
                           int x, int y, int w, int h)
 {
@@ -341,8 +343,55 @@ static void curses_setup(void)
     nodelay(stdscr, TRUE); nonl(); keypad(stdscr, TRUE);
     start_color(); raw(); scrollok(stdscr, FALSE);
 
-    for (i = 0; i < 64; i ++)
+    for (i = 0; i < 64; i++) {
         init_pair(i, colour_default[i & 7], colour_default[i >> 3]);
+    }
+    /* Set default color for more than 64. (monitor uses 0x74xx for example) */
+    for (i = 64; i < COLOR_PAIRS; i++) {
+        init_pair(i, COLOR_WHITE, COLOR_BLACK);
+    }
+
+    /*
+     * Setup mapping for vga to curses line graphics.
+     * FIXME: for better font, have to use ncursesw and setlocale()
+     */
+#if 0
+    /* FIXME: map from where? */
+    ACS_S1;
+    ACS_S3;
+    ACS_S7;
+    ACS_S9;
+#endif
+    /* ACS_* is not constant. So, we can't initialize statically. */
+    vga_to_curses['\0'] = ' ';
+    vga_to_curses[0x04] = ACS_DIAMOND;
+    vga_to_curses[0x0a] = ACS_RARROW;
+    vga_to_curses[0x0b] = ACS_LARROW;
+    vga_to_curses[0x18] = ACS_UARROW;
+    vga_to_curses[0x19] = ACS_DARROW;
+    vga_to_curses[0x9c] = ACS_STERLING;
+    vga_to_curses[0xb0] = ACS_BOARD;
+    vga_to_curses[0xb1] = ACS_CKBOARD;
+    vga_to_curses[0xb3] = ACS_VLINE;
+    vga_to_curses[0xb4] = ACS_RTEE;
+    vga_to_curses[0xbf] = ACS_URCORNER;
+    vga_to_curses[0xc0] = ACS_LLCORNER;
+    vga_to_curses[0xc1] = ACS_BTEE;
+    vga_to_curses[0xc2] = ACS_TTEE;
+    vga_to_curses[0xc3] = ACS_LTEE;
+    vga_to_curses[0xc4] = ACS_HLINE;
+    vga_to_curses[0xc5] = ACS_PLUS;
+    vga_to_curses[0xce] = ACS_LANTERN;
+    vga_to_curses[0xd8] = ACS_NEQUAL;
+    vga_to_curses[0xd9] = ACS_LRCORNER;
+    vga_to_curses[0xda] = ACS_ULCORNER;
+    vga_to_curses[0xdb] = ACS_BLOCK;
+    vga_to_curses[0xe3] = ACS_PI;
+    vga_to_curses[0xf1] = ACS_PLMINUS;
+    vga_to_curses[0xf2] = ACS_GEQUAL;
+    vga_to_curses[0xf3] = ACS_LEQUAL;
+    vga_to_curses[0xf8] = ACS_DEGREE;
+    vga_to_curses[0xfe] = ACS_BULLET;
 }
 
 static void curses_keyboard_setup(void)
@@ -382,7 +431,7 @@ void curses_display_init(DisplayState *ds, int full_screen)
 
     curses_winch_init();
 
-    dcl = (DisplayChangeListener *) g_malloc0(sizeof(DisplayChangeListener));
+    dcl = g_new0(DisplayChangeListener, 1);
     dcl->ops = &dcl_ops;
     register_displaychangelistener(dcl);
 
diff --git a/ui/curses_keys.h b/ui/curses_keys.h
index 18ce6dcee..f7467449b 100644
--- a/ui/curses_keys.h
+++ b/ui/curses_keys.h
@@ -29,8 +29,7 @@
 #include "keymaps.h"
 
 
-#define KEY_RELEASE         0x80
-#define KEY_MASK            0x7f
+#define KEY_MASK            SCANCODE_KEYMASK
 #define GREY_CODE           0xe0
 #define GREY                SCANCODE_GREY
 #define SHIFT_CODE          0x2a
@@ -60,6 +59,8 @@ static const int curses2keysym[CURSES_KEYS] = {
     ['\n'] = KEY_ENTER,
     [27] = 27,
     [KEY_BTAB] = '\t' | KEYSYM_SHIFT,
+    [KEY_SPREVIOUS] = KEY_PPAGE | KEYSYM_SHIFT,
+    [KEY_SNEXT] = KEY_NPAGE | KEYSYM_SHIFT,
 };
 
 static const int curses2keycode[CURSES_KEYS] = {
@@ -149,6 +150,9 @@ static const int curses2keycode[CURSES_KEYS] = {
     [KEY_IC] = 82 | GREY, /* Insert */
     [KEY_DC] = 83 | GREY, /* Delete */
 
+    [KEY_SPREVIOUS] = 73 | GREY | SHIFT, /* Shift + Page Up */
+    [KEY_SNEXT] = 81 | GREY | SHIFT, /* Shift + Page Down */
+
     ['!'] = 2 | SHIFT,
     ['@'] = 3 | SHIFT,
     ['#'] = 4 | SHIFT,
diff --git a/ui/egl-context.c b/ui/egl-context.c
new file mode 100644
index 000000000..40102e392
--- /dev/null
+++ b/ui/egl-context.c
@@ -0,0 +1,34 @@
+#include "qemu-common.h"
+#include "ui/egl-context.h"
+
+QEMUGLContext qemu_egl_create_context(DisplayChangeListener *dcl,
+                                      QEMUGLParams *params)
+{
+   EGLContext ctx;
+   EGLint ctx_att[] = {
+      EGL_CONTEXT_CLIENT_VERSION, params->major_ver,
+      EGL_CONTEXT_MINOR_VERSION_KHR, params->minor_ver,
+      EGL_NONE
+   };
+
+   ctx = eglCreateContext(qemu_egl_display, qemu_egl_config,
+                          eglGetCurrentContext(), ctx_att);
+   return ctx;
+}
+
+void qemu_egl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    eglDestroyContext(qemu_egl_display, ctx);
+}
+
+int qemu_egl_make_context_current(DisplayChangeListener *dcl,
+                                  QEMUGLContext ctx)
+{
+   return eglMakeCurrent(qemu_egl_display,
+                         EGL_NO_SURFACE, EGL_NO_SURFACE, ctx);
+}
+
+QEMUGLContext qemu_egl_get_current_context(DisplayChangeListener *dcl)
+{
+    return eglGetCurrentContext();
+}
diff --git a/ui/gtk-egl.c b/ui/gtk-egl.c
index 15b41f2ba..500c42c9f 100644
--- a/ui/gtk-egl.c
+++ b/ui/gtk-egl.c
@@ -21,6 +21,29 @@
 
 #include "sysemu/sysemu.h"
 
+static void gtk_egl_set_scanout_mode(VirtualConsole *vc, bool scanout)
+{
+    if (vc->gfx.scanout_mode == scanout) {
+        return;
+    }
+
+    vc->gfx.scanout_mode = scanout;
+    if (!vc->gfx.scanout_mode) {
+        if (vc->gfx.fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            glDeleteFramebuffers(1, &vc->gfx.fbo_id);
+            vc->gfx.fbo_id = 0;
+        }
+        if (vc->gfx.surface) {
+            surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+}
+
 /** DisplayState Callbacks (opengl version) **/
 
 void gd_egl_init(VirtualConsole *vc)
@@ -50,19 +73,26 @@ void gd_egl_draw(VirtualConsole *vc)
     GdkWindow *window;
     int ww, wh;
 
-    if (!vc->gfx.gls || !vc->gfx.ds) {
+    if (!vc->gfx.gls) {
         return;
     }
 
-    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
-                   vc->gfx.esurface, vc->gfx.ectx);
+    if (vc->gfx.scanout_mode) {
+        gd_egl_scanout_flush(&vc->gfx.dcl, 0, 0, vc->gfx.w, vc->gfx.h);
+    } else {
+        if (!vc->gfx.ds) {
+            return;
+        }
+        eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                       vc->gfx.esurface, vc->gfx.ectx);
 
-    window = gtk_widget_get_window(vc->gfx.drawing_area);
-    gdk_drawable_get_size(window, &ww, &wh);
-    surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
-    surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
+        window = gtk_widget_get_window(vc->gfx.drawing_area);
+        gdk_drawable_get_size(window, &ww, &wh);
+        surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
+        surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
 
-    eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+        eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+    }
 }
 
 void gd_egl_update(DisplayChangeListener *dcl,
@@ -99,6 +129,7 @@ void gd_egl_refresh(DisplayChangeListener *dcl)
 
     if (vc->gfx.glupdates) {
         vc->gfx.glupdates = 0;
+        gtk_egl_set_scanout_mode(vc, false);
         gd_egl_draw(vc);
     }
 }
@@ -128,6 +159,81 @@ void gd_egl_switch(DisplayChangeListener *dcl,
     }
 }
 
+QEMUGLContext gd_egl_create_context(DisplayChangeListener *dcl,
+                                    QEMUGLParams *params)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+    return qemu_egl_create_context(dcl, params);
+}
+
+void gd_egl_scanout(DisplayChangeListener *dcl,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y,
+                    uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    vc->gfx.x = x;
+    vc->gfx.y = y;
+    vc->gfx.w = w;
+    vc->gfx.h = h;
+    vc->gfx.tex_id = backing_id;
+    vc->gfx.y0_top = backing_y_0_top;
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+
+    if (vc->gfx.tex_id == 0 || vc->gfx.w == 0 || vc->gfx.h == 0) {
+        gtk_egl_set_scanout_mode(vc, false);
+        return;
+    }
+
+    gtk_egl_set_scanout_mode(vc, true);
+    if (!vc->gfx.fbo_id) {
+        glGenFramebuffers(1, &vc->gfx.fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, vc->gfx.tex_id, 0);
+}
+
+void gd_egl_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    GdkWindow *window;
+    int ww, wh, y1, y2;
+
+    if (!vc->gfx.scanout_mode) {
+        return;
+    }
+    if (!vc->gfx.fbo_id) {
+        return;
+    }
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, vc->gfx.fbo_id);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+
+    window = gtk_widget_get_window(vc->gfx.drawing_area);
+    gdk_drawable_get_size(window, &ww, &wh);
+    glViewport(0, 0, ww, wh);
+    y1 = vc->gfx.y0_top ? 0 : vc->gfx.h;
+    y2 = vc->gfx.y0_top ? vc->gfx.h : 0;
+    glBlitFramebuffer(0, y1, vc->gfx.w, y2,
+                      0, 0, ww, wh,
+                      GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+
+    eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+}
+
 void gtk_egl_init(void)
 {
     GdkDisplay *gdk_display = gdk_display_get_default();
@@ -139,3 +245,12 @@ void gtk_egl_init(void)
 
     display_opengl = 1;
 }
+
+int gd_egl_make_current(DisplayChangeListener *dcl,
+                        QEMUGLContext ctx)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    return eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                          vc->gfx.esurface, ctx);
+}
diff --git a/ui/gtk-gl-area.c b/ui/gtk-gl-area.c
new file mode 100644
index 000000000..dec3edb29
--- /dev/null
+++ b/ui/gtk-gl-area.c
@@ -0,0 +1,223 @@
+/*
+ * GTK UI -- glarea opengl code.
+ *
+ * Requires 3.16+ (GtkGLArea widget).
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+
+#include "trace.h"
+
+#include "ui/console.h"
+#include "ui/gtk.h"
+#include "ui/egl-helpers.h"
+
+#include "sysemu/sysemu.h"
+
+static void gtk_gl_area_set_scanout_mode(VirtualConsole *vc, bool scanout)
+{
+    if (vc->gfx.scanout_mode == scanout) {
+        return;
+    }
+
+    vc->gfx.scanout_mode = scanout;
+    if (!vc->gfx.scanout_mode) {
+        if (vc->gfx.fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            glDeleteFramebuffers(1, &vc->gfx.fbo_id);
+            vc->gfx.fbo_id = 0;
+        }
+        if (vc->gfx.surface) {
+            surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+}
+
+/** DisplayState Callbacks (opengl version) **/
+
+void gd_gl_area_draw(VirtualConsole *vc)
+{
+    int ww, wh, y1, y2;
+
+    if (!vc->gfx.gls) {
+        return;
+    }
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    ww = gtk_widget_get_allocated_width(vc->gfx.drawing_area);
+    wh = gtk_widget_get_allocated_height(vc->gfx.drawing_area);
+
+    if (vc->gfx.scanout_mode) {
+        if (!vc->gfx.fbo_id) {
+            return;
+        }
+
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, vc->gfx.fbo_id);
+        /* GtkGLArea sets GL_DRAW_FRAMEBUFFER for us */
+
+        glViewport(0, 0, ww, wh);
+        y1 = vc->gfx.y0_top ? 0 : vc->gfx.h;
+        y2 = vc->gfx.y0_top ? vc->gfx.h : 0;
+        glBlitFramebuffer(0, y1, vc->gfx.w, y2,
+                          0, 0, ww, wh,
+                          GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    } else {
+        if (!vc->gfx.ds) {
+            return;
+        }
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+
+        surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
+        surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
+    }
+}
+
+void gd_gl_area_update(DisplayChangeListener *dcl,
+                   int x, int y, int w, int h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.gls || !vc->gfx.ds) {
+        return;
+    }
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    surface_gl_update_texture(vc->gfx.gls, vc->gfx.ds, x, y, w, h);
+    vc->gfx.glupdates++;
+}
+
+void gd_gl_area_refresh(DisplayChangeListener *dcl)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.gls) {
+        if (!gtk_widget_get_realized(vc->gfx.drawing_area)) {
+            return;
+        }
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+        vc->gfx.gls = console_gl_init_context();
+        if (vc->gfx.ds) {
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+
+    graphic_hw_update(dcl->con);
+
+    if (vc->gfx.glupdates) {
+        vc->gfx.glupdates = 0;
+        gtk_gl_area_set_scanout_mode(vc, false);
+        gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+    }
+}
+
+void gd_gl_area_switch(DisplayChangeListener *dcl,
+                       DisplaySurface *surface)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    bool resized = true;
+
+    trace_gd_switch(vc->label, surface_width(surface), surface_height(surface));
+
+    if (vc->gfx.ds &&
+        surface_width(vc->gfx.ds) == surface_width(surface) &&
+        surface_height(vc->gfx.ds) == surface_height(surface)) {
+        resized = false;
+    }
+
+    if (vc->gfx.gls) {
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+        surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+        surface_gl_create_texture(vc->gfx.gls, surface);
+    }
+    vc->gfx.ds = surface;
+
+    if (resized) {
+        gd_update_windowsize(vc);
+    }
+}
+
+QEMUGLContext gd_gl_area_create_context(DisplayChangeListener *dcl,
+                                        QEMUGLParams *params)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    GdkWindow *window;
+    GdkGLContext *ctx;
+    GError *err = NULL;
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    window = gtk_widget_get_window(vc->gfx.drawing_area);
+    ctx = gdk_window_create_gl_context(window, &err);
+    gdk_gl_context_set_required_version(ctx,
+                                        params->major_ver,
+                                        params->minor_ver);
+    gdk_gl_context_realize(ctx, &err);
+    return ctx;
+}
+
+void gd_gl_area_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    /* FIXME */
+}
+
+void gd_gl_area_scanout(DisplayChangeListener *dcl,
+                        uint32_t backing_id, bool backing_y_0_top,
+                        uint32_t x, uint32_t y,
+                        uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    vc->gfx.x = x;
+    vc->gfx.y = y;
+    vc->gfx.w = w;
+    vc->gfx.h = h;
+    vc->gfx.tex_id = backing_id;
+    vc->gfx.y0_top = backing_y_0_top;
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+
+    if (vc->gfx.tex_id == 0 || vc->gfx.w == 0 || vc->gfx.h == 0) {
+        gtk_gl_area_set_scanout_mode(vc, false);
+        return;
+    }
+
+    gtk_gl_area_set_scanout_mode(vc, true);
+    if (!vc->gfx.fbo_id) {
+        glGenFramebuffers(1, &vc->gfx.fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, vc->gfx.tex_id, 0);
+}
+
+void gd_gl_area_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+}
+
+void gtk_gl_area_init(void)
+{
+    display_opengl = 1;
+}
+
+QEMUGLContext gd_gl_area_get_current_context(DisplayChangeListener *dcl)
+{
+    return gdk_gl_context_get_current();
+}
+
+int gd_gl_area_make_current(DisplayChangeListener *dcl,
+                            QEMUGLContext ctx)
+{
+    gdk_gl_context_make_current(ctx);
+    return 0;
+}
diff --git a/ui/gtk.c b/ui/gtk.c
index 11ea2cf37..47b37e1f1 100644
--- a/ui/gtk.c
+++ b/ui/gtk.c
@@ -104,6 +104,15 @@
 #define GDK_KEY_Pause GDK_Pause
 #endif
 
+/* Some older mingw versions lack this constant or have
+ * it conditionally defined */
+#ifdef _WIN32
+# ifndef MAPVK_VK_TO_VSC
+#  define MAPVK_VK_TO_VSC 0
+# endif
+#endif
+
+
 #define HOTKEY_MODIFIERS        (GDK_CONTROL_MASK | GDK_MOD1_MASK)
 
 static const int modifier_keycode[] = {
@@ -165,8 +174,10 @@ struct GtkDisplayState {
     bool ignore_keys;
 };
 
-static void gd_grab_pointer(VirtualConsole *vc);
+static void gd_grab_pointer(VirtualConsole *vc, const char *reason);
 static void gd_ungrab_pointer(GtkDisplayState *s);
+static void gd_grab_keyboard(VirtualConsole *vc, const char *reason);
+static void gd_ungrab_keyboard(GtkDisplayState *s);
 
 /** Utility Functions **/
 
@@ -356,6 +367,12 @@ static void gd_update_full_redraw(VirtualConsole *vc)
     GtkWidget *area = vc->gfx.drawing_area;
     int ww, wh;
     gdk_drawable_get_size(gtk_widget_get_window(area), &ww, &wh);
+#if defined(CONFIG_GTK_GL)
+    if (vc->gfx.gls) {
+        gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+        return;
+    }
+#endif
     gtk_widget_queue_draw_area(area, 0, 0, ww, wh);
 }
 
@@ -596,6 +613,27 @@ static const DisplayChangeListenerOps dcl_ops = {
 
 /** DisplayState Callbacks (opengl version) **/
 
+#if defined(CONFIG_GTK_GL)
+
+static const DisplayChangeListenerOps dcl_gl_area_ops = {
+    .dpy_name             = "gtk-egl",
+    .dpy_gfx_update       = gd_gl_area_update,
+    .dpy_gfx_switch       = gd_gl_area_switch,
+    .dpy_gfx_check_format = console_gl_check_format,
+    .dpy_refresh          = gd_gl_area_refresh,
+    .dpy_mouse_set        = gd_mouse_set,
+    .dpy_cursor_define    = gd_cursor_define,
+
+    .dpy_gl_ctx_create       = gd_gl_area_create_context,
+    .dpy_gl_ctx_destroy      = gd_gl_area_destroy_context,
+    .dpy_gl_ctx_make_current = gd_gl_area_make_current,
+    .dpy_gl_ctx_get_current  = gd_gl_area_get_current_context,
+    .dpy_gl_scanout          = gd_gl_area_scanout,
+    .dpy_gl_update           = gd_gl_area_scanout_flush,
+};
+
+#else
+
 static const DisplayChangeListenerOps dcl_egl_ops = {
     .dpy_name             = "gtk-egl",
     .dpy_gfx_update       = gd_egl_update,
@@ -604,9 +642,17 @@ static const DisplayChangeListenerOps dcl_egl_ops = {
     .dpy_refresh          = gd_egl_refresh,
     .dpy_mouse_set        = gd_mouse_set,
     .dpy_cursor_define    = gd_cursor_define,
+
+    .dpy_gl_ctx_create       = gd_egl_create_context,
+    .dpy_gl_ctx_destroy      = qemu_egl_destroy_context,
+    .dpy_gl_ctx_make_current = gd_egl_make_current,
+    .dpy_gl_ctx_get_current  = qemu_egl_get_current_context,
+    .dpy_gl_scanout          = gd_egl_scanout,
+    .dpy_gl_update           = gd_egl_scanout_flush,
 };
 
-#endif
+#endif /* CONFIG_GTK_GL */
+#endif /* CONFIG_OPENGL */
 
 /** QEMU Events **/
 
@@ -656,6 +702,39 @@ static gboolean gd_window_close(GtkWidget *widget, GdkEvent *event,
     return TRUE;
 }
 
+static void gd_set_ui_info(VirtualConsole *vc, gint width, gint height)
+{
+    QemuUIInfo info;
+
+    memset(&info, 0, sizeof(info));
+    info.width = width;
+    info.height = height;
+    dpy_set_ui_info(vc->gfx.dcl.con, &info);
+}
+
+#if defined(CONFIG_GTK_GL)
+
+static gboolean gd_render_event(GtkGLArea *area, GdkGLContext *context,
+                                void *opaque)
+{
+    VirtualConsole *vc = opaque;
+
+    if (vc->gfx.gls) {
+        gd_gl_area_draw(vc);
+    }
+    return TRUE;
+}
+
+static void gd_resize_event(GtkGLArea *area,
+                            gint width, gint height, gpointer *opaque)
+{
+    VirtualConsole *vc = (void *)opaque;
+
+    gd_set_ui_info(vc, width, height);
+}
+
+#endif
+
 static gboolean gd_draw_event(GtkWidget *widget, cairo_t *cr, void *opaque)
 {
     VirtualConsole *vc = opaque;
@@ -666,8 +745,13 @@ static gboolean gd_draw_event(GtkWidget *widget, cairo_t *cr, void *opaque)
 
 #if defined(CONFIG_OPENGL)
     if (vc->gfx.gls) {
+#if defined(CONFIG_GTK_GL)
+        /* invoke render callback please */
+        return FALSE;
+#else
         gd_egl_draw(vc);
         return TRUE;
+#endif
     }
 #endif
 
@@ -849,13 +933,11 @@ static gboolean gd_button_event(GtkWidget *widget, GdkEventButton *button,
     /* implicitly grab the input at the first click in the relative mode */
     if (button->button == 1 && button->type == GDK_BUTTON_PRESS &&
         !qemu_input_is_absolute() && s->ptr_owner != vc) {
-        gd_ungrab_pointer(s);
         if (!vc->window) {
             gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                            TRUE);
         } else {
-            gd_grab_pointer(vc);
-            gd_update_caption(s);
+            gd_grab_pointer(vc, "relative-mode-click");
         }
         return TRUE;
     }
@@ -1092,9 +1174,8 @@ static gboolean gd_win_grab(void *opaque)
     if (vc->s->ptr_owner) {
         gd_ungrab_pointer(vc->s);
     } else {
-        gd_grab_pointer(vc);
+        gd_grab_pointer(vc, "user-request-detached-tab");
     }
-    gd_update_caption(vc->s);
     return TRUE;
 }
 
@@ -1141,10 +1222,6 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
         gtk_widget_hide(s->menu_bar);
         if (vc->type == GD_VC_GFX) {
             gtk_widget_set_size_request(vc->gfx.drawing_area, -1, -1);
-            if (qemu_console_is_graphic(vc->gfx.dcl.con)) {
-                gtk_check_menu_item_set_active
-                    (GTK_CHECK_MENU_ITEM(s->grab_item), TRUE);
-            }
         }
         gtk_window_fullscreen(GTK_WINDOW(s->window));
         s->full_screen = TRUE;
@@ -1157,8 +1234,6 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
             vc->gfx.scale_x = 1.0;
             vc->gfx.scale_y = 1.0;
             gd_update_windowsize(vc);
-            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
-                                           FALSE);
         }
     }
 
@@ -1257,8 +1332,16 @@ static void gd_grab_devices(VirtualConsole *vc, bool grab,
 }
 #endif
 
-static void gd_grab_keyboard(VirtualConsole *vc)
+static void gd_grab_keyboard(VirtualConsole *vc, const char *reason)
 {
+    if (vc->s->kbd_owner) {
+        if (vc->s->kbd_owner == vc) {
+            return;
+        } else {
+            gd_ungrab_keyboard(vc->s);
+        }
+    }
+
 #if GTK_CHECK_VERSION(3, 0, 0)
     gd_grab_devices(vc, true, GDK_SOURCE_KEYBOARD,
                    GDK_KEY_PRESS_MASK | GDK_KEY_RELEASE_MASK,
@@ -1269,7 +1352,8 @@ static void gd_grab_keyboard(VirtualConsole *vc)
                       GDK_CURRENT_TIME);
 #endif
     vc->s->kbd_owner = vc;
-    trace_gd_grab(vc->label, "kbd", true);
+    gd_update_caption(vc->s);
+    trace_gd_grab(vc->label, "kbd", reason);
 }
 
 static void gd_ungrab_keyboard(GtkDisplayState *s)
@@ -1286,12 +1370,22 @@ static void gd_ungrab_keyboard(GtkDisplayState *s)
 #else
     gdk_keyboard_ungrab(GDK_CURRENT_TIME);
 #endif
-    trace_gd_grab(vc->label, "kbd", false);
+    gd_update_caption(s);
+    trace_gd_ungrab(vc->label, "kbd");
 }
 
-static void gd_grab_pointer(VirtualConsole *vc)
+static void gd_grab_pointer(VirtualConsole *vc, const char *reason)
 {
     GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
+
+    if (vc->s->ptr_owner) {
+        if (vc->s->ptr_owner == vc) {
+            return;
+        } else {
+            gd_ungrab_pointer(vc->s);
+        }
+    }
+
 #if GTK_CHECK_VERSION(3, 0, 0)
     GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
     gd_grab_devices(vc, true, GDK_SOURCE_MOUSE,
@@ -1318,7 +1412,8 @@ static void gd_grab_pointer(VirtualConsole *vc)
                             &vc->s->grab_x_root, &vc->s->grab_y_root, NULL);
 #endif
     vc->s->ptr_owner = vc;
-    trace_gd_grab(vc->label, "ptr", true);
+    gd_update_caption(vc->s);
+    trace_gd_grab(vc->label, "ptr", reason);
 }
 
 static void gd_ungrab_pointer(GtkDisplayState *s)
@@ -1343,7 +1438,8 @@ static void gd_ungrab_pointer(GtkDisplayState *s)
                              gtk_widget_get_screen(vc->gfx.drawing_area),
                              vc->s->grab_x_root, vc->s->grab_y_root);
 #endif
-    trace_gd_grab(vc->label, "ptr", false);
+    gd_update_caption(s);
+    trace_gd_ungrab(vc->label, "ptr");
 }
 
 static void gd_menu_grab_input(GtkMenuItem *item, void *opaque)
@@ -1352,16 +1448,13 @@ static void gd_menu_grab_input(GtkMenuItem *item, void *opaque)
     VirtualConsole *vc = gd_vc_find_current(s);
 
     if (gd_is_grab_active(s)) {
-        if (!gd_grab_on_hover(s)) {
-            gd_grab_keyboard(vc);
-        }
-        gd_grab_pointer(vc);
+        gd_grab_keyboard(vc, "user-request-main-window");
+        gd_grab_pointer(vc, "user-request-main-window");
     } else {
         gd_ungrab_keyboard(s);
         gd_ungrab_pointer(s);
     }
 
-    gd_update_caption(s);
     gd_update_cursor(vc);
 }
 
@@ -1415,9 +1508,7 @@ static gboolean gd_enter_event(GtkWidget *widget, GdkEventCrossing *crossing,
     GtkDisplayState *s = vc->s;
 
     if (gd_grab_on_hover(s)) {
-        gd_ungrab_keyboard(s);
-        gd_grab_keyboard(vc);
-        gd_update_caption(s);
+        gd_grab_keyboard(vc, "grab-on-hover");
     }
     return TRUE;
 }
@@ -1430,7 +1521,6 @@ static gboolean gd_leave_event(GtkWidget *widget, GdkEventCrossing *crossing,
 
     if (gd_grab_on_hover(s)) {
         gd_ungrab_keyboard(s);
-        gd_update_caption(s);
     }
     return TRUE;
 }
@@ -1449,12 +1539,8 @@ static gboolean gd_configure(GtkWidget *widget,
                              GdkEventConfigure *cfg, gpointer opaque)
 {
     VirtualConsole *vc = opaque;
-    QemuUIInfo info;
 
-    memset(&info, 0, sizeof(info));
-    info.width = cfg->width;
-    info.height = cfg->height;
-    dpy_set_ui_info(vc->gfx.dcl.con, &info);
+    gd_set_ui_info(vc, cfg->width, cfg->height);
     return FALSE;
 }
 
@@ -1505,7 +1591,7 @@ static int gd_vc_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 static int nb_vcs;
 static CharDriverState *vcs[MAX_VCS];
 
-static CharDriverState *gd_vc_handler(ChardevVC *unused)
+static CharDriverState *gd_vc_handler(ChardevVC *unused, Error **errp)
 {
     CharDriverState *chr;
 
@@ -1611,6 +1697,15 @@ static void gd_connect_vc_gfx_signals(VirtualConsole *vc)
 #if GTK_CHECK_VERSION(3, 0, 0)
     g_signal_connect(vc->gfx.drawing_area, "draw",
                      G_CALLBACK(gd_draw_event), vc);
+#if defined(CONFIG_GTK_GL)
+    if (display_opengl) {
+        /* wire up GtkGlArea events */
+        g_signal_connect(vc->gfx.drawing_area, "render",
+                         G_CALLBACK(gd_render_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "resize",
+                         G_CALLBACK(gd_resize_event), vc);
+    }
+#endif
 #else
     g_signal_connect(vc->gfx.drawing_area, "expose-event",
                      G_CALLBACK(gd_expose_event), vc);
@@ -1719,26 +1814,13 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
     vc->gfx.scale_x = 1.0;
     vc->gfx.scale_y = 1.0;
 
-    vc->gfx.drawing_area = gtk_drawing_area_new();
-    gtk_widget_add_events(vc->gfx.drawing_area,
-                          GDK_POINTER_MOTION_MASK |
-                          GDK_BUTTON_PRESS_MASK |
-                          GDK_BUTTON_RELEASE_MASK |
-                          GDK_BUTTON_MOTION_MASK |
-                          GDK_ENTER_NOTIFY_MASK |
-                          GDK_LEAVE_NOTIFY_MASK |
-                          GDK_SCROLL_MASK |
-                          GDK_KEY_PRESS_MASK);
-    gtk_widget_set_can_focus(vc->gfx.drawing_area, TRUE);
-
-    vc->type = GD_VC_GFX;
-    vc->tab_item = vc->gfx.drawing_area;
-    vc->focus = vc->gfx.drawing_area;
-    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook),
-                             vc->tab_item, gtk_label_new(vc->label));
-
 #if defined(CONFIG_OPENGL)
     if (display_opengl) {
+#if defined(CONFIG_GTK_GL)
+        vc->gfx.drawing_area = gtk_gl_area_new();
+        vc->gfx.dcl.ops = &dcl_gl_area_ops;
+#else
+        vc->gfx.drawing_area = gtk_drawing_area_new();
         /*
          * gtk_widget_set_double_buffered() was deprecated in 3.14.
          * It is required for opengl rendering on X11 though.  A
@@ -1754,12 +1836,32 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
 #pragma GCC diagnostic pop
 #endif
         vc->gfx.dcl.ops = &dcl_egl_ops;
+#endif /* CONFIG_GTK_GL */
     } else
 #endif
     {
+        vc->gfx.drawing_area = gtk_drawing_area_new();
         vc->gfx.dcl.ops = &dcl_ops;
     }
 
+
+    gtk_widget_add_events(vc->gfx.drawing_area,
+                          GDK_POINTER_MOTION_MASK |
+                          GDK_BUTTON_PRESS_MASK |
+                          GDK_BUTTON_RELEASE_MASK |
+                          GDK_BUTTON_MOTION_MASK |
+                          GDK_ENTER_NOTIFY_MASK |
+                          GDK_LEAVE_NOTIFY_MASK |
+                          GDK_SCROLL_MASK |
+                          GDK_KEY_PRESS_MASK);
+    gtk_widget_set_can_focus(vc->gfx.drawing_area, TRUE);
+
+    vc->type = GD_VC_GFX;
+    vc->tab_item = vc->gfx.drawing_area;
+    vc->focus = vc->gfx.drawing_area;
+    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook),
+                             vc->tab_item, gtk_label_new(vc->label));
+
     vc->gfx.dcl.con = con;
     register_displaychangelistener(&vc->gfx.dcl);
 
@@ -1768,6 +1870,7 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
 
     if (dpy_ui_info_supported(vc->gfx.dcl.con)) {
         gtk_menu_item_activate(GTK_MENU_ITEM(s->zoom_fit_item));
+        s->free_scale = true;
     }
 
     return group;
@@ -2041,8 +2144,12 @@ void early_gtk_display_init(int opengl)
         break;
     case 1: /* on */
 #if defined(CONFIG_OPENGL)
+#if defined(CONFIG_GTK_GL)
+        gtk_gl_area_init();
+#else
         gtk_egl_init();
 #endif
+#endif
         break;
     default:
         g_assert_not_reached();
diff --git a/ui/input-keymap.c b/ui/input-keymap.c
index 7635cb0dc..d36be4b60 100644
--- a/ui/input-keymap.c
+++ b/ui/input-keymap.c
@@ -139,11 +139,11 @@ static int number_to_qcode[0x100];
 
 int qemu_input_key_value_to_number(const KeyValue *value)
 {
-    if (value->kind == KEY_VALUE_KIND_QCODE) {
-        return qcode_to_number[value->qcode];
+    if (value->type == KEY_VALUE_KIND_QCODE) {
+        return qcode_to_number[value->u.qcode];
     } else {
-        assert(value->kind == KEY_VALUE_KIND_NUMBER);
-        return value->number;
+        assert(value->type == KEY_VALUE_KIND_NUMBER);
+        return value->u.number;
     }
 }
 
@@ -166,11 +166,11 @@ int qemu_input_key_number_to_qcode(uint8_t nr)
 
 int qemu_input_key_value_to_qcode(const KeyValue *value)
 {
-    if (value->kind == KEY_VALUE_KIND_QCODE) {
-        return value->qcode;
+    if (value->type == KEY_VALUE_KIND_QCODE) {
+        return value->u.qcode;
     } else {
-        assert(value->kind == KEY_VALUE_KIND_NUMBER);
-        return qemu_input_key_number_to_qcode(value->number);
+        assert(value->type == KEY_VALUE_KIND_NUMBER);
+        return qemu_input_key_number_to_qcode(value->u.number);
     }
 }
 
@@ -180,8 +180,8 @@ int qemu_input_key_value_to_scancode(const KeyValue *value, bool down,
     int keycode = qemu_input_key_value_to_number(value);
     int count = 0;
 
-    if (value->kind == KEY_VALUE_KIND_QCODE &&
-        value->qcode == Q_KEY_CODE_PAUSE) {
+    if (value->type == KEY_VALUE_KIND_QCODE &&
+        value->u.qcode == Q_KEY_CODE_PAUSE) {
         /* specific case */
         int v = down ? 0 : 0x80;
         codes[count++] = 0xe1;
diff --git a/ui/input-legacy.c b/ui/input-legacy.c
index e50f2968e..3f28bbce6 100644
--- a/ui/input-legacy.c
+++ b/ui/input-legacy.c
@@ -57,12 +57,13 @@ struct QEMUPutLEDEntry {
 static QTAILQ_HEAD(, QEMUPutLEDEntry) led_handlers =
     QTAILQ_HEAD_INITIALIZER(led_handlers);
 
-int index_from_key(const char *key)
+int index_from_key(const char *key, size_t key_length)
 {
     int i;
 
     for (i = 0; QKeyCode_lookup[i] != NULL; i++) {
-        if (!strcmp(key, QKeyCode_lookup[i])) {
+        if (!strncmp(key, QKeyCode_lookup[i], key_length) &&
+            !QKeyCode_lookup[i][key_length]) {
             break;
         }
     }
@@ -113,8 +114,8 @@ static void legacy_kbd_event(DeviceState *dev, QemuConsole *src,
     if (!entry || !entry->put_kbd) {
         return;
     }
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(evt->u.key->key,
+                                             evt->u.key->down,
                                              scancodes);
     for (i = 0; i < count; i++) {
         entry->put_kbd(entry->opaque, scancodes[i]);
@@ -150,21 +151,22 @@ static void legacy_mouse_event(DeviceState *dev, QemuConsole *src,
     };
     QEMUPutMouseEntry *s = (QEMUPutMouseEntry *)dev;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            s->buttons |= bmap[evt->btn->button];
+        if (evt->u.btn->down) {
+            s->buttons |= bmap[evt->u.btn->button];
         } else {
-            s->buttons &= ~bmap[evt->btn->button];
+            s->buttons &= ~bmap[evt->u.btn->button];
         }
-        if (evt->btn->down && evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        if (evt->u.btn->down && evt->u.btn->button == INPUT_BUTTON_WHEEL_UP) {
             s->qemu_put_mouse_event(s->qemu_put_mouse_event_opaque,
                                     s->axis[INPUT_AXIS_X],
                                     s->axis[INPUT_AXIS_Y],
                                     -1,
                                     s->buttons);
         }
-        if (evt->btn->down && evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+        if (evt->u.btn->down &&
+            evt->u.btn->button == INPUT_BUTTON_WHEEL_DOWN) {
             s->qemu_put_mouse_event(s->qemu_put_mouse_event_opaque,
                                     s->axis[INPUT_AXIS_X],
                                     s->axis[INPUT_AXIS_Y],
@@ -173,10 +175,10 @@ static void legacy_mouse_event(DeviceState *dev, QemuConsole *src,
         }
         break;
     case INPUT_EVENT_KIND_ABS:
-        s->axis[evt->abs->axis] = evt->abs->value;
+        s->axis[evt->u.abs->axis] = evt->u.abs->value;
         break;
     case INPUT_EVENT_KIND_REL:
-        s->axis[evt->rel->axis] += evt->rel->value;
+        s->axis[evt->u.rel->axis] += evt->u.rel->value;
         break;
     default:
         break;
@@ -205,7 +207,7 @@ QEMUPutMouseEntry *qemu_add_mouse_event_handler(QEMUPutMouseEvent *func,
 {
     QEMUPutMouseEntry *s;
 
-    s = g_malloc0(sizeof(QEMUPutMouseEntry));
+    s = g_new0(QEMUPutMouseEntry, 1);
 
     s->qemu_put_mouse_event = func;
     s->qemu_put_mouse_event_opaque = opaque;
@@ -239,7 +241,7 @@ QEMUPutLEDEntry *qemu_add_led_event_handler(QEMUPutLEDEvent *func,
 {
     QEMUPutLEDEntry *s;
 
-    s = g_malloc0(sizeof(QEMUPutLEDEntry));
+    s = g_new0(QEMUPutLEDEntry, 1);
 
     s->put_led = func;
     s->opaque = opaque;
diff --git a/ui/input.c b/ui/input.c
index 1a552d1de..a0f9873f5 100644
--- a/ui/input.c
+++ b/ui/input.c
@@ -6,6 +6,7 @@
 #include "trace.h"
 #include "ui/input.h"
 #include "ui/console.h"
+#include "sysemu/replay.h"
 
 struct QemuInputHandlerState {
     DeviceState       *dev;
@@ -147,10 +148,10 @@ void qmp_x_input_send_event(bool has_console, int64_t console,
     for (e = events; e != NULL; e = e->next) {
         InputEvent *event = e->value;
 
-        if (!qemu_input_find_handler(1 << event->kind, con)) {
+        if (!qemu_input_find_handler(1 << event->type, con)) {
             error_setg(errp, "Input handler not found for "
                              "event type %s",
-                            InputEventKind_lookup[event->kind]);
+                            InputEventKind_lookup[event->type]);
             return;
         }
     }
@@ -168,22 +169,22 @@ static void qemu_input_transform_abs_rotate(InputEvent *evt)
 {
     switch (graphic_rotate) {
     case 90:
-        if (evt->abs->axis == INPUT_AXIS_X) {
-            evt->abs->axis = INPUT_AXIS_Y;
-        } else if (evt->abs->axis == INPUT_AXIS_Y) {
-            evt->abs->axis = INPUT_AXIS_X;
-            evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
+        if (evt->u.abs->axis == INPUT_AXIS_X) {
+            evt->u.abs->axis = INPUT_AXIS_Y;
+        } else if (evt->u.abs->axis == INPUT_AXIS_Y) {
+            evt->u.abs->axis = INPUT_AXIS_X;
+            evt->u.abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->u.abs->value;
         }
         break;
     case 180:
-        evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
+        evt->u.abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->u.abs->value;
         break;
     case 270:
-        if (evt->abs->axis == INPUT_AXIS_X) {
-            evt->abs->axis = INPUT_AXIS_Y;
-            evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
-        } else if (evt->abs->axis == INPUT_AXIS_Y) {
-            evt->abs->axis = INPUT_AXIS_X;
+        if (evt->u.abs->axis == INPUT_AXIS_X) {
+            evt->u.abs->axis = INPUT_AXIS_Y;
+            evt->u.abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->u.abs->value;
+        } else if (evt->u.abs->axis == INPUT_AXIS_Y) {
+            evt->u.abs->axis = INPUT_AXIS_X;
         }
         break;
     }
@@ -197,18 +198,18 @@ static void qemu_input_event_trace(QemuConsole *src, InputEvent *evt)
     if (src) {
         idx = qemu_console_get_index(src);
     }
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_KEY:
-        switch (evt->key->key->kind) {
+        switch (evt->u.key->key->type) {
         case KEY_VALUE_KIND_NUMBER:
-            qcode = qemu_input_key_number_to_qcode(evt->key->key->number);
+            qcode = qemu_input_key_number_to_qcode(evt->u.key->key->u.number);
             name = QKeyCode_lookup[qcode];
-            trace_input_event_key_number(idx, evt->key->key->number,
-                                         name, evt->key->down);
+            trace_input_event_key_number(idx, evt->u.key->key->u.number,
+                                         name, evt->u.key->down);
             break;
         case KEY_VALUE_KIND_QCODE:
-            name = QKeyCode_lookup[evt->key->key->qcode];
-            trace_input_event_key_qcode(idx, name, evt->key->down);
+            name = QKeyCode_lookup[evt->u.key->key->u.qcode];
+            trace_input_event_key_qcode(idx, name, evt->u.key->down);
             break;
         case KEY_VALUE_KIND_MAX:
             /* keep gcc happy */
@@ -216,16 +217,16 @@ static void qemu_input_event_trace(QemuConsole *src, InputEvent *evt)
         }
         break;
     case INPUT_EVENT_KIND_BTN:
-        name = InputButton_lookup[evt->btn->button];
-        trace_input_event_btn(idx, name, evt->btn->down);
+        name = InputButton_lookup[evt->u.btn->button];
+        trace_input_event_btn(idx, name, evt->u.btn->down);
         break;
     case INPUT_EVENT_KIND_REL:
-        name = InputAxis_lookup[evt->rel->axis];
-        trace_input_event_rel(idx, name, evt->rel->value);
+        name = InputAxis_lookup[evt->u.rel->axis];
+        trace_input_event_rel(idx, name, evt->u.rel->value);
         break;
     case INPUT_EVENT_KIND_ABS:
-        name = InputAxis_lookup[evt->abs->axis];
-        trace_input_event_abs(idx, name, evt->abs->value);
+        name = InputAxis_lookup[evt->u.abs->axis];
+        trace_input_event_abs(idx, name, evt->u.abs->value);
         break;
     case INPUT_EVENT_KIND_MAX:
         /* keep gcc happy */
@@ -300,23 +301,19 @@ static void qemu_input_queue_sync(struct QemuInputEventQueueHead *queue)
     QTAILQ_INSERT_TAIL(queue, item, node);
 }
 
-void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
+void qemu_input_event_send_impl(QemuConsole *src, InputEvent *evt)
 {
     QemuInputHandlerState *s;
 
-    if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
-        return;
-    }
-
     qemu_input_event_trace(src, evt);
 
     /* pre processing */
-    if (graphic_rotate && (evt->kind == INPUT_EVENT_KIND_ABS)) {
+    if (graphic_rotate && (evt->type == INPUT_EVENT_KIND_ABS)) {
             qemu_input_transform_abs_rotate(evt);
     }
 
     /* send event */
-    s = qemu_input_find_handler(1 << evt->kind, src);
+    s = qemu_input_find_handler(1 << evt->type, src);
     if (!s) {
         return;
     }
@@ -324,14 +321,19 @@ void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
     s->events++;
 }
 
-void qemu_input_event_sync(void)
+void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
 {
-    QemuInputHandlerState *s;
-
     if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
         return;
     }
 
+    replay_input_event(src, evt);
+}
+
+void qemu_input_event_sync_impl(void)
+{
+    QemuInputHandlerState *s;
+
     trace_input_event_sync();
 
     QTAILQ_FOREACH(s, &handlers, node) {
@@ -345,13 +347,22 @@ void qemu_input_event_sync(void)
     }
 }
 
+void qemu_input_event_sync(void)
+{
+    if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
+        return;
+    }
+
+    replay_input_sync_event();
+}
+
 InputEvent *qemu_input_event_new_key(KeyValue *key, bool down)
 {
     InputEvent *evt = g_new0(InputEvent, 1);
-    evt->key = g_new0(InputKeyEvent, 1);
-    evt->kind = INPUT_EVENT_KIND_KEY;
-    evt->key->key = key;
-    evt->key->down = down;
+    evt->u.key = g_new0(InputKeyEvent, 1);
+    evt->type = INPUT_EVENT_KIND_KEY;
+    evt->u.key->key = key;
+    evt->u.key->down = down;
     return evt;
 }
 
@@ -372,16 +383,16 @@ void qemu_input_event_send_key(QemuConsole *src, KeyValue *key, bool down)
 void qemu_input_event_send_key_number(QemuConsole *src, int num, bool down)
 {
     KeyValue *key = g_new0(KeyValue, 1);
-    key->kind = KEY_VALUE_KIND_NUMBER;
-    key->number = num;
+    key->type = KEY_VALUE_KIND_NUMBER;
+    key->u.number = num;
     qemu_input_event_send_key(src, key, down);
 }
 
 void qemu_input_event_send_key_qcode(QemuConsole *src, QKeyCode q, bool down)
 {
     KeyValue *key = g_new0(KeyValue, 1);
-    key->kind = KEY_VALUE_KIND_QCODE;
-    key->qcode = q;
+    key->type = KEY_VALUE_KIND_QCODE;
+    key->u.qcode = q;
     qemu_input_event_send_key(src, key, down);
 }
 
@@ -398,10 +409,10 @@ void qemu_input_event_send_key_delay(uint32_t delay_ms)
 InputEvent *qemu_input_event_new_btn(InputButton btn, bool down)
 {
     InputEvent *evt = g_new0(InputEvent, 1);
-    evt->btn = g_new0(InputBtnEvent, 1);
-    evt->kind = INPUT_EVENT_KIND_BTN;
-    evt->btn->button = btn;
-    evt->btn->down = down;
+    evt->u.btn = g_new0(InputBtnEvent, 1);
+    evt->type = INPUT_EVENT_KIND_BTN;
+    evt->u.btn->button = btn;
+    evt->u.btn->down = down;
     return evt;
 }
 
@@ -451,8 +462,8 @@ InputEvent *qemu_input_event_new_move(InputEventKind kind,
     InputEvent *evt = g_new0(InputEvent, 1);
     InputMoveEvent *move = g_new0(InputMoveEvent, 1);
 
-    evt->kind = kind;
-    evt->data = move;
+    evt->type = kind;
+    evt->u.data = move;
     move->axis = axis;
     move->value = value;
     return evt;
diff --git a/ui/keymaps.c b/ui/keymaps.c
index 49410ae9d..1b9ba3fa2 100644
--- a/ui/keymaps.c
+++ b/ui/keymaps.c
@@ -109,7 +109,7 @@ static kbd_layout_t *parse_keyboard_layout(const name2keysym_t *table,
     }
 
     if (!k) {
-        k = g_malloc0(sizeof(kbd_layout_t));
+        k = g_new0(kbd_layout_t, 1);
     }
 
     for(;;) {
diff --git a/ui/sdl.c b/ui/sdl.c
index 3be29101e..570cb99f0 100644
--- a/ui/sdl.c
+++ b/ui/sdl.c
@@ -985,7 +985,7 @@ void sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
         sdl_grab_start();
     }
 
-    dcl = g_malloc0(sizeof(DisplayChangeListener));
+    dcl = g_new0(DisplayChangeListener, 1);
     dcl->ops = &dcl_ops;
     register_displaychangelistener(dcl);
 
diff --git a/ui/sdl2-2d.c b/ui/sdl2-2d.c
index d0b340f95..191ee3b52 100644
--- a/ui/sdl2-2d.c
+++ b/ui/sdl2-2d.c
@@ -45,10 +45,23 @@ void sdl2_2d_update(DisplayChangeListener *dcl,
         return;
     }
 
+    /*
+     * SDL2 seems to do some double-buffering, and trying to only
+     * update the changed areas results in only one of the two buffers
+     * being updated.  Which flickers alot.  So lets not try to be
+     * clever do a full update every time ...
+     */
+#if 0
     rect.x = x;
     rect.y = y;
     rect.w = w;
     rect.h = h;
+#else
+    rect.x = 0;
+    rect.y = 0;
+    rect.w = surface_width(surf);
+    rect.h = surface_height(surf);
+#endif
 
     SDL_UpdateTexture(scon->texture, NULL, surface_data(surf),
                       surface_stride(surf));
diff --git a/ui/shader.c b/ui/shader.c
index 52a463293..0588655cf 100644
--- a/ui/shader.c
+++ b/ui/shader.c
@@ -29,21 +29,42 @@
 
 /* ---------------------------------------------------------------------- */
 
-void qemu_gl_run_texture_blit(GLint texture_blit_prog)
+GLuint qemu_gl_init_texture_blit(GLint texture_blit_prog)
 {
-    GLfloat in_position[] = {
+    static const GLfloat in_position[] = {
         -1, -1,
         1,  -1,
         -1,  1,
         1,   1,
     };
     GLint l_position;
+    GLuint vao, buffer;
+
+    glGenVertexArrays(1, &vao);
+    glBindVertexArray(vao);
+
+    /* this is the VBO that holds the vertex data */
+    glGenBuffers(1, &buffer);
+    glBindBuffer(GL_ARRAY_BUFFER, buffer);
+    glBufferData(GL_ARRAY_BUFFER, sizeof(in_position), in_position,
+                 GL_STATIC_DRAW);
 
-    glUseProgram(texture_blit_prog);
     l_position = glGetAttribLocation(texture_blit_prog, "in_position");
-    glVertexAttribPointer(l_position, 2, GL_FLOAT, GL_FALSE, 0, in_position);
+    glVertexAttribPointer(l_position, 2, GL_FLOAT, GL_FALSE, 0, 0);
     glEnableVertexAttribArray(l_position);
-    glDrawArrays(GL_TRIANGLE_STRIP, l_position, 4);
+
+    glBindBuffer(GL_ARRAY_BUFFER, 0);
+    glBindVertexArray(0);
+
+    return vao;
+}
+
+void qemu_gl_run_texture_blit(GLint texture_blit_prog,
+                              GLint texture_blit_vao)
+{
+    glUseProgram(texture_blit_prog);
+    glBindVertexArray(texture_blit_vao);
+    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
 }
 
 /* ---------------------------------------------------------------------- */
diff --git a/ui/spice-core.c b/ui/spice-core.c
index bf4fd0749..6a62d712f 100644
--- a/ui/spice-core.c
+++ b/ui/spice-core.c
@@ -200,8 +200,6 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
 {
     SpiceServerInfo *server = g_malloc0(sizeof(*server));
     SpiceChannel *client = g_malloc0(sizeof(*client));
-    server->base = g_malloc0(sizeof(*server->base));
-    client->base = g_malloc0(sizeof(*client->base));
 
     /*
      * Spice server might have called us from spice worker thread
@@ -218,9 +216,11 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
     }
 
     if (info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
-        add_addr_info(client->base, (struct sockaddr *)&info->paddr_ext,
+        add_addr_info(qapi_SpiceChannel_base(client),
+                      (struct sockaddr *)&info->paddr_ext,
                       info->plen_ext);
-        add_addr_info(server->base, (struct sockaddr *)&info->laddr_ext,
+        add_addr_info(qapi_SpiceServerInfo_base(server),
+                      (struct sockaddr *)&info->laddr_ext,
                       info->llen_ext);
     } else {
         error_report("spice: %s, extended address is expected",
@@ -229,7 +229,9 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
 
     switch (event) {
     case SPICE_CHANNEL_EVENT_CONNECTED:
-        qapi_event_send_spice_connected(server->base, client->base, &error_abort);
+        qapi_event_send_spice_connected(qapi_SpiceServerInfo_base(server),
+                                        qapi_SpiceChannel_base(client),
+                                        &error_abort);
         break;
     case SPICE_CHANNEL_EVENT_INITIALIZED:
         if (auth) {
@@ -242,7 +244,9 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
         break;
     case SPICE_CHANNEL_EVENT_DISCONNECTED:
         channel_list_del(info);
-        qapi_event_send_spice_disconnected(server->base, client->base, &error_abort);
+        qapi_event_send_spice_disconnected(qapi_SpiceServerInfo_base(server),
+                                           qapi_SpiceChannel_base(client),
+                                           &error_abort);
         break;
     default:
         break;
@@ -378,16 +382,15 @@ static SpiceChannelList *qmp_query_spice_channels(void)
 
         chan = g_malloc0(sizeof(*chan));
         chan->value = g_malloc0(sizeof(*chan->value));
-        chan->value->base = g_malloc0(sizeof(*chan->value->base));
 
         paddr = (struct sockaddr *)&item->info->paddr_ext;
         plen = item->info->plen_ext;
         getnameinfo(paddr, plen,
                     host, sizeof(host), port, sizeof(port),
                     NI_NUMERICHOST | NI_NUMERICSERV);
-        chan->value->base->host = g_strdup(host);
-        chan->value->base->port = g_strdup(port);
-        chan->value->base->family = inet_netfamily(paddr->sa_family);
+        chan->value->host = g_strdup(host);
+        chan->value->port = g_strdup(port);
+        chan->value->family = inet_netfamily(paddr->sa_family);
 
         chan->value->connection_id = item->info->connection_id;
         chan->value->channel_type = item->info->type;
diff --git a/ui/spice-display.c b/ui/spice-display.c
index 0360abfd2..048913196 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -408,7 +408,8 @@ void qemu_spice_display_switch(SimpleSpiceDisplay *ssd,
 
     if (surface && ssd->surface &&
         surface_width(surface) == pixman_image_get_width(ssd->surface) &&
-        surface_height(surface) == pixman_image_get_height(ssd->surface)) {
+        surface_height(surface) == pixman_image_get_height(ssd->surface) &&
+        surface_format(surface) == pixman_image_get_format(ssd->surface)) {
         /* no-resize fast path: just swap backing store */
         dprint(1, "%s/%d: fast (%dx%d)\n", __func__, ssd->qxl.id,
                surface_width(surface), surface_height(surface));
@@ -737,9 +738,7 @@ static void display_mouse_set(DisplayChangeListener *dcl,
     qemu_mutex_lock(&ssd->lock);
     ssd->ptr_x = x;
     ssd->ptr_y = y;
-    if (ssd->ptr_move) {
-        g_free(ssd->ptr_move);
-    }
+    g_free(ssd->ptr_move);
     ssd->ptr_move = qemu_spice_create_cursor_update(ssd, NULL, on);
     qemu_mutex_unlock(&ssd->lock);
 }
@@ -752,13 +751,9 @@ static void display_mouse_define(DisplayChangeListener *dcl,
     qemu_mutex_lock(&ssd->lock);
     ssd->hot_x = c->hot_x;
     ssd->hot_y = c->hot_y;
-    if (ssd->ptr_move) {
-        g_free(ssd->ptr_move);
-        ssd->ptr_move = NULL;
-    }
-    if (ssd->ptr_define) {
-        g_free(ssd->ptr_define);
-    }
+    g_free(ssd->ptr_move);
+    ssd->ptr_move = NULL;
+    g_free(ssd->ptr_define);
     ssd->ptr_define = qemu_spice_create_cursor_update(ssd, c, 0);
     qemu_mutex_unlock(&ssd->lock);
 }
diff --git a/ui/vnc-auth-sasl.c b/ui/vnc-auth-sasl.c
index 62a5fc4bf..fc732bdba 100644
--- a/ui/vnc-auth-sasl.c
+++ b/ui/vnc-auth-sasl.c
@@ -525,21 +525,24 @@ void start_auth_sasl(VncState *vs)
         goto authabort;
     }
 
-#ifdef CONFIG_VNC_TLS
     /* Inform SASL that we've got an external SSF layer from TLS/x509 */
     if (vs->auth == VNC_AUTH_VENCRYPT &&
         vs->subauth == VNC_AUTH_VENCRYPT_X509SASL) {
-        gnutls_cipher_algorithm_t cipher;
+        Error *local_err = NULL;
+        int keysize;
         sasl_ssf_t ssf;
 
-        cipher = gnutls_cipher_get(vs->tls.session);
-        if (!(ssf = (sasl_ssf_t)gnutls_cipher_get_key_size(cipher))) {
-            VNC_DEBUG("%s", "cannot TLS get cipher size\n");
+        keysize = qcrypto_tls_session_get_key_size(vs->tls,
+                                                   &local_err);
+        if (keysize < 0) {
+            VNC_DEBUG("cannot TLS get cipher size: %s\n",
+                      error_get_pretty(local_err));
+            error_free(local_err);
             sasl_dispose(&vs->sasl.conn);
             vs->sasl.conn = NULL;
             goto authabort;
         }
-        ssf *= 8; /* tls key size is bytes, sasl wants bits */
+        ssf = keysize * CHAR_BIT; /* tls key size is bytes, sasl wants bits */
 
         err = sasl_setprop(vs->sasl.conn, SASL_SSF_EXTERNAL, &ssf);
         if (err != SASL_OK) {
@@ -549,20 +552,19 @@ void start_auth_sasl(VncState *vs)
             vs->sasl.conn = NULL;
             goto authabort;
         }
-    } else
-#endif /* CONFIG_VNC_TLS */
+    } else {
         vs->sasl.wantSSF = 1;
+    }
 
     memset (&secprops, 0, sizeof secprops);
-    /* Inform SASL that we've got an external SSF layer from TLS */
-    if (vs->vd->is_unix
-#ifdef CONFIG_VNC_TLS
-        /* Disable SSF, if using TLS+x509+SASL only. TLS without x509
-           is not sufficiently strong */
-        || (vs->auth == VNC_AUTH_VENCRYPT &&
-            vs->subauth == VNC_AUTH_VENCRYPT_X509SASL)
-#endif /* CONFIG_VNC_TLS */
-        ) {
+    /* Inform SASL that we've got an external SSF layer from TLS.
+     *
+     * Disable SSF, if using TLS+x509+SASL only. TLS without x509
+     * is not sufficiently strong
+     */
+    if (vs->vd->is_unix ||
+        (vs->auth == VNC_AUTH_VENCRYPT &&
+         vs->subauth == VNC_AUTH_VENCRYPT_X509SASL)) {
         /* If we've got TLS or UNIX domain sock, we don't care about SSF */
         secprops.min_ssf = 0;
         secprops.max_ssf = 0;
diff --git a/ui/vnc-auth-vencrypt.c b/ui/vnc-auth-vencrypt.c
index 8fc965b4a..44ac2fae6 100644
--- a/ui/vnc-auth-vencrypt.c
+++ b/ui/vnc-auth-vencrypt.c
@@ -67,38 +67,42 @@ static void vnc_tls_handshake_io(void *opaque);
 
 static int vnc_start_vencrypt_handshake(VncState *vs)
 {
-    int ret;
-
-    if ((ret = gnutls_handshake(vs->tls.session)) < 0) {
-       if (!gnutls_error_is_fatal(ret)) {
-           VNC_DEBUG("Handshake interrupted (blocking)\n");
-           if (!gnutls_record_get_direction(vs->tls.session))
-               qemu_set_fd_handler(vs->csock, vnc_tls_handshake_io, NULL, vs);
-           else
-               qemu_set_fd_handler(vs->csock, NULL, vnc_tls_handshake_io, vs);
-           return 0;
-       }
-       VNC_DEBUG("Handshake failed %s\n", gnutls_strerror(ret));
-       vnc_client_error(vs);
-       return -1;
+    Error *err = NULL;
+
+    if (qcrypto_tls_session_handshake(vs->tls, &err) < 0) {
+        goto error;
     }
 
-    if (vs->vd->tls.x509verify) {
-        if (vnc_tls_validate_certificate(vs) < 0) {
-            VNC_DEBUG("Client verification failed\n");
-            vnc_client_error(vs);
-            return -1;
-        } else {
-            VNC_DEBUG("Client verification passed\n");
+    switch (qcrypto_tls_session_get_handshake_status(vs->tls)) {
+    case QCRYPTO_TLS_HANDSHAKE_COMPLETE:
+        VNC_DEBUG("Handshake done, checking credentials\n");
+        if (qcrypto_tls_session_check_credentials(vs->tls, &err) < 0) {
+            goto error;
         }
-    }
+        VNC_DEBUG("Client verification passed, starting TLS I/O\n");
+        qemu_set_fd_handler(vs->csock, vnc_client_read, vnc_client_write, vs);
+
+        start_auth_vencrypt_subauth(vs);
+        break;
 
-    VNC_DEBUG("Handshake done, switching to TLS data mode\n");
-    qemu_set_fd_handler(vs->csock, vnc_client_read, vnc_client_write, vs);
+    case QCRYPTO_TLS_HANDSHAKE_RECVING:
+        VNC_DEBUG("Handshake interrupted (blocking read)\n");
+        qemu_set_fd_handler(vs->csock, vnc_tls_handshake_io, NULL, vs);
+        break;
 
-    start_auth_vencrypt_subauth(vs);
+    case QCRYPTO_TLS_HANDSHAKE_SENDING:
+        VNC_DEBUG("Handshake interrupted (blocking write)\n");
+        qemu_set_fd_handler(vs->csock, NULL, vnc_tls_handshake_io, vs);
+        break;
+    }
 
     return 0;
+
+ error:
+    VNC_DEBUG("Handshake failed %s\n", error_get_pretty(err));
+    error_free(err);
+    vnc_client_error(vs);
+    return -1;
 }
 
 static void vnc_tls_handshake_io(void *opaque)
@@ -110,14 +114,6 @@ static void vnc_tls_handshake_io(void *opaque)
 }
 
 
-
-#define NEED_X509_AUTH(vs)                              \
-    ((vs)->subauth == VNC_AUTH_VENCRYPT_X509NONE ||   \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509VNC ||    \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509PLAIN ||  \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509SASL)
-
-
 static int protocol_client_vencrypt_auth(VncState *vs, uint8_t *data, size_t len)
 {
     int auth = read_u32(data, 0);
@@ -128,15 +124,29 @@ static int protocol_client_vencrypt_auth(VncState *vs, uint8_t *data, size_t len
         vnc_flush(vs);
         vnc_client_error(vs);
     } else {
+        Error *err = NULL;
         VNC_DEBUG("Accepting auth %d, setting up TLS for handshake\n", auth);
         vnc_write_u8(vs, 1); /* Accept auth */
         vnc_flush(vs);
 
-        if (vnc_tls_client_setup(vs, NEED_X509_AUTH(vs)) < 0) {
-            VNC_DEBUG("Failed to setup TLS\n");
+        vs->tls = qcrypto_tls_session_new(vs->vd->tlscreds,
+                                          NULL,
+                                          vs->vd->tlsaclname,
+                                          QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+                                          &err);
+        if (!vs->tls) {
+            VNC_DEBUG("Failed to setup TLS %s\n",
+                      error_get_pretty(err));
+            error_free(err);
+            vnc_client_error(vs);
             return 0;
         }
 
+        qcrypto_tls_session_set_callbacks(vs->tls,
+                                          vnc_tls_push,
+                                          vnc_tls_pull,
+                                          vs);
+
         VNC_DEBUG("Start TLS VeNCrypt handshake process\n");
         if (vnc_start_vencrypt_handshake(vs) < 0) {
             VNC_DEBUG("Failed to start TLS handshake\n");
diff --git a/ui/vnc-jobs.c b/ui/vnc-jobs.c
index 22c9abce5..aa21191ea 100644
--- a/ui/vnc-jobs.c
+++ b/ui/vnc-jobs.c
@@ -29,6 +29,7 @@
 #include "vnc.h"
 #include "vnc-jobs.h"
 #include "qemu/sockets.h"
+#include "qemu/main-loop.h"
 #include "block/aio.h"
 
 /*
@@ -54,7 +55,6 @@ struct VncJobQueue {
     QemuCond cond;
     QemuMutex mutex;
     QemuThread thread;
-    Buffer buffer;
     bool exit;
     QTAILQ_HEAD(, VncJob) jobs;
 };
@@ -79,7 +79,7 @@ static void vnc_unlock_queue(VncJobQueue *queue)
 
 VncJob *vnc_job_new(VncState *vs)
 {
-    VncJob *job = g_malloc0(sizeof(VncJob));
+    VncJob *job = g_new0(VncJob, 1);
 
     job->vs = vs;
     vnc_lock_queue(queue);
@@ -90,7 +90,7 @@ VncJob *vnc_job_new(VncState *vs)
 
 int vnc_job_add_rect(VncJob *job, int x, int y, int w, int h)
 {
-    VncRectEntry *entry = g_malloc0(sizeof(VncRectEntry));
+    VncRectEntry *entry = g_new0(VncRectEntry, 1);
 
     entry->rect.x = x;
     entry->rect.y = y;
@@ -166,8 +166,11 @@ void vnc_jobs_consume_buffer(VncState *vs)
 
     vnc_lock_output(vs);
     if (vs->jobs_buffer.offset) {
-        vnc_write(vs, vs->jobs_buffer.buffer, vs->jobs_buffer.offset);
-        buffer_reset(&vs->jobs_buffer);
+        if (vs->csock != -1 && buffer_empty(&vs->output)) {
+            qemu_set_fd_handler(vs->csock, vnc_client_read,
+                                vnc_client_write, vs);
+        }
+        buffer_move(&vs->output, &vs->jobs_buffer);
     }
     flush = vs->csock != -1 && vs->abort != true;
     vnc_unlock_output(vs);
@@ -182,6 +185,9 @@ void vnc_jobs_consume_buffer(VncState *vs)
  */
 static void vnc_async_encoding_start(VncState *orig, VncState *local)
 {
+    buffer_init(&local->output, "vnc-worker-output");
+    local->csock = -1; /* Don't do any network work on this thread */
+
     local->vnc_encoding = orig->vnc_encoding;
     local->features = orig->features;
     local->vd = orig->vd;
@@ -193,10 +199,6 @@ static void vnc_async_encoding_start(VncState *orig, VncState *local)
     local->zlib = orig->zlib;
     local->hextile = orig->hextile;
     local->zrle = orig->zrle;
-    local->output =  queue->buffer;
-    local->csock = -1; /* Don't do any network work on this thread */
-
-    buffer_reset(&local->output);
 }
 
 static void vnc_async_encoding_end(VncState *orig, VncState *local)
@@ -206,15 +208,13 @@ static void vnc_async_encoding_end(VncState *orig, VncState *local)
     orig->hextile = local->hextile;
     orig->zrle = local->zrle;
     orig->lossy_rect = local->lossy_rect;
-
-    queue->buffer = local->output;
 }
 
 static int vnc_worker_thread_loop(VncJobQueue *queue)
 {
     VncJob *job;
     VncRectEntry *entry, *tmp;
-    VncState vs;
+    VncState vs = {};
     int n_rectangles;
     int saved_offset;
 
@@ -235,6 +235,14 @@ static int vnc_worker_thread_loop(VncJobQueue *queue)
         vnc_unlock_output(job->vs);
         goto disconnected;
     }
+    if (buffer_empty(&job->vs->output)) {
+        /*
+         * Looks like a NOP as it obviously moves no data.  But it
+         * moves the empty buffer, so we don't have to malloc a new
+         * one for vs.output
+         */
+        buffer_move_empty(&vs.output, &job->vs->output);
+    }
     vnc_unlock_output(job->vs);
 
     /* Make a local copy of vs and switch output buffers */
@@ -274,14 +282,13 @@ static int vnc_worker_thread_loop(VncJobQueue *queue)
 
     vnc_lock_output(job->vs);
     if (job->vs->csock != -1) {
-        buffer_reserve(&job->vs->jobs_buffer, vs.output.offset);
-        buffer_append(&job->vs->jobs_buffer, vs.output.buffer,
-                      vs.output.offset);
+        buffer_move(&job->vs->jobs_buffer, &vs.output);
         /* Copy persistent encoding data */
         vnc_async_encoding_end(job->vs, &vs);
 
 	qemu_bh_schedule(job->vs->bh);
     }  else {
+        buffer_reset(&vs.output);
         /* Copy persistent encoding data */
         vnc_async_encoding_end(job->vs, &vs);
     }
@@ -298,7 +305,7 @@ disconnected:
 
 static VncJobQueue *vnc_queue_init(void)
 {
-    VncJobQueue *queue = g_malloc0(sizeof(VncJobQueue));
+    VncJobQueue *queue = g_new0(VncJobQueue, 1);
 
     qemu_cond_init(&queue->cond);
     qemu_mutex_init(&queue->mutex);
@@ -310,7 +317,6 @@ static void vnc_queue_clear(VncJobQueue *q)
 {
     qemu_cond_destroy(&queue->cond);
     qemu_mutex_destroy(&queue->mutex);
-    buffer_free(&queue->buffer);
     g_free(q);
     queue = NULL; /* Unset global queue */
 }
diff --git a/ui/vnc-tls.c b/ui/vnc-tls.c
deleted file mode 100644
index 028fc4db1..000000000
--- a/ui/vnc-tls.c
+++ /dev/null
@@ -1,474 +0,0 @@
-/*
- * QEMU VNC display driver: TLS helpers
- *
- * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2006 Fabrice Bellard
- * Copyright (C) 2009 Red Hat, Inc
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-x509.h"
-#include "vnc.h"
-#include "qemu/sockets.h"
-
-#if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
-/* Very verbose, so only enabled for _VNC_DEBUG >= 2 */
-static void vnc_debug_gnutls_log(int level, const char* str) {
-    VNC_DEBUG("%d %s", level, str);
-}
-#endif /* defined(_VNC_DEBUG) && _VNC_DEBUG >= 2 */
-
-
-#define DH_BITS 1024
-static gnutls_dh_params_t dh_params;
-
-static int vnc_tls_initialize(void)
-{
-    static int tlsinitialized = 0;
-
-    if (tlsinitialized)
-        return 1;
-
-    if (gnutls_global_init () < 0)
-        return 0;
-
-    /* XXX ought to re-generate diffie-hellman params periodically */
-    if (gnutls_dh_params_init (&dh_params) < 0)
-        return 0;
-    if (gnutls_dh_params_generate2 (dh_params, DH_BITS) < 0)
-        return 0;
-
-#if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
-    gnutls_global_set_log_level(10);
-    gnutls_global_set_log_function(vnc_debug_gnutls_log);
-#endif
-
-    tlsinitialized = 1;
-
-    return 1;
-}
-
-static ssize_t vnc_tls_push(gnutls_transport_ptr_t transport,
-                            const void *data,
-                            size_t len) {
-    VncState *vs = (VncState *)transport;
-    int ret;
-
- retry:
-    ret = send(vs->csock, data, len, 0);
-    if (ret < 0) {
-        if (errno == EINTR)
-            goto retry;
-        return -1;
-    }
-    return ret;
-}
-
-
-static ssize_t vnc_tls_pull(gnutls_transport_ptr_t transport,
-                            void *data,
-                            size_t len) {
-    VncState *vs = (VncState *)transport;
-    int ret;
-
- retry:
-    ret = qemu_recv(vs->csock, data, len, 0);
-    if (ret < 0) {
-        if (errno == EINTR)
-            goto retry;
-        return -1;
-    }
-    return ret;
-}
-
-
-static gnutls_anon_server_credentials_t vnc_tls_initialize_anon_cred(void)
-{
-    gnutls_anon_server_credentials_t anon_cred;
-    int ret;
-
-    if ((ret = gnutls_anon_allocate_server_credentials(&anon_cred)) < 0) {
-        VNC_DEBUG("Cannot allocate credentials %s\n", gnutls_strerror(ret));
-        return NULL;
-    }
-
-    gnutls_anon_set_server_dh_params(anon_cred, dh_params);
-
-    return anon_cred;
-}
-
-
-static gnutls_certificate_credentials_t vnc_tls_initialize_x509_cred(VncDisplay *vd)
-{
-    gnutls_certificate_credentials_t x509_cred;
-    int ret;
-
-    if (!vd->tls.x509cacert) {
-        VNC_DEBUG("No CA x509 certificate specified\n");
-        return NULL;
-    }
-    if (!vd->tls.x509cert) {
-        VNC_DEBUG("No server x509 certificate specified\n");
-        return NULL;
-    }
-    if (!vd->tls.x509key) {
-        VNC_DEBUG("No server private key specified\n");
-        return NULL;
-    }
-
-    if ((ret = gnutls_certificate_allocate_credentials(&x509_cred)) < 0) {
-        VNC_DEBUG("Cannot allocate credentials %s\n", gnutls_strerror(ret));
-        return NULL;
-    }
-    if ((ret = gnutls_certificate_set_x509_trust_file(x509_cred,
-                                                      vd->tls.x509cacert,
-                                                      GNUTLS_X509_FMT_PEM)) < 0) {
-        VNC_DEBUG("Cannot load CA certificate %s\n", gnutls_strerror(ret));
-        gnutls_certificate_free_credentials(x509_cred);
-        return NULL;
-    }
-
-    if ((ret = gnutls_certificate_set_x509_key_file (x509_cred,
-                                                     vd->tls.x509cert,
-                                                     vd->tls.x509key,
-                                                     GNUTLS_X509_FMT_PEM)) < 0) {
-        VNC_DEBUG("Cannot load certificate & key %s\n", gnutls_strerror(ret));
-        gnutls_certificate_free_credentials(x509_cred);
-        return NULL;
-    }
-
-    if (vd->tls.x509cacrl) {
-        if ((ret = gnutls_certificate_set_x509_crl_file(x509_cred,
-                                                        vd->tls.x509cacrl,
-                                                        GNUTLS_X509_FMT_PEM)) < 0) {
-            VNC_DEBUG("Cannot load CRL %s\n", gnutls_strerror(ret));
-            gnutls_certificate_free_credentials(x509_cred);
-            return NULL;
-        }
-    }
-
-    gnutls_certificate_set_dh_params (x509_cred, dh_params);
-
-    return x509_cred;
-}
-
-
-int vnc_tls_validate_certificate(VncState *vs)
-{
-    int ret;
-    unsigned int status;
-    const gnutls_datum_t *certs;
-    unsigned int nCerts, i;
-    time_t now;
-
-    VNC_DEBUG("Validating client certificate\n");
-    if ((ret = gnutls_certificate_verify_peers2 (vs->tls.session, &status)) < 0) {
-        VNC_DEBUG("Verify failed %s\n", gnutls_strerror(ret));
-        return -1;
-    }
-
-    if ((now = time(NULL)) == ((time_t)-1)) {
-        return -1;
-    }
-
-    if (status != 0) {
-        if (status & GNUTLS_CERT_INVALID)
-            VNC_DEBUG("The certificate is not trusted.\n");
-
-        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND)
-            VNC_DEBUG("The certificate hasn't got a known issuer.\n");
-
-        if (status & GNUTLS_CERT_REVOKED)
-            VNC_DEBUG("The certificate has been revoked.\n");
-
-        if (status & GNUTLS_CERT_INSECURE_ALGORITHM)
-            VNC_DEBUG("The certificate uses an insecure algorithm\n");
-
-        return -1;
-    } else {
-        VNC_DEBUG("Certificate is valid!\n");
-    }
-
-    /* Only support x509 for now */
-    if (gnutls_certificate_type_get(vs->tls.session) != GNUTLS_CRT_X509)
-        return -1;
-
-    if (!(certs = gnutls_certificate_get_peers(vs->tls.session, &nCerts)))
-        return -1;
-
-    for (i = 0 ; i < nCerts ; i++) {
-        gnutls_x509_crt_t cert;
-        VNC_DEBUG ("Checking certificate chain %d\n", i);
-        if (gnutls_x509_crt_init (&cert) < 0)
-            return -1;
-
-        if (gnutls_x509_crt_import(cert, &certs[i], GNUTLS_X509_FMT_DER) < 0) {
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_expiration_time (cert) < now) {
-            VNC_DEBUG("The certificate has expired\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_activation_time (cert) > now) {
-            VNC_DEBUG("The certificate is not yet activated\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_activation_time (cert) > now) {
-            VNC_DEBUG("The certificate is not yet activated\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (i == 0) {
-            size_t dnameSize = 1024;
-            vs->tls.dname = g_malloc(dnameSize);
-        requery:
-            if ((ret = gnutls_x509_crt_get_dn (cert, vs->tls.dname, &dnameSize)) != 0) {
-                if (ret == GNUTLS_E_SHORT_MEMORY_BUFFER) {
-                    vs->tls.dname = g_realloc(vs->tls.dname, dnameSize);
-                    goto requery;
-                }
-                gnutls_x509_crt_deinit (cert);
-                VNC_DEBUG("Cannot get client distinguished name: %s",
-                          gnutls_strerror (ret));
-                return -1;
-            }
-
-            if (vs->vd->tls.x509verify) {
-                int allow;
-                if (!vs->vd->tls.acl) {
-                    VNC_DEBUG("no ACL activated, allowing access");
-                    gnutls_x509_crt_deinit (cert);
-                    continue;
-                }
-
-                allow = qemu_acl_party_is_allowed(vs->vd->tls.acl,
-                                                  vs->tls.dname);
-
-                VNC_DEBUG("TLS x509 ACL check for %s is %s\n",
-                          vs->tls.dname, allow ? "allowed" : "denied");
-                if (!allow) {
-                    gnutls_x509_crt_deinit (cert);
-                    return -1;
-                }
-            }
-        }
-
-        gnutls_x509_crt_deinit (cert);
-    }
-
-    return 0;
-}
-
-#if defined(GNUTLS_VERSION_NUMBER) && \
-    GNUTLS_VERSION_NUMBER >= 0x020200 /* 2.2.0 */
-
-static int vnc_set_gnutls_priority(gnutls_session_t s, int x509)
-{
-    const char *priority = x509 ? "NORMAL" : "NORMAL:+ANON-DH";
-    int rc;
-
-    rc = gnutls_priority_set_direct(s, priority, NULL);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-    return 0;
-}
-
-#else
-
-static int vnc_set_gnutls_priority(gnutls_session_t s, int x509)
-{
-    static const int cert_types[] = { GNUTLS_CRT_X509, 0 };
-    static const int protocols[] = {
-        GNUTLS_TLS1_1, GNUTLS_TLS1_0, GNUTLS_SSL3, 0
-    };
-    static const int kx_anon[] = { GNUTLS_KX_ANON_DH, 0 };
-    static const int kx_x509[] = {
-        GNUTLS_KX_DHE_DSS, GNUTLS_KX_RSA,
-        GNUTLS_KX_DHE_RSA, GNUTLS_KX_SRP, 0
-    };
-    int rc;
-
-    rc = gnutls_kx_set_priority(s, x509 ? kx_x509 : kx_anon);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-
-    rc = gnutls_certificate_type_set_priority(s, cert_types);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-
-    rc = gnutls_protocol_set_priority(s, protocols);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-    return 0;
-}
-
-#endif
-
-int vnc_tls_client_setup(VncState *vs,
-                         int needX509Creds) {
-    VNC_DEBUG("Do TLS setup\n");
-    if (vnc_tls_initialize() < 0) {
-        VNC_DEBUG("Failed to init TLS\n");
-        vnc_client_error(vs);
-        return -1;
-    }
-    if (vs->tls.session == NULL) {
-        if (gnutls_init(&vs->tls.session, GNUTLS_SERVER) < 0) {
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (gnutls_set_default_priority(vs->tls.session) < 0) {
-            gnutls_deinit(vs->tls.session);
-            vs->tls.session = NULL;
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (vnc_set_gnutls_priority(vs->tls.session, needX509Creds) < 0) {
-            gnutls_deinit(vs->tls.session);
-            vs->tls.session = NULL;
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (needX509Creds) {
-            gnutls_certificate_server_credentials x509_cred =
-                vnc_tls_initialize_x509_cred(vs->vd);
-            if (!x509_cred) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (gnutls_credentials_set(vs->tls.session,
-                                       GNUTLS_CRD_CERTIFICATE, x509_cred) < 0) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                gnutls_certificate_free_credentials(x509_cred);
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (vs->vd->tls.x509verify) {
-                VNC_DEBUG("Requesting a client certificate\n");
-                gnutls_certificate_server_set_request(vs->tls.session,
-                                                      GNUTLS_CERT_REQUEST);
-            }
-
-        } else {
-            gnutls_anon_server_credentials_t anon_cred =
-                vnc_tls_initialize_anon_cred();
-            if (!anon_cred) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (gnutls_credentials_set(vs->tls.session,
-                                       GNUTLS_CRD_ANON, anon_cred) < 0) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                gnutls_anon_free_server_credentials(anon_cred);
-                vnc_client_error(vs);
-                return -1;
-            }
-        }
-
-        gnutls_transport_set_ptr(vs->tls.session, (gnutls_transport_ptr_t)vs);
-        gnutls_transport_set_push_function(vs->tls.session, vnc_tls_push);
-        gnutls_transport_set_pull_function(vs->tls.session, vnc_tls_pull);
-    }
-    return 0;
-}
-
-
-void vnc_tls_client_cleanup(VncState *vs)
-{
-    if (vs->tls.session) {
-        gnutls_deinit(vs->tls.session);
-        vs->tls.session = NULL;
-    }
-    g_free(vs->tls.dname);
-}
-
-
-
-static int vnc_set_x509_credential(VncDisplay *vd,
-                                   const char *certdir,
-                                   const char *filename,
-                                   char **cred,
-                                   int ignoreMissing)
-{
-    struct stat sb;
-
-    g_free(*cred);
-    *cred = g_malloc(strlen(certdir) + strlen(filename) + 2);
-
-    strcpy(*cred, certdir);
-    strcat(*cred, "/");
-    strcat(*cred, filename);
-
-    VNC_DEBUG("Check %s\n", *cred);
-    if (stat(*cred, &sb) < 0) {
-        g_free(*cred);
-        *cred = NULL;
-        if (ignoreMissing && errno == ENOENT)
-            return 0;
-        return -1;
-    }
-
-    return 0;
-}
-
-
-int vnc_tls_set_x509_creds_dir(VncDisplay *vd,
-                               const char *certdir)
-{
-    if (vnc_set_x509_credential(vd, certdir, X509_CA_CERT_FILE, &vd->tls.x509cacert, 0) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_CA_CRL_FILE, &vd->tls.x509cacrl, 1) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_SERVER_CERT_FILE, &vd->tls.x509cert, 0) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_SERVER_KEY_FILE, &vd->tls.x509key, 0) < 0)
-        goto cleanup;
-
-    return 0;
-
- cleanup:
-    g_free(vd->tls.x509cacert);
-    g_free(vd->tls.x509cacrl);
-    g_free(vd->tls.x509cert);
-    g_free(vd->tls.x509key);
-    vd->tls.x509cacert = vd->tls.x509cacrl = vd->tls.x509cert = vd->tls.x509key = NULL;
-    return -1;
-}
-
diff --git a/ui/vnc-ws.c b/ui/vnc-ws.c
index b4cb6bde7..175ea50b4 100644
--- a/ui/vnc-ws.c
+++ b/ui/vnc-ws.c
@@ -22,60 +22,70 @@
 #include "qemu/main-loop.h"
 #include "crypto/hash.h"
 
-#ifdef CONFIG_VNC_TLS
-#include "qemu/sockets.h"
-
 static int vncws_start_tls_handshake(VncState *vs)
 {
-    int ret = gnutls_handshake(vs->tls.session);
-
-    if (ret < 0) {
-        if (!gnutls_error_is_fatal(ret)) {
-            VNC_DEBUG("Handshake interrupted (blocking)\n");
-            if (!gnutls_record_get_direction(vs->tls.session)) {
-                qemu_set_fd_handler(vs->csock, vncws_tls_handshake_io,
-                                    NULL, vs);
-            } else {
-                qemu_set_fd_handler(vs->csock, NULL, vncws_tls_handshake_io,
-                                    vs);
-            }
-            return 0;
-        }
-        VNC_DEBUG("Handshake failed %s\n", gnutls_strerror(ret));
-        vnc_client_error(vs);
-        return -1;
+    Error *err = NULL;
+
+    if (qcrypto_tls_session_handshake(vs->tls, &err) < 0) {
+        goto error;
     }
 
-    if (vs->vd->tls.x509verify) {
-        if (vnc_tls_validate_certificate(vs) < 0) {
-            VNC_DEBUG("Client verification failed\n");
-            vnc_client_error(vs);
-            return -1;
-        } else {
-            VNC_DEBUG("Client verification passed\n");
+    switch (qcrypto_tls_session_get_handshake_status(vs->tls)) {
+    case QCRYPTO_TLS_HANDSHAKE_COMPLETE:
+        VNC_DEBUG("Handshake done, checking credentials\n");
+        if (qcrypto_tls_session_check_credentials(vs->tls, &err) < 0) {
+            goto error;
         }
+        VNC_DEBUG("Client verification passed, starting TLS I/O\n");
+        qemu_set_fd_handler(vs->csock, vncws_handshake_read, NULL, vs);
+        break;
+
+    case QCRYPTO_TLS_HANDSHAKE_RECVING:
+        VNC_DEBUG("Handshake interrupted (blocking read)\n");
+        qemu_set_fd_handler(vs->csock, vncws_tls_handshake_io, NULL, vs);
+        break;
+
+    case QCRYPTO_TLS_HANDSHAKE_SENDING:
+        VNC_DEBUG("Handshake interrupted (blocking write)\n");
+        qemu_set_fd_handler(vs->csock, NULL, vncws_tls_handshake_io, vs);
+        break;
     }
 
-    VNC_DEBUG("Handshake done, switching to TLS data mode\n");
-    qemu_set_fd_handler(vs->csock, vncws_handshake_read, NULL, vs);
-
     return 0;
+
+ error:
+    VNC_DEBUG("Handshake failed %s\n", error_get_pretty(err));
+    error_free(err);
+    vnc_client_error(vs);
+    return -1;
 }
 
 void vncws_tls_handshake_io(void *opaque)
 {
     VncState *vs = (VncState *)opaque;
+    Error *err = NULL;
 
-    if (!vs->tls.session) {
-        VNC_DEBUG("TLS Websocket setup\n");
-        if (vnc_tls_client_setup(vs, vs->vd->tls.x509cert != NULL) < 0) {
-            return;
-        }
+    vs->tls = qcrypto_tls_session_new(vs->vd->tlscreds,
+                                      NULL,
+                                      vs->vd->tlsaclname,
+                                      QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+                                      &err);
+    if (!vs->tls) {
+        VNC_DEBUG("Failed to setup TLS %s\n",
+                  error_get_pretty(err));
+        error_free(err);
+        vnc_client_error(vs);
+        return;
     }
-    VNC_DEBUG("Handshake IO continue\n");
+
+    qcrypto_tls_session_set_callbacks(vs->tls,
+                                      vnc_tls_push,
+                                      vnc_tls_pull,
+                                      vs);
+
+    VNC_DEBUG("Start TLS WS handshake process\n");
     vncws_start_tls_handshake(vs);
 }
-#endif /* CONFIG_VNC_TLS */
 
 void vncws_handshake_read(void *opaque)
 {
diff --git a/ui/vnc-ws.h b/ui/vnc-ws.h
index 94942258e..4ab0a8c89 100644
--- a/ui/vnc-ws.h
+++ b/ui/vnc-ws.h
@@ -72,9 +72,7 @@ enum {
     WS_OPCODE_PONG = 0xA
 };
 
-#ifdef CONFIG_VNC_TLS
 void vncws_tls_handshake_io(void *opaque);
-#endif /* CONFIG_VNC_TLS */
 void vncws_handshake_read(void *opaque);
 long vnc_client_write_ws(VncState *vs);
 long vnc_client_read_ws(VncState *vs);
diff --git a/ui/vnc.c b/ui/vnc.c
index caf82f56f..cbe4d3395 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -41,6 +41,9 @@
 #include "ui/input.h"
 #include "qapi-event.h"
 #include "crypto/hash.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredsx509.h"
+#include "qom/object_interfaces.h"
 
 #define VNC_REFRESH_INTERVAL_BASE GUI_REFRESH_INTERVAL_DEFAULT
 #define VNC_REFRESH_INTERVAL_INC  50
@@ -153,10 +156,11 @@ char *vnc_socket_remote_addr(const char *format, int fd) {
     return addr_to_string(format, &sa, salen);
 }
 
-static VncBasicInfo *vnc_basic_info_get(struct sockaddr_storage *sa,
-                                        socklen_t salen)
+static void vnc_init_basic_info(struct sockaddr_storage *sa,
+                                socklen_t salen,
+                                VncBasicInfo *info,
+                                Error **errp)
 {
-    VncBasicInfo *info;
     char host[NI_MAXHOST];
     char serv[NI_MAXSERV];
     int err;
@@ -165,42 +169,44 @@ static VncBasicInfo *vnc_basic_info_get(struct sockaddr_storage *sa,
                            host, sizeof(host),
                            serv, sizeof(serv),
                            NI_NUMERICHOST | NI_NUMERICSERV)) != 0) {
-        VNC_DEBUG("Cannot resolve address %d: %s\n",
-                  err, gai_strerror(err));
-        return NULL;
+        error_setg(errp, "Cannot resolve address: %s",
+                   gai_strerror(err));
+        return;
     }
 
-    info = g_malloc0(sizeof(VncBasicInfo));
     info->host = g_strdup(host);
     info->service = g_strdup(serv);
     info->family = inet_netfamily(sa->ss_family);
-    return info;
 }
 
-static VncBasicInfo *vnc_basic_info_get_from_server_addr(int fd)
+static void vnc_init_basic_info_from_server_addr(int fd, VncBasicInfo *info,
+                                                 Error **errp)
 {
     struct sockaddr_storage sa;
     socklen_t salen;
 
     salen = sizeof(sa);
     if (getsockname(fd, (struct sockaddr*)&sa, &salen) < 0) {
-        return NULL;
+        error_setg_errno(errp, errno, "getsockname failed");
+        return;
     }
 
-    return vnc_basic_info_get(&sa, salen);
+    vnc_init_basic_info(&sa, salen, info, errp);
 }
 
-static VncBasicInfo *vnc_basic_info_get_from_remote_addr(int fd)
+static void vnc_init_basic_info_from_remote_addr(int fd, VncBasicInfo *info,
+                                                 Error **errp)
 {
     struct sockaddr_storage sa;
     socklen_t salen;
 
     salen = sizeof(sa);
     if (getpeername(fd, (struct sockaddr*)&sa, &salen) < 0) {
-        return NULL;
+        error_setg_errno(errp, errno, "getpeername failed");
+        return;
     }
 
-    return vnc_basic_info_get(&sa, salen);
+    vnc_init_basic_info(&sa, salen, info, errp);
 }
 
 static const char *vnc_auth_name(VncDisplay *vd) {
@@ -222,7 +228,6 @@ static const char *vnc_auth_name(VncDisplay *vd) {
     case VNC_AUTH_TLS:
         return "tls";
     case VNC_AUTH_VENCRYPT:
-#ifdef CONFIG_VNC_TLS
         switch (vd->subauth) {
         case VNC_AUTH_VENCRYPT_PLAIN:
             return "vencrypt+plain";
@@ -245,9 +250,6 @@ static const char *vnc_auth_name(VncDisplay *vd) {
         default:
             return "vencrypt";
         }
-#else
-        return "vencrypt";
-#endif
     case VNC_AUTH_SASL:
         return "sasl";
     }
@@ -257,15 +259,18 @@ static const char *vnc_auth_name(VncDisplay *vd) {
 static VncServerInfo *vnc_server_info_get(VncDisplay *vd)
 {
     VncServerInfo *info;
-    VncBasicInfo *bi = vnc_basic_info_get_from_server_addr(vd->lsock);
-    if (!bi) {
-        return NULL;
-    }
+    Error *err = NULL;
 
     info = g_malloc(sizeof(*info));
-    info->base = bi;
+    vnc_init_basic_info_from_server_addr(vd->lsock,
+                                         qapi_VncServerInfo_base(info), &err);
     info->has_auth = true;
     info->auth = g_strdup(vnc_auth_name(vd));
+    if (err) {
+        qapi_free_VncServerInfo(info);
+        info = NULL;
+        error_free(err);
+    }
     return info;
 }
 
@@ -275,13 +280,12 @@ static void vnc_client_cache_auth(VncState *client)
         return;
     }
 
-#ifdef CONFIG_VNC_TLS
-    if (client->tls.session &&
-        client->tls.dname) {
-        client->info->has_x509_dname = true;
-        client->info->x509_dname = g_strdup(client->tls.dname);
+    if (client->tls) {
+        client->info->x509_dname =
+            qcrypto_tls_session_get_peer_name(client->tls);
+        client->info->has_x509_dname =
+            client->info->x509_dname != NULL;
     }
-#endif
 #ifdef CONFIG_VNC_SASL
     if (client->sasl.conn &&
         client->sasl.username) {
@@ -293,11 +297,16 @@ static void vnc_client_cache_auth(VncState *client)
 
 static void vnc_client_cache_addr(VncState *client)
 {
-    VncBasicInfo *bi = vnc_basic_info_get_from_remote_addr(client->csock);
+    Error *err = NULL;
 
-    if (bi) {
-        client->info = g_malloc0(sizeof(*client->info));
-        client->info->base = bi;
+    client->info = g_malloc0(sizeof(*client->info));
+    vnc_init_basic_info_from_remote_addr(client->csock,
+                                         qapi_VncClientInfo_base(client->info),
+                                         &err);
+    if (err) {
+        qapi_free_VncClientInfo(client->info);
+        client->info = NULL;
+        error_free(err);
     }
 }
 
@@ -308,7 +317,6 @@ static void vnc_qmp_event(VncState *vs, QAPIEvent event)
     if (!vs->info) {
         return;
     }
-    g_assert(vs->info->base);
 
     si = vnc_server_info_get(vs->vd);
     if (!si) {
@@ -317,7 +325,8 @@ static void vnc_qmp_event(VncState *vs, QAPIEvent event)
 
     switch (event) {
     case QAPI_EVENT_VNC_CONNECTED:
-        qapi_event_send_vnc_connected(si, vs->info->base, &error_abort);
+        qapi_event_send_vnc_connected(si, qapi_VncClientInfo_base(vs->info),
+                                      &error_abort);
         break;
     case QAPI_EVENT_VNC_INITIALIZED:
         qapi_event_send_vnc_initialized(si, vs->info, &error_abort);
@@ -352,18 +361,15 @@ static VncClientInfo *qmp_query_vnc_client(const VncState *client)
     }
 
     info = g_malloc0(sizeof(*info));
-    info->base = g_malloc0(sizeof(*info->base));
-    info->base->host = g_strdup(host);
-    info->base->service = g_strdup(serv);
-    info->base->family = inet_netfamily(sa.ss_family);
-    info->base->websocket = client->websocket;
+    info->host = g_strdup(host);
+    info->service = g_strdup(serv);
+    info->family = inet_netfamily(sa.ss_family);
+    info->websocket = client->websocket;
 
-#ifdef CONFIG_VNC_TLS
-    if (client->tls.session && client->tls.dname) {
-        info->has_x509_dname = true;
-        info->x509_dname = g_strdup(client->tls.dname);
+    if (client->tls) {
+        info->x509_dname = qcrypto_tls_session_get_peer_name(client->tls);
+        info->has_x509_dname = info->x509_dname != NULL;
     }
-#endif
 #ifdef CONFIG_VNC_SASL
     if (client->sasl.conn && client->sasl.username) {
         info->has_sasl_username = true;
@@ -513,7 +519,6 @@ static void qmp_query_auth(VncDisplay *vd, VncInfo2 *info)
         break;
     case VNC_AUTH_VENCRYPT:
         info->auth = VNC_PRIMARY_AUTH_VENCRYPT;
-#ifdef CONFIG_VNC_TLS
         info->has_vencrypt = true;
         switch (vd->subauth) {
         case VNC_AUTH_VENCRYPT_PLAIN:
@@ -547,7 +552,6 @@ static void qmp_query_auth(VncDisplay *vd, VncInfo2 *info)
             info->has_vencrypt = false;
             break;
         }
-#endif
         break;
     case VNC_AUTH_SASL:
         info->auth = VNC_PRIMARY_AUTH_SASL;
@@ -611,10 +615,25 @@ static void framebuffer_update_request(VncState *vs, int incremental,
 static void vnc_refresh(DisplayChangeListener *dcl);
 static int vnc_refresh_server_surface(VncDisplay *vd);
 
+static int vnc_width(VncDisplay *vd)
+{
+    return MIN(VNC_MAX_WIDTH, ROUND_UP(surface_width(vd->ds),
+                                       VNC_DIRTY_PIXELS_PER_BIT));
+}
+
+static int vnc_height(VncDisplay *vd)
+{
+    return MIN(VNC_MAX_HEIGHT, surface_height(vd->ds));
+}
+
 static void vnc_set_area_dirty(DECLARE_BITMAP(dirty[VNC_MAX_HEIGHT],
                                VNC_MAX_WIDTH / VNC_DIRTY_PIXELS_PER_BIT),
-                               int width, int height,
-                               int x, int y, int w, int h) {
+                               VncDisplay *vd,
+                               int x, int y, int w, int h)
+{
+    int width = vnc_width(vd);
+    int height = vnc_height(vd);
+
     /* this is needed this to ensure we updated all affected
      * blocks if x % VNC_DIRTY_PIXELS_PER_BIT != 0 */
     w += (x % VNC_DIRTY_PIXELS_PER_BIT);
@@ -636,10 +655,8 @@ static void vnc_dpy_update(DisplayChangeListener *dcl,
 {
     VncDisplay *vd = container_of(dcl, VncDisplay, dcl);
     struct VncSurface *s = &vd->guest;
-    int width = pixman_image_get_width(vd->server);
-    int height = pixman_image_get_height(vd->server);
 
-    vnc_set_area_dirty(s->dirty, width, height, x, y, w, h);
+    vnc_set_area_dirty(s->dirty, vd, x, y, w, h);
 }
 
 void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
@@ -653,49 +670,6 @@ void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
     vnc_write_s32(vs, encoding);
 }
 
-void buffer_reserve(Buffer *buffer, size_t len)
-{
-    if ((buffer->capacity - buffer->offset) < len) {
-        buffer->capacity += (len + 1024);
-        buffer->buffer = g_realloc(buffer->buffer, buffer->capacity);
-    }
-}
-
-static int buffer_empty(Buffer *buffer)
-{
-    return buffer->offset == 0;
-}
-
-uint8_t *buffer_end(Buffer *buffer)
-{
-    return buffer->buffer + buffer->offset;
-}
-
-void buffer_reset(Buffer *buffer)
-{
-        buffer->offset = 0;
-}
-
-void buffer_free(Buffer *buffer)
-{
-    g_free(buffer->buffer);
-    buffer->offset = 0;
-    buffer->capacity = 0;
-    buffer->buffer = NULL;
-}
-
-void buffer_append(Buffer *buffer, const void *data, size_t len)
-{
-    memcpy(buffer->buffer + buffer->offset, data, len);
-    buffer->offset += len;
-}
-
-void buffer_advance(Buffer *buf, size_t len)
-{
-    memmove(buf->buffer, buf->buffer + len,
-            (buf->offset - len));
-    buf->offset -= len;
-}
 
 static void vnc_desktop_resize(VncState *vs)
 {
@@ -752,6 +726,21 @@ void *vnc_server_fb_ptr(VncDisplay *vd, int x, int y)
     return ptr;
 }
 
+static void vnc_update_server_surface(VncDisplay *vd)
+{
+    qemu_pixman_image_unref(vd->server);
+    vd->server = NULL;
+
+    if (QTAILQ_EMPTY(&vd->clients)) {
+        return;
+    }
+
+    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
+                                          vnc_width(vd),
+                                          vnc_height(vd),
+                                          NULL, 0);
+}
+
 static void vnc_dpy_switch(DisplayChangeListener *dcl,
                            DisplaySurface *surface)
 {
@@ -760,26 +749,19 @@ static void vnc_dpy_switch(DisplayChangeListener *dcl,
     int width, height;
 
     vnc_abort_display_jobs(vd);
+    vd->ds = surface;
 
     /* server surface */
-    qemu_pixman_image_unref(vd->server);
-    vd->ds = surface;
-    width = MIN(VNC_MAX_WIDTH, ROUND_UP(surface_width(vd->ds),
-                                        VNC_DIRTY_PIXELS_PER_BIT));
-    height = MIN(VNC_MAX_HEIGHT, surface_height(vd->ds));
-    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
-                                          width, height, NULL, 0);
+    vnc_update_server_surface(vd);
 
     /* guest surface */
-#if 0 /* FIXME */
-    if (ds_get_bytes_per_pixel(ds) != vd->guest.ds->pf.bytes_per_pixel)
-        console_color_init(ds);
-#endif
     qemu_pixman_image_unref(vd->guest.fb);
     vd->guest.fb = pixman_image_ref(surface->image);
     vd->guest.format = surface->format;
+    width = vnc_width(vd);
+    height = vnc_height(vd);
     memset(vd->guest.dirty, 0x00, sizeof(vd->guest.dirty));
-    vnc_set_area_dirty(vd->guest.dirty, width, height, 0, 0,
+    vnc_set_area_dirty(vd->guest.dirty, vd, 0, 0,
                        width, height);
 
     QTAILQ_FOREACH(vs, &vd->clients, next) {
@@ -789,7 +771,7 @@ static void vnc_dpy_switch(DisplayChangeListener *dcl,
             vnc_cursor_define(vs);
         }
         memset(vs->dirty, 0x00, sizeof(vs->dirty));
-        vnc_set_area_dirty(vs->dirty, width, height, 0, 0,
+        vnc_set_area_dirty(vs->dirty, vd, 0, 0,
                            width, height);
     }
 }
@@ -879,6 +861,8 @@ int vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 int vnc_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     int n = 0;
+    bool encode_raw = false;
+    size_t saved_offs = vs->output.offset;
 
     switch(vs->vnc_encoding) {
         case VNC_ENCODING_ZLIB:
@@ -901,10 +885,24 @@ int vnc_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
             n = vnc_zywrle_send_framebuffer_update(vs, x, y, w, h);
             break;
         default:
-            vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
-            n = vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+            encode_raw = true;
             break;
     }
+
+    /* If the client has the same pixel format as our internal buffer and
+     * a RAW encoding would need less space fall back to RAW encoding to
+     * save bandwidth and processing power in the client. */
+    if (!encode_raw && vs->write_pixels == vnc_write_pixels_copy &&
+        12 + h * w * VNC_SERVER_FB_BYTES <= (vs->output.offset - saved_offs)) {
+        vs->output.offset = saved_offs;
+        encode_raw = true;
+    }
+
+    if (encode_raw) {
+        vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
+        n = vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+    }
+
     return n;
 }
 
@@ -933,6 +931,11 @@ static void vnc_dpy_copy(DisplayChangeListener *dcl,
     int i, x, y, pitch, inc, w_lim, s;
     int cmp_bytes;
 
+    if (!vd->server) {
+        /* no client connected */
+        return;
+    }
+
     vnc_refresh_server_surface(vd);
     QTAILQ_FOREACH_SAFE(vs, &vd->clients, next, vn) {
         if (vnc_has_feature(vs, VNC_FEATURE_COPYRECT)) {
@@ -1237,9 +1240,7 @@ void vnc_disconnect_finish(VncState *vs)
     vnc_tight_clear(vs);
     vnc_zrle_clear(vs);
 
-#ifdef CONFIG_VNC_TLS
-    vnc_tls_client_cleanup(vs);
-#endif /* CONFIG_VNC_TLS */
+    qcrypto_tls_session_free(vs->tls);
 #ifdef CONFIG_VNC_SASL
     vnc_sasl_client_cleanup(vs);
 #endif /* CONFIG_VNC_SASL */
@@ -1249,6 +1250,10 @@ void vnc_disconnect_finish(VncState *vs)
     if (vs->initialized) {
         QTAILQ_REMOVE(&vs->vd->clients, vs, next);
         qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
+        if (QTAILQ_EMPTY(&vs->vd->clients)) {
+            /* last client gone */
+            vnc_update_server_surface(vs->vd);
+        }
     }
 
     if (vs->vd->lock_key_sync)
@@ -1268,7 +1273,7 @@ void vnc_disconnect_finish(VncState *vs)
     g_free(vs);
 }
 
-int vnc_client_io_error(VncState *vs, int ret, int last_errno)
+ssize_t vnc_client_io_error(VncState *vs, ssize_t ret, int last_errno)
 {
     if (ret == 0 || ret == -1) {
         if (ret == -1) {
@@ -1284,7 +1289,7 @@ int vnc_client_io_error(VncState *vs, int ret, int last_errno)
             }
         }
 
-        VNC_DEBUG("Closing down client sock: ret %d, errno %d\n",
+        VNC_DEBUG("Closing down client sock: ret %zd, errno %d\n",
                   ret, ret < 0 ? last_errno : 0);
         vnc_disconnect_start(vs);
 
@@ -1300,23 +1305,40 @@ void vnc_client_error(VncState *vs)
     vnc_disconnect_start(vs);
 }
 
-#ifdef CONFIG_VNC_TLS
-static long vnc_client_write_tls(gnutls_session_t *session,
-                                 const uint8_t *data,
-                                 size_t datalen)
+
+ssize_t vnc_tls_pull(char *buf, size_t len, void *opaque)
 {
-    long ret = gnutls_write(*session, data, datalen);
+    VncState *vs = opaque;
+    ssize_t ret;
+
+ retry:
+    ret = qemu_recv(vs->csock, buf, len, 0);
     if (ret < 0) {
-        if (ret == GNUTLS_E_AGAIN) {
-            errno = EAGAIN;
-        } else {
-            errno = EIO;
+        if (errno == EINTR) {
+            goto retry;
         }
-        ret = -1;
+        return -1;
     }
     return ret;
 }
-#endif /* CONFIG_VNC_TLS */
+
+
+ssize_t vnc_tls_push(const char *buf, size_t len, void *opaque)
+{
+    VncState *vs = opaque;
+    ssize_t ret;
+
+ retry:
+    ret = send(vs->csock, buf, len, 0);
+    if (ret < 0) {
+        if (errno == EINTR) {
+            goto retry;
+        }
+        return -1;
+    }
+    return ret;
+}
+
 
 /*
  * Called to write a chunk of data to the client socket. The data may
@@ -1333,20 +1355,23 @@ static long vnc_client_write_tls(gnutls_session_t *session,
  * the requested 'datalen' if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
+ssize_t vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
 {
-    long ret;
-#ifdef CONFIG_VNC_TLS
-    if (vs->tls.session) {
-        ret = vnc_client_write_tls(&vs->tls.session, data, datalen);
+    ssize_t ret;
+    int err = 0;
+    if (vs->tls) {
+        ret = qcrypto_tls_session_write(vs->tls, (const char *)data, datalen);
+        if (ret < 0) {
+            err = errno;
+        }
     } else {
-#endif /* CONFIG_VNC_TLS */
         ret = send(vs->csock, (const void *)data, datalen, 0);
-#ifdef CONFIG_VNC_TLS
+        if (ret < 0) {
+            err = socket_error();
+        }
     }
-#endif /* CONFIG_VNC_TLS */
     VNC_DEBUG("Wrote wire %p %zd -> %ld\n", data, datalen, ret);
-    return vnc_client_io_error(vs, ret, socket_error());
+    return vnc_client_io_error(vs, ret, err);
 }
 
 
@@ -1360,9 +1385,9 @@ long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
  * the buffered output data if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-static long vnc_client_write_plain(VncState *vs)
+static ssize_t vnc_client_write_plain(VncState *vs)
 {
-    long ret;
+    ssize_t ret;
 
 #ifdef CONFIG_VNC_SASL
     VNC_DEBUG("Write Plain: Pending output %p size %zd offset %zd. Wait SSF %d\n",
@@ -1435,22 +1460,6 @@ void vnc_read_when(VncState *vs, VncReadEvent *func, size_t expecting)
     vs->read_handler_expect = expecting;
 }
 
-#ifdef CONFIG_VNC_TLS
-static long vnc_client_read_tls(gnutls_session_t *session, uint8_t *data,
-                                size_t datalen)
-{
-    long ret = gnutls_read(*session, data, datalen);
-    if (ret < 0) {
-        if (ret == GNUTLS_E_AGAIN) {
-            errno = EAGAIN;
-        } else {
-            errno = EIO;
-        }
-        ret = -1;
-    }
-    return ret;
-}
-#endif /* CONFIG_VNC_TLS */
 
 /*
  * Called to read a chunk of data from the client socket. The data may
@@ -1467,20 +1476,23 @@ static long vnc_client_read_tls(gnutls_session_t *session, uint8_t *data,
  * the requested 'datalen' if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
+ssize_t vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
 {
-    long ret;
-#ifdef CONFIG_VNC_TLS
-    if (vs->tls.session) {
-        ret = vnc_client_read_tls(&vs->tls.session, data, datalen);
+    ssize_t ret;
+    int err = -1;
+    if (vs->tls) {
+        ret = qcrypto_tls_session_read(vs->tls, (char *)data, datalen);
+        if (ret < 0) {
+            err = errno;
+        }
     } else {
-#endif /* CONFIG_VNC_TLS */
         ret = qemu_recv(vs->csock, data, datalen, 0);
-#ifdef CONFIG_VNC_TLS
+        if (ret < 0) {
+            err = socket_error();
+        }
     }
-#endif /* CONFIG_VNC_TLS */
     VNC_DEBUG("Read wire %p %zd -> %ld\n", data, datalen, ret);
-    return vnc_client_io_error(vs, ret, socket_error());
+    return vnc_client_io_error(vs, ret, err);
 }
 
 
@@ -1492,9 +1504,9 @@ long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
  * Returns the number of bytes read. Returns -1 on error, and
  * disconnects the client socket.
  */
-static long vnc_client_read_plain(VncState *vs)
+static ssize_t vnc_client_read_plain(VncState *vs)
 {
-    int ret;
+    ssize_t ret;
     VNC_DEBUG("Read plain %p size %zd offset %zd\n",
               vs->input.buffer, vs->input.capacity, vs->input.offset);
     buffer_reserve(&vs->input, 4096);
@@ -1520,7 +1532,7 @@ static void vnc_jobs_bh(void *opaque)
 void vnc_client_read(void *opaque)
 {
     VncState *vs = opaque;
-    long ret;
+    ssize_t ret;
 
 #ifdef CONFIG_VNC_SASL
     if (vs->sasl.conn && vs->sasl.runSSF)
@@ -2024,9 +2036,6 @@ static void ext_key_event(VncState *vs, int down,
 static void framebuffer_update_request(VncState *vs, int incremental,
                                        int x, int y, int w, int h)
 {
-    int width = pixman_image_get_width(vs->vd->server);
-    int height = pixman_image_get_height(vs->vd->server);
-
     vs->need_update = 1;
 
     if (incremental) {
@@ -2034,7 +2043,7 @@ static void framebuffer_update_request(VncState *vs, int incremental,
     }
 
     vs->force_update = 1;
-    vnc_set_area_dirty(vs->dirty, width, height, x, y, w, h);
+    vnc_set_area_dirty(vs->dirty, vs->vd, x, y, w, h);
 }
 
 static void send_ext_key_event_ack(VncState *vs)
@@ -2189,15 +2198,15 @@ static void set_pixel_format(VncState *vs,
         return;
     }
 
-    vs->client_pf.rmax = red_max;
+    vs->client_pf.rmax = red_max ? red_max : 0xFF;
     vs->client_pf.rbits = hweight_long(red_max);
     vs->client_pf.rshift = red_shift;
     vs->client_pf.rmask = red_max << red_shift;
-    vs->client_pf.gmax = green_max;
+    vs->client_pf.gmax = green_max ? green_max : 0xFF;
     vs->client_pf.gbits = hweight_long(green_max);
     vs->client_pf.gshift = green_shift;
     vs->client_pf.gmask = green_max << green_shift;
-    vs->client_pf.bmax = blue_max;
+    vs->client_pf.bmax = blue_max ? blue_max : 0xFF;
     vs->client_pf.bbits = hweight_long(blue_max);
     vs->client_pf.bshift = blue_shift;
     vs->client_pf.bmask = blue_max << blue_shift;
@@ -2631,12 +2640,10 @@ static int protocol_client_auth(VncState *vs, uint8_t *data, size_t len)
            start_auth_vnc(vs);
            break;
 
-#ifdef CONFIG_VNC_TLS
        case VNC_AUTH_VENCRYPT:
            VNC_DEBUG("Accept VeNCrypt auth\n");
            start_auth_vencrypt(vs);
            break;
-#endif /* CONFIG_VNC_TLS */
 
 #ifdef CONFIG_VNC_SASL
        case VNC_AUTH_SASL:
@@ -3002,12 +3009,32 @@ static void vnc_refresh(DisplayChangeListener *dcl)
 static void vnc_connect(VncDisplay *vd, int csock,
                         bool skipauth, bool websocket)
 {
-    VncState *vs = g_malloc0(sizeof(VncState));
+    VncState *vs = g_new0(VncState, 1);
     int i;
 
     vs->csock = csock;
     vs->vd = vd;
 
+    buffer_init(&vs->input,          "vnc-input/%d", csock);
+    buffer_init(&vs->output,         "vnc-output/%d", csock);
+    buffer_init(&vs->ws_input,       "vnc-ws_input/%d", csock);
+    buffer_init(&vs->ws_output,      "vnc-ws_output/%d", csock);
+    buffer_init(&vs->jobs_buffer,    "vnc-jobs_buffer/%d", csock);
+
+    buffer_init(&vs->tight.tight,    "vnc-tight/%d", csock);
+    buffer_init(&vs->tight.zlib,     "vnc-tight-zlib/%d", csock);
+    buffer_init(&vs->tight.gradient, "vnc-tight-gradient/%d", csock);
+#ifdef CONFIG_VNC_JPEG
+    buffer_init(&vs->tight.jpeg,     "vnc-tight-jpeg/%d", csock);
+#endif
+#ifdef CONFIG_VNC_PNG
+    buffer_init(&vs->tight.png,      "vnc-tight-png/%d", csock);
+#endif
+    buffer_init(&vs->zlib.zlib,      "vnc-zlib/%d", csock);
+    buffer_init(&vs->zrle.zrle,      "vnc-zrle/%d", csock);
+    buffer_init(&vs->zrle.fb,        "vnc-zrle-fb/%d", csock);
+    buffer_init(&vs->zrle.zlib,      "vnc-zrle-zlib/%d", csock);
+
     if (skipauth) {
 	vs->auth = VNC_AUTH_NONE;
 	vs->subauth = VNC_AUTH_INVALID;
@@ -3025,7 +3052,7 @@ static void vnc_connect(VncDisplay *vd, int csock,
 
     vs->lossy_rect = g_malloc0(VNC_STAT_ROWS * sizeof (*vs->lossy_rect));
     for (i = 0; i < VNC_STAT_ROWS; ++i) {
-        vs->lossy_rect[i] = g_malloc0(VNC_STAT_COLS * sizeof (uint8_t));
+        vs->lossy_rect[i] = g_new0(uint8_t, VNC_STAT_COLS);
     }
 
     VNC_DEBUG("New client on socket %d\n", csock);
@@ -3033,12 +3060,9 @@ static void vnc_connect(VncDisplay *vd, int csock,
     qemu_set_nonblock(vs->csock);
     if (websocket) {
         vs->websocket = 1;
-#ifdef CONFIG_VNC_TLS
         if (vd->ws_tls) {
             qemu_set_fd_handler(vs->csock, vncws_tls_handshake_io, NULL, vs);
-        } else
-#endif /* CONFIG_VNC_TLS */
-        {
+        } else {
             qemu_set_fd_handler(vs->csock, vncws_handshake_read, NULL, vs);
         }
     } else
@@ -3068,6 +3092,7 @@ void vnc_init_state(VncState *vs)
 {
     vs->initialized = true;
     VncDisplay *vd = vs->vd;
+    bool first_client = QTAILQ_EMPTY(&vd->clients);
 
     vs->last_x = -1;
     vs->last_y = -1;
@@ -3081,6 +3106,9 @@ void vnc_init_state(VncState *vs)
     vs->bh = qemu_bh_new(vnc_jobs_bh, vs);
 
     QTAILQ_INSERT_TAIL(&vd->clients, vs, next);
+    if (first_client) {
+        vnc_update_server_surface(vd);
+    }
 
     graphic_hw_update(vd->dcl.con);
 
@@ -3194,9 +3222,11 @@ static void vnc_display_close(VncDisplay *vs)
     }
     vs->auth = VNC_AUTH_INVALID;
     vs->subauth = VNC_AUTH_INVALID;
-#ifdef CONFIG_VNC_TLS
-    vs->tls.x509verify = 0;
-#endif
+    if (vs->tlscreds) {
+        object_unparent(OBJECT(vs->tlscreds));
+    }
+    g_free(vs->tlsaclname);
+    vs->tlsaclname = NULL;
 }
 
 int vnc_display_password(const char *id, const char *password)
@@ -3250,6 +3280,10 @@ static QemuOptsList qemu_vnc_opts = {
             .name = "websocket",
             .type = QEMU_OPT_STRING,
         },{
+            .name = "tls-creds",
+            .type = QEMU_OPT_STRING,
+        },{
+            /* Deprecated in favour of tls-creds */
             .name = "x509",
             .type = QEMU_OPT_STRING,
         },{
@@ -3286,9 +3320,11 @@ static QemuOptsList qemu_vnc_opts = {
             .name = "sasl",
             .type = QEMU_OPT_BOOL,
         },{
+            /* Deprecated in favour of tls-creds */
             .name = "tls",
             .type = QEMU_OPT_BOOL,
         },{
+            /* Deprecated in favour of tls-creds */
             .name = "x509verify",
             .type = QEMU_OPT_STRING,
         },{
@@ -3306,13 +3342,12 @@ static QemuOptsList qemu_vnc_opts = {
 };
 
 
-static void
+static int
 vnc_display_setup_auth(VncDisplay *vs,
                        bool password,
                        bool sasl,
-                       bool tls,
-                       bool x509,
-                       bool websocket)
+                       bool websocket,
+                       Error **errp)
 {
     /*
      * We have a choice of 3 authentication options
@@ -3362,17 +3397,24 @@ vnc_display_setup_auth(VncDisplay *vs,
      * result has the same security characteristics.
      */
     if (password) {
-        if (tls) {
+        if (vs->tlscreds) {
             vs->auth = VNC_AUTH_VENCRYPT;
             if (websocket) {
                 vs->ws_tls = true;
             }
-            if (x509) {
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
                 VNC_DEBUG("Initializing VNC server with x509 password auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_X509VNC;
-            } else {
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
                 VNC_DEBUG("Initializing VNC server with TLS password auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_TLSVNC;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
             }
         } else {
             VNC_DEBUG("Initializing VNC server with password auth\n");
@@ -3385,17 +3427,24 @@ vnc_display_setup_auth(VncDisplay *vs,
             vs->ws_auth = VNC_AUTH_INVALID;
         }
     } else if (sasl) {
-        if (tls) {
+        if (vs->tlscreds) {
             vs->auth = VNC_AUTH_VENCRYPT;
             if (websocket) {
                 vs->ws_tls = true;
             }
-            if (x509) {
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
                 VNC_DEBUG("Initializing VNC server with x509 SASL auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_X509SASL;
-            } else {
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
                 VNC_DEBUG("Initializing VNC server with TLS SASL auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_TLSSASL;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
             }
         } else {
             VNC_DEBUG("Initializing VNC server with SASL auth\n");
@@ -3408,17 +3457,24 @@ vnc_display_setup_auth(VncDisplay *vs,
             vs->ws_auth = VNC_AUTH_INVALID;
         }
     } else {
-        if (tls) {
+        if (vs->tlscreds) {
             vs->auth = VNC_AUTH_VENCRYPT;
             if (websocket) {
                 vs->ws_tls = true;
             }
-            if (x509) {
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
                 VNC_DEBUG("Initializing VNC server with x509 no auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_X509NONE;
-            } else {
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
                 VNC_DEBUG("Initializing VNC server with TLS no auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_TLSNONE;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
             }
         } else {
             VNC_DEBUG("Initializing VNC server with no auth\n");
@@ -3431,34 +3487,72 @@ vnc_display_setup_auth(VncDisplay *vs,
             vs->ws_auth = VNC_AUTH_INVALID;
         }
     }
+    return 0;
 }
 
+
+/*
+ * Handle back compat with old CLI syntax by creating some
+ * suitable QCryptoTLSCreds objects
+ */
+static QCryptoTLSCreds *
+vnc_display_create_creds(bool x509,
+                         bool x509verify,
+                         const char *dir,
+                         const char *id,
+                         Error **errp)
+{
+    gchar *credsid = g_strdup_printf("tlsvnc%s", id);
+    Object *parent = object_get_objects_root();
+    Object *creds;
+    Error *err = NULL;
+
+    if (x509) {
+        creds = object_new_with_props(TYPE_QCRYPTO_TLS_CREDS_X509,
+                                      parent,
+                                      credsid,
+                                      &err,
+                                      "endpoint", "server",
+                                      "dir", dir,
+                                      "verify-peer", x509verify ? "yes" : "no",
+                                      NULL);
+    } else {
+        creds = object_new_with_props(TYPE_QCRYPTO_TLS_CREDS_ANON,
+                                      parent,
+                                      credsid,
+                                      &err,
+                                      "endpoint", "server",
+                                      NULL);
+    }
+
+    g_free(credsid);
+
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
 void vnc_display_open(const char *id, Error **errp)
 {
     VncDisplay *vs = vnc_display_find(id);
     QemuOpts *opts = qemu_opts_find(&qemu_vnc_opts, id);
-    QemuOpts *sopts, *wsopts;
+    SocketAddress *saddr = NULL, *wsaddr = NULL;
     const char *share, *device_id;
     QemuConsole *con;
     bool password = false;
     bool reverse = false;
     const char *vnc;
-    const char *has_to;
     char *h;
-    bool has_ipv4 = false;
-    bool has_ipv6 = false;
-    const char *websocket;
-    bool tls = false, x509 = false;
-#ifdef CONFIG_VNC_TLS
-    const char *path;
-#endif
+    const char *credid;
     bool sasl = false;
 #ifdef CONFIG_VNC_SASL
     int saslErr;
 #endif
-#if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
     int acl = 0;
-#endif
     int lock_key_sync = 1;
 
     if (!vs) {
@@ -3475,44 +3569,83 @@ void vnc_display_open(const char *id, Error **errp)
         return;
     }
 
-    sopts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    wsopts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-
     h = strrchr(vnc, ':');
     if (h) {
-        char *host;
         size_t hlen = h - vnc;
 
-        if (vnc[0] == '[' && vnc[hlen - 1] == ']') {
-            host = g_strndup(vnc + 1, hlen - 2);
+        const char *websocket = qemu_opt_get(opts, "websocket");
+        int to = qemu_opt_get_number(opts, "to", 0);
+        bool has_ipv4 = qemu_opt_get_bool(opts, "ipv4", false);
+        bool has_ipv6 = qemu_opt_get_bool(opts, "ipv6", false);
+
+        saddr = g_new0(SocketAddress, 1);
+        if (websocket) {
+            if (!qcrypto_hash_supports(QCRYPTO_HASH_ALG_SHA1)) {
+                error_setg(errp,
+                           "SHA1 hash support is required for websockets");
+                goto fail;
+            }
+
+            wsaddr = g_new0(SocketAddress, 1);
+            vs->ws_enabled = true;
+        }
+
+        if (strncmp(vnc, "unix:", 5) == 0) {
+            saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+            saddr->u.q_unix = g_new0(UnixSocketAddress, 1);
+            saddr->u.q_unix->path = g_strdup(vnc + 5);
+
+            if (vs->ws_enabled) {
+                error_setg(errp, "UNIX sockets not supported with websock");
+                goto fail;
+            }
         } else {
-            host = g_strndup(vnc, hlen);
+            unsigned long long baseport;
+            saddr->type = SOCKET_ADDRESS_KIND_INET;
+            saddr->u.inet = g_new0(InetSocketAddress, 1);
+            if (vnc[0] == '[' && vnc[hlen - 1] == ']') {
+                saddr->u.inet->host = g_strndup(vnc + 1, hlen - 2);
+            } else {
+                saddr->u.inet->host = g_strndup(vnc, hlen);
+            }
+            if (parse_uint_full(h + 1, &baseport, 10) < 0) {
+                error_setg(errp, "can't convert to a number: %s", h + 1);
+                goto fail;
+            }
+            if (baseport > 65535 ||
+                baseport + 5900 > 65535) {
+                error_setg(errp, "port %s out of range", h + 1);
+                goto fail;
+            }
+            saddr->u.inet->port = g_strdup_printf(
+                "%d", (int)baseport + 5900);
+
+            if (to) {
+                saddr->u.inet->has_to = true;
+                saddr->u.inet->to = to + 5900;
+            }
+            saddr->u.inet->ipv4 = saddr->u.inet->has_ipv4 = has_ipv4;
+            saddr->u.inet->ipv6 = saddr->u.inet->has_ipv6 = has_ipv6;
+
+            if (vs->ws_enabled) {
+                wsaddr->type = SOCKET_ADDRESS_KIND_INET;
+                wsaddr->u.inet = g_new0(InetSocketAddress, 1);
+                wsaddr->u.inet->host = g_strdup(saddr->u.inet->host);
+                wsaddr->u.inet->port = g_strdup(websocket);
+
+                if (to) {
+                    wsaddr->u.inet->has_to = true;
+                    wsaddr->u.inet->to = to;
+                }
+                wsaddr->u.inet->ipv4 = wsaddr->u.inet->has_ipv4 = has_ipv4;
+                wsaddr->u.inet->ipv6 = wsaddr->u.inet->has_ipv6 = has_ipv6;
+            }
         }
-        qemu_opt_set(sopts, "host", host, &error_abort);
-        qemu_opt_set(wsopts, "host", host, &error_abort);
-        qemu_opt_set(sopts, "port", h+1, &error_abort);
-        g_free(host);
     } else {
         error_setg(errp, "no vnc port specified");
         goto fail;
     }
 
-    has_to = qemu_opt_get(opts, "to");
-    has_ipv4 = qemu_opt_get_bool(opts, "ipv4", false);
-    has_ipv6 = qemu_opt_get_bool(opts, "ipv6", false);
-    if (has_to) {
-        qemu_opt_set(sopts, "to", has_to, &error_abort);
-        qemu_opt_set(wsopts, "to", has_to, &error_abort);
-    }
-    if (has_ipv4) {
-        qemu_opt_set(sopts, "ipv4", "on", &error_abort);
-        qemu_opt_set(wsopts, "ipv4", "on", &error_abort);
-    }
-    if (has_ipv6) {
-        qemu_opt_set(sopts, "ipv6", "on", &error_abort);
-        qemu_opt_set(wsopts, "ipv6", "on", &error_abort);
-    }
-
     password = qemu_opt_get_bool(opts, "password", false);
     if (password) {
         if (fips_get_state()) {
@@ -3539,32 +3672,67 @@ void vnc_display_open(const char *id, Error **errp)
         goto fail;
     }
 #endif /* CONFIG_VNC_SASL */
-    tls  = qemu_opt_get_bool(opts, "tls", false);
-#ifdef CONFIG_VNC_TLS
-    path = qemu_opt_get(opts, "x509");
-    if (!path) {
-        path = qemu_opt_get(opts, "x509verify");
-        if (path) {
-            vs->tls.x509verify = true;
-        }
-    }
-    if (path) {
-        x509 = true;
-        if (vnc_tls_set_x509_creds_dir(vs, path) < 0) {
-            error_setg(errp, "Failed to find x509 certificates/keys in %s",
-                       path);
+    credid = qemu_opt_get(opts, "tls-creds");
+    if (credid) {
+        Object *creds;
+        if (qemu_opt_get(opts, "tls") ||
+            qemu_opt_get(opts, "x509") ||
+            qemu_opt_get(opts, "x509verify")) {
+            error_setg(errp,
+                       "'credid' parameter is mutually exclusive with "
+                       "'tls', 'x509' and 'x509verify' parameters");
             goto fail;
         }
+
+        creds = object_resolve_path_component(
+            object_get_objects_root(), credid);
+        if (!creds) {
+            error_setg(errp, "No TLS credentials with id '%s'",
+                       credid);
+            goto fail;
+        }
+        vs->tlscreds = (QCryptoTLSCreds *)
+            object_dynamic_cast(creds,
+                                TYPE_QCRYPTO_TLS_CREDS);
+        if (!vs->tlscreds) {
+            error_setg(errp, "Object with id '%s' is not TLS credentials",
+                       credid);
+            goto fail;
+        }
+        object_ref(OBJECT(vs->tlscreds));
+
+        if (vs->tlscreds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            error_setg(errp,
+                       "Expecting TLS credentials with a server endpoint");
+            goto fail;
+        }
+    } else {
+        const char *path;
+        bool tls = false, x509 = false, x509verify = false;
+        tls  = qemu_opt_get_bool(opts, "tls", false);
+        if (tls) {
+            path = qemu_opt_get(opts, "x509");
+
+            if (path) {
+                x509 = true;
+            } else {
+                path = qemu_opt_get(opts, "x509verify");
+                if (path) {
+                    x509 = true;
+                    x509verify = true;
+                }
+            }
+            vs->tlscreds = vnc_display_create_creds(x509,
+                                                    x509verify,
+                                                    path,
+                                                    vs->id,
+                                                    errp);
+            if (!vs->tlscreds) {
+                goto fail;
+            }
+        }
     }
-#else /* ! CONFIG_VNC_TLS */
-    if (tls) {
-        error_setg(errp, "VNC TLS auth requires gnutls support");
-        goto fail;
-    }
-#endif /* ! CONFIG_VNC_TLS */
-#if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
     acl = qemu_opt_get_bool(opts, "acl", false);
-#endif
 
     share = qemu_opt_get(opts, "share");
     if (share) {
@@ -3583,16 +3751,6 @@ void vnc_display_open(const char *id, Error **errp)
     }
     vs->connections_limit = qemu_opt_get_number(opts, "connections", 32);
 
-    websocket = qemu_opt_get(opts, "websocket");
-    if (websocket) {
-        vs->ws_enabled = true;
-        qemu_opt_set(wsopts, "port", websocket, &error_abort);
-        if (!qcrypto_hash_supports(QCRYPTO_HASH_ALG_SHA1)) {
-            error_setg(errp, "SHA1 hash support is required for websockets");
-            goto fail;
-        }
-    }
-
 #ifdef CONFIG_VNC_JPEG
     vs->lossy = qemu_opt_get_bool(opts, "lossy", false);
 #endif
@@ -3604,19 +3762,14 @@ void vnc_display_open(const char *id, Error **errp)
         vs->non_adaptive = true;
     }
 
-#ifdef CONFIG_VNC_TLS
-    if (acl && x509 && vs->tls.x509verify) {
-        char *aclname;
-
+    if (acl) {
         if (strcmp(vs->id, "default") == 0) {
-            aclname = g_strdup("vnc.x509dname");
+            vs->tlsaclname = g_strdup("vnc.x509dname");
         } else {
-            aclname = g_strdup_printf("vnc.%s.x509dname", vs->id);
+            vs->tlsaclname = g_strdup_printf("vnc.%s.x509dname", vs->id);
         }
-        vs->tls.acl = qemu_acl_init(aclname);
-        g_free(aclname);
-    }
-#endif
+        qemu_acl_init(vs->tlsaclname);
+     }
 #ifdef CONFIG_VNC_SASL
     if (acl && sasl) {
         char *aclname;
@@ -3631,7 +3784,9 @@ void vnc_display_open(const char *id, Error **errp)
     }
 #endif
 
-    vnc_display_setup_auth(vs, password, sasl, tls, x509, websocket);
+    if (vnc_display_setup_auth(vs, password, sasl, vs->ws_enabled, errp) < 0) {
+        goto fail;
+    }
 
 #ifdef CONFIG_VNC_SASL
     if ((saslErr = sasl_server_init(NULL, "qemu")) != SASL_OK) {
@@ -3674,37 +3829,31 @@ void vnc_display_open(const char *id, Error **errp)
         int csock;
         vs->lsock = -1;
         vs->lwebsock = -1;
-        if (strncmp(vnc, "unix:", 5) == 0) {
-            csock = unix_connect(vnc+5, errp);
-        } else {
-            csock = inet_connect(vnc, errp);
+        if (vs->ws_enabled) {
+            error_setg(errp, "Cannot use websockets in reverse mode");
+            goto fail;
         }
+        csock = socket_connect(saddr, errp, NULL, NULL);
         if (csock < 0) {
             goto fail;
         }
+        vs->is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
         vnc_connect(vs, csock, false, false);
     } else {
         /* listen for connects */
-        if (strncmp(vnc, "unix:", 5) == 0) {
-            vs->lsock = unix_listen(vnc+5, NULL, 0, errp);
-            if (vs->lsock < 0) {
-                goto fail;
-            }
-            vs->is_unix = true;
-        } else {
-            vs->lsock = inet_listen_opts(sopts, 5900, errp);
-            if (vs->lsock < 0) {
-                goto fail;
-            }
-            if (vs->ws_enabled) {
-                vs->lwebsock = inet_listen_opts(wsopts, 0, errp);
-                if (vs->lwebsock < 0) {
-                    if (vs->lsock != -1) {
-                        close(vs->lsock);
-                        vs->lsock = -1;
-                    }
-                    goto fail;
+        vs->lsock = socket_listen(saddr, errp);
+        if (vs->lsock < 0) {
+            goto fail;
+        }
+        vs->is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
+        if (vs->ws_enabled) {
+            vs->lwebsock = socket_listen(wsaddr, errp);
+            if (vs->lwebsock < 0) {
+                if (vs->lsock != -1) {
+                    close(vs->lsock);
+                    vs->lsock = -1;
                 }
+                goto fail;
             }
         }
         vs->enabled = true;
@@ -3714,13 +3863,14 @@ void vnc_display_open(const char *id, Error **errp)
                                 NULL, vs);
         }
     }
-    qemu_opts_del(sopts);
-    qemu_opts_del(wsopts);
+
+    qapi_free_SocketAddress(saddr);
+    qapi_free_SocketAddress(wsaddr);
     return;
 
 fail:
-    qemu_opts_del(sopts);
-    qemu_opts_del(wsopts);
+    qapi_free_SocketAddress(saddr);
+    qapi_free_SocketAddress(wsaddr);
     vs->enabled = false;
     vs->ws_enabled = false;
 }
diff --git a/ui/vnc.h b/ui/vnc.h
index 814d720df..2863f583a 100644
--- a/ui/vnc.h
+++ b/ui/vnc.h
@@ -33,6 +33,8 @@
 #include "ui/console.h"
 #include "audio/audio.h"
 #include "qemu/bitmap.h"
+#include "crypto/tlssession.h"
+#include "qemu/buffer.h"
 #include <zlib.h>
 #include <stdbool.h>
 
@@ -55,13 +57,6 @@
  *
  *****************************************************************************/
 
-typedef struct Buffer
-{
-    size_t capacity;
-    size_t offset;
-    uint8_t *buffer;
-} Buffer;
-
 typedef struct VncState VncState;
 typedef struct VncJob VncJob;
 typedef struct VncRect VncRect;
@@ -101,10 +96,7 @@ typedef void VncSendHextileTile(VncState *vs,
 
 typedef struct VncDisplay VncDisplay;
 
-#ifdef CONFIG_VNC_TLS
-#include "vnc-tls.h"
 #include "vnc-auth-vencrypt.h"
-#endif
 #ifdef CONFIG_VNC_SASL
 #include "vnc-auth-sasl.h"
 #endif
@@ -181,9 +173,8 @@ struct VncDisplay
     bool ws_tls; /* Used by websockets */
     bool lossy;
     bool non_adaptive;
-#ifdef CONFIG_VNC_TLS
-    VncDisplayTLS tls;
-#endif
+    QCryptoTLSCreds *tlscreds;
+    char *tlsaclname;
 #ifdef CONFIG_VNC_SASL
     VncDisplaySASL sasl;
 #endif
@@ -284,9 +275,7 @@ struct VncState
     int auth;
     int subauth; /* Used by VeNCrypt */
     char challenge[VNC_AUTH_CHALLENGE_SIZE];
-#ifdef CONFIG_VNC_TLS
-    VncStateTLS tls;
-#endif
+    QCryptoTLSSession *tls;
 #ifdef CONFIG_VNC_SASL
     VncStateSASL sasl;
 #endif
@@ -513,8 +502,10 @@ enum {
 void vnc_client_read(void *opaque);
 void vnc_client_write(void *opaque);
 
-long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen);
-long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen);
+ssize_t vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen);
+ssize_t vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen);
+ssize_t vnc_tls_pull(char *buf, size_t len, void *opaque);
+ssize_t vnc_tls_push(const char *buf, size_t len, void *opaque);
 
 /* Protocol I/O functions */
 void vnc_write(VncState *vs, const void *data, size_t len);
@@ -533,19 +524,11 @@ uint32_t read_u32(uint8_t *data, size_t offset);
 
 /* Protocol stage functions */
 void vnc_client_error(VncState *vs);
-int vnc_client_io_error(VncState *vs, int ret, int last_errno);
+ssize_t vnc_client_io_error(VncState *vs, ssize_t ret, int last_errno);
 
 void start_client_init(VncState *vs);
 void start_auth_vnc(VncState *vs);
 
-/* Buffer management */
-void buffer_reserve(Buffer *buffer, size_t len);
-void buffer_reset(Buffer *buffer);
-void buffer_free(Buffer *buffer);
-void buffer_append(Buffer *buffer, const void *data, size_t len);
-void buffer_advance(Buffer *buf, size_t len);
-uint8_t *buffer_end(Buffer *buffer);
-
 
 /* Misc helpers */
 
diff --git a/user-exec.c b/user-exec.c
index ed9a07f15..8ad89a466 100644
--- a/user-exec.c
+++ b/user-exec.c
@@ -92,8 +92,8 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
     int ret;
 
 #if defined(DEBUG_SIGNAL)
-    qemu_printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
-                pc, address, is_write, *(unsigned long *)old_set);
+    printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
+           pc, address, is_write, *(unsigned long *)old_set);
 #endif
     /* XXX: locking issue */
     if (is_write && h2g_valid(address)
diff --git a/util/Makefile.objs b/util/Makefile.objs
index 114d6578c..89dd80ef8 100644
--- a/util/Makefile.objs
+++ b/util/Makefile.objs
@@ -1,13 +1,20 @@
 util-obj-y = osdep.o cutils.o unicode.o qemu-timer-common.o
-util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o event_notifier-win32.o
-util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o event_notifier-posix.o qemu-openpty.o
+util-obj-$(CONFIG_POSIX) += compatfd.o
+util-obj-$(CONFIG_POSIX) += event_notifier-posix.o
+util-obj-$(CONFIG_POSIX) += mmap-alloc.o
+util-obj-$(CONFIG_POSIX) += oslib-posix.o
+util-obj-$(CONFIG_POSIX) += qemu-openpty.o
+util-obj-$(CONFIG_POSIX) += qemu-thread-posix.o
+util-obj-$(CONFIG_WIN32) += event_notifier-win32.o
+util-obj-$(CONFIG_POSIX) += memfd.o
+util-obj-$(CONFIG_WIN32) += oslib-win32.o
+util-obj-$(CONFIG_WIN32) += qemu-thread-win32.o
 util-obj-y += envlist.o path.o module.o
 util-obj-$(call lnot,$(CONFIG_INT128)) += host-utils.o
 util-obj-y += bitmap.o bitops.o hbitmap.o
 util-obj-y += fifo8.o
 util-obj-y += acl.o
 util-obj-y += error.o qemu-error.o
-util-obj-$(CONFIG_POSIX) += compatfd.o
 util-obj-y += id.o
 util-obj-y += iov.o qemu-config.o qemu-sockets.o uri.o notify.o
 util-obj-y += qemu-option.o qemu-progress.o
@@ -18,3 +25,8 @@ util-obj-y += getauxval.o
 util-obj-y += readline.o
 util-obj-y += rfifolock.o
 util-obj-y += rcu.o
+util-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
+util-obj-y += qemu-coroutine-sleep.o
+util-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
+util-obj-y += buffer.o
+util-obj-y += timed-average.o
diff --git a/util/bitmap.c b/util/bitmap.c
index 300a68e38..44f0f481b 100644
--- a/util/bitmap.c
+++ b/util/bitmap.c
@@ -14,7 +14,7 @@
 #include "qemu/atomic.h"
 
 /*
- * bitmaps provide an array of bits, implemented using an an
+ * bitmaps provide an array of bits, implemented using an
  * array of unsigned longs.  The number of valid bits in a
  * given bitmap does _not_ need to be an exact multiple of
  * BITS_PER_LONG.
diff --git a/util/buffer.c b/util/buffer.c
new file mode 100644
index 000000000..8b27c08aa
--- /dev/null
+++ b/util/buffer.c
@@ -0,0 +1,171 @@
+/*
+ * QEMU generic buffers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/buffer.h"
+#include "trace.h"
+
+#define BUFFER_MIN_INIT_SIZE     4096
+#define BUFFER_MIN_SHRINK_SIZE  65536
+
+/* define the factor alpha for the expentional smoothing
+ * that is used in the average size calculation. a shift
+ * of 7 results in an alpha of 1/2^7. */
+#define BUFFER_AVG_SIZE_SHIFT       7
+
+static size_t buffer_req_size(Buffer *buffer, size_t len)
+{
+    return MAX(BUFFER_MIN_INIT_SIZE,
+               pow2ceil(buffer->offset + len));
+}
+
+static void buffer_adj_size(Buffer *buffer, size_t len)
+{
+    size_t old = buffer->capacity;
+    buffer->capacity = buffer_req_size(buffer, len);
+    buffer->buffer = g_realloc(buffer->buffer, buffer->capacity);
+    trace_buffer_resize(buffer->name ?: "unnamed",
+                        old, buffer->capacity);
+
+    /* make it even harder for the buffer to shrink, reset average size
+     * to currenty capacity if it is larger than the average. */
+    buffer->avg_size = MAX(buffer->avg_size,
+                           buffer->capacity << BUFFER_AVG_SIZE_SHIFT);
+}
+
+void buffer_init(Buffer *buffer, const char *name, ...)
+{
+    va_list ap;
+
+    va_start(ap, name);
+    buffer->name = g_strdup_vprintf(name, ap);
+    va_end(ap);
+}
+
+static uint64_t buffer_get_avg_size(Buffer *buffer)
+{
+    return buffer->avg_size >> BUFFER_AVG_SIZE_SHIFT;
+}
+
+void buffer_shrink(Buffer *buffer)
+{
+    size_t new;
+
+    /* Calculate the average size of the buffer as
+     * avg_size = avg_size * ( 1 - a ) + required_size * a
+     * where a is 1 / 2 ^ BUFFER_AVG_SIZE_SHIFT. */
+    buffer->avg_size *= (1 << BUFFER_AVG_SIZE_SHIFT) - 1;
+    buffer->avg_size >>= BUFFER_AVG_SIZE_SHIFT;
+    buffer->avg_size += buffer_req_size(buffer, 0);
+
+    /* And then only shrink if the average size of the buffer is much
+     * too big, to avoid bumping up & down the buffers all the time.
+     * realloc() isn't exactly cheap ...  */
+    new = buffer_req_size(buffer, buffer_get_avg_size(buffer));
+    if (new < buffer->capacity >> 3 &&
+        new >= BUFFER_MIN_SHRINK_SIZE) {
+        buffer_adj_size(buffer, buffer_get_avg_size(buffer));
+    }
+
+    buffer_adj_size(buffer, 0);
+}
+
+void buffer_reserve(Buffer *buffer, size_t len)
+{
+    if ((buffer->capacity - buffer->offset) < len) {
+        buffer_adj_size(buffer, len);
+    }
+}
+
+gboolean buffer_empty(Buffer *buffer)
+{
+    return buffer->offset == 0;
+}
+
+uint8_t *buffer_end(Buffer *buffer)
+{
+    return buffer->buffer + buffer->offset;
+}
+
+void buffer_reset(Buffer *buffer)
+{
+    buffer->offset = 0;
+    buffer_shrink(buffer);
+}
+
+void buffer_free(Buffer *buffer)
+{
+    trace_buffer_free(buffer->name ?: "unnamed", buffer->capacity);
+    g_free(buffer->buffer);
+    g_free(buffer->name);
+    buffer->offset = 0;
+    buffer->capacity = 0;
+    buffer->buffer = NULL;
+    buffer->name = NULL;
+}
+
+void buffer_append(Buffer *buffer, const void *data, size_t len)
+{
+    memcpy(buffer->buffer + buffer->offset, data, len);
+    buffer->offset += len;
+}
+
+void buffer_advance(Buffer *buffer, size_t len)
+{
+    memmove(buffer->buffer, buffer->buffer + len,
+            (buffer->offset - len));
+    buffer->offset -= len;
+    buffer_shrink(buffer);
+}
+
+void buffer_move_empty(Buffer *to, Buffer *from)
+{
+    trace_buffer_move_empty(to->name ?: "unnamed",
+                            from->offset,
+                            from->name ?: "unnamed");
+    assert(to->offset == 0);
+
+    g_free(to->buffer);
+    to->offset = from->offset;
+    to->capacity = from->capacity;
+    to->buffer = from->buffer;
+
+    from->offset = 0;
+    from->capacity = 0;
+    from->buffer = NULL;
+}
+
+void buffer_move(Buffer *to, Buffer *from)
+{
+    if (to->offset == 0) {
+        buffer_move_empty(to, from);
+        return;
+    }
+
+    trace_buffer_move(to->name ?: "unnamed",
+                      from->offset,
+                      from->name ?: "unnamed");
+    buffer_reserve(to, from->offset);
+    buffer_append(to, from->buffer, from->offset);
+
+    g_free(from->buffer);
+    from->offset = 0;
+    from->capacity = 0;
+    from->buffer = NULL;
+}
diff --git a/coroutine-gthread.c b/util/coroutine-gthread.c
index 6bd6d6b22..0bcd77867 100644
--- a/coroutine-gthread.c
+++ b/util/coroutine-gthread.c
@@ -20,7 +20,7 @@
 
 #include <glib.h>
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 typedef struct {
     Coroutine base;
diff --git a/coroutine-sigaltstack.c b/util/coroutine-sigaltstack.c
index 63519fffc..39842a4a9 100644
--- a/coroutine-sigaltstack.c
+++ b/util/coroutine-sigaltstack.c
@@ -31,7 +31,7 @@
 #include <pthread.h>
 #include <signal.h>
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 typedef struct {
     Coroutine base;
diff --git a/coroutine-ucontext.c b/util/coroutine-ucontext.c
index 259fcb48a..26cbebb7a 100644
--- a/coroutine-ucontext.c
+++ b/util/coroutine-ucontext.c
@@ -27,7 +27,7 @@
 #include <stdint.h>
 #include <ucontext.h>
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 #ifdef CONFIG_VALGRIND_H
 #include <valgrind/valgrind.h>
diff --git a/coroutine-win32.c b/util/coroutine-win32.c
index 17ace37de..4f922c53a 100644
--- a/coroutine-win32.c
+++ b/util/coroutine-win32.c
@@ -23,7 +23,7 @@
  */
 
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 typedef struct
 {
diff --git a/util/cutils.c b/util/cutils.c
index 5d1c9ebe0..cfeb848d1 100644
--- a/util/cutils.c
+++ b/util/cutils.c
@@ -145,11 +145,6 @@ time_t mktimegm(struct tm *tm)
     return t;
 }
 
-int qemu_fls(int i)
-{
-    return 32 - clz32(i);
-}
-
 /*
  * Make sure data goes on disk, but if possible do not bother to
  * write out the inode just for timestamp updates.
@@ -281,19 +276,19 @@ int fcntl_setfl(int fd, int flag)
 static int64_t suffix_mul(char suffix, int64_t unit)
 {
     switch (qemu_toupper(suffix)) {
-    case STRTOSZ_DEFSUFFIX_B:
+    case QEMU_STRTOSZ_DEFSUFFIX_B:
         return 1;
-    case STRTOSZ_DEFSUFFIX_KB:
+    case QEMU_STRTOSZ_DEFSUFFIX_KB:
         return unit;
-    case STRTOSZ_DEFSUFFIX_MB:
+    case QEMU_STRTOSZ_DEFSUFFIX_MB:
         return unit * unit;
-    case STRTOSZ_DEFSUFFIX_GB:
+    case QEMU_STRTOSZ_DEFSUFFIX_GB:
         return unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_TB:
+    case QEMU_STRTOSZ_DEFSUFFIX_TB:
         return unit * unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_PB:
+    case QEMU_STRTOSZ_DEFSUFFIX_PB:
         return unit * unit * unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_EB:
+    case QEMU_STRTOSZ_DEFSUFFIX_EB:
         return unit * unit * unit * unit * unit * unit;
     }
     return -1;
@@ -305,7 +300,7 @@ static int64_t suffix_mul(char suffix, int64_t unit)
  * in *end, if not NULL. Return -ERANGE on overflow, Return -EINVAL on
  * other error.
  */
-int64_t strtosz_suffix_unit(const char *nptr, char **end,
+int64_t qemu_strtosz_suffix_unit(const char *nptr, char **end,
                             const char default_suffix, int64_t unit)
 {
     int64_t retval = -EINVAL;
@@ -348,14 +343,165 @@ fail:
     return retval;
 }
 
-int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix)
+int64_t qemu_strtosz_suffix(const char *nptr, char **end,
+                            const char default_suffix)
 {
-    return strtosz_suffix_unit(nptr, end, default_suffix, 1024);
+    return qemu_strtosz_suffix_unit(nptr, end, default_suffix, 1024);
 }
 
-int64_t strtosz(const char *nptr, char **end)
+int64_t qemu_strtosz(const char *nptr, char **end)
+{
+    return qemu_strtosz_suffix(nptr, end, QEMU_STRTOSZ_DEFSUFFIX_MB);
+}
+
+/**
+ * Helper function for qemu_strto*l() functions.
+ */
+static int check_strtox_error(const char *p, char *endptr, const char **next,
+                              int err)
+{
+    /* If no conversion was performed, prefer BSD behavior over glibc
+     * behavior.
+     */
+    if (err == 0 && endptr == p) {
+        err = EINVAL;
+    }
+    if (!next && *endptr) {
+        return -EINVAL;
+    }
+    if (next) {
+        *next = endptr;
+    }
+    return -err;
+}
+
+/**
+ * QEMU wrappers for strtol(), strtoll(), strtoul(), strotull() C functions.
+ *
+ * Convert ASCII string @nptr to a long integer value
+ * from the given @base. Parameters @nptr, @endptr, @base
+ * follows same semantics as strtol() C function.
+ *
+ * Unlike from strtol() function, if @endptr is not NULL, this
+ * function will return -EINVAL whenever it cannot fully convert
+ * the string in @nptr with given @base to a long. This function returns
+ * the result of the conversion only through the @result parameter.
+ *
+ * If NULL is passed in @endptr, then the whole string in @ntpr
+ * is a number otherwise it returns -EINVAL.
+ *
+ * RETURN VALUE
+ * Unlike from strtol() function, this wrapper returns either
+ * -EINVAL or the errno set by strtol() function (e.g -ERANGE).
+ * If the conversion overflows, -ERANGE is returned, and @result
+ * is set to the max value of the desired type
+ * (e.g. LONG_MAX, LLONG_MAX, ULONG_MAX, ULLONG_MAX). If the case
+ * of underflow, -ERANGE is returned, and @result is set to the min
+ * value of the desired type. For strtol(), strtoll(), @result is set to
+ * LONG_MIN, LLONG_MIN, respectively, and for strtoul(), strtoull() it
+ * is set to 0.
+ */
+int qemu_strtol(const char *nptr, const char **endptr, int base,
+                long *result)
 {
-    return strtosz_suffix(nptr, end, STRTOSZ_DEFSUFFIX_MB);
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtol(nptr, &p, base);
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to an unsigned long integer.
+ *
+ * If string contains a negative number, value will be converted to
+ * the unsigned representation of the signed value, unless the original
+ * (nonnegated) value would overflow, in this case, it will set @result
+ * to ULONG_MAX, and return ERANGE.
+ *
+ * The same behavior holds, for qemu_strtoull() but sets @result to
+ * ULLONG_MAX instead of ULONG_MAX.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoul(const char *nptr, const char **endptr, int base,
+                 unsigned long *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoul(nptr, &p, base);
+        /* Windows returns 1 for negative out-of-range values.  */
+        if (errno == ERANGE) {
+            *result = -1;
+        }
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to a long long integer.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoll(const char *nptr, const char **endptr, int base,
+                 int64_t *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoll(nptr, &p, base);
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to an unsigned long long integer.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoull(const char *nptr, const char **endptr, int base,
+                  uint64_t *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoull(nptr, &p, base);
+        /* Windows returns 1 for negative out-of-range values.  */
+        if (errno == ERANGE) {
+            *result = -1;
+        }
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
 }
 
 /**
@@ -474,29 +620,6 @@ int qemu_parse_fd(const char *param)
     return fd;
 }
 
-/* round down to the nearest power of 2*/
-int64_t pow2floor(int64_t value)
-{
-    if (!is_power_of_2(value)) {
-        value = 0x8000000000000000ULL >> clz64(value);
-    }
-    return value;
-}
-
-/* round up to the nearest power of 2 (0 if overflow) */
-uint64_t pow2ceil(uint64_t value)
-{
-    uint8_t nlz = clz64(value);
-
-    if (is_power_of_2(value)) {
-        return value;
-    }
-    if (!nlz) {
-        return 0;
-    }
-    return 1ULL << (64 - nlz);
-}
-
 /*
  * Implementation of  ULEB128 (http://en.wikipedia.org/wiki/LEB128)
  * Input is limited to 14-bit numbers
diff --git a/util/error.c b/util/error.c
index 14f435187..80c89a207 100644
--- a/util/error.c
+++ b/util/error.c
@@ -2,9 +2,11 @@
  * QEMU Error Objects
  *
  * Copyright IBM, Corp. 2011
+ * Copyright (C) 2011-2015 Red Hat, Inc.
  *
  * Authors:
  *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Markus Armbruster <armbru@redhat.com>,
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.  See
  * the COPYING.LIB file in the top-level directory.
@@ -18,14 +20,33 @@ struct Error
 {
     char *msg;
     ErrorClass err_class;
+    const char *src, *func;
+    int line;
+    GString *hint;
 };
 
 Error *error_abort;
+Error *error_fatal;
 
-void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
+static void error_handle_fatal(Error **errp, Error *err)
+{
+    if (errp == &error_abort) {
+        fprintf(stderr, "Unexpected error in %s() at %s:%d:\n",
+                err->func, err->src, err->line);
+        error_report_err(err);
+        abort();
+    }
+    if (errp == &error_fatal) {
+        error_report_err(err);
+        exit(1);
+    }
+}
+
+static void error_setv(Error **errp,
+                       const char *src, int line, const char *func,
+                       ErrorClass err_class, const char *fmt, va_list ap)
 {
     Error *err;
-    va_list ap;
     int saved_errno = errno;
 
     if (errp == NULL) {
@@ -34,99 +55,120 @@ void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
     assert(*errp == NULL);
 
     err = g_malloc0(sizeof(*err));
-
-    va_start(ap, fmt);
     err->msg = g_strdup_vprintf(fmt, ap);
-    va_end(ap);
     err->err_class = err_class;
+    err->src = src;
+    err->line = line;
+    err->func = func;
 
-    if (errp == &error_abort) {
-        error_report_err(err);
-        abort();
-    }
-
+    error_handle_fatal(errp, err);
     *errp = err;
 
     errno = saved_errno;
 }
 
-void error_set_errno(Error **errp, int os_errno, ErrorClass err_class,
-                     const char *fmt, ...)
+void error_set_internal(Error **errp,
+                        const char *src, int line, const char *func,
+                        ErrorClass err_class, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_setv(errp, src, line, func, err_class, fmt, ap);
+    va_end(ap);
+}
+
+void error_setg_internal(Error **errp,
+                         const char *src, int line, const char *func,
+                         const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR, fmt, ap);
+    va_end(ap);
+}
+
+void error_setg_errno_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int os_errno, const char *fmt, ...)
 {
-    Error *err;
-    char *msg1;
     va_list ap;
+    char *msg;
     int saved_errno = errno;
 
     if (errp == NULL) {
         return;
     }
-    assert(*errp == NULL);
-
-    err = g_malloc0(sizeof(*err));
 
     va_start(ap, fmt);
-    msg1 = g_strdup_vprintf(fmt, ap);
-    if (os_errno != 0) {
-        err->msg = g_strdup_printf("%s: %s", msg1, strerror(os_errno));
-        g_free(msg1);
-    } else {
-        err->msg = msg1;
-    }
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR, fmt, ap);
     va_end(ap);
-    err->err_class = err_class;
 
-    if (errp == &error_abort) {
-        error_report_err(err);
-        abort();
+    if (os_errno != 0) {
+        msg = (*errp)->msg;
+        (*errp)->msg = g_strdup_printf("%s: %s", msg, strerror(os_errno));
+        g_free(msg);
     }
 
-    *errp = err;
-
     errno = saved_errno;
 }
 
-void error_setg_file_open(Error **errp, int os_errno, const char *filename)
+void error_setg_file_open_internal(Error **errp,
+                                   const char *src, int line, const char *func,
+                                   int os_errno, const char *filename)
 {
-    error_setg_errno(errp, os_errno, "Could not open '%s'", filename);
+    error_setg_errno_internal(errp, src, line, func, os_errno,
+                              "Could not open '%s'", filename);
+}
+
+void error_append_hint(Error **errp, const char *fmt, ...)
+{
+    va_list ap;
+    int saved_errno = errno;
+    Error *err;
+
+    if (!errp) {
+        return;
+    }
+    err = *errp;
+    assert(err && errp != &error_abort);
+
+    if (!err->hint) {
+        err->hint = g_string_new(NULL);
+    }
+    va_start(ap, fmt);
+    g_string_append_vprintf(err->hint, fmt, ap);
+    va_end(ap);
+
+    errno = saved_errno;
 }
 
 #ifdef _WIN32
 
-void error_set_win32(Error **errp, int win32_err, ErrorClass err_class,
-                     const char *fmt, ...)
+void error_setg_win32_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int win32_err, const char *fmt, ...)
 {
-    Error *err;
-    char *msg1;
     va_list ap;
+    char *msg1, *msg2;
 
     if (errp == NULL) {
         return;
     }
-    assert(*errp == NULL);
-
-    err = g_malloc0(sizeof(*err));
 
     va_start(ap, fmt);
-    msg1 = g_strdup_vprintf(fmt, ap);
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR, fmt, ap);
+    va_end(ap);
+
     if (win32_err != 0) {
-        char *msg2 = g_win32_error_message(win32_err);
-        err->msg = g_strdup_printf("%s: %s (error: %x)", msg1, msg2,
-                                   (unsigned)win32_err);
+        msg1 = (*errp)->msg;
+        msg2 = g_win32_error_message(win32_err);
+        (*errp)->msg = g_strdup_printf("%s: %s (error: %x)", msg1, msg2,
+                                       (unsigned)win32_err);
         g_free(msg2);
         g_free(msg1);
-    } else {
-        err->msg = msg1;
-    }
-    va_end(ap);
-    err->err_class = err_class;
-
-    if (errp == &error_abort) {
-        error_report_err(err);
-        abort();
     }
-
-    *errp = err;
 }
 
 #endif
@@ -138,6 +180,12 @@ Error *error_copy(const Error *err)
     err_new = g_malloc0(sizeof(*err));
     err_new->msg = g_strdup(err->msg);
     err_new->err_class = err->err_class;
+    err_new->src = err->src;
+    err_new->line = err->line;
+    err_new->func = err->func;
+    if (err->hint) {
+        err_new->hint = g_string_new(err->hint->str);
+    }
 
     return err_new;
 }
@@ -155,6 +203,9 @@ const char *error_get_pretty(Error *err)
 void error_report_err(Error *err)
 {
     error_report("%s", error_get_pretty(err));
+    if (err->hint) {
+        error_printf_unless_qmp("%s\n", err->hint->str);
+    }
     error_free(err);
 }
 
@@ -162,18 +213,29 @@ void error_free(Error *err)
 {
     if (err) {
         g_free(err->msg);
+        if (err->hint) {
+            g_string_free(err->hint, true);
+        }
         g_free(err);
     }
 }
 
+void error_free_or_abort(Error **errp)
+{
+    assert(errp && *errp);
+    error_free(*errp);
+    *errp = NULL;
+}
+
 void error_propagate(Error **dst_errp, Error *local_err)
 {
-    if (local_err && dst_errp == &error_abort) {
-        error_report_err(local_err);
-        abort();
-    } else if (dst_errp && !*dst_errp) {
+    if (!local_err) {
+        return;
+    }
+    error_handle_fatal(dst_errp, local_err);
+    if (dst_errp && !*dst_errp) {
         *dst_errp = local_err;
-    } else if (local_err) {
+    } else {
         error_free(local_err);
     }
 }
diff --git a/util/event_notifier-posix.c b/util/event_notifier-posix.c
index ed4ca2b01..d4a0c63e1 100644
--- a/util/event_notifier-posix.c
+++ b/util/event_notifier-posix.c
@@ -77,7 +77,7 @@ void event_notifier_cleanup(EventNotifier *e)
     close(e->wfd);
 }
 
-int event_notifier_get_fd(EventNotifier *e)
+int event_notifier_get_fd(const EventNotifier *e)
 {
     return e->rfd;
 }
diff --git a/util/id.c b/util/id.c
index 09b22fb8f..7883fbec7 100644
--- a/util/id.c
+++ b/util/id.c
@@ -26,3 +26,40 @@ bool id_wellformed(const char *id)
     }
     return true;
 }
+
+#define ID_SPECIAL_CHAR '#'
+
+static const char *const id_subsys_str[ID_MAX] = {
+    [ID_QDEV]  = "qdev",
+    [ID_BLOCK] = "block",
+};
+
+/*
+ *  Generates an ID of the form PREFIX SUBSYSTEM NUMBER
+ *  where:
+ *
+ *  - PREFIX is the reserved character '#'
+ *  - SUBSYSTEM identifies the subsystem creating the ID
+ *  - NUMBER is a decimal number unique within SUBSYSTEM.
+ *
+ *    Example: "#block146"
+ *
+ * Note that these IDs do not satisfy id_wellformed().
+ *
+ * The caller is responsible for freeing the returned string with g_free()
+ */
+char *id_generate(IdSubSystems id)
+{
+    static uint64_t id_counters[ID_MAX];
+    uint32_t rnd;
+
+    assert(id < ARRAY_SIZE(id_subsys_str));
+    assert(id_subsys_str[id]);
+
+    rnd = g_random_int_range(0, 100);
+
+    return g_strdup_printf("%c%s%" PRIu64 "%02" PRId32, ID_SPECIAL_CHAR,
+                                                        id_subsys_str[id],
+                                                        id_counters[id]++,
+                                                        rnd);
+}
diff --git a/util/memfd.c b/util/memfd.c
new file mode 100644
index 000000000..7c406914c
--- /dev/null
+++ b/util/memfd.c
@@ -0,0 +1,162 @@
+/*
+ * memfd.c
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * QEMU library functions on POSIX which are shared between QEMU and
+ * the QEMU tools.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+#include <glib.h>
+#include <glib/gprintf.h>
+
+#include <sys/mman.h>
+
+#include "qemu/memfd.h"
+
+#ifdef CONFIG_MEMFD
+#include <sys/memfd.h>
+#elif defined CONFIG_LINUX
+#include <sys/syscall.h>
+#include <asm/unistd.h>
+
+static int memfd_create(const char *name, unsigned int flags)
+{
+#ifdef __NR_memfd_create
+    return syscall(__NR_memfd_create, name, flags);
+#else
+    return -1;
+#endif
+}
+#endif
+
+#ifndef MFD_CLOEXEC
+#define MFD_CLOEXEC 0x0001U
+#endif
+
+#ifndef MFD_ALLOW_SEALING
+#define MFD_ALLOW_SEALING 0x0002U
+#endif
+
+/*
+ * This is a best-effort helper for shared memory allocation, with
+ * optional sealing. The helper will do his best to allocate using
+ * memfd with sealing, but may fallback on other methods without
+ * sealing.
+ */
+void *qemu_memfd_alloc(const char *name, size_t size, unsigned int seals,
+                       int *fd)
+{
+    void *ptr;
+    int mfd = -1;
+
+    *fd = -1;
+
+#ifdef CONFIG_LINUX
+    if (seals) {
+        mfd = memfd_create(name, MFD_ALLOW_SEALING | MFD_CLOEXEC);
+    }
+
+    if (mfd == -1) {
+        /* some systems have memfd without sealing */
+        mfd = memfd_create(name, MFD_CLOEXEC);
+        seals = 0;
+    }
+#endif
+
+    if (mfd != -1) {
+        if (ftruncate(mfd, size) == -1) {
+            perror("ftruncate");
+            close(mfd);
+            return NULL;
+        }
+
+        if (seals && fcntl(mfd, F_ADD_SEALS, seals) == -1) {
+            perror("fcntl");
+            close(mfd);
+            return NULL;
+        }
+    } else {
+        const char *tmpdir = g_get_tmp_dir();
+        gchar *fname;
+
+        fname = g_strdup_printf("%s/memfd-XXXXXX", tmpdir);
+        mfd = mkstemp(fname);
+        unlink(fname);
+        g_free(fname);
+
+        if (mfd == -1) {
+            perror("mkstemp");
+            return NULL;
+        }
+
+        if (ftruncate(mfd, size) == -1) {
+            perror("ftruncate");
+            close(mfd);
+            return NULL;
+        }
+    }
+
+    ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
+    if (ptr == MAP_FAILED) {
+        perror("mmap");
+        close(mfd);
+        return NULL;
+    }
+
+    *fd = mfd;
+    return ptr;
+}
+
+void qemu_memfd_free(void *ptr, size_t size, int fd)
+{
+    if (ptr) {
+        munmap(ptr, size);
+    }
+
+    if (fd != -1) {
+        close(fd);
+    }
+}
+
+enum {
+    MEMFD_KO,
+    MEMFD_OK,
+    MEMFD_TODO
+};
+
+bool qemu_memfd_check(void)
+{
+    static int memfd_check = MEMFD_TODO;
+
+    if (memfd_check == MEMFD_TODO) {
+        int fd;
+        void *ptr;
+
+        ptr = qemu_memfd_alloc("test", 4096, 0, &fd);
+        memfd_check = ptr ? MEMFD_OK : MEMFD_KO;
+        qemu_memfd_free(ptr, 4096, fd);
+    }
+
+    return memfd_check == MEMFD_OK;
+}
diff --git a/util/mmap-alloc.c b/util/mmap-alloc.c
new file mode 100644
index 000000000..54793a5dc
--- /dev/null
+++ b/util/mmap-alloc.c
@@ -0,0 +1,110 @@
+/*
+ * Support for RAM backed by mmaped host memory.
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+#include <qemu/mmap-alloc.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <assert.h>
+
+#define HUGETLBFS_MAGIC       0x958458f6
+
+#ifdef CONFIG_LINUX
+#include <sys/vfs.h>
+#endif
+
+size_t qemu_fd_getpagesize(int fd)
+{
+#ifdef CONFIG_LINUX
+    struct statfs fs;
+    int ret;
+
+    if (fd != -1) {
+        do {
+            ret = fstatfs(fd, &fs);
+        } while (ret != 0 && errno == EINTR);
+
+        if (ret == 0 && fs.f_type == HUGETLBFS_MAGIC) {
+            return fs.f_bsize;
+        }
+    }
+#endif
+
+    return getpagesize();
+}
+
+void *qemu_ram_mmap(int fd, size_t size, size_t align, bool shared)
+{
+    /*
+     * Note: this always allocates at least one extra page of virtual address
+     * space, even if size is already aligned.
+     */
+    size_t total = size + align;
+#if defined(__powerpc64__) && defined(__linux__)
+    /* On ppc64 mappings in the same segment (aka slice) must share the same
+     * page size. Since we will be re-allocating part of this segment
+     * from the supplied fd, we should make sure to use the same page size,
+     * unless we are using the system page size, in which case anonymous memory
+     * is OK. Use align as a hint for the page size.
+     * In this case, set MAP_NORESERVE to avoid allocating backing store memory.
+     */
+    int anonfd = fd == -1 || qemu_fd_getpagesize(fd) == getpagesize() ? -1 : fd;
+    int flags = anonfd == -1 ? MAP_ANONYMOUS : MAP_NORESERVE;
+    void *ptr = mmap(0, total, PROT_NONE, flags | MAP_PRIVATE, anonfd, 0);
+#else
+    void *ptr = mmap(0, total, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+#endif
+    size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
+    void *ptr1;
+
+    if (ptr == MAP_FAILED) {
+        return MAP_FAILED;
+    }
+
+    /* Make sure align is a power of 2 */
+    assert(!(align & (align - 1)));
+    /* Always align to host page size */
+    assert(align >= getpagesize());
+
+    ptr1 = mmap(ptr + offset, size, PROT_READ | PROT_WRITE,
+                MAP_FIXED |
+                (fd == -1 ? MAP_ANONYMOUS : 0) |
+                (shared ? MAP_SHARED : MAP_PRIVATE),
+                fd, 0);
+    if (ptr1 == MAP_FAILED) {
+        munmap(ptr, total);
+        return MAP_FAILED;
+    }
+
+    ptr += offset;
+    total -= offset;
+
+    if (offset > 0) {
+        munmap(ptr - offset, offset);
+    }
+
+    /*
+     * Leave a single PROT_NONE page allocated after the RAM block, to serve as
+     * a guard page guarding against potential buffer overflows.
+     */
+    if (total > size + getpagesize()) {
+        munmap(ptr + size + getpagesize(), total - size - getpagesize());
+    }
+
+    return ptr;
+}
+
+void qemu_ram_munmap(void *ptr, size_t size)
+{
+    if (ptr) {
+        /* Unmap both the RAM block and the guard page */
+        munmap(ptr, size + getpagesize());
+    }
+}
diff --git a/util/osdep.c b/util/osdep.c
index 0092bb61b..534b51147 100644
--- a/util/osdep.c
+++ b/util/osdep.c
@@ -52,7 +52,14 @@ extern int madvise(caddr_t, size_t, int);
 
 static bool fips_enabled = false;
 
-static const char *qemu_version = QEMU_VERSION;
+/* Starting on QEMU 2.5, qemu_hw_version() returns "2.5+" by default
+ * instead of QEMU_VERSION, so setting hw_version on MachineClass
+ * is no longer mandatory.
+ *
+ * Do NOT change this string, or it will break compatibility on all
+ * machine classes that don't set hw_version.
+ */
+static const char *hw_version = "2.5+";
 
 int socket_set_cork(int fd, int v)
 {
@@ -311,14 +318,14 @@ int qemu_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
     return ret;
 }
 
-void qemu_set_version(const char *version)
+void qemu_set_hw_version(const char *version)
 {
-    qemu_version = version;
+    hw_version = version;
 }
 
-const char *qemu_get_version(void)
+const char *qemu_hw_version(void)
 {
-    return qemu_version;
+    return hw_version;
 }
 
 void fips_set_state(bool requested)
diff --git a/util/oslib-posix.c b/util/oslib-posix.c
index 3ae4987b6..d25f6715c 100644
--- a/util/oslib-posix.c
+++ b/util/oslib-posix.c
@@ -46,7 +46,6 @@ extern int daemon(int, int);
 #else
 #  define QEMU_VMALLOC_ALIGN getpagesize()
 #endif
-#define HUGETLBFS_MAGIC       0x958458f6
 
 #include <termios.h>
 #include <unistd.h>
@@ -65,13 +64,14 @@ extern int daemon(int, int);
 
 #ifdef CONFIG_LINUX
 #include <sys/syscall.h>
-#include <sys/vfs.h>
 #endif
 
 #ifdef __FreeBSD__
 #include <sys/sysctl.h>
 #endif
 
+#include <qemu/mmap-alloc.h>
+
 int qemu_get_thread_id(void)
 {
 #if defined(__linux__)
@@ -128,10 +128,7 @@ void *qemu_memalign(size_t alignment, size_t size)
 void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment)
 {
     size_t align = QEMU_VMALLOC_ALIGN;
-    size_t total = size + align - getpagesize();
-    void *ptr = mmap(0, total, PROT_READ | PROT_WRITE,
-                     MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
-    size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
+    void *ptr = qemu_ram_mmap(-1, size, align, false);
 
     if (ptr == MAP_FAILED) {
         return NULL;
@@ -140,15 +137,6 @@ void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment)
     if (alignment) {
         *alignment = align;
     }
-    ptr += offset;
-    total -= offset;
-
-    if (offset > 0) {
-        munmap(ptr - offset, offset);
-    }
-    if (total > size) {
-        munmap(ptr + size, total - size);
-    }
 
     trace_qemu_anon_ram_alloc(size, ptr);
     return ptr;
@@ -163,9 +151,7 @@ void qemu_vfree(void *ptr)
 void qemu_anon_ram_free(void *ptr, size_t size)
 {
     trace_qemu_anon_ram_free(ptr, size);
-    if (ptr) {
-        munmap(ptr, size);
-    }
+    qemu_ram_munmap(ptr, size);
 }
 
 void qemu_set_block(int fd)
@@ -352,26 +338,6 @@ static void sigbus_handler(int signal)
     siglongjmp(sigjump, 1);
 }
 
-static size_t fd_getpagesize(int fd)
-{
-#ifdef CONFIG_LINUX
-    struct statfs fs;
-    int ret;
-
-    if (fd != -1) {
-        do {
-            ret = fstatfs(fd, &fs);
-        } while (ret != 0 && errno == EINTR);
-
-        if (ret == 0 && fs.f_type == HUGETLBFS_MAGIC) {
-            return fs.f_bsize;
-        }
-    }
-#endif
-
-    return getpagesize();
-}
-
 void os_mem_prealloc(int fd, char *area, size_t memory)
 {
     int ret;
@@ -399,7 +365,7 @@ void os_mem_prealloc(int fd, char *area, size_t memory)
         exit(1);
     } else {
         int i;
-        size_t hpagesize = fd_getpagesize(fd);
+        size_t hpagesize = qemu_fd_getpagesize(fd);
         size_t numpages = DIV_ROUND_UP(memory, hpagesize);
 
         /* MAP_POPULATE silently ignores failures */
@@ -482,3 +448,74 @@ int qemu_read_password(char *buf, int buf_size)
     printf("\n");
     return ret;
 }
+
+
+pid_t qemu_fork(Error **errp)
+{
+    sigset_t oldmask, newmask;
+    struct sigaction sig_action;
+    int saved_errno;
+    pid_t pid;
+
+    /*
+     * Need to block signals now, so that child process can safely
+     * kill off caller's signal handlers without a race.
+     */
+    sigfillset(&newmask);
+    if (pthread_sigmask(SIG_SETMASK, &newmask, &oldmask) != 0) {
+        error_setg_errno(errp, errno,
+                         "cannot block signals");
+        return -1;
+    }
+
+    pid = fork();
+    saved_errno = errno;
+
+    if (pid < 0) {
+        /* attempt to restore signal mask, but ignore failure, to
+         * avoid obscuring the fork failure */
+        (void)pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+        error_setg_errno(errp, saved_errno,
+                         "cannot fork child process");
+        errno = saved_errno;
+        return -1;
+    } else if (pid) {
+        /* parent process */
+
+        /* Restore our original signal mask now that the child is
+         * safely running. Only documented failures are EFAULT (not
+         * possible, since we are using just-grabbed mask) or EINVAL
+         * (not possible, since we are using correct arguments).  */
+        (void)pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+    } else {
+        /* child process */
+        size_t i;
+
+        /* Clear out all signal handlers from parent so nothing
+         * unexpected can happen in our child once we unblock
+         * signals */
+        sig_action.sa_handler = SIG_DFL;
+        sig_action.sa_flags = 0;
+        sigemptyset(&sig_action.sa_mask);
+
+        for (i = 1; i < NSIG; i++) {
+            /* Only possible errors are EFAULT or EINVAL The former
+             * won't happen, the latter we expect, so no need to check
+             * return value */
+            (void)sigaction(i, &sig_action, NULL);
+        }
+
+        /* Unmask all signals in child, since we've no idea what the
+         * caller's done with their signal mask and don't want to
+         * propagate that to children */
+        sigemptyset(&newmask);
+        if (pthread_sigmask(SIG_SETMASK, &newmask, NULL) != 0) {
+            Error *local_err = NULL;
+            error_setg_errno(&local_err, errno,
+                             "cannot unblock signals");
+            error_report_err(local_err);
+            _exit(1);
+        }
+    }
+    return pid;
+}
diff --git a/util/oslib-win32.c b/util/oslib-win32.c
index 730a6707a..6a47019df 100644
--- a/util/oslib-win32.c
+++ b/util/oslib-win32.c
@@ -95,6 +95,7 @@ void qemu_anon_ram_free(void *ptr, size_t size)
     }
 }
 
+#ifndef CONFIG_LOCALTIME_R
 /* FIXME: add proper locking */
 struct tm *gmtime_r(const time_t *timep, struct tm *result)
 {
@@ -118,6 +119,7 @@ struct tm *localtime_r(const time_t *timep, struct tm *result)
     }
     return p;
 }
+#endif /* CONFIG_LOCALTIME_R */
 
 void qemu_set_block(int fd)
 {
@@ -452,7 +454,7 @@ gint g_poll(GPollFD *fds, guint nfds, gint timeout)
     return retval;
 }
 
-size_t getpagesize(void)
+int getpagesize(void)
 {
     SYSTEM_INFO system_info;
 
@@ -494,3 +496,12 @@ int qemu_read_password(char *buf, int buf_size)
     buf[i] = '\0';
     return 0;
 }
+
+
+pid_t qemu_fork(Error **errp)
+{
+    errno = ENOSYS;
+    error_setg_errno(errp, errno,
+                     "cannot fork child process");
+    return -1;
+}
diff --git a/qemu-coroutine-io.c b/util/qemu-coroutine-io.c
index 28dc7351a..e1eae7331 100644
--- a/qemu-coroutine-io.c
+++ b/util/qemu-coroutine-io.c
@@ -24,7 +24,7 @@
  */
 #include "qemu-common.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qemu/iov.h"
 #include "qemu/main-loop.h"
 
diff --git a/qemu-coroutine-lock.c b/util/qemu-coroutine-lock.c
index 6b4903334..130ee19d1 100644
--- a/qemu-coroutine-lock.c
+++ b/util/qemu-coroutine-lock.c
@@ -23,8 +23,8 @@
  */
 
 #include "qemu-common.h"
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
 #include "qemu/queue.h"
 #include "trace.h"
 
diff --git a/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.c
index 9abb7fdf3..b35db5635 100644
--- a/qemu-coroutine-sleep.c
+++ b/util/qemu-coroutine-sleep.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qemu/timer.h"
 #include "block/aio.h"
 
diff --git a/qemu-coroutine.c b/util/qemu-coroutine.c
index c17a92b10..895356022 100644
--- a/qemu-coroutine.c
+++ b/util/qemu-coroutine.c
@@ -16,8 +16,8 @@
 #include "qemu-common.h"
 #include "qemu/thread.h"
 #include "qemu/atomic.h"
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
 
 enum {
     POOL_BATCH_SIZE = 64,
diff --git a/util/qemu-error.c b/util/qemu-error.c
index 77ea6c614..c1574bb34 100644
--- a/util/qemu-error.c
+++ b/util/qemu-error.c
@@ -210,7 +210,7 @@ void error_vreport(const char *fmt, va_list ap)
     GTimeVal tv;
     gchar *timestr;
 
-    if (enable_timestamp_msg) {
+    if (enable_timestamp_msg && !cur_mon) {
         g_get_current_time(&tv);
         timestr = g_time_val_to_iso8601(&tv);
         error_printf("%s ", timestr);
diff --git a/util/qemu-option.c b/util/qemu-option.c
index efe9d279c..a50eceae4 100644
--- a/util/qemu-option.c
+++ b/util/qemu-option.c
@@ -180,6 +180,11 @@ void parse_option_size(const char *name, const char *value,
 
     if (value != NULL) {
         sizef = strtod(value, &postfix);
+        if (sizef < 0 || sizef > UINT64_MAX) {
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name,
+                             "a non-negative number below 2^64");
+            return;
+        }
         switch (*postfix) {
         case 'T':
             sizef *= 1024;
@@ -200,10 +205,8 @@ void parse_option_size(const char *name, const char *value,
             break;
         default:
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            error_printf_unless_qmp("You may use k, M, G or T suffixes for "
-                    "kilobytes, megabytes, gigabytes and terabytes.\n");
-#endif
+            error_append_hint(errp, "You may use k, M, G or T suffixes for "
+                    "kilobytes, megabytes, gigabytes and terabytes.");
             return;
         }
     } else {
@@ -643,9 +646,8 @@ QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
         if (!id_wellformed(id)) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "id",
                        "an identifier");
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            error_printf_unless_qmp("Identifiers consist of letters, digits, '-', '.', '_', starting with a letter.\n");
-#endif
+            error_append_hint(errp, "Identifiers consist of letters, digits, "
+                              "'-', '.', '_', starting with a letter.");
             return NULL;
         }
         opts = qemu_opts_find(list, id);
@@ -730,14 +732,35 @@ void qemu_opts_del(QemuOpts *opts)
     g_free(opts);
 }
 
-void qemu_opts_print(QemuOpts *opts, const char *sep)
+/* print value, escaping any commas in value */
+static void escaped_print(const char *value)
+{
+    const char *ptr;
+
+    for (ptr = value; *ptr; ++ptr) {
+        if (*ptr == ',') {
+            putchar(',');
+        }
+        putchar(*ptr);
+    }
+}
+
+void qemu_opts_print(QemuOpts *opts, const char *separator)
 {
     QemuOpt *opt;
     QemuOptDesc *desc = opts->list->desc;
+    const char *sep = "";
+
+    if (opts->id) {
+        printf("id=%s", opts->id); /* passed id_wellformed -> no commas */
+        sep = separator;
+    }
 
     if (desc[0].name == NULL) {
         QTAILQ_FOREACH(opt, &opts->head, next) {
-            printf("%s%s=\"%s\"", sep, opt->name, opt->str);
+            printf("%s%s=", sep, opt->name);
+            escaped_print(opt->str);
+            sep = separator;
         }
         return;
     }
@@ -750,13 +773,15 @@ void qemu_opts_print(QemuOpts *opts, const char *sep)
             continue;
         }
         if (desc->type == QEMU_OPT_STRING) {
-            printf("%s%s='%s'", sep, desc->name, value);
+            printf("%s%s=", sep, desc->name);
+            escaped_print(value);
         } else if ((desc->type == QEMU_OPT_SIZE ||
                     desc->type == QEMU_OPT_NUMBER) && opt) {
             printf("%s%s=%" PRId64, sep, desc->name, opt->value.uint);
         } else {
             printf("%s%s=%s", sep, desc->name, value);
         }
+        sep = separator;
     }
 }
 
diff --git a/util/qemu-sockets.c b/util/qemu-sockets.c
index 2add83a0f..5a31d164d 100644
--- a/util/qemu-sockets.c
+++ b/util/qemu-sockets.c
@@ -25,6 +25,9 @@
 #include "monitor/monitor.h"
 #include "qemu/sockets.h"
 #include "qemu/main-loop.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi-visit.h"
 
 #ifndef AI_ADDRCONFIG
 # define AI_ADDRCONFIG 0
@@ -125,12 +128,15 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
     ai.ai_family = PF_UNSPEC;
     ai.ai_socktype = SOCK_STREAM;
 
-    if ((qemu_opt_get(opts, "host") == NULL) ||
-        (qemu_opt_get(opts, "port") == NULL)) {
-        error_setg(errp, "host and/or port not specified");
+    if ((qemu_opt_get(opts, "host") == NULL)) {
+        error_setg(errp, "host not specified");
         return -1;
     }
-    pstrcpy(port, sizeof(port), qemu_opt_get(opts, "port"));
+    if (qemu_opt_get(opts, "port") != NULL) {
+        pstrcpy(port, sizeof(port), qemu_opt_get(opts, "port"));
+    } else {
+        port[0] = '\0';
+    }
     addr = qemu_opt_get(opts, "host");
 
     to = qemu_opt_get_number(opts, "to", 0);
@@ -142,6 +148,10 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
     /* lookup */
     if (port_offset) {
         unsigned long long baseport;
+        if (strlen(port) == 0) {
+            error_setg(errp, "port not specified");
+            return -1;
+        }
         if (parse_uint_full(port, &baseport, 10) < 0) {
             error_setg(errp, "can't convert to a number: %s", port);
             return -1;
@@ -153,7 +163,8 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
         }
         snprintf(port, sizeof(port), "%d", (int)baseport + port_offset);
     }
-    rc = getaddrinfo(strlen(addr) ? addr : NULL, port, &ai, &res);
+    rc = getaddrinfo(strlen(addr) ? addr : NULL,
+                     strlen(port) ? port : NULL, &ai, &res);
     if (rc != 0) {
         error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
                    gai_strerror(rc));
@@ -586,12 +597,15 @@ fail:
 
 static void inet_addr_to_opts(QemuOpts *opts, const InetSocketAddress *addr)
 {
-    bool ipv4 = addr->ipv4 || !addr->has_ipv4;
-    bool ipv6 = addr->ipv6 || !addr->has_ipv6;
+    bool ipv4 = addr->has_ipv4 && addr->ipv4;
+    bool ipv6 = addr->has_ipv6 && addr->ipv6;
 
-    if (!ipv4 || !ipv6) {
+    if (ipv4 || ipv6) {
         qemu_opt_set_bool(opts, "ipv4", ipv4, &error_abort);
         qemu_opt_set_bool(opts, "ipv6", ipv6, &error_abort);
+    } else if (addr->has_ipv4 || addr->has_ipv6) {
+        qemu_opt_set_bool(opts, "ipv4", !addr->has_ipv4, &error_abort);
+        qemu_opt_set_bool(opts, "ipv6", !addr->has_ipv6, &error_abort);
     }
     if (addr->has_to) {
         qemu_opt_set_number(opts, "to", addr->to, &error_abort);
@@ -737,8 +751,7 @@ int unix_listen_opts(QemuOpts *opts, Error **errp)
         qemu_opt_set(opts, "path", un.sun_path, &error_abort);
     }
 
-    if ((access(un.sun_path, F_OK) == 0) &&
-        unlink(un.sun_path) < 0) {
+    if (unlink(un.sun_path) < 0 && errno != ENOENT) {
         error_setg_errno(errp, errno,
                          "Failed to unlink socket %s", un.sun_path);
         goto err;
@@ -904,23 +917,23 @@ SocketAddress *socket_parse(const char *str, Error **errp)
             error_setg(errp, "invalid Unix socket address");
             goto fail;
         } else {
-            addr->kind = SOCKET_ADDRESS_KIND_UNIX;
-            addr->q_unix = g_new(UnixSocketAddress, 1);
-            addr->q_unix->path = g_strdup(str + 5);
+            addr->type = SOCKET_ADDRESS_KIND_UNIX;
+            addr->u.q_unix = g_new(UnixSocketAddress, 1);
+            addr->u.q_unix->path = g_strdup(str + 5);
         }
     } else if (strstart(str, "fd:", NULL)) {
         if (str[3] == '\0') {
             error_setg(errp, "invalid file descriptor address");
             goto fail;
         } else {
-            addr->kind = SOCKET_ADDRESS_KIND_FD;
-            addr->fd = g_new(String, 1);
-            addr->fd->str = g_strdup(str + 3);
+            addr->type = SOCKET_ADDRESS_KIND_FD;
+            addr->u.fd = g_new(String, 1);
+            addr->u.fd->str = g_strdup(str + 3);
         }
     } else {
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = inet_parse(str, errp);
-        if (addr->inet == NULL) {
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet = inet_parse(str, errp);
+        if (addr->u.inet == NULL) {
             goto fail;
         }
     }
@@ -938,19 +951,19 @@ int socket_connect(SocketAddress *addr, Error **errp,
     int fd;
 
     opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, addr->inet);
+        inet_addr_to_opts(opts, addr->u.inet);
         fd = inet_connect_opts(opts, errp, callback, opaque);
         break;
 
     case SOCKET_ADDRESS_KIND_UNIX:
-        qemu_opt_set(opts, "path", addr->q_unix->path, &error_abort);
+        qemu_opt_set(opts, "path", addr->u.q_unix->path, &error_abort);
         fd = unix_connect_opts(opts, errp, callback, opaque);
         break;
 
     case SOCKET_ADDRESS_KIND_FD:
-        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        fd = monitor_get_fd(cur_mon, addr->u.fd->str, errp);
         if (fd >= 0 && callback) {
             qemu_set_nonblock(fd);
             callback(fd, NULL, opaque);
@@ -970,19 +983,19 @@ int socket_listen(SocketAddress *addr, Error **errp)
     int fd;
 
     opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, addr->inet);
+        inet_addr_to_opts(opts, addr->u.inet);
         fd = inet_listen_opts(opts, 0, errp);
         break;
 
     case SOCKET_ADDRESS_KIND_UNIX:
-        qemu_opt_set(opts, "path", addr->q_unix->path, &error_abort);
+        qemu_opt_set(opts, "path", addr->u.q_unix->path, &error_abort);
         fd = unix_listen_opts(opts, errp);
         break;
 
     case SOCKET_ADDRESS_KIND_FD:
-        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        fd = monitor_get_fd(cur_mon, addr->u.fd->str, errp);
         break;
 
     default:
@@ -998,12 +1011,12 @@ int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp)
     int fd;
 
     opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (remote->kind) {
+    switch (remote->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, remote->inet);
+        inet_addr_to_opts(opts, remote->u.inet);
         if (local) {
-            qemu_opt_set(opts, "localaddr", local->inet->host, &error_abort);
-            qemu_opt_set(opts, "localport", local->inet->port, &error_abort);
+            qemu_opt_set(opts, "localaddr", local->u.inet->host, &error_abort);
+            qemu_opt_set(opts, "localport", local->u.inet->port, &error_abort);
         }
         fd = inet_dgram_opts(opts, errp);
         break;
@@ -1015,3 +1028,140 @@ int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp)
     qemu_opts_del(opts);
     return fd;
 }
+
+
+static SocketAddress *
+socket_sockaddr_to_address_inet(struct sockaddr_storage *sa,
+                                socklen_t salen,
+                                Error **errp)
+{
+    char host[NI_MAXHOST];
+    char serv[NI_MAXSERV];
+    SocketAddress *addr;
+    int ret;
+
+    ret = getnameinfo((struct sockaddr *)sa, salen,
+                      host, sizeof(host),
+                      serv, sizeof(serv),
+                      NI_NUMERICHOST | NI_NUMERICSERV);
+    if (ret != 0) {
+        error_setg(errp, "Cannot format numeric socket address: %s",
+                   gai_strerror(ret));
+        return NULL;
+    }
+
+    addr = g_new0(SocketAddress, 1);
+    addr->type = SOCKET_ADDRESS_KIND_INET;
+    addr->u.inet = g_new0(InetSocketAddress, 1);
+    addr->u.inet->host = g_strdup(host);
+    addr->u.inet->port = g_strdup(serv);
+    if (sa->ss_family == AF_INET) {
+        addr->u.inet->has_ipv4 = addr->u.inet->ipv4 = true;
+    } else {
+        addr->u.inet->has_ipv6 = addr->u.inet->ipv6 = true;
+    }
+
+    return addr;
+}
+
+
+#ifndef WIN32
+static SocketAddress *
+socket_sockaddr_to_address_unix(struct sockaddr_storage *sa,
+                                socklen_t salen,
+                                Error **errp)
+{
+    SocketAddress *addr;
+    struct sockaddr_un *su = (struct sockaddr_un *)sa;
+
+    addr = g_new0(SocketAddress, 1);
+    addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    addr->u.q_unix = g_new0(UnixSocketAddress, 1);
+    if (su->sun_path[0]) {
+        addr->u.q_unix->path = g_strndup(su->sun_path,
+                                         sizeof(su->sun_path));
+    }
+
+    return addr;
+}
+#endif /* WIN32 */
+
+static SocketAddress *
+socket_sockaddr_to_address(struct sockaddr_storage *sa,
+                           socklen_t salen,
+                           Error **errp)
+{
+    switch (sa->ss_family) {
+    case AF_INET:
+    case AF_INET6:
+        return socket_sockaddr_to_address_inet(sa, salen, errp);
+
+#ifndef WIN32
+    case AF_UNIX:
+        return socket_sockaddr_to_address_unix(sa, salen, errp);
+#endif /* WIN32 */
+
+    default:
+        error_setg(errp, "socket family %d unsupported",
+                   sa->ss_family);
+        return NULL;
+    }
+    return 0;
+}
+
+
+SocketAddress *socket_local_address(int fd, Error **errp)
+{
+    struct sockaddr_storage ss;
+    socklen_t sslen = sizeof(ss);
+
+    if (getsockname(fd, (struct sockaddr *)&ss, &sslen) < 0) {
+        error_setg_errno(errp, socket_error(), "%s",
+                         "Unable to query local socket address");
+        return NULL;
+    }
+
+    return socket_sockaddr_to_address(&ss, sslen, errp);
+}
+
+
+SocketAddress *socket_remote_address(int fd, Error **errp)
+{
+    struct sockaddr_storage ss;
+    socklen_t sslen = sizeof(ss);
+
+    if (getpeername(fd, (struct sockaddr *)&ss, &sslen) < 0) {
+        error_setg_errno(errp, socket_error(), "%s",
+                         "Unable to query remote socket address");
+        return NULL;
+    }
+
+    return socket_sockaddr_to_address(&ss, sslen, errp);
+}
+
+
+void qapi_copy_SocketAddress(SocketAddress **p_dest,
+                             SocketAddress *src)
+{
+    QmpOutputVisitor *qov;
+    QmpInputVisitor *qiv;
+    Visitor *ov, *iv;
+    QObject *obj;
+
+    *p_dest = NULL;
+
+    qov = qmp_output_visitor_new();
+    ov = qmp_output_get_visitor(qov);
+    visit_type_SocketAddress(ov, &src, NULL, &error_abort);
+    obj = qmp_output_get_qobject(qov);
+    qmp_output_visitor_cleanup(qov);
+    if (!obj) {
+        return;
+    }
+
+    qiv = qmp_input_visitor_new(obj);
+    iv = qmp_input_get_visitor(qiv);
+    visit_type_SocketAddress(iv, p_dest, NULL, &error_abort);
+    qmp_input_visitor_cleanup(qiv);
+    qobject_decref(obj);
+}
diff --git a/util/qemu-thread-posix.c b/util/qemu-thread-posix.c
index ba67cec62..dbd8094fc 100644
--- a/util/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -298,7 +298,16 @@ static inline void futex_wake(QemuEvent *ev, int n)
 
 static inline void futex_wait(QemuEvent *ev, unsigned val)
 {
-    futex(ev, FUTEX_WAIT, (int) val, NULL, NULL, 0);
+    while (futex(ev, FUTEX_WAIT, (int) val, NULL, NULL, 0)) {
+        switch (errno) {
+        case EWOULDBLOCK:
+            return;
+        case EINTR:
+            break; /* get out of switch and retry */
+        default:
+            abort();
+        }
+    }
 }
 #else
 static inline void futex_wake(QemuEvent *ev, int n)
@@ -389,7 +398,7 @@ void qemu_event_wait(QemuEvent *ev)
             /*
              * Leave the event reset and tell qemu_event_set that there
              * are waiters.  No need to retry, because there cannot be
-             * a concurent busy->free transition.  After the CAS, the
+             * a concurrent busy->free transition.  After the CAS, the
              * event will be either set or busy.
              */
             if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
diff --git a/util/qemu-thread-win32.c b/util/qemu-thread-win32.c
index 406b52f91..6cdd553e9 100644
--- a/util/qemu-thread-win32.c
+++ b/util/qemu-thread-win32.c
@@ -238,10 +238,34 @@ void qemu_sem_wait(QemuSemaphore *sem)
     }
 }
 
+/* Wrap a Win32 manual-reset event with a fast userspace path.  The idea
+ * is to reset the Win32 event lazily, as part of a test-reset-test-wait
+ * sequence.  Such a sequence is, indeed, how QemuEvents are used by
+ * RCU and other subsystems!
+ *
+ * Valid transitions:
+ * - free->set, when setting the event
+ * - busy->set, when setting the event, followed by futex_wake
+ * - set->free, when resetting the event
+ * - free->busy, when waiting
+ *
+ * set->busy does not happen (it can be observed from the outside but
+ * it really is set->free->busy).
+ *
+ * busy->free provably cannot happen; to enforce it, the set->free transition
+ * is done with an OR, which becomes a no-op if the event has concurrently
+ * transitioned to free or busy (and is faster than cmpxchg).
+ */
+
+#define EV_SET         0
+#define EV_FREE        1
+#define EV_BUSY       -1
+
 void qemu_event_init(QemuEvent *ev, bool init)
 {
     /* Manual reset.  */
-    ev->event = CreateEvent(NULL, TRUE, init, NULL);
+    ev->event = CreateEvent(NULL, TRUE, TRUE, NULL);
+    ev->value = (init ? EV_SET : EV_FREE);
 }
 
 void qemu_event_destroy(QemuEvent *ev)
@@ -251,17 +275,51 @@ void qemu_event_destroy(QemuEvent *ev)
 
 void qemu_event_set(QemuEvent *ev)
 {
-    SetEvent(ev->event);
+    if (atomic_mb_read(&ev->value) != EV_SET) {
+        if (atomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
+            /* There were waiters, wake them up.  */
+            SetEvent(ev->event);
+        }
+    }
 }
 
 void qemu_event_reset(QemuEvent *ev)
 {
-    ResetEvent(ev->event);
+    if (atomic_mb_read(&ev->value) == EV_SET) {
+        /* If there was a concurrent reset (or even reset+wait),
+         * do nothing.  Otherwise change EV_SET->EV_FREE.
+         */
+        atomic_or(&ev->value, EV_FREE);
+    }
 }
 
 void qemu_event_wait(QemuEvent *ev)
 {
-    WaitForSingleObject(ev->event, INFINITE);
+    unsigned value;
+
+    value = atomic_mb_read(&ev->value);
+    if (value != EV_SET) {
+        if (value == EV_FREE) {
+            /* qemu_event_set is not yet going to call SetEvent, but we are
+             * going to do another check for EV_SET below when setting EV_BUSY.
+             * At that point it is safe to call WaitForSingleObject.
+             */
+            ResetEvent(ev->event);
+
+            /* Tell qemu_event_set that there are waiters.  No need to retry
+             * because there cannot be a concurent busy->free transition.
+             * After the CAS, the event will be either set or busy.
+             */
+            if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
+                value = EV_SET;
+            } else {
+                value = EV_BUSY;
+            }
+        }
+        if (value == EV_BUSY) {
+            WaitForSingleObject(ev->event, INFINITE);
+        }
+    }
 }
 
 struct QemuThreadData {
diff --git a/util/rcu.c b/util/rcu.c
index cdcad678b..8ba304dc4 100644
--- a/util/rcu.c
+++ b/util/rcu.c
@@ -47,7 +47,8 @@
 unsigned long rcu_gp_ctr = RCU_GP_LOCKED;
 
 QemuEvent rcu_gp_event;
-static QemuMutex rcu_gp_lock;
+static QemuMutex rcu_registry_lock;
+static QemuMutex rcu_sync_lock;
 
 /*
  * Check whether a quiescent state was crossed between the beginning of
@@ -66,7 +67,7 @@ static inline int rcu_gp_ongoing(unsigned long *ctr)
  */
 __thread struct rcu_reader_data rcu_reader;
 
-/* Protected by rcu_gp_lock.  */
+/* Protected by rcu_registry_lock.  */
 typedef QLIST_HEAD(, rcu_reader_data) ThreadList;
 static ThreadList registry = QLIST_HEAD_INITIALIZER(registry);
 
@@ -114,10 +115,26 @@ static void wait_for_readers(void)
             break;
         }
 
-        /* Wait for one thread to report a quiescent state and
-         * try again.
+        /* Wait for one thread to report a quiescent state and try again.
+         * Release rcu_registry_lock, so rcu_(un)register_thread() doesn't
+         * wait too much time.
+         *
+         * rcu_register_thread() may add nodes to &registry; it will not
+         * wake up synchronize_rcu, but that is okay because at least another
+         * thread must exit its RCU read-side critical section before
+         * synchronize_rcu is done.  The next iteration of the loop will
+         * move the new thread's rcu_reader from &registry to &qsreaders,
+         * because rcu_gp_ongoing() will return false.
+         *
+         * rcu_unregister_thread() may remove nodes from &qsreaders instead
+         * of &registry if it runs during qemu_event_wait.  That's okay;
+         * the node then will not be added back to &registry by QLIST_SWAP
+         * below.  The invariant is that the node is part of one list when
+         * rcu_registry_lock is released.
          */
+        qemu_mutex_unlock(&rcu_registry_lock);
         qemu_event_wait(&rcu_gp_event);
+        qemu_mutex_lock(&rcu_registry_lock);
     }
 
     /* put back the reader list in the registry */
@@ -126,7 +143,8 @@ static void wait_for_readers(void)
 
 void synchronize_rcu(void)
 {
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_sync_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
 
     if (!QLIST_EMPTY(&registry)) {
         /* In either case, the atomic_mb_set below blocks stores that free
@@ -149,7 +167,8 @@ void synchronize_rcu(void)
         wait_for_readers();
     }
 
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
+    qemu_mutex_unlock(&rcu_sync_lock);
 }
 
 
@@ -273,23 +292,24 @@ void call_rcu1(struct rcu_head *node, void (*func)(struct rcu_head *node))
 void rcu_register_thread(void)
 {
     assert(rcu_reader.ctr == 0);
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
     QLIST_INSERT_HEAD(&registry, &rcu_reader, node);
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
 }
 
 void rcu_unregister_thread(void)
 {
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
     QLIST_REMOVE(&rcu_reader, node);
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
 }
 
 static void rcu_init_complete(void)
 {
     QemuThread thread;
 
-    qemu_mutex_init(&rcu_gp_lock);
+    qemu_mutex_init(&rcu_registry_lock);
+    qemu_mutex_init(&rcu_sync_lock);
     qemu_event_init(&rcu_gp_event, true);
 
     qemu_event_init(&rcu_call_ready_event, false);
@@ -306,12 +326,14 @@ static void rcu_init_complete(void)
 #ifdef CONFIG_POSIX
 static void rcu_init_lock(void)
 {
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_sync_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
 }
 
 static void rcu_init_unlock(void)
 {
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
+    qemu_mutex_unlock(&rcu_sync_lock);
 }
 #endif
 
diff --git a/util/throttle.c b/util/throttle.c
index 706c13111..1113671ec 100644
--- a/util/throttle.c
+++ b/util/throttle.c
@@ -300,6 +300,21 @@ bool throttle_is_valid(ThrottleConfig *cfg)
     return !invalid;
 }
 
+/* check if bps_max/iops_max is used without bps/iops
+ * @cfg: the throttling configuration to inspect
+ */
+bool throttle_max_is_missing_limit(ThrottleConfig *cfg)
+{
+    int i;
+
+    for (i = 0; i < BUCKETS_COUNT; i++) {
+        if (cfg->buckets[i].max && !cfg->buckets[i].avg) {
+            return true;
+        }
+    }
+    return false;
+}
+
 /* fix bucket parameters */
 static void throttle_fix_bucket(LeakyBucket *bkt)
 {
diff --git a/util/timed-average.c b/util/timed-average.c
new file mode 100644
index 000000000..a2dfb4834
--- /dev/null
+++ b/util/timed-average.c
@@ -0,0 +1,231 @@
+/*
+ * QEMU timed average computation
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free sofware: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Sofware Foundation, either version 2 of the License, or
+ * (at your option) version 3 or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string.h>
+
+#include "qemu/timed-average.h"
+
+/* This module computes an average of a set of values within a time
+ * window.
+ *
+ * Algorithm:
+ *
+ * - Create two windows with a certain expiration period, and
+ *   offsetted by period / 2.
+ * - Each time you want to account a new value, do it in both windows.
+ * - The minimum / maximum / average values are always returned from
+ *   the oldest window.
+ *
+ * Example:
+ *
+ *        t=0          |t=0.5           |t=1          |t=1.5            |t=2
+ *        wnd0: [0,0.5)|wnd0: [0.5,1.5) |             |wnd0: [1.5,2.5)  |
+ *        wnd1: [0,1)  |                |wnd1: [1,2)  |                 |
+ *
+ * Values are returned from:
+ *
+ *        wnd0---------|wnd1------------|wnd0---------|wnd1-------------|
+ */
+
+/* Update the expiration of a time window
+ *
+ * @w:      the window used
+ * @now:    the current time in nanoseconds
+ * @period: the expiration period in nanoseconds
+ */
+static void update_expiration(TimedAverageWindow *w, int64_t now,
+                              int64_t period)
+{
+    /* time elapsed since the last theoretical expiration */
+    int64_t elapsed = (now - w->expiration) % period;
+    /* time remaininging until the next expiration */
+    int64_t remaining = period - elapsed;
+    /* compute expiration */
+    w->expiration = now + remaining;
+}
+
+/* Reset a window
+ *
+ * @w: the window to reset
+ */
+static void window_reset(TimedAverageWindow *w)
+{
+    w->min = UINT64_MAX;
+    w->max = 0;
+    w->sum = 0;
+    w->count = 0;
+}
+
+/* Get the current window (that is, the one with the earliest
+ * expiration time).
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: a pointer to the current window
+ */
+static TimedAverageWindow *current_window(TimedAverage *ta)
+{
+     return &ta->windows[ta->current];
+}
+
+/* Initialize a TimedAverage structure
+ *
+ * @ta:         the TimedAverage structure
+ * @clock_type: the type of clock to use
+ * @period:     the time window period in nanoseconds
+ */
+void timed_average_init(TimedAverage *ta, QEMUClockType clock_type,
+                        uint64_t period)
+{
+    int64_t now = qemu_clock_get_ns(clock_type);
+
+    /* Returned values are from the oldest window, so they belong to
+     * the interval [ta->period/2,ta->period). By adjusting the
+     * requested period by 4/3, we guarantee that they're in the
+     * interval [2/3 period,4/3 period), closer to the requested
+     * period on average */
+    ta->period = (uint64_t) period * 4 / 3;
+    ta->clock_type = clock_type;
+    ta->current = 0;
+
+    window_reset(&ta->windows[0]);
+    window_reset(&ta->windows[1]);
+
+    /* Both windows are offsetted by half a period */
+    ta->windows[0].expiration = now + ta->period / 2;
+    ta->windows[1].expiration = now + ta->period;
+}
+
+/* Check if the time windows have expired, updating their counters and
+ * expiration time if that's the case.
+ *
+ * @ta: the TimedAverage structure
+ * @elapsed: if non-NULL, the elapsed time (in ns) within the current
+ *           window will be stored here
+ */
+static void check_expirations(TimedAverage *ta, uint64_t *elapsed)
+{
+    int64_t now = qemu_clock_get_ns(ta->clock_type);
+    int i;
+
+    assert(ta->period != 0);
+
+    /* Check if the windows have expired */
+    for (i = 0; i < 2; i++) {
+        TimedAverageWindow *w = &ta->windows[i];
+        if (w->expiration <= now) {
+            window_reset(w);
+            update_expiration(w, now, ta->period);
+        }
+    }
+
+    /* Make ta->current point to the oldest window */
+    if (ta->windows[0].expiration < ta->windows[1].expiration) {
+        ta->current = 0;
+    } else {
+        ta->current = 1;
+    }
+
+    /* Calculate the elapsed time within the current window */
+    if (elapsed) {
+        int64_t remaining = ta->windows[ta->current].expiration - now;
+        *elapsed = ta->period - remaining;
+    }
+}
+
+/* Account a value
+ *
+ * @ta:    the TimedAverage structure
+ * @value: the value to account
+ */
+void timed_average_account(TimedAverage *ta, uint64_t value)
+{
+    int i;
+    check_expirations(ta, NULL);
+
+    /* Do the accounting in both windows at the same time */
+    for (i = 0; i < 2; i++) {
+        TimedAverageWindow *w = &ta->windows[i];
+
+        w->sum += value;
+        w->count++;
+
+        if (value < w->min) {
+            w->min = value;
+        }
+
+        if (value > w->max) {
+            w->max = value;
+        }
+    }
+}
+
+/* Get the minimum value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the minimum value
+ */
+uint64_t timed_average_min(TimedAverage *ta)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, NULL);
+    w = current_window(ta);
+    return w->min < UINT64_MAX ? w->min : 0;
+}
+
+/* Get the average value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the average value
+ */
+uint64_t timed_average_avg(TimedAverage *ta)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, NULL);
+    w = current_window(ta);
+    return w->count > 0 ? w->sum / w->count : 0;
+}
+
+/* Get the maximum value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the maximum value
+ */
+uint64_t timed_average_max(TimedAverage *ta)
+{
+    check_expirations(ta, NULL);
+    return current_window(ta)->max;
+}
+
+/* Get the sum of all accounted values
+ * @ta:      the TimedAverage structure
+ * @elapsed: if non-NULL, the elapsed time (in ns) will be stored here
+ * @ret:     the sum of all accounted values
+ */
+uint64_t timed_average_sum(TimedAverage *ta, uint64_t *elapsed)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, elapsed);
+    w = current_window(ta);
+    return w->sum;
+}
diff --git a/vl.c b/vl.c
index 6b739348c..873d26547 100644
--- a/vl.c
+++ b/vl.c
@@ -68,7 +68,7 @@ int main(int argc, char **argv)
 #include "hw/isa/isa.h"
 #include "hw/bt.h"
 #include "sysemu/watchdog.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/xen/xen.h"
 #include "hw/qdev.h"
 #include "hw/loader.h"
@@ -122,6 +122,8 @@ int main(int argc, char **argv)
 #include "qapi-event.h"
 #include "exec/semihost.h"
 #include "crypto/init.h"
+#include "sysemu/replay.h"
+#include "qapi/qmp/qerror.h"
 
 #define MAX_VIRTIO_CONSOLES 1
 #define MAX_SCLP_CONSOLES 1
@@ -474,6 +476,12 @@ static QemuOptsList qemu_icount_opts = {
         }, {
             .name = "sleep",
             .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "rr",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "rrfile",
+            .type = QEMU_OPT_STRING,
         },
         { /* end of list */ }
     },
@@ -512,6 +520,10 @@ static QemuOptsList qemu_fw_cfg_opts = {
             .type = QEMU_OPT_STRING,
             .help = "Sets the name of the file from which\n"
                     "the fw_cfg blob will be loaded",
+        }, {
+            .name = "string",
+            .type = QEMU_OPT_STRING,
+            .help = "Sets content of the blob to be inserted from a string",
         },
         { /* end of list */ }
     },
@@ -534,10 +546,8 @@ const char *qemu_get_vm_name(void)
 
 static void res_free(void)
 {
-    if (boot_splash_filedata != NULL) {
-        g_free(boot_splash_filedata);
-        boot_splash_filedata = NULL;
-    }
+    g_free(boot_splash_filedata);
+    boot_splash_filedata = NULL;
 }
 
 static int default_driver_check(void *opaque, QemuOpts *opts, Error **errp)
@@ -582,6 +592,7 @@ static const RunStateTransition runstate_transitions_def[] = {
     { RUN_STATE_INMIGRATE, RUN_STATE_SUSPENDED },
     { RUN_STATE_INMIGRATE, RUN_STATE_WATCHDOG },
     { RUN_STATE_INMIGRATE, RUN_STATE_GUEST_PANICKED },
+    { RUN_STATE_INMIGRATE, RUN_STATE_FINISH_MIGRATE },
 
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_PAUSED },
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_FINISH_MIGRATE },
@@ -671,9 +682,9 @@ void runstate_set(RunState new_state)
     assert(new_state < RUN_STATE_MAX);
 
     if (!runstate_valid_transitions[current_run_state][new_state]) {
-        fprintf(stderr, "ERROR: invalid runstate transition: '%s' -> '%s'\n",
-                RunState_lookup[current_run_state],
-                RunState_lookup[new_state]);
+        error_report("invalid runstate transition: '%s' -> '%s'",
+                     RunState_lookup[current_run_state],
+                     RunState_lookup[new_state]);
         abort();
     }
     trace_runstate_set(new_state);
@@ -825,8 +836,9 @@ static void configure_rtc_date_offset(const char *startdate, int legacy)
         rtc_start_date = mktimegm(&tm);
         if (rtc_start_date == -1) {
         date_fail:
-            fprintf(stderr, "Invalid date format. Valid formats are:\n"
-                            "'2006-06-17T16:01:21' or '2006-06-17'\n");
+            error_report("invalid date format");
+            error_printf("valid formats: "
+                         "'2006-06-17T16:01:21' or '2006-06-17'\n");
             exit(1);
         }
         rtc_date_offset = qemu_time() - rtc_start_date;
@@ -842,7 +854,11 @@ static void configure_rtc(QemuOpts *opts)
         if (!strcmp(value, "utc")) {
             rtc_utc = 1;
         } else if (!strcmp(value, "localtime")) {
+            Error *blocker = NULL;
             rtc_utc = 0;
+            error_setg(&blocker, QERR_REPLAY_NOT_SUPPORTED,
+                      "-rtc base=localtime");
+            replay_add_blocker(blocker);
         } else {
             configure_rtc_date_offset(value, 0);
         }
@@ -856,7 +872,7 @@ static void configure_rtc(QemuOpts *opts)
         } else if (!strcmp(value, "vm")) {
             rtc_clock = QEMU_CLOCK_VIRTUAL;
         } else {
-            fprintf(stderr, "qemu: invalid option value '%s'\n", value);
+            error_report("invalid option value '%s'", value);
             exit(1);
         }
     }
@@ -876,7 +892,7 @@ static void configure_rtc(QemuOpts *opts)
         } else if (!strcmp(value, "none")) {
             /* discard is default */
         } else {
-            fprintf(stderr, "qemu: invalid option value '%s'\n", value);
+            error_report("invalid option value '%s'", value);
             exit(1);
         }
     }
@@ -902,7 +918,7 @@ static int bt_hci_parse(const char *str)
     bdaddr_t bdaddr;
 
     if (nb_hcis >= MAX_NICS) {
-        fprintf(stderr, "qemu: Too many bluetooth HCIs (max %i).\n", MAX_NICS);
+        error_report("too many bluetooth HCIs (max %i)", MAX_NICS);
         return -1;
     }
 
@@ -928,8 +944,8 @@ static void bt_vhci_add(int vlan_id)
     struct bt_scatternet_s *vlan = qemu_find_bt_vlan(vlan_id);
 
     if (!vlan->slave)
-        fprintf(stderr, "qemu: warning: adding a VHCI to "
-                        "an empty scatternet %i\n", vlan_id);
+        error_report("warning: adding a VHCI to an empty scatternet %i",
+                     vlan_id);
 
     bt_vhci_init(bt_new_hci(vlan));
 }
@@ -947,7 +963,7 @@ static struct bt_device_s *bt_device_add(const char *opt)
     if (endp) {
         vlan_id = strtol(endp + 6, &endp, 0);
         if (*endp) {
-            fprintf(stderr, "qemu: unrecognised bluetooth vlan Id\n");
+            error_report("unrecognised bluetooth vlan Id");
             return 0;
         }
     }
@@ -955,13 +971,13 @@ static struct bt_device_s *bt_device_add(const char *opt)
     vlan = qemu_find_bt_vlan(vlan_id);
 
     if (!vlan->slave)
-        fprintf(stderr, "qemu: warning: adding a slave device to "
-                        "an empty scatternet %i\n", vlan_id);
+        error_report("warning: adding a slave device to an empty scatternet %i",
+                     vlan_id);
 
     if (!strcmp(devname, "keyboard"))
         return bt_keyboard_init(vlan);
 
-    fprintf(stderr, "qemu: unsupported bluetooth device `%s'\n", devname);
+    error_report("unsupported bluetooth device '%s'", devname);
     return 0;
 }
 
@@ -984,11 +1000,11 @@ static int bt_parse(const char *opt)
                 if (strstart(endp, ",vlan=", &p)) {
                     vlan = strtol(p, (char **) &endp, 0);
                     if (*endp) {
-                        fprintf(stderr, "qemu: bad scatternet '%s'\n", p);
+                        error_report("bad scatternet '%s'", p);
                         return 1;
                     }
                 } else {
-                    fprintf(stderr, "qemu: bad parameter '%s'\n", endp + 1);
+                    error_report("bad parameter '%s'", endp + 1);
                     return 1;
                 }
             } else
@@ -1000,7 +1016,7 @@ static int bt_parse(const char *opt)
     } else if (strstart(opt, "device:", &endp))
         return !bt_device_add(endp);
 
-    fprintf(stderr, "qemu: bad bluetooth parameter '%s'\n", opt);
+    error_report("bad bluetooth parameter '%s'", opt);
     return 1;
 }
 
@@ -1015,8 +1031,7 @@ static int parse_sandbox(void *opaque, QemuOpts *opts, Error **errp)
             return -1;
         }
 #else
-        error_report("sandboxing request but seccomp is not compiled "
-                     "into this build");
+        error_report("seccomp support is disabled");
         return -1;
 #endif
     }
@@ -1097,7 +1112,7 @@ static int parse_add_fd(void *opaque, QemuOpts *opts, Error **errp)
     }
 #endif
     if (dupfd == -1) {
-        error_report("Error duplicating fd: %s", strerror(errno));
+        error_report("error duplicating fd: %s", strerror(errno));
         return -1;
     }
 
@@ -1217,14 +1232,21 @@ static void smp_parse(QemuOpts *opts)
         } else if (threads == 0) {
             threads = cpus / (cores * sockets);
         } else if (sockets * cores * threads < cpus) {
-            fprintf(stderr, "cpu topology: error: "
-                    "sockets (%u) * cores (%u) * threads (%u) < "
-                    "smp_cpus (%u)\n",
-                    sockets, cores, threads, cpus);
+            error_report("cpu topology: "
+                         "sockets (%u) * cores (%u) * threads (%u) < "
+                         "smp_cpus (%u)",
+                         sockets, cores, threads, cpus);
             exit(1);
         }
 
-        max_cpus = qemu_opt_get_number(opts, "maxcpus", 0);
+        max_cpus = qemu_opt_get_number(opts, "maxcpus", cpus);
+        if (sockets * cores * threads > max_cpus) {
+            error_report("cpu topology: "
+                         "sockets (%u) * cores (%u) * threads (%u) > "
+                         "maxcpus (%u)",
+                         sockets, cores, threads, max_cpus);
+            exit(1);
+        }
 
         smp_cpus = cpus;
         smp_cores = cores > 0 ? cores : 1;
@@ -1237,21 +1259,26 @@ static void smp_parse(QemuOpts *opts)
     }
 
     if (max_cpus > MAX_CPUMASK_BITS) {
-        fprintf(stderr, "Unsupported number of maxcpus\n");
+        error_report("unsupported number of maxcpus");
         exit(1);
     }
     if (max_cpus < smp_cpus) {
-        fprintf(stderr, "maxcpus must be equal to or greater than smp\n");
+        error_report("maxcpus must be equal to or greater than smp");
         exit(1);
     }
 
+    if (smp_cpus > 1 || smp_cores > 1 || smp_threads > 1) {
+        Error *blocker = NULL;
+        error_setg(&blocker, QERR_REPLAY_NOT_SUPPORTED, "smp");
+        replay_add_blocker(blocker);
+    }
 }
 
 static void realtime_init(void)
 {
     if (enable_mlock) {
         if (os_mlock() < 0) {
-            fprintf(stderr, "qemu: locking memory failed\n");
+            error_report("locking memory failed");
             exit(1);
         }
     }
@@ -1338,6 +1365,13 @@ static inline void semihosting_arg_fallback(const char *file, const char *cmd)
     }
 }
 
+/* Now we still need this for compatibility with XEN. */
+bool has_igd_gfx_passthru;
+static void igd_gfx_passthru(void)
+{
+    has_igd_gfx_passthru = current_machine->igd_gfx_passthru;
+}
+
 /***********************************************************/
 /* USB devices */
 
@@ -1398,7 +1432,7 @@ static int usb_parse(const char *cmdline)
     int r;
     r = usb_device_add(cmdline);
     if (r < 0) {
-        fprintf(stderr, "qemu: could not add USB device '%s'\n", cmdline);
+        error_report("could not add USB device '%s'", cmdline);
     }
     return r;
 }
@@ -1424,43 +1458,6 @@ void hmp_usb_del(Monitor *mon, const QDict *qdict)
 
 MachineState *current_machine;
 
-/*
- * Transitional class registration/init used for converting from
- * legacy QEMUMachine to MachineClass.
- */
-static void qemu_machine_class_init(ObjectClass *oc, void *data)
-{
-    MachineClass *mc = MACHINE_CLASS(oc);
-    QEMUMachine *qm = data;
-    mc->name = qm->name;
-    mc->desc = qm->desc;
-    mc->init = qm->init;
-    mc->kvm_type = qm->kvm_type;
-    mc->block_default_type = qm->block_default_type;
-    mc->max_cpus = qm->max_cpus;
-    mc->no_sdcard = qm->no_sdcard;
-    mc->has_dynamic_sysbus = qm->has_dynamic_sysbus;
-    mc->is_default = qm->is_default;
-    mc->default_machine_opts = qm->default_machine_opts;
-    mc->default_boot_order = qm->default_boot_order;
-}
-
-int qemu_register_machine(QEMUMachine *m)
-{
-    char *name = g_strconcat(m->name, TYPE_MACHINE_SUFFIX, NULL);
-    TypeInfo ti = {
-        .name       = name,
-        .parent     = TYPE_MACHINE,
-        .class_init = qemu_machine_class_init,
-        .class_data = (void *)m,
-    };
-
-    type_register(&ti);
-    g_free(name);
-
-    return 0;
-}
-
 static MachineClass *find_machine(const char *name)
 {
     GSList *el, *machines = object_class_get_list(TYPE_MACHINE, false);
@@ -1539,12 +1536,14 @@ MachineInfoList *qmp_query_machines(Error **errp)
 static int machine_help_func(QemuOpts *opts, MachineState *machine)
 {
     ObjectProperty *prop;
+    ObjectPropertyIterator *iter;
 
     if (!qemu_opt_has_help_opt(opts)) {
         return 0;
     }
 
-    QTAILQ_FOREACH(prop, &OBJECT(machine)->properties, node) {
+    iter = object_property_iter_init(OBJECT(machine));
+    while ((prop = object_property_iter_next(iter))) {
         if (!prop->set) {
             continue;
         }
@@ -1557,6 +1556,7 @@ static int machine_help_func(QemuOpts *opts, MachineState *machine)
             error_printf("\n");
         }
     }
+    object_property_iter_free(iter);
 
     return 1;
 }
@@ -1645,14 +1645,14 @@ static int qemu_shutdown_requested(void)
 static void qemu_kill_report(void)
 {
     if (!qtest_driver() && shutdown_signal != -1) {
-        fprintf(stderr, "qemu: terminating on signal %d", shutdown_signal);
         if (shutdown_pid == 0) {
             /* This happens for eg ^C at the terminal, so it's worth
              * avoiding printing an odd message in that case.
              */
-            fputc('\n', stderr);
+            error_report("terminating on signal %d", shutdown_signal);
         } else {
-            fprintf(stderr, " from pid " FMT_pid "\n", shutdown_pid);
+            error_report("terminating on signal %d from pid " FMT_pid,
+                         shutdown_signal, shutdown_pid);
         }
         shutdown_signal = -1;
     }
@@ -1661,15 +1661,21 @@ static void qemu_kill_report(void)
 static int qemu_reset_requested(void)
 {
     int r = reset_requested;
-    reset_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_RESET_REQUESTED)) {
+        reset_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static int qemu_suspend_requested(void)
 {
     int r = suspend_requested;
-    suspend_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_SUSPEND_REQUESTED)) {
+        suspend_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static WakeupReason qemu_wakeup_requested(void)
@@ -1740,6 +1746,15 @@ void qemu_system_reset(bool report)
     cpu_synchronize_all_post_reset();
 }
 
+void qemu_system_guest_panicked(void)
+{
+    if (current_cpu) {
+        current_cpu->crash_occurred = true;
+    }
+    qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE, &error_abort);
+    vm_stop(RUN_STATE_GUEST_PANICKED);
+}
+
 void qemu_system_reset_request(void)
 {
     if (no_reboot) {
@@ -1808,12 +1823,18 @@ void qemu_system_killed(int signal, pid_t pid)
     shutdown_signal = signal;
     shutdown_pid = pid;
     no_shutdown = 0;
-    qemu_system_shutdown_request();
+
+    /* Cannot call qemu_system_shutdown_request directly because
+     * we are in a signal handler.
+     */
+    shutdown_requested = 1;
+    qemu_notify_event();
 }
 
 void qemu_system_shutdown_request(void)
 {
     trace_qemu_system_shutdown_request();
+    replay_shutdown_request();
     shutdown_requested = 1;
     qemu_notify_event();
 }
@@ -1992,28 +2013,28 @@ static void select_vgahw (const char *p)
         if (vga_available()) {
             vga_interface_type = VGA_STD;
         } else {
-            fprintf(stderr, "Error: standard VGA not available\n");
+            error_report("standard VGA not available");
             exit(0);
         }
     } else if (strstart(p, "cirrus", &opts)) {
         if (cirrus_vga_available()) {
             vga_interface_type = VGA_CIRRUS;
         } else {
-            fprintf(stderr, "Error: Cirrus VGA not available\n");
+            error_report("Cirrus VGA not available");
             exit(0);
         }
     } else if (strstart(p, "vmware", &opts)) {
         if (vmware_vga_available()) {
             vga_interface_type = VGA_VMWARE;
         } else {
-            fprintf(stderr, "Error: VMWare SVGA not available\n");
+            error_report("VMWare SVGA not available");
             exit(0);
         }
     } else if (strstart(p, "virtio", &opts)) {
         if (virtio_vga_available()) {
             vga_interface_type = VGA_VIRTIO;
         } else {
-            fprintf(stderr, "Error: Virtio VGA not available\n");
+            error_report("Virtio VGA not available");
             exit(0);
         }
     } else if (strstart(p, "xenfb", &opts)) {
@@ -2022,26 +2043,26 @@ static void select_vgahw (const char *p)
         if (qxl_vga_available()) {
             vga_interface_type = VGA_QXL;
         } else {
-            fprintf(stderr, "Error: QXL VGA not available\n");
+            error_report("QXL VGA not available");
             exit(0);
         }
     } else if (strstart(p, "tcx", &opts)) {
         if (tcx_vga_available()) {
             vga_interface_type = VGA_TCX;
         } else {
-            fprintf(stderr, "Error: TCX framebuffer not available\n");
+            error_report("TCX framebuffer not available");
             exit(0);
         }
     } else if (strstart(p, "cg3", &opts)) {
         if (cg3_vga_available()) {
             vga_interface_type = VGA_CG3;
         } else {
-            fprintf(stderr, "Error: CG3 framebuffer not available\n");
+            error_report("CG3 framebuffer not available");
             exit(0);
         }
     } else if (!strstart(p, "none", &opts)) {
     invalid_vga:
-        fprintf(stderr, "Unknown vga type: %s\n", p);
+        error_report("unknown vga type: %s", p);
         exit(1);
     }
     while (*opts) {
@@ -2117,7 +2138,7 @@ static DisplayType select_display(const char *p)
                 }
             } else {
             invalid_sdl_args:
-                error_report("Invalid SDL option string");
+                error_report("invalid SDL option string");
                 exit(1);
             }
             opts = nextopt;
@@ -2146,7 +2167,7 @@ static DisplayType select_display(const char *p)
 #ifdef CONFIG_CURSES
         display = DT_CURSES;
 #else
-        error_report("Curses support is disabled");
+        error_report("curses support is disabled");
         exit(1);
 #endif
     } else if (strstart(p, "gtk", &opts)) {
@@ -2175,7 +2196,7 @@ static DisplayType select_display(const char *p)
                 }
             } else {
             invalid_gtk_args:
-                error_report("Invalid GTK option string");
+                error_report("invalid GTK option string");
                 exit(1);
             }
             opts = nextopt;
@@ -2187,7 +2208,7 @@ static DisplayType select_display(const char *p)
     } else if (strstart(p, "none", &opts)) {
         display = DT_NONE;
     } else {
-        error_report("Unknown display type");
+        error_report("unknown display type");
         exit(1);
     }
 
@@ -2255,11 +2276,16 @@ char *qemu_find_file(int type, const char *name)
     return NULL;
 }
 
+static inline bool nonempty_str(const char *str)
+{
+    return str && *str;
+}
+
 static int parse_fw_cfg(void *opaque, QemuOpts *opts, Error **errp)
 {
     gchar *buf;
     size_t size;
-    const char *name, *file;
+    const char *name, *file, *str;
 
     if (opaque == NULL) {
         error_report("fw_cfg device not available");
@@ -2267,8 +2293,15 @@ static int parse_fw_cfg(void *opaque, QemuOpts *opts, Error **errp)
     }
     name = qemu_opt_get(opts, "name");
     file = qemu_opt_get(opts, "file");
-    if (name == NULL || *name == '\0' || file == NULL || *file == '\0') {
-        error_report("invalid argument value");
+    str = qemu_opt_get(opts, "string");
+
+    /* we need name and either a file or the content string */
+    if (!(nonempty_str(name) && (nonempty_str(file) || nonempty_str(str)))) {
+        error_report("invalid argument(s)");
+        return -1;
+    }
+    if (nonempty_str(file) && nonempty_str(str)) {
+        error_report("file and string are mutually exclusive");
         return -1;
     }
     if (strlen(name) > FW_CFG_MAX_FILE_PATH - 1) {
@@ -2276,12 +2309,17 @@ static int parse_fw_cfg(void *opaque, QemuOpts *opts, Error **errp)
         return -1;
     }
     if (strncmp(name, "opt/", 4) != 0) {
-        error_report("WARNING: externally provided fw_cfg item names "
-                     "should be prefixed with \"opt/\"!");
+        error_report("warning: externally provided fw_cfg item names "
+                     "should be prefixed with \"opt/\"");
     }
-    if (!g_file_get_contents(file, &buf, &size, NULL)) {
-        error_report("can't load %s", file);
-        return -1;
+    if (nonempty_str(str)) {
+        size = strlen(str); /* NUL terminator NOT included in fw_cfg blob */
+        buf = g_memdup(str, size);
+    } else {
+        if (!g_file_get_contents(file, &buf, &size, NULL)) {
+            error_report("can't load %s", file);
+            return -1;
+        }
     }
     fw_cfg_add_file((FWCfgState *)opaque, name, buf, size);
     return 0;
@@ -2344,7 +2382,7 @@ static int mon_init_func(void *opaque, QemuOpts *opts, Error **errp)
     } else if (strcmp(mode, "control") == 0) {
         flags = MONITOR_USE_CONTROL;
     } else {
-        fprintf(stderr, "unknown monitor mode \"%s\"\n", mode);
+        error_report("unknown monitor mode \"%s\"", mode);
         exit(1);
     }
 
@@ -2357,7 +2395,7 @@ static int mon_init_func(void *opaque, QemuOpts *opts, Error **errp)
     chardev = qemu_opt_get(opts, "chardev");
     chr = qemu_chr_find(chardev);
     if (chr == NULL) {
-        fprintf(stderr, "chardev \"%s\" not found\n", chardev);
+        error_report("chardev \"%s\" not found", chardev);
         exit(1);
     }
 
@@ -2385,7 +2423,7 @@ static void monitor_parse(const char *optarg, const char *mode, bool pretty)
         }
         opts = qemu_chr_parse_compat(label, optarg);
         if (!opts) {
-            fprintf(stderr, "parse error: %s\n", optarg);
+            error_report("parse error: %s", optarg);
             exit(1);
         }
     }
@@ -2459,14 +2497,14 @@ static int serial_parse(const char *devname)
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_SERIAL_PORTS) {
-        fprintf(stderr, "qemu: too many serial ports\n");
+        error_report("too many serial ports");
         exit(1);
     }
     snprintf(label, sizeof(label), "serial%d", index);
     serial_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!serial_hds[index]) {
-        fprintf(stderr, "qemu: could not connect serial device"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect serial device"
+                     " to character backend '%s'", devname);
         return -1;
     }
     index++;
@@ -2481,14 +2519,14 @@ static int parallel_parse(const char *devname)
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_PARALLEL_PORTS) {
-        fprintf(stderr, "qemu: too many parallel ports\n");
+        error_report("too many parallel ports");
         exit(1);
     }
     snprintf(label, sizeof(label), "parallel%d", index);
     parallel_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!parallel_hds[index]) {
-        fprintf(stderr, "qemu: could not connect parallel device"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect parallel device"
+                     " to character backend '%s'", devname);
         return -1;
     }
     index++;
@@ -2505,7 +2543,7 @@ static int virtcon_parse(const char *devname)
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_VIRTIO_CONSOLES) {
-        fprintf(stderr, "qemu: too many virtio consoles\n");
+        error_report("too many virtio consoles");
         exit(1);
     }
 
@@ -2522,8 +2560,8 @@ static int virtcon_parse(const char *devname)
     snprintf(label, sizeof(label), "virtcon%d", index);
     virtcon_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!virtcon_hds[index]) {
-        fprintf(stderr, "qemu: could not connect virtio console"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect virtio console"
+                     " to character backend '%s'", devname);
         return -1;
     }
     qemu_opt_set(dev_opts, "chardev", label, &error_abort);
@@ -2543,7 +2581,7 @@ static int sclp_parse(const char *devname)
         return 0;
     }
     if (index == MAX_SCLP_CONSOLES) {
-        fprintf(stderr, "qemu: too many sclp consoles\n");
+        error_report("too many sclp consoles");
         exit(1);
     }
 
@@ -2555,8 +2593,8 @@ static int sclp_parse(const char *devname)
     snprintf(label, sizeof(label), "sclpcon%d", index);
     sclp_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!sclp_hds[index]) {
-        fprintf(stderr, "qemu: could not connect sclp console"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect sclp console"
+                     " to character backend '%s'", devname);
         return -1;
     }
     qemu_opt_set(dev_opts, "chardev", label, &error_abort);
@@ -2574,7 +2612,7 @@ static int debugcon_parse(const char *devname)
     }
     opts = qemu_opts_create(qemu_find_opts("device"), "debugcon", 1, NULL);
     if (!opts) {
-        fprintf(stderr, "qemu: already have a debugcon device\n");
+        error_report("already have a debugcon device");
         exit(1);
     }
     qemu_opt_set(opts, "driver", "isa-debugcon", &error_abort);
@@ -2629,8 +2667,8 @@ static gint machine_class_cmp(gconstpointer a, gconstpointer b)
         return mc;
     }
     if (name && !is_help_option(name)) {
-        error_report("Unsupported machine type");
-        error_printf("Use -machine help to list supported machines!\n");
+        error_report("unsupported machine type");
+        error_printf("Use -machine help to list supported machines\n");
     } else {
         printf("Supported machines are:\n");
         machines = g_slist_sort(machines, machine_class_cmp);
@@ -2719,6 +2757,31 @@ static const QEMUOption *lookup_opt(int argc, char **argv,
     return popt;
 }
 
+static void set_machine_options(MachineClass **machine_class)
+{
+    const char *optarg;
+    QemuOpts *opts;
+    Location loc;
+
+    loc_push_none(&loc);
+
+    opts = qemu_get_machine_opts();
+    qemu_opts_loc_restore(opts);
+
+    optarg = qemu_opt_get(opts, "type");
+    if (optarg) {
+        *machine_class = machine_parse(optarg);
+    }
+
+    if (*machine_class == NULL) {
+        error_report("No machine specified, and there is no default");
+        error_printf("Use -machine help to list supported machines\n");
+        exit(1);
+    }
+
+    loc_pop(&loc);
+}
+
 static int machine_set_property(void *opaque,
                                 const char *name, const char *value,
                                 Error **errp)
@@ -2763,13 +2826,23 @@ static bool object_create_initial(const char *type)
     if (g_str_equal(type, "rng-egd")) {
         return false;
     }
+
+    /*
+     * return false for concrete netfilters since
+     * they depend on netdevs already existing
+     */
+    if (g_str_equal(type, "filter-buffer") ||
+        g_str_equal(type, "filter-dump")) {
+        return false;
+    }
+
     return true;
 }
 
 
 /*
  * The remainder of object creation happens after the
- * creation of chardev, fsdev and device data types.
+ * creation of chardev, fsdev, net clients and device data types.
  */
 static bool object_create_delayed(const char *type)
 {
@@ -2995,8 +3068,7 @@ int main(int argc, char **argv, char **envp)
     runstate_init();
 
     if (qcrypto_init(&err) < 0) {
-        fprintf(stderr, "Cannot initialize crypto: %s\n",
-                error_get_pretty(err));
+        error_report("cannot initialize crypto: %s", error_get_pretty(err));
         exit(1);
     }
     rtc_clock = QEMU_CLOCK_HOST;
@@ -3154,7 +3226,7 @@ int main(int argc, char **argv, char **envp)
                         }
                     } else if (*p != '\0') {
                     chs_fail:
-                        fprintf(stderr, "qemu: invalid physical CHS format\n");
+                        error_report("invalid physical CHS format");
                         exit(1);
                     }
                     if (hda_opts != NULL) {
@@ -3197,7 +3269,7 @@ int main(int argc, char **argv, char **envp)
 #ifdef CONFIG_CURSES
                 display_type = DT_CURSES;
 #else
-                fprintf(stderr, "Curses support is disabled\n");
+                error_report("curses support is disabled");
                 exit(1);
 #endif
                 break;
@@ -3208,8 +3280,7 @@ int main(int argc, char **argv, char **envp)
                 graphic_rotate = strtol(optarg, (char **) &optarg, 10);
                 if (graphic_rotate != 0 && graphic_rotate != 90 &&
                     graphic_rotate != 180 && graphic_rotate != 270) {
-                    fprintf(stderr,
-                        "qemu: only 90, 180, 270 deg rotation is available\n");
+                    error_report("only 90, 180, 270 deg rotation is available");
                     exit(1);
                 }
                 break;
@@ -3360,7 +3431,7 @@ int main(int argc, char **argv, char **envp)
                     w = strtol(p, (char **)&p, 10);
                     if (w <= 0) {
                     graphic_error:
-                        fprintf(stderr, "qemu: invalid resolution or depth\n");
+                        error_report("invalid resolution or depth");
                         exit(1);
                     }
                     if (*p != 'x')
@@ -3426,7 +3497,7 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_fsdev:
                 olist = qemu_find_opts("fsdev");
                 if (!olist) {
-                    fprintf(stderr, "fsdev is not supported by this qemu build.\n");
+                    error_report("fsdev support is disabled");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(olist, optarg, true);
@@ -3441,7 +3512,7 @@ int main(int argc, char **argv, char **envp)
 
                 olist = qemu_find_opts("virtfs");
                 if (!olist) {
-                    fprintf(stderr, "virtfs is not supported by this qemu build.\n");
+                    error_report("virtfs support is disabled");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(olist, optarg, true);
@@ -3451,15 +3522,15 @@ int main(int argc, char **argv, char **envp)
 
                 if (qemu_opt_get(opts, "fsdriver") == NULL ||
                     qemu_opt_get(opts, "mount_tag") == NULL) {
-                    fprintf(stderr, "Usage: -virtfs fsdriver,mount_tag=tag.\n");
+                    error_report("Usage: -virtfs fsdriver,mount_tag=tag");
                     exit(1);
                 }
                 fsdev = qemu_opts_create(qemu_find_opts("fsdev"),
                                          qemu_opt_get(opts, "mount_tag"),
                                          1, NULL);
                 if (!fsdev) {
-                    fprintf(stderr, "duplicate fsdev id: %s\n",
-                            qemu_opt_get(opts, "mount_tag"));
+                    error_report("duplicate fsdev id: %s",
+                                 qemu_opt_get(opts, "mount_tag"));
                     exit(1);
                 }
 
@@ -3468,8 +3539,8 @@ int main(int argc, char **argv, char **envp)
 #ifdef CONFIG_SYNC_FILE_RANGE
                     qemu_opt_set(fsdev, "writeout", writeout, &error_abort);
 #else
-                    fprintf(stderr, "writeout=immediate not supported on "
-                            "this platform\n");
+                    error_report("writeout=immediate not supported "
+                                 "on this platform");
                     exit(1);
 #endif
                 }
@@ -3508,7 +3579,7 @@ int main(int argc, char **argv, char **envp)
                 fsdev = qemu_opts_create(qemu_find_opts("fsdev"), "v_synth",
                                          1, NULL);
                 if (!fsdev) {
-                    fprintf(stderr, "duplicate option: %s\n", "virtfs_synth");
+                    error_report("duplicate option: %s", "virtfs_synth");
                     exit(1);
                 }
                 qemu_opt_set(fsdev, "fsdriver", "synth", &error_abort);
@@ -3529,15 +3600,14 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_watchdog:
                 if (watchdog) {
-                    fprintf(stderr,
-                            "qemu: only one watchdog option may be given\n");
+                    error_report("only one watchdog option may be given");
                     return 1;
                 }
                 watchdog = optarg;
                 break;
             case QEMU_OPTION_watchdog_action:
                 if (select_watchdog_action(optarg) == -1) {
-                    fprintf(stderr, "Unknown -watchdog-action parameter\n");
+                    error_report("unknown -watchdog-action parameter");
                     exit(1);
                 }
                 break;
@@ -3581,7 +3651,7 @@ int main(int argc, char **argv, char **envp)
                 display_type = DT_SDL;
                 break;
 #else
-                fprintf(stderr, "SDL support is disabled\n");
+                error_report("SDL support is disabled");
                 exit(1);
 #endif
             case QEMU_OPTION_pidfile:
@@ -3643,8 +3713,7 @@ int main(int argc, char **argv, char **envp)
                 qemu_opts_parse_noisily(olist, "accel=tcg", false);
                 break;
             case QEMU_OPTION_no_kvm_pit: {
-                fprintf(stderr, "Warning: KVM PIT can no longer be disabled "
-                                "separately.\n");
+                error_report("warning: ignoring deprecated option");
                 break;
             }
             case QEMU_OPTION_no_kvm_pit_reinjection: {
@@ -3657,8 +3726,8 @@ int main(int argc, char **argv, char **envp)
                     { /* end of list */ }
                 };
 
-                fprintf(stderr, "Warning: option deprecated, use "
-                        "lost_tick_policy property of kvm-pit instead.\n");
+                error_report("warning: deprecated, replaced by "
+                             "-global kvm-pit.lost_tick_policy=discard");
                 qdev_prop_register_global_list(kvm_pit_lost_tick_policy);
                 break;
             }
@@ -3693,7 +3762,7 @@ int main(int argc, char **argv, char **envp)
                     exit(1);
                 }
 #else
-                fprintf(stderr, "VNC support is disabled\n");
+                error_report("VNC support is disabled");
                 exit(1);
 #endif
                 break;
@@ -3706,7 +3775,7 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_balloon:
                 if (balloon_parse(optarg) < 0) {
-                    fprintf(stderr, "Unknown -balloon argument %s\n", optarg);
+                    error_report("unknown -balloon argument %s", optarg);
                     exit(1);
                 }
                 break;
@@ -3721,15 +3790,14 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_uuid:
                 if(qemu_uuid_parse(optarg, qemu_uuid) < 0) {
-                    fprintf(stderr, "Fail to parse UUID string."
-                            " Wrong format.\n");
+                    error_report("failed to parse UUID string: wrong format");
                     exit(1);
                 }
                 qemu_uuid_set = true;
                 break;
             case QEMU_OPTION_option_rom:
                 if (nb_option_roms >= MAX_OPTION_ROMS) {
-                    fprintf(stderr, "Too many option ROMs\n");
+                    error_report("too many option ROMs");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(qemu_find_opts("option-rom"),
@@ -3741,7 +3809,7 @@ int main(int argc, char **argv, char **envp)
                 option_rom[nb_option_roms].bootindex =
                     qemu_opt_get_number(opts, "bootindex", -1);
                 if (!option_rom[nb_option_roms].name) {
-                    fprintf(stderr, "Option ROM file is not specified\n");
+                    error_report("Option ROM file is not specified");
                     exit(1);
                 }
                 nb_option_roms++;
@@ -3766,9 +3834,8 @@ int main(int argc, char **argv, char **envp)
                         } else  if (strcmp("auto", target) == 0) {
                             semihosting.target = SEMIHOSTING_TARGET_AUTO;
                         } else {
-                            fprintf(stderr, "Unsupported semihosting-config"
-                                    " %s\n",
-                                optarg);
+                            error_report("unsupported semihosting-config %s",
+                                         optarg);
                             exit(1);
                         }
                     } else {
@@ -3778,14 +3845,12 @@ int main(int argc, char **argv, char **envp)
                     qemu_opt_foreach(opts, add_semihosting_arg,
                                      &semihosting, NULL);
                 } else {
-                    fprintf(stderr, "Unsupported semihosting-config %s\n",
-                            optarg);
+                    error_report("unsupported semihosting-config %s", optarg);
                     exit(1);
                 }
                 break;
             case QEMU_OPTION_tdf:
-                fprintf(stderr, "Warning: user space PIT time drift fix "
-                                "is no longer supported.\n");
+                error_report("warning: ignoring deprecated option");
                 break;
             case QEMU_OPTION_name:
                 opts = qemu_opts_parse_noisily(qemu_find_opts("name"),
@@ -3796,7 +3861,7 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_prom_env:
                 if (nb_prom_envs >= MAX_PROM_ENVS) {
-                    fprintf(stderr, "Too many prom variables\n");
+                    error_report("too many prom variables");
                     exit(1);
                 }
                 prom_envs[nb_prom_envs] = optarg;
@@ -3879,8 +3944,8 @@ int main(int argc, char **argv, char **envp)
                 {
                     int ret = qemu_read_config_file(optarg);
                     if (ret < 0) {
-                        fprintf(stderr, "read config %s: %s\n", optarg,
-                            strerror(-ret));
+                        error_report("read config %s: %s", optarg,
+                                     strerror(-ret));
                         exit(1);
                     }
                     break;
@@ -3888,7 +3953,7 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_spice:
                 olist = qemu_find_opts("spice");
                 if (!olist) {
-                    fprintf(stderr, "spice is not supported by this qemu build.\n");
+                    error_report("spice support is disabled");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(olist, optarg, false);
@@ -3905,7 +3970,8 @@ int main(int argc, char **argv, char **envp)
                     } else {
                         fp = fopen(optarg, "w");
                         if (fp == NULL) {
-                            fprintf(stderr, "open %s: %s\n", optarg, strerror(errno));
+                            error_report("open %s: %s", optarg,
+                                         strerror(errno));
                             exit(1);
                         }
                     }
@@ -3966,13 +4032,13 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_dump_vmstate:
                 if (vmstate_dump_file) {
-                    fprintf(stderr, "qemu: only one '-dump-vmstate' "
-                            "option may be given\n");
+                    error_report("only one '-dump-vmstate' "
+                                 "option may be given");
                     exit(1);
                 }
                 vmstate_dump_file = fopen(optarg, "w");
                 if (vmstate_dump_file == NULL) {
-                    fprintf(stderr, "open %s: %s\n", optarg, strerror(errno));
+                    error_report("open %s: %s", optarg, strerror(errno));
                     exit(1);
                 }
                 break;
@@ -3982,17 +4048,9 @@ int main(int argc, char **argv, char **envp)
         }
     }
 
-    opts = qemu_get_machine_opts();
-    optarg = qemu_opt_get(opts, "type");
-    if (optarg) {
-        machine_class = machine_parse(optarg);
-    }
+    replay_configure(icount_opts);
 
-    if (machine_class == NULL) {
-        fprintf(stderr, "No machine specified, and there is no default.\n"
-                "Use -machine help to list supported machines!\n");
-        exit(1);
-    }
+    set_machine_options(&machine_class);
 
     set_memory_options(&ram_slots, &maxram_size, machine_class);
 
@@ -4037,7 +4095,7 @@ int main(int argc, char **argv, char **envp)
     cpu_exec_init_all();
 
     if (machine_class->hw_version) {
-        qemu_set_version(machine_class->hw_version);
+        qemu_set_hw_version(machine_class->hw_version);
     }
 
     /* Init CPU def lists, based on config
@@ -4091,9 +4149,9 @@ int main(int argc, char **argv, char **envp)
 
     machine_class->max_cpus = machine_class->max_cpus ?: 1; /* Default to UP */
     if (max_cpus > machine_class->max_cpus) {
-        fprintf(stderr, "Number of SMP cpus requested (%d), exceeds max cpus "
-                "supported by machine `%s' (%d)\n", max_cpus,
-                machine_class->name, machine_class->max_cpus);
+        error_report("Number of SMP CPUs requested (%d) exceeds max CPUs "
+                     "supported by machine '%s' (%d)", max_cpus,
+                     machine_class->name, machine_class->max_cpus);
         exit(1);
     }
 
@@ -4154,12 +4212,12 @@ int main(int argc, char **argv, char **envp)
         if (display_type == DT_NOGRAPHIC
             && (default_parallel || default_serial
                 || default_monitor || default_virtcon)) {
-            fprintf(stderr, "-nographic can not be used with -daemonize\n");
+            error_report("-nographic cannot be used with -daemonize");
             exit(1);
         }
 #ifdef CONFIG_CURSES
         if (display_type == DT_CURSES) {
-            fprintf(stderr, "curses display can not be used with -daemonize\n");
+            error_report("curses display cannot be used with -daemonize");
             exit(1);
         }
 #endif
@@ -4218,12 +4276,12 @@ int main(int argc, char **argv, char **envp)
     }
 
     if ((no_frame || alt_grab || ctrl_grab) && display_type != DT_SDL) {
-        fprintf(stderr, "-no-frame, -alt-grab and -ctrl-grab are only valid "
-                        "for SDL, ignoring option\n");
+        error_report("-no-frame, -alt-grab and -ctrl-grab are only valid "
+                     "for SDL, ignoring option");
     }
     if (no_quit && (display_type != DT_GTK && display_type != DT_SDL)) {
-        fprintf(stderr, "-no-quit is only valid for GTK and SDL, "
-                        "ignoring option\n");
+        error_report("-no-quit is only valid for GTK and SDL, "
+                     "ignoring option");
     }
 
 #if defined(CONFIG_GTK)
@@ -4238,13 +4296,14 @@ int main(int argc, char **argv, char **envp)
 #endif
     if (request_opengl == 1 && display_opengl == 0) {
 #if defined(CONFIG_OPENGL)
-        fprintf(stderr, "OpenGL is not supported by the display.\n");
+        error_report("OpenGL is not supported by the display");
 #else
-        fprintf(stderr, "QEMU was built without opengl support.\n");
+        error_report("OpenGL support is disabled");
 #endif
         exit(1);
     }
 
+    page_size_init();
     socket_init();
 
     if (qemu_opts_foreach(qemu_find_opts("object"),
@@ -4266,7 +4325,7 @@ int main(int argc, char **argv, char **envp)
 #endif
 
     if (pid_file && qemu_create_pidfile(pid_file) != 0) {
-        fprintf(stderr, "Could not acquire pid file: %s\n", strerror(errno));
+        error_report("could not acquire pid file: %s", strerror(errno));
         exit(1);
     }
 
@@ -4275,12 +4334,6 @@ int main(int argc, char **argv, char **envp)
         exit(0);
     }
 
-    if (qemu_opts_foreach(qemu_find_opts("object"),
-                          object_create,
-                          object_create_delayed, NULL)) {
-        exit(1);
-    }
-
     machine_opts = qemu_get_machine_opts();
     if (qemu_opt_foreach(machine_opts, machine_set_property, current_machine,
                          NULL)) {
@@ -4343,17 +4396,17 @@ int main(int argc, char **argv, char **envp)
     linux_boot = (kernel_filename != NULL);
 
     if (!linux_boot && *kernel_cmdline != '\0') {
-        fprintf(stderr, "-append only allowed with -kernel option\n");
+        error_report("-append only allowed with -kernel option");
         exit(1);
     }
 
     if (!linux_boot && initrd_filename != NULL) {
-        fprintf(stderr, "-initrd only allowed with -kernel option\n");
+        error_report("-initrd only allowed with -kernel option");
         exit(1);
     }
 
     if (!linux_boot && qemu_opt_get(machine_opts, "dtb")) {
-        fprintf(stderr, "-dtb only allowed with -kernel option\n");
+        error_report("-dtb only allowed with -kernel option");
         exit(1);
     }
 
@@ -4372,7 +4425,7 @@ int main(int argc, char **argv, char **envp)
     cpu_ticks_init();
     if (icount_opts) {
         if (kvm_enabled() || xen_enabled()) {
-            fprintf(stderr, "-icount is not allowed with kvm or xen\n");
+            error_report("-icount is not allowed with kvm or xen");
             exit(1);
         }
         configure_icount(icount_opts, &error_abort);
@@ -4386,6 +4439,12 @@ int main(int argc, char **argv, char **envp)
         exit(1);
     }
 
+    if (qemu_opts_foreach(qemu_find_opts("object"),
+                          object_create,
+                          object_create_delayed, NULL)) {
+        exit(1);
+    }
+
 #ifdef CONFIG_TPM
     if (tpm_init() < 0) {
         exit(1);
@@ -4399,7 +4458,7 @@ int main(int argc, char **argv, char **envp)
     if (!xen_enabled()) {
         /* On 32-bit hosts, QEMU is limited by virtual address space */
         if (ram_size > (2047 << 20) && HOST_LONG_BITS == 32) {
-            fprintf(stderr, "qemu: at most 2047 MB RAM can be simulated\n");
+            error_report("at most 2047 MB RAM can be simulated");
             exit(1);
         }
     }
@@ -4415,9 +4474,10 @@ int main(int argc, char **argv, char **envp)
     }
 
     /* open the virtual block devices */
-    if (snapshot)
-        qemu_opts_foreach(qemu_find_opts("drive"),
-                          drive_enable_snapshot, NULL, NULL);
+    if (snapshot || replay_mode != REPLAY_MODE_NONE) {
+        qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot,
+                          NULL, NULL);
+    }
     if (qemu_opts_foreach(qemu_find_opts("drive"), drive_init_func,
                           &machine_class->block_default_type, NULL)) {
         exit(1);
@@ -4472,6 +4532,10 @@ int main(int argc, char **argv, char **envp)
     }
     qemu_add_globals();
 
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_INIT);
     qdev_machine_init();
 
     current_machine->ram_size = ram_size;
@@ -4501,6 +4565,9 @@ int main(int argc, char **argv, char **envp)
             exit(1);
     }
 
+    /* Check if IGD GFX passthrough. */
+    igd_gfx_passthru();
+
     /* init generic devices */
     if (qemu_opts_foreach(qemu_find_opts("device"),
                           device_init_func, NULL, NULL)) {
@@ -4561,7 +4628,7 @@ int main(int argc, char **argv, char **envp)
                       vnc_init_func, NULL, NULL);
     if (show_vnc_port) {
         char *ret = vnc_display_local_addr("default");
-        printf("VNC server running on `%s'\n", ret);
+        printf("VNC server running on '%s'\n", ret);
         g_free(ret);
     }
 #endif
@@ -4583,10 +4650,16 @@ int main(int argc, char **argv, char **envp)
     qemu_run_machine_init_done_notifiers();
 
     if (rom_check_and_register_reset() != 0) {
-        fprintf(stderr, "rom check and register reset failed\n");
+        error_report("rom check and register reset failed");
         exit(1);
     }
 
+    replay_start();
+
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_RESET);
     qemu_system_reset(VMRESET_SILENT);
     register_global_state();
     if (loadvm) {
@@ -4624,6 +4697,8 @@ int main(int argc, char **argv, char **envp)
     }
 
     main_loop();
+    replay_disable_events();
+
     bdrv_close_all();
     pause_all_vcpus();
     res_free();
diff --git a/xen-hvm-stub.c b/xen-hvm-stub.c
index 46867d87d..6a39425da 100644
--- a/xen-hvm-stub.c
+++ b/xen-hvm-stub.c
@@ -47,8 +47,7 @@ void xen_modified_memory(ram_addr_t start, ram_addr_t length)
 {
 }
 
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory)
+int xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory)
 {
     return 0;
 }
diff --git a/xen-hvm.c b/xen-hvm.c
index 040846236..3d78a0c52 100644
--- a/xen-hvm.c
+++ b/xen-hvm.c
@@ -180,8 +180,7 @@ qemu_irq *xen_interrupt_controller_init(void)
 
 /* Memory Ops */
 
-static void xen_ram_init(ram_addr_t *below_4g_mem_size,
-                         ram_addr_t *above_4g_mem_size,
+static void xen_ram_init(PCMachineState *pcms,
                          ram_addr_t ram_size, MemoryRegion **ram_memory_p)
 {
     MemoryRegion *sysmem = get_system_memory();
@@ -198,23 +197,23 @@ static void xen_ram_init(ram_addr_t *below_4g_mem_size,
     }
 
     if (ram_size >= user_lowmem) {
-        *above_4g_mem_size = ram_size - user_lowmem;
-        *below_4g_mem_size = user_lowmem;
+        pcms->above_4g_mem_size = ram_size - user_lowmem;
+        pcms->below_4g_mem_size = user_lowmem;
     } else {
-        *above_4g_mem_size = 0;
-        *below_4g_mem_size = ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = ram_size;
     }
-    if (!*above_4g_mem_size) {
+    if (!pcms->above_4g_mem_size) {
         block_len = ram_size;
     } else {
         /*
          * Xen does not allocate the memory continuously, it keeps a
          * hole of the size computed above or passed in.
          */
-        block_len = (1ULL << 32) + *above_4g_mem_size;
+        block_len = (1ULL << 32) + pcms->above_4g_mem_size;
     }
     memory_region_init_ram(&ram_memory, NULL, "xen.ram", block_len,
-                           &error_abort);
+                           &error_fatal);
     *ram_memory_p = &ram_memory;
     vmstate_register_ram_global(&ram_memory);
 
@@ -229,12 +228,12 @@ static void xen_ram_init(ram_addr_t *below_4g_mem_size,
      */
     memory_region_init_alias(&ram_lo, NULL, "xen.ram.lo",
                              &ram_memory, 0xc0000,
-                             *below_4g_mem_size - 0xc0000);
+                             pcms->below_4g_mem_size - 0xc0000);
     memory_region_add_subregion(sysmem, 0xc0000, &ram_lo);
-    if (*above_4g_mem_size > 0) {
+    if (pcms->above_4g_mem_size > 0) {
         memory_region_init_alias(&ram_hi, NULL, "xen.ram.hi",
                                  &ram_memory, 0x100000000ULL,
-                                 *above_4g_mem_size);
+                                 pcms->above_4g_mem_size);
         memory_region_add_subregion(sysmem, 0x100000000ULL, &ram_hi);
     }
 }
@@ -345,10 +344,10 @@ go_physmap:
         unsigned long idx = pfn + i;
         xen_pfn_t gpfn = start_gpfn + i;
 
-        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
+        rc = xen_xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
         if (rc) {
             DPRINTF("add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
-                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
+                    PRI_xen_pfn" failed: %d (errno: %d)\n", idx, gpfn, rc, errno);
             return -rc;
         }
     }
@@ -422,10 +421,10 @@ static int xen_remove_from_physmap(XenIOState *state,
         xen_pfn_t idx = start_addr + i;
         xen_pfn_t gpfn = phys_offset + i;
 
-        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
+        rc = xen_xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
         if (rc) {
             fprintf(stderr, "add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
-                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
+                    PRI_xen_pfn" failed: %d (errno: %d)\n", idx, gpfn, rc, errno);
             return -rc;
         }
     }
@@ -814,9 +813,14 @@ static void cpu_ioreq_pio(ioreq_t *req)
 {
     uint32_t i;
 
+    trace_cpu_ioreq_pio(req, req->dir, req->df, req->data_is_ptr, req->addr,
+                         req->data, req->count, req->size);
+
     if (req->dir == IOREQ_READ) {
         if (!req->data_is_ptr) {
             req->data = do_inp(req->addr, req->size);
+            trace_cpu_ioreq_pio_read_reg(req, req->data, req->addr,
+                                         req->size);
         } else {
             uint32_t tmp;
 
@@ -827,6 +831,8 @@ static void cpu_ioreq_pio(ioreq_t *req)
         }
     } else if (req->dir == IOREQ_WRITE) {
         if (!req->data_is_ptr) {
+            trace_cpu_ioreq_pio_write_reg(req, req->data, req->addr,
+                                          req->size);
             do_outp(req->addr, req->size, req->data);
         } else {
             for (i = 0; i < req->count; i++) {
@@ -843,6 +849,9 @@ static void cpu_ioreq_move(ioreq_t *req)
 {
     uint32_t i;
 
+    trace_cpu_ioreq_move(req, req->dir, req->df, req->data_is_ptr, req->addr,
+                         req->data, req->count, req->size);
+
     if (!req->data_is_ptr) {
         if (req->dir == IOREQ_READ) {
             for (i = 0; i < req->count; i++) {
@@ -915,11 +924,18 @@ static void handle_vmport_ioreq(XenIOState *state, ioreq_t *req)
 
 static void handle_ioreq(XenIOState *state, ioreq_t *req)
 {
+    trace_handle_ioreq(req, req->type, req->dir, req->df, req->data_is_ptr,
+                       req->addr, req->data, req->count, req->size);
+
     if (!req->data_is_ptr && (req->dir == IOREQ_WRITE) &&
             (req->size < sizeof (target_ulong))) {
         req->data &= ((target_ulong) 1 << (8 * req->size)) - 1;
     }
 
+    if (req->dir == IOREQ_WRITE)
+        trace_handle_ioreq_write(req, req->type, req->df, req->data_is_ptr,
+                                 req->addr, req->data, req->count, req->size);
+
     switch (req->type) {
         case IOREQ_TYPE_PIO:
             cpu_ioreq_pio(req);
@@ -959,23 +975,38 @@ static void handle_ioreq(XenIOState *state, ioreq_t *req)
         default:
             hw_error("Invalid ioreq type 0x%x\n", req->type);
     }
+    if (req->dir == IOREQ_READ) {
+        trace_handle_ioreq_read(req, req->type, req->df, req->data_is_ptr,
+                                req->addr, req->data, req->count, req->size);
+    }
 }
 
 static int handle_buffered_iopage(XenIOState *state)
 {
+    buffered_iopage_t *buf_page = state->buffered_io_page;
     buf_ioreq_t *buf_req = NULL;
     ioreq_t req;
     int qw;
 
-    if (!state->buffered_io_page) {
+    if (!buf_page) {
         return 0;
     }
 
     memset(&req, 0x00, sizeof(req));
 
-    while (state->buffered_io_page->read_pointer != state->buffered_io_page->write_pointer) {
-        buf_req = &state->buffered_io_page->buf_ioreq[
-            state->buffered_io_page->read_pointer % IOREQ_BUFFER_SLOT_NUM];
+    for (;;) {
+        uint32_t rdptr = buf_page->read_pointer, wrptr;
+
+        xen_rmb();
+        wrptr = buf_page->write_pointer;
+        xen_rmb();
+        if (rdptr != buf_page->read_pointer) {
+            continue;
+        }
+        if (rdptr == wrptr) {
+            break;
+        }
+        buf_req = &buf_page->buf_ioreq[rdptr % IOREQ_BUFFER_SLOT_NUM];
         req.size = 1UL << buf_req->size;
         req.count = 1;
         req.addr = buf_req->addr;
@@ -987,15 +1018,14 @@ static int handle_buffered_iopage(XenIOState *state)
         req.data_is_ptr = 0;
         qw = (req.size == 8);
         if (qw) {
-            buf_req = &state->buffered_io_page->buf_ioreq[
-                (state->buffered_io_page->read_pointer + 1) % IOREQ_BUFFER_SLOT_NUM];
+            buf_req = &buf_page->buf_ioreq[(rdptr + 1) %
+                                           IOREQ_BUFFER_SLOT_NUM];
             req.data |= ((uint64_t)buf_req->data) << 32;
         }
 
         handle_ioreq(state, &req);
 
-        xen_mb();
-        state->buffered_io_page->read_pointer += qw ? 2 : 1;
+        atomic_add(&buf_page->read_pointer, qw + 1);
     }
 
     return req.count;
@@ -1159,7 +1189,7 @@ static void xen_wakeup_notifier(Notifier *notifier, void *data)
 }
 
 /* return 0 means OK, or -1 means critical issue -- will exit(1) */
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
+int xen_hvm_init(PCMachineState *pcms,
                  MemoryRegion **ram_memory)
 {
     int i, rc;
@@ -1270,7 +1300,7 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
 
     /* Init RAM management */
     xen_map_cache_init(xen_phys_offset_to_gaddr, state);
-    xen_ram_init(below_4g_mem_size, above_4g_mem_size, ram_size, ram_memory);
+    xen_ram_init(pcms, ram_size, ram_memory);
 
     qemu_add_vm_change_state_handler(xen_hvm_change_state_handler, state);
 
diff --git a/xen-mapcache.c b/xen-mapcache.c
index 8cefd0c62..97fece235 100644
--- a/xen-mapcache.c
+++ b/xen-mapcache.c
@@ -169,10 +169,8 @@ static void xen_remap_bucket(MapCacheEntry *entry,
             exit(-1);
         }
     }
-    if (entry->valid_mapping != NULL) {
-        g_free(entry->valid_mapping);
-        entry->valid_mapping = NULL;
-    }
+    g_free(entry->valid_mapping);
+    entry->valid_mapping = NULL;
 
     for (i = 0; i < nb_pfn; i++) {
         pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;